commit 1c4adf53ac1829b8df82ece2b7673cefb156a272 Author: wehub-resource-sync Date: Mon Jul 13 13:13:07 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 0000000..193fe72 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,13 @@ +# cargo-audit configuration. +# +# The advisories below are "unmaintained" notices (not vulnerabilities) for +# transitive dependencies we don't pull directly and can't drop without a major +# bump of an upstream crate. They're listed here — with intent — so a real +# vulnerability is never lost in warning noise. Revisit when upstreams migrate +# (e.g. rustls-pemfile -> rustls-pki-types). +[advisories] +ignore = [ + "RUSTSEC-2024-0436", # paste — unmaintained (transitive, macro deps) + "RUSTSEC-2026-0173", # proc-macro-error2 — unmaintained (transitive) + "RUSTSEC-2025-0134", # rustls-pemfile — unmaintained (transitive, TLS) +] diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..3550a30 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..e9dc616 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +lib/libkrun.dylib filter=lfs diff=lfs merge=lfs -text +lib/libkrunfw.5.dylib filter=lfs diff=lfs merge=lfs -text +lib/libvirglrenderer.1.dylib filter=lfs diff=lfs merge=lfs -text +lib/libMoltenVK.dylib filter=lfs diff=lfs merge=lfs -text +lib/libepoxy.0.dylib filter=lfs diff=lfs merge=lfs -text +lib/linux-x86_64/*.so* filter=lfs diff=lfs merge=lfs -text +lib/linux-aarch64/*.so* filter=lfs diff=lfs merge=lfs -text +lib/windows-x86_64/*.dll filter=lfs diff=lfs merge=lfs -text diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..8f0eb83 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: BinSquare diff --git a/.github/workflows/build-libkrun.yml b/.github/workflows/build-libkrun.yml new file mode 100644 index 0000000..c86b551 --- /dev/null +++ b/.github/workflows/build-libkrun.yml @@ -0,0 +1,111 @@ +name: Build libkrun (linux, glibc 2.35) + +# Rebuild the bundled linux libkrun.so on ubuntu-22.04 so its glibc floor is +# 2.35 — the same floor the release binaries get (they also build on 22.04, see +# release.yml). A libkrun built on a newer distro (e.g. Ubuntu 24.04 / glibc +# 2.39) loads fine there but fails on 22.04 with "GLIBC_2.39 not found", which is +# exactly how a hand-built lib slipped a 2.39 floor into v1.2.0. +# +# SKIP_LIBKRUNFW=1: reuse the committed lib/linux-/libkrunfw.so (it wraps a +# guest-kernel blob, floor GLIBC_2.2 — host-glibc-independent), so we skip the +# multi-minute kernel rebuild and only recompile libkrun itself. +# +# Dispatch-only. Uploads lib/linux-/ as an artifact; the rebuilt .so is +# then committed into the repo (git-LFS) and shipped in the next release. + +on: + workflow_dispatch: + inputs: + rebuild_libkrunfw: + description: "Rebuild libkrunfw from source (full guest-kernel rebuild) instead of reusing the committed .so. Needed when a libkrunfw patch changed the guest kernel." + type: boolean + default: false + +jobs: + build: + name: libkrun ${{ matrix.arch }} + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runner: ubuntu-22.04 + - arch: aarch64 + runner: ubuntu-22.04-arm + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive # libkrun links against the libkrunfw source + lfs: true # committed libkrunfw.so reused under SKIP_LIBKRUNFW + + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android \ + /opt/hostedtoolcache/CodeQL 2>/dev/null || true + df -h / | tail -1 + + # By default reuse the committed libkrunfw.so (skips the multi-minute guest + # kernel rebuild). When a libkrunfw patch changed the guest kernel, dispatch + # with rebuild_libkrunfw=true so libkrunfw.so is rebuilt from the submodule + # source and picked up here. + - name: Build libkrun (GPU=1) + run: | + if [ "${{ inputs.rebuild_libkrunfw }}" = "true" ]; then + echo "Rebuilding libkrunfw from source" + GPU=1 ./scripts/build-libkrun-linux.sh + else + SKIP_LIBKRUNFW=1 GPU=1 ./scripts/build-libkrun-linux.sh + fi + + - name: Assert glibc floor <= 2.35 + run: | + so="lib/linux-${{ matrix.arch }}/libkrun.so" + maxv=$(objdump -T "$so" | grep -oE 'GLIBC_[0-9]+\.[0-9]+' \ + | sed 's/GLIBC_//' | sort -V | tail -1) + echo "max GLIBC required by $so: ${maxv:-none}" + if [ -n "$maxv" ] && [ "$(printf '%s\n2.35\n' "$maxv" | sort -V | tail -1)" != "2.35" ]; then + echo "::error::$so requires glibc $maxv (> 2.35) — floor not lowered" + exit 1 + fi + echo "OK: glibc floor is <= 2.35" + + - uses: actions/upload-artifact@v4 + with: + name: libkrun-linux-${{ matrix.arch }} + path: lib/linux-${{ matrix.arch }}/ + if-no-files-found: error + + # Cross-build the Windows libkrunfw.dll (guest kernel wrapped for a Windows + # host) with mingw-w64 on Linux. The kernel builds with native gcc; only the + # final bundle link uses the mingw toolchain (Makefile OS=Windows path). Only + # runs when refreshing libkrunfw, since the .dll wraps the guest kernel. + windows-libkrunfw: + name: libkrunfw.dll (windows cross) + if: ${{ inputs.rebuild_libkrunfw }} + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + lfs: false + + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android \ + /opt/hostedtoolcache/CodeQL 2>/dev/null || true + + - name: Install build + mingw toolchain + run: | + sudo apt-get update + sudo apt-get install -y make gcc bc bison flex libelf-dev libssl-dev \ + python3-pyelftools curl xz-utils cpio kmod gcc-mingw-w64-x86-64 + + - name: Build libkrunfw.dll (mingw cross) + run: cd libkrunfw && make OS=Windows -j"$(nproc)" + + - uses: actions/upload-artifact@v4 + with: + name: libkrunfw-windows-x86_64 + path: libkrunfw/libkrunfw.dll + if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0d1b228 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,337 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_call: + +env: + CARGO_TERM_COLOR: always + +jobs: + # ========================================================================== + # Quick checks (no platform-specific deps needed) + # ========================================================================== + fmt: + name: Format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - run: cargo fmt --all -- --check + + secrets-guards: + name: Secrets invariants + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check secrets guardrails + run: ./scripts/check-secrets-guards.sh + + libkrun-provenance: + name: libkrun freshness + runs-on: ubuntu-latest + steps: + # LFS on: check-krun-exports.sh reads the actual bundled lib binaries. + # (check-libkrun-provenance.sh itself only needs the pinned gitlinks.) + - uses: actions/checkout@v4 + with: + lfs: true + - name: Bundled libkrun matches the pinned submodule + run: ./scripts/check-libkrun-provenance.sh + # nm (ELF), llvm-nm (Mach-O dylib), mingw objdump (PE dll exports). + - name: Install symbol-reading tools + run: | + sudo apt-get update + sudo apt-get install -y binutils binutils-mingw-w64-x86-64 llvm + - name: Bundled libkrun exports every symbol smolvm requires + run: ./scripts/check-krun-exports.sh + + audit: + name: Cargo Audit + runs-on: ubuntu-latest + # Scans Cargo.lock for RustSec advisories — no engine/libkrun needed. Honors + # .cargo/audit.toml, which ignores documented unmaintained-transitive notices + # so a real vulnerability stands out instead of hiding in warning noise. + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: taiki-e/install-action@cargo-audit + - run: cargo audit + + # ========================================================================== + # macOS Build & Test (Apple Silicon) + # ========================================================================== + macos: + name: macOS (arm64) + runs-on: macos-14 + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Cache cargo + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: macos-arm64-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + restore-keys: macos-arm64-cargo- + + - name: Setup Git LFS + run: | + git lfs install + git lfs pull + + - name: Verify LFS dylibs + run: | + ls -la lib/ + file lib/*.dylib + + - name: Build + run: cargo build --release + env: + LIBRARY_PATH: ${{ github.workspace }}/lib + + - name: Clippy + run: cargo clippy --all-targets -- -D warnings + env: + LIBRARY_PATH: ${{ github.workspace }}/lib + + - name: Run unit tests + run: cargo test --lib + env: + LIBRARY_PATH: ${{ github.workspace }}/lib + DYLD_LIBRARY_PATH: ${{ github.workspace }}/lib + + - name: Run agent unit tests + run: cargo test -p smolvm-agent --bins + env: + LIBRARY_PATH: ${{ github.workspace }}/lib + DYLD_LIBRARY_PATH: ${{ github.workspace }}/lib + + - name: Sign binary + run: codesign --force --sign - --entitlements smolvm.entitlements target/release/smolvm + + - name: Verify binary runs + run: ./target/release/smolvm --version + env: + DYLD_LIBRARY_PATH: ${{ github.workspace }}/lib + # Note: Integration tests (E2E VM tests) require Hypervisor.framework access, + # which GitHub runners don't provide. Run integration tests locally. + + # ========================================================================== + # Linux Build & Test + # ========================================================================== + linux: + name: Linux (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runner: ubuntu-latest + target: x86_64-unknown-linux-gnu + - arch: aarch64 + runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + targets: ${{ matrix.target }} + + - name: Cache cargo + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: linux-${{ matrix.arch }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + restore-keys: linux-${{ matrix.arch }}-cargo- + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + libssl-dev \ + pkg-config + + # libkrun is not easily available on Ubuntu, so we create a stub library + # for linking. This allows us to compile and run unit tests. + # Full integration tests require macOS or a Linux machine with KVM. + - name: Create libkrun stub for linking + run: | + # Create minimal stub that satisfies the linker + cat > /tmp/libkrun_stub.c << 'EOF' + // Stub implementations - these won't be called in unit tests + int krun_create_ctx() { return -1; } + int krun_free_ctx(int ctx) { return 0; } + int krun_set_vm_config(int ctx, int cpus, int mem) { return -1; } + int krun_set_root(int ctx, const char* path) { return -1; } + int krun_set_workdir(int ctx, const char* path) { return -1; } + int krun_set_exec(int ctx, const char* path, char** argv, char** env) { return -1; } + int krun_start_enter(int ctx) { return -1; } + int krun_set_log_level(int level) { return 0; } + int krun_add_disk2(int ctx, const char* id, const char* path, int flags, int ro) { return -1; } + int krun_set_port_map(int ctx, char** map) { return -1; } + int krun_add_vsock_port(int ctx, int port, const char* path) { return -1; } + int krun_add_vsock_port2(int ctx, int port, const char* path, int flags) { return -1; } + int krun_disable_implicit_vsock(int ctx) { return -1; } + int krun_add_vsock(int ctx, int tsi_features) { return -1; } + int krun_set_passt_fd(int ctx, int fd) { return -1; } + int krun_add_virtiofs(int ctx, const char* tag, const char* path) { return -1; } + int krun_add_virtiofs2(int ctx, const char* tag, const char* path, int flags) { return -1; } + int krun_set_console_output(int ctx, const char* path) { return -1; } + int krun_add_virtio_console_default(int ctx) { return -1; } + EOF + gcc -shared -fPIC -o /tmp/libkrun.so /tmp/libkrun_stub.c + sudo cp /tmp/libkrun.so /usr/local/lib/ + sudo ln -sf /usr/local/lib/libkrun.so /usr/local/lib/libkrun.1.dylib + sudo ldconfig + + - name: Build + run: cargo build --release --target ${{ matrix.target }} + env: + LIBRARY_PATH: /usr/local/lib + + - name: Clippy + run: cargo clippy --all-targets --target ${{ matrix.target }} -- -D warnings + env: + LIBRARY_PATH: /usr/local/lib + + - name: Run unit tests + run: cargo test --lib --target ${{ matrix.target }} + env: + LIBRARY_PATH: /usr/local/lib + LD_LIBRARY_PATH: /usr/local/lib + + - name: Run agent unit tests + run: cargo test -p smolvm-agent --bins --target ${{ matrix.target }} + env: + LIBRARY_PATH: /usr/local/lib + LD_LIBRARY_PATH: /usr/local/lib + # Note: Full VM integration tests require KVM + real libkrun. + # Run integration tests locally on a machine with KVM access. + + # ========================================================================== + # Windows Build & Lint (cross-compiled from Linux, mingw-w64 / GNU target) + # ========================================================================== + windows: + name: Windows (x86_64, cross) + runs-on: ubuntu-latest + env: + CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER: x86_64-w64-mingw32-gcc + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + targets: x86_64-pc-windows-gnu + + - name: Cache cargo + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: windows-gnu-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + restore-keys: windows-gnu-cargo- + + - name: Install mingw-w64 cross toolchain + run: | + sudo apt-get update + sudo apt-get install -y gcc-mingw-w64-x86-64 clang libclang-dev + + # The Windows host dlopens libkrun at runtime (libloading), so unlike the + # macOS/Linux jobs there is no link-time libkrun dependency and no stub is + # needed. This gate catches host-side compile + lint regressions on + # Windows; WHP VM integration tests run out-of-band on real hardware via + # the WHP QA harness (unit tests can additionally be run under Wine with + # CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUNNER=wine). + - name: Build + run: cargo build --release --target x86_64-pc-windows-gnu + + - name: Clippy + run: cargo clippy --release --target x86_64-pc-windows-gnu --all-targets -- -D warnings + + # ========================================================================== + # Build agent (Linux musl for static binary) + # ========================================================================== + agent: + name: Agent (Linux musl) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + with: + targets: aarch64-unknown-linux-musl, x86_64-unknown-linux-musl + + - name: Install musl tools + run: | + sudo apt-get update + sudo apt-get install -y musl-tools + + - name: Install cross-compilation tools + run: | + # For aarch64 cross-compilation + sudo apt-get install -y gcc-aarch64-linux-gnu + + - name: Build agent (x86_64) + run: cargo build --release --target x86_64-unknown-linux-musl -p smolvm-agent + working-directory: . + + - name: Build agent (aarch64) + run: | + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc + cargo build --release --target aarch64-unknown-linux-musl -p smolvm-agent + working-directory: . + + - name: Verify static binaries + run: | + echo "x86_64 agent:" + file target/x86_64-unknown-linux-musl/release/smolvm-agent + + echo "aarch64 agent:" + file target/aarch64-unknown-linux-musl/release/smolvm-agent + + # ========================================================================== + # Summary job for branch protection + # ========================================================================== + ci-success: + name: CI Success + needs: [fmt, macos, linux, windows, agent] + runs-on: ubuntu-latest + if: always() + steps: + - name: Check all jobs passed + run: | + if [[ "${{ needs.fmt.result }}" != "success" ]] || \ + [[ "${{ needs.macos.result }}" != "success" ]] || \ + [[ "${{ needs.linux.result }}" != "success" ]] || \ + [[ "${{ needs.windows.result }}" != "success" ]] || \ + [[ "${{ needs.agent.result }}" != "success" ]]; then + echo "One or more jobs failed" + exit 1 + fi + echo "All CI jobs passed!" diff --git a/.github/workflows/pacman-repo.yml b/.github/workflows/pacman-repo.yml new file mode 100644 index 0000000..d27b72e --- /dev/null +++ b/.github/workflows/pacman-repo.yml @@ -0,0 +1,101 @@ +name: Publish pacman repo + +# Builds Arch packages from the release binaries and publishes them to the +# GitHub Pages `pacman-repo` branch, served at +# https://smol-machines.github.io/smolvm/pacman/$arch — a single stable +# repository endpoint (no per-arch release entries). See docs/install-arch.md. +# +# Triggers: +# - workflow_call: invoked by release.yml right after it creates the GitHub +# Release. This is the real automation path — the `release: published` event +# does NOT fire here on its own, because that release is created with the +# default GITHUB_TOKEN and GitHub does not start workflows from token-created +# releases (same reason release.yml pings the Homebrew tap explicitly). +# - workflow_dispatch: manual re-run / backfill for a given tag. +# - release: kept as a safety net for releases published by a human/PAT. +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: 'Release tag to package (e.g. v1.5.0)' + required: true + workflow_call: + inputs: + tag: + description: 'Release tag to package (e.g. v1.5.0)' + required: true + type: string + +permissions: + contents: write + +jobs: + publish: + runs-on: ubuntu-latest + container: + image: archlinux:base-devel + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Resolve tag + id: tag + run: | + TAG='${{ github.event.release.tag_name || inputs.tag }}' + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "pkgver=${TAG#v}" >> "$GITHUB_OUTPUT" + + - name: Install tooling + run: pacman -Syu --noconfirm pacman-contrib git + + - uses: actions/checkout@v4 + + - name: Prepare build user + run: | + useradd -m builder + echo 'builder ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/builder + cp -r packaging/arch /home/builder/build + chown -R builder /home/builder/build + + - name: Build packages (both arches; repackage-only, no compilation) + run: | + cd /home/builder/build + sed -i "s/^pkgver=.*/pkgver=${{ steps.tag.outputs.pkgver }}/; s/^pkgrel=.*/pkgrel=1/" PKGBUILD + sudo -u builder updpkgsums + sudo -u builder makepkg -df --noconfirm + sudo -u builder env CARCH=aarch64 makepkg -df --noconfirm --ignorearch + ls -l *.pkg.tar.zst + + - name: Publish to the GitHub Pages pacman repo (branch pacman-repo) + run: | + cd /home/builder/build + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + git config --global --add safe.directory '*' + # Pages serves the pacman-repo branch at https://.github.io// + # (Deploy from a branch); the pacman endpoint is pacman/$arch/. Pushing + # to the branch triggers the Pages build. + auth="https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git clone --depth 1 --branch pacman-repo "$auth" pages + for arch in x86_64 aarch64; do + dir="pages/pacman/$arch" + mkdir -p "$dir" + # Rolling repo: keep only the current version's package so the branch + # stays small. repo-add then builds a fresh single-entry db. + rm -f "$dir"/smolvm-*.pkg.tar.zst "$dir"/smol-machines.db* "$dir"/smol-machines.files* + cp smolvm-*-"$arch".pkg.tar.zst "$dir/" + ( + cd "$dir" + repo-add smol-machines.db.tar.gz smolvm-*-"$arch".pkg.tar.zst + # GitHub Pages does not serve broken symlinks — materialize the + # convenience `.db`/`.files` names as real copies of their targets. + cp --remove-destination "$(readlink -f smol-machines.db)" smol-machines.db + cp --remove-destination "$(readlink -f smol-machines.files)" smol-machines.files + ) + done + cd pages + git add -A + git commit -m "pacman: publish ${{ steps.tag.outputs.tag }}" \ + || { echo "no package changes to publish"; exit 0; } + git push origin pacman-repo diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml new file mode 100644 index 0000000..50ae297 --- /dev/null +++ b/.github/workflows/publish-crates.yml @@ -0,0 +1,91 @@ +name: Publish crates to crates.io + +# Publishes the smolvm *library* crates to crates.io so external consumers can +# `cargo add smolvm-protocol` (etc.) at the same version as the released engine. +# +# The crate versions track the workspace — they're bumped in lockstep with the +# engine — so this publishes whatever version is on the checked-out ref. Runs on +# a version tag (the engine release) so publishing can never lapse behind a +# release again, and on manual dispatch for a one-off catch-up. +# +# REQUIRES the `CARGO_REGISTRY_TOKEN` repo secret: a crates.io API token that +# owns/can-publish the smolvm-* + smolfile crates. Without it the job skips with +# a warning rather than failing a release. +# +# Publishes in dependency order (a dependency must be on crates.io before a +# dependent's `version = "x"` requirement can resolve): +# smolvm-protocol, smolvm-network (leaves) +# → smolvm-pack, smolvm-smolfile (need protocol) +# → smolvm-registry (needs pack) +# smolvm-cuda / smolvm-cuda-guest are intentionally NOT published. + +on: + push: + tags: ["v1.*"] + workflow_dispatch: + inputs: + dry_run: + description: "Package + verify only (cargo publish --dry-run); don't upload." + type: boolean + default: false + +permissions: + contents: read + +# Never let two publish runs race on the same crate versions. +concurrency: + group: publish-crates + cancel-in-progress: false + +jobs: + publish: + name: publish library crates + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - name: Publish library crates in dependency order + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + DRY_RUN: ${{ inputs.dry_run }} + run: | + set -euo pipefail + + if [ "${DRY_RUN:-false}" != "true" ] && [ -z "${CARGO_REGISTRY_TOKEN:-}" ]; then + echo "::warning::CARGO_REGISTRY_TOKEN is not set — skipping crates.io publish. \ + Add the secret to enable it." + exit 0 + fi + + # Dependency order: a dep must be published (and indexed) before a + # dependent, so its `version = "x"` requirement resolves on crates.io. + crates=(smolvm-protocol smolvm-network smolvm-pack smolvm-smolfile smolvm-registry) + + for c in "${crates[@]}"; do + name="$(grep -m1 '^name' "crates/$c/Cargo.toml" | sed -E 's/.*"([^"]+)".*/\1/')" + ver="$(grep -m1 '^version' "crates/$c/Cargo.toml" | sed -E 's/.*"([^"]+)".*/\1/')" + echo "::group::$name $ver" + + if [ "${DRY_RUN:-false}" = "true" ]; then + cargo publish -p "$name" --dry-run + echo "::endgroup::" + continue + fi + + # Idempotent: skip a version that's already on crates.io, so a re-run + # after a partial failure resumes instead of erroring on duplicates. + if curl -sf -H "User-Agent: smolvm-publish (ops@smolmachines.com)" \ + "https://crates.io/api/v1/crates/$name/$ver" >/dev/null 2>&1; then + echo "$name $ver already on crates.io — skipping" + echo "::endgroup::" + continue + fi + + # `cargo publish` blocks until the new version is in the registry + # index, so the next (dependent) crate can resolve it. + cargo publish -p "$name" + echo "published $name $ver" + echo "::endgroup::" + done diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ff61a9f --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,460 @@ +name: Release + +on: + push: + tags: ['v*'] + +env: + CARGO_TERM_COLOR: always + +jobs: + # ========================================================================== + # Run CI checks first + # ========================================================================== + ci: + uses: ./.github/workflows/ci.yml + + # ========================================================================== + # Build agent binaries (both architectures) + # ========================================================================== + build-agent: + name: Build agent (${{ matrix.arch }}) + needs: ci + runs-on: ubuntu-latest + strategy: + matrix: + include: + - arch: x86_64 + target: x86_64-unknown-linux-musl + linker_pkg: "" + linker_env: "" + - arch: aarch64 + target: aarch64-unknown-linux-musl + linker_pkg: gcc-aarch64-linux-gnu + linker_env: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=aarch64-linux-gnu-gcc + steps: + - uses: actions/checkout@v6 + + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Cache cargo + uses: actions/cache@v5 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: agent-${{ matrix.arch }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: agent-${{ matrix.arch }}-cargo- + + - name: Install cross-compilation tools + if: matrix.linker_pkg != '' + run: | + sudo apt-get update + sudo apt-get install -y musl-tools ${{ matrix.linker_pkg }} + + - name: Install musl-tools + if: matrix.linker_pkg == '' + run: | + sudo apt-get update + sudo apt-get install -y musl-tools + + - name: Build agent + run: | + ${{ matrix.linker_env }} \ + cargo build --profile release-small -p smolvm-agent --target ${{ matrix.target }} + + - name: Verify static binary + run: file target/${{ matrix.target }}/release-small/smolvm-agent + + - uses: actions/upload-artifact@v7 + with: + name: agent-${{ matrix.arch }} + path: target/${{ matrix.target }}/release-small/smolvm-agent + + # ========================================================================== + # Build agent rootfs (both architectures) + # ========================================================================== + build-rootfs: + name: Build rootfs (${{ matrix.arch }}) + needs: build-agent + runs-on: ubuntu-latest + strategy: + matrix: + include: + - arch: aarch64 + - arch: x86_64 + steps: + - uses: actions/checkout@v6 + + - name: Download agent binary + uses: actions/download-artifact@v8 + with: + name: agent-${{ matrix.arch }} + path: /tmp/agent/ + + - name: Build rootfs + run: | + chmod +x /tmp/agent/smolvm-agent + AGENT_BINARY=/tmp/agent/smolvm-agent \ + ./scripts/build-agent-rootfs.sh --arch ${{ matrix.arch }} --no-build-agent + + - name: Package rootfs as tarball + run: | + # Tar preserves ownership/permissions without needing read access + # to restricted dirs like /var/run/chrony (owned by chrony:chrony 700) + sudo tar -cf target/agent-rootfs.tar -C target/agent-rootfs . + + - uses: actions/upload-artifact@v7 + with: + name: rootfs-${{ matrix.arch }} + path: target/agent-rootfs.tar + + # ========================================================================== + # Build distribution tarballs (per platform) + # ========================================================================== + build-dist: + name: Build dist (${{ matrix.platform }}) + needs: [build-agent, build-rootfs] + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - platform: darwin-arm64 + runner: macos-14 + rootfs_arch: aarch64 + agent_arch: aarch64 + lib_dir: lib + # Linux dist binaries build on 22.04 (glibc 2.35), NOT ubuntu-latest + # (24.04 = glibc 2.39). Symbol versioning is backward-compatible only, + # so the build host's glibc is the MINIMUM a user needs at runtime — + # a 2.39-built binary dies on 22.04 with "GLIBC_2.39 not found". 2.35 + # covers Ubuntu 22.04+ / Debian 12+ / RHEL 9+. + - platform: linux-x86_64 + runner: ubuntu-22.04 + rootfs_arch: x86_64 + agent_arch: x86_64 + lib_dir: lib/linux-x86_64 + - platform: linux-arm64 + runner: ubuntu-22.04-arm + rootfs_arch: aarch64 + agent_arch: aarch64 + lib_dir: lib/linux-aarch64 + steps: + - uses: actions/checkout@v6 + with: + lfs: true + submodules: ${{ startsWith(matrix.platform, 'linux-') && 'recursive' || 'false' }} + + - name: Pull LFS files + run: git lfs pull + + - uses: dtolnay/rust-toolchain@stable + + - name: Cache cargo + uses: actions/cache@v5 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + # Include lib dir hash so libkrun updates bust the cache + key: dist-${{ matrix.platform }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ hashFiles(format('{0}/*', matrix.lib_dir)) }} + restore-keys: dist-${{ matrix.platform }}-cargo- + + # Force relink if cached binary was linked against a different libkrun + - name: Clean stale build artifacts + run: rm -f target/release/smolvm target/release/deps/smolvm-* + + - name: Install build dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + # patchelf: build-dist.sh strips libkrun's hard libvirglrenderer NEEDED + # so non-GPU Linux hosts can dlopen it (GPU stays optional at runtime). + # Without patchelf the strip is silently skipped and `smolvm run` fails + # with "libvirglrenderer.so.1: cannot open shared object file". + sudo apt-get install -y build-essential libssl-dev pkg-config e2fsprogs patchelf + + - name: Install build dependencies (macOS) + if: runner.os == 'macOS' + run: brew install e2fsprogs + + - name: Verify LFS libraries + run: | + echo "Library directory: ${{ matrix.lib_dir }}" + ls -la ${{ matrix.lib_dir }}/ + file ${{ matrix.lib_dir }}/* + + - name: Download rootfs + uses: actions/download-artifact@v8 + with: + name: rootfs-${{ matrix.rootfs_arch }} + path: /tmp/rootfs-dl/ + + - name: Extract rootfs tarball + run: | + mkdir -p target/agent-rootfs + sudo tar -xf /tmp/rootfs-dl/agent-rootfs.tar -C target/agent-rootfs + + - name: Download agent binary + uses: actions/download-artifact@v8 + with: + name: agent-${{ matrix.agent_arch }} + path: /tmp/agent/ + + - name: Place agent binary for build-dist.sh + run: | + mkdir -p target/release-small + cp /tmp/agent/smolvm-agent target/release-small/smolvm-agent + chmod +x target/release-small/smolvm-agent + + # Import Developer ID certificate if available (macOS only). + # Set secrets APPLE_CERTIFICATE_P12 (base64-encoded .p12) and + # APPLE_CERTIFICATE_PASSWORD in your GitHub repo settings. + - name: Import signing certificate (macOS) + if: runner.os == 'macOS' && env.APPLE_CERTIFICATE_P12 != '' + env: + APPLE_CERTIFICATE_P12: ${{ secrets.APPLE_CERTIFICATE_P12 }} + APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} + run: | + echo "$APPLE_CERTIFICATE_P12" | base64 --decode > /tmp/certificate.p12 + KEYCHAIN_PASSWORD="$(openssl rand -hex 16)" + security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain + security default-keychain -s build.keychain + security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain + security import /tmp/certificate.p12 -k build.keychain \ + -P "$APPLE_CERTIFICATE_PASSWORD" -T /usr/bin/codesign + security set-key-partition-list -S apple-tool:,apple: \ + -s -k "$KEYCHAIN_PASSWORD" build.keychain + rm /tmp/certificate.p12 + + # Check out the unified `smol` CLI (a separate repo) into ./smol so + # build-dist.sh builds and bundles it alongside the engine — one tarball + # serves both `smol` and `smolvm`. Requires repo secret SMOL_REPO_TOKEN + # with read access to smol-machines/smol. If unset, the step is skipped and + # an engine-only tarball is produced. Ref is left at the smol default + # branch (its versioning is independent of the engine tag); pin a ref here + # if you want reproducible smol-CLI releases. + - name: Check out smol CLI source + if: env.SMOL_REPO_TOKEN != '' + uses: actions/checkout@v6 + with: + repository: smol-machines/smol + path: smol + token: ${{ secrets.SMOL_REPO_TOKEN }} + env: + SMOL_REPO_TOKEN: ${{ secrets.SMOL_REPO_TOKEN }} + + # No separate guest-init build: libkrun 2.0.0 embeds the guest init + # (krun-init) directly in libkrun.so (INIT_BLOB is on by default), so the + # old per-arch `cc init/init.c` step is gone — init.c no longer exists in + # the 2.0.0 submodule, and a VM boots from the embedded init with no + # external init.krun (verified). build-dist.sh no longer bundles init.krun. + + - name: Build distribution tarball + run: ./scripts/build-dist.sh --skip-agent-build + env: + LIB_DIR: ${{ matrix.lib_dir }} + CODESIGN_IDENTITY: ${{ secrets.APPLE_CERTIFICATE_P12 != '' && secrets.CODESIGN_IDENTITY || '-' }} + + # Notarize the macOS binary if Developer ID signing was used. + # Requires secrets: APPLE_ID, APPLE_TEAM_ID, APPLE_APP_PASSWORD. + - name: Notarize (macOS) + if: runner.os == 'macOS' && env.APPLE_CERTIFICATE_P12 != '' + env: + APPLE_CERTIFICATE_P12: ${{ secrets.APPLE_CERTIFICATE_P12 }} + APPLE_ID: ${{ secrets.APPLE_ID }} + APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }} + APPLE_APP_PASSWORD: ${{ secrets.APPLE_APP_PASSWORD }} + run: | + TARBALL=$(ls dist/smolvm-*.tar.gz) + xcrun notarytool submit "$TARBALL" \ + --apple-id "$APPLE_ID" \ + --team-id "$APPLE_TEAM_ID" \ + --password "$APPLE_APP_PASSWORD" \ + --wait + + - name: Verify bundled libraries match source + run: | + VERSION="$(grep '^version' Cargo.toml | head -1 | cut -d'"' -f2)" + DIST_LIB="dist/smolvm-${VERSION}-${{ matrix.platform }}/lib" + echo "Source libraries:" + ls -la ${{ matrix.lib_dir }}/ + echo "Bundled libraries:" + ls -la "$DIST_LIB"/ + # Verify sizes match for all bundled libraries (catches stale LFS or copy issues). + # Covers both required libs (libkrun, libkrunfw) and optional GPU libs. + for lib in "$DIST_LIB"/lib*; do + [[ -L "$lib" ]] && continue # skip symlinks + name=$(basename "$lib") + src="${{ matrix.lib_dir }}/$name" + if [[ -f "$src" ]]; then + src_size=$(wc -c < "$src") + dst_size=$(wc -c < "$lib") + if [[ "$src_size" != "$dst_size" ]]; then + echo "ERROR: $name size mismatch: source=$src_size bundled=$dst_size" + exit 1 + fi + echo "OK: $name ($dst_size bytes)" + fi + done + + - name: List distribution contents + run: | + VERSION="$(grep '^version' Cargo.toml | head -1 | cut -d'"' -f2)" + ls -la "dist/smolvm-${VERSION}-"*/ + + - uses: actions/upload-artifact@v7 + with: + name: dist-${{ matrix.platform }} + path: dist/smolvm-*.tar.gz + + # ========================================================================== + # Build Windows distribution (cross-compiled; zipped) + # ========================================================================== + build-dist-windows: + name: Build dist (windows-x86_64) + needs: [build-agent, build-rootfs] + runs-on: ubuntu-latest + env: + CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER: x86_64-w64-mingw32-gcc + steps: + - uses: actions/checkout@v6 + with: + lfs: true + + - name: Pull LFS files + run: git lfs pull + + - uses: dtolnay/rust-toolchain@stable + with: + targets: x86_64-pc-windows-gnu + + - name: Install build dependencies + run: | + sudo apt-get update + # mingw: cross-link smolvm.exe; clang/libclang: bindgen in build deps + # (matches the CI windows job); e2fsprogs: pre-format disk templates + # (Windows has no host mkfs.ext4); zip: package the dist. + sudo apt-get install -y gcc-mingw-w64-x86-64 clang libclang-dev e2fsprogs zip + + - name: Verify Windows libraries + run: | + ls -la lib/windows-x86_64/ + file lib/windows-x86_64/*.dll + + - name: Download rootfs (x86_64) + uses: actions/download-artifact@v8 + with: + name: rootfs-x86_64 + path: /tmp/rootfs-dl/ + + - name: Extract rootfs + run: | + mkdir -p target/agent-rootfs + sudo tar -xf /tmp/rootfs-dl/agent-rootfs.tar -C target/agent-rootfs + + - name: Download agent binary (x86_64) + uses: actions/download-artifact@v8 + with: + name: agent-x86_64 + path: /tmp/agent/ + + # The rootfs artifact ships agent-less; inject the freshly built agent and + # repoint /sbin/init at it (same as the Unix build-dist.sh path). + - name: Inject agent into rootfs + run: | + sudo cp /tmp/agent/smolvm-agent target/agent-rootfs/usr/local/bin/smolvm-agent + sudo chmod +x target/agent-rootfs/usr/local/bin/smolvm-agent + sudo ln -sf /usr/local/bin/smolvm-agent target/agent-rootfs/sbin/init + + - name: Build Windows distribution + run: ./scripts/build-dist-windows.sh + env: + LIB_DIR: lib/windows-x86_64 + + - uses: actions/upload-artifact@v7 + with: + name: dist-windows-x86_64 + path: dist/smolvm-*-windows-x86_64.zip + + # ========================================================================== + # Create GitHub Release + # ========================================================================== + release: + name: Create Release + needs: [build-dist, build-dist-windows] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v6 + + - name: Download all dist artifacts + uses: actions/download-artifact@v8 + with: + pattern: dist-* + path: dist/ + merge-multiple: true + + - name: List release artifacts + run: ls -la dist/ + + - name: Generate checksums + run: | + cd dist + sha256sum *.tar.gz *.zip > checksums.sha256 + cat checksums.sha256 + + - name: Create GitHub Release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${{ github.ref_name }}" \ + --title "smolvm ${{ github.ref_name }}" \ + --generate-notes \ + dist/*.tar.gz \ + dist/*.zip \ + dist/checksums.sha256 + + # Ping the Homebrew tap to bump its formula to this release immediately, + # instead of waiting for its scheduled poll. Best-effort: if the token + # secret isn't configured, this no-ops (the tap's cron picks the release up + # within a few hours anyway) and never fails the release. The token is a + # fine-grained PAT with Contents:write on smol-machines/homebrew-tap. + - name: Notify Homebrew tap of the new release + env: + GH_TOKEN: ${{ secrets.HOMEBREW_TAP_DISPATCH_TOKEN }} + TAG: ${{ github.ref_name }} + run: | + if [ -z "${GH_TOKEN:-}" ]; then + echo "HOMEBREW_TAP_DISPATCH_TOKEN not set — skipping instant bump (the tap polls on a schedule)." + exit 0 + fi + version="${TAG#v}" + gh api --method POST repos/smol-machines/homebrew-tap/dispatches \ + -f event_type=smolvm-release \ + -f "client_payload[version]=$version" + echo "Dispatched smolvm-release ($version) to the Homebrew tap." + + # Build + publish the Arch/pacman packages for this release. Invoked directly + # (not via the `release: published` event) because that event never fires for a + # release created with the default GITHUB_TOKEN — the same GitHub restriction + # the Homebrew step above works around. Runs after the release exists so the + # PKGBUILD can download its release binaries. + publish-pacman: + name: Publish pacman repo + needs: release + permissions: + contents: write + uses: ./.github/workflows/pacman-repo.yml + with: + tag: ${{ github.ref_name }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2f7242b --- /dev/null +++ b/.gitignore @@ -0,0 +1,48 @@ +# Rust +/target/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo +*~ + +# macOS +.DS_Store + +# Build artifacts +*.so +*.a +# ...except the vendored runtime libs (else base libkrun.so/libkrunfw.so get +# dropped, leaving dangling symlinks that break `hashFiles` in release CI). +!lib/linux-x86_64/*.so +!lib/linux-aarch64/*.so + +# Distribution build output +/dist/smolvm-*/ +/dist/*.tar.gz + +# Nix/devshell artifacts +/.direnv/ +/result +/result-* + +# Bundled libraries are in the /lib (large binaries - download or build locally) +# These are bundled into release tarballs by build-dist.sh + +# Agent rootfs is generated in target/ by build-agent-rootfs.sh + +# Local config +.env +.env.local + +# Test artifacts +/tmp/ +*.log + +# smol machines artifacts +*.smolmachine + +# Registry infrastructure (separate git repo) +/smolmachines-registry/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..1962663 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,9 @@ +[submodule "libkrun"] + path = libkrun + url = git@github.com:smol-machines/libkrun.git +[submodule "libkrunfw"] + path = libkrunfw + url = git@github.com:smol-machines/libkrunfw.git +[submodule "smolvm-sdk"] + path = smolvm-sdk + url = git@github.com:smol-machines/smolvm-sdk.git diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..fe9edda --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,610 @@ +# smolvm — Agent Reference + +A tool to build and run portable, self-contained virtual machines locally. <200ms boot time. No daemon, no Docker. + +## Platform Support + +| Host | Guest | Hypervisor | Requirements | +|------|-------|------------|--------------| +| macOS Apple Silicon | arm64 Linux | Hypervisor.framework | macOS 11+ | +| Linux x86_64 / aarch64 | matching Linux | KVM | `/dev/kvm` | +| Windows x86_64 | x86_64 Linux | Windows Hypervisor Platform (WHP) | WHP feature enabled | + +Windows caveats (run, persistent machines, volumes, port-forwarding, pack create/run, and interactive TTY all work): networking is TSI-only (TCP/UDP + inbound `-p`, no virtio-net); no GPU acceleration; no fork/snapshot. `pack create` needs `storage-template.ext4` / `overlay-template.ext4` beside `smolvm.exe` (Windows has no host `mkfs.ext4`). Set `SMOLVM_LIB_DIR` (folder holding `krun.dll` + `libkrunfw.dll`) and `SMOLVM_AGENT_ROOTFS` when running from a non-standard layout. + +## Quick Reference + +```bash +# Ephemeral (cleaned up after exit) +smolvm machine run --net --image alpine -- echo hello +smolvm machine run --net -it --image alpine -- /bin/sh # interactive shell +smolvm machine run --net --image python:3.12-alpine -- python3 script.py + +# Persistent (survives across exec sessions and stop/start) +smolvm machine create --net --name myvm +smolvm machine start --name myvm +smolvm machine exec --name myvm -- apk add python3 # installs persist +smolvm machine exec --name myvm -- which python3 # still there +smolvm machine shell --name myvm # interactive shell (auto-starts if stopped) +smolvm machine stop --name myvm +smolvm machine delete --name myvm + +# Image-based persistent (filesystem changes persist across exec sessions) +smolvm machine create --net --image ubuntu --name myvm +smolvm machine start --name myvm +smolvm machine exec --name myvm -- apt-get update +smolvm machine exec --name myvm -- apt-get install -y python3 +smolvm machine exec --name myvm -- which python3 # still there after exit+re-exec + +# SSH agent forwarding (git/ssh without exposing keys) — see "SSH Agent Forwarding" below +smolvm machine run --ssh-agent --net --image alpine -- sh -c "apk add -q openssh-client && ssh-add -l" +smolvm machine create --name myvm --image alpine --ssh-agent --net # persistent: then start + exec git/ssh + +# Inject secrets into workload env (referenced from host env var / file) +smolvm machine run --secret-env OPENAI_API_KEY=OPENAI_API_KEY -- ./app +smolvm machine run -s Smolfile -- ./app # Smolfile [secrets] resolves at launch + +# Pack into portable executable +smolvm pack create --image python:3.12-alpine -o ./my-python +./my-python run -- python3 -c "print('hello')" + +# Create machine from packed artifact (fast start, no pull) +smolvm machine create --name my-vm --from ./my-python.smolmachine +smolvm machine start --name my-vm +smolvm machine exec --name my-vm -- pip install requests + +# Use local container images (CI, air-gapped, fast iteration) +docker save myapp:latest -o myapp.tar +smolvm machine run --image ./myapp.tar -- ./app # from a docker/podman save archive +docker save myapp:latest | smolvm machine run --image - -- ./app # from stdin +smolvm machine run --image ./rootfs/ -- ./app # from an unpacked rootfs dir +smolvm machine create --name myvm --image ./myapp.tar # persistent, from a local archive +``` + +## When to Use What + +| Goal | Command | +|------|---------| +| Run a one-off command in isolation | `smolvm machine run --net --image IMAGE -- CMD` | +| Interactive shell (ephemeral) | `smolvm machine run --net -it --image IMAGE -- /bin/sh` | +| Interactive shell (persistent) | `smolvm machine shell --name NAME` | +| Persistent dev environment | `machine create` → `machine start` → `machine exec` | +| Ship software as a binary | `smolvm pack create --image IMAGE -o OUTPUT` | +| Fast persistent machine from packed artifact | `machine create --name NAME --from FILE.smolmachine` | +| Use local container images (CI / air-gapped / fast iteration) | `--image ./archive.tar`, `--image -` (stdin), or `--image ./rootfs/` | +| Use git/ssh with private keys safely | Add `--ssh-agent` to run or create | +| Inject API keys / tokens without putting them on the command line | `--secret-env`/`--secret-file` flags or Smolfile `[secrets]` | +| Minimal VM without image | `smolvm machine run -s Smolfile` (bare VM) | +| Change mounts/ports/resources on existing VM | `machine update --name NAME -v ./src:/app -p 8080:8080` | +| Declarative VM config | Create a Smolfile, use `--smolfile`/`-s` flag | + +### Persistence Model + +- **`machine run`** — ephemeral. All changes are discarded when the command exits. +- **`machine exec`** — persistent. Filesystem changes (package installs, config edits) persist across exec sessions for the same machine, whether bare or image-based. Changes are stored in an overlay on the machine's storage disk. +- **`machine stop` + `start`** — changes persist across restarts. The persistent overlay is remounted preserving previous changes. +- **`pack run`** — ephemeral. Each run starts fresh from the packed image. +- **`pack start` + `exec`** — daemon mode. `/workspace` persists across exec sessions and stop/start. Container overlay resets per exec (package installs don't persist — use `/workspace` for durable data). +- **`machine create --from .smolmachine`** — creates a persistent named machine from a packed artifact. Boots from pre-extracted layers (~250ms, no image pull). Full `machine exec` persistence — package installs, file writes all survive across exec and stop/start. + +## CLI Structure + +All commands use named flags (no positional args except `machine create --name NAME` and `machine delete --name NAME`). + +``` +smolvm machine run --image IMAGE [-- COMMAND] # ephemeral +smolvm machine exec --name NAME [-- COMMAND] # run in existing VM +smolvm machine shell [--name NAME] # interactive shell (auto-starts) +smolvm machine create --name NAME [OPTIONS] # create persistent +smolvm machine create --name NAME --from FILE.smolmachine # from packed artifact +smolvm machine start [--name NAME] # start (default: "default") +smolvm machine stop [--name NAME] # stop +smolvm machine delete --name NAME [-f] # delete +smolvm machine status [--name NAME] # check state +smolvm machine ls [--json] # list all +smolvm machine update --name NAME [OPTIONS] # modify stopped machine settings +smolvm machine cp SRC DST # copy files (host↔VM) +smolvm machine exec --stream --name NAME -- CMD # streaming output +smolvm machine monitor [--name NAME] # foreground health + restart + +smolvm pack create --image IMAGE -o PATH # package +smolvm pack create --from-vm NAME -o PATH # pack from VM snapshot +smolvm pack run [--sidecar PATH] [-- CMD] # run .smolmachine + +smolvm serve start [--listen ADDR:PORT|PATH] # HTTP API +smolvm config registries edit # registry auth + +# Secrets are references to host env vars / files, resolved at launch — no +# built-in store. Attach them on the command line or in a Smolfile [secrets]. +smolvm machine run --secret-env GUEST_VAR=HOST_VAR # from host env var +smolvm machine run --secret-file GUEST_VAR=/abs/path # from host file +smolvm machine create --name NAME --secret-env GUEST_VAR=HOST_VAR # persists the ref +smolvm machine exec --name NAME --secret-env GUEST_VAR=HOST_VAR -- cmd +``` + +## Artifact References + +Artifact references follow OCI conventions and support both tags and digests: + +``` +python-dev # bare name (default registry + latest) +python-dev:v1.0 # name + tag +binsquare/custom:v1 # namespace + name + tag +smolmachines.com/python-dev:latest # registry + name + tag +smolmachines.com/binsquare/custom:v1 # registry + namespace + name + tag +python-dev@sha256:abcdef0123... # digest reference (immutable) +``` + +Default registry: `registry.smolmachines.com`. Digest references require `sha256:` followed by exactly 64 hex characters. + +### Local container images + +`--image` also accepts a local source — useful for CI, air-gapped hosts, and fast +local iteration. smolvm stays a microVM runtime and delegates all image work +(flatten, whiteouts, config) to container tooling (`crane`/`docker`/`podman`); the +archive is flattened with `crane export`. + +``` +./image.tar ./image.tar.gz ./image.tgz # a `docker save` / `podman save` archive (gzip ok) +- # the same archive streamed on stdin +./rootfs/ # an already-unpacked root filesystem directory +``` + +A source is treated as local when it starts with `/`, `./`, `../`, is `-`, or ends in +`.tar`/`.tar.gz`/`.tgz`; everything else is a registry reference (so bare `alpine` +still pulls). Archives are cached content-addressed by hash and re-resolved on +`machine start`. `--image -` cannot be combined with `-i`/`-t` (both read stdin). + +smolvm boots images, it does not build them: a Dockerfile passed to `--image` is +rejected with a hint to build first (`docker build … && docker save … | … --image -`). + +## Key Flags + +| Flag | Short | Used on | Description | +|------|-------|---------|-------------| +| `--image` | `-I` | run, create, pack create | OCI image, or a local source: a `docker save` archive (`./img.tar`, or `-` for stdin) or unpacked rootfs dir (`./rootfs/`) | +| `--name` | `-n` | run, start, stop, status, exec, update | Machine name (default: "default") | +| `--net` | | run, create | Enable outbound networking (off by default) | +| `--gpu` | | run, create | Enable GPU acceleration (Vulkan via virtio-gpu) | +| `--gpu-vram` | | run, create | GPU shared-memory region size in MiB (default: 4096). Ignored without `--gpu`. | +| `--volume` | `-v` | run, create, update | Mount host dir: `HOST:GUEST[:ro]` | +| `--port` | `-p` | run, create, update | Port mapping: `HOST:GUEST` | +| `--smolfile` | `-s` | run, create, pack create | Load config from Smolfile | +| `--interactive` | `-i` | run, exec | Keep stdin open | +| `--tty` | `-t` | run, exec | Allocate pseudo-TTY | +| `--allow-cidr` | | run, create | CIDR egress filter (implies --net) | +| `--allow-host` | | run, create | Hostname egress filter, resolved at VM start (implies --net) | +| `--ssh-agent` | | run, create | Forward host SSH agent (git/ssh without exposing keys) | + +## Smolfile Reference + +A Smolfile is a TOML file declaring a VM workload. Use with `--smolfile`/`-s`. + +```toml +# Top-level: workload definition +image = "python:3.12-alpine" # OCI image (omit for bare Alpine) +entrypoint = ["/app/run"] # overrides image ENTRYPOINT +cmd = ["serve"] # overrides image CMD +env = ["PORT=8080", "DEBUG=1"] # environment variables +workdir = "/app" # working directory + +# Resources +cpus = 2 # vCPUs (default: 4) +memory = 1024 # MiB (default: 8192, elastic via balloon) +net = true # outbound networking (default: false) +gpu = true # GPU acceleration (default: false) +gpu_vram = 4096 # GPU VRAM MiB (default: 4096, ignored unless gpu=true) +storage = 40 # storage disk GiB (default: 20) +overlay = 4 # overlay disk GiB (default: 2) + +# Network policy — egress filtering by hostname and/or CIDR +[network] +allow_hosts = ["api.stripe.com"] # resolved at VM start (implies net) +allow_cidrs = ["10.0.0.0/8"] # IP/CIDR ranges (implies net) + +# Dev profile (used by `machine run` and `machine create`) +[dev] +volumes = ["./src:/app"] # host bind mounts +ports = ["8080:8080"] # port forwarding +init = ["pip install -r requirements.txt"] # run on every VM start +env = ["APP_MODE=dev"] # dev-only env (extends top-level) +workdir = "/app" # dev-only workdir + +# Artifact profile (used by `pack create`) +[artifact] +cpus = 4 # override resources for distribution +memory = 2048 +entrypoint = ["/app/run"] # override entrypoint for packed binary +oci_platform = "linux/amd64" # target OCI platform + +# Health check (used by `machine monitor`) +[health] +exec = ["curl", "-f", "http://127.0.0.1:8080/health"] +interval = "10s" +timeout = "2s" +retries = 3 +startup_grace = "20s" + +# Credential forwarding +[auth] +ssh_agent = true # forward host SSH agent into the VM + +# Secrets — references to host sources, resolved at workload launch +[secrets] +DATABASE_URL = { from_env = "PROD_DB_URL" } # host env var (at launch) +GCP_CREDS = { from_file = "/abs/creds.json" } # host file (at launch) +``` + +### Merge Precedence + +CLI flags override Smolfile values: + +``` +image: --image flag > Smolfile image > None (bare Alpine) +entrypoint: Smolfile entrypoint > image metadata +cmd: trailing args (after --) > Smolfile cmd > image metadata +env: top-level env + [dev].env + CLI -e (all merged) +volumes: [dev].volumes + CLI -v (all merged) +ports: [dev].ports + CLI -p (all merged) +init: [dev].init + CLI --init (all merged) +cpus/mem: CLI flag > Smolfile > defaults (4 CPU, 8192 MiB) +``` + +## Networking + +- **Off by default** — VMs have no outbound access unless `--net` is specified +- `--net` enables full outbound (TCP/UDP, DNS) +- `--allow-host api.stripe.com` enables egress only to resolved IPs of that hostname (implies `--net`). Also enables DNS filtering — only allowed hostnames can be resolved. +- `--allow-cidr 10.0.0.0/8` enables egress only to specified IP ranges (implies `--net`) +- `--allow-host` and `--allow-cidr` can be combined and used multiple times +- `--outbound-localhost-only` restricts to 127.0.0.0/8 and ::1 (implies `--net`) +- `-p HOST:GUEST` forwards a host port to the VM (TCP) +- Smolfile: use `[network] allow_hosts` and `[network] allow_cidrs` + +### Proxy Support + +Pass proxy settings into VMs with `-e` when behind a corporate proxy or VPN: + +```bash +smolvm machine run --net \ + -e https_proxy=http://proxy.corp:3128 \ + -e http_proxy=http://proxy.corp:3128 \ + -e no_proxy=localhost,127.0.0.1 \ + --image alpine -- wget -q -O /dev/null https://example.com +``` + +Or declare them in a Smolfile: + +```toml +net = true +env = [ + "https_proxy=http://proxy.corp:3128", + "http_proxy=http://proxy.corp:3128", + "no_proxy=localhost,127.0.0.1" +] +``` + +Proxy vars are NOT forwarded automatically — each VM gets exactly the env you specify. The VM uses the host's DNS server (from `/etc/resolv.conf`) for name resolution. + +## SSH Agent Forwarding + +Forward the host's SSH agent into the VM so git, ssh, and scp work with your keys — without the private keys ever entering the VM. + +```bash +# Quick check the agent is forwarded (alpine needs openssh-client for ssh-add): +smolvm machine run --ssh-agent --net --image alpine -- sh -c "apk add -q openssh-client && ssh-add -l" + +# Persistent VM — create, start, install tooling, then use the agent: +smolvm machine create --name myvm --image alpine --ssh-agent --net +smolvm machine start --name myvm +smolvm machine exec --name myvm -- apk add -q git openssh-client + +# Smolfile +# [auth] +# ssh_agent = true +``` + +Inside the VM, `SSH_AUTH_SOCK` is set automatically. Any tool that uses the SSH agent protocol (git, ssh, scp) works transparently: + +```bash +smolvm machine exec --name myvm -- git clone git@github.com:org/private-repo.git +smolvm machine exec --name myvm -- ssh deploy@server "systemctl restart app" +``` + +The host SSH agent signs challenges but never sends private keys across the boundary. Even with root inside the VM, keys cannot be extracted — this is enforced by the SSH agent protocol and the hypervisor isolation. + +Requires `SSH_AUTH_SOCK` to be set on the host. If missing, smolvm exits with an error and remediation instructions. + +## GPU Acceleration + +Enable the host GPU inside a VM with `--gpu`. Guest Vulkan talks to the host GPU via virtio-gpu/Venus; ANGLE uses it as the WebGL/OpenGL ES backend. + +Not available on Windows (WHP) — `--gpu` has no effect there. + +**Host setup:** +- macOS — bundled, no extra installs needed. +- Linux — install virglrenderer from the system package manager before use: + - Alpine: `apk add virglrenderer mesa-vulkan-intel` (or `mesa-vulkan-ati` for AMD) + - Debian/Ubuntu: `apt install virglrenderer0 mesa-vulkan-drivers` + +```bash +# One-shot GPU workload +smolvm machine run --gpu --image alpine -- sh -c ' + apk add --no-cache mesa-vulkan-virtio vulkan-tools + VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/virtio_icd.x86_64.json \ + vulkaninfo --summary 2>/dev/null | grep deviceName +' +# → deviceName = Virtio-GPU Venus (Intel(R) UHD Graphics ...) + +# Persistent GPU machine +smolvm machine create --name browser --gpu --gpu-vram 2048 +smolvm machine start --name browser +smolvm machine exec --name browser -- \ + chromium --headless=new --no-sandbox --use-gl=angle --use-angle=vulkan \ + --screenshot=/tmp/out.png --window-size=1280,800 https://example.com +``` + +The guest must set `VK_ICD_FILENAMES` so the Vulkan loader finds the virtio ICD. Put it in `env` in a Smolfile to avoid repeating it on every exec: + +```toml +gpu = true +gpu_vram = 2048 +env = ["VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/virtio_icd.x86_64.json"] +``` + +For a complete working example see [`examples/headless-browser/browser.smolfile`](examples/headless-browser/browser.smolfile). + +## CUDA (experimental) + +`--cuda` remotes guest CUDA **Driver-API** calls to the host NVIDIA GPU over +vsock — separate from `--gpu` (Vulkan graphics). The host needs a working +NVIDIA driver (`libcuda.so.1` + loaded kernel module); no CUDA toolkit is +required on host or guest. Enable with `--cuda` on `machine run`/`create`, or +`cuda = true` in a Smolfile. + +Guest programs reach the GPU through the drop-in driver library built from +[`crates/smolvm-cuda-shim`](crates/smolvm-cuda-shim) (a `cdylib` with soname +`libcuda.so.1`). Mount or install it in the guest and unmodified Driver-API +programs work with no code changes: + +```bash +cargo build --release -p smolvm-cuda-shim # → target/release/libcuda.so +mkdir -p /tmp/shim && cp target/release/libcuda.so /tmp/shim/libcuda.so.1 + +smolvm machine run --net --cuda -v /tmp/shim:/opt/shim:ro --image debian:bookworm-slim -- \ + sh -c 'gcc myapp.c -o myapp -L/opt/shim -l:libcuda.so.1 && LD_LIBRARY_PATH=/opt/shim ./myapp' +``` + +Covered surface: init/device queries (name, attributes, uuid, total/free mem), +contexts (incl. the primary-context flow the CUDA runtime uses), module +load/unload (PTX, cubin, fatbin), mem alloc/free/HtoD/DtoH/DtoD/memset, +kernel launch (param sizes come from the host via `cuFuncGetParamInfo`, +CUDA 12.4+ drivers), streams, events, `cuGetProcAddress`. All work executes +synchronously host-side; `*Async` calls complete before returning (permitted +by the CUDA contract). + +**This covers programs written against the Driver API (the `cu*` C API), not +the Runtime API.** A program built with `nvcc` (or PyTorch, RAPIDS, etc.) links +NVIDIA's `libcudart`, which bootstraps by requesting a *private* internal +driver interface via `cuGetExportTable` (a versioned UUID whose function ABIs +are undocumented). A pure `libcuda` shim cannot provide it, so `libcudart` +aborts during context init. Hosting Runtime-API workloads therefore requires +remoting at the `libcudart` level instead — a separate, larger effort (see +`docs/cuda-support-plan.md`, Phase 4). Set `SMOLVM_CUDA_SHIM_TRACE=1` to log +which driver entry points a program resolves through the shim. + +Without an NVIDIA driver the host serves a CPU-emulation backend (test-only: +it knows the `vecadd` test kernel), so the transport stays testable anywhere. +## Secrets + +smolvm stores no secret material. A secret is a *reference* to a value that +already lives on the host — a host environment variable or a host file — and is +resolved into the workload's process environment at launch time. Bring your own +secrets manager (Vault, 1Password, AWS, sops, your shell): render the value into +an env var or file, then point a ref at it. Only the reference is ever +persisted; the resolved value never lands in the VM record, the database, or a +`.smolmachine` pack. + +Attach refs on the command line: + +```bash +# From a host environment variable (GUEST_VAR=HOST_VAR) +smolvm machine run --secret-env OPENAI_API_KEY=OPENAI_API_KEY -- ./app +smolvm machine create --name web --secret-env DATABASE_URL=PROD_DB_URL # persists the ref +smolvm machine exec --name web --secret-env TOKEN=CI_TOKEN -- ./deploy + +# From a host file (GUEST_VAR=/absolute/path) +smolvm machine run --secret-file GCP_CREDS=/abs/creds.json -- ./app + +# Bridge any external manager through the env/file seam, e.g. 1Password: +op run --env-file=secrets.env -- smolvm machine run -- ./app +``` + +Or reference them from a Smolfile. The left-hand key becomes the env var name in +the guest workload: + +```toml +[secrets] +DATABASE_URL = { from_env = "PROD_DB_URL" } # host env var (at launch) +GCP_CREDS = { from_file = "/abs/creds.json" } # absolute host file (at launch) +``` + +Exactly one of `from_env`, `from_file` must be set per entry; `from_file` paths +must be absolute. Resolved values are appended *after* top-level `env` and CLI +`-e` flags. Resolution is late-bound, so rotating the underlying env var or file +takes effect at the next launch with nothing to re-sync. + +**Threat model:** this is defense-in-depth, not zero-knowledge. The target +process sees plaintext in its own environment, and root inside the guest can +read any `/proc/*/environ`. Use SSH agent forwarding instead when a secret must +never leave the host. + +**Where they're resolved:** `machine run`, `machine create` + `machine start`, +and `machine exec` resolve refs against *this host* under a trusted-local scope. +Untrusted surfaces — HTTP API request bodies and portable `.smolmachine` packs — +are treated as untrusted callers and may carry **no** resolvable secret ref: +`from_env` would expose the server's env and `from_file` would be an arbitrary +host-file read, so both are rejected. Configure secrets locally instead. + +## File Copy + +Copy files between the host and a running machine using `machine:path` syntax: + +```bash +# Upload a file to the VM +smolvm machine cp ./script.py myvm:/workspace/script.py + +# Download a file from the VM +smolvm machine cp myvm:/workspace/output.json ./output.json +``` + +**Image-based VMs (--image):** Files copied with `cp` are visible to +`exec` at the same path, and vice versa. This works for any path — +`/tmp`, `/home`, `/workspace`, etc. Under the hood, `cp` routes +through the container's overlay filesystem so both commands see the +same files. + +**`/workspace` shared directory:** Every machine has a `/workspace` +directory — bare VMs, image-based VMs, and machines created from +`.smolmachine` artifacts. It persists across `exec` sessions and +across `stop`/`start` cycles. It's a good default location for +scripts, data, and results. Passing `-v /host/dir:/workspace` replaces +the default storage-disk workspace with your host directory for that +run — the host mount takes priority and the storage workspace is skipped: + +```bash +# Typical agent workflow: copy code in, execute, extract results +smolvm machine create --name r-sandbox --image r-base:latest --net +smolvm machine start --name r-sandbox + +smolvm machine cp analysis.R r-sandbox:/workspace/analysis.R +smolvm machine exec --name r-sandbox -- Rscript /workspace/analysis.R +smolvm machine cp r-sandbox:/workspace/results.csv ./results.csv + +smolvm machine stop --name r-sandbox +``` + +**Behavior and limits:** + +- Files up to 1 MiB transfer as a single message — no perceptible + overhead beyond the agent round-trip. +- Larger files stream automatically: 1 MiB chunks for upload, 16 MiB + chunks for download. The split is asymmetric because the + host→guest direction has tighter socket-buffer headroom. +- Per-transfer cap is **4 GiB** in either direction. Files at or + above this size are rejected up front (`total_size exceeds maximum` + on upload; `exceeding the byte cap` on download). For larger + blobs, mount a host directory with `--volume` instead of copying. +- A throttled progress line prints to stderr while large transfers + run, including bytes-so-far, percentage (uploads), and rate. + Pipe captures (`> file`) only see the upload/download summary, + not the progress noise. +- Atomic on the guest side: a partially-written file never appears + at the target path. If the transfer fails or the connection drops + mid-stream, the staging file is cleaned up and the original + destination (if any) is unaffected. + +Typical throughput on macOS (Apple Silicon): ~35-42 MB/s upload, +~170 MB/s download. + +## Streaming Exec + +Stream command output in real-time instead of buffering: + +```bash +# CLI — prints output as it arrives +smolvm machine exec --stream --name myvm -- python3 train.py + +# API — Server-Sent Events +POST /api/v1/machines/:name/exec/stream +Content-Type: application/json +{"command": ["python3", "train.py"]} + +# Response: text/event-stream +# event: stdout +# data: Epoch 1/10... +# event: exit +# data: {"exitCode":0} +``` +## Bare VM Mode + +`machine run` works without `--image` when a Smolfile provides the workload config, or for direct Alpine shell access: + +```bash +# Bare Alpine shell +smolvm machine run -it + +# Smolfile with entrypoint/cmd (no container overhead) +smolvm machine run -s Smolfile + +# Bare VM with init setup, detached +smolvm machine run -d -s Smolfile +``` + +Bare VMs run commands directly in the Alpine rootfs — no OCI image pull needed. Use this when you need a minimal Linux environment. + +## Packed Binaries (.smolmachine) + +`smolvm pack create` produces two files: +- `my-app` — stub binary with embedded VM runtime (platform-specific) +- `my-app.smolmachine` — VM payload: rootfs, OCI layers, storage (cross-platform) + +The packed binary runs as a normal executable: +```bash +./my-app run -- python3 -c "print('hello')" # ephemeral, cleaned up after exit +./my-app start # persistent daemon mode +./my-app exec -- pip install x # exec into daemon +./my-app stop # stop daemon +``` + +Alternatively, create a named machine from the `.smolmachine` for full lifecycle management: +```bash +smolvm machine create --name my-vm --from my-app.smolmachine +smolvm machine start --name my-vm # ~250ms boot, no image pull +smolvm machine exec --name my-vm -- pip install x # fully persistent +smolvm machine stop --name my-vm +smolvm machine ls # shows my-vm +``` + +The `.smolmachine` manifest includes registry-oriented metadata: +- `host_platform` — host OS+arch this machine runs on (e.g., `darwin/arm64`), distinct from `platform` which is the guest +- `created` — RFC 3339 timestamp of when the machine was packed +- `smolvm_version` — version of smolvm that built it + +## HTTP API + +Start with `smolvm serve start --listen 127.0.0.1:8080` or `smolvm serve start --listen $XDG_RUNTIME_DIR/smolvm.sock`. Key endpoints: + +``` +POST /api/v1/machines Create machine +GET /api/v1/machines List machines +GET /api/v1/machines/:name Get machine +POST /api/v1/machines/:name/start Start machine +POST /api/v1/machines/:name/stop Stop machine +DELETE /api/v1/machines/:name Delete machine +POST /api/v1/machines/:name/exec Execute command +POST /api/v1/machines/:name/exec/stream Streaming exec (SSE) +PUT /api/v1/machines/:name/files/*path Upload file +GET /api/v1/machines/:name/files/*path Download file +GET /api/v1/machines/:name/logs Stream logs (SSE) +POST /api/v1/machines/:name/images/pull Pull OCI image +``` + +OpenAPI spec: `smolvm serve openapi` + +## Important Defaults + +- Machine name defaults to `"default"` when `--name` is omitted +- Network is **off** by default (security-first) +- CPUs: 4, Memory: 8192 MiB, Storage: 20 GiB, Overlay: 2 GiB +- Packed binaries use the same defaults (CPUs: 4, Memory: 8192 MiB) +- Memory and CPU are elastic via virtio balloon — the host only commits what the guest actually uses and reclaims the rest + +## Important Behaviors + +- **Observational commands don't stop running VMs.** `machine images`, `machine status`, `machine ls` and similar read-only commands leave a running VM in its current state. If the VM was already running before the command, it stays running after. +- **`machine prune` works on a running VM.** Regular prune only removes unreferenced layers and is safe while containers are active. `prune --all` requires the VM to be stopped first since it deletes manifests for layers that may be in use. +- **`machine exec` persists filesystem changes.** Package installs, config edits, and file writes inside `exec` survive across sessions. This works for both bare VMs and image-based VMs (created with `--image`). +- **`machine update` modifies a stopped machine.** Add/remove mounts, ports, env vars, or change CPU/memory without recreating the VM. Changes take effect on next `machine start`. Requires the machine to be stopped. +- **`machine run` is always ephemeral.** The VM is created, the command runs, and everything is cleaned up. No state carries over. +- **`-v host:/workspace` replaces the default workspace.** Every image-based VM exposes `/workspace` backed by the VM's storage disk. Mounting a host directory at `/workspace` takes priority — the host share is used instead and the storage-disk workspace is not mounted. Any other target path (e.g. `/data`, `/app`) does not affect `/workspace`. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..ed9e81a --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3836 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "axum-macros", + "base64", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", + "sync_wrapper", + "tokio", + "tokio-tungstenite", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "axum-server" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ab4a3ec9ea8a657c72d99a03a824af695bd0fb5ec639ccbd9cd3543b41a5f9" +dependencies = [ + "arc-swap", + "bytes", + "fs-err", + "http", + "http-body", + "hyper", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + +[[package]] +name = "defmt" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0963443817029b2024136fc4dd07a5107eb8f977eaf18fcd1fdeb11306b64ad" +dependencies = [ + "defmt 1.1.0", +] + +[[package]] +name = "defmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +dependencies = [ + "defmt-parser", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "enumflags2" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1027f7680c853e056ebcec683615fb6fbbc07dbaa13b4d5d9442b146ded4ecef" +dependencies = [ + "enumflags2_derive", +] + +[[package]] +name = "enumflags2_derive" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs-err" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" +dependencies = [ + "autocfg", + "tokio", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "heapless" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" +dependencies = [ + "hash32", + "stable_deref_trait", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "hyper" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea94e891b3606826e9c998be69ddca42247dad8ad50b1649a5cb7e1c9ae06fd" +dependencies = [ + "libc", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "kqueue" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273c0752728918e0ac4976f2b275b6fefb9ecd400585dec929419f3844cd87b5" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" +dependencies = [ + "bitflags 2.13.0", + "libc", +] + +[[package]] +name = "landlock" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "635839550ae8b90d9fd2571460a6645dc0aec070225956ca7a2831ed31d2795d" +dependencies = [ + "enumflags2", + "libc", + "thiserror 2.0.18", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libredox" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +dependencies = [ + "libc", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd7399781913e5393588a8d8c6a2867bf85fb38eaf2502fdce465aad2dc6f034" +dependencies = [ + "base64", + "indexmap", + "metrics", + "metrics-util", + "quanta", + "thiserror 1.0.69", +] + +[[package]] +name = "metrics-util" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.5", + "metrics", + "quanta", + "rand", + "rand_xoshiro", + "sketches-ddsketch", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "mio" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "libc", + "memoffset", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.0", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.13.0", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio 0.8.11", + "walkdir", + "windows-sys 0.48.0", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rapidhash" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.13.0", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.0", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags 2.13.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rust-embed" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04113cb9355a377d83f06ef1f0a45b8ab8cd7d8b1288160717d66df5c7988d27" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0902e4c7c8e997159ab384e6d0fc91c221375f6894346ae107f47dd0f3ccaa" +dependencies = [ + "proc-macro2", + "quote", + "rust-embed-utils", + "syn", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcdef0be6fe7f6fa333b1073c949729274b05f123a0ad7efcb8efd878e5c3b1" +dependencies = [ + "sha2", + "walkdir", +] + +[[package]] +name = "rustc-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ruzstd" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fad02996bfc73da3e301efe90b1837be9ed8f4a462b6ed410aa35d00381de89f" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "seccompiler" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "345a3e4dddf721a478089d4697b83c6c0a8f5bf16086f6c13397e4534eb6e2e5" +dependencies = [ + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "smolfile" +version = "1.5.2" +dependencies = [ + "serde", + "smolvm-protocol", + "tempfile", + "thiserror 1.0.69", + "toml", +] + +[[package]] +name = "smoltcp" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f73d40463bba65efc9adc6370b56df76d563cc46e2482bba58351b4afb7535e" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "cfg-if", + "defmt 0.3.100", + "heapless", + "log", + "managed", +] + +[[package]] +name = "smolvm" +version = "1.5.2" +dependencies = [ + "async-stream", + "axum", + "axum-server", + "base64", + "clap", + "dirs", + "futures-util", + "hex", + "humantime", + "ipnet", + "landlock", + "libc", + "libloading", + "metrics", + "metrics-exporter-prometheus", + "notify", + "parking_lot", + "pkg-config", + "regex", + "rusqlite", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "seccompiler", + "serde", + "serde_json", + "serde_yaml", + "sha2", + "smolfile", + "smolvm-cuda", + "smolvm-network", + "smolvm-pack", + "smolvm-protocol", + "smolvm-registry", + "socket2", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tokio-util", + "toml", + "tower-http", + "tracing", + "tracing-subscriber", + "utoipa", + "utoipa-axum", + "utoipa-swagger-ui", + "windows-sys 0.60.2", + "zeroize", +] + +[[package]] +name = "smolvm-agent" +version = "1.5.2" +dependencies = [ + "flate2", + "lazy_static", + "libc", + "nix 0.27.1", + "parking_lot", + "rustix", + "ruzstd", + "serde", + "serde_json", + "smolvm-protocol", + "tar", + "tempfile", + "tracing", + "tracing-subscriber", + "zstd", +] + +[[package]] +name = "smolvm-cuda" +version = "1.5.2" +dependencies = [ + "libc", + "libloading", +] + +[[package]] +name = "smolvm-cuda-codegen" +version = "1.5.2" + +[[package]] +name = "smolvm-cuda-guest" +version = "1.5.2" +dependencies = [ + "smolvm-cuda", + "vsock", +] + +[[package]] +name = "smolvm-cuda-shim" +version = "1.5.2" +dependencies = [ + "smolvm-cuda", + "vsock", +] + +[[package]] +name = "smolvm-cudart-shim" +version = "1.5.2" +dependencies = [ + "libc", + "smolvm-cuda", + "vsock", +] + +[[package]] +name = "smolvm-network" +version = "1.5.2" +dependencies = [ + "crossbeam-queue", + "humantime", + "polling", + "smoltcp", + "socket2", + "tracing", +] + +[[package]] +name = "smolvm-pack" +version = "1.5.2" +dependencies = [ + "crc32fast", + "dirs", + "libc", + "serde", + "serde_json", + "sha2", + "smolvm-protocol", + "tar", + "tempfile", + "thiserror 1.0.69", + "time", + "windows-sys 0.60.2", + "zstd", +] + +[[package]] +name = "smolvm-protocol" +version = "1.5.2" +dependencies = [ + "base64", + "serde", + "serde_json", + "tracing", +] + +[[package]] +name = "smolvm-registry" +version = "1.5.2" +dependencies = [ + "bytes", + "dirs", + "filetime", + "futures-util", + "hex", + "reqwest", + "serde", + "serde_json", + "sha2", + "smolvm-pack", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-util", + "tracing", + "wiremock", +] + +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio 1.2.1", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.0", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "tungstenite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "sha1", + "thiserror 2.0.18", +] + +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "utoipa" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bde15df68e80b16c7d16b9616e80770ad158988daa56a27dccd1e55558b0160" +dependencies = [ + "indexmap", + "serde", + "serde_json", + "utoipa-gen", +] + +[[package]] +name = "utoipa-axum" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c25bae5bccc842449ec0c5ddc5cbb6a3a1eaeac4503895dc105a1138f8234a0" +dependencies = [ + "axum", + "paste", + "tower-layer", + "tower-service", + "utoipa", +] + +[[package]] +name = "utoipa-gen" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0b99ee52df3028635d93840c797102da61f8a7bb3cf751032455895b52ef8" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn", +] + +[[package]] +name = "utoipa-swagger-ui" +version = "9.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d047458f1b5b65237c2f6dc6db136945667f40a7668627b3490b9513a3d43a55" +dependencies = [ + "axum", + "base64", + "mime_guess", + "regex", + "rust-embed", + "serde", + "serde_json", + "url", + "utoipa", + "utoipa-swagger-ui-vendored", + "zip", +] + +[[package]] +name = "utoipa-swagger-ui-vendored" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2eebbbfe4093922c2b6734d7c679ebfebd704a0d7e56dfcb0d05818ce28977d" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsock" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba782755fc073877e567c2253c0be48e4aa9a254c232d36d3985dfae0bd5205" +dependencies = [ + "libc", + "nix 0.31.3", +] + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.3+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.123" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.13.0", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.13.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12598812502ed0105f607f941c386f43d441e00148fce9dec3ca5ffb0bde9308" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap", + "memchr", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..3be4da4 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,151 @@ +[workspace] +members = [ + ".", + "crates/smolvm-agent", + "crates/smolvm-cuda", + "crates/smolvm-cuda-guest", + "crates/smolvm-cuda-shim", + "crates/smolvm-cuda-codegen", + "crates/smolvm-cudart-shim", + "crates/smolvm-network", + "crates/smolvm-pack", + "crates/smolvm-protocol", + "crates/smolvm-registry", + "crates/smolvm-smolfile", +] +resolver = "2" + +[package] +name = "smolvm" +version = "1.5.2" +edition = "2021" +description = "OCI-native microVM runtime" +license = "Apache-2.0" +repository = "https://github.com/smol-machines/smolvm" +keywords = ["microvm", "container", "virtualization"] +categories = ["virtualization", "command-line-utilities"] + +[lib] +name = "smolvm" +path = "src/lib.rs" + +[[bin]] +name = "smolvm" +path = "src/main.rs" + +[dependencies] +smolvm-network = { path = "crates/smolvm-network", version = "1.5.2" } +smolvm-protocol = { path = "crates/smolvm-protocol", version = "1.5.2" } +smolvm-cuda = { path = "crates/smolvm-cuda", version = "1.5.2" } +smolvm-pack = { path = "crates/smolvm-pack", version = "1.5.2" } +smolvm-registry = { path = "crates/smolvm-registry", version = "1.5.2" } +smolfile = { path = "crates/smolvm-smolfile", version = "1.5.2" } +thiserror = "1" +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_yaml = "0.9" +toml = "0.8" +rusqlite = { version = "0.32", features = ["bundled"] } +libc = "0.2" +# Cross-platform dynamic library loading for libkrun (dlopen on Unix, +# LoadLibraryW on Windows). Replaces the previous direct libc::dlopen path. +libloading = "0.8" +# Cross-platform AF_UNIX sockets for the agent control channel and vsock-port +# bridges. std has no UnixStream on Windows, but socket2 (Domain::UNIX) works +# on Windows 10 1809+. See src/platform/uds.rs. +socket2 = "0.6" +dirs = "5" +sha2 = "0.10" +hex = "0.4" +# Host filesystem watcher (FSEvents on macOS, inotify on Linux) that drives +# host→guest fsnotify propagation for -v mounts. See src/agent/fsnotify_watch.rs. +notify = "6" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] } + +base64 = { workspace = true } + +# Secrets management. Resolved plaintext is scrubbed on drop; smolvm stores no +# secret material itself (refs only), so no at-rest crypto/store deps are needed. +zeroize = { version = "1", features = ["zeroize_derive"] } +humantime = "2" +tempfile = "3" + +# HTTP API server +tokio = { version = "1", features = ["full"] } +axum = { version = "0.8", features = ["macros", "ws"] } +# mTLS for the fleet serve API (control↔node). axum-server's rustls acceptor +# does the TLS handshake + client-cert verification; rustls-pemfile parses the +# CA/cert/key PEMs handed to us at enrollment. +axum-server = { version = "0.7", features = ["tls-rustls"] } +rustls = { version = "0.23", default-features = false, features = ["ring"] } +rustls-pemfile = "2" +rustls-pki-types = "1" +tower-http = { version = "0.6", features = ["cors", "trace", "timeout"] } +tokio-stream = "0.1" +# ReaderStream for streaming a cached blob from disk to a peer (GET /p2p/blob). +tokio-util = { version = "0.7", features = ["io"] } +parking_lot = "0.12" +async-stream = "0.3" +futures-util = "0.3" + +# OpenAPI documentation +utoipa = { version = "5", features = ["axum_extras"] } +utoipa-axum = "0.2" +utoipa-swagger-ui = { version = "9", features = ["axum", "vendored"] } +ipnet = "2.12.0" +metrics = "0.24" + +metrics-exporter-prometheus = { version = "0.16", default-features = false } + +# Seccomp + Landlock for the VM boot subprocess (runtime isolation). Linux-only +# — macOS/HVF has neither; the code is cfg-gated to linux (seccomp also to x86_64). +[target.'cfg(target_os = "linux")'.dependencies] +seccompiler = "0.4" +landlock = "0.4" + +# Windows process management (OpenProcess / TerminateProcess / Wait) and +# DLL-search-path control (SetDllDirectoryW). Used to replace the Unix +# kill/waitpid/dlopen primitives on the Windows host build. +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.60", features = [ + "Win32_Foundation", + "Win32_System_Threading", + "Win32_System_ProcessStatus", + "Win32_System_LibraryLoader", + "Win32_System_Console", + "Win32_System_IO", + "Win32_Storage_FileSystem", + "Win32_Networking_WinSock", +] } + +[dev-dependencies] +tempfile = "3" +smolvm-protocol = { path = "crates/smolvm-protocol", version = "1.5.2" } +regex = "1" + +[build-dependencies] +pkg-config = "0.3" + +[workspace.dependencies] +base64 = "0.22" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "1" +tracing = "0.1" + +# Release profile optimizations +[profile.release] +lto = true +codegen-units = 1 +strip = true + +# Smaller binary profile for agent (use with --profile release-small) +[profile.release-small] +inherits = "release" +opt-level = "z" # Optimize for size +lto = true +codegen-units = 1 +panic = "abort" # No unwinding - saves ~10% binary size +strip = true diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2703352 --- /dev/null +++ b/LICENSE @@ -0,0 +1,190 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024-2025 smolvm contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Licenses.md b/Licenses.md new file mode 100644 index 0000000..455bd4c --- /dev/null +++ b/Licenses.md @@ -0,0 +1,40 @@ +# Third-Party Licenses + +smolvm bundles the following third-party libraries in the `lib/` directory. + +## libkrun + +- **Version:** 1.15.1 +- **License:** Apache License 2.0 +- **Source:** https://github.com/containers/libkrun +- **Copyright:** The libkrun Authors + +Licensed under the Apache License, Version 2.0. You may obtain a copy of the License at: +http://www.apache.org/licenses/LICENSE-2.0 + +## libkrunfw + +- **Version:** 4.x +- **License:** LGPL-2.1-only (library), GPL-2.0-only (bundled Linux kernel) +- **Source:** https://github.com/containers/libkrunfw +- **Copyright:** The libkrunfw Authors + +libkrunfw is a library that bundles the Linux kernel for use with libkrun. + +### Source Code Availability + +In compliance with LGPL-2.1 and GPL-2.0, the complete source code for libkrunfw and the bundled Linux kernel is available at: + +- **libkrunfw:** https://github.com/containers/libkrunfw +- **Linux kernel (with patches):** https://github.com/containers/libkrunfw/tree/main/patches + +To obtain the exact source code corresponding to the bundled binary, check out the version tag matching the library version from the repository above. + +### Your Rights Under LGPL-2.1 + +You have the right to: +- Use this library in your own projects +- Modify the library and distribute your modifications +- Reverse engineer the library for debugging purposes + +If you distribute a modified version of libkrunfw, you must make your modifications available under the same license. diff --git a/Makefile.toml b/Makefile.toml new file mode 100644 index 0000000..6396498 --- /dev/null +++ b/Makefile.toml @@ -0,0 +1,276 @@ +# cargo-make configuration for smolvm +# https://github.com/sagiegurari/cargo-make +# +# Usage: +# cargo make dev # Build and codesign (macOS) +# cargo make smolvm # Run with env vars configured for local resources +# cargo make dist # Build distribution package +# cargo make test # Run all tests +# cargo make lint # Clippy + fmt checks + +[config] +min_version = "0.37.0" +default_to_workspace = false +skip_core_tasks = true + +# ============================================================================= +# Development Tasks +# ============================================================================= + +[tasks.dev] +description = "Build and codesign dev binary (macOS auto-signs)" +category = "Development" + +script_runner = "bash" +script = [ +''' +if [ -z "${LIBKRUN_BUNDLE:-}" ]; then + if [ "$(uname -s)" = "Linux" ]; then + export LIBKRUN_BUNDLE="${CARGO_MAKE_WORKING_DIRECTORY}/lib/linux-$(uname -m)" + else + export LIBKRUN_BUNDLE="${CARGO_MAKE_WORKING_DIRECTORY}/lib" + fi +fi + +cargo build --release +if [ "$(uname -s)" = "Darwin" ]; then + codesign --force --sign - --entitlements smolvm.entitlements ./target/release/smolvm + echo "✓ Codesigned ./target/release/smolvm" +else + echo "✓ Linux: no codesigning needed" +fi +echo "" +echo "Binary ready! Run it with:" +echo " cargo make smolvm --version" +echo " cargo make smolvm sandbox run --net alpine:latest -- echo hello" +''' +] + +[tasks.ensure-binary-signed] +description = "Ensure binary is built and signed (macOS)" +category = "Development" +script_runner = "bash" +script = [ +''' +if [ ! -f ./target/release/smolvm ]; then + echo "Binary not found, building..." + cargo build --release +fi +if [ "$(uname -s)" = "Darwin" ]; then + codesign --force --sign - --entitlements smolvm.entitlements ./target/release/smolvm 2>/dev/null || true +fi +''' +] + +[tasks.smolvm] +description = "Run smolvm with environment variables set up" +category = "Development" +dependencies = ["ensure-binary-signed"] +env = { DYLD_LIBRARY_PATH = "${CARGO_MAKE_WORKING_DIRECTORY}/lib", SMOLVM_AGENT_ROOTFS = "${CARGO_MAKE_WORKING_DIRECTORY}/target/agent-rootfs" } +script_runner = "bash" +script = [ +''' +IFS=';' read -ra ARGS <<< "${CARGO_MAKE_TASK_ARGS}" +exec ./target/release/smolvm "${ARGS[@]}" +''' +] + +# ============================================================================= +# Build Tasks +# ============================================================================= + +[tasks.build] +description = "Build release binary" +category = "Build" +command = "cargo" +args = ["build", "--release"] + +[tasks.build-agent] +description = "Build agent for Linux (size-optimized)" +category = "Build" +script_runner = "bash" +script = [ +''' +if [ "$(uname -s)" = "Linux" ]; then + cargo build --profile release-small -p smolvm-agent --target x86_64-unknown-linux-musl +else + docker run --rm -v "$(pwd):/work" -w /work rust:alpine sh -c 'apk add musl-dev && cargo build --profile release-small -p smolvm-agent' +fi +''' +] + +# ============================================================================= +# libkrun Tasks +# ============================================================================= + +[tasks.build-libkrunfw] +description = "Build libkrunfw guest kernel and update lib/ (slow: downloads and compiles Linux 6.x)" +category = "Build" +script_runner = "bash" +script = [ +''' +set -e +ARCH=$(uname -m) +FULL_VERSION=$(grep '^FULL_VERSION' libkrunfw/Makefile | cut -d= -f2 | tr -d ' ') +KERNEL_SRC=$(grep '^KERNEL_VERSION' libkrunfw/Makefile | cut -d= -f2 | tr -d ' ') + +# If the kernel source tree already exists, update its .config and force +# kernel.c to regenerate — otherwise make skips both steps. +if [ -d "libkrunfw/${KERNEL_SRC}" ]; then + cp libkrunfw/config-libkrunfw_x86_64 libkrunfw/${KERNEL_SRC}/.config + make -C libkrunfw/${KERNEL_SRC} olddefconfig + rm -f libkrunfw/kernel.c +fi + +make -C libkrunfw +mkdir -p lib/linux-${ARCH} +cp libkrunfw/libkrunfw.so.${FULL_VERSION} lib/linux-${ARCH}/libkrunfw.so.${FULL_VERSION} +cd lib/linux-${ARCH} +ln -sf libkrunfw.so.${FULL_VERSION} libkrunfw.so.5 +ln -sf libkrunfw.so.5 libkrunfw.so +echo "✓ libkrunfw ${FULL_VERSION} built and installed to lib/linux-${ARCH}/" +''' +] + +[tasks.build-libkrun] +description = "Build libkrun with BLK and GPU support and copy into lib/ (run build-libkrunfw first if kernel config changed)" +category = "Build" +script_runner = "bash" +script = [ +''' +set -e +ARCH=$(uname -m) +make -C libkrun BLK=1 NET=1 GPU=1 +if [ "$(uname -s)" = "Darwin" ]; then + # macOS: the dylib lives in lib/ (alongside libkrunfw.5.dylib). + cp libkrun/target/release/libkrun.dylib lib/libkrun.dylib + ./scripts/stamp-libkrun-provenance.sh lib --skip-libkrunfw + echo "✓ libkrun built and copied to lib/libkrun.dylib" +else + mkdir -p lib/linux-${ARCH} + cp libkrun/target/release/libkrun.so lib/linux-${ARCH}/libkrun.so + ./scripts/stamp-libkrun-provenance.sh lib/linux-${ARCH} --skip-libkrunfw + echo "✓ libkrun built and copied to lib/linux-${ARCH}/libkrun.so" +fi +''' +] + +# ============================================================================= +# Distribution Tasks +# ============================================================================= + +[tasks.dist] +description = "Build distribution package" +category = "Distribution" +script_runner = "bash" +script = [ +''' +./scripts/build-dist.sh "$@" +''' +] + +# ============================================================================= +# Agent Tasks +# ============================================================================= + +[tasks.agent-rootfs] +description = "Build agent rootfs" +category = "Agent" +script_runner = "bash" +script = [ +''' +./scripts/build-agent-rootfs.sh +''' +] + +[tasks.agent-rebuild] +description = "Rebuild agent and update rootfs" +category = "Agent" +dependencies = ["build-agent", "agent-rootfs"] +script_runner = "bash" +script = [ +''' +./scripts/rebuild-agent.sh +''' +] + +# ============================================================================= +# Test Tasks +# ============================================================================= + +[tasks.test] +description = "Run all feature suites (11 groups, ~10 min)" +category = "Test" +dependencies = ["test-lib"] +script_runner = "bash" +script = [ +''' +./tests/run_tests.sh +''' +] + +[tasks.test-cli] +description = "Run CLI tests only" +category = "Test" +script_runner = "bash" +script = [ +''' +./tests/run_tests.sh cli +''' +] + +[tasks.test-pack] +description = "Run pack tests only" +category = "Test" +script_runner = "bash" +script = [ +''' +./tests/run_tests.sh pack +''' +] + +[tasks.test-lib] +description = "Run unit tests (no VM required)" +category = "Test" +command = "cargo" +args = ["test", "--lib"] + +# ============================================================================= +# Install Tasks +# ============================================================================= + +[tasks.install] +description = "Install locally from dist package" +category = "Install" +script_runner = "bash" +script = [ +''' +./scripts/install-local.sh +''' +] + +# ============================================================================= +# Code Quality Tasks +# ============================================================================= + +[tasks.lint] +description = "Run clippy and fmt checks" +category = "Quality" +script_runner = "bash" +script = [ +''' +cargo fmt --all -- --check +cargo clippy --all-targets -- -D warnings +''' +] + +[tasks.fix-lints] +description = "Auto-fix linting issues" +category = "Quality" +script_runner = "bash" +script = [ +''' +cargo fmt --all +cargo clippy --all-targets --fix --allow-dirty +''' +] diff --git a/README.md b/README.md new file mode 100644 index 0000000..66f62b4 --- /dev/null +++ b/README.md @@ -0,0 +1,236 @@ +

+ smol machines +

+ +

+ Discord + Release + License +

+ +smolvm +====== + +Ship and run software with isolation by default. + +This is a CLI tool that lets you: +1. Manage and run custom Linux virtual machines locally with: sub-second cold start, cross-platform (macOS, Linux, Windows), elastic memory usage. +2. Pack a stateful virtual machine into a single file (.smolmachine) to rehydrate on any supported platform. + +Install +------- + +```bash +# install (macOS + Linux) +curl -sSL https://smolmachines.com/install.sh | bash + +# for coding agents — install + discover all commands +curl -sSL https://smolmachines.com/install.sh | bash && smolvm --help +``` + +Or download from [GitHub Releases](https://github.com/smol-machines/smolvm/releases), and place it into `~/.local/share/`. + +**Windows:** download the `windows-x86_64` release (bundles `krun.dll` + `libkrunfw.dll`), unzip it, and run `smolvm.exe`. Requires the [Windows Hypervisor Platform](https://learn.microsoft.com/en-us/virtualization/api/) (WHP) feature enabled. + +Quick Start +----------- + +```bash +# run a command in an ephemeral VM (cleaned up after exit) +smolvm machine run --net --image alpine -- sh -c "echo 'Hello world from a microVM' && uname -a" + +# interactive shell +smolvm machine run --net -it --image alpine -- /bin/sh +# inside the VM: apk add sl && sl && exit +``` + +Use This For +------------ + +**Sandbox untrusted code** — run untrusted programs in a hardware-isolated VM. Host filesystem, network, and credentials are separated by a hypervisor boundary. + +```bash +# network is off by default — untrusted code can't phone home +smolvm machine run --image alpine -- nslookup example.com +# fails — no network access + +# lock down egress — only allow specific hosts +smolvm machine run --net --image alpine --allow-host registry.npmjs.org -- wget -q -O /dev/null https://registry.npmjs.org +# works — allowed host + +smolvm machine run --net --image alpine --allow-host registry.npmjs.org -- wget -q -O /dev/null https://google.com +# fails — not in allow list +``` + +**Pack into portable executables** — turn any workload into a self-contained binary. All dependencies are pre-baked — no install step, no runtime downloads, boots in <200ms. + +```bash +smolvm pack create --image python:3.12-alpine -o ./python312 +./python312 run -- python3 --version +# Python 3.12.x — isolated, no pyenv/venv/conda needed +``` + +**Use local container images** — for CI, air-gapped hosts, and fast iteration. Feed `--image` a `docker save` / `podman save` archive, pipe one on stdin, or point it at an unpacked rootfs directory. Image work is delegated to your container tooling; smolvm just boots the result. + +```bash +# build locally, run in the VM with no push/pull +docker build -t myapp . +docker save myapp | smolvm machine run --image - -- ./app + +# from an archive file (boots with no network) +smolvm machine run --image ./myapp.tar -- ./app + +# from an already-unpacked rootfs directory +smolvm machine run --image ./rootfs/ -- ./app +``` + +**Persistent machines for development** — create, stop, start. Installed packages survive restarts. + +```bash +smolvm machine create --net --name myvm +smolvm machine start --name myvm +smolvm machine exec --name myvm -- apk add sl +smolvm machine exec --name myvm -it -- /bin/sh +# inside: sl, ls, uname -a — type 'exit' to leave +smolvm machine stop --name myvm +``` + +**Use git and SSH without copying private keys into the guest.** Forward your host SSH agent into the VM. The guest can ask the agent to sign with any forwarded key while the socket is available, so forward it only to workloads you trust. Requires an SSH agent running on your host (`ssh-add -l` to check). + +```bash +smolvm machine run --ssh-agent --net --image alpine -- sh -c "apk add -q openssh-client && ssh-add -l" +# lists your host keys; private key material remains in the host agent + +smolvm machine exec --name myvm -- git clone git@github.com:org/private-repo.git +``` + +**Declare environments with a Smolfile** — reproducible VM config in a simple TOML file. + +```toml +image = "python:3.12-alpine" +net = true + +[network] +allow_hosts = ["api.stripe.com", "db.example.com"] + +[dev] +init = ["pip install -r requirements.txt"] +volumes = ["./src:/app"] + +[auth] +ssh_agent = true +``` + +```bash +smolvm machine create --name myvm -s Smolfile +smolvm machine start --name myvm +``` + +More examples: [python](https://github.com/smol-machines/smolvm/tree/main/examples/python-app) · [node](https://github.com/smol-machines/smolvm/tree/main/examples/node-app) · [doom](https://github.com/smol-machines/smolvm/tree/main/examples/doom-web) + +How It Works +------------ + +Each workload runs in a hardware-virtualized VM with its own guest kernel on [Hypervisor.framework](https://developer.apple.com/documentation/hypervisor) (macOS), KVM (Linux), or the [Windows Hypervisor Platform](https://learn.microsoft.com/en-us/virtualization/api/) (Windows). [libkrun](https://github.com/containers/libkrun) is the VMM and [libkrunfw](https://github.com/smol-machines/libkrunfw) supplies the guest kernel. Pack it into a `.smolmachine` and it runs anywhere the host architecture matches, with zero dependencies. + +Images use the [OCI](https://opencontainers.org/) format — the same open standard Docker uses. Any image on Docker Hub, ghcr.io, or other OCI registries can be pulled and booted as a microVM. No Docker daemon required. + +Defaults: 4 vCPUs, 8 GiB RAM. Memory is elastic via virtio balloon — the host only commits what the guest actually uses and reclaims the rest automatically. vCPU threads sleep in the hypervisor when idle, so over-provisioning has near-zero cost. Override with `--cpus` and `--mem`. + +Security Model +-------------- + +smolvm strengthens the guest/host boundary by giving each workload a separate VM and guest kernel. It is not, by itself, a hardened multi-user control plane: + +* The `smolvm` CLI and VMM processes run with the permissions of the invoking host user. That user account, the host OS, the hypervisor backend, libkrun, and smolvm are in the trusted computing base. +* Host directories passed with `--volume` are intentionally exposed to the guest with the requested access. Do not mount secrets or sensitive paths into an untrusted workload. +* `--ssh-agent` does not copy private key material into the guest, but it grants the guest access to the forwarded agent socket and therefore the ability to request signatures while the VM is running. +* Networking is disabled by default. Enabling `--net`, port forwarding, or host services expands the workload's reachable surface. +* In standalone local use, smolvm's state and control endpoints are scoped to the invoking user's environment. For hostile local co-tenants, add host-level account separation and OS confinement around the VMM process. This section does not describe the separate smolmachines cloud control plane or its tenant-isolation guarantees. +* Release archives publish SHA-256 checksums and the installer rejects a mismatch when the checksum file is available. Releases are not currently signed or accompanied by provenance attestations, and the installer permits installation when the checksum file cannot be downloaded. + +Treat root in the guest as untrusted. The VM boundary limits its direct access to the host, while every explicitly forwarded capability, including mounts, network access, ports, and SSH agent access, becomes part of the workload's authority. + +Comparison +---------- + +| | smolvm | Containers | Colima | QEMU | Firecracker | Kata | +|---------------------|--------|------------|--------|------|-------------|------| +| Workload boundary | VM + guest kernel | Namespace + shared kernel | Namespace inside shared VM | VM + guest kernel | VM + guest kernel | VM per container | +| Boot time | <200ms | ~100ms | ~seconds | ~15-30s | <125ms | ~500ms | +| Architecture | Library (libkrun) | Daemon | Daemon (in VM) | Process | Process | Runtime stack | +| Per-workload VMs | Yes | No | No (shared) | Yes | Yes | Yes | +| macOS native | Yes | Via Docker VM | Yes (krunkit) | Yes | No | No | +| Embeddable SDK | Yes | No | No | No | No | No | +| Portable artifacts | `.smolmachine` | Images (need daemon) | No | No | No | No | + +Platform Support +---------------- + +| Host | Guest | Requirements | +|------|-------|-------------| +| macOS Apple Silicon | arm64 Linux | macOS 11+ | +| macOS Intel | x86_64 Linux | macOS 11+ (untested) | +| Linux x86_64 | x86_64 Linux | KVM (`/dev/kvm`) | +| Linux aarch64 | aarch64 Linux | KVM (`/dev/kvm`) | +| Windows x86_64 | x86_64 Linux | Windows Hypervisor Platform (WHP) enabled | + +Known Limitations +----------------- + +* Network is opt-in (`--net` on `machine create`). TCP/UDP only, no ICMP. +* Volume mounts: directories only (no single files). Mounting at `/workspace` (`-v /host/dir:/workspace`) takes priority over the default storage-disk workspace — your host directory is used instead. +* macOS: binary must be signed with Hypervisor.framework entitlements. +* `--ssh-agent` requires an SSH agent running on the host (`SSH_AUTH_SOCK` must be set). +* GPU acceleration requires libkrun built with `GPU=1` and virglrenderer + a Vulkan driver on the host (see [GPU Acceleration](#gpu-acceleration) below). +* Windows: `--net` works the same as on other platforms (virtio-net with inbound port-forwarding; TSI for outbound-only VMs), as do `machine exec` / interactive sessions and `machine stats`. Not yet available on Windows: GPU acceleration and `machine fork` / snapshot. Pack *create* needs `storage-template.ext4` / `overlay-template.ext4` next to `smolvm.exe` (Windows has no host `mkfs.ext4`). + +GPU Acceleration +---------------- + +smolvm exposes the host GPU to guests via **virtio-gpu / Venus** (Vulkan-over-virtio). Guest workloads see a real Vulkan device; on Linux + Intel this renders as: + +``` +ANGLE (Intel, Vulkan 1.4 (Virtio-GPU Venus (Intel(R) UHD Graphics ...)), venus) +``` + +### Host requirements + +**macOS** — virglrenderer and MoltenVK are bundled in the smolvm distribution. No extra installs needed. + +**Linux** — virglrenderer and a host Vulkan driver must be installed from the system package manager: + +| Distro | Packages | +|--------|----------| +| Alpine | `apk add virglrenderer mesa-vulkan-intel` (or `mesa-vulkan-ati` for AMD) | +| Debian/Ubuntu | `apt install virglrenderer0 mesa-vulkan-drivers` | + +> virglrenderer depends on libEGL and libdrm from the host GPU driver stack — these are hardware-specific and cannot be bundled. Any GPU-capable Linux host will already have them installed via its GPU driver. + +### Usage + +```bash +# CLI +smolvm machine run --gpu --image alpine -- vulkaninfo --summary + +# Smolfile +# gpu = true +# gpu_vram = 2048 # MiB, default 4096 +``` + +The guest Vulkan loader must be pointed at the virtio ICD: + +```bash +export VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/virtio_icd.x86_64.json +``` + +### Headless browser example + +See [`examples/headless-browser/`](examples/headless-browser/) for a working Chromium setup using ANGLE + Venus for hardware-accelerated WebGL inside a headless VM. + +Development +----------- + +See [docs/DEVELOPMENT.md](docs/DEVELOPMENT.md). + +[Apache-2.0](LICENSE) · made by [@binsquare](https://github.com/BinSquare) · [twitter](https://x.com/binsquares) · [github](https://github.com/smol-machines/smolvm) diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..ce9df7a --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目:`smol-machines/smolvm` +- 原始仓库:https://github.com/smol-machines/smolvm +- 导入方式:上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源,不代表 WeHub 是原项目作者 diff --git a/assets/logo.png b/assets/logo.png new file mode 100644 index 0000000..e1ed183 Binary files /dev/null and b/assets/logo.png differ diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..a6cfc15 --- /dev/null +++ b/build.rs @@ -0,0 +1,436 @@ +//! Build script for smolvm. +//! +//! Handles macOS weak-link setup for libkrun. +//! +//! Linux loads libkrun at runtime with `dlopen` so packed stubs can start +//! before bundled libraries are extracted. +//! +//! # Linking Options +//! +//! ## Dynamic (default) +//! macOS can weak-link against libkrun installed on the system: +//! ```sh +//! brew install libkrun # macOS +//! cargo build +//! ``` +//! +//! ## Bundle Pre-built Library (Recommended for Distribution) +//! Copy Homebrew's libkrun to your bundle directory and build: +//! ```sh +//! mkdir -p lib +//! cp /opt/homebrew/opt/libkrun/lib/libkrun.dylib lib/ +//! # Also copy libkrunfw if bundling +//! cp /opt/homebrew/opt/libkrunfw/lib/libkrunfw.dylib lib/ +//! LIBKRUN_BUNDLE=$PWD/lib cargo build --release +//! ``` +//! The binary will use @rpath to find libraries in ./lib or ../lib. +//! +//! ## Build from Submodule (Experimental) +//! Build libkrun from the vendored submodule: +//! ```sh +//! LIBKRUN_BUILD=1 cargo build +//! ``` +//! Note: This builds a minimal libkrun. Block device support (`krun_add_disk2`) +//! requires building libkrun separately with `make BLK=1` and using LIBKRUN_BUNDLE. +//! +//! ## Static (libkrun only) +//! Statically link libkrun (still dynamically links libkrunfw): +//! ```sh +//! LIBKRUN_STATIC=/path/to/libkrun.a cargo build +//! ``` + +#[cfg(target_os = "macos")] +use std::path::PathBuf; +#[cfg(target_os = "macos")] +use std::process::Command; + +/// Link libkrun for legacy macOS weak-link behavior. +/// +/// Linux intentionally does not link libkrun at build time: packed stubs must +/// be able to start before bundled libraries are extracted, so Linux VM launch +/// paths use `dlopen` instead. +#[cfg(target_os = "macos")] +fn link_krun() { + println!("cargo:rustc-link-arg=-Wl,-weak-lkrun"); +} + +#[cfg(target_os = "macos")] +fn has_library(name: &str) -> bool { + pkg_config::Config::new().probe(name).is_ok() +} + +fn main() { + // Build scripts run on the HOST: `cfg!(target_os)` here describes the + // build machine, not the artifact. Cross-compiling smolvm from macOS to + // Linux (zigbuild) must NOT emit macOS-only linker args (-sectcreate, + // -weak-lkrun) — gate every emission on the actual compile target. + let target_os = std::env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + if target_os == "macos" { + // On macOS, create a placeholder __SMOLVM,__smolvm Mach-O section. + // This section is replaced with real data by `smolvm pack --single-file`. + // The placeholder marker is NOT the SMOLSECT magic, so detect.rs won't + // false-positive on a normal smolvm binary. + #[cfg(target_os = "macos")] + { + use std::io::Write; + let out_dir = std::env::var("OUT_DIR").unwrap(); + let placeholder_path = format!("{}/smolvm_placeholder.bin", out_dir); + let mut f = std::fs::File::create(&placeholder_path).unwrap(); + f.write_all(b"SMOLVM_SECTION_PLACEHOLDER_V1").unwrap(); + f.write_all(&[0u8; 4]).unwrap(); + println!( + "cargo:rustc-link-arg=-Wl,-sectcreate,__SMOLVM,__smolvm,{}", + placeholder_path + ); + } + + #[cfg(target_os = "macos")] + link_libkrun(); + } +} + +#[cfg(target_os = "macos")] +fn link_libkrun() { + println!("cargo:rerun-if-env-changed=LIBKRUN_STATIC"); + println!("cargo:rerun-if-env-changed=LIBKRUN_BUNDLE"); + println!("cargo:rerun-if-env-changed=LIBKRUN_DIR"); + println!("cargo:rerun-if-env-changed=LIBKRUN_BUILD"); + + // Option 0: Build from submodule + if std::env::var("LIBKRUN_BUILD").is_ok() { + if let Some(lib_path) = build_libkrun_from_submodule() { + println!("cargo:rustc-link-search=native={}", lib_path.display()); + link_krun(); + + // Set rpath to find libraries relative to executable + #[cfg(target_os = "macos")] + { + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/lib"); + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/../lib"); + // Also add the build output path for development + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_path.display()); + } + #[cfg(target_os = "linux")] + { + println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/lib"); + println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/../lib"); + } + return; + } + } + + // Option 1: Bundle libraries with the binary + if let Ok(bundle_path) = std::env::var("LIBKRUN_BUNDLE") { + // On Linux, check that the library is not an LFS pointer before linking + #[cfg(target_os = "linux")] + { + let lib_path = std::path::Path::new(&bundle_path).join("libkrun.so"); + if lib_path.exists() && is_lfs_pointer(&lib_path) { + println!("cargo:error=libkrun.so is a Git LFS pointer, not the actual library."); + println!("cargo:error=Run 'git lfs pull' to fetch the actual library binary."); + panic!("Git LFS pointer detected"); + } + } + + println!("cargo:rustc-link-search=native={}", bundle_path); + link_krun(); + + // Set rpath to find libraries relative to executable + #[cfg(target_os = "macos")] + { + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/lib"); + println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/../lib"); + + // Change the library's install_name to use @rpath and re-sign + let lib_path = std::path::Path::new(&bundle_path).join("libkrun.dylib"); + if lib_path.exists() { + let _ = Command::new("install_name_tool") + .args(["-id", "@rpath/libkrun.dylib", lib_path.to_str().unwrap()]) + .status(); + // Re-sign after modification (macOS requires valid signature) + let _ = Command::new("codesign") + .args(["--force", "--sign", "-", lib_path.to_str().unwrap()]) + .status(); + } + } + #[cfg(target_os = "linux")] + { + println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/lib"); + println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/../lib"); + } + return; + } + + // Option 2: Static linking + if let Ok(static_path) = std::env::var("LIBKRUN_STATIC") { + let path = std::path::Path::new(&static_path); + + if path.is_dir() { + println!("cargo:rustc-link-search=native={}", static_path); + } else if path.is_file() { + if let Some(dir) = path.parent() { + println!("cargo:rustc-link-search=native={}", dir.display()); + } + } else { + panic!("LIBKRUN_STATIC path does not exist: {}", static_path); + } + + println!("cargo:rustc-link-lib=static=krun"); + + // Static libkrun requires these frameworks on macOS + #[cfg(target_os = "macos")] + { + println!("cargo:rustc-link-lib=framework=Hypervisor"); + println!("cargo:rustc-link-lib=framework=vmnet"); + } + return; + } + + // Option 3: Custom directory + if let Ok(dir) = std::env::var("LIBKRUN_DIR") { + println!("cargo:rustc-link-search=native={}", dir); + link_krun(); + return; + } + + // Option 4: Bundled libraries in lib/linux-{arch}/ (for distribution builds) + #[cfg(target_os = "linux")] + { + let arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| ".".to_string()); + let lib_dir = format!("{}/lib/linux-{}", manifest_dir, arch); + let lib_path = std::path::Path::new(&lib_dir); + let libkrun_path = lib_path.join("libkrun.so"); + + // Check if the library exists and is a real library (not an LFS pointer) + if libkrun_path.exists() && !is_lfs_pointer(&libkrun_path) { + println!( + "cargo:warning=Using bundled Linux libraries from {}", + lib_dir + ); + println!("cargo:rustc-link-search=native={}", lib_dir); + link_krun(); + + // Set rpath to find libraries relative to executable + println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/lib"); + println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/../lib"); + // Also add the source directory for development builds + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir); + return; + } + } + + // Option 5: System installation via pkg-config + if pkg_config::Config::new() + .atleast_version("1.0") + .probe("libkrun") + .is_ok() + { + return; + } + + // Option 6: Common installation paths + #[cfg(target_os = "macos")] + { + let paths = [ + "/opt/homebrew/lib", + "/usr/local/lib", + "/opt/homebrew/opt/libkrun/lib", + "/usr/local/opt/libkrun/lib", + ]; + + for path in paths { + if std::path::Path::new(path).join("libkrun.dylib").exists() { + println!("cargo:rustc-link-search=native={}", path); + link_krun(); + // Set rpath so runtime linker can find dependencies + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", path); + return; + } + } + } + + #[cfg(target_os = "linux")] + { + let paths = [ + "/usr/lib", + "/usr/local/lib", + "/usr/lib64", + "/usr/local/lib64", + "/usr/lib/x86_64-linux-gnu", + "/usr/lib/aarch64-linux-gnu", + ]; + + for path in paths { + if std::path::Path::new(path).join("libkrun.so").exists() { + println!("cargo:rustc-link-search=native={}", path); + link_krun(); + return; + } + } + } + + // Fallback + link_krun(); +} + +/// Build libkrun from the vendored submodule. +/// +/// Returns the path to the directory containing the built library. +#[cfg(target_os = "macos")] +fn build_libkrun_from_submodule() -> Option { + let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").ok()?); + let libkrun_dir = manifest_dir.join("libkrun"); + + // Check if submodule exists + if !libkrun_dir.join("Cargo.toml").exists() { + println!( + "cargo:warning=libkrun submodule not found at {}. \ + Run: git submodule update --init", + libkrun_dir.display() + ); + return None; + } + + // On macOS, libkrun embeds the init binary via include_bytes!() + // We need to ensure init/init exists (copy from init/init.krun if needed) + #[cfg(target_os = "macos")] + { + let init_dst = libkrun_dir.join("init/init"); + let init_src = libkrun_dir.join("init/init.krun"); + if !init_dst.exists() && init_src.exists() { + println!("cargo:warning=Copying init.krun to init for embedding..."); + if let Err(e) = std::fs::copy(&init_src, &init_dst) { + println!("cargo:warning=Failed to copy init binary: {}", e); + return None; + } + } else if !init_dst.exists() { + println!("cargo:warning=init binary not found. Need init/init or init/init.krun"); + return None; + } + } + + // Determine profile + let profile = std::env::var("PROFILE").unwrap_or_else(|_| "release".to_string()); + + println!( + "cargo:warning=Building libkrun from submodule ({} build)...", + profile + ); + + // Build libkrun using cargo directly (make has issues on macOS) + let libkrun_manifest = libkrun_dir.join("src/libkrun/Cargo.toml"); + let mut cmd = Command::new("cargo"); + cmd.arg("build") + .arg("--manifest-path") + .arg(&libkrun_manifest); + + if profile == "release" { + cmd.arg("--release"); + } + + // Auto-detect GPU support: enable if virglrenderer is installed on the host. + // GPU feature requires virglrenderer + libclang (for krun_display bindgen). + // On macOS, also needs MoltenVK for Vulkan → Metal translation. + let gpu_available = has_library("virglrenderer"); + if gpu_available { + cmd.arg("--features").arg("gpu"); + + // krun_display uses bindgen which needs libclang. + // Help it find libclang on macOS (Homebrew LLVM). + #[cfg(target_os = "macos")] + { + let llvm_lib = std::path::Path::new("/opt/homebrew/opt/llvm/lib"); + if llvm_lib.exists() { + cmd.env("LIBCLANG_PATH", llvm_lib); + } + } + + println!("cargo:warning=GPU support enabled (virglrenderer found)"); + } + + let status = cmd.status(); + + match status { + Ok(s) if s.success() => { + println!("cargo:warning=libkrun built successfully"); + } + Ok(s) => { + println!( + "cargo:warning=libkrun build failed with exit code: {:?}", + s.code() + ); + return None; + } + Err(e) => { + println!("cargo:warning=Failed to run cargo: {}", e); + return None; + } + } + + // Find the built library + let target_dir = if profile == "release" { + libkrun_dir.join("target/release") + } else { + libkrun_dir.join("target/debug") + }; + + #[cfg(target_os = "macos")] + let lib_name = "libkrun.dylib"; + #[cfg(target_os = "linux")] + let lib_name = "libkrun.so"; + + if target_dir.join(lib_name).exists() { + // Copy library to smolvm's target directory for bundling + let out_dir = PathBuf::from(std::env::var("OUT_DIR").ok()?); + let lib_out_dir = out_dir.join("lib"); + std::fs::create_dir_all(&lib_out_dir).ok()?; + + let src = target_dir.join(lib_name); + let dst = lib_out_dir.join(lib_name); + std::fs::copy(&src, &dst).ok()?; + + println!( + "cargo:warning=Copied {} to {}", + src.display(), + dst.display() + ); + + // On macOS, change the install_name to use @rpath so the binary finds + // the bundled library instead of a system-installed one + #[cfg(target_os = "macos")] + { + let install_name = format!("@rpath/{}", lib_name); + let status = Command::new("install_name_tool") + .args(["-id", &install_name, dst.to_str().unwrap()]) + .status(); + if let Ok(s) = status { + if s.success() { + println!("cargo:warning=Set install_name to {}", install_name); + // Re-sign after modification + let _ = Command::new("codesign") + .args(["--force", "--sign", "-", dst.to_str().unwrap()]) + .status(); + } + } + } + + // Tell cargo to rebuild if libkrun source changes + println!( + "cargo:rerun-if-changed={}", + libkrun_dir.join("src").display() + ); + println!( + "cargo:rerun-if-changed={}", + libkrun_dir.join("init").display() + ); + + Some(lib_out_dir) + } else { + println!( + "cargo:warning=Built library not found at {}", + target_dir.join(lib_name).display() + ); + None + } +} diff --git a/crates/smolvm-agent/Cargo.toml b/crates/smolvm-agent/Cargo.toml new file mode 100644 index 0000000..6978850 --- /dev/null +++ b/crates/smolvm-agent/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "smolvm-agent" +version = "1.5.2" +edition = "2021" +description = "Guest agent for smolvm VMs (handles OCI operations and command execution)" +license = "Apache-2.0" +repository = "https://github.com/smol-machines/smolvm" + +[[bin]] +name = "smolvm-agent" +path = "src/main.rs" + +[dependencies] +smolvm-protocol = { path = "../smolvm-protocol", version = "1.5.2" } +serde = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } +lazy_static = "1.4" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } +libc = "0.2" +parking_lot = "0.12" +tempfile = "3" +# Read OCI layer tars in-process so extraction can apply overlayfs whiteouts, +# and decompress them in-process too: layers may be gzip- OR zstd-compressed +# (`skopeo`/`smolvm pack` default to zstd), and the guest rootfs ships no zstd +# tool — so the old external `gunzip`-only path silently broke every zstd layer. +tar = "0.4" +# gzip via `flate2` (defaults to the pure-Rust `miniz_oxide` backend) and zstd +# via the pure-Rust `ruzstd` decoder. The C-backed `zstd` crate can't link +# against musl (its zstd-sys references glibc's `__memcpy_chk`) and the agent is +# a static-musl binary, so runtime decompression must be pure Rust. +flate2 = "1" +ruzstd = "0.7" + +# `zstd` (C-backed) is used ONLY to encode fixtures in unit tests; dev-deps are +# excluded from the musl release binary, so it never hits the link issue above. +[dev-dependencies] +zstd = "0.13" + +# Linux-specific dependencies for vsock +[target.'cfg(target_os = "linux")'.dependencies] +nix = { version = "0.27", features = ["socket", "fs"] } +# The new mount API (fsopen/fsconfig/fsmount/move_mount) for overlay mounts — lets +# us append each lower layer via `lowerdir+` instead of shelling out to `mount(8)`, +# whose `-o lowerdir=` value is capped at ~255 bytes. Already in the workspace lock. +rustix = { version = "1", features = ["mount", "fs"] } diff --git a/crates/smolvm-agent/src/crun.rs b/crates/smolvm-agent/src/crun.rs new file mode 100644 index 0000000..5479d2d --- /dev/null +++ b/crates/smolvm-agent/src/crun.rs @@ -0,0 +1,356 @@ +//! Crun OCI runtime command builder. +//! +//! This module provides a consistent interface for invoking crun commands +//! with the correct configuration (cgroup-manager, etc.). + +use std::path::Path; +use std::process::{Command, Stdio}; + +use crate::paths; + +/// Default PATH for container execution. +/// +/// This is passed explicitly when using `crun exec --env` because crun doesn't +/// preserve the container's PATH for command lookup when custom env vars are set. +pub const DEFAULT_CONTAINER_PATH: &str = + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + +/// Ensure PATH is included in environment variables for crun exec. +/// +/// When crun exec is called with `--env`, it doesn't search PATH for executables +/// unless PATH is explicitly set. This function ensures PATH is always present. +fn ensure_path_in_env(env: &[(String, String)]) -> Vec<(String, String)> { + let has_path = env.iter().any(|(k, _)| k == "PATH"); + if has_path { + env.to_vec() + } else { + let mut result = env.to_vec(); + result.push(("PATH".to_string(), DEFAULT_CONTAINER_PATH.to_string())); + result + } +} + +/// Builder for crun commands with consistent configuration. +/// +/// This ensures all crun invocations use the same cgroup-manager setting +/// and other common options. +pub struct CrunCommand { + cmd: Command, + /// Trailing positional arguments (e.g. the container id for `crun run`, or + /// the container id followed by the command for `crun exec`). Appended at + /// the very end in `spawn`/`output`/`status` so options added later (e.g. + /// `--console-socket` via `console_socket()`) still land before them. + pending_positionals: Vec, +} + +impl CrunCommand { + /// Create a new crun command with standard configuration. + /// + /// Uses `--root` to store container state on the persistent storage disk + /// instead of the default `/run/crun`, which may not be writable when the + /// rootfs is an overlayfs with an initramfs lower layer. + fn new() -> Self { + let mut cmd = Command::new(paths::CRUN_PATH); + cmd.args(["--root", paths::CRUN_ROOT_DIR]); + cmd.args(["--cgroup-manager", paths::CRUN_CGROUP_MANAGER]); + Self { + cmd, + pending_positionals: Vec::new(), + } + } + + /// Create a container: `crun create --bundle ` + /// + /// This puts the container in "created" state, ready for `crun start`. + /// Stdio defaults to null because capturing pipes can block when child + /// processes inherit file descriptors. + pub fn create(bundle_dir: &Path, container_id: &str) -> Self { + let mut c = Self::new(); + c.cmd.args([ + "create", + "--bundle", + &bundle_dir.to_string_lossy(), + container_id, + ]); + c.cmd.stdin(Stdio::null()); + c.cmd.stdout(Stdio::null()); + c.cmd.stderr(Stdio::null()); + c + } + + /// Run a container: `crun run [options] --bundle ` + /// + /// Creates, starts, waits, and deletes the container in one operation. + /// The container id is deferred so later builder calls (e.g. + /// `console_socket`) can still insert options before the positional. + pub fn run(bundle_dir: &Path, container_id: &str) -> Self { + let mut c = Self::new(); + c.cmd + .args(["run", "--bundle", &bundle_dir.to_string_lossy()]); + c.pending_positionals = vec![container_id.to_string()]; + c + } + + /// Run a container with its console handed back over a socket: + /// `crun run --bundle --console-socket `. + /// + /// crun creates the container's PTY and sends its master fd to `console_socket` + /// (via SCM_RIGHTS), instead of relaying through crun's own stdio. This is the + /// only way the agent can own the real console — and therefore drive window + /// size / resize, which crun does not propagate in stdio-relay mode. crun's own + /// stdio is unused, so it's nulled. + pub fn run_with_console(bundle_dir: &Path, container_id: &str, console_socket: &Path) -> Self { + let mut c = Self::new(); + c.cmd.args([ + "run", + "--bundle", + &bundle_dir.to_string_lossy(), + "--console-socket", + &console_socket.to_string_lossy(), + container_id, + ]); + c.cmd.stdin(Stdio::null()); + c.cmd.stdout(Stdio::null()); + // Pipe stderr so the caller can surface crun's reason if the console + // handshake fails (and it falls back to a stdio PTY). + c.cmd.stderr(Stdio::piped()); + c + } + + /// `crun exec --tty --console-socket [--env ...] [--cwd ] `. + /// The TTY counterpart to [`Self::exec`] that takes the console over a socket + /// so the agent owns the real PTY master (for resize). crun's stdio is nulled. + pub fn exec_with_console( + container_id: &str, + env: &[(String, String)], + command: &[String], + workdir: Option<&str>, + console_socket: &Path, + ) -> Self { + let mut c = Self::new(); + c.cmd.arg("exec").arg("--tty"); + c.cmd.arg("--console-socket").arg(console_socket); + let env_with_path = crate::cuda::augment_exec_env(ensure_path_in_env(env)); + for (key, value) in &env_with_path { + c.cmd.arg("--env").arg(format!("{}={}", key, value)); + } + if let Some(wd) = workdir { + c.cmd.args(["--cwd", wd]); + } + c.cmd.arg(container_id).args(command); + c.cmd.stdin(Stdio::null()); + c.cmd.stdout(Stdio::null()); + c.cmd.stderr(Stdio::piped()); + c + } + + /// Start a container: `crun start ` + pub fn start(container_id: &str) -> Self { + let mut c = Self::new(); + c.cmd.args(["start", container_id]); + c + } + + /// Execute a command in a running container. + /// + /// Supports optional working directory and TTY allocation. + /// Automatically ensures PATH is set if not provided, because crun doesn't + /// search PATH for executables when `--env` is used. + /// + /// The container id and command are deferred (see `pending_positionals`) so + /// later builder calls — notably `console_socket()` for the interactive TTY + /// path — still insert their options before the positional arguments. + pub fn exec( + container_id: &str, + env: &[(String, String)], + command: &[String], + workdir: Option<&str>, + tty: bool, + ) -> Self { + let mut c = Self::new(); + c.cmd.arg("exec"); + if tty { + c.cmd.arg("--tty"); + } + // Ensure PATH is set for command lookup; forward CUDA zero-copy opt-in. + let env_with_path = crate::cuda::augment_exec_env(ensure_path_in_env(env)); + for (key, value) in &env_with_path { + c.cmd.arg("--env").arg(format!("{}={}", key, value)); + } + if let Some(wd) = workdir { + c.cmd.args(["--cwd", wd]); + } + c.pending_positionals = std::iter::once(container_id.to_string()) + .chain(command.iter().cloned()) + .collect(); + c + } + + /// Kill a container: `crun kill ` + pub fn kill(container_id: &str, signal: &str) -> Self { + let mut c = Self::new(); + c.cmd.args(["kill", container_id, signal]); + c + } + + /// Delete a container: `crun delete [-f] ` + pub fn delete(container_id: &str, force: bool) -> Self { + let mut c = Self::new(); + if force { + c.cmd.args(["delete", "-f", container_id]); + } else { + c.cmd.args(["delete", container_id]); + } + c + } + + /// Get container state: `crun state ` + pub fn state(container_id: &str) -> Self { + let mut c = Self::new(); + c.cmd.args(["state", container_id]); + c + } + + /// Set stdin to null. + pub fn stdin_null(mut self) -> Self { + self.cmd.stdin(Stdio::null()); + self + } + + /// Set stdin to piped. + pub fn stdin_piped(mut self) -> Self { + self.cmd.stdin(Stdio::piped()); + self + } + + /// Set stdin from a raw fd (e.g., PTY slave). + /// + /// # Safety + /// The fd must be a valid open file descriptor. Ownership is transferred. + #[cfg(unix)] + pub unsafe fn stdin_from_fd(mut self, fd: std::os::unix::io::RawFd) -> Self { + use std::os::unix::io::FromRawFd; + self.cmd.stdin(Stdio::from_raw_fd(fd)); + self + } + + /// Set stdout from a raw fd (e.g., PTY slave). + /// + /// # Safety + /// The fd must be a valid open file descriptor. Ownership is transferred. + #[cfg(unix)] + pub unsafe fn stdout_from_fd(mut self, fd: std::os::unix::io::RawFd) -> Self { + use std::os::unix::io::FromRawFd; + self.cmd.stdout(Stdio::from_raw_fd(fd)); + self + } + + /// Set stderr from a raw fd (e.g., PTY slave). + /// + /// # Safety + /// The fd must be a valid open file descriptor. Ownership is transferred. + #[cfg(unix)] + pub unsafe fn stderr_from_fd(mut self, fd: std::os::unix::io::RawFd) -> Self { + use std::os::unix::io::FromRawFd; + self.cmd.stderr(Stdio::from_raw_fd(fd)); + self + } + + /// Capture stdout. + pub fn stdout_piped(mut self) -> Self { + self.cmd.stdout(Stdio::piped()); + self + } + + /// Capture stderr. + pub fn stderr_piped(mut self) -> Self { + self.cmd.stderr(Stdio::piped()); + self + } + + /// Capture both stdout and stderr. + pub fn capture_output(self) -> Self { + self.stdout_piped().stderr_piped() + } + + /// Discard both stdout and stderr. + pub fn discard_output(mut self) -> Self { + self.cmd.stdout(Stdio::null()); + self.cmd.stderr(Stdio::null()); + self + } + + /// Append any deferred positional arguments right before the command is + /// launched, so options added by the caller (e.g. `--console-socket`) land + /// before them. + fn apply_pending(&mut self) { + for arg in std::mem::take(&mut self.pending_positionals) { + self.cmd.arg(arg); + } + } + + /// Spawn the command. + pub fn spawn(mut self) -> std::io::Result { + self.apply_pending(); + self.cmd.spawn() + } + + /// Run and wait for output. + pub fn output(mut self) -> std::io::Result { + self.apply_pending(); + self.cmd.output() + } + + /// Run and wait for status. + pub fn status(mut self) -> std::io::Result { + self.apply_pending(); + self.cmd.status() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_container_path_value() { + assert!(DEFAULT_CONTAINER_PATH.contains("/usr/bin")); + assert!(DEFAULT_CONTAINER_PATH.contains("/bin")); + } + + #[test] + fn test_ensure_path_in_env_adds_path_when_missing() { + let env = vec![("HOME".to_string(), "/root".to_string())]; + let result = ensure_path_in_env(&env); + assert_eq!(result.len(), 2); + assert!(result + .iter() + .any(|(k, v)| k == "PATH" && v == DEFAULT_CONTAINER_PATH)); + } + + #[test] + fn test_ensure_path_in_env_preserves_existing_path() { + let custom_path = "/custom/bin:/other/bin"; + let env = vec![("PATH".to_string(), custom_path.to_string())]; + let result = ensure_path_in_env(&env); + assert_eq!(result.len(), 1); + assert_eq!(result[0], ("PATH".to_string(), custom_path.to_string())); + } + + #[test] + fn test_ensure_path_in_env_empty_input() { + let env: Vec<(String, String)> = vec![]; + let result = ensure_path_in_env(&env); + assert_eq!(result.len(), 1); + assert_eq!(result[0].0, "PATH"); + } + + #[test] + fn test_ensure_path_in_env_case_sensitive() { + // "path" (lowercase) should not be treated as PATH + let env = vec![("path".to_string(), "/lowercase".to_string())]; + let result = ensure_path_in_env(&env); + assert_eq!(result.len(), 2); + assert!(result.iter().any(|(k, _)| k == "PATH")); + } +} diff --git a/crates/smolvm-agent/src/cuda.rs b/crates/smolvm-agent/src/cuda.rs new file mode 100644 index 0000000..aa053e6 --- /dev/null +++ b/crates/smolvm-agent/src/cuda.rs @@ -0,0 +1,291 @@ +//! Guest-side CUDA forwarding wiring for the workload container. +//! +//! When a VM is launched with `--cuda`, the launcher sets `SMOLVM_CUDA_ZEROCOPY` +//! in the agent's (PID 1) environment. The workload container gets its env from +//! the image plus the request — not from the agent's own env — so the zero-copy +//! opt-in has to be forwarded into the container spec explicitly, the same way +//! [`crate::ssh_agent`] forwards `SSH_AUTH_SOCK`. +//! +//! With the flag set, the guest CUDA shim (`libcudart.so`) backs +//! `cudaHostAlloc`/`cudaMallocHost` with page-locked guest RAM whose +//! guest-physical frames it reads from `/proc/self/pagemap`, so a memcpy ships +//! only a guest-physical descriptor and the host DMAs straight from guest RAM. +//! It degrades to byte-shipping wherever that path is unavailable (no +//! `CAP_SYS_ADMIN`, older libkrun), so forwarding it is always safe. + +/// The env var the launcher sets on the agent, and that the guest shim reads. +const ZEROCOPY_ENV: &str = "SMOLVM_CUDA_ZEROCOPY"; + +/// Whether CUDA guest-RAM zero-copy was requested for this VM. +pub fn zerocopy_enabled() -> bool { + std::env::var(ZEROCOPY_ENV).as_deref() == Ok("1") +} + +/// Where the guest CUDA shims ship inside the VM rootfs (from the agent +/// rootfs). Absent on builds without CUDA shim bundling — every staging step +/// degrades to a no-op so `--cuda` still works in the manual-setup mode. +const GUEST_SHIM_DIR: &str = "/usr/local/lib/smolvm-cuda"; +/// Where the shim dir is bind-mounted inside the workload container, and put +/// on `LD_LIBRARY_PATH` so the loader finds `libcuda.so.1` there. +const CONTAINER_SHIM_DIR: &str = "/opt/smolvm-cuda"; +/// The runtime shim file inside [`GUEST_SHIM_DIR`] (one cdylib exports the +/// whole cudart/cuBLAS/cuBLASLt/cuDNN surface; it is staged under each soname). +const RUNTIME_SHIM: &str = "libcudart-shim.so"; +/// The driver shim (`libcuda.so.1`) inside [`GUEST_SHIM_DIR`]. +const DRIVER_SHIM: &str = "libcuda.so.1"; + +/// The pip-bundled NVIDIA sonames PyTorch's `libtorch_cuda.so` resolves via +/// DT_RPATH. RPATH is searched *before* `LD_LIBRARY_PATH`, so the only way to +/// interpose is to place the shim at these exact paths (a read-only bind mount +/// over each file). CUDA 12 and 11 wheel layouts. +const RPATH_PINNED_SONAMES: &[&str] = &[ + "libcudart.so.12", + "libcublas.so.12", + "libcublasLt.so.12", + "libcudnn.so.9", + "libcudart.so.11.0", + "libcublas.so.11", + "libcublasLt.so.11", + "libcudnn.so.8", +]; + +/// Forward the CUDA opt-in into the workload container spec and, when the +/// guest shims are bundled, stage them so an unmodified PyTorch works with no +/// user setup: the shim dir rides `LD_LIBRARY_PATH` (covers `libcuda.so.1`, +/// which nothing RPATH-pins) and the runtime shim is bind-mounted over each +/// pip-bundled NVIDIA library found in the image rootfs (RPATH-pinned, so env +/// vars can't reach them). Used on the fresh-container path (`crun run`/ +/// `create`). No-op unless CUDA was requested. +pub fn inject_into_container(spec: &mut crate::oci::OciSpec, rootfs: &std::path::Path) { + inject_into_container_if(spec, rootfs, zerocopy_enabled()); +} + +/// Testable core of [`inject_into_container`]. +fn inject_into_container_if( + spec: &mut crate::oci::OciSpec, + rootfs: &std::path::Path, + enabled: bool, +) { + if !enabled { + return; + } + spec.add_env(ZEROCOPY_ENV, "1"); + stage_shims(spec, rootfs, std::path::Path::new(GUEST_SHIM_DIR)); +} + +/// Bind-mount the bundled shims into the container. Split from the gate so +/// tests can point `shim_dir` at a fixture. +fn stage_shims( + spec: &mut crate::oci::OciSpec, + rootfs: &std::path::Path, + shim_dir: &std::path::Path, +) { + let runtime = shim_dir.join(RUNTIME_SHIM); + let driver = shim_dir.join(DRIVER_SHIM); + if !runtime.is_file() || !driver.is_file() { + return; // shims not bundled — manual staging still works + } + + // Driver shim: the whole shim dir at a stable path + LD_LIBRARY_PATH. + // `libcuda.so.1` is resolved through the normal loader search (no RPATH in + // the way), and this also lets users link the runtime shim directly. + spec.add_bind_mount(&shim_dir.to_string_lossy(), CONTAINER_SHIM_DIR, true); + append_ld_library_path(&mut spec.process.env, CONTAINER_SHIM_DIR); + + // Runtime shim over each RPATH-pinned pip-bundled NVIDIA library. + let runtime_src = runtime.to_string_lossy(); + for hit in find_rpath_pinned_libs(rootfs) { + let dest = format!( + "/{}", + hit.strip_prefix(rootfs).unwrap_or(&hit).to_string_lossy() + ); + spec.add_bind_mount(&runtime_src, &dest, true); + } +} + +/// Append `dir` to the spec's `LD_LIBRARY_PATH`, preserving an image-provided +/// value; creates the variable if absent, skips if already present. +fn append_ld_library_path(env: &mut Vec, dir: &str) { + for e in env.iter_mut() { + if let Some(v) = e.strip_prefix("LD_LIBRARY_PATH=") { + if v.split(':').any(|p| p == dir) { + return; + } + *e = format!("LD_LIBRARY_PATH={v}:{dir}"); + return; + } + } + env.push(format!("LD_LIBRARY_PATH={dir}")); +} + +/// Find pip-bundled NVIDIA libraries in the image rootfs: files named like the +/// RPATH-pinned sonames under a `site-packages`/`dist-packages` → `nvidia` +/// wheel layout. Bounded walk: skips pseudo-filesystems and never follows +/// symlinks (wheel layouts don't use them and cycles would hang the boot). +fn find_rpath_pinned_libs(rootfs: &std::path::Path) -> Vec { + const SKIP_TOP: &[&str] = &["proc", "sys", "dev", "run", "tmp", "boot"]; + const MAX_DEPTH: usize = 16; + let mut hits = Vec::new(); + let mut stack: Vec<(std::path::PathBuf, usize)> = vec![(rootfs.to_path_buf(), 0)]; + while let Some((dir, depth)) = stack.pop() { + if depth >= MAX_DEPTH { + continue; + } + let entries = match std::fs::read_dir(&dir) { + Ok(e) => e, + Err(_) => continue, + }; + for entry in entries.flatten() { + let ft = match entry.file_type() { + Ok(t) => t, + Err(_) => continue, + }; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if ft.is_dir() && !ft.is_symlink() { + if depth == 0 && SKIP_TOP.contains(&name.as_ref()) { + continue; + } + stack.push((entry.path(), depth + 1)); + } else if ft.is_file() && RPATH_PINNED_SONAMES.contains(&name.as_ref()) { + let p = entry.path(); + let s = p.to_string_lossy(); + if s.contains("/nvidia/") + && (s.contains("/site-packages/") || s.contains("/dist-packages/")) + { + hits.push(p); + } + } + } + } + hits.sort(); + hits +} + +/// Append the CUDA opt-in (and the shim dir's loader path) to an explicit exec +/// env when enabled. Used on the `crun exec` path (joining a persistent +/// machine's keep-alive container), where the workload env is passed via +/// `--env` rather than inherited from the container spec, so the spec injection +/// above doesn't reach it. The bind mounts themselves were established when the +/// keep-alive container was created. No-op when disabled or already present. +pub fn augment_exec_env(mut env: Vec<(String, String)>) -> Vec<(String, String)> { + if !zerocopy_enabled() { + return env; + } + if !env.iter().any(|(k, _)| k == ZEROCOPY_ENV) { + env.push((ZEROCOPY_ENV.to_string(), "1".to_string())); + } + match env.iter_mut().find(|(k, _)| k == "LD_LIBRARY_PATH") { + Some((_, v)) => { + if !v.split(':').any(|p| p == CONTAINER_SHIM_DIR) { + *v = format!("{v}:{CONTAINER_SHIM_DIR}"); + } + } + None => env.push(( + "LD_LIBRARY_PATH".to_string(), + CONTAINER_SHIM_DIR.to_string(), + )), + } + env +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::oci::{OciSpec, ProcessIdentity}; + + fn spec() -> OciSpec { + OciSpec::new( + &["true".to_string()], + &[], + "/", + false, + &ProcessIdentity::root(), + false, + ) + } + + #[test] + fn injects_when_enabled() { + let mut s = spec(); + inject_into_container_if(&mut s, std::path::Path::new("/nonexistent"), true); + assert!(s.process.env.iter().any(|e| e == "SMOLVM_CUDA_ZEROCOPY=1")); + } + + #[test] + fn noop_when_disabled() { + let mut s = spec(); + inject_into_container_if(&mut s, std::path::Path::new("/nonexistent"), false); + assert!(!s + .process + .env + .iter() + .any(|e| e.starts_with("SMOLVM_CUDA_ZEROCOPY"))); + } + + #[test] + fn augment_exec_env_is_idempotent() { + let base = vec![("SMOLVM_CUDA_ZEROCOPY".to_string(), "1".to_string())]; + // Gate off in tests (env var unset) → unchanged. + assert_eq!(augment_exec_env(base.clone()), base); + } + + #[test] + fn ld_library_path_append_preserves_existing() { + let mut env = vec!["LD_LIBRARY_PATH=/usr/local/nvidia/lib".to_string()]; + append_ld_library_path(&mut env, CONTAINER_SHIM_DIR); + assert_eq!( + env[0], + format!("LD_LIBRARY_PATH=/usr/local/nvidia/lib:{CONTAINER_SHIM_DIR}") + ); + // Idempotent. + append_ld_library_path(&mut env, CONTAINER_SHIM_DIR); + assert_eq!(env.len(), 1); + assert_eq!(env[0].matches(CONTAINER_SHIM_DIR).count(), 1); + } + + #[test] + fn ld_library_path_created_when_absent() { + let mut env = vec!["PATH=/usr/bin".to_string()]; + append_ld_library_path(&mut env, CONTAINER_SHIM_DIR); + assert!(env.contains(&format!("LD_LIBRARY_PATH={CONTAINER_SHIM_DIR}"))); + } + + #[test] + fn finds_and_overmounts_wheel_libs() { + let tmp = std::env::temp_dir().join(format!("cuda-stage-test-{}", std::process::id())); + let rootfs = tmp.join("rootfs"); + let wheel = rootfs.join("usr/lib/python3.11/site-packages/nvidia/cublas/lib"); + std::fs::create_dir_all(&wheel).unwrap(); + std::fs::write(wheel.join("libcublas.so.12"), b"real").unwrap(); + std::fs::write(wheel.join("libnvblas.so.12"), b"other").unwrap(); // not pinned + // A same-named file outside a wheel layout must NOT match. + let stray = rootfs.join("opt/other"); + std::fs::create_dir_all(&stray).unwrap(); + std::fs::write(stray.join("libcublas.so.12"), b"stray").unwrap(); + + let hits = find_rpath_pinned_libs(&rootfs); + assert_eq!(hits.len(), 1); + assert!(hits[0].ends_with("nvidia/cublas/lib/libcublas.so.12")); + + // With a shim fixture present, staging adds the dir mount + overmount. + let shim_dir = tmp.join("shims"); + std::fs::create_dir_all(&shim_dir).unwrap(); + std::fs::write(shim_dir.join(RUNTIME_SHIM), b"shim").unwrap(); + std::fs::write(shim_dir.join(DRIVER_SHIM), b"shim").unwrap(); + let mut s = spec(); + stage_shims(&mut s, &rootfs, &shim_dir); + assert!(s.mounts.iter().any(|m| m.destination == CONTAINER_SHIM_DIR)); + assert!(s.mounts.iter().any(|m| m + .destination + .ends_with("nvidia/cublas/lib/libcublas.so.12") + && m.source.ends_with(RUNTIME_SHIM))); + assert!(s + .process + .env + .iter() + .any(|e| e.starts_with("LD_LIBRARY_PATH=") && e.contains(CONTAINER_SHIM_DIR))); + + std::fs::remove_dir_all(&tmp).ok(); + } +} diff --git a/crates/smolvm-agent/src/dns_proxy.rs b/crates/smolvm-agent/src/dns_proxy.rs new file mode 100644 index 0000000..fa54654 --- /dev/null +++ b/crates/smolvm-agent/src/dns_proxy.rs @@ -0,0 +1,232 @@ +//! Guest-side DNS filtering proxy. +//! +//! Listens on `127.0.0.1:53` (both UDP and TCP) inside the VM and forwards +//! raw DNS packets to the host via vsock for filtering. The host decides +//! whether to resolve the query (domain is in the allowlist) or return +//! NXDOMAIN. +//! +//! This is a dumb bridge — no DNS parsing happens in the agent. The host +//! does all filtering, keeping the agent binary small. +//! +//! TCP/53 is required for: +//! - Responses that exceed 512 bytes (TC bit set in the UDP reply) +//! - Clients that prefer TCP (Go's `net` resolver, some systemd-resolved configs) +//! +//! Framing over the vsock stream (stream-oriented, so we need packet +//! boundaries): +//! [2-byte BE length] [raw DNS packet bytes] +//! +//! The host responds with the same framing. This matches the standard TCP DNS +//! wire format (RFC 1035 §4.2.2) exactly, so no extra wrapping is needed for +//! the TCP path. + +use smolvm_protocol::guest_env; +use smolvm_protocol::ports; +use std::io::{self, Read, Write}; +use std::net::{TcpListener, UdpSocket}; +use std::thread; + +/// Maximum DNS packet size over UDP (RFC 1035 §2.3.4). +const MAX_DNS_UDP_PACKET: usize = 512; + +/// Maximum DNS response size over TCP (2-byte length field limit). +const MAX_DNS_TCP_PACKET: usize = 65535; + +/// Start the guest-side DNS proxy in background threads. +/// +/// Spawns one thread for UDP/53 and one for TCP/53. Both reuse the same +/// `forward_to_host` path — the vsock channel framing is identical for both +/// protocols since TCP DNS already uses 2-byte length prefixes. +/// +/// Rewrites `/etc/resolv.conf` to point to localhost so all DNS queries +/// from guest applications go through this proxy. +pub fn start() { + // Rewrite resolv.conf to route DNS through our proxy + if let Err(e) = std::fs::write("/etc/resolv.conf", "nameserver 127.0.0.1\n") { + tracing::warn!(error = %e, "failed to rewrite /etc/resolv.conf for DNS filtering"); + return; + } + + thread::Builder::new() + .name("dns-proxy-udp".into()) + .spawn(|| { + if let Err(e) = run_udp_proxy() { + tracing::warn!(error = %e, "guest UDP DNS proxy stopped"); + } + }) + .ok(); + + thread::Builder::new() + .name("dns-proxy-tcp".into()) + .spawn(|| { + if let Err(e) = run_tcp_proxy() { + tracing::warn!(error = %e, "guest TCP DNS proxy stopped"); + } + }) + .ok(); +} + +/// Check if DNS filtering is enabled via environment variable. +/// The host sets this when `--allow-host` is used. +pub fn is_enabled() -> bool { + std::env::var(guest_env::DNS_FILTER).as_deref() == Ok("1") +} + +fn run_udp_proxy() -> io::Result<()> { + let udp = UdpSocket::bind("127.0.0.1:53")?; + tracing::info!("guest DNS proxy listening on UDP 127.0.0.1:53"); + + let mut buf = [0u8; MAX_DNS_UDP_PACKET]; + + loop { + let (len, src_addr) = udp.recv_from(&mut buf)?; + if len == 0 { + continue; + } + + let response = match forward_to_host(&buf[..len]) { + Ok(resp) => resp, + Err(e) => { + tracing::debug!(error = %e, "DNS forward to host failed, returning SERVFAIL"); + build_servfail(&buf[..len]) + } + }; + + if let Err(e) = udp.send_to(&response, src_addr) { + tracing::debug!(error = %e, "failed to send DNS response"); + } + } +} + +fn run_tcp_proxy() -> io::Result<()> { + let listener = TcpListener::bind("127.0.0.1:53")?; + tracing::info!("guest DNS proxy listening on TCP 127.0.0.1:53"); + + for stream in listener.incoming() { + match stream { + Ok(mut stream) => { + thread::spawn(move || { + if let Err(e) = handle_tcp_connection(&mut stream) { + tracing::debug!(error = %e, "TCP DNS connection error"); + } + }); + } + Err(e) => tracing::debug!(error = %e, "TCP DNS accept error"), + } + } + Ok(()) +} + +/// Handle a single TCP DNS connection. +/// +/// RFC 1035 §4.2.2 allows multiple queries on a single TCP connection +/// (pipelining). Each query is length-prefixed with a 2-byte BE value, +/// matching our vsock framing exactly — `forward_to_host` is used unchanged. +fn handle_tcp_connection(stream: &mut (impl Read + Write)) -> io::Result<()> { + loop { + let mut len_buf = [0u8; 2]; + match stream.read_exact(&mut len_buf) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => break, + Err(e) => return Err(e), + } + let query_len = u16::from_be_bytes(len_buf) as usize; + if query_len == 0 { + break; + } + + let mut query = vec![0u8; query_len]; + stream.read_exact(&mut query)?; + + let response = match forward_to_host(&query) { + Ok(resp) => resp, + Err(e) => { + tracing::debug!(error = %e, "DNS forward to host failed, returning SERVFAIL"); + build_servfail(&query) + } + }; + + let resp_len = response.len() as u16; + stream.write_all(&resp_len.to_be_bytes())?; + stream.write_all(&response)?; + stream.flush()?; + } + Ok(()) +} + +/// Forward a raw DNS packet to the host via vsock and return the response. +/// +/// One vsock connection per query. The host may return a response larger than +/// 512 bytes if it performed a TCP retry on the guest's behalf (TC-bit path), +/// so the cap is the protocol maximum (65535) for both UDP and TCP callers. +fn forward_to_host(query: &[u8]) -> io::Result> { + let mut stream = super::vsock::connect(ports::DNS_FILTER)?; + + // Send: [2-byte BE length] [query bytes] + let len = query.len() as u16; + stream.write_all(&len.to_be_bytes())?; + stream.write_all(query)?; + stream.flush()?; + + // Receive: [2-byte BE length] [response bytes] + let mut len_buf = [0u8; 2]; + stream.read_exact(&mut len_buf)?; + let resp_len = u16::from_be_bytes(len_buf) as usize; + + if resp_len > MAX_DNS_TCP_PACKET { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("DNS response too large: {resp_len}"), + )); + } + + let mut resp = vec![0u8; resp_len]; + stream.read_exact(&mut resp)?; + + Ok(resp) +} + +/// Build a minimal SERVFAIL response from a query. +/// Copies the query ID and question, sets the SERVFAIL rcode. +fn build_servfail(query: &[u8]) -> Vec { + if query.len() < 12 { + return vec![]; + } + + let mut resp = query.to_vec(); + // Set QR bit (response), keep opcode, set RCODE=2 (SERVFAIL) + resp[2] = 0x80 | (resp[2] & 0x78); // QR=1, preserve opcode + resp[3] = (resp[3] & 0xF0) | 0x02; // RCODE=SERVFAIL + // Zero answer/authority/additional counts + resp[6..12].fill(0); + resp +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_servfail_preserves_id() { + // Minimal DNS query header (12 bytes) + let query = vec![ + 0xAB, 0xCD, // ID + 0x01, 0x00, // Flags: standard query + 0x00, 0x01, // QDCOUNT: 1 + 0x00, 0x00, // ANCOUNT: 0 + 0x00, 0x00, // NSCOUNT: 0 + 0x00, 0x00, // ARCOUNT: 0 + ]; + let resp = build_servfail(&query); + assert_eq!(resp[0], 0xAB); // ID preserved + assert_eq!(resp[1], 0xCD); + assert_eq!(resp[2] & 0x80, 0x80); // QR=1 (response) + assert_eq!(resp[3] & 0x0F, 0x02); // RCODE=SERVFAIL + } + + #[test] + fn test_build_servfail_short_query() { + let resp = build_servfail(&[0x00, 0x01]); + assert!(resp.is_empty()); + } +} diff --git a/crates/smolvm-agent/src/main.rs b/crates/smolvm-agent/src/main.rs new file mode 100644 index 0000000..4853dcf --- /dev/null +++ b/crates/smolvm-agent/src/main.rs @@ -0,0 +1,6195 @@ +//! smolvm guest agent. +//! +//! This agent runs inside smolvm VMs and handles: +//! - OCI image pulling via crane +//! - Layer extraction and storage management +//! - Overlay filesystem preparation for workloads +//! - Command execution with optional interactive/TTY support +//! +//! Communication is via vsock on port 6000. + +use smolvm_protocol::{ + error_codes, guest_env, ports, AgentRequest, AgentResponse, Envelope, FsNotifyEvent, + RegistryAuth, AGENT_READY_MARKER, LAYER_CHUNK_SIZE, PROTOCOL_VERSION, +}; +use std::io::{Read, Write}; +use std::os::unix::io::AsRawFd; +use std::process::{Child, Command, Stdio}; +use std::sync::OnceLock; +use tracing::{debug, error, info, warn}; + +mod crun; + +/// Ensures storage disk is mounted exactly once. The mount happens either during +/// deferred init (the common case) or on the first request that needs storage +/// (if a request arrives before deferred init reaches the mount step). +/// This eliminates the race between early ready signaling and storage access. +static STORAGE_MOUNTED: OnceLock = OnceLock::new(); + +fn ensure_storage_mounted() -> bool { + *STORAGE_MOUNTED.get_or_init(|| { + let t0 = uptime_ms(); + let ok = mount_storage_disk(); + // Log after tracing may or may not be initialized — use boot_log for safety. + if ok { + boot_log( + "INFO", + &format!("storage disk mounted (duration_ms={})", uptime_ms() - t0), + ); + } else { + boot_log( + "ERROR", + "storage disk NOT mounted — image pulls and container overlays will fail", + ); + } + ok + }) +} + +/// Format a structured JSON log line for early boot (before tracing is up). +fn format_boot_log(level: &str, msg: &str) -> String { + let escaped = serde_json::to_string(msg).unwrap_or_else(|_| format!("\"{}\"", msg)); + format!( + r#"{{"level":"{}","message":{},"target":"smolvm_agent::boot"}}"#, + level, escaped + ) +} + +/// Write a structured JSON log line to stderr during early boot, +/// before tracing_subscriber is initialized. This keeps +/// agent-console.log as valid JSON throughout. +/// Also mirrors to /dev/kmsg so messages appear in `dmesg` inside the VM. +/// +/// These timing lines (`boot agent_entry`, `mounts_done`, `rootfs_done`, etc.) +/// are intentionally permanent support logs — they are not gated on RUST_LOG +/// because they run before the tracing subscriber exists. Output goes only to +/// `agent-console.log` (readable via `~/Library/Caches/smolvm/vms//`) and +/// the in-VM kernel ring buffer, neither of which is visible to end users in +/// normal operation. +fn boot_log(level: &str, msg: &str) { + let line = format_boot_log(level, msg); + eprintln!("{}", line); + // Mirror to /dev/kmsg so dmesg captures early-boot messages even when + // the console-log pipe isn't receiving eprintln! output. + #[cfg(target_os = "linux")] + { + use std::io::Write; + if let Ok(mut f) = std::fs::OpenOptions::new().write(true).open("/dev/kmsg") { + let _ = writeln!(f, "smolvm-agent: {}", line); + } + } +} +mod cuda; +mod dns_proxy; +mod network; +mod oci; +mod paths; +mod process; +#[cfg(target_os = "linux")] +mod pty; +mod retry; +mod rosetta; +mod ssh_agent; +mod storage; +#[cfg(target_os = "linux")] +mod timesync; +mod vsock; + +// ============================================================================ +// Configuration Constants +// ============================================================================ + +/// Initial buffer size for reading requests from the vsock socket. +const REQUEST_BUFFER_SIZE: usize = 64 * 1024; // 64KB + +/// Maximum allowed message size to prevent DoS via memory exhaustion. +const MAX_MESSAGE_SIZE: usize = 16 * 1024 * 1024; // 16MB + +/// Buffer size for streaming stdout/stderr in interactive mode. +const IO_BUFFER_SIZE: usize = 4096; + +/// Default poll timeout in milliseconds for interactive I/O loop. +const INTERACTIVE_POLL_TIMEOUT_MS: i32 = 100; + +/// Timeout for network connectivity test operations. +/// Used in diagnostics/troubleshooting functions. +const NETWORK_TEST_TIMEOUT_SECS: u64 = 10; + +/// Poll interval for checking process completion in VM exec. +const PROCESS_POLL_INTERVAL_MS: u64 = 10; + +/// Get system uptime in milliseconds (for timing relative to boot). +fn uptime_ms() -> u64 { + if let Ok(contents) = std::fs::read_to_string("/proc/uptime") { + if let Some(uptime_str) = contents.split_whitespace().next() { + if let Ok(uptime_secs) = uptime_str.parse::() { + return (uptime_secs * 1000.0) as u64; + } + } + } + 0 +} + +/// Get uptime via CLOCK_BOOTTIME syscall — works before /proc is mounted. +#[cfg(target_os = "linux")] +fn boottime_ms() -> u64 { + let mut ts = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + unsafe { + libc::clock_gettime(libc::CLOCK_BOOTTIME, &mut ts); + } + (ts.tv_sec as u64) * 1000 + (ts.tv_nsec as u64) / 1_000_000 +} + +/// Decide what to write to `/etc/resolv.conf` on the TSI (no-virtio) path, or +/// `None` to leave the existing file untouched. +/// +/// The host delivers a `--dns` override through one of two mechanisms depending +/// on the boot path: it either forwards it as `guest_env::DNS`, or it writes the +/// nameserver straight into the rootfs's resolv.conf before boot (the libkrun +/// backend's `setup_dns`). This one function honors both: an explicit override +/// wins, and otherwise we only repair a stale loopback/empty resolv.conf (left +/// by a prior `--allow-host` run that pointed at the local DNS proxy) — we never +/// clobber a valid nameserver the host already wrote. The old code wrote the +/// hardcoded public resolvers unconditionally, which both dropped `--dns` and +/// overwrote `setup_dns`'s value, so `--dns` never took effect on TSI. +fn tsi_resolv_conf(dns_override: Option<&str>, current: &str) -> Option { + match dns_override.map(str::trim) { + Some(dns) if !dns.is_empty() => Some(format!("nameserver {dns}\n")), + _ if current.contains("127.0.0.1") || current.trim().is_empty() => { + Some("nameserver 1.1.1.1\nnameserver 8.8.8.8\n".to_string()) + } + _ => None, + } +} + +/// Seed the guest wall clock from `SMOLVM_HOST_TIME_NS` (the host's launch time), +/// but only when the guest clock is obviously wrong (before 2020). Hypervisors +/// without a guest-readable paravirt clock (WHP on Windows) boot the guest at +/// ~1999, breaking all TLS cert validation; this fixes that without fighting an +/// already-correct kvmclock (KVM) or HVF-seeded clock (macOS). +#[cfg(target_os = "linux")] +fn maybe_set_clock_from_host() { + const Y2020_SECS: i64 = 1_577_836_800; + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + if now_secs >= Y2020_SECS { + return; + } + let Ok(ns_str) = std::env::var(smolvm_protocol::guest_env::HOST_TIME_NS) else { + return; + }; + let Ok(ns) = ns_str.parse::() else { + return; + }; + let ts = libc::timespec { + tv_sec: (ns / 1_000_000_000) as libc::time_t, + tv_nsec: (ns % 1_000_000_000) as libc::c_long, + }; + // SAFETY: ts is a valid timespec; the agent runs as PID-1 with CAP_SYS_TIME. + let _ = unsafe { libc::clock_settime(libc::CLOCK_REALTIME, &ts) }; +} + +fn main() { + // Quick --version check (used by init script to detect rootfs updates) + if std::env::args().any(|a| a == "--version") { + println!("{}", env!("CARGO_PKG_VERSION")); + std::process::exit(0); + } + + // Earliest possible timing point — CLOCK_BOOTTIME works before /proc exists. + #[cfg(target_os = "linux")] + boot_log( + "INFO", + &format!("boot agent_entry uptime_ms={}", boottime_ms()), + ); + + // Seed the guest wall clock from the host's launch time when the hypervisor + // gives the guest no readable paravirt clock and it boots at ~1999 (WHP on + // Windows); without this every TLS handshake fails ("cert not yet valid"). + // No-op when the clock already looks sane (kvmclock on KVM, HVF on macOS). + #[cfg(target_os = "linux")] + maybe_set_clock_from_host(); + + // Keep the guest clock synced with the host for the machine's whole life: + // libkrun's host-side TimesyncThread pushes the host time over a vsock DGRAM + // at boot, every 60s, and — critically — right after the host resumes from + // sleep (when the frozen VM's clock has fallen behind). Boot-only seeding + // above doesn't cover that; without this the guest drifts behind after macOS + // sleep until TLS breaks (issue #521). + #[cfg(target_os = "linux")] + timesync::spawn(); + + // CRITICAL: Mount essential filesystems FIRST, before anything else. + // When running as init (PID 1), we need these for the system to function. + // This must happen before logging (which needs /dev for output). + mount_essential_filesystems(); + boot_log( + "INFO", + &format!("boot mounts_done uptime_ms={}", uptime_ms()), + ); + + // Create /dev/dri device nodes only when GPU is enabled. The setup + // function polls up to 500ms for the virtio-gpu driver to finish probing — + // running it on non-GPU machines wastes the full polling window. + #[cfg(target_os = "linux")] + if std::env::var(guest_env::GPU).as_deref() == Ok(guest_env::VALUE_ON) { + setup_gpu_dev_nodes(); + } + + // Set up persistent rootfs overlay (if /dev/vdb exists). + // This does overlayfs + pivot_root before anything else touches the filesystem. + setup_persistent_rootfs(); + boot_log( + "INFO", + &format!("boot rootfs_done uptime_ms={}", uptime_ms()), + ); + + // Start seatd AFTER pivot_root so its socket lands at /run/seatd.sock in + // the live root. Wayland compositors (Weston, sway) and DRI3-capable X + // servers (Xwayland) connect to this socket via libseat to acquire DRM + // devices. Without seatd, GPU-accelerated display workloads fail with + // "No backend was able to open a seat." + #[cfg(target_os = "linux")] + if std::env::var(guest_env::GPU).as_deref() == Ok(guest_env::VALUE_ON) { + start_seatd(); + } + + // CRITICAL: Create vsock listener IMMEDIATELY after mounts. + // This must happen before logging setup to minimize time to listener ready. + // The kernel boots in ~30ms and host connects immediately after. + let listener = match vsock::listen(ports::AGENT_CONTROL) { + Ok(l) => l, + Err(e) => { + boot_log("ERROR", &format!("FAILED to create vsock listener: {}", e)); + std::process::exit(1); + } + }; + boot_log( + "INFO", + &format!("boot vsock_bound uptime_ms={}", uptime_ms()), + ); + + // Set up signal handlers for graceful shutdown (sync before exit) + setup_signal_handlers(); + + // --- Deferred init: runs before the host is allowed to send requests --- + // + // Keep the ready marker hidden until the guest has completed the boot-time + // work that init commands may depend on: + // - guest networking + // - storage mount + // - packed layers + // - volume mounts and /workspace setup + // - SSH agent / DNS proxy bridges + // + // `machine run` can reach init very quickly, so signaling readiness too + // early creates a race where init starts before the guest is actually ready + // to resolve or connect to the network. Named-machine startup has enough + // extra host-side work to mostly hide this, but the guest itself should not + // report "ready" until the boot prerequisites are complete. + // Storage mount is behind a OnceLock (ensure_storage_mounted) so it + // happens exactly once — either here or on first storage-dependent request. + + let start_uptime = uptime_ms(); + + // Initialize logging after deferred boot work begins; boot_log is still + // used for the earliest readiness breadcrumbs. + tracing_subscriber::fmt() + .json() + .with_env_filter( + tracing_subscriber::EnvFilter::from_default_env() + .add_directive("smolvm_agent=info".parse().expect("valid directive")), + ) + .init(); + + info!( + version = env!("CARGO_PKG_VERSION"), + uptime_ms = start_uptime, + "smolvm-agent started, deferred boot work in progress" + ); + + let t0 = uptime_ms(); + match network::configure_from_env() { + Ok(true) => { + info!( + duration_ms = uptime_ms() - t0, + "guest virtio network configured" + ); + } + Ok(false) => { + // TSI mode or no network. The overlay may contain a stale + // "nameserver 127.0.0.1" written by a previous --allow-host run. + // TSI forwards UDP to the resolver directly, so reset resolv.conf to + // a known-good value unless the DNS filter proxy is about to + // overwrite it with 127.0.0.1 anyway. Honor an explicit `--dns` + // override (forwarded by the host as guest_env::DNS) — the virtio + // path already derives the guest resolver from the same env var, so + // both backends now agree on which nameserver the guest uses. This + // is what makes `--dns` work on a network where the public resolvers + // (1.1.1.1/8.8.8.8) are blocked. + if !dns_proxy::is_enabled() { + let dns_override = std::env::var(guest_env::DNS).ok(); + let current = std::fs::read_to_string("/etc/resolv.conf").unwrap_or_default(); + if let Some(new) = tsi_resolv_conf(dns_override.as_deref(), ¤t) { + let _ = std::fs::write("/etc/resolv.conf", new); + } + } + } + Err(err) => { + error!(error = %err, "failed to configure guest network"); + std::process::exit(1); + } + } + + // Initialize packed layers support (if SMOLVM_PACKED_LAYERS env var is set) + let t0 = uptime_ms(); + if let Some(packed_dir) = storage::get_packed_layers_dir() { + info!( + duration_ms = uptime_ms() - t0, + packed_dir = %packed_dir.display(), + "packed layers initialized" + ); + } + + // Initialize volume mounts from SMOLVM_MOUNT_* env vars. + // MUST run before the /workspace symlink below — if the user passed + // `-v host:/workspace`, the bind mount claims /workspace first and + // the symlink's `!exists()` guard correctly skips creation. + let t0 = uptime_ms(); + let boot_mounts = storage::init_volume_mounts(); + if !boot_mounts.is_empty() { + info!( + duration_ms = uptime_ms() - t0, + mount_count = boot_mounts.len(), + "volume mounts initialized at boot" + ); + } + + // Create /workspace symlink for bare VMs (no -v targeting /workspace). + // Image-based VMs get /workspace via a bind mount in the container spec, + // but bare VMs run directly in the VM rootfs where /workspace doesn't + // exist. The symlink makes /workspace available in both modes. + // Placed AFTER init_volume_mounts so that `-v host:/workspace` takes + // priority — the exists() check sees the bind mount and skips. + // + // The target `/storage/workspace` is created by the storage mount, which + // now runs *after* this block (it was moved below signal_ready_to_host() + // to overlap with the host connect). So the symlink is created even when + // the target doesn't exist yet — a dangling symlink is fine, it resolves + // once storage mounts, which always completes before the accept loop reads + // any request. Gating on `workspace_target.exists()` here would silently + // skip the symlink for bare VMs and is what regressed `/workspace`. + { + let workspace_link = std::path::Path::new("/workspace"); + let workspace_target = std::path::Path::new("/storage/workspace"); + if !workspace_link.exists() { + let _ = std::os::unix::fs::symlink(workspace_target, workspace_link); + } + } + + // Registry load+reconcile deferred to first container operation via + // REGISTRY.ensure_loaded(). On fresh boot, no containers from a previous + // instance survive, so this work (~30-50ms for crun list + JSON parse) + // is wasted if no container operations are requested. + + // Start SSH agent forwarding bridge if enabled by host + if ssh_agent::is_enabled() { + info!("SSH agent forwarding enabled, starting guest bridge"); + ssh_agent::start(); + // Set env so all child processes (git, ssh, etc.) find the agent socket + std::env::set_var("SSH_AUTH_SOCK", ssh_agent::GUEST_SSH_AUTH_SOCK); + } + + // Mount the Rosetta 2 runtime and register the binfmt_misc handler if the + // host attached it. Must run after pivot_root (the wrapper lives in the + // rootfs) and after /proc is mounted (binfmt_misc registration). + if rosetta::is_enabled() { + info!("Rosetta 2 requested, setting up x86_64 translation"); + rosetta::setup(); + } + + // Start DNS filtering proxy if enabled by host (when --allow-host is used) + if dns_proxy::is_enabled() { + info!("DNS filtering enabled, starting guest proxy"); + dns_proxy::start(); + } + + // If the host started us with --gpu, sanity-check that the guest + // kernel actually sees a virtio-gpu device. libkrun accepts the + // GPU config call regardless of whether the embedded kernel has + // the `virtio-gpu` driver compiled in, so the only place this + // mismatch surfaces is here. Without this log, the user discovers + // the missing GPU much later — when their workload makes a + // rendering call and crashes with a confused EGL/Vulkan error. + #[cfg(target_os = "linux")] + if std::env::var(guest_env::GPU).as_deref() == Ok(guest_env::VALUE_ON) { + log_gpu_status(); + } + + info!( + total_startup_ms = uptime_ms() - start_uptime, + uptime_ms = uptime_ms(), + "agent init complete, entering accept loop" + ); + + // Only now is the guest safe to accept host requests for init/exec. + // Network is configured above; storage is mounted below while the host is + // establishing its vsock connection (~10-30ms), so the accept loop always + // sees storage ready without that mount adding to the user-visible boot time. + signal_ready_to_host(); + boot_log( + "INFO", + &format!("boot ready_sent uptime_ms={}", uptime_ms()), + ); + + // Mount storage after signaling ready. The accept loop has not started yet, + // so no request can arrive before this completes. The OnceLock in + // ensure_storage_mounted() also guards any concurrent call from a request + // that races in the brief window on very fast hosts. + ensure_storage_mounted(); + + // Start accepting connections (listener already bound) + if let Err(e) = run_server_with_listener(listener) { + error!(error = %e, "server error"); + std::process::exit(1); + } +} + +/// Signal to the host that the agent is fully initialized and ready. +/// +/// Writes a marker file to the virtiofs rootfs. The host polls for this file. +/// The virtiofs FUSE write is visible on the host filesystem within ~1ms +/// (hv_gic_set_spi interrupt injection is 0–15µs; no event_manager involvement). +fn signal_ready_to_host() { + use std::path::Path; + + let content = uptime_ms().to_string(); + + // The host gives each VM its own marker name (SMOLVM_READY_MARKER) so + // concurrent boots don't race on one shared rootfs file and, under uid + // isolation, the host can pre-create it owned by this VM's uid. Fall back to + // the shared protocol constant if the host didn't set it (older host). + let marker = + std::env::var(guest_env::READY_MARKER).unwrap_or_else(|_| AGENT_READY_MARKER.to_string()); + + // Try /oldroot first (overlay mode: virtiofs is the lower layer after pivot_root) + // Before pivot_root: virtiofs is at /, so the / path works. + let paths = [format!("/oldroot/{}", marker), format!("/{}", marker)]; + + for path in &paths { + if Path::new(path).parent().map_or(false, |p| p.exists()) { + // Write + fsync so the host sees the marker immediately. A plain + // write() can sit in the guest's virtiofs writeback cache for + // seconds before the host fs backend observes it, leaving the host's + // marker poll empty until the socket-probe grace expires — a + // multi-second boot-time regression. fsync forces the FUSE write + // through to the host now. + use std::io::Write as _; + if let Ok(mut f) = std::fs::File::create(path) { + if f.write_all(content.as_bytes()).is_ok() && f.sync_all().is_ok() { + boot_log("INFO", &format!("ready marker written: {path}")); + return; + } + } + } + } +} + +/// Helper to create a CString from a static str. +/// Used by boot functions that call libc mount/mknod/pivot_root. +#[cfg(target_os = "linux")] +fn cstr(s: &str) -> std::ffi::CString { + std::ffi::CString::new(s).expect("static string without null bytes") +} + +/// A single mount entry for `mount_essential_filesystems`. +#[cfg(target_os = "linux")] +struct MountEntry { + source: &'static str, + target: &'static str, + fstype: &'static str, + flags: libc::c_ulong, + data: Option<&'static str>, +} + +#[cfg(target_os = "linux")] +impl MountEntry { + fn mount(&self) -> Result<(), String> { + if let Err(e) = std::fs::create_dir_all(self.target) { + // Clean up any partial directories left by create_dir_all + let _ = std::fs::remove_dir(self.target); + return Err(format!("failed to create {}: {}", self.target, e)); + } + + // Bind the optional data CString so it lives through the libc::mount call. + let data_cstr = self.data.map(cstr); + let data_ptr = match &data_cstr { + Some(d) => d.as_ptr() as *const libc::c_void, + None => std::ptr::null(), + }; + + // SAFETY: libc::mount with valid CString pointers for filesystem mounting. + // All CString values (from cstr() calls and data_cstr) are alive for the + // duration of this call. + let ret = unsafe { + libc::mount( + cstr(self.source).as_ptr(), + cstr(self.target).as_ptr(), + cstr(self.fstype).as_ptr(), + self.flags, + data_ptr, + ) + }; + + if ret != 0 { + return Err(format!( + "failed to mount {} at {}: {}", + self.fstype, + self.target, + std::io::Error::last_os_error() + )); + } + + Ok(()) + } +} + +/// Mount essential filesystems (proc, sysfs, devtmpfs, devpts). +/// This must be done first when running as init (PID 1). +/// Uses direct syscalls to avoid any overhead. +#[cfg(target_os = "linux")] +fn mount_essential_filesystems() { + // The guest root filesystem is read-only, and neither libkrun's init.c nor + // the kernel mounts /tmp. A writable /tmp is the idiomatic home for + // ephemeral runtime files (the registry-auth config, crun console sockets, + // the ssh-agent socket, …), so mount a tmpfs over it. Best-effort: a + // failure here must not abort boot, and the individual writers that target + // /storage still work without it. Runs unconditionally — before the + // init.c-already-mounted early return below — because init.c never mounts + // /tmp. + let tmp = MountEntry { + source: "tmpfs", + target: "/tmp", + fstype: "tmpfs", + flags: libc::MS_NOSUID | libc::MS_NODEV, + data: Some("mode=1777"), + }; + if let Err(e) = tmp.mount() { + warn!("smolvm-agent: failed to mount tmpfs on /tmp: {}", e); + } + + // libkrun's init.c mounts /proc, /sys, /dev, /dev/pts before exec'ing + // the agent. Skip redundant mounts if already present. + if std::path::Path::new("/proc/uptime").exists() { + // Ensure /dev/ptmx symlink exists (not set up by init.c) + let _ = std::os::unix::fs::symlink("pts/ptmx", "/dev/ptmx"); + return; + } + + let mounts = [ + MountEntry { + source: "proc", + target: "/proc", + fstype: "proc", + flags: 0, + data: None, + }, + MountEntry { + source: "sysfs", + target: "/sys", + fstype: "sysfs", + flags: 0, + data: None, + }, + MountEntry { + source: "devtmpfs", + target: "/dev", + fstype: "devtmpfs", + flags: 0, + data: None, + }, + MountEntry { + source: "devpts", + target: "/dev/pts", + fstype: "devpts", + flags: 0, + data: Some("mode=0620,ptmxmode=0666"), + }, + ]; + + for entry in &mounts { + if let Err(e) = entry.mount() { + error!("smolvm-agent: {}", e); + return; + } + } + + // Create /dev/ptmx symlink pointing to pts/ptmx + // This ensures openpty() can find the PTY multiplexer + let _ = std::os::unix::fs::symlink("pts/ptmx", "/dev/ptmx"); + + // Set up loopback interface (non-blocking, best effort) + unsafe { + let fd = libc::socket(libc::AF_INET, libc::SOCK_DGRAM, 0); + if fd >= 0 { + // This would require more complex ioctl calls, skip for now + // The networking will be set up by TSI anyway + libc::close(fd); + } + } +} + +/// Stub for non-Linux platforms (agent only runs on Linux inside VM). +#[cfg(not(target_os = "linux"))] +fn mount_essential_filesystems() { + // No-op on non-Linux platforms +} + +/// Create /dev/dri device nodes for virtio-gpu if present. +/// +/// libkrun's init.c mounts /dev as a basic tmpfs so the kernel's devtmpfs +/// doesn't auto-populate DRM device nodes. This function reads each render +/// node and card from /sys/class/drm/ and creates the corresponding +/// character device node in /dev/dri/ so containers can access the GPU. +#[cfg(target_os = "linux")] +fn setup_gpu_dev_nodes() { + let sysfs_drm = std::path::Path::new("/sys/class/drm"); + if !sysfs_drm.exists() { + return; + } + + // The virtio-gpu driver may not have finished probing when the agent + // starts — sysfs entries appear only after probe() completes. Poll + // for up to ~500 ms in 10 ms increments rather than racing with the + // driver. On fast boots the first read already wins; the loop just + // handles the small window where the driver is still initialising. + let entries = { + let mut found = Vec::new(); + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(500); + loop { + let Ok(rd) = std::fs::read_dir(sysfs_drm) else { + break; + }; + let candidates: Vec<_> = rd + .flatten() + .filter(|e| { + let n = e.file_name(); + let s = n.to_string_lossy(); + s.starts_with("renderD") || s.starts_with("card") + }) + .collect(); + if !candidates.is_empty() { + found = candidates; + break; + } + if std::time::Instant::now() >= deadline { + break; + } + // SAFETY: nanosleep — always safe + unsafe { + libc::nanosleep( + &libc::timespec { + tv_sec: 0, + tv_nsec: 10_000_000, + }, + std::ptr::null_mut(), + ); + } + } + found + }; + + if entries.is_empty() { + return; + } + + let _ = std::fs::create_dir_all("/dev/dri"); + + for entry in &entries { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + // Read major:minor from sysfs (e.g. "226:128\n") + let dev_file = sysfs_drm.join(&*name_str).join("dev"); + let Ok(dev_str) = std::fs::read_to_string(&dev_file) else { + continue; + }; + let parts: Vec<&str> = dev_str.trim().split(':').collect(); + if parts.len() != 2 { + continue; + } + let Ok(major) = parts[0].parse::() else { + continue; + }; + let Ok(minor) = parts[1].parse::() else { + continue; + }; + + let node_path = format!("/dev/dri/{}", name_str); + let Ok(node_cstr) = std::ffi::CString::new(node_path) else { + continue; + }; + + // SAFETY: mknod a character device node with the DRM major:minor from sysfs + unsafe { + libc::mknod( + node_cstr.as_ptr(), + libc::S_IFCHR | 0o666, + libc::makedev(major, minor), + ); + } + } +} + +/// Start the seatd seat manager daemon. +/// +/// seatd provides the seat management API that Wayland compositors (Weston, sway) +/// and DRI3-capable X servers (Xwayland) need to access /dev/dri devices. +/// Without it, GPU-accelerated display workloads fail with: +/// "No backend was able to open a seat" +/// +/// seatd runs as a background daemon on a Unix socket at /run/seatd.sock. +/// It's only started when GPU is enabled — zero overhead otherwise. +#[cfg(target_os = "linux")] +fn start_seatd() { + use std::process::{Command, Stdio}; + + let seatd_path = "/usr/bin/seatd"; + if !std::path::Path::new(seatd_path).exists() { + boot_log( + "WARN", + "seatd not found in rootfs, GPU display workloads may fail", + ); + return; + } + + // seatd needs /run for its socket + let _ = std::fs::create_dir_all("/run"); + + match Command::new(seatd_path) + .args(["-g", "root"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + { + Ok(child) => { + boot_log("INFO", &format!("seatd started (PID {})", child.id())); + // Detach — seatd runs for the lifetime of the VM + std::mem::forget(child); + } + Err(e) => { + boot_log("WARN", &format!("failed to start seatd: {}", e)); + } + } +} + +/// Log whether the GPU is accessible in the guest. +/// +/// Called at startup when `guest_env::GPU` is set. Checks whether the virtio-gpu +/// render node appeared in `/dev/dri/` after `setup_gpu_dev_nodes` ran. If not, +/// the kernel was built without `virtio-gpu` or `drm` support and the workload +/// will fail at Vulkan/EGL init rather than here — log a clear warning so the +/// user knows to check the kernel config. +#[cfg(target_os = "linux")] +fn log_gpu_status() { + let render_node = std::path::Path::new("/dev/dri/renderD128"); + if render_node.exists() { + info!("GPU: virtio-gpu render node present at /dev/dri/renderD128"); + } else { + warn!( + "GPU: requested but /dev/dri/renderD128 not found — \ + the guest kernel may lack virtio-gpu/drm support; \ + Vulkan/EGL workloads will fail" + ); + } +} + +/// Ensure a mount-point directory exists on the agent rootfs, logging a WARN +/// (rather than failing) if it can't be created. +/// +/// The boot-critical mount points (/mnt/{overlay,storage,newroot}) are baked +/// into the rootfs at build time — see the `mkdir` block in +/// scripts/build-agent-rootfs.sh. This runtime `create_dir_all` is a backstop +/// for rootfs images that predate that change or are assembled differently. +/// +/// On a writable rootfs the dir already exists (no-op) or is created here. On a +/// read-only rootfs the mkdir fails; we surface that as a WARN so the resulting +/// failed overlay/storage mount is diagnosable instead of presenting as a +/// silent boot-without-persistence. It also flags drift: a newly added `/mnt` +/// mount point that wasn't baked into the build script will WARN here on a RO +/// rootfs rather than fail mysteriously. +#[cfg(target_os = "linux")] +fn ensure_mount_dir(path: &str) { + if let Err(e) = std::fs::create_dir_all(path) { + boot_log( + "WARN", + &format!( + "could not create mount point {} ({}); rootfs may be read-only and \ + missing baked-in mount dirs — the following mount will likely fail", + path, e + ), + ); + } +} + +/// Set up persistent rootfs overlay using overlayfs on /dev/vdb. +/// +/// If /dev/vdb exists (overlay disk attached by host), this function: +/// 1. Mounts /dev/vdb as ext4 (formats on first boot) +/// 2. Creates overlayfs with initramfs as lower layer, /dev/vdb as upper +/// 3. Moves /proc, /sys, /dev into the new root +/// 4. Calls pivot_root to switch to the overlayfs root +/// +/// After pivot_root, the old initramfs stays at /oldroot (needed as +/// overlay lower layer). All subsequent writes go through overlayfs +/// and are persisted to /dev/vdb. +/// +/// If /dev/vdb doesn't exist, this is a no-op (backward compatible). +#[cfg(target_os = "linux")] +fn setup_persistent_rootfs() { + use std::path::Path; + + const OVERLAY_DEVICE: &str = "/dev/vdb"; + const OVERLAY_MOUNT: &str = "/mnt/overlay"; + const STORAGE_DEVICE: &str = "/dev/vda"; + const STORAGE_TEMP_MOUNT: &str = "/mnt/storage"; + const NEWROOT: &str = "/mnt/newroot"; + + // pivot_root requires that the current root mount is NOT shared. + // The kernel mounts virtiofs as shared:1 by default. We must make it + // private before pivot_root will accept it. + // + // Strategy: + // 1. Try MS_PRIVATE|MS_REC directly (works if CAP_SYS_ADMIN + not locked). + // 2. If that fails, unshare the mount namespace first (creates a private + // copy of the namespace for this process), then retry. + let root = cstr("/"); + // SAFETY: mount with MS_PRIVATE|MS_REC on root — no fstype needed + let ms_private_ret = unsafe { + libc::mount( + std::ptr::null(), + root.as_ptr(), + std::ptr::null(), + libc::MS_PRIVATE | libc::MS_REC, + std::ptr::null(), + ) + }; + if ms_private_ret != 0 { + boot_log( + "WARN", + &format!( + "MS_PRIVATE on / failed ({}), trying unshare", + std::io::Error::last_os_error() + ), + ); + // SAFETY: unshare mount namespace so we get a private copy + let unshare_ret = unsafe { libc::unshare(libc::CLONE_NEWNS) }; + if unshare_ret != 0 { + boot_log( + "WARN", + &format!( + "unshare(CLONE_NEWNS) failed ({}), pivot_root may fail", + std::io::Error::last_os_error() + ), + ); + } + // Retry MS_PRIVATE after unshare (now we own the namespace) + unsafe { + libc::mount( + std::ptr::null(), + root.as_ptr(), + std::ptr::null(), + libc::MS_PRIVATE | libc::MS_REC, + std::ptr::null(), + ); + } + } + + // If overlay device doesn't exist, no overlay disk attached — skip. + // On devtmpfs, the kernel creates /dev/vdb automatically when libkrun + // attaches a second virtio-blk disk. No mknod needed. + if !Path::new(OVERLAY_DEVICE).exists() { + return; + } + + ensure_mount_dir(OVERLAY_MOUNT); + + // Probe and mount storage disk in parallel with the overlay ext4 operations. + // Spawning here (before the /dev/vdb ext4 mount) lets storage work overlap + // with both the virtiofs activity between probe and mount (~27ms gap) and + // the /dev/vdb ext4 mount itself (~10ms, req=4–17). On warm boots this + // removes storage mount latency from the critical path entirely. + let storage_handle = if Path::new(STORAGE_DEVICE).exists() { + ensure_mount_dir(STORAGE_TEMP_MOUNT); + Some(std::thread::spawn(|| { + // Resize before mount — template may be smaller than device. + // Skip if filesystem already fills the device (subsequent boots). + // If resize fails (e.g. macOS-created template with incompatible features), + // skip mount — mount_storage_disk() will handle mkfs fallback. + if !ext4_already_full_size(STORAGE_DEVICE) + && !resize_ext4_if_needed(STORAGE_DEVICE, "storage") + { + boot_log( + "WARN", + "storage: resize failed, deferring to mount_storage_disk", + ); + return false; + } + + let dev = cstr(STORAGE_DEVICE); + let mnt = cstr(STORAGE_TEMP_MOUNT); + let ext4 = cstr("ext4"); + // SAFETY: mount /dev/vda as ext4 at /mnt/storage with noatime + let mounted = unsafe { + libc::mount( + dev.as_ptr(), + mnt.as_ptr(), + ext4.as_ptr(), + libc::MS_NOATIME, + std::ptr::null(), + ) == 0 + }; + if !mounted { + let err = std::io::Error::last_os_error(); + boot_log( + "WARN", + &format!( + "storage: parallel mount failed ({}), deferring to mount_storage_disk", + err + ), + ); + } + mounted + })) + } else { + None + }; + + // Resize ext4 on the UNMOUNTED device before mounting. The host copies + // from a small template (~512MB) then extends the sparse file. resize2fs + // on a mounted device fails with "Resource busy" — must resize first. + // Skip if filesystem already fills the device (subsequent boots). + // If resize fails (macOS-created template), the mount+mkfs fallback below handles it. + if !ext4_already_full_size(OVERLAY_DEVICE) { + let _ = resize_ext4_if_needed(OVERLAY_DEVICE, "overlay"); + } + + // Try to mount overlay disk (should be pre-formatted ext4) + let dev = cstr(OVERLAY_DEVICE); + let mnt = cstr(OVERLAY_MOUNT); + let ext4 = cstr("ext4"); + // SAFETY: mount /dev/vdb as ext4 at /mnt/overlay with noatime + let mounted = unsafe { + libc::mount( + dev.as_ptr(), + mnt.as_ptr(), + ext4.as_ptr(), + libc::MS_NOATIME, + std::ptr::null(), + ) == 0 + }; + + if !mounted { + // First boot — format the disk + let _ = std::process::Command::new("mkfs.ext4") + .args([ + "-F", + "-q", + "-O", + "^has_journal", + "-L", + "smolvm-overlay", + OVERLAY_DEVICE, + ]) + .status(); + + let dev = cstr(OVERLAY_DEVICE); + let mnt = cstr(OVERLAY_MOUNT); + let ext4 = cstr("ext4"); + // SAFETY: retry mount after formatting with noatime + if unsafe { + libc::mount( + dev.as_ptr(), + mnt.as_ptr(), + ext4.as_ptr(), + libc::MS_NOATIME, + std::ptr::null(), + ) + } != 0 + { + boot_log("ERROR", "failed to mount overlay disk after formatting"); + // Clean up the parallel storage mount so mount_storage_disk() fallback + // does not hit EBUSY from an already-mounted /dev/vda. + if let Some(handle) = storage_handle { + if handle.join().unwrap_or(false) { + let mnt = cstr(STORAGE_TEMP_MOUNT); + // SAFETY: umount /mnt/storage that the parallel thread mounted + unsafe { + libc::umount(mnt.as_ptr()); + } + } + } + return; + } + } + + // Create overlay directories + let _ = std::fs::create_dir_all(format!("{}/upper", OVERLAY_MOUNT)); + let _ = std::fs::create_dir_all(format!("{}/work", OVERLAY_MOUNT)); + ensure_mount_dir(NEWROOT); + + // Mount overlayfs: initramfs (lower, read-only) + persistent disk (upper) + let overlay_src = cstr("overlay"); + let newroot = cstr(NEWROOT); + let overlay_type = cstr("overlay"); + // Simple overlayfs without index/redirect_dir/uuid — these options trigger + // ext4 xattr writes + journal flushes on every mount. On macOS/HVF each + // ext4 journal flush (FUA/barrier) serializes to an APFS fsync (~10-30ms). + // With 15-30 flushes per mount, they add 200-900ms to every boot. + // + // These options also do NOT enable Docker overlay2 on this overlay — the + // lower layer is the initramfs (ramfs) which has no file-handle support, + // so the kernel falls back to index=off for any overlay2 upper dir on this + // root regardless. Docker's data root is on /storage/ (bare ext4), not here. + let overlay_opts = cstr(&format!( + "lowerdir=/,upperdir={}/upper,workdir={}/work", + OVERLAY_MOUNT, OVERLAY_MOUNT + )); + // SAFETY: mount overlayfs with the specified options + let result = unsafe { + libc::mount( + overlay_src.as_ptr(), + newroot.as_ptr(), + overlay_type.as_ptr(), + 0, + overlay_opts.as_ptr() as *const libc::c_void, + ) + }; + if result != 0 { + let err = std::io::Error::last_os_error(); + boot_log("ERROR", &format!("failed to mount overlayfs ({})", err)); + // Clean up parallel storage mount to avoid double-mount in + // mount_storage_disk() fallback path. + if let Some(handle) = storage_handle { + if handle.join().unwrap_or(false) { + let mnt = cstr(STORAGE_TEMP_MOUNT); + // SAFETY: umount the temp storage mount + unsafe { + libc::umount(mnt.as_ptr()); + } + } + } + return; + } + + // pivot_root also requires the new root mount not to be shared. + // Make the overlayfs mount private immediately after creating it. + // SAFETY: mount(NULL, newroot, NULL, MS_PRIVATE|MS_REC, NULL) + unsafe { + libc::mount( + std::ptr::null(), + newroot.as_ptr(), + std::ptr::null(), + libc::MS_PRIVATE | libc::MS_REC, + std::ptr::null(), + ); + } + + // Create mount point directories in new root and move special mounts + for dir in &["proc", "sys", "dev"] { + let _ = std::fs::create_dir_all(format!("{}/{}", NEWROOT, dir)); + let src = cstr(&format!("/{}", dir)); + let dst = cstr(&format!("{}/{}", NEWROOT, dir)); + // SAFETY: mount --move for each special filesystem + unsafe { + libc::mount( + src.as_ptr(), + dst.as_ptr(), + std::ptr::null(), + libc::MS_MOVE, + std::ptr::null(), + ); + } + } + + // Join parallel storage mount and move it into new root. + // On subsequent boots, the ext4 mount succeeds and overlaps with the + // overlayfs setup above. On first boot from macOS template, mount fails + // and mount_storage_disk() handles it with full fsck/mkfs recovery. + if let Some(handle) = storage_handle { + match handle.join() { + Ok(true) => { + let _ = std::fs::create_dir_all(format!("{}/storage", NEWROOT)); + let src = cstr(STORAGE_TEMP_MOUNT); + let dst = cstr(&format!("{}/storage", NEWROOT)); + // SAFETY: mount --move /mnt/storage to newroot/storage + let result = unsafe { + libc::mount( + src.as_ptr(), + dst.as_ptr(), + std::ptr::null(), + libc::MS_MOVE, + std::ptr::null(), + ) + }; + if result != 0 { + let err = std::io::Error::last_os_error(); + boot_log( + "WARN", + &format!( + "storage: mount-move to newroot failed ({}), will retry after pivot_root", + err + ), + ); + // Unmount temp so mount_storage_disk() can try fresh + let mnt = cstr(STORAGE_TEMP_MOUNT); + unsafe { + libc::umount(mnt.as_ptr()); + } + } + } + Ok(false) => { + // Thread reported failure — mount_storage_disk() will handle it + } + Err(_) => { + boot_log("WARN", "storage: parallel mount thread panicked"); + } + } + } + + // Prepare for pivot_root + let _ = std::fs::create_dir_all(format!("{}/oldroot", NEWROOT)); + + if std::env::set_current_dir(NEWROOT).is_err() { + boot_log("ERROR", "failed to chdir to new root"); + return; + } + + // pivot_root — switch to overlayed root. + // Old root stays at /oldroot (needed as overlay lower layer, ~44MB RAM). + let dot = cstr("."); + let oldroot = cstr("oldroot"); + // SAFETY: pivot_root syscall with valid path arguments + let result = unsafe { libc::syscall(libc::SYS_pivot_root, dot.as_ptr(), oldroot.as_ptr()) }; + if result != 0 { + let err = std::io::Error::last_os_error(); + boot_log("ERROR", &format!("pivot_root failed: {}", err)); + return; + } + + // Set working directory to new root + let _ = std::env::set_current_dir("/"); + + boot_log("INFO", "persistent rootfs overlay active (pivot_root done)"); +} + +/// Stub for non-Linux platforms. +#[cfg(not(target_os = "linux"))] +fn setup_persistent_rootfs() { + // No-op on non-Linux platforms +} + +/// Sync filesystem caches before shutdown. +/// This prevents ext4 corruption when the VM is terminated. +#[cfg(target_os = "linux")] +fn sync_and_unmount_storage() { + info!("syncing filesystems before shutdown"); + + // Sync all filesystem caches to disk + // SAFETY: sync() is always safe to call + unsafe { + libc::sync(); + } + + // Note: We don't unmount /storage here because: + // 1. The overlay filesystem uses /storage/layers and /storage/overlays + // 2. Unmounting /storage while overlay is active causes issues + // 3. The sync() call ensures all pending writes are flushed to disk + // 4. When the VM terminates, the kernel will clean up mounts +} + +/// Stub for non-Linux platforms. +#[cfg(not(target_os = "linux"))] +fn sync_and_unmount_storage() { + // No-op on non-Linux platforms +} + +/// Set up signal handlers to sync filesystem on SIGTERM/SIGINT. +/// This prevents ext4 corruption when the VM is forcefully stopped. +#[cfg(target_os = "linux")] +fn setup_signal_handlers() { + // SAFETY: Signal handler that calls sync() - sync is async-signal-safe + unsafe extern "C" fn handle_term_signal(_sig: libc::c_int) { + // sync() is async-signal-safe, so we can call it from a signal handler + libc::sync(); + // Exit cleanly + libc::_exit(0); + } + + // SAFETY: Setting up signal handlers with valid function pointers + unsafe { + // Handle SIGTERM (sent by VM stop) + libc::signal( + libc::SIGTERM, + handle_term_signal as *const () as libc::sighandler_t, + ); + // Handle SIGINT (Ctrl+C, if attached to console) + libc::signal( + libc::SIGINT, + handle_term_signal as *const () as libc::sighandler_t, + ); + // Note: We do NOT install a SIGCHLD handler here because it would + // race with Child::wait() in synchronous exec paths. Instead, + // background exec children are reaped by reap_background_children() + // called periodically in the accept loop. + } +} + +/// Stub for non-Linux platforms. +#[cfg(not(target_os = "linux"))] +fn setup_signal_handlers() { + // No-op on non-Linux platforms +} + +#[cfg(target_os = "linux")] +/// Resize an ext4 filesystem on an unmounted device to fill the block device. +/// +/// The host creates disks by copying a small pre-formatted template (~512MB) +/// then extending the sparse file to the target size (e.g. 20GB). The ext4 +/// filesystem inside still thinks it's 512MB. This function expands it to +/// fill the full block device. +/// +/// MUST be called BEFORE mounting — resize2fs on a mounted device fails with +/// "Resource busy" because the kernel holds the block device exclusively. +/// +/// Tries resize2fs directly first. Only falls back to e2fsck if resize2fs +/// fails (e.g., due to actual corruption). ext4 journal replay handles +/// `needs_recovery` on mount in ~1-2ms, so a full e2fsck is unnecessary +/// on the happy path. Uses boot_log instead of tracing because this runs +/// before tracing_subscriber is initialized. +fn resize_ext4_if_needed(device: &str, label: &str) -> bool { + use std::process::Command; + + // Try resize2fs directly — skip e2fsck on the happy path. + // ext4 journal replay handles needs_recovery on mount, so resize2fs + // usually succeeds without a prior fsck. + match Command::new("resize2fs").arg(device).output() { + Ok(output) if output.status.success() => { + let msg = String::from_utf8_lossy(&output.stderr); + if msg.contains("Nothing to do") { + boot_log( + "DEBUG", + &format!("{} filesystem already at full device size", label), + ); + } else { + boot_log( + "INFO", + &format!("{} filesystem resized to fill block device", label), + ); + } + return true; + } + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + boot_log( + "WARN", + &format!( + "{} resize2fs failed (exit {}): {}, trying e2fsck", + label, + output.status.code().unwrap_or(-1), + stderr.trim() + ), + ); + } + Err(e) => { + boot_log("WARN", &format!("{} resize2fs not found: {}", label, e)); + return false; + } + } + + // Fallback: resize2fs failed, run e2fsck -y (without -f) then retry. + // Without -f, e2fsck skips clean filesystems instantly. With needs_recovery, + // it replays the journal (~10ms) instead of a full forced scan (~128ms). + match Command::new("e2fsck").args(["-y", device]).output() { + Ok(output) => { + let code = output.status.code().unwrap_or(-1); + // e2fsck exit codes (bit flags, may be OR'd together): + // 0 = clean + // 1 = errors corrected + // 2 = errors corrected, reboot needed (unsafe to proceed) + // 4 = errors left uncorrected + // 8 = operational error + if code >= 2 { + let stderr = String::from_utf8_lossy(&output.stderr); + boot_log( + "WARN", + &format!( + "{} e2fsck could not fully repair (exit {}): {}", + label, + code, + stderr.trim() + ), + ); + return false; + } + if code == 1 { + boot_log("INFO", &format!("{} e2fsck fixed errors", label)); + } + } + Err(e) => { + boot_log("WARN", &format!("{} e2fsck not found: {}", label, e)); + return false; + } + } + + // Retry resize2fs after e2fsck + match Command::new("resize2fs").arg(device).output() { + Ok(output) if output.status.success() => { + boot_log( + "INFO", + &format!("{} filesystem resized after e2fsck", label), + ); + true + } + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + boot_log( + "WARN", + &format!( + "{} resize2fs still failed after e2fsck (exit {}): {}", + label, + output.status.code().unwrap_or(-1), + stderr.trim() + ), + ); + false + } + Err(e) => { + boot_log("WARN", &format!("{} resize2fs failed: {}", label, e)); + false + } + } +} + +#[cfg(target_os = "linux")] +/// Check if ext4 filesystem already fills the block device. +/// +/// Reads the ext4 superblock (at offset 1024) to get block_count and block_size, +/// then compares against the device size. Returns true if the filesystem already +/// spans the full device, meaning resize2fs would be a no-op. This avoids the +/// ~5ms cost of spawning resize2fs on every subsequent boot. +/// +/// Returns false (conservative, triggers resize path) on any error: unformatted +/// device, non-ext4 filesystem, corrupt superblock, or I/O failure. +fn ext4_already_full_size(device: &str) -> bool { + use std::fs::File; + use std::io::{Read, Seek, SeekFrom}; + + let mut f = match File::open(device) { + Ok(f) => f, + Err(_) => return false, + }; + + // For block devices, metadata().len() returns 0. Use seek to find size. + let dev_size = match f.seek(SeekFrom::End(0)) { + Ok(s) if s > 0 => s, + _ => return false, + }; + + // ext4 superblock starts at byte offset 1024. We need: + // offset 4: s_blocks_count_lo (4 bytes) + // offset 24: s_log_block_size (4 bytes) + // offset 56: s_magic (2 bytes) — must be 0xEF53 + let mut sb = [0u8; 64]; + if f.seek(SeekFrom::Start(1024)).is_err() || f.read_exact(&mut sb).is_err() { + return false; + } + + // Validate ext4 magic number before trusting any fields. + let magic = u16::from_le_bytes([sb[56], sb[57]]); + if magic != 0xEF53 { + return false; + } + + let log_block_size = u32::from_le_bytes([sb[24], sb[25], sb[26], sb[27]]); + // Sanity check: log_block_size > 6 means block_size > 64 MB, not valid ext4. + if log_block_size > 6 { + return false; + } + let block_size: u64 = 1024u64 << log_block_size; + + let blocks_lo = u32::from_le_bytes([sb[4], sb[5], sb[6], sb[7]]) as u64; + let fs_size = blocks_lo * block_size; + + // Allow 1 block of slack — filesystem may not use the very last block. + // Note: only uses s_blocks_count_lo (sufficient for disks up to 16 TB at 4K blocks). + fs_size + block_size >= dev_size +} + +#[cfg(target_os = "linux")] +/// Check /proc/mounts to see if anything is mounted at the given path. +fn is_mounted_at(mount_point: &str) -> bool { + if let Ok(mounts) = std::fs::read_to_string("/proc/mounts") { + return mounts + .lines() + .any(|line| line.split_whitespace().nth(1) == Some(mount_point)); + } + false +} + +#[cfg(target_os = "linux")] +/// Create required subdirectories under the storage mount point. +fn create_storage_dirs(mount_point: &str) { + let dirs = [ + "layers", + "configs", + "manifests", + "overlays", + "workspace", + "containers/run", + "containers/logs", + "containers/exit", + "containers/crun", + ]; + for dir in dirs { + let _ = std::fs::create_dir_all(std::path::Path::new(mount_point).join(dir)); + } +} + +/// Mount ext4 /dev/vda at /storage using direct syscall (avoids ~3-5ms fork+exec). +#[cfg(target_os = "linux")] +fn try_mount_storage_ext4() -> bool { + let dev = cstr("/dev/vda"); + let mnt = cstr("/storage"); + let ext4 = cstr("ext4"); + // SAFETY: mount /dev/vda as ext4 at /storage with noatime + unsafe { + libc::mount( + dev.as_ptr(), + mnt.as_ptr(), + ext4.as_ptr(), + libc::MS_NOATIME, + std::ptr::null(), + ) == 0 + } +} + +/// Mount the storage disk at /storage. Returns true if successfully mounted. +/// +/// Three-attempt fallback chain: +/// 1. resize + mount (works on subsequent boots with Linux-native FS) +/// 2. fsck + resize + mount (may fix minor corruption) +/// 3. mkfs + mount (first boot from macOS template, or unrecoverable) +#[cfg(target_os = "linux")] +fn mount_storage_disk() -> bool { + use std::process::Command; + + const STORAGE_DEVICE: &str = "/dev/vda"; + const STORAGE_MOUNT: &str = "/storage"; + + // Create mount point if needed + let _ = std::fs::create_dir_all(STORAGE_MOUNT); + + // Check if device exists + if !std::path::Path::new(STORAGE_DEVICE).exists() { + let dev_path = cstr(STORAGE_DEVICE); + // SAFETY: mknod with block device type, major 253 minor 0 + unsafe { + libc::mknod( + dev_path.as_ptr(), + libc::S_IFBLK | 0o660, + libc::makedev(253, 0), + ); + } + } + + // Check if already mounted (pre-mounted during setup_persistent_rootfs) + if is_mounted_at(STORAGE_MOUNT) { + debug!("storage already mounted at /storage"); + create_storage_dirs(STORAGE_MOUNT); + return true; + } + + // --- Attempt 1: resize (if needed) + mount (works on subsequent boots) --- + let resized = + ext4_already_full_size(STORAGE_DEVICE) || resize_ext4_if_needed(STORAGE_DEVICE, "storage"); + if resized && try_mount_storage_ext4() { + info!("storage disk mounted after resize"); + create_storage_dirs(STORAGE_MOUNT); + return true; + } + + // --- Attempt 2: fsck + resize + mount --- + if resized { + warn!("mount failed after resize, attempting fsck repair"); + } else { + warn!("resize failed, attempting fsck repair before mount"); + } + + let fsck_ok = match Command::new("fsck.ext4") + .args(["-y", "-f", STORAGE_DEVICE]) + .status() + { + Ok(status) => { + let code = status.code().unwrap_or(-1); + if code <= 1 { + info!(exit_code = code, "fsck completed"); + true + } else { + warn!(exit_code = code, "fsck could not fully repair filesystem"); + false + } + } + Err(e) => { + warn!(error = %e, "fsck.ext4 not available"); + false + } + }; + + if fsck_ok { + let _ = resize_ext4_if_needed(STORAGE_DEVICE, "storage"); + if try_mount_storage_ext4() { + info!("storage disk mounted after fsck repair"); + create_storage_dirs(STORAGE_MOUNT); + return true; + } + warn!("mount still failed after fsck, will format"); + } + + // --- Attempt 3: mkfs (last resort, destroys data) --- + info!("formatting storage disk (first boot or unrecoverable)"); + match Command::new("mkfs.ext4") + .args(["-F", "-q", "-O", "^has_journal", STORAGE_DEVICE]) + .status() + { + Ok(status) if status.success() => {} + Ok(status) => { + error!(exit_code = status.code().unwrap_or(-1), "mkfs.ext4 failed"); + return false; + } + Err(e) => { + error!(error = %e, "mkfs.ext4 not available"); + return false; + } + } + + if try_mount_storage_ext4() { + info!("storage disk mounted after format"); + create_storage_dirs(STORAGE_MOUNT); + return true; + } + + error!("CRITICAL: could not mount storage disk after all recovery attempts"); + false +} + +/// Stub for non-Linux platforms. +#[cfg(not(target_os = "linux"))] +fn mount_storage_disk() -> bool { + false +} + +/// Maximum number of vsock connections serviced concurrently. +/// Bounds thread count and prevents vsock channel saturation. +/// 8 (2^3) gives headroom for several parallel execs plus the state probe. +const MAX_CONCURRENT_CONNECTIONS: usize = 8; + +/// A counting semaphore for bounding concurrent connection handlers. +struct ConnectionSemaphore { + inner: std::sync::Arc<(std::sync::Mutex, std::sync::Condvar)>, +} + +impl ConnectionSemaphore { + fn new(n: usize) -> Self { + Self { + inner: std::sync::Arc::new((std::sync::Mutex::new(n), std::sync::Condvar::new())), + } + } + + fn acquire(&self) -> ConnectionPermit { + let (lock, cvar) = &*self.inner; + let mut count = lock.lock().unwrap(); + while *count == 0 { + count = cvar.wait(count).unwrap(); + } + *count -= 1; + ConnectionPermit { + inner: self.inner.clone(), + } + } +} + +/// RAII guard that releases a semaphore slot when dropped. +struct ConnectionPermit { + inner: std::sync::Arc<(std::sync::Mutex, std::sync::Condvar)>, +} + +impl Drop for ConnectionPermit { + fn drop(&mut self) { + let (lock, cvar) = &*self.inner; + *lock.lock().unwrap() += 1; + cvar.notify_one(); + } +} + +/// Run the vsock server with a pre-created listener. +/// The listener is created early (before initialization) to ensure the kernel +/// has a listener ready when the host connects. +fn run_server_with_listener( + listener: vsock::VsockListener, +) -> Result<(), Box> { + let mut first_connection = true; + let listen_start = uptime_ms(); + let semaphore = std::sync::Arc::new(ConnectionSemaphore::new(MAX_CONCURRENT_CONNECTIONS)); + + info!(uptime_ms = uptime_ms(), "entering vsock accept loop"); + + loop { + // Reap any exited background children to prevent zombie accumulation. + reap_background_children(); + + match listener.accept() { + Ok(mut stream) => { + if first_connection { + info!( + wait_for_first_connection_ms = uptime_ms() - listen_start, + uptime_ms = uptime_ms(), + "first connection accepted" + ); + first_connection = false; + } + info!("accepted connection"); + + // Acquire a slot before spawning; blocks if MAX_CONCURRENT_CONNECTIONS + // threads are already running. This bounds thread count and prevents + // vsock channel saturation under heavy concurrency. + let permit = semaphore.acquire(); + + // Service the connection in its own thread so a long-running + // exec doesn't block subsequent requests on the same listener. + // Before this landed, a single held-open `machine exec` would + // stall the 250ms state probe, flip the VM to Unreachable, + // and make every following exec fail with "not running". + std::thread::spawn(move || { + let _permit = permit; + if let Err(e) = handle_connection(&mut stream) { + warn!(error = %e, "connection error"); + } + }); + } + Err(e) => { + warn!(error = %e, "accept error"); + } + } + } +} + +/// Patch `timeout_ms` on request variants where serde may have silently dropped +/// the field due to the flatten + internally-tagged enum limitation. +/// Re-extracts the value from the raw JSON payload. +fn patch_timeout_ms(request: &mut AgentRequest, raw: &[u8]) { + let needs_patch = matches!( + request, + AgentRequest::VmExec { + timeout_ms: None, + .. + } | AgentRequest::Run { + timeout_ms: None, + .. + } + ); + if !needs_patch { + return; + } + + if let Ok(map) = serde_json::from_slice::(raw) { + if let Some(ms) = map.get("timeout_ms").and_then(|v| v.as_u64()) { + match request { + AgentRequest::VmExec { timeout_ms, .. } | AgentRequest::Run { timeout_ms, .. } => { + *timeout_ms = Some(ms); + } + _ => {} + } + } + } +} + +/// Handle a single connection. +/// Replay host-originated filesystem changes as guest fsnotify events. +/// +/// Writes one `" \n"` line per event to `/proc/smolvm-fsnotify` +/// (provided by the libkrunfw kernel patch), which resolves the path and fires +/// the matching fsnotify event on the guest inode. Because the container's view +/// of a `-v` mount is a bind of the same virtiofs inode, a watcher inside the +/// container wakes up exactly as if the change had happened in the guest. +/// +/// Best-effort: a path that no longer resolves (e.g. a delete arriving after the +/// host watcher already saw the unlink) is skipped, not fatal. Returns the count +/// of events that fired. +fn handle_fsnotify(events: &[FsNotifyEvent]) -> AgentResponse { + const PROC_PATH: &str = "/proc/smolvm-fsnotify"; + + let mut file = match std::fs::OpenOptions::new().write(true).open(PROC_PATH) { + Ok(f) => f, + Err(e) => { + // Kernel without the patch (older libkrunfw): degrade gracefully so + // hosts on new + old kernels both work — the mount still serves + // reads, only event delivery is unavailable. + debug!(error = %e, "fsnotify inject unavailable ({PROC_PATH} missing)"); + return AgentResponse::ok_with_data( + serde_json::json!({ "fired": 0, "supported": false }), + ); + } + }; + + let mut fired = 0u32; + for ev in events { + // Each write() is one event; the kernel handler parses exactly one line. + let line = format!("{:x} {}\n", ev.mask, ev.path); + match file.write_all(line.as_bytes()) { + Ok(()) => fired += 1, + Err(e) => { + debug!(path = %ev.path, mask = ev.mask, error = %e, "fsnotify inject skipped") + } + } + } + + AgentResponse::ok_with_data(serde_json::json!({ "fired": fired, "supported": true })) +} + +fn handle_connection(stream: &mut impl ReadWrite) -> Result<(), Box> { + let mut buf = vec![0u8; REQUEST_BUFFER_SIZE]; + + // Per-connection streaming-upload session. `Option` + // guarantees cleanup via Drop when the connection closes, when + // the session is replaced, or when a protocol violation (e.g., + // another request type arriving mid-stream) takes it. + let mut write_session: Option = None; + + loop { + // Read length header + let mut header = [0u8; 4]; + match stream.read_exact(&mut header) { + Ok(_) => {} + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => { + debug!("connection closed"); + return Ok(()); + } + Err(e) => return Err(e.into()), + } + + let len = u32::from_be_bytes(header) as usize; + + // Validate message size to prevent DoS via memory exhaustion + if len > MAX_MESSAGE_SIZE { + warn!( + len = len, + max = MAX_MESSAGE_SIZE, + "message too large, rejecting" + ); + send_response( + stream, + &AgentResponse::error( + format!("message size {} exceeds maximum {}", len, MAX_MESSAGE_SIZE), + error_codes::MESSAGE_TOO_LARGE, + ), + )?; + continue; + } + + if len > buf.len() { + buf.resize(len, 0); + } + + // Read payload + stream.read_exact(&mut buf[..len])?; + + // Parse request (Envelope wraps the request with an optional trace_id). + // Falls back to bare AgentRequest for backward compatibility with old hosts. + let (mut request, trace_id) = + match serde_json::from_slice::>(&buf[..len]) { + Ok(env) => (env.body, env.trace_id), + Err(_) => match serde_json::from_slice::(&buf[..len]) { + Ok(req) => (req, None), + Err(e) => { + warn!(error = %e, "invalid request"); + send_response( + stream, + &AgentResponse::error( + format!("invalid request: {}", e), + error_codes::INVALID_REQUEST, + ), + )?; + continue; + } + }, + }; + + // Work around serde flatten + internally-tagged enum limitation: + // fields with #[serde(default)] can be silently dropped during + // deserialization. Re-extract timeout_ms from the raw JSON. + patch_timeout_ms(&mut request, &buf[..len]); + + let _span = if let Some(ref tid) = trace_id { + tracing::info_span!("request", trace_id = %tid, method = %request.log_summary()) + } else { + tracing::info_span!("request", method = %request.log_summary()) + }; + let _guard = _span.enter(); + + debug!(method = %request.log_summary(), "received request"); + + // Check if this is an interactive run request + if let AgentRequest::Run { + interactive: true, .. + } + | AgentRequest::Run { tty: true, .. } = &request + { + // Handle interactive session + handle_interactive_run(stream, request)?; + continue; + } + + // Detached run — start container in background and return its container ID. + if let AgentRequest::Run { detached: true, .. } = &request { + handle_run_detached(stream, request)?; + continue; + } + + // Check if this is an interactive VM exec request + if let AgentRequest::VmExec { + interactive: true, .. + } + | AgentRequest::VmExec { tty: true, .. } = &request + { + // Handle interactive VM exec session + handle_interactive_vm_exec(stream, request)?; + continue; + } + + // Handle Pull with progress streaming + if let AgentRequest::Pull { + ref image, + ref oci_platform, + ref auth, + ref proxy, + ref no_proxy, + } = request + { + handle_streaming_pull( + stream, + image, + oci_platform.as_deref(), + auth.as_ref(), + proxy.as_deref(), + no_proxy.as_deref(), + )?; + continue; + } + + // Handle ExportLayer with chunked streaming + if let AgentRequest::ExportLayer { + ref image_digest, + layer_index, + } = request + { + handle_streaming_export_layer(stream, image_digest, layer_index)?; + continue; + } + + // Handle FileRead with chunked streaming (replaces the old + // single-shot FileData path that capped files at ~16 MiB). + if let AgentRequest::FileRead { ref path } = request { + handle_streaming_file_read(stream, path)?; + continue; + } + + // Streaming file upload: Begin opens a session, Chunk appends + // or finalizes. Any other request type closes the session + // implicitly (Drop runs on the Option assignment to None). + if let AgentRequest::FileWriteBegin { + path, + mode, + total_size, + } = request + { + // Drop any leftover session now (Drop cleans its tmp file) before + // starting a new one. `take()` makes the drop explicit and keeps the + // value from looking like a dead store. + let _ = write_session.take(); + let (new_session, response) = handle_file_write_begin(path, mode, total_size); + write_session = new_session; + send_response(stream, &response)?; + continue; + } + if let AgentRequest::FileWriteChunk { data, done } = request { + let (new_session, response) = + handle_file_write_chunk(write_session.take(), &data, done); + write_session = new_session; + send_response(stream, &response)?; + continue; + } + + // Any other request mid-session is a protocol error. Drop the + // session (Drop cleans the staging file) and proceed — the + // operator's new request is honored rather than failed; the + // alternative (error out) buys no safety since the drop + // already handled cleanup. + if write_session.is_some() { + debug!( + method = %request.log_summary(), + "dropping in-flight FileWrite session: non-chunk request arrived" + ); + write_session = None; + } + + // Handle regular request. Pass the stream's fd so long-running + // commands (exec, run) can detect client disconnect and kill their + // children instead of blocking the accept loop. + let client_fd = stream.as_raw_fd(); + let response = handle_request(request, Some(client_fd)); + + // Check for client disconnection BEFORE writing — if the peer is gone, + // write_all may succeed (kernel buffers) but the next read_exact would + // block forever waiting for bytes that will never arrive. Close the + // connection now and let the accept loop pick up the next client. + if process::is_peer_closed(client_fd) { + debug!("client disconnected during request, closing connection"); + return Ok(()); + } + + if let Err(e) = send_response(stream, &response) { + debug!(error = %e, "send_response failed (client disconnected?)"); + return Ok(()); + } + + // Check for shutdown + if matches!(response, AgentResponse::Ok { .. }) { + if let AgentResponse::Ok { data: Some(ref d) } = response { + if d.get("shutdown").and_then(|v| v.as_bool()) == Some(true) { + info!("shutdown requested"); + return Ok(()); + } + } + } + } +} + +/// Handle a single non-interactive request. +/// +/// `client_fd` is the vsock file descriptor of the requesting client. It's +/// used by long-running handlers (Run, VmExec) to detect when the client +/// disconnects so the child process can be killed promptly — freeing the +/// accept loop for the next request instead of waiting for the orphan. +fn handle_request( + request: AgentRequest, + client_fd: Option, +) -> AgentResponse { + // Ensure storage is mounted for operations that need it. + // Ping, NetworkTest, VmExec, and Shutdown don't access /storage. + match &request { + AgentRequest::Ping + | AgentRequest::NetworkTest { .. } + | AgentRequest::VmExec { .. } + | AgentRequest::Shutdown => {} + _ => { + ensure_storage_mounted(); + } + } + + match request { + AgentRequest::Ping => AgentResponse::Pong { + version: PROTOCOL_VERSION, + }, + + AgentRequest::FsNotify { events } => handle_fsnotify(&events), + + // Pull is handled separately in handle_streaming_pull for progress streaming + AgentRequest::Pull { .. } => unreachable!("Pull handled before match"), + + AgentRequest::Query { image } => handle_query(&image), + + AgentRequest::ListImages => handle_list_images(), + + AgentRequest::GarbageCollect { dry_run, purge_all } => handle_gc(dry_run, purge_all), + + AgentRequest::PrepareOverlay { image, workload_id } => { + handle_prepare_overlay(&image, &workload_id) + } + + AgentRequest::CleanupOverlay { workload_id } => handle_cleanup_overlay(&workload_id), + + AgentRequest::FormatStorage => handle_format_storage(), + + AgentRequest::StorageStatus => handle_storage_status(), + + AgentRequest::NetworkTest { url } => { + info!(url = %url, "testing network connectivity directly from agent"); + + // Extract host:port for TCP test from URL + let tcp_target = extract_host_port(&url).unwrap_or_else(|| "1.1.1.1:80".to_string()); + + // Test 1: Pure syscall TCP connect test (bypass C library) + let syscall_result = test_tcp_syscall(&tcp_target); + + // Test 2: Try wget (busybox/musl) + let wget_result = match std::process::Command::new("wget") + .args(["-q", "-O-", "-T", "10", &url]) + .output() + { + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + serde_json::json!({ + "tool": "wget", + "success": output.status.success(), + "exit_code": output.status.code(), + "stdout_len": output.stdout.len(), + "stderr": stderr, + }) + } + Err(e) => serde_json::json!({ + "tool": "wget", + "error": format!("{}", e), + }), + }; + + // Test 3: Try crane (Go static binary) - fetch manifest + let crane_result = match std::process::Command::new("crane") + .args(["manifest", "alpine:latest"]) + .env("HOME", "/root") + .output() + { + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + serde_json::json!({ + "tool": "crane", + "success": output.status.success(), + "exit_code": output.status.code(), + "stdout_len": output.stdout.len(), + "stderr": stderr, + }) + } + Err(e) => serde_json::json!({ + "tool": "crane", + "error": format!("{}", e), + }), + }; + + AgentResponse::Ok { + data: Some(serde_json::json!({ + "syscall_tcp": syscall_result, + "wget": wget_result, + "crane": crane_result, + })), + } + } + + AgentRequest::Shutdown => { + info!("shutdown requested"); + // Sync filesystem before shutdown to prevent corruption + sync_and_unmount_storage(); + AgentResponse::Ok { + data: Some(serde_json::json!({"shutdown": true})), + } + } + + // VM-level background exec — spawn and return PID immediately + AgentRequest::VmExec { + command, + env, + workdir, + background: true, + .. + } => handle_vm_exec_background(&command, &env, workdir.as_deref()), + + // VM-level exec (direct command execution in VM, not container) + AgentRequest::VmExec { + command, + env, + workdir, + timeout_ms, + interactive: false, + tty: false, + stdin_data, + .. + } => handle_vm_exec( + &command, + &env, + workdir.as_deref(), + timeout_ms, + client_fd, + stdin_data.as_deref(), + ), + + AgentRequest::VmExec { .. } => { + // Interactive mode should be handled by handle_interactive_vm_exec + AgentResponse::error( + "interactive VM exec not handled here", + error_codes::INTERNAL_ERROR, + ) + } + + AgentRequest::Run { + image, + command, + env, + workdir, + user, + mounts, + timeout_ms, + interactive: false, + tty: false, + detached: false, + unprivileged, + persistent_overlay_id, + stdin_data, + background, + } => { + if background { + handle_run_background( + &image, + &command, + &env, + workdir.as_deref(), + user.as_deref(), + &mounts, + persistent_overlay_id.as_deref(), + unprivileged, + ) + } else { + handle_run( + &image, + &command, + &env, + workdir.as_deref(), + user.as_deref(), + &mounts, + timeout_ms, + persistent_overlay_id.as_deref(), + stdin_data.as_deref(), + client_fd, + unprivileged, + ) + } + } + + AgentRequest::Run { .. } => { + // Interactive mode should be handled by handle_interactive_run + AgentResponse::error( + "interactive mode not handled here", + error_codes::INTERNAL_ERROR, + ) + } + + AgentRequest::Stdin { .. } | AgentRequest::Resize { .. } => AgentResponse::error( + "stdin/resize only valid during interactive session", + error_codes::INVALID_REQUEST, + ), + + AgentRequest::ExportLayer { .. } => { + // Streaming export is handled by handle_streaming_export_layer + AgentResponse::error("export layer not handled here", error_codes::INTERNAL_ERROR) + } + + AgentRequest::FileWrite { path, data, mode } => handle_file_write(&path, &data, mode), + + // Streaming uploads go through `handle_connection`'s + // per-connection session state so they can't land here. + AgentRequest::FileWriteBegin { .. } | AgentRequest::FileWriteChunk { .. } => { + AgentResponse::error( + "streaming file write must be handled at connection level", + error_codes::INTERNAL_ERROR, + ) + } + + // Streaming read goes through `handle_connection`'s explicit + // dispatch so it can emit multiple responses per request. + AgentRequest::FileRead { .. } => AgentResponse::error( + "streaming file read must be handled at connection level", + error_codes::INTERNAL_ERROR, + ), + } +} + +// ============================================================================ +// File I/O Handlers +// ============================================================================ + +/// Unique-per-call staging suffix. Using the PID + a counter avoids +/// collisions when two connections write to the same path and avoids +/// the predictable-filename class of symlink races. +fn staging_suffix() -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + format!(".smolvm-upload.{}.{}", std::process::id(), n) +} + +/// Ensure `path`'s parent exists, returning an AgentResponse on error +/// (so the two file-write entry points don't duplicate this block). +fn ensure_parent_dir(path: &std::path::Path) -> std::result::Result<(), AgentResponse> { + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + if let Err(e) = std::fs::create_dir_all(parent) { + return Err(AgentResponse::error( + format!("failed to create directory {}: {}", parent.display(), e), + error_codes::FILE_IO_FAILED, + )); + } + } + } + Ok(()) +} + +/// Apply a Unix mode, logging but not failing if the permissions +/// can't be set (matches prior single-shot behavior). +#[cfg(unix)] +fn apply_mode_best_effort(path: &std::path::Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + if let Err(e) = std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)) { + info!(path = %path.display(), error = %e, "failed to set file mode (non-fatal)"); + } +} +#[cfg(not(unix))] +fn apply_mode_best_effort(_path: &std::path::Path, _mode: u32) {} + +#[derive(Clone, Copy, Debug)] +enum FilePathAccess { + Read, + Write, +} + +fn invalid_guest_path_error(path: &str, reason: &str) -> AgentResponse { + AgentResponse::error( + format!("invalid guest path {}: {}", path, reason), + error_codes::INVALID_REQUEST, + ) +} + +fn normalize_guest_path(path: &str) -> std::result::Result { + if path.is_empty() { + return Err(invalid_guest_path_error(path, "path is empty")); + } + + let normalized = if path.starts_with('/') { + path.to_string() + } else { + format!("/{}", path) + }; + + let p = std::path::Path::new(&normalized); + for component in p.components() { + match component { + std::path::Component::ParentDir => { + return Err(invalid_guest_path_error( + path, + "parent traversal is not allowed", + )); + } + std::path::Component::CurDir => { + return Err(invalid_guest_path_error( + path, + "current-dir segments are not allowed", + )); + } + std::path::Component::Prefix(_) => { + return Err(invalid_guest_path_error( + path, + "path prefixes are not allowed", + )); + } + std::path::Component::RootDir | std::path::Component::Normal(_) => {} + } + } + + Ok(normalized) +} + +fn active_persistent_overlay_merged_root() -> Option { + let overlays_dir = std::path::Path::new("/storage/overlays"); + let entries = std::fs::read_dir(overlays_dir).ok()?; + for entry in entries.flatten() { + if !entry + .file_name() + .to_string_lossy() + .starts_with("persistent-") + { + continue; + } + let merged = entry.path().join("merged"); + if merged.join("bin").exists() || merged.join("usr").exists() { + return Some(merged); + } + } + None +} + +fn ensure_descendant_after_canonicalize( + path: &std::path::Path, + root: &std::path::Path, + original_path: &str, + access: FilePathAccess, +) -> std::result::Result<(), AgentResponse> { + if matches!(access, FilePathAccess::Write) && !root.exists() { + std::fs::create_dir_all(root).map_err(|e| { + AgentResponse::error( + format!("failed to create root {}: {}", root.display(), e), + error_codes::FILE_IO_FAILED, + ) + })?; + } + + let root_canon = root.canonicalize().map_err(|e| { + AgentResponse::error( + format!("failed to canonicalize root {}: {}", root.display(), e), + error_codes::FILE_IO_FAILED, + ) + })?; + + match access { + FilePathAccess::Read => { + let path_canon = path.canonicalize().map_err(|e| { + AgentResponse::error( + format!("failed to canonicalize target {}: {}", original_path, e), + error_codes::FILE_IO_FAILED, + ) + })?; + if !path_canon.starts_with(&root_canon) { + return Err(invalid_guest_path_error( + original_path, + "resolved path escapes allowed root", + )); + } + } + FilePathAccess::Write => { + let parent = path.parent().ok_or_else(|| { + invalid_guest_path_error(original_path, "target has no parent directory") + })?; + + std::fs::create_dir_all(parent).map_err(|e| { + AgentResponse::error( + format!("failed to create directory {}: {}", parent.display(), e), + error_codes::FILE_IO_FAILED, + ) + })?; + + let parent_canon = parent.canonicalize().map_err(|e| { + AgentResponse::error( + format!("failed to canonicalize parent {}: {}", parent.display(), e), + error_codes::FILE_IO_FAILED, + ) + })?; + if !parent_canon.starts_with(&root_canon) { + return Err(invalid_guest_path_error( + original_path, + "resolved parent escapes allowed root", + )); + } + + if path.exists() { + let path_canon = path.canonicalize().map_err(|e| { + AgentResponse::error( + format!("failed to canonicalize target {}: {}", original_path, e), + error_codes::FILE_IO_FAILED, + ) + })?; + if !path_canon.starts_with(&root_canon) { + return Err(invalid_guest_path_error( + original_path, + "resolved path escapes allowed root", + )); + } + } + } + } + + Ok(()) +} + +fn resolve_guest_io_path_with_roots( + path: &str, + access: FilePathAccess, + overlay_merged_root: Option<&std::path::Path>, + workspace_root: &std::path::Path, +) -> std::result::Result { + let normalized = normalize_guest_path(path)?; + + let Some(overlay_root) = overlay_merged_root else { + return Ok(std::path::PathBuf::from(normalized)); + }; + + let (target, allowed_root): (std::path::PathBuf, &std::path::Path) = if let Some(relative) = + normalized.strip_prefix("/workspace/").or_else(|| { + if normalized == "/workspace" { + Some("") + } else { + None + } + }) { + (workspace_root.join(relative), workspace_root) + } else { + let relative = normalized.strip_prefix('/').unwrap_or(&normalized); + (overlay_root.join(relative), overlay_root) + }; + + ensure_descendant_after_canonicalize(&target, allowed_root, &normalized, access)?; + Ok(target) +} + +/// Resolve guest file I/O path with strict boundary checks. +/// +/// For image-based persistent VMs, paths are mapped into the active +/// `persistent-*` overlay's `merged` root (except `/workspace`, which +/// maps to `/storage/workspace`, the bind-mount source). Both read and +/// write flows enforce canonicalized descendant checks against the +/// selected root to prevent traversal and symlink escapes. +fn resolve_guest_io_path( + path: &str, + access: FilePathAccess, +) -> std::result::Result { + let overlay = active_persistent_overlay_merged_root(); + resolve_guest_io_path_with_roots( + path, + access, + overlay.as_deref(), + std::path::Path::new("/storage/workspace"), + ) +} + +/// Shared between single-shot [`handle_file_write`] and the streaming +/// finalize step. The atomic-rename pattern is the thing both paths +/// need to guarantee: partial contents never appear at `path` under +/// any error or kill scenario. +fn install_file_atomic(path: &str, data: &[u8], mode: Option) -> AgentResponse { + let resolved = match resolve_guest_io_path(path, FilePathAccess::Write) { + Ok(p) => p, + Err(resp) => return resp, + }; + let target = resolved.as_path(); + if let Err(resp) = ensure_parent_dir(target) { + return resp; + } + + let tmp_name = format!( + "{}{}", + target + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(), + staging_suffix() + ); + let tmp_path = target + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .map(|p| p.join(&tmp_name)) + .unwrap_or_else(|| std::path::PathBuf::from(&tmp_name)); + + if let Err(e) = std::fs::write(&tmp_path, data) { + let _ = std::fs::remove_file(&tmp_path); + return AgentResponse::error( + format!("failed to write {}: {}", tmp_path.display(), e), + error_codes::FILE_IO_FAILED, + ); + } + + if let Err(e) = std::fs::rename(&tmp_path, target) { + let _ = std::fs::remove_file(&tmp_path); + return AgentResponse::error( + format!("failed to rename onto {}: {}", path, e), + error_codes::FILE_IO_FAILED, + ); + } + + if let Some(m) = mode { + apply_mode_best_effort(target, m); + } + info!(path = %path, size = data.len(), "file written"); + AgentResponse::Ok { data: None } +} + +/// Write a file inside the VM filesystem (single-shot path). +fn handle_file_write(path: &str, data: &[u8], mode: Option) -> AgentResponse { + install_file_atomic(path, data, mode) +} + +/// State for an in-progress streaming file upload on one connection. +/// +/// One session lives inside `handle_connection`'s stack, so it's +/// scoped to a single client. `Drop` cleans up the staging file if +/// the connection drops (or the session is replaced) before the +/// final chunk arrives — this is how we guarantee no partial file +/// ever appears at the target path. +struct WriteSession { + /// User-requested target path inside the guest. + target: std::path::PathBuf, + /// Staging file we append to; renamed to `target` on done. + tmp_path: std::path::PathBuf, + /// Handle we keep open for the lifetime of the session. + tmp_file: std::fs::File, + /// Permissions to apply after rename. + mode: Option, + /// Running total — compared against `total_size` as a DoS guard. + bytes_written: u64, + /// Caller-declared total; the agent refuses chunks that would + /// push `bytes_written` past it. + total_size: u64, +} + +impl WriteSession { + /// Open a fresh staging file for `target`. + fn open( + target: std::path::PathBuf, + mode: Option, + total_size: u64, + ) -> std::io::Result { + if let Some(parent) = target.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent)?; + } + } + let tmp_name = format!( + "{}{}", + target + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(), + staging_suffix() + ); + let tmp_path = target + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .map(|p| p.join(&tmp_name)) + .unwrap_or_else(|| std::path::PathBuf::from(&tmp_name)); + + let tmp_file = std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .truncate(true) + .open(&tmp_path)?; + Ok(Self { + target, + tmp_path, + tmp_file, + mode, + bytes_written: 0, + total_size, + }) + } + + /// Append a chunk; returns AgentResponse on error. + fn write_chunk(&mut self, data: &[u8]) -> std::result::Result<(), AgentResponse> { + let new_total = self.bytes_written.saturating_add(data.len() as u64); + if new_total > self.total_size { + return Err(AgentResponse::error( + format!( + "chunk overflows declared total_size ({} > {})", + new_total, self.total_size + ), + error_codes::INVALID_REQUEST, + )); + } + use std::io::Write; + if let Err(e) = self.tmp_file.write_all(data) { + return Err(AgentResponse::error( + format!("failed to write chunk to staging file: {}", e), + error_codes::FILE_IO_FAILED, + )); + } + self.bytes_written = new_total; + Ok(()) + } + + /// Fsync, rename onto target, apply mode. Consumes the session. + /// + /// Takes `&mut self` rather than `self` so we can do a + /// `mem::take` on `tmp_path` to disarm the Drop-based cleanup + /// after the rename has moved the file onto its final path. + /// (Moving individual fields out of a struct with Drop isn't + /// allowed — `mem::take` swaps in a default `PathBuf` so the + /// subsequent Drop is a no-op.) + /// + /// Linux + macOS allow renaming an open file, so we don't need + /// to close the handle first; it drops naturally when this + /// function returns via the by-value caller pattern. + fn finalize(&mut self) -> AgentResponse { + use std::io::Write; + if let Err(e) = self.tmp_file.flush() { + return AgentResponse::error( + format!("failed to flush staging file: {}", e), + error_codes::FILE_IO_FAILED, + ); + } + if let Err(e) = self.tmp_file.sync_all() { + return AgentResponse::error( + format!("failed to sync staging file: {}", e), + error_codes::FILE_IO_FAILED, + ); + } + // Disarm Drop before rename; if the rename fails we'll + // re-arm below by restoring the path. + let tmp = std::mem::take(&mut self.tmp_path); + if let Err(e) = std::fs::rename(&tmp, &self.target) { + // Re-arm Drop so the staging file still gets cleaned up + // when the session is dropped by the caller. + self.tmp_path = tmp; + return AgentResponse::error( + format!("failed to rename onto {}: {}", self.target.display(), e), + error_codes::FILE_IO_FAILED, + ); + } + if let Some(m) = self.mode { + apply_mode_best_effort(&self.target, m); + } + info!( + path = %self.target.display(), + size = self.bytes_written, + "file written (streaming)" + ); + AgentResponse::Ok { data: None } + } +} + +impl Drop for WriteSession { + fn drop(&mut self) { + // If finalize consumed the session, `tmp_path` was emptied. + if !self.tmp_path.as_os_str().is_empty() { + let _ = std::fs::remove_file(&self.tmp_path); + } + } +} + +/// Open a streaming upload session. Called from the connection loop. +/// +/// Returns the new session plus the response to send back. On error +/// the session is not created. +fn handle_file_write_begin( + path: String, + mode: Option, + total_size: u64, +) -> (Option, AgentResponse) { + if total_size > smolvm_protocol::FILE_TRANSFER_MAX_TOTAL { + return ( + None, + AgentResponse::error( + format!( + "total_size {} exceeds maximum {}", + total_size, + smolvm_protocol::FILE_TRANSFER_MAX_TOTAL + ), + error_codes::INVALID_REQUEST, + ), + ); + } + let resolved = match resolve_guest_io_path(&path, FilePathAccess::Write) { + Ok(p) => p, + Err(resp) => return (None, resp), + }; + match WriteSession::open(resolved, mode, total_size) { + Ok(session) => (Some(session), AgentResponse::Ok { data: None }), + Err(e) => ( + None, + AgentResponse::error( + format!("failed to open staging file: {}", e), + error_codes::FILE_IO_FAILED, + ), + ), + } +} + +/// Append a chunk to the open session (if any). Called from the +/// connection loop. On `done`, the session is consumed and the file +/// is finalized. +/// +/// Returns the (possibly consumed) session plus the response. +fn handle_file_write_chunk( + session: Option, + data: &[u8], + done: bool, +) -> (Option, AgentResponse) { + let Some(mut s) = session else { + return ( + None, + AgentResponse::error( + "no FileWriteBegin issued on this connection", + error_codes::INVALID_REQUEST, + ), + ); + }; + if let Err(resp) = s.write_chunk(data) { + // Session is dropped by returning None, cleaning the tmp file. + return (None, resp); + } + if done { + let resp = s.finalize(); + // On success `tmp_path` was cleared so Drop is a no-op; + // on failure the session Drop will still clean the staging + // file when `s` falls out of scope here. + (None, resp) + } else { + (Some(s), AgentResponse::Ok { data: None }) + } +} + +/// Stream a reader's bytes to the client as a sequence of +/// `AgentResponse::DataChunk` responses. +/// +/// Shared between `FileRead` (reader = open file) and `ExportLayer` +/// (reader = `tar` child stdout). Each chunk is at most `chunk_size` +/// bytes; EOF is always signaled with a trailing `done: true` frame +/// (possibly empty) so the client's receive loop terminates uniformly. +/// +/// On read error, emits a structured Error response with the +/// caller-supplied `error_code` (so operators can distinguish +/// "file-IO failed" from "export failed" in logs and status codes) +/// and returns the `io::Error` to the caller for producer-specific +/// cleanup (e.g., killing a child process). +fn send_data_chunks( + stream: &mut impl Write, + reader: &mut R, + chunk_size: usize, + error_context: &str, + error_code: &'static str, +) -> Result<(), Box> { + let mut buf = vec![0u8; chunk_size]; + loop { + // Fill as much of the buffer as possible in one chunk. + // Partial reads are common when the source is a pipe + // (e.g. tar subprocess) — we keep reading until the buffer + // is full or EOF arrives. + let mut pending = 0; + while pending < buf.len() { + match reader.read(&mut buf[pending..]) { + Ok(0) => break, + Ok(n) => pending += n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => { + send_response( + stream, + &AgentResponse::error(format!("{}: {}", error_context, e), error_code), + )?; + return Err(Box::new(e)); + } + } + } + + if pending == 0 { + // EOF — emit the terminator frame. + send_response( + stream, + &AgentResponse::DataChunk { + data: vec![], + done: true, + }, + )?; + return Ok(()); + } + + send_response( + stream, + &AgentResponse::DataChunk { + data: buf[..pending].to_vec(), + done: false, + }, + )?; + } +} + +/// Stream a file from the guest filesystem to the host as a +/// sequence of `DataChunk` responses. Called from the connection +/// loop (not the generic `handle_request` match) so it can emit +/// multiple responses per request. +fn handle_streaming_file_read( + stream: &mut impl ReadWrite, + path: &str, +) -> Result<(), Box> { + let resolved = match resolve_guest_io_path(path, FilePathAccess::Read) { + Ok(p) => p, + Err(resp) => { + send_response(stream, &resp)?; + return Ok(()); + } + }; + let mut file = match std::fs::File::open(&resolved) { + Ok(f) => f, + Err(e) => { + send_response( + stream, + &AgentResponse::error( + format!("failed to open {}: {}", path, e), + error_codes::FILE_IO_FAILED, + ), + )?; + return Ok(()); + } + }; + // Only stream regular files. Directories and special files (e.g. /dev/zero, + // FIFOs) `open()` successfully but misbehave on read — a directory fails with + // EISDIR *after* the chunk stream has started (desyncing the wire and bricking + // the connection), and an unbounded device never EOFs (hangs the caller). We + // must reject them with a clean error BEFORE the first DataChunk frame. + let metadata = match file.metadata() { + Ok(m) => m, + Err(e) => { + send_response( + stream, + &AgentResponse::error( + format!("failed to stat {}: {}", path, e), + error_codes::FILE_IO_FAILED, + ), + )?; + return Ok(()); + } + }; + if !metadata.is_file() { + send_response( + stream, + &AgentResponse::error( + format!("not a regular file: {}", path), + error_codes::FILE_IO_FAILED, + ), + )?; + return Ok(()); + } + let size = metadata.len(); + info!(path = %path, size, "streaming file read"); + send_data_chunks( + stream, + &mut file, + smolvm_protocol::LAYER_CHUNK_SIZE, + "failed to read file", + error_codes::FILE_IO_FAILED, + ) +} + +/// Handle an interactive run session with streaming I/O. +fn handle_interactive_run( + stream: &mut impl ReadWrite, + request: AgentRequest, +) -> Result<(), Box> { + ensure_storage_mounted(); + let ( + image, + command, + env, + workdir, + user, + mounts, + timeout_ms, + tty, + persistent_overlay_id, + unprivileged, + ) = match request { + AgentRequest::Run { + image, + command, + env, + workdir, + user, + mounts, + timeout_ms, + tty, + persistent_overlay_id, + unprivileged, + .. + } => ( + image, + command, + env, + workdir, + user, + mounts, + timeout_ms, + tty, + persistent_overlay_id, + unprivileged, + ), + _ => { + send_response( + stream, + &AgentResponse::error("expected Run request", error_codes::INVALID_REQUEST), + )?; + return Ok(()); + } + }; + + let is_persistent = persistent_overlay_id.is_some(); + info!(image = %image, command = ?command, tty = tty, persistent = is_persistent, "starting interactive run"); + + // Prepare the overlay and get the rootfs path + let prepared = match &persistent_overlay_id { + Some(id) => storage::prepare_for_run_persistent(&image, id), + None => storage::prepare_for_run(&image), + }; + let prepared = match prepared { + Ok(prepared) => prepared, + Err(e) => { + send_response(stream, &AgentResponse::from_err(e, error_codes::RUN_FAILED))?; + return Ok(()); + } + }; + + // Setup virtiofs mounts at staging area (crun will bind-mount them via OCI spec) + // Helper: only clean up ephemeral overlays, not persistent ones + let maybe_cleanup = |wid: &str| { + if !is_persistent { + let _ = storage::cleanup_overlay(wid); + } + }; + + if let Err(e) = storage::setup_mounts(&prepared.rootfs_path, &mounts) { + maybe_cleanup(&prepared.workload_id); + send_response( + stream, + &AgentResponse::from_err(e, error_codes::MOUNT_FAILED), + )?; + return Ok(()); + } + + // Resolve the container's launch settings from the image's OCI config (with + // request overrides). Required to call spawn_interactive_command, so the + // interactive path can't drop the image's Env/WorkingDir/User either. + let launch = match ResolvedLaunch::resolve(&image, command, env, workdir, user) { + Ok(l) => l, + Err(e) => { + maybe_cleanup(&prepared.workload_id); + send_response( + stream, + &AgentResponse::error(e.to_string(), error_codes::INVALID_REQUEST), + )?; + return Ok(()); + } + }; + + // Spawn the command with crun + let (mut child, pty_master) = match spawn_interactive_command( + &prepared.rootfs_path, + &launch, + &mounts, + tty, + persistent_overlay_id.as_deref(), + unprivileged, + ) { + Ok(result) => result, + Err(e) => { + maybe_cleanup(&prepared.workload_id); + send_response( + stream, + &AgentResponse::from_err(e, error_codes::SPAWN_FAILED), + )?; + return Ok(()); + } + }; + + // Send Started response + send_response(stream, &AgentResponse::Started)?; + + // Run the appropriate interactive I/O loop + let exit_code = match pty_master { + #[cfg(target_os = "linux")] + Some(pty) => match run_interactive_loop_pty(stream, &mut child, pty, timeout_ms) { + Ok(exit_code) => exit_code, + Err(e) => { + maybe_cleanup(&prepared.workload_id); + return Err(e); + } + }, + _ => match run_interactive_loop(stream, &mut child, timeout_ms) { + Ok(exit_code) => exit_code, + Err(e) => { + maybe_cleanup(&prepared.workload_id); + return Err(e); + } + }, + }; + + // Send Exited response + send_response(stream, &AgentResponse::Exited { exit_code })?; + maybe_cleanup(&prepared.workload_id); + + Ok(()) +} + +/// The fully-resolved launch settings for an image container: the image's OCI +/// config (Entrypoint/Cmd, Env, WorkingDir, User) merged with the request. +/// +/// Fields are private and the ONLY constructor is [`ResolvedLaunch::resolve`], +/// which performs the merge — and [`write_oci_bundle`], the single path that +/// creates an OCI container, requires a `&ResolvedLaunch`. So every container +/// launch necessarily honors the image config: the detached and interactive +/// paths both go through it, and a *future* launch path won't compile without +/// resolving. That's what keeps Env/WorkingDir/User from being silently dropped +/// on some path — the failure mode this type exists to make impossible. +/// +/// Defined on all platforms: the shared `handle_interactive_run` constructs one, +/// and the macOS build compiles the agent (as stubs) for `cargo test`. +struct ResolvedLaunch { + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + user: Option, +} + +impl ResolvedLaunch { + /// Merge the request's launch params with the image's OCI config: + /// - **command**: the request's, else the image's `ENTRYPOINT` + `CMD` + /// (errors if neither exists), so a service-style image runs as authored. + /// - **env**: the image's `Env` (notably `PATH`) with the request layered on + /// top — the request wins per key. + /// - **workdir / user**: the request's, else the image's — an image `CMD` is + /// relative to its `WORKDIR`, and its `USER` is the uid it expects to run as. + fn resolve( + image: &str, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + user: Option, + ) -> Result> { + let info = storage::query_image(image)?.ok_or_else(|| -> Box { + format!("image not found: {image}").into() + })?; + let command = if command.is_empty() { + let mut resolved = info.entrypoint; + resolved.extend(info.cmd); + if resolved.is_empty() { + return Err(format!( + "no command given and image '{image}' defines no entrypoint or cmd" + ) + .into()); + } + resolved + } else { + command + }; + Ok(Self { + command, + env: merge_image_env(info.env, env), + workdir: workdir.or(info.workdir), + user: user.or(info.user), + }) + } +} + +/// Layer the request's env over an image's OCI `Env` (each `"KEY=VAL"`): the +/// request wins on key conflicts, image entries fill in the rest — matching how +/// a container runtime composes image + run-time environment. +fn merge_image_env( + image_env: Vec, + request_env: Vec<(String, String)>, +) -> Vec<(String, String)> { + let request_keys: std::collections::HashSet<&str> = + request_env.iter().map(|(k, _)| k.as_str()).collect(); + let mut merged: Vec<(String, String)> = image_env + .iter() + .filter_map(|entry| entry.split_once('=')) + .filter(|(k, _)| !request_keys.contains(*k)) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + merged.extend(request_env); + merged +} + +/// Write an OCI bundle (`config.json`) and return a freshly generated container ID. +/// +/// Shared by [`handle_run_detached`] and [`spawn_interactive_command`] to avoid +/// duplicating the identity-resolve → spec-build → mount-wiring → write sequence. +/// The only caller-controlled variation is `tty` (detached: always `false`). +#[cfg(target_os = "linux")] +#[allow(clippy::too_many_arguments)] +fn write_oci_bundle( + rootfs_path: &std::path::Path, + bundle_path: &std::path::Path, + launch: &ResolvedLaunch, + mounts: &[(String, String, bool)], + tty: bool, + unprivileged: bool, +) -> Result> { + use std::path::Path; + + let workdir_str = launch.workdir.as_deref().unwrap_or("/"); + let identity = oci::resolve_process_identity(rootfs_path, launch.user.as_deref())?; + let mut spec = oci::OciSpec::new( + &launch.command, + &launch.env, + workdir_str, + tty, + &identity, + unprivileged, + ); + + if tty { + // Give the PTY a non-zero starting size. The host follows up with a + // Resize message carrying the real terminal dimensions as soon as + // the interactive session starts. + spec.process.console_size = Some(oci::OciConsoleSize { + height: 24, + width: 80, + }); + } + + // Interactive and detached containers use this bundle writer instead of + // storage::run_command(). Mirror that path's GPU wiring so `-i`/`-t` + // shells see /dev/dri when the VM was started with --gpu. + spec.add_gpu_devices_if_available(); + + for (tag, container_path, read_only) in mounts { + let virtiofs_mount = Path::new(paths::VIRTIOFS_MOUNT_ROOT).join(tag); + spec.add_bind_mount( + &virtiofs_mount.to_string_lossy(), + container_path, + *read_only, + ); + } + + storage::add_workspace_fallback(&mut spec, mounts); + storage::add_storage_fallback(&mut spec, mounts, unprivileged); + + ssh_agent::inject_into_container(&mut spec); + rosetta::inject_into_container(&mut spec); + cuda::inject_into_container(&mut spec, rootfs_path); + spec.write_to(bundle_path) + .map_err(|e| format!("failed to write OCI spec: {}", e))?; + + Ok(oci::generate_container_id()) +} + +/// Handle a detached run request: start a container in the background and +/// return its container ID to the caller. +/// +/// The container ID is saved to `main_container_id_path(workload_id)` so that +/// subsequent `machine exec` calls can join the container via `crun exec` +/// instead of creating a new isolated container. +/// +/// Requires `persistent_overlay_id` to be set — detached containers must +/// persist across exec sessions, which requires a named overlay. +#[cfg(target_os = "linux")] +fn handle_run_detached( + stream: &mut impl ReadWrite, + request: AgentRequest, +) -> Result<(), Box> { + use std::path::Path; + + ensure_storage_mounted(); + + let (image, command, env, workdir, user, mounts, persistent_overlay_id, unprivileged) = + match request { + AgentRequest::Run { + image, + command, + env, + workdir, + user, + mounts, + persistent_overlay_id, + unprivileged, + .. + } => ( + image, + command, + env, + workdir, + user, + mounts, + persistent_overlay_id, + unprivileged, + ), + _ => { + send_response( + stream, + &AgentResponse::error("expected Run request", error_codes::INVALID_REQUEST), + )?; + return Ok(()); + } + }; + + // An empty command is allowed here: it means "run the image's own + // ENTRYPOINT/CMD". We resolve it from the image config below, after the + // image has been prepared (so its config is guaranteed present). + + let overlay_id = match persistent_overlay_id { + Some(id) => id, + None => { + send_response( + stream, + &AgentResponse::error( + "detached mode requires persistent_overlay_id", + error_codes::INVALID_REQUEST, + ), + )?; + return Ok(()); + } + }; + + info!( + image = %image, + command = ?command, + overlay_id = %overlay_id, + "starting detached container" + ); + + let prepared = match storage::prepare_for_run_persistent(&image, &overlay_id) { + Ok(p) => p, + Err(e) => { + send_response(stream, &AgentResponse::from_err(e, error_codes::RUN_FAILED))?; + return Ok(()); + } + }; + + // Resolve the container's launch settings from the image's OCI config + // (command, Env, WorkingDir, User) with the request layered on top. + // `write_oci_bundle` requires a `ResolvedLaunch`, so the image config can't be + // silently dropped here or on any other launch path. + let launch = match ResolvedLaunch::resolve(&image, command, env, workdir, user) { + Ok(l) => l, + Err(e) => { + send_response( + stream, + &AgentResponse::error(e.to_string(), error_codes::INVALID_REQUEST), + )?; + return Ok(()); + } + }; + info!(image = %image, command = ?launch.command, workdir = ?launch.workdir, user = ?launch.user, "resolved launch from request + image config"); + + if let Err(e) = storage::setup_mounts(&prepared.rootfs_path, &mounts) { + send_response( + stream, + &AgentResponse::from_err(e, error_codes::MOUNT_FAILED), + )?; + return Ok(()); + } + + let rootfs_path = Path::new(&prepared.rootfs_path); + let bundle_path = match rootfs_path.parent() { + Some(p) => p.join("bundle"), + None => { + send_response( + stream, + &AgentResponse::error( + "invalid rootfs path: no parent", + error_codes::INTERNAL_ERROR, + ), + )?; + return Ok(()); + } + }; + + if !bundle_path.exists() { + send_response( + stream, + &AgentResponse::error( + format!("bundle directory not found: {}", bundle_path.display()), + error_codes::INTERNAL_ERROR, + ), + )?; + return Ok(()); + } + + let workload_id = format!("persistent-{}", overlay_id); + + // Detached containers always run non-interactively (tty: false). + let container_id = match write_oci_bundle( + rootfs_path, + &bundle_path, + &launch, + &mounts, + false, + unprivileged, + ) { + Ok(id) => id, + Err(e) => { + send_response( + stream, + &AgentResponse::from_err(e, error_codes::INTERNAL_ERROR), + )?; + return Ok(()); + } + }; + + info!( + container_id = %container_id, + workload_id = %workload_id, + "running detached container" + ); + + // Use `crun create` + `crun start` (two-step OCI lifecycle) instead of + // `crun run --detach` which hangs in the smolvm VM environment. The + // two-step approach registers the container state so `crun exec` can join + // it, and `crun start` returns immediately once the container is running. + let create_output = crun::CrunCommand::create(&bundle_path, &container_id).output(); + match create_output { + Ok(output) if output.status.success() => {} + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + send_response( + stream, + &AgentResponse::error( + format!("crun create failed: {}", stderr.trim()), + error_codes::SPAWN_FAILED, + ), + )?; + return Ok(()); + } + Err(e) => { + send_response( + stream, + &AgentResponse::from_err(e, error_codes::SPAWN_FAILED), + )?; + return Ok(()); + } + } + + let start_output = crun::CrunCommand::start(&container_id).output(); + match start_output { + Ok(output) if output.status.success() => { + // Save the container ID so subsequent execs can join this container. + // If the write fails, kill the container and return an error — exec + // join won't work without the persisted ID. + let id_path = paths::main_container_id_path(&workload_id); + if let Err(e) = std::fs::write(&id_path, container_id.as_bytes()) { + let _ = crun::CrunCommand::kill(&container_id, "SIGKILL").status(); + let _ = crun::CrunCommand::delete(&container_id, true).output(); + send_response( + stream, + &AgentResponse::error( + format!("failed to persist container ID: {}", e), + error_codes::INTERNAL_ERROR, + ), + )?; + return Ok(()); + } + info!( + container_id = %container_id, + "detached container started via create+start" + ); + send_response( + stream, + &AgentResponse::Completed { + exit_code: 0, + stdout: container_id.into_bytes(), + stderr: vec![], + }, + )?; + } + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + // Clean up the created-but-not-started container + let _ = crun::CrunCommand::delete(&container_id, true).output(); + send_response( + stream, + &AgentResponse::error( + format!("crun start failed: {}", stderr.trim()), + error_codes::SPAWN_FAILED, + ), + )?; + } + Err(e) => { + let _ = crun::CrunCommand::delete(&container_id, true).output(); + send_response( + stream, + &AgentResponse::from_err(e, error_codes::SPAWN_FAILED), + )?; + } + } + + Ok(()) +} + +/// Non-Linux stub: the agent only runs on Linux; this exists so the host-side +/// `cargo check` on macOS compiles the dispatch in `handle_connection`. +#[cfg(not(target_os = "linux"))] +fn handle_run_detached( + stream: &mut impl ReadWrite, + _request: AgentRequest, +) -> Result<(), Box> { + send_response( + stream, + &AgentResponse::error( + "detached mode not supported on this platform", + error_codes::INTERNAL_ERROR, + ), + )?; + Ok(()) +} + +/// Check whether a crun container is currently running by querying `crun state` +/// and verifying the PID is still alive. +/// +/// Returns `false` on any error (missing state files, parse failures, etc.) +/// so callers fall through to starting a fresh container. +#[cfg(target_os = "linux")] +pub fn is_container_running(container_id: &str) -> bool { + let output = match crun::CrunCommand::state(container_id) + .capture_output() + .output() + { + Ok(o) => o, + Err(_) => return false, + }; + if !output.status.success() { + return false; + } + let stdout = match std::str::from_utf8(&output.stdout) { + Ok(s) => s, + Err(_) => return false, + }; + let v: serde_json::Value = match serde_json::from_str(stdout) { + Ok(v) => v, + Err(_) => return false, + }; + + let is_running = v + .get("status") + .and_then(|s| s.as_str()) + .map(|s| s == "running") + .unwrap_or(false); + if !is_running { + return false; + } + + let pid = v.get("pid").and_then(|p| p.as_i64()).unwrap_or(0); + if pid <= 0 { + return false; + } + + // kill(pid, 0) checks process existence without sending a signal. + // Handles stale "running" state left after SIGKILL of a container. + // SAFETY: signal 0 never terminates a process; checks existence only. + unsafe { libc::kill(pid as libc::pid_t, 0) == 0 } +} + +/// Non-Linux stub. +#[cfg(not(target_os = "linux"))] +pub fn is_container_running(_container_id: &str) -> bool { + false +} + +/// Spawn a command by joining a running container via `crun exec`. +/// +/// Used when a main workload container is already running for this overlay — +/// the new command joins its existing PID/mount/cgroup namespaces rather than +/// creating an isolated new container. +/// +/// When `tty` is true the PTY is obtained from crun via `--console-socket`, +/// exactly like the fresh `crun run` path in [`spawn_interactive_command`]. +/// Letting crun allocate the PTY (rather than handing it a slave from an +/// agent-owned pair) is what makes `TIOCSWINSZ` resizes reach the process +/// inside the container; without it TUIs never redraw on host resize. See +/// GH #156. +#[cfg(target_os = "linux")] +fn spawn_exec_in_container( + container_id: &str, + launch: &ResolvedLaunch, + tty: bool, +) -> Result<(Child, Option), Box> { + use std::io::Read as _; + use std::os::unix::io::AsRawFd as _; + use std::sync::atomic::Ordering; + + // An exec joining a running container inherits the same image-resolved env / + // workdir as the container's main process. + let command: &[String] = &launch.command; + let env: &[(String, String)] = &launch.env; + let workdir: Option<&str> = launch.workdir.as_deref(); + + info!( + container_id = %container_id, + command = ?command, + tty = tty, + "joining running container via crun exec" + ); + + if tty { + // Preferred: console socket (resizable). Mirrors the create path. + if CONSOLE_SOCKET_WORKS.load(Ordering::Relaxed) { + let console = pty::ConsoleSocket::bind(container_id)?; + let mut child = crun::CrunCommand::exec_with_console( + container_id, + env, + command, + workdir, + console.path(), + ) + .spawn()?; + match console.recv_master(std::time::Duration::from_secs(3)) { + Ok(pty_master) => { + let _ = pty_master.set_window_size(80, 24); + if let Some(mut err) = child.stderr.take() { + std::thread::spawn(move || { + let mut sink = Vec::new(); + let _ = err.read_to_end(&mut sink); + }); + } + return Ok((child, Some(pty_master))); + } + Err(e) => { + // A transient timeout (crun slow to hand back the console) + // must not permanently disable console sockets for the rest + // of the VM's life — that would silently lose resize on + // every later session. Only latch off when the runtime + // genuinely doesn't support them (a non-timeout failure). + if e.kind() != std::io::ErrorKind::TimedOut { + CONSOLE_SOCKET_WORKS.store(false, Ordering::Relaxed); + } + let _ = child.kill(); + let _ = child.wait(); + let stderr = child + .stderr + .take() + .map(|mut s| { + let mut buf = String::new(); + let _ = s.read_to_string(&mut buf); + buf + }) + .unwrap_or_default(); + warn!(error = %e, crun_stderr = %stderr.trim(), "exec console socket unavailable; falling back to stdio PTY"); + } + } + } + + // Fallback: attach the agent's PTY slave as crun exec's stdio. + let (pty_master, slave_fd) = pty::open_pty(80, 24)?; + let slave_raw = slave_fd.as_raw_fd(); + // SAFETY: slave_fd is a valid open fd from openpty. + let child = unsafe { + crun::CrunCommand::exec(container_id, env, command, workdir, true) + .stdin_from_fd(libc::dup(slave_raw)) + .stdout_from_fd(libc::dup(slave_raw)) + .stderr_from_fd(libc::dup(slave_raw)) + .spawn()? + }; + drop(slave_fd); + Ok((child, Some(pty_master))) + } else { + let child = crun::CrunCommand::exec(container_id, env, command, workdir, false) + .stdin_piped() + .capture_output() + .spawn()?; + Ok((child, None)) + } +} + +/// Look up a running main workload container for the given overlay ID. +/// +/// Returns `Some(container_id)` if a container is registered and alive. +/// Cleans up stale state (dead container ID file, orphaned crun state) +/// and returns `None` so the caller falls through to a fresh `crun run`. +/// +/// Used by both interactive and non-interactive exec paths. +#[cfg(target_os = "linux")] +pub fn resolve_main_container(persistent_overlay_id: Option<&str>) -> Option { + let overlay_id = persistent_overlay_id?; + let workload_id = format!("persistent-{}", overlay_id); + let id_path = paths::main_container_id_path(&workload_id); + + let cid = std::fs::read_to_string(&id_path).ok()?; + let cid = cid.trim().to_string(); + if cid.is_empty() { + return None; + } + + if is_container_running(&cid) { + return Some(cid); + } + + // Stale: container died. Clean up the ID file and crun state. + info!(container_id = %cid, "main container not running, cleaning up stale state"); + let _ = std::fs::remove_file(&id_path); + let _ = crun::CrunCommand::delete(&cid, true).output(); + None +} + +/// Non-Linux stub. +#[cfg(not(target_os = "linux"))] +pub fn resolve_main_container(_persistent_overlay_id: Option<&str>) -> Option { + None +} + +/// Whether the runtime's `--console-socket` handshake works in this environment. +/// We attempt it once; if the runtime never hands back the console master (older +/// crun, or a guest where it doesn't work), we flip this off and use the +/// stdio-PTY fallback for the rest of the process's life — so only the first +/// interactive session pays the connect timeout. The console path gives dynamic +/// terminal resize; the fallback works everywhere but doesn't propagate resize. +#[cfg(target_os = "linux")] +static CONSOLE_SOCKET_WORKS: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(true); + +/// Spawn a command for interactive execution using crun OCI runtime. +/// +/// When `tty` is true, the OCI spec sets `terminal: true` and the agent +/// asks crun to allocate the container's PTY via `--console-socket`. +/// crun sends the PTY master fd back to the agent over an AF_UNIX socket +/// using `SCM_RIGHTS`; the agent then reads, writes, and resizes that +/// master directly. This is the only way to make `TIOCSWINSZ` reach the +/// process inside the container. Without `--console-socket` crun +/// allocates its own PTY that the agent has no handle on, and every +/// resize message is silently applied to the wrong terminal. See GH +/// #156. +/// Establish the long-lived "main" container for a persistent overlay and return +/// its id. PID 1 is a keep-alive (`tail -f /dev/null`, present on both busybox and +/// coreutils) that never exits, so processes a later `crun exec` backgrounds +/// inside it survive across exec calls for the machine's lifetime. Env / workdir / +/// user are inherited from the image so exec'd commands see the right environment. +/// Uses the same two-step `crun create` + `crun start` as [`handle_run_detached`] +/// (`crun run --detach` hangs in the smolvm VM environment). +#[cfg(target_os = "linux")] +fn ensure_main_container( + rootfs: &str, + overlay_id: &str, + mounts: &[(String, String, bool)], + unprivileged: bool, + base_launch: &ResolvedLaunch, +) -> Result> { + use std::path::Path; + + let keepalive = ResolvedLaunch { + command: vec![ + "tail".to_string(), + "-f".to_string(), + "/dev/null".to_string(), + ], + env: base_launch.env.clone(), + workdir: base_launch.workdir.clone(), + user: base_launch.user.clone(), + }; + + let rootfs_path = Path::new(rootfs); + let bundle_path = rootfs_path + .parent() + .ok_or("invalid rootfs path: no parent")? + .join("bundle"); + if !bundle_path.exists() { + return Err(format!("bundle directory not found: {}", bundle_path.display()).into()); + } + + let container_id = write_oci_bundle( + rootfs_path, + &bundle_path, + &keepalive, + mounts, + false, + unprivileged, + )?; + + let create = crun::CrunCommand::create(&bundle_path, &container_id).output()?; + if !create.status.success() { + return Err(format!( + "keep-alive crun create failed: {}", + String::from_utf8_lossy(&create.stderr).trim() + ) + .into()); + } + let start = crun::CrunCommand::start(&container_id).output()?; + if !start.status.success() { + let _ = crun::CrunCommand::delete(&container_id, true).output(); + return Err(format!( + "keep-alive crun start failed: {}", + String::from_utf8_lossy(&start.stderr).trim() + ) + .into()); + } + + let workload_id = format!("persistent-{}", overlay_id); + if let Err(e) = std::fs::write( + paths::main_container_id_path(&workload_id), + container_id.as_bytes(), + ) { + let _ = crun::CrunCommand::kill(&container_id, "SIGKILL").status(); + let _ = crun::CrunCommand::delete(&container_id, true).output(); + return Err(format!("failed to persist main container id: {}", e).into()); + } + info!(container_id = %container_id, overlay_id = %overlay_id, "established keep-alive main container for persistent machine"); + Ok(container_id) +} + +#[cfg(target_os = "linux")] +#[allow(clippy::too_many_arguments)] +fn spawn_interactive_command( + rootfs: &str, + launch: &ResolvedLaunch, + mounts: &[(String, String, bool)], + tty: bool, + persistent_overlay_id: Option<&str>, + unprivileged: bool, +) -> Result<(Child, Option), Box> { + use std::io::Read as _; + use std::os::unix::io::AsRawFd as _; + use std::path::Path; + use std::sync::atomic::Ordering; + + if launch.command.is_empty() { + return Err("empty command".into()); + } + + // If a main workload container is running for this overlay, join it. + if let Some(cid) = resolve_main_container(persistent_overlay_id) { + return spawn_exec_in_container(&cid, launch, tty); + } + + // On a persistent machine with no main container yet, establish a long-lived + // keep-alive container (PID 1 = `tail -f /dev/null`) and exec the command INTO + // it, rather than running the command AS the container. Without this, PID 1 is + // the command itself, so the container — and anything the command backgrounds + // (a `dockerd`, a dev server, a k3d cluster) — is torn down the moment the + // command returns, and the next exec sees a fresh container. Joining a + // keep-alive container makes backgrounded processes survive across execs for + // the machine's lifetime. On failure, fall through to the fresh-container path + // so exec never breaks outright. + if let Some(overlay_id) = persistent_overlay_id { + match ensure_main_container(rootfs, overlay_id, mounts, unprivileged, launch) { + Ok(cid) => return spawn_exec_in_container(&cid, launch, tty), + Err(e) => { + warn!(error = %e, "keep-alive main container setup failed; running in a fresh container") + } + } + } + + let rootfs_path = Path::new(rootfs); + let overlay_root = rootfs_path + .parent() + .ok_or("invalid rootfs path: no parent")?; + let bundle_path = overlay_root.join("bundle"); + + if !bundle_path.exists() { + return Err(format!("bundle directory not found: {}", bundle_path.display()).into()); + } + + // Build the OCI bundle (config.json) and get a fresh container ID. When + // tty=true the spec sets terminal:true and a starting consoleSize, and the + // PTY master is obtained from crun via --console-socket below. + let container_id = + write_oci_bundle(rootfs_path, &bundle_path, launch, mounts, tty, unprivileged)?; + + // Persist the container ID so subsequent execs join this container. + // Written before spawn: if spawn fails the ID is stale, but + // is_container_running() will return false and the next exec starts fresh. + if let Some(overlay_id) = persistent_overlay_id { + let workload_id = format!("persistent-{}", overlay_id); + let _ = std::fs::write( + paths::main_container_id_path(&workload_id), + container_id.as_bytes(), + ); + } + + info!( + command = ?launch.command, + container_id = %container_id, + bundle = %bundle_path.display(), + mounts = mounts.len(), + tty = tty, + "spawning interactive container with crun" + ); + + if tty { + // Preferred path: take the container's console over a socket so the + // master we hold is the process's real tty (resize works). + if CONSOLE_SOCKET_WORKS.load(Ordering::Relaxed) { + let console = pty::ConsoleSocket::bind(&container_id)?; + let mut child = + crun::CrunCommand::run_with_console(&bundle_path, &container_id, console.path()) + .spawn()?; + match console.recv_master(std::time::Duration::from_secs(3)) { + Ok(pty_master) => { + let _ = pty_master.set_window_size(80, 24); + // Drain crun's stderr so a chatty runtime can't fill the pipe. + if let Some(mut err) = child.stderr.take() { + std::thread::spawn(move || { + let mut sink = Vec::new(); + let _ = err.read_to_end(&mut sink); + }); + } + return Ok((child, Some(pty_master))); + } + Err(e) => { + // A transient timeout (crun slow to hand back the console) + // must not permanently disable console sockets for the rest + // of the VM's life — that would silently lose resize on + // every later session. Only latch off when the runtime + // genuinely doesn't support them (a non-timeout failure). + if e.kind() != std::io::ErrorKind::TimedOut { + CONSOLE_SOCKET_WORKS.store(false, Ordering::Relaxed); + } + let _ = child.kill(); + let _ = child.wait(); + let stderr = child + .stderr + .take() + .map(|mut s| { + let mut buf = String::new(); + let _ = s.read_to_string(&mut buf); + buf + }) + .unwrap_or_default(); + warn!(error = %e, crun_stderr = %stderr.trim(), "console socket unavailable; falling back to stdio PTY (resize will not propagate)"); + // The failed `crun run --console-socket` may have registered + // container state under this id; clear it so the fallback + // `crun run` with the same id doesn't hit "already exists". + let _ = crun::CrunCommand::delete(&container_id, true).output(); + } + } + } + + // Fallback: attach the agent's PTY slave as crun's stdio. + let (pty_master, slave_fd) = pty::open_pty(80, 24)?; + let slave_raw = slave_fd.as_raw_fd(); + // SAFETY: slave_fd is a valid open fd from openpty. + let child = unsafe { + crun::CrunCommand::run(&bundle_path, &container_id) + .stdin_from_fd(libc::dup(slave_raw)) + .stdout_from_fd(libc::dup(slave_raw)) + .stderr_from_fd(libc::dup(slave_raw)) + .spawn()? + }; + drop(slave_fd); + Ok((child, Some(pty_master))) + } else { + let child = crun::CrunCommand::run(&bundle_path, &container_id) + .stdin_piped() + .capture_output() + .spawn()?; + Ok((child, None)) + } +} + +/// Non-Linux stub for spawn_interactive_command. +#[cfg(not(target_os = "linux"))] +fn spawn_interactive_command( + rootfs: &str, + launch: &ResolvedLaunch, + mounts: &[(String, String, bool)], + _tty: bool, + _persistent_overlay_id: Option<&str>, + unprivileged: bool, +) -> Result<(Child, Option<()>), Box> { + use std::path::Path; + + let command: &[String] = &launch.command; + let env: &[(String, String)] = &launch.env; + let workdir: Option<&str> = launch.workdir.as_deref(); + let user: Option<&str> = launch.user.as_deref(); + + if command.is_empty() { + return Err("empty command".into()); + } + + let rootfs_path = Path::new(rootfs); + let overlay_root = rootfs_path + .parent() + .ok_or("invalid rootfs path: no parent")?; + let bundle_path = overlay_root.join("bundle"); + + if !bundle_path.exists() { + return Err(format!("bundle directory not found: {}", bundle_path.display()).into()); + } + + let workdir_str = workdir.unwrap_or("/"); + let identity = oci::resolve_process_identity(rootfs_path, user)?; + let mut spec = oci::OciSpec::new(command, env, workdir_str, false, &identity, unprivileged); + spec.add_gpu_devices_if_available(); + + for (tag, container_path, read_only) in mounts { + let virtiofs_mount = Path::new(paths::VIRTIOFS_MOUNT_ROOT).join(tag); + spec.add_bind_mount( + &virtiofs_mount.to_string_lossy(), + container_path, + *read_only, + ); + } + + storage::add_workspace_fallback(&mut spec, mounts); + storage::add_storage_fallback(&mut spec, mounts, unprivileged); + + // Forward SSH agent into the container if enabled at boot. + ssh_agent::inject_into_container(&mut spec); + rosetta::inject_into_container(&mut spec); + cuda::inject_into_container(&mut spec, rootfs_path); + + spec.write_to(&bundle_path) + .map_err(|e| format!("failed to write OCI spec: {}", e))?; + + let container_id = oci::generate_container_id(); + + let child = crun::CrunCommand::run(&bundle_path, &container_id) + .stdin_piped() + .capture_output() + .spawn()?; + + Ok((child, None)) +} + +/// Run the interactive I/O loop using poll() for efficient I/O multiplexing. +/// Kill a child process and return a timeout exit code. Used when the host +/// disconnects during an interactive exec — the agent must clean up the +/// child and continue accepting new connections rather than propagating +/// the I/O error. +fn kill_child_on_disconnect(child: &mut Child) -> i32 { + let _ = child.kill(); + let _ = child.wait(); + 124 +} + +fn run_interactive_loop( + stream: &mut impl ReadWrite, + child: &mut Child, + timeout_ms: Option, +) -> Result> { + use std::io::Read as _; + use std::time::{Duration, Instant}; + + let start = Instant::now(); + let deadline = timeout_ms.map(|ms| start + Duration::from_millis(ms)); + + // Get handles to child's stdio + let mut child_stdout = child.stdout.take(); + let mut child_stderr = child.stderr.take(); + let mut child_stdin = child.stdin.take(); + + // Set non-blocking mode on stdout/stderr + if let Some(ref stdout) = child_stdout { + if !set_nonblocking(stdout.as_raw_fd()) { + warn!("failed to set stdout to non-blocking mode"); + } + } + if let Some(ref stderr) = child_stderr { + if !set_nonblocking(stderr.as_raw_fd()) { + warn!("failed to set stderr to non-blocking mode"); + } + } + + let mut stdout_buf = [0u8; IO_BUFFER_SIZE]; + let mut stderr_buf = [0u8; IO_BUFFER_SIZE]; + + loop { + // Check if child has exited + if let Some(status) = child.try_wait()? { + // Drain any remaining output + drain_remaining_output( + stream, + &mut child_stdout, + &mut child_stderr, + &mut stdout_buf, + &mut stderr_buf, + )?; + return Ok(process::exit_code_from_status(&status)); + } + + // Check timeout + if let Some(deadline) = deadline { + if Instant::now() >= deadline { + warn!("interactive command timed out, killing process"); + if let Err(e) = child.kill() { + warn!(error = %e, "failed to kill timed out process"); + } + // Wait to reap the process and avoid zombies + if let Err(e) = child.wait() { + warn!(error = %e, "failed to wait for killed process"); + } + return Ok(124); // Timeout exit code + } + } + + // Calculate poll timeout: either remaining time until deadline, or 100ms default + let poll_timeout_ms = match deadline { + Some(dl) => { + let remaining = dl.saturating_duration_since(Instant::now()); + // Cap at 100ms to periodically check child exit status + remaining + .as_millis() + .min(INTERACTIVE_POLL_TIMEOUT_MS as u128) as i32 + } + None => INTERACTIVE_POLL_TIMEOUT_MS, + }; + + // Build poll fds array for stdout, stderr, and vsock stream + let stdout_fd = child_stdout.as_ref().map(|s| s.as_raw_fd()).unwrap_or(-1); + let stderr_fd = child_stderr.as_ref().map(|s| s.as_raw_fd()).unwrap_or(-1); + let stream_fd = stream.as_raw_fd(); + + let mut poll_fds = [ + libc::pollfd { + fd: stdout_fd, + events: if stdout_fd >= 0 { libc::POLLIN } else { 0 }, + revents: 0, + }, + libc::pollfd { + fd: stderr_fd, + events: if stderr_fd >= 0 { libc::POLLIN } else { 0 }, + revents: 0, + }, + libc::pollfd { + fd: stream_fd, + events: libc::POLLIN, + revents: 0, + }, + ]; + + // Wait for I/O or timeout using poll() + let poll_result = unsafe { libc::poll(poll_fds.as_mut_ptr(), 3, poll_timeout_ms) }; + + if poll_result < 0 { + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::Interrupted { + debug!(error = %err, "poll error"); + } + continue; + } + + // Read available stdout. If send_response fails (host disconnected), + // kill the child and return gracefully. + if poll_fds[0].revents & libc::POLLIN != 0 { + if let Some(ref mut stdout) = child_stdout { + loop { + match stdout.read(&mut stdout_buf) { + Ok(0) => break, + Ok(n) => { + if send_response( + stream, + &AgentResponse::Stdout { + data: stdout_buf[..n].to_vec(), + }, + ) + .is_err() + { + debug!("host disconnected while sending stdout"); + return Ok(kill_child_on_disconnect(child)); + } + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break, + Err(e) => { + debug!(error = %e, "stdout read error"); + break; + } + } + } + } + } + + // Read available stderr. Same disconnection handling as stdout. + if poll_fds[1].revents & libc::POLLIN != 0 { + if let Some(ref mut stderr) = child_stderr { + loop { + match stderr.read(&mut stderr_buf) { + Ok(0) => break, + Ok(n) => { + if send_response( + stream, + &AgentResponse::Stderr { + data: stderr_buf[..n].to_vec(), + }, + ) + .is_err() + { + debug!("host disconnected while sending stderr"); + return Ok(kill_child_on_disconnect(child)); + } + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break, + Err(e) => { + debug!(error = %e, "stderr read error"); + break; + } + } + } + } + } + + // Read incoming request from host (stdin data, resize) — only when + // poll confirms data is available, then use blocking read_exact which + // is safe because the data is already in the kernel buffer. + // + // If the host disconnects (client killed, timeout), read_exact returns + // an error. In that case, kill the child and return gracefully — the + // agent must survive client disconnections. + if poll_fds[2].revents & (libc::POLLIN | libc::POLLHUP) != 0 { + let mut header = [0u8; 4]; + if let Err(e) = stream.read_exact(&mut header) { + debug!(error = %e, "host disconnected during interactive exec"); + return Ok(kill_child_on_disconnect(child)); + } + let len = u32::from_be_bytes(header) as usize; + if len > MAX_MESSAGE_SIZE { + return Err(format!("message too large: {} bytes", len).into()); + } + let mut buf = vec![0u8; len]; + if let Err(e) = stream.read_exact(&mut buf) { + debug!(error = %e, "host disconnected during interactive exec payload"); + return Ok(kill_child_on_disconnect(child)); + } + let request: AgentRequest = serde_json::from_slice(&buf)?; + + match request { + AgentRequest::Stdin { data } => { + if data.is_empty() { + drop(child_stdin.take()); + } else if let Some(ref mut stdin) = child_stdin { + let _ = stdin.write_all(&data); + let _ = stdin.flush(); + } + } + AgentRequest::Resize { cols, rows } => { + debug!(cols, rows, "resize requested (no PTY in pipe mode)"); + } + _ => { + warn!("unexpected request during interactive session"); + } + } + } + } +} + +/// Run the interactive I/O loop for PTY-based sessions. +/// +/// Unlike `run_interactive_loop`, this polls a single PTY master fd +/// (PTY merges stdout and stderr) and supports terminal resize. +#[cfg(target_os = "linux")] +fn run_interactive_loop_pty( + stream: &mut impl ReadWrite, + child: &mut Child, + pty_master: pty::PtyMaster, + timeout_ms: Option, +) -> Result> { + use std::time::{Duration, Instant}; + + let start = Instant::now(); + let deadline = timeout_ms.map(|ms| start + Duration::from_millis(ms)); + + // Set the master fd to non-blocking so we can poll it. + if !set_nonblocking(pty_master.as_raw_fd()) { + warn!("failed to set PTY master to non-blocking mode"); + } + + let mut buf = [0u8; IO_BUFFER_SIZE]; + + loop { + // Check if child has exited. + if let Some(status) = child.try_wait()? { + // Drain remaining PTY output. + loop { + match pty_master.read(&mut buf) { + Ok(0) => break, + Ok(n) => { + send_response( + stream, + &AgentResponse::Stdout { + data: buf[..n].to_vec(), + }, + )?; + } + Err(e) + if e.kind() == std::io::ErrorKind::WouldBlock + || e.raw_os_error() == Some(libc::EIO) => + { + // EIO is expected when the slave side is closed. + break; + } + Err(_) => break, + } + } + return Ok(process::exit_code_from_status(&status)); + } + + // Check timeout. + if let Some(deadline) = deadline { + if Instant::now() >= deadline { + warn!("interactive PTY command timed out, killing process"); + if let Err(e) = child.kill() { + warn!(error = %e, "failed to kill timed out process"); + } + if let Err(e) = child.wait() { + warn!(error = %e, "failed to wait for killed process"); + } + return Ok(124); + } + } + + // Poll the PTY master fd for readable data. + let poll_timeout_ms = match deadline { + Some(dl) => { + let remaining = dl.saturating_duration_since(Instant::now()); + remaining + .as_millis() + .min(INTERACTIVE_POLL_TIMEOUT_MS as u128) as i32 + } + None => INTERACTIVE_POLL_TIMEOUT_MS, + }; + + // Poll PTY master and vsock stream for readable data. + let stream_fd = stream.as_raw_fd(); + let mut poll_fds = [ + libc::pollfd { + fd: pty_master.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }, + libc::pollfd { + fd: stream_fd, + events: libc::POLLIN, + revents: 0, + }, + ]; + + let poll_result = unsafe { libc::poll(poll_fds.as_mut_ptr(), 2, poll_timeout_ms) }; + + if poll_result < 0 { + let err = std::io::Error::last_os_error(); + if err.kind() != std::io::ErrorKind::Interrupted { + debug!(error = %err, "poll error on PTY master"); + } + continue; + } + + // Read available data from PTY master. If send_response fails + // (host disconnected), kill the child and return gracefully. + let mut slave_closed = false; + if poll_fds[0].revents & (libc::POLLIN | libc::POLLHUP) != 0 { + loop { + match pty_master.read(&mut buf) { + Ok(0) => break, + Ok(n) => { + if send_response( + stream, + &AgentResponse::Stdout { + data: buf[..n].to_vec(), + }, + ) + .is_err() + { + debug!("host disconnected while sending PTY stdout"); + return Ok(kill_child_on_disconnect(child)); + } + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break, + Err(e) if e.raw_os_error() == Some(libc::EIO) => { + // Slave side closed — child is exiting. Reap immediately + // instead of waiting for the next poll cycle. + slave_closed = true; + break; + } + Err(e) => { + debug!(error = %e, "PTY master read error"); + slave_closed = true; + break; + } + } + } + } + + // If the slave closed, the process is exiting — reap it now. + if slave_closed { + let status = child.wait()?; + return Ok(process::exit_code_from_status(&status)); + } + + // Read incoming request from host — only when poll confirms data + // is available, then use blocking read_exact (safe, data is buffered). + // If the host disconnects, kill the child and return gracefully. + if poll_fds[1].revents & (libc::POLLIN | libc::POLLHUP) != 0 { + let mut header = [0u8; 4]; + if let Err(e) = stream.read_exact(&mut header) { + debug!(error = %e, "host disconnected during PTY interactive exec"); + return Ok(kill_child_on_disconnect(child)); + } + let len = u32::from_be_bytes(header) as usize; + if len > MAX_MESSAGE_SIZE { + return Err(format!("message too large: {} bytes", len).into()); + } + let mut msg_buf = vec![0u8; len]; + if let Err(e) = stream.read_exact(&mut msg_buf) { + debug!(error = %e, "host disconnected during PTY interactive exec payload"); + return Ok(kill_child_on_disconnect(child)); + } + let request: AgentRequest = serde_json::from_slice(&msg_buf)?; + + match request { + AgentRequest::Stdin { data } => { + if data.is_empty() { + // Host stdin reached EOF. A PTY cannot have one + // direction closed, so signal end-of-input to the + // child by writing the EOF control character (VEOF, + // Ctrl-D / 0x04). In canonical mode VEOF makes the + // pending line available to the child's read() + // immediately; a read() that finds an empty line + // buffer returns 0, which is the EOF the child waits + // for. Without it a stdin-reading child (cat, sh, + // read) never terminates and the exec session hangs. + // + // Send it twice: if the host's final stdin chunk was + // not newline-terminated, the first VEOF only flushes + // that partial line (delivered as data, not EOF), so a + // second VEOF — now at an empty line buffer — is what + // produces the zero-length read. When the buffer is + // already empty the first VEOF yields EOF and the + // second is harmlessly consumed after the child exits. + let _ = pty_master.write_all(&[0x04, 0x04]); + } else { + let _ = pty_master.write_all(&data); + } + } + AgentRequest::Resize { cols, rows } => { + if let Err(e) = pty_master.set_window_size(cols, rows) { + debug!(error = %e, cols, rows, "failed to set PTY window size"); + } + } + _ => { + warn!("unexpected request during interactive PTY session"); + } + } + } + } +} + +/// Drain any remaining output from stdout/stderr after child exits. +fn drain_remaining_output( + stream: &mut impl Write, + child_stdout: &mut Option, + child_stderr: &mut Option, + stdout_buf: &mut [u8], + stderr_buf: &mut [u8], +) -> Result<(), Box> { + use std::io::Read as _; + + if let Some(ref mut stdout) = child_stdout { + loop { + match stdout.read(stdout_buf) { + Ok(0) => break, + Ok(n) => { + send_response( + stream, + &AgentResponse::Stdout { + data: stdout_buf[..n].to_vec(), + }, + )?; + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break, + Err(_) => break, + } + } + } + if let Some(ref mut stderr) = child_stderr { + loop { + match stderr.read(stderr_buf) { + Ok(0) => break, + Ok(n) => { + send_response( + stream, + &AgentResponse::Stderr { + data: stderr_buf[..n].to_vec(), + }, + )?; + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break, + Err(_) => break, + } + } + } + Ok(()) +} + +/// Set a file descriptor to non-blocking mode. +/// +/// Returns true if successful, false if fcntl() failed. +fn set_nonblocking(fd: i32) -> bool { + unsafe { + let flags = libc::fcntl(fd, libc::F_GETFL); + if flags < 0 { + debug!(fd, "fcntl(F_GETFL) failed"); + return false; + } + let result = libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK); + if result < 0 { + debug!(fd, "fcntl(F_SETFL, O_NONBLOCK) failed"); + return false; + } + true + } +} + +/// Extract host:port from a URL for TCP connection testing. +/// +/// Supports URLs like: +/// - `http://example.com` -> `example.com:80` +/// - `https://example.com` -> `example.com:443` +/// - `http://example.com:8080` -> `example.com:8080` +/// - `example.com:80` -> `example.com:80` +fn extract_host_port(url: &str) -> Option { + // Remove protocol prefix if present + let without_proto = url + .strip_prefix("http://") + .or_else(|| url.strip_prefix("https://")) + .unwrap_or(url); + + // Extract host (remove path) + let host_port = without_proto.split('/').next()?; + + // If no port, add default based on protocol + if host_port.contains(':') { + Some(host_port.to_string()) + } else if url.starts_with("https://") { + Some(format!("{}:443", host_port)) + } else { + Some(format!("{}:80", host_port)) + } +} + +/// Test TCP connection using pure syscalls (bypass C library). +/// Connects to the specified target and sends HTTP GET request. +/// +/// # Arguments +/// * `target` - Host:port to connect to (e.g., "1.1.1.1:80", "example.com:443") +fn test_tcp_syscall(target: &str) -> serde_json::Value { + use std::io::{Read as _, Write as _}; + use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; + use std::time::Duration; + + info!(target = %target, "testing TCP with pure Rust std::net"); + + // Resolve the target to socket address + let addr: SocketAddr = match target.to_socket_addrs() { + Ok(mut addrs) => match addrs.next() { + Some(addr) => addr, + None => { + return serde_json::json!({ + "success": false, + "error": "could not resolve target address", + "target": target, + }); + } + }, + Err(e) => { + return serde_json::json!({ + "success": false, + "error": format!("failed to resolve {}: {}", target, e), + "target": target, + }); + } + }; + + // Extract host for HTTP Host header + let host = target.split(':').next().unwrap_or(target); + + let connect_result = + match TcpStream::connect_timeout(&addr, Duration::from_secs(NETWORK_TEST_TIMEOUT_SECS)) { + Ok(mut stream) => { + // Try to set timeouts + let _ = + stream.set_read_timeout(Some(Duration::from_secs(NETWORK_TEST_TIMEOUT_SECS))); + let _ = + stream.set_write_timeout(Some(Duration::from_secs(NETWORK_TEST_TIMEOUT_SECS))); + + // Send a simple HTTP request + let request = format!("GET / HTTP/1.0\r\nHost: {}\r\n\r\n", host); + match stream.write_all(request.as_bytes()) { + Ok(_) => { + // Try to read the response + let mut response = vec![0u8; 1024]; + match stream.read(&mut response) { + Ok(n) => { + let response_str = + String::from_utf8_lossy(&response[..n.min(200)]).to_string(); + serde_json::json!({ + "success": true, + "connected": true, + "sent_request": true, + "received_bytes": n, + "response_preview": response_str, + }) + } + Err(e) => { + serde_json::json!({ + "success": false, + "connected": true, + "sent_request": true, + "read_error": format!("{}", e), + "read_error_kind": format!("{:?}", e.kind()), + }) + } + } + } + Err(e) => { + serde_json::json!({ + "success": false, + "connected": true, + "write_error": format!("{}", e), + }) + } + } + } + Err(e) => { + // Get more details about the error + let raw_os_error = e.raw_os_error(); + serde_json::json!({ + "success": false, + "connected": false, + "error": format!("{}", e), + "error_kind": format!("{:?}", e.kind()), + "raw_os_error": raw_os_error, + }) + } + }; + + // Also test socket syscall and lseek behavior using safe nix APIs + #[cfg(target_os = "linux")] + let socket_test = { + use nix::sys::socket::{socket, AddressFamily, SockFlag, SockType}; + use nix::unistd::{lseek, Whence}; + use std::os::fd::AsRawFd; + + match socket( + AddressFamily::Inet, + SockType::Stream, + SockFlag::empty(), + None, + ) { + Ok(fd) => { + let raw_fd = fd.as_raw_fd(); + + // Test lseek on the socket - this should return ESPIPE (29) for normal sockets + let (lseek_result, lseek_errno) = match lseek(raw_fd, 0, Whence::SeekCur) { + Ok(offset) => (offset, None), + Err(e) => (-1, Some((e as i32, e.desc().to_string()))), + }; + + // fd is automatically closed when OwnedFd drops + serde_json::json!({ + "socket_created": true, + "fd": raw_fd, + "sock_type": libc::SOCK_STREAM, // We know we created SOCK_STREAM + "lseek_result": lseek_result, + "lseek_errno": lseek_errno.map(|(e, s)| serde_json::json!({"code": e, "str": s})), + "expected_errno_espipe": 29, // ESPIPE = 29 on Linux + }) + } + Err(e) => { + serde_json::json!({ + "socket_created": false, + "errno": e as i32, + "errno_str": e.desc().to_string(), + }) + } + } + }; + + #[cfg(not(target_os = "linux"))] + let socket_test = serde_json::json!({ + "skipped": true, + "reason": "socket test only available on Linux" + }); + + // Test 3: Try nc (netcat) if available + let nc_result = match std::process::Command::new("nc") + .args(["-w", "5", "1.1.1.1", "80"]) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + { + Ok(mut child) => { + // Send HTTP request via stdin + if let Some(ref mut stdin) = child.stdin { + let _ = stdin.write_all(b"GET / HTTP/1.0\r\nHost: 1.1.1.1\r\n\r\n"); + } + drop(child.stdin.take()); + + match child.wait_with_output() { + Ok(output) => { + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + serde_json::json!({ + "tool": "nc", + "success": output.status.success(), + "exit_code": output.status.code(), + "stdout_preview": stdout.chars().take(200).collect::(), + "stderr": stderr.to_string(), + }) + } + Err(e) => serde_json::json!({ + "tool": "nc", + "error": format!("wait error: {}", e), + }), + } + } + Err(e) => serde_json::json!({ + "tool": "nc", + "error": format!("spawn error: {}", e), + }), + }; + + // Test 4: Try curl if available + let curl_result = match std::process::Command::new("curl") + .args([ + "-s", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--connect-timeout", + "10", + "http://1.1.1.1", + ]) + .output() + { + Ok(output) => { + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + serde_json::json!({ + "tool": "curl", + "success": output.status.success(), + "exit_code": output.status.code(), + "http_code": stdout, + "stderr": stderr, + }) + } + Err(e) => serde_json::json!({ + "tool": "curl", + "error": format!("{}", e), + }), + }; + + serde_json::json!({ + "rust_std_net": connect_result, + "raw_socket": socket_test, + "nc": nc_result, + "curl": curl_result, + }) +} + +/// Handle command execution request (non-interactive). +/// Handle a background `Run` request — spawn the container and return its PID. +/// +/// Background mode requires a persistent overlay ID; an ephemeral overlay +/// would leak because nothing waits for the container to exit to clean it +/// up. The returned PID is the crun process, which stays alive as long as +/// the container's init process runs. +#[allow(clippy::too_many_arguments)] +fn handle_run_background( + image: &str, + command: &[String], + env: &[(String, String)], + workdir: Option<&str>, + user: Option<&str>, + mounts: &[(String, String, bool)], + persistent_overlay_id: Option<&str>, + unprivileged: bool, +) -> AgentResponse { + info!(image = %image, command = ?command, mounts = ?mounts, "running command in background"); + + let Some(overlay_id) = persistent_overlay_id else { + return AgentResponse::error( + "background run requires persistent_overlay_id", + error_codes::INVALID_REQUEST, + ); + }; + + match storage::spawn_in_overlay( + image, + command, + env, + workdir, + user, + mounts, + overlay_id, + unprivileged, + ) { + Ok(pid) => AgentResponse::Completed { + exit_code: 0, + stdout: format!("{}", pid).into_bytes(), + stderr: Vec::new(), + }, + Err(e) => AgentResponse::from_err(e, error_codes::RUN_FAILED), + } +} + +/// Non-streaming exec/run returns the whole output in a single wire frame. If it +/// would exceed the frame budget, return a clear error instead of attempting an +/// oversized frame — which otherwise fails mid-send and surfaces to the caller as +/// a confusing connection drop / SIGPIPE-truncated result. Large output should +/// use streaming exec (`exec_stream`/`execStream`). The budget leaves headroom +/// under MAX_FRAME_SIZE for base64 of the byte fields (~4/3) plus the JSON envelope. +fn oversized_output_error(stdout: &[u8], stderr: &[u8]) -> Option { + const BUDGET: usize = 20 * 1024 * 1024; // ~20 MiB raw → ~27 MiB base64, under the 32 MiB frame + let total = stdout.len() + stderr.len(); + if total > BUDGET { + return Some(AgentResponse::error( + format!( + "command output too large ({total} bytes; limit {BUDGET}). \ + Use streaming exec (exec_stream / execStream) for large output." + ), + error_codes::EXEC_FAILED, + )); + } + None +} + +#[allow(clippy::too_many_arguments)] +/// Run a non-interactive command inside the machine's long-lived keep-alive +/// container (establishing it on first use), capturing its output. Joining the +/// keep-alive container — rather than spawning a fresh one per exec — is what +/// lets a process this command backgrounds (a `dockerd`, a dev server, a k3d +/// cluster) survive into later execs for the machine's lifetime. Returns the +/// captured result; the caller falls back to a fresh container on error. +#[cfg(target_os = "linux")] +fn run_in_keepalive_container( + overlay_id: &str, + image: &str, + command: &[String], + env: &[(String, String)], + workdir: Option<&str>, + user: Option<&str>, + mounts: &[(String, String, bool)], + unprivileged: bool, + stdin_data: Option<&str>, +) -> Result> { + use std::io::Write as _; + + let launch = ResolvedLaunch::resolve( + image, + command.to_vec(), + env.to_vec(), + workdir.map(str::to_string), + user.map(str::to_string), + )?; + + // Reuse the running keep-alive container, or establish one now so this and + // every later exec join the same container (PID 1 = `tail -f /dev/null`). + let cid = match resolve_main_container(Some(overlay_id)) { + Some(c) => c, + None => { + let prepared = storage::prepare_for_run_persistent(image, overlay_id)?; + storage::setup_mounts(&prepared.rootfs_path, mounts)?; + ensure_main_container( + &prepared.rootfs_path, + overlay_id, + mounts, + unprivileged, + &launch, + )? + } + }; + + let output = if let Some(data) = stdin_data { + let mut child = crun::CrunCommand::exec( + &cid, + &launch.env, + &launch.command, + launch.workdir.as_deref(), + false, + ) + .capture_output() + .stdin_piped() + .spawn()?; + if let Some(mut stdin) = child.stdin.take() { + let _ = stdin.write_all(data.as_bytes()); + } + child.wait_with_output()? + } else { + crun::CrunCommand::exec( + &cid, + &launch.env, + &launch.command, + launch.workdir.as_deref(), + false, + ) + .capture_output() + .stdin_null() + .output()? + }; + + Ok(AgentResponse::Completed { + exit_code: output.status.code().unwrap_or(-1), + stdout: output.stdout, + stderr: output.stderr, + }) +} + +fn handle_run( + image: &str, + command: &[String], + env: &[(String, String)], + workdir: Option<&str>, + user: Option<&str>, + mounts: &[(String, String, bool)], + timeout_ms: Option, + persistent_overlay_id: Option<&str>, + stdin_data: Option<&str>, + client_fd: Option, + unprivileged: bool, +) -> AgentResponse { + info!(image = %image, command = ?command, mounts = ?mounts, timeout_ms = ?timeout_ms, persistent = persistent_overlay_id.is_some(), stdin = stdin_data.is_some(), "running command"); + + // SSH agent forwarding: make SSH_AUTH_SOCK part of the command env so it + // survives the keep-alive `crun exec` path (#542), which runs commands with + // this env rather than the keep-alive container's own. No-op when forwarding + // is off; harmless on the fresh-container path. + let mut env = env.to_vec(); + ssh_agent::inject_into_env(&mut env); + let env = &env[..]; + + // On a persistent machine, run inside the long-lived keep-alive container so + // backgrounded processes survive across execs. Fall back to a fresh container + // if the keep-alive can't be established, so exec never breaks outright. + #[cfg(target_os = "linux")] + { + if let Some(overlay_id) = persistent_overlay_id { + match run_in_keepalive_container( + overlay_id, + image, + command, + env, + workdir, + user, + mounts, + unprivileged, + stdin_data, + ) { + Ok(resp) => return resp, + Err(e) => { + warn!(error = %e, "keep-alive exec failed; running in a fresh container") + } + } + } + } + + match storage::run_command( + image, + command, + env, + workdir, + user, + mounts, + timeout_ms, + persistent_overlay_id, + stdin_data, + client_fd, + unprivileged, + ) { + Ok(result) => oversized_output_error(&result.stdout, &result.stderr).unwrap_or( + AgentResponse::Completed { + exit_code: result.exit_code, + stdout: result.stdout, + stderr: result.stderr, + }, + ), + Err(e) => AgentResponse::from_err(e, error_codes::RUN_FAILED), + } +} + +/// Handle image pull request with progress streaming. +fn handle_streaming_pull( + stream: &mut S, + image: &str, + oci_platform: Option<&str>, + auth: Option<&RegistryAuth>, + proxy: Option<&str>, + no_proxy: Option<&str>, +) -> Result<(), Box> { + ensure_storage_mounted(); + info!( + image = %image, + ?oci_platform, + has_auth = auth.is_some(), + has_proxy = proxy.is_some(), + "pulling image with progress" + ); + + // Create a progress callback that sends updates over the stream + let progress_callback = |current: usize, total: usize, layer: &str| { + let percent = if total > 0 { + ((current as f64 / total as f64) * 100.0) as u8 + } else { + 0 + }; + let response = AgentResponse::Progress { + message: format!("Pulling layer {}/{}", current, total), + percent: Some(percent), + layer: Some(layer.to_string()), + }; + // Ignore errors from progress updates - non-critical + let _ = send_response(stream, &response); + }; + + let response = AgentResponse::from_result( + storage::pull_image_with_progress_and_auth( + image, + oci_platform, + auth, + proxy, + no_proxy, + progress_callback, + ), + error_codes::PULL_FAILED, + ); + + send_response(stream, &response) +} + +/// Handle image query request. +fn handle_query(image: &str) -> AgentResponse { + match storage::query_image(image) { + Ok(Some(info)) => AgentResponse::ok_with_data(info), + Ok(None) => AgentResponse::error( + format!("image not found: {}", image), + error_codes::NOT_FOUND, + ), + Err(e) => AgentResponse::from_err(e, error_codes::QUERY_FAILED), + } +} + +/// Handle list images request. +fn handle_list_images() -> AgentResponse { + AgentResponse::from_result(storage::list_images(), error_codes::LIST_FAILED) +} + +/// Handle garbage collection request. +fn handle_gc(dry_run: bool, purge_all: bool) -> AgentResponse { + if purge_all && !dry_run { + if let Err(e) = storage::purge_all_images() { + return AgentResponse::from_err(e, error_codes::GC_FAILED); + } + } + match storage::garbage_collect(dry_run) { + Ok(freed) => AgentResponse::ok_with_data(serde_json::json!({ + "freed_bytes": freed, + "dry_run": dry_run, + })), + Err(e) => AgentResponse::from_err(e, error_codes::GC_FAILED), + } +} + +/// Handle overlay preparation request. +fn handle_prepare_overlay(image: &str, workload_id: &str) -> AgentResponse { + info!(image = %image, workload_id = %workload_id, "preparing overlay"); + AgentResponse::from_result( + storage::prepare_overlay(image, workload_id), + error_codes::OVERLAY_FAILED, + ) +} + +/// Handle overlay cleanup request. +fn handle_cleanup_overlay(workload_id: &str) -> AgentResponse { + info!(workload_id = %workload_id, "cleaning up overlay"); + match storage::cleanup_overlay(workload_id) { + Ok(_) => AgentResponse::ok(None), + Err(e) => AgentResponse::from_err(e, error_codes::CLEANUP_FAILED), + } +} + +/// Handle storage format request. +fn handle_format_storage() -> AgentResponse { + info!("formatting storage"); + match storage::format() { + Ok(_) => AgentResponse::ok(None), + Err(e) => AgentResponse::from_err(e, error_codes::FORMAT_FAILED), + } +} + +/// Handle export layer request with chunked streaming. +/// +/// Pipes `tar -cf -` stdout directly to the vsock stream in LAYER_CHUNK_SIZE +/// chunks. No temp tar file is created — this allows exporting layers of any +/// size without filling the storage disk. +fn handle_streaming_export_layer( + stream: &mut impl Write, + image_digest: &str, + layer_index: usize, +) -> Result<(), Box> { + ensure_storage_mounted(); + info!(image_digest = %image_digest, layer_index = layer_index, "exporting layer (streamed)"); + + // Find the layer directory without creating a temp tar file. + let layer_dir = match storage::find_layer_path(image_digest, layer_index) { + Ok(path) => path, + Err(e) => { + send_response( + stream, + &AgentResponse::from_err(e, error_codes::EXPORT_FAILED), + )?; + return Ok(()); + } + }; + + // Pipe tar stdout directly — no temp file on disk. + let mut child = match std::process::Command::new("tar") + .args(["-cf", "-", "-C"]) + .arg(&layer_dir) + .arg(".") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn() + { + Ok(c) => c, + Err(e) => { + send_response( + stream, + &AgentResponse::error( + format!("failed to spawn tar: {}", e), + error_codes::EXPORT_FAILED, + ), + )?; + return Ok(()); + } + }; + + let mut stdout = child.stdout.take().unwrap(); + + // Shared streaming path — same helper used by FileRead. + let result = send_data_chunks( + stream, + &mut stdout, + LAYER_CHUNK_SIZE, + "failed to read tar output", + error_codes::EXPORT_FAILED, + ); + // If the helper returned an Err, it already sent an Error + // response; we still need to clean up the tar subprocess. + if result.is_err() { + let _ = child.kill(); + } + let _ = child.wait(); + result +} + +/// Handle storage status request. +fn handle_storage_status() -> AgentResponse { + AgentResponse::from_result(storage::status(), error_codes::STATUS_FAILED) +} + +// ============================================================================ +// VM-Level Exec Handlers (Direct Execution in VM) +// ============================================================================ + +/// PIDs of background children the agent owns and must reap. +/// +/// Populated by [`register_background_child`] when a background-mode +/// handler (`handle_vm_exec_background`, `handle_run_background`) +/// spawns + forgets a process. Cleared by [`reap_background_children`]. +/// +/// Scoping to known PIDs is required once the accept loop is +/// multi-threaded: an unscoped `waitpid(-1, WNOHANG)` would steal the +/// exit status from *any* exited child — including the foreground +/// crun processes that per-request handlers are actively waiting on — +/// and produce ECHILD races under concurrent load. +static BG_CHILDREN: OnceLock>> = OnceLock::new(); + +fn bg_children() -> &'static std::sync::Mutex> { + BG_CHILDREN.get_or_init(|| std::sync::Mutex::new(Vec::new())) +} + +/// Track a PID so a later [`reap_background_children`] waits on it. +/// +/// Callers should pair this with `std::mem::forget(child)` so the +/// Rust `Child` doesn't also race to reap the process on drop. +fn register_background_child(pid: u32) { + bg_children().lock().unwrap().push(pid); +} + +/// Reap any exited background children to prevent zombie accumulation. +/// +/// Called periodically in the accept loop. Walks the registered PID +/// list and issues a per-PID `waitpid(..., WNOHANG)` — non-blocking +/// and scoped, so it never steals exit statuses from foreground +/// handlers running in sibling threads. +#[cfg(target_os = "linux")] +fn reap_background_children() { + let mut guard = bg_children().lock().unwrap(); + guard.retain(|&pid| { + let ret = unsafe { libc::waitpid(pid as i32, std::ptr::null_mut(), libc::WNOHANG) }; + match ret { + // >0 = child was reaped; drop from tracking. + r if r > 0 => { + debug!(pid, "reaped background child"); + false + } + // 0 = still running; keep tracking for the next sweep. + 0 => true, + // <0 = error (typically ECHILD — already reaped elsewhere or the + // PID was detached in a way we don't own). Drop either way. + _ => false, + } + }); +} + +#[cfg(not(target_os = "linux"))] +fn reap_background_children() {} + +/// Handle background VM exec — spawn and return PID immediately. +/// +/// The process runs detached from the agent's control. stdout/stderr +/// go to /dev/null. Zombie children are reaped by reap_background_children() +/// in the accept loop. +fn handle_vm_exec_background( + command: &[String], + env: &[(String, String)], + workdir: Option<&str>, +) -> AgentResponse { + info!(command = ?command, "background VM exec"); + + if command.is_empty() { + return AgentResponse::error("command cannot be empty", error_codes::INVALID_REQUEST); + } + + let mut cmd = Command::new(&command[0]); + cmd.args(&command[1..]); + + for (key, value) in env { + cmd.env(key, value); + } + if let Some(wd) = workdir { + cmd.current_dir(wd); + } + + // Detach: stdout/stderr to /dev/null so the process doesn't block on pipe writes + cmd.stdout(Stdio::null()); + cmd.stderr(Stdio::null()); + cmd.stdin(Stdio::null()); + + match cmd.spawn() { + Ok(child) => { + let pid = child.id(); + // Don't wait — let the child run independently. The Rust + // `Child` is forgotten so drop doesn't race our reaper, and + // the PID is registered so reap_background_children() + // collects the eventual exit status. + std::mem::forget(child); + register_background_child(pid); + info!(pid = pid, "background process started"); + AgentResponse::Completed { + exit_code: 0, + stdout: format!("{}", pid).into_bytes(), + stderr: Vec::new(), + } + } + Err(e) => AgentResponse::error( + format!("failed to spawn background command: {}", e), + error_codes::SPAWN_FAILED, + ), + } +} + +fn handle_vm_exec( + command: &[String], + env: &[(String, String)], + workdir: Option<&str>, + timeout_ms: Option, + client_fd: Option, + stdin_data: Option<&str>, +) -> AgentResponse { + info!(command = ?command, "executing directly in VM"); + + if command.is_empty() { + return AgentResponse::error("command cannot be empty", error_codes::INVALID_REQUEST); + } + + let mut cmd = Command::new(&command[0]); + cmd.args(&command[1..]); + + // Set environment variables + for (key, value) in env { + cmd.env(key, value); + } + + // Set working directory + if let Some(wd) = workdir { + cmd.current_dir(wd); + } + + // If stdin data is provided, pipe it to the command. + // Otherwise, give the command immediate EOF via /dev/null. + if stdin_data.is_some() { + cmd.stdin(Stdio::piped()); + } else { + cmd.stdin(Stdio::null()); + } + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + // Put the child in its own process group so we can signal the whole + // tree (e.g., `sh -c 'sleep 30'` — killing sh alone leaves sleep + // orphaned and holding the stdout pipe, blocking reader threads). + #[cfg(target_os = "linux")] + { + use std::os::unix::process::CommandExt; + unsafe { + cmd.pre_exec(|| { + // setsid creates a new session and process group rooted at + // this child. killpg(pgid, SIGKILL) later hits all descendants. + if libc::setsid() < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + }); + } + } + + // Spawn the command + let mut child = match cmd.spawn() { + Ok(child) => child, + Err(e) => { + return AgentResponse::error( + format!("failed to spawn command: {}", e), + error_codes::SPAWN_FAILED, + ); + } + }; + + // Drain stdout and stderr concurrently in background threads to prevent + // pipe deadlock. Without this, a child writing >64KB to stderr blocks on + // write() while the agent blocks waiting for the child to exit — neither + // side makes progress. See docs/exec-streaming-unification.md for the + // long-term fix (streaming exec). + const MAX_OUTPUT: usize = crate::process::MAX_EXEC_OUTPUT; + + // Use read_to_end (not read_to_string) so binary output (image bytes, + // tarballs, any non-UTF-8 data) is preserved through the protocol. + // The protocol serializes Vec as base64 JSON string. + let stdout_handle = child.stdout.take().map(|out| { + std::thread::Builder::new() + .name("exec-stdout".into()) + .spawn(move || { + // Read ONE byte past the cap so the handler can tell "exactly at + // cap" from "overflowed" and return a clear error instead of a + // silently truncated result (+ a SIGPIPE exit on the child). + let mut buf = Vec::new(); + let _ = out.take(MAX_OUTPUT as u64 + 1).read_to_end(&mut buf); + buf + }) + }); + + let stderr_handle = child.stderr.take().map(|err| { + std::thread::Builder::new() + .name("exec-stderr".into()) + .spawn(move || { + let mut buf = Vec::new(); + let _ = err.take(MAX_OUTPUT as u64 + 1).read_to_end(&mut buf); + buf + }) + }); + + // Write stdin on a separate thread after stdout/stderr drains are live. + // This keeps the timeout/disconnect loop below active even when the child + // never reads stdin and the pipe buffer fills. + let stdin_handle = stdin_data.and_then(|data| { + child.stdin.take().map(|mut child_stdin| { + let data = data.to_owned(); + std::thread::Builder::new() + .name("exec-stdin".into()) + .spawn(move || { + use std::io::Write; + child_stdin.write_all(data.as_bytes()) + // child_stdin is dropped here, closing the pipe → child sees EOF. + }) + }) + }); + + // Wait for exit with timeout + let deadline = + timeout_ms.map(|ms| std::time::Instant::now() + std::time::Duration::from_millis(ms)); + + let exit_code = loop { + match child.try_wait() { + Ok(Some(status)) => break process::exit_code_from_status(&status), + Ok(None) => { + // Client disconnected — kill the orphan child so the accept + // loop isn't blocked waiting for it. Fixes BUG-12/20: SIGTERM + // on the host-side exec client used to leave the agent stuck. + if let Some(fd) = client_fd { + if process::is_peer_closed(fd) { + warn!( + pid = child.id(), + "client disconnected during VM exec, killing child group" + ); + // Kill the entire process group — `sh -c 'sleep 30'` + // creates child processes that inherit the stdout pipe. + // Killing just `sh` leaves `sleep` holding the pipe, + // blocking the reader threads' EOF. killpg hits them all. + #[cfg(target_os = "linux")] + unsafe { + libc::killpg(child.id() as libc::pid_t, libc::SIGKILL); + } + let _ = child.kill(); + let _ = child.wait(); + break 129; // SIGHUP convention: killed by disconnect + } + } + if let Some(deadline) = deadline { + if std::time::Instant::now() >= deadline { + warn!("VM exec command timed out, killing process group"); + #[cfg(target_os = "linux")] + unsafe { + libc::killpg(child.id() as libc::pid_t, libc::SIGKILL); + } + let _ = child.kill(); + let _ = child.wait(); + break 124; // Standard timeout exit code + } + } + std::thread::sleep(std::time::Duration::from_millis(PROCESS_POLL_INTERVAL_MS)); + } + Err(e) => { + return AgentResponse::error( + format!("failed to check process status: {}", e), + error_codes::WAIT_FAILED, + ); + } + } + }; + + // Join reader threads — they return EOF because the child and all its + // descendants in the process group have been killed (pipes closed). + let stdout = stdout_handle + .and_then(|h| h.ok()) + .and_then(|h| h.join().ok()) + .unwrap_or_default(); + let stderr = stderr_handle + .and_then(|h| h.ok()) + .and_then(|h| h.join().ok()) + .unwrap_or_default(); + + if let Some(Ok(handle)) = stdin_handle { + if handle.is_finished() { + match handle.join() { + Ok(Ok(())) => {} + Ok(Err(e)) if e.kind() == std::io::ErrorKind::BrokenPipe => {} + Ok(Err(e)) => debug!(error = %e, "stdin writer finished with error"), + Err(_) => debug!("stdin writer thread panicked"), + } + } else { + debug!("stdin writer still blocked after command exit; detaching"); + } + } + + // If either stream exceeded the cap (we read one byte past it), the output + // was truncated and the child likely took a SIGPIPE — return a clear error + // instead of a silently-truncated success. Large output should stream. + if stdout.len() > MAX_OUTPUT || stderr.len() > MAX_OUTPUT { + return AgentResponse::error( + format!( + "command output exceeded {MAX_OUTPUT} bytes and was truncated. \ + Use streaming exec (exec_stream / execStream) for large output." + ), + error_codes::EXEC_FAILED, + ); + } + AgentResponse::Completed { + exit_code, + stdout, + stderr, + } +} + +/// Handle interactive VM-level exec with streaming I/O. +fn handle_interactive_vm_exec( + stream: &mut impl ReadWrite, + request: AgentRequest, +) -> Result<(), Box> { + let (command, env, workdir, timeout_ms, tty) = match request { + AgentRequest::VmExec { + command, + env, + workdir, + timeout_ms, + tty, + .. + } => (command, env, workdir, timeout_ms, tty), + _ => { + send_response( + stream, + &AgentResponse::error("expected VmExec request", error_codes::INVALID_REQUEST), + )?; + return Ok(()); + } + }; + + info!(command = ?command, tty = tty, "starting interactive VM exec"); + + if command.is_empty() { + send_response( + stream, + &AgentResponse::error("command cannot be empty", error_codes::INVALID_REQUEST), + )?; + return Ok(()); + } + + // Spawn the command directly + let (mut child, pty_master) = + match spawn_direct_interactive_command(&command, &env, workdir.as_deref(), tty) { + Ok(result) => result, + Err(e) => { + send_response( + stream, + &AgentResponse::from_err(e, error_codes::SPAWN_FAILED), + )?; + return Ok(()); + } + }; + + // Send Started response + send_response(stream, &AgentResponse::Started)?; + + // Run the appropriate interactive I/O loop. Every Ok path inside the loops + // already reaps the child (try_wait auto-reaps on exit; timeout and host + // disconnect both kill+wait). The Err paths — a malformed/oversized inbound + // frame, a Stdin parse failure, a try_wait/drain error — do NOT, so reap + // here before propagating. The agent is PID 1 with no global waitpid(-1) + // reaper, so an unreaped interactive child would linger as a zombie holding + // its stdio/PTY fds. Mirrors the loop's own kill_child_on_disconnect. + let loop_result = match pty_master { + #[cfg(target_os = "linux")] + Some(pty) => run_interactive_loop_pty(stream, &mut child, pty, timeout_ms), + _ => run_interactive_loop(stream, &mut child, timeout_ms), + }; + let exit_code = match loop_result { + Ok(code) => code, + Err(e) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(e); + } + }; + + // Send Exited response + send_response(stream, &AgentResponse::Exited { exit_code })?; + + Ok(()) +} + +/// Spawn a command directly in the VM for interactive execution. +/// +/// When `tty` is true, allocates a PTY pair and attaches the slave side +/// to the child process. Returns the child and an optional `PtyMaster`. +#[cfg(target_os = "linux")] +fn spawn_direct_interactive_command( + command: &[String], + env: &[(String, String)], + workdir: Option<&str>, + tty: bool, +) -> Result<(Child, Option), Box> { + use std::os::unix::io::{AsRawFd as _, FromRawFd as _}; + use std::os::unix::process::CommandExt; + + let mut cmd = Command::new(&command[0]); + cmd.args(&command[1..]); + + for (key, value) in env { + cmd.env(key, value); + } + if let Some(wd) = workdir { + cmd.current_dir(wd); + } + + if tty { + // Allocate a PTY pair with default 80x24 size (host will send Resize). + let (pty_master, slave_fd) = pty::open_pty(80, 24)?; + let slave_raw = slave_fd.as_raw_fd(); + + // Set up stdio from the slave fd. We dup because Stdio::from_raw_fd + // takes ownership and we need the fd for all three handles + pre_exec. + // SAFETY: slave_fd is a valid open fd from openpty. + unsafe { + cmd.stdin(Stdio::from_raw_fd(libc::dup(slave_raw))); + cmd.stdout(Stdio::from_raw_fd(libc::dup(slave_raw))); + cmd.stderr(Stdio::from_raw_fd(libc::dup(slave_raw))); + } + + // SAFETY: pre_exec closure calls only async-signal-safe functions. + unsafe { + cmd.pre_exec(pty::slave_pre_exec(slave_raw)); + } + + let child = cmd.spawn()?; + + // Close the slave fd in the parent — the child has its own copies. + drop(slave_fd); + + Ok((child, Some(pty_master))) + } else { + cmd.stdin(Stdio::piped()); + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + let child = cmd.spawn()?; + Ok((child, None)) + } +} + +/// Stub for non-Linux platforms. +#[cfg(not(target_os = "linux"))] +fn spawn_direct_interactive_command( + command: &[String], + env: &[(String, String)], + workdir: Option<&str>, + _tty: bool, +) -> Result<(Child, Option<()>), Box> { + let mut cmd = Command::new(&command[0]); + cmd.args(&command[1..]); + + for (key, value) in env { + cmd.env(key, value); + } + if let Some(wd) = workdir { + cmd.current_dir(wd); + } + + cmd.stdin(Stdio::piped()); + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + + let child = cmd.spawn()?; + Ok((child, None)) +} + +/// Send a response to the client. +fn send_response( + stream: &mut impl Write, + response: &AgentResponse, +) -> Result<(), Box> { + let json = serde_json::to_vec(response)?; + let len = json.len() as u32; + + stream.write_all(&len.to_be_bytes())?; + stream.write_all(&json)?; + stream.flush()?; + + debug!(?response, "sent response"); + Ok(()) +} + +/// Trait for read+write streams with raw fd access. +trait ReadWrite: Read + Write + AsRawFd {} +impl ReadWrite for T {} + +/// Regression tests for the scoped background-child reaper. +/// +/// These are the companion to the accept-loop threading change. The bug +/// they guard against: once the accept loop spawns a thread per +/// connection, an unscoped `waitpid(-1, WNOHANG)` in the reaper steals +/// exit statuses from any foreground crun process that a sibling thread +/// is waiting on, producing ECHILD races and "command died mid-run" +/// failures. Scoped reaping must only touch PIDs registered as +/// background. +/// +/// Linux-only because `waitpid` behavior + the agent crate as a whole +/// is Linux-specific. `cargo test -p smolvm-agent --target +/// aarch64-unknown-linux-musl` on a Linux runner. +#[cfg(test)] +#[cfg(target_os = "linux")] +mod bg_reap_tests { + use super::*; + use std::process::Command; + use std::time::Duration; + + #[test] + fn reaper_leaves_unregistered_children_waitable() { + // Foreground child: the test owns it and will wait on it. The + // reaper must NOT steal it. + let mut foreground = Command::new("/bin/true") + .spawn() + .expect("spawn foreground /bin/true"); + let fg_pid = foreground.id(); + + // Background child: registered, and the Rust Child handle is + // forgotten so drop doesn't race the reaper. + let background = Command::new("/bin/true") + .spawn() + .expect("spawn background /bin/true"); + let bg_pid = background.id(); + register_background_child(bg_pid); + std::mem::forget(background); + + // Let both exit before we reap. + std::thread::sleep(Duration::from_millis(150)); + + reap_background_children(); + + // Foreground must still be reapable via the Rust Child. If the + // unscoped reaper stole it, wait() returns ECHILD and this + // expect() fires — that's exactly the concurrent-exec bug. + let status = foreground.wait().expect( + "foreground child must still be waitable — reaper must not touch unregistered PIDs", + ); + assert!(status.success(), "foreground /bin/true should succeed"); + + // Background PID should be gone from tracking (reaped). Check + // only this test's PID so parallel tests don't interfere. + let tracked = bg_children().lock().unwrap().clone(); + assert!( + !tracked.contains(&bg_pid), + "reaped bg PID {} must be removed from tracking", + bg_pid + ); + assert!( + !tracked.contains(&fg_pid), + "unregistered fg PID {} must never enter tracking", + fg_pid + ); + } + + #[test] + fn reaper_retains_still_running_background_children() { + // A registered but still-alive child must stay in tracking so a + // subsequent sweep collects it after it exits. + let mut child = Command::new("sleep") + .arg("30") + .spawn() + .expect("spawn sleep 30"); + let pid = child.id(); + register_background_child(pid); + + reap_background_children(); + + let tracked = bg_children().lock().unwrap().clone(); + assert!( + tracked.contains(&pid), + "still-running bg PID {} must remain in tracking", + pid + ); + + // Clean up so the test doesn't leak a 30-second sleep. + let _ = child.kill(); + let _ = child.wait(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Regression for `--dns` being silently dropped on the TSI backend: an + // explicit override must win over whatever is currently in resolv.conf, + // otherwise a guest on a network where 1.1.1.1/8.8.8.8 are unreachable can + // never pull an image. + #[test] + fn tsi_resolv_conf_override_wins_over_current() { + assert_eq!( + tsi_resolv_conf(Some("9.9.9.9"), "nameserver 8.8.8.8\n").as_deref(), + Some("nameserver 9.9.9.9\n") + ); + // Surrounding whitespace is trimmed rather than written into resolv.conf. + assert_eq!( + tsi_resolv_conf(Some(" 10.0.0.2 "), "").as_deref(), + Some("nameserver 10.0.0.2\n") + ); + } + + // Without an override, a valid host-written resolv.conf (e.g. the libkrun + // backend's setup_dns) must be PRESERVED, not clobbered — this is the other + // half of the same bug. + #[test] + fn tsi_resolv_conf_preserves_valid_host_written_file() { + assert_eq!(tsi_resolv_conf(None, "nameserver 9.9.9.9\n"), None); + // A blank override is treated as "no override", so preservation applies. + assert_eq!(tsi_resolv_conf(Some(" "), "nameserver 10.0.0.2\n"), None); + } + + // A stale loopback (left by a prior --allow-host run) or an empty file is + // repaired to the public resolvers. + #[test] + fn tsi_resolv_conf_repairs_stale_loopback_or_empty() { + let public = "nameserver 1.1.1.1\nnameserver 8.8.8.8\n"; + assert_eq!( + tsi_resolv_conf(None, "nameserver 127.0.0.1\n").as_deref(), + Some(public) + ); + assert_eq!(tsi_resolv_conf(None, "").as_deref(), Some(public)); + assert_eq!(tsi_resolv_conf(None, " \n").as_deref(), Some(public)); + } + + #[test] + #[cfg(unix)] + fn vm_exec_timeout_is_not_blocked_by_unread_stdin() { + let stdin_data = "x".repeat(8 * 1024 * 1024); + let start = std::time::Instant::now(); + + let response = handle_vm_exec( + &["sleep".to_string(), "5".to_string()], + &[], + None, + Some(100), + None, + Some(&stdin_data), + ); + + let AgentResponse::Completed { exit_code, .. } = response else { + panic!("expected completed response"); + }; + assert_eq!(exit_code, 124); + assert!( + start.elapsed() < std::time::Duration::from_secs(2), + "blocked stdin must not prevent timeout handling" + ); + } + + // ======================================================================== + // Streaming file-upload session tests + // + // These exercise the agent-side state machine in isolation — no + // vsock, no connection, just the handlers and the WriteSession + // struct. End-to-end protocol testing would require booting a + // real VM, which is covered by the integration harness. + // ======================================================================== + + fn tmp_target(tmp: &tempfile::TempDir, name: &str) -> std::path::PathBuf { + tmp.path().join(name) + } + + /// Collect every file in a directory whose name starts with the + /// staging prefix. Used to assert there are no orphan staging + /// files after a test runs. + fn staging_files_in(dir: &std::path::Path) -> Vec { + std::fs::read_dir(dir) + .map(|rd| { + rd.filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().contains(".smolvm-upload.")) + .map(|e| e.path()) + .collect() + }) + .unwrap_or_default() + } + + /// Decode a length-prefixed (4-byte BE) JSON response from a + /// byte slice. Returns the response and how many bytes it + /// consumed. Used by the `send_data_chunks` tests to walk the + /// stream of frames the helper wrote into a buffer. + fn pop_one_response(buf: &[u8]) -> (AgentResponse, usize) { + assert!(buf.len() >= 4, "buffer too short for length header"); + let len = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize; + assert!(buf.len() >= 4 + len, "incomplete frame in buffer"); + let resp: AgentResponse = + serde_json::from_slice(&buf[4..4 + len]).expect("decode response"); + (resp, 4 + len) + } + + #[test] + fn send_data_chunks_emits_terminator_for_empty_source() { + // Source is empty → exactly one DataChunk { data: [], done: true }. + let mut sink: Vec = Vec::new(); + let mut empty: &[u8] = &[]; + send_data_chunks( + &mut sink, + &mut empty, + 4096, + "test", + error_codes::FILE_IO_FAILED, + ) + .unwrap(); + + let (resp, consumed) = pop_one_response(&sink); + match resp { + AgentResponse::DataChunk { data, done } => { + assert!(data.is_empty()); + assert!(done); + } + other => panic!("wrong variant: {:?}", other), + } + assert_eq!(consumed, sink.len(), "exactly one frame expected"); + } + + #[test] + fn send_data_chunks_concatenates_in_order_with_done_terminator() { + // 1024 bytes through a 256-byte chunk → 4 full chunks + 1 + // empty terminator. The agent's implementation always emits a + // separate done-frame on EOF, even when EOF lands on a chunk + // boundary; the host's read_file relies on that. + let payload: Vec = (0..1024).map(|i| (i & 0xFF) as u8).collect(); + let mut sink: Vec = Vec::new(); + let mut src = std::io::Cursor::new(payload.clone()); + send_data_chunks( + &mut sink, + &mut src, + 256, + "test", + error_codes::FILE_IO_FAILED, + ) + .unwrap(); + + let mut offset = 0usize; + let mut reconstructed: Vec = Vec::new(); + let mut saw_done = false; + while offset < sink.len() { + let (resp, consumed) = pop_one_response(&sink[offset..]); + match resp { + AgentResponse::DataChunk { data, done } => { + reconstructed.extend_from_slice(&data); + if done { + saw_done = true; + } + } + other => panic!("wrong variant: {:?}", other), + } + offset += consumed; + } + assert!(saw_done, "stream missing done terminator"); + assert_eq!(reconstructed, payload); + } + + #[test] + fn send_data_chunks_partial_final_chunk_is_handled() { + // 1000 bytes through a 256-byte chunk → 3 full chunks (768 + // bytes) + 1 partial chunk (232 bytes, done: false) + 1 + // empty terminator (done: true). This separates the + // "partial chunk" case from the "EOF" case so the helper + // doesn't have to detect short reads. + let payload: Vec = (0..1000).map(|i| (i & 0xFF) as u8).collect(); + let mut sink: Vec = Vec::new(); + let mut src = std::io::Cursor::new(payload.clone()); + send_data_chunks( + &mut sink, + &mut src, + 256, + "test", + error_codes::FILE_IO_FAILED, + ) + .unwrap(); + + let mut offset = 0usize; + let mut chunks: Vec<(Vec, bool)> = Vec::new(); + while offset < sink.len() { + let (resp, consumed) = pop_one_response(&sink[offset..]); + if let AgentResponse::DataChunk { data, done } = resp { + chunks.push((data, done)); + } + offset += consumed; + } + // Last frame must be the empty terminator. + let last = chunks.last().expect("at least one frame"); + assert!( + last.0.is_empty() && last.1, + "last frame must be empty + done" + ); + // All earlier frames carry data and are not done. + for c in &chunks[..chunks.len() - 1] { + assert!(!c.1, "non-final chunk had done=true"); + assert!(!c.0.is_empty(), "non-final chunk was empty"); + } + // Concatenated data matches the source. + let concatenated: Vec = chunks.iter().flat_map(|(d, _)| d.iter().copied()).collect(); + assert_eq!(concatenated, payload); + } + + #[test] + fn streaming_write_rejects_when_total_size_exceeds_cap() { + let tmp = tempfile::tempdir().unwrap(); + let target = tmp_target(&tmp, "big"); + let (session, resp) = handle_file_write_begin( + target.to_string_lossy().into(), + None, + smolvm_protocol::FILE_TRANSFER_MAX_TOTAL + 1, + ); + assert!(session.is_none(), "session must not be created"); + assert!( + matches!(resp, AgentResponse::Error { .. }), + "expected error, got {:?}", + resp + ); + } + + #[test] + fn streaming_write_happy_path_writes_file_atomically() { + let tmp = tempfile::tempdir().unwrap(); + let target = tmp_target(&tmp, "hello.bin"); + + // Build a recognizable payload that also crosses a chunk boundary + // via the test-only helper. Use a known-odd size so no power-of- + // two alignment could accidentally hide a bug. + let payload = { + let mut v = Vec::with_capacity(37); + for i in 0..37u8 { + v.push(i); + } + v + }; + + let (session, resp) = handle_file_write_begin( + target.to_string_lossy().into(), + Some(0o600), + payload.len() as u64, + ); + assert!(matches!(resp, AgentResponse::Ok { .. })); + + let (session, resp) = handle_file_write_chunk(session, &payload, true); + assert!( + matches!(resp, AgentResponse::Ok { .. }), + "finalize failed: {:?}", + resp + ); + assert!(session.is_none()); + + // File exists with correct contents. + let got = std::fs::read(&target).unwrap(); + assert_eq!(got, payload); + + // No staging file left behind. + assert!( + staging_files_in(tmp.path()).is_empty(), + "staging file leaked" + ); + + // Mode applied (unix only). + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&target).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + } + } + + #[test] + fn streaming_write_multi_chunk_concatenates_in_order() { + let tmp = tempfile::tempdir().unwrap(); + let target = tmp_target(&tmp, "multi.bin"); + let total = 1024usize; + + let (mut session, resp) = + handle_file_write_begin(target.to_string_lossy().into(), None, total as u64); + assert!(matches!(resp, AgentResponse::Ok { .. })); + + // Three chunks: 400 + 400 + 224 bytes, each a distinct fill byte. + let chunks: [(&[u8], bool); 3] = [ + (&[b'A'; 400], false), + (&[b'B'; 400], false), + (&[b'C'; 224], true), + ]; + for (data, done) in chunks { + let (new_session, resp) = handle_file_write_chunk(session, data, done); + assert!( + matches!(resp, AgentResponse::Ok { .. }), + "chunk failed: {:?}", + resp + ); + session = new_session; + } + assert!(session.is_none(), "session must be consumed on done"); + + let got = std::fs::read(&target).unwrap(); + let mut expected = Vec::with_capacity(total); + expected.extend(std::iter::repeat(b'A').take(400)); + expected.extend(std::iter::repeat(b'B').take(400)); + expected.extend(std::iter::repeat(b'C').take(224)); + assert_eq!(got, expected); + assert!(staging_files_in(tmp.path()).is_empty()); + } + + #[test] + fn streaming_write_overflow_aborts_with_no_partial_file() { + let tmp = tempfile::tempdir().unwrap(); + let target = tmp_target(&tmp, "overflow.bin"); + + let (session, _resp) = handle_file_write_begin(target.to_string_lossy().into(), None, 10); + assert!(session.is_some()); + + // First chunk fits. + let (session, resp) = handle_file_write_chunk(session, &[0u8; 5], false); + assert!(matches!(resp, AgentResponse::Ok { .. })); + assert!(session.is_some()); + + // Second chunk would push bytes_written to 15, over total_size=10. + let (session, resp) = handle_file_write_chunk(session, &[0u8; 10], false); + assert!(matches!(resp, AgentResponse::Error { .. })); + // Session must be dropped (cleans staging). + assert!(session.is_none()); + + // Target never appeared (promise: no partial file). + assert!(!target.exists()); + // Staging file cleaned up via Drop. + assert!(staging_files_in(tmp.path()).is_empty()); + } + + #[test] + fn streaming_write_drop_cleans_staging_file() { + // Simulates connection dropping mid-stream. We open a session, + // write one chunk, then drop the session without calling done. + // The Drop impl must unlink the staging file so no partial + // content lingers. + let tmp = tempfile::tempdir().unwrap(); + let target = tmp_target(&tmp, "dropped.bin"); + + let (session, _) = handle_file_write_begin(target.to_string_lossy().into(), None, 100); + let (session, _) = handle_file_write_chunk(session, &[0u8; 50], false); + assert!(session.is_some()); + // Staging file exists mid-stream. + assert_eq!(staging_files_in(tmp.path()).len(), 1); + + drop(session); + // After drop, the staging file is gone and target never + // appeared. + assert!(staging_files_in(tmp.path()).is_empty()); + assert!(!target.exists()); + } + + #[test] + fn streaming_write_chunk_without_begin_errors() { + let (session, resp) = handle_file_write_chunk(None, &[0u8; 10], true); + assert!(session.is_none()); + assert!(matches!(resp, AgentResponse::Error { .. })); + } + + #[test] + fn streaming_write_zero_length_file() { + // Host sends empty `FileWriteChunk { data: [], done: true }` + // to finalize an empty file. Agent must create an empty file + // at the target, not leave it missing. + let tmp = tempfile::tempdir().unwrap(); + let target = tmp_target(&tmp, "empty.bin"); + + let (session, _) = handle_file_write_begin(target.to_string_lossy().into(), None, 0); + let (session, resp) = handle_file_write_chunk(session, &[], true); + assert!(matches!(resp, AgentResponse::Ok { .. })); + assert!(session.is_none()); + + assert!(target.exists()); + assert_eq!(std::fs::metadata(&target).unwrap().len(), 0); + assert!(staging_files_in(tmp.path()).is_empty()); + } + + #[test] + fn single_shot_write_uses_atomic_rename_too() { + // Regression guard on the shared finalizer: handle_file_write + // and handle_file_write_chunk(done=true) both go through + // install_file_atomic / WriteSession::finalize, so a failure + // mid-rename must leave no partial file at the target. + // Here we just verify the success path — install_file_atomic + // produces a correct file — and that no staging artifact + // leaks. + let tmp = tempfile::tempdir().unwrap(); + let target = tmp_target(&tmp, "single.bin"); + let payload = b"small file contents".to_vec(); + + let resp = handle_file_write(&target.to_string_lossy(), &payload, Some(0o644)); + assert!( + matches!(resp, AgentResponse::Ok { .. }), + "write failed: {:?}", + resp + ); + assert_eq!(std::fs::read(&target).unwrap(), payload); + assert!(staging_files_in(tmp.path()).is_empty()); + } + + #[test] + fn resolve_guest_path_rejects_parent_traversal() { + let tmp = tempfile::tempdir().unwrap(); + let merged = tmp.path().join("merged"); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&merged).unwrap(); + std::fs::create_dir_all(&workspace).unwrap(); + + let res = resolve_guest_io_path_with_roots( + "/../../storage/secret.txt", + FilePathAccess::Write, + Some(&merged), + &workspace, + ); + assert!(matches!(res, Err(AgentResponse::Error { .. }))); + } + + #[cfg(unix)] + #[test] + fn resolve_guest_read_rejects_symlink_escape_from_overlay() { + use std::os::unix::fs::symlink; + + let tmp = tempfile::tempdir().unwrap(); + let merged = tmp.path().join("merged"); + let workspace = tmp.path().join("workspace"); + let outside = tmp.path().join("outside"); + std::fs::create_dir_all(&merged).unwrap(); + std::fs::create_dir_all(&workspace).unwrap(); + std::fs::create_dir_all(&outside).unwrap(); + + let outside_file = outside.join("secret.txt"); + std::fs::write(&outside_file, b"secret").unwrap(); + symlink(&outside_file, merged.join("link-outside")).unwrap(); + + let res = resolve_guest_io_path_with_roots( + "/link-outside", + FilePathAccess::Read, + Some(&merged), + &workspace, + ); + assert!(matches!(res, Err(AgentResponse::Error { .. }))); + } + + #[cfg(unix)] + #[test] + fn resolve_guest_write_rejects_symlink_escape_from_overlay() { + use std::os::unix::fs::symlink; + + let tmp = tempfile::tempdir().unwrap(); + let merged = tmp.path().join("merged"); + let workspace = tmp.path().join("workspace"); + let outside = tmp.path().join("outside"); + std::fs::create_dir_all(&merged).unwrap(); + std::fs::create_dir_all(&workspace).unwrap(); + std::fs::create_dir_all(&outside).unwrap(); + + symlink(&outside, merged.join("escape-dir")).unwrap(); + + let res = resolve_guest_io_path_with_roots( + "/escape-dir/pwned.txt", + FilePathAccess::Write, + Some(&merged), + &workspace, + ); + assert!(matches!(res, Err(AgentResponse::Error { .. }))); + } + + #[test] + fn resolve_guest_workspace_path_maps_to_workspace_root() { + let tmp = tempfile::tempdir().unwrap(); + let merged = tmp.path().join("merged"); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&merged).unwrap(); + std::fs::create_dir_all(&workspace).unwrap(); + + let resolved = resolve_guest_io_path_with_roots( + "/workspace/subdir/file.txt", + FilePathAccess::Write, + Some(&merged), + &workspace, + ) + .unwrap(); + + assert!(resolved.starts_with(&workspace)); + assert_eq!(resolved, workspace.join("subdir").join("file.txt")); + } + + #[test] + fn resolve_guest_relative_path_is_normalized_under_root() { + let tmp = tempfile::tempdir().unwrap(); + let merged = tmp.path().join("merged"); + let workspace = tmp.path().join("workspace"); + std::fs::create_dir_all(&merged).unwrap(); + std::fs::create_dir_all(&workspace).unwrap(); + + let resolved = resolve_guest_io_path_with_roots( + "etc/hosts", + FilePathAccess::Write, + Some(&merged), + &workspace, + ) + .unwrap(); + + assert_eq!(resolved, merged.join("etc").join("hosts")); + } + + #[test] + fn test_boot_log_valid_json() { + let line = format_boot_log("ERROR", "something failed"); + let parsed: serde_json::Value = serde_json::from_str(&line) + .unwrap_or_else(|e| panic!("Invalid JSON: {}\nLine: {}", e, line)); + assert_eq!(parsed["level"], "ERROR"); + assert_eq!(parsed["message"], "something failed"); + assert_eq!(parsed["target"], "smolvm_agent::boot"); + } + + #[test] + fn test_boot_log_escapes_quotes() { + let line = format_boot_log("ERROR", r#"failed: "device" not found"#); + let parsed: serde_json::Value = serde_json::from_str(&line) + .unwrap_or_else(|e| panic!("Invalid JSON: {}\nLine: {}", e, line)); + assert!(parsed["message"].as_str().unwrap().contains("\"device\"")); + } +} diff --git a/crates/smolvm-agent/src/network/linux.rs b/crates/smolvm-agent/src/network/linux.rs new file mode 100644 index 0000000..b3f1e20 --- /dev/null +++ b/crates/smolvm-agent/src/network/linux.rs @@ -0,0 +1,702 @@ +//! Linux network configuration helpers for a given host NIC. + +use std::net::{Ipv4Addr, Ipv6Addr}; + +/// `IFA_F_NODAD` from `linux/if_addr.h`: skip Duplicate Address Detection for a +/// statically assigned IPv6 address. On the two-node virtual link the addresses +/// are deterministic, and skipping DAD avoids the ~1s "tentative" window before +/// the guest can use its IPv6 address. +const IFA_F_NODAD: u8 = 0x02; + +/// Configure a guest interface for the virtio-net. +/// +/// High-level flow: +/// +/// ```text +/// 1. find eth0's ifindex (SIOCGIFINDEX) +/// 2. set link-layer MAC address (SIOCSIFHWADDR) +/// 3. set MTU (SIOCSIFMTU) +/// 4. add IPv4 address/prefix (RTM_NEWADDR via NETLINK_ROUTE) +/// 5. mark the interface UP (SIOCGIFFLAGS + SIOCSIFFLAGS) +/// 6. add the default route (RTM_NEWROUTE via NETLINK_ROUTE) +/// 7. write /etc/resolv.conf (plain file write) +/// ``` +/// +/// Read these functions as: +/// +/// ```text +/// set_mac_address() ~= ip link set dev eth0 address ... +/// set_mtu() ~= ip link set dev eth0 mtu ... +/// add_address_v4() ~= ip addr add ... +/// bring_interface_up()~= ip link set dev eth0 up +/// add_default_route() ~= ip route add default via ... +/// ``` +/// +/// Outcome: +/// - the guest ends up with a configured `eth0` +/// - traffic to non-local destinations is sent to `gateway` +/// - libc DNS resolution uses `dns_server` +/// +/// Why this order: +/// - MAC and MTU are link attributes, so we set them before bringing the +/// interface fully up +/// - the IPv4 address and route are programmed explicitly instead of relying on +/// DHCP or helper tools +/// - the function is fail-fast: any kernel call failure aborts boot for the +/// requested virtio-net path +/// - the address is installed before the route so the kernel already knows the +/// guest's on-link subnet when the default gateway is added +/// +/// The Linux kernel interfaces used here are all C ABI calls: +/// +/// - `SIOCGIFINDEX`: asks the kernel for the numeric interface index for +/// `ifname`. Netlink messages use that numeric id, not the human-readable +/// string. +/// - `SIOCSIFHWADDR`: updates the NIC MAC address. +/// - `SIOCSIFMTU`: updates the NIC MTU. +/// - `SIOCGIFFLAGS` / `SIOCSIFFLAGS`: read-modify-write the interface flags so +/// we can set `IFF_UP`. +/// - `RTM_NEWADDR`: asks the kernel routing stack to add an IPv4 address. +/// - `RTM_NEWROUTE`: asks the kernel routing stack to install the default route. +#[allow(clippy::too_many_arguments)] +pub fn configure_interface( + ifname: &str, + mac: [u8; 6], + mtu: u16, + address: Ipv4Addr, + prefix_len: u8, + gateway: Ipv4Addr, + ipv6: Option<(Ipv6Addr, u8, Ipv6Addr)>, + dns_server: Ipv4Addr, +) -> Result<(), String> { + let ifindex = get_ifindex(ifname)?; + set_mac_address(ifname, &mac)?; + set_mtu(ifname, mtu)?; + add_address_v4(ifindex, address, prefix_len)?; + bring_interface_up(ifname)?; + add_default_route_v4(gateway)?; + if let Some((address6, prefix_len6, gateway6)) = ipv6 { + add_address_v6(ifindex, address6, prefix_len6)?; + add_default_route_v6(gateway6)?; + } + write_resolv_conf(dns_server)?; + Ok(()) +} + +/// Resolve the Linux interface index used by rtnetlink messages. +/// +/// C ABI context: +/// - opens an `AF_INET` datagram socket only as an ioctl handle +/// - fills an `ifreq` +/// - calls `ioctl(..., SIOCGIFINDEX, ...)` +/// +/// Usage: +/// - `ifname` is the human-readable name, such as `eth0` +/// +/// Outcome: +/// - returns the kernel's numeric interface id, which is later embedded into +/// `RTM_NEWADDR` +/// +/// Why this exists: +/// - `ioctl` interface operations identify the device by name in `ifreq` +/// - rtnetlink address operations identify the device by numeric index +/// - this is the bridge between those two APIs +fn get_ifindex(ifname: &str) -> Result { + // SAFETY: `ifreq` is plain old data; zeroed initialization is valid. + unsafe { + let mut ifr: libc::ifreq = std::mem::zeroed(); + copy_ifname(&mut ifr, ifname)?; + + let sock = socket_fd()?; + if libc::ioctl(sock, libc::SIOCGIFINDEX as _, &mut ifr) < 0 { + let err = std::io::Error::last_os_error(); + libc::close(sock); + return Err(format!("SIOCGIFINDEX failed for {}: {}", ifname, err)); + } + libc::close(sock); + + Ok(ifr.ifr_ifru.ifru_ifindex as u32) + } +} + +/// Program the NIC MAC address with `SIOCSIFHWADDR`. +/// +/// C ABI context: +/// - `ifreq.ifru_hwaddr` carries a `sockaddr`-shaped payload +/// - `sa_family = ARPHRD_ETHER` tells the kernel the address is Ethernet +/// - the first 6 bytes of `sa_data` hold the MAC octets +/// +/// Outcome: +/// - future packets emitted by this guest NIC use the requested source MAC +/// +/// Shell equivalent: +/// +/// ```text +/// ip link set dev address +/// ``` +fn set_mac_address(ifname: &str, mac: &[u8; 6]) -> Result<(), String> { + // SAFETY: `ifreq` is plain old data; zeroed initialization is valid. + unsafe { + let mut ifr: libc::ifreq = std::mem::zeroed(); + copy_ifname(&mut ifr, ifname)?; + + ifr.ifr_ifru.ifru_hwaddr.sa_family = libc::ARPHRD_ETHER; + ifr.ifr_ifru.ifru_hwaddr.sa_data[..6] + .copy_from_slice(&mac.map(|byte| byte as libc::c_char)); + + let sock = socket_fd()?; + if libc::ioctl(sock, libc::SIOCSIFHWADDR as _, &ifr) < 0 { + let err = std::io::Error::last_os_error(); + libc::close(sock); + return Err(format!("SIOCSIFHWADDR failed for {}: {}", ifname, err)); + } + libc::close(sock); + } + Ok(()) +} + +/// Program the link MTU with `SIOCSIFMTU`. +/// +/// C ABI context: +/// - `ifreq.ifru_mtu` is interpreted by the kernel as the requested MTU value +/// +/// Outcome: +/// - the kernel enforces this frame size for the interface +/// +/// Shell equivalent: +/// +/// ```text +/// ip link set dev mtu +/// ``` +fn set_mtu(ifname: &str, mtu: u16) -> Result<(), String> { + // SAFETY: `ifreq` is plain old data; zeroed initialization is valid. + unsafe { + let mut ifr: libc::ifreq = std::mem::zeroed(); + copy_ifname(&mut ifr, ifname)?; + ifr.ifr_ifru.ifru_mtu = mtu as libc::c_int; + + let sock = socket_fd()?; + if libc::ioctl(sock, libc::SIOCSIFMTU as _, &ifr) < 0 { + let err = std::io::Error::last_os_error(); + libc::close(sock); + return Err(format!("SIOCSIFMTU failed for {}: {}", ifname, err)); + } + libc::close(sock); + } + Ok(()) +} + +/// Mark the interface `IFF_UP`. +/// +/// C ABI context: +/// - `SIOCGIFFLAGS` reads the current flag word +/// - we OR in `IFF_UP` +/// - `SIOCSIFFLAGS` writes the updated flag word back +/// +/// Outcome: +/// - the kernel considers the interface administratively up and will start +/// using the configured address and route +/// +/// Shell equivalent: +/// +/// ```text +/// ip link set dev up +/// ``` +fn bring_interface_up(ifname: &str) -> Result<(), String> { + // SAFETY: `ifreq` is plain old data; zeroed initialization is valid. + unsafe { + let mut ifr: libc::ifreq = std::mem::zeroed(); + copy_ifname(&mut ifr, ifname)?; + + let sock = socket_fd()?; + if libc::ioctl(sock, libc::SIOCGIFFLAGS as _, &mut ifr) < 0 { + let err = std::io::Error::last_os_error(); + libc::close(sock); + return Err(format!("SIOCGIFFLAGS failed for {}: {}", ifname, err)); + } + + ifr.ifr_ifru.ifru_flags |= libc::IFF_UP as libc::c_short; + if libc::ioctl(sock, libc::SIOCSIFFLAGS as _, &ifr) < 0 { + let err = std::io::Error::last_os_error(); + libc::close(sock); + return Err(format!("SIOCSIFFLAGS failed for {}: {}", ifname, err)); + } + libc::close(sock); + } + Ok(()) +} + +/// Add the guest IPv4 address through rtnetlink. +/// +/// This is the programmatic equivalent of: +/// +/// ```text +/// ip addr add
/ dev +/// ``` +/// +/// Outcome: +/// - the kernel records the IPv4 address on the interface identified by +/// `ifindex` +/// +/// Why netlink here: +/// - there is no classic `ioctl` that cleanly expresses modern address +/// creation the way `ip addr add` does +/// - rtnetlink is the kernel's structured control plane for addresses and +/// routes +fn add_address_v4(ifindex: u32, address: Ipv4Addr, prefix_len: u8) -> Result<(), String> { + let address_bytes = address.octets(); + netlink_newaddr(ifindex, libc::AF_INET as u8, 0, prefix_len, &address_bytes).map_err(|err| { + format!( + "failed to add IPv4 address {}/{}: {}", + address, prefix_len, err + ) + }) +} + +/// Add the guest IPv6 address through rtnetlink (`ip -6 addr add ...`), with +/// DAD skipped (see [`IFA_F_NODAD`]). +fn add_address_v6(ifindex: u32, address: Ipv6Addr, prefix_len: u8) -> Result<(), String> { + let address_bytes = address.octets(); + netlink_newaddr( + ifindex, + libc::AF_INET6 as u8, + IFA_F_NODAD, + prefix_len, + &address_bytes, + ) + .map_err(|err| { + format!( + "failed to add IPv6 address {}/{}: {}", + address, prefix_len, err + ) + }) +} + +/// Install the default IPv4 route through the provided gateway. +/// +/// This is the programmatic equivalent of: +/// +/// ```text +/// ip route add default via +/// ``` +/// +/// Outcome: +/// - traffic for non-local destinations is sent to `gateway` +/// +/// Why only a gateway attribute is needed here: +/// - the default route says "for destinations not matched by a more specific +/// route, send to this next hop" +/// - because the guest interface address was installed first, the kernel can +/// resolve that gateway as reachable on the connected subnet +fn add_default_route_v4(gateway: Ipv4Addr) -> Result<(), String> { + let gateway_bytes = gateway.octets(); + netlink_newroute(libc::AF_INET as u8, &gateway_bytes) + .map_err(|err| format!("failed to add default route via {}: {}", gateway, err)) +} + +/// Install the default IPv6 route through the provided gateway +/// (`ip -6 route add default via ...`). +fn add_default_route_v6(gateway: Ipv6Addr) -> Result<(), String> { + let gateway_bytes = gateway.octets(); + netlink_newroute(libc::AF_INET6 as u8, &gateway_bytes) + .map_err(|err| format!("failed to add default IPv6 route via {}: {}", gateway, err)) +} + +/// Replace `/etc/resolv.conf` with the gateway-side resolver. +/// +/// Outcome: +/// - standard guest libc resolution (`getaddrinfo`, `nslookup`, etc.) sends DNS +/// traffic to the host-provided resolver path +/// +/// This step is intentionally plain file I/O rather than a C networking API. +/// DNS configuration in a minimal Linux guest is usually conveyed through +/// `/etc/resolv.conf`, and that is enough for the MVP. +fn write_resolv_conf(dns_server: Ipv4Addr) -> Result<(), String> { + let contents = format!("nameserver {}\n", dns_server); + let direct_err = match std::fs::write("/etc/resolv.conf", &contents) { + Ok(()) => return Ok(()), + Err(err) => err, + }; + + // Read-only /etc (the Linux agent rootfs boots read-only — same trap class + // as the read-only /tmp): stage the file on the tmpfs /tmp and bind-mount + // it over /etc/resolv.conf. And in NO case may DNS config kill the boot — + // the interface, routes, and egress NAT are already up; a guest without + // resolv.conf still serves published ports and numeric egress — so any + // residual failure degrades to a WARN instead of an Err (which the caller + // treats as fatal for PID 1). + let staged = "/tmp/.smolvm-resolv.conf"; + if let Err(e) = std::fs::write(staged, &contents) { + tracing::warn!(error = %e, original = %direct_err, + "resolv.conf unwritable and tmpfs staging failed; continuing without DNS config"); + return Ok(()); + } + let src = std::ffi::CString::new(staged).expect("static path"); + let dst = std::ffi::CString::new("/etc/resolv.conf").expect("static path"); + let rc = unsafe { + libc::mount( + src.as_ptr(), + dst.as_ptr(), + std::ptr::null(), + libc::MS_BIND, + std::ptr::null(), + ) + }; + if rc != 0 { + tracing::warn!( + error = %std::io::Error::last_os_error(), + original = %direct_err, + "resolv.conf bind-mount fallback failed; continuing without DNS config" + ); + } + Ok(()) +} + +/// Create a datagram socket used only as an ioctl control handle. +/// +/// C ABI context: +/// - many legacy interface ioctls operate on any socket fd from the right +/// address family +/// - this socket is not used for packet I/O +/// +/// Outcome: +/// - returns an fd suitable for `SIOCGIFINDEX`, `SIOCSIFHWADDR`, +/// `SIOCSIFMTU`, and `SIOCSIFFLAGS` +/// +/// Important distinction: +/// - this is not the data path for guest traffic +/// - it is just a capability handle the kernel accepts for network ioctls +fn socket_fd() -> Result { + // SAFETY: `socket` is a standard libc call with valid arguments. + let fd = unsafe { libc::socket(libc::AF_INET, libc::SOCK_DGRAM | libc::SOCK_CLOEXEC, 0) }; + if fd < 0 { + return Err(format!( + "failed to create socket: {}", + std::io::Error::last_os_error() + )); + } + Ok(fd) +} + +/// Copy an interface name into `ifreq.ifr_name`. +/// +/// C ABI context: +/// - most interface ioctls use `ifreq` +/// - the kernel matches the request to an interface through the fixed-width +/// `ifr_name` buffer +/// - `ifreq` was zero-initialized, so copying only the visible bytes leaves the +/// trailing NUL padding in place +fn copy_ifname(ifr: &mut libc::ifreq, ifname: &str) -> Result<(), String> { + let bytes = ifname.as_bytes(); + if bytes.len() >= libc::IFNAMSIZ { + return Err(format!("interface name too long: {}", ifname)); + } + + // SAFETY: `ifr_name` is large enough because of the length check above. + unsafe { + std::ptr::copy_nonoverlapping( + bytes.as_ptr(), + ifr.ifr_name.as_mut_ptr().cast(), + bytes.len(), + ); + } + + Ok(()) +} + +/// Minimal Linux `ifaddrmsg` layout used by `RTM_NEWADDR`. +/// +/// We define the struct locally instead of relying on higher-level helpers so +/// the agent stays self-contained in the guest environment. +/// +/// Meaning of the fields we populate: +/// - `ifa_family = AF_INET`: this is an IPv4 address operation +/// - `ifa_prefixlen`: subnet length, for example `24` +/// - `ifa_scope = RT_SCOPE_UNIVERSE`: globally scoped address, not host-local +/// - `ifa_index`: target interface id returned by `SIOCGIFINDEX` +#[repr(C)] +struct IfAddrMsg { + ifa_family: u8, + ifa_prefixlen: u8, + ifa_flags: u8, + ifa_scope: u8, + ifa_index: u32, +} + +/// Minimal Linux `rtmsg` layout used by `RTM_NEWROUTE`. +/// +/// Meaning of the fields we populate: +/// - `rtm_family = AF_INET`: this is an IPv4 route +/// - `rtm_dst_len = 0`: zero-bit destination prefix, which means "default route" +/// - `rtm_table = RT_TABLE_MAIN`: install into the main routing table +/// - `rtm_protocol = RTPROT_BOOT`: route was installed during boot/runtime init +/// - `rtm_scope = RT_SCOPE_UNIVERSE`: globally reachable route +/// - `rtm_type = RTN_UNICAST`: normal unicast forwarding entry +#[repr(C)] +struct RtMsg { + rtm_family: u8, + rtm_dst_len: u8, + rtm_src_len: u8, + rtm_tos: u8, + rtm_table: u8, + rtm_protocol: u8, + rtm_scope: u8, + rtm_type: u8, + rtm_flags: u32, +} + +const NLMSG_HDRLEN: usize = 16; +const IFADDRMSG_LEN: usize = 8; +const RTMSG_LEN: usize = 12; +const RTA_HDRLEN: usize = 4; + +const _: () = assert!(std::mem::size_of::() == NLMSG_HDRLEN); +const _: () = assert!(std::mem::size_of::() == IFADDRMSG_LEN); +const _: () = assert!(std::mem::size_of::() == RTMSG_LEN); + +/// Build and send an `RTM_NEWADDR` netlink message. +/// +/// C ABI context: +/// - `nlmsghdr` is the outer netlink envelope +/// - `IfAddrMsg` is the `RTM_NEWADDR` body +/// - `IFA_ADDRESS` and `IFA_LOCAL` are route attributes appended after the body +/// +/// Outcome: +/// - asks the kernel to attach an IPv4 address/prefix to `ifindex` +/// +/// Message shape: +/// +/// ```text +/// nlmsghdr +/// type = RTM_NEWADDR +/// flags = REQUEST | ACK | CREATE | EXCL +/// ifaddrmsg +/// family = AF_INET +/// prefixlen = +/// index = +/// rtattr IFA_ADDRESS = +/// rtattr IFA_LOCAL = +/// ``` +/// +/// Why both `IFA_ADDRESS` and `IFA_LOCAL` are present: +/// - for a plain unicast interface address, both effectively describe the same +/// address +/// - including both matches the common shape produced by tools like `ip` +/// for local interface address assignment +/// +/// `family` selects IPv4 (`AF_INET`, 4-byte address) or IPv6 (`AF_INET6`, +/// 16-byte address); `flags` carries e.g. [`IFA_F_NODAD`]. +fn netlink_newaddr( + ifindex: u32, + family: u8, + flags: u8, + prefix_len: u8, + address: &[u8], +) -> std::io::Result<()> { + let rta_len = rta_space(address.len()); + let msg_len = NLMSG_HDRLEN + IFADDRMSG_LEN + (rta_len * 2); + let mut buf = vec![0u8; nlmsg_align(msg_len)]; + + let nlh = buf.as_mut_ptr().cast::(); + // SAFETY: `buf` is large enough for `nlmsghdr`. + unsafe { + (*nlh).nlmsg_len = msg_len as u32; + (*nlh).nlmsg_type = libc::RTM_NEWADDR; + (*nlh).nlmsg_flags = + (libc::NLM_F_REQUEST | libc::NLM_F_ACK | libc::NLM_F_CREATE | libc::NLM_F_EXCL) as u16; + (*nlh).nlmsg_seq = 1; + } + + let ifa = unsafe { buf.as_mut_ptr().add(NLMSG_HDRLEN).cast::() }; + // SAFETY: `buf` is large enough for `IfAddrMsg`. + unsafe { + (*ifa).ifa_family = family; + (*ifa).ifa_prefixlen = prefix_len; + (*ifa).ifa_flags = flags; + (*ifa).ifa_scope = libc::RT_SCOPE_UNIVERSE; + (*ifa).ifa_index = ifindex; + } + + let mut offset = NLMSG_HDRLEN + IFADDRMSG_LEN; + write_rta(&mut buf[offset..], libc::IFA_ADDRESS, address); + offset += rta_space(address.len()); + write_rta(&mut buf[offset..], libc::IFA_LOCAL, address); + + netlink_send(&buf) +} + +/// Build and send an `RTM_NEWROUTE` netlink message for the default route. +/// +/// C ABI context: +/// - `rtm_dst_len = 0` means "default route" +/// - `RTA_GATEWAY` carries the next-hop IPv4 address +/// +/// Outcome: +/// - asks the kernel to install a unicast route in the main table +/// +/// Message shape: +/// +/// ```text +/// nlmsghdr +/// type = RTM_NEWROUTE +/// flags = REQUEST | ACK | CREATE | EXCL +/// rtmsg +/// family = AF_INET +/// dst_len = 0 +/// table = RT_TABLE_MAIN +/// protocol = RTPROT_BOOT +/// scope = RT_SCOPE_UNIVERSE +/// type = RTN_UNICAST +/// rtattr RTA_GATEWAY = +/// ``` +/// +/// `family` selects IPv4 (`AF_INET`, 4-byte gateway) or IPv6 (`AF_INET6`, +/// 16-byte gateway). +fn netlink_newroute(family: u8, gateway: &[u8]) -> std::io::Result<()> { + let rta_len = rta_space(gateway.len()); + let msg_len = NLMSG_HDRLEN + RTMSG_LEN + rta_len; + let mut buf = vec![0u8; nlmsg_align(msg_len)]; + + let nlh = buf.as_mut_ptr().cast::(); + // SAFETY: `buf` is large enough for `nlmsghdr`. + unsafe { + (*nlh).nlmsg_len = msg_len as u32; + (*nlh).nlmsg_type = libc::RTM_NEWROUTE; + (*nlh).nlmsg_flags = + (libc::NLM_F_REQUEST | libc::NLM_F_ACK | libc::NLM_F_CREATE | libc::NLM_F_EXCL) as u16; + (*nlh).nlmsg_seq = 2; + } + + let rtm = unsafe { buf.as_mut_ptr().add(NLMSG_HDRLEN).cast::() }; + // SAFETY: `buf` is large enough for `RtMsg`. + unsafe { + (*rtm).rtm_family = family; + (*rtm).rtm_dst_len = 0; + (*rtm).rtm_src_len = 0; + (*rtm).rtm_tos = 0; + (*rtm).rtm_table = libc::RT_TABLE_MAIN; + (*rtm).rtm_protocol = libc::RTPROT_BOOT; + (*rtm).rtm_scope = libc::RT_SCOPE_UNIVERSE; + (*rtm).rtm_type = libc::RTN_UNICAST; + (*rtm).rtm_flags = 0; + } + + let offset = NLMSG_HDRLEN + RTMSG_LEN; + write_rta(&mut buf[offset..], libc::RTA_GATEWAY, gateway); + netlink_send(&buf) +} + +/// Send one netlink request and wait for the kernel ack. +/// +/// Walkthrough: +/// +/// ```text +/// userspace buffer +/// -> socket(AF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE) +/// -> bind(local netlink sockaddr) +/// -> send(message) +/// -> recv(ack) +/// -> inspect NLMSG_ERROR +/// ``` +/// +/// C ABI context: +/// - `socket(AF_NETLINK, SOCK_DGRAM, NETLINK_ROUTE)` opens a routing netlink +/// endpoint to the kernel +/// - `bind` attaches a local netlink address so the kernel can reply +/// - `send` transmits the prepared `nlmsghdr + payload` +/// - `recv` collects the kernel ack +/// - the ack payload for `NLMSG_ERROR` contains a signed errno: +/// - `0` means success +/// - a negative errno means failure +/// +/// Outcome: +/// - returns `Ok(())` once the kernel accepts the request +/// - returns an `io::Error` if socket creation, send/recv, or the kernel ack +/// reports a failure +/// +/// Important distinction from the virtio-net data path: +/// - this socket is not carrying guest Ethernet traffic +/// - it is a control-plane socket used only to ask the Linux kernel to mutate +/// routing/address state inside the guest +fn netlink_send(msg: &[u8]) -> std::io::Result<()> { + // SAFETY: all libc calls use valid buffers and checked lengths. + unsafe { + let sock = libc::socket(libc::AF_NETLINK, libc::SOCK_DGRAM, libc::NETLINK_ROUTE); + if sock < 0 { + return Err(std::io::Error::last_os_error()); + } + + let mut sockaddr: libc::sockaddr_nl = std::mem::zeroed(); + sockaddr.nl_family = libc::AF_NETLINK as u16; + if libc::bind( + sock, + (&sockaddr as *const libc::sockaddr_nl).cast(), + std::mem::size_of::() as u32, + ) < 0 + { + let err = std::io::Error::last_os_error(); + libc::close(sock); + return Err(err); + } + + if libc::send(sock, msg.as_ptr().cast(), msg.len(), 0) < 0 { + let err = std::io::Error::last_os_error(); + libc::close(sock); + return Err(err); + } + + let mut ack = [0u8; 1024]; + let bytes = libc::recv(sock, ack.as_mut_ptr().cast(), ack.len(), 0); + libc::close(sock); + if bytes < 0 { + return Err(std::io::Error::last_os_error()); + } + + if (bytes as usize) >= NLMSG_HDRLEN + 4 { + let nlh = ack.as_ptr().cast::(); + if (*nlh).nlmsg_type == libc::NLMSG_ERROR as u16 { + let err = + i32::from_ne_bytes(ack[NLMSG_HDRLEN..NLMSG_HDRLEN + 4].try_into().unwrap()); + if err < 0 { + return Err(std::io::Error::from_raw_os_error(-err)); + } + } + } + + Ok(()) + } +} + +/// Netlink/rtnetlink attributes are 4-byte aligned. +/// +/// The kernel expects both message bodies and nested attributes to start on +/// 4-byte boundaries. This helper rounds a length up to that alignment. +fn nlmsg_align(len: usize) -> usize { + (len + 3) & !3 +} + +/// Space consumed by one route attribute including alignment padding. +/// +/// An `rtattr` is: +/// +/// ```text +/// [u16 len][u16 type][payload bytes][optional padding] +/// ``` +/// +/// `rta_space` returns the full reserved byte count, not just the visible +/// header + payload. +fn rta_space(data_len: usize) -> usize { + nlmsg_align(RTA_HDRLEN + data_len) +} + +/// Encode one `rtattr` header plus its payload bytes into `buf`. +/// +/// `buf` is expected to be large enough for `rta_space(data.len())`. The caller +/// is responsible for advancing the offset by the aligned size so the next +/// attribute starts on a 4-byte boundary. +fn write_rta(buf: &mut [u8], rta_type: u16, data: &[u8]) { + let rta_len = (RTA_HDRLEN + data.len()) as u16; + buf[0..2].copy_from_slice(&rta_len.to_ne_bytes()); + buf[2..4].copy_from_slice(&rta_type.to_ne_bytes()); + buf[RTA_HDRLEN..RTA_HDRLEN + data.len()].copy_from_slice(data); +} diff --git a/crates/smolvm-agent/src/network/mod.rs b/crates/smolvm-agent/src/network/mod.rs new file mode 100644 index 0000000..3acb6a0 --- /dev/null +++ b/crates/smolvm-agent/src/network/mod.rs @@ -0,0 +1,250 @@ +//! Guest-side virtio-net configuration from `SMOLVM_NETWORK_*`. +//! +//! Context +//! ======= +//! +//! The host side of the virtio-net design decides whether a VM should use: +//! - the legacy TSI networking path, or +//! - a real virtio-net device exposed to the guest +//! +//! When virtio-net is selected, the launcher does not run guest shell +//! commands like `ip link`, `ip addr`, or `ip route`. Instead it passes a +//! small, explicit configuration contract into the guest as environment +//! variables. The agent reads those values very early in boot and programs +//! the kernel network state directly. +//! +//! That gives us a narrow host/guest boundary: +//! +//! ```text +//! host launcher +//! -> decides backend = virtio-net +//! -> chooses guest IP / gateway / DNS / MAC +//! -> exports SMOLVM_NETWORK_* env +//! -> starts guest agent +//! +//! guest agent +//! -> parses SMOLVM_NETWORK_* env +//! -> configures eth0 inside the guest kernel +//! -> continues normal boot +//! ``` +//! +//! In shell terms, the Linux implementation in `linux.rs` is effectively a +//! built-in replacement for this class of commands: +//! +//! ```text +//! ip link set dev eth0 address +//! ip link set dev eth0 mtu +//! ip addr add / dev eth0 +//! ip link set dev eth0 up +//! ip route add default via +//! printf 'nameserver \n' > /etc/resolv.conf +//! ``` +//! +//! We do it inside the agent rather than by spawning external tools because the +//! guest image is intentionally small and boots before we can assume userspace +//! helpers are present. +//! +//! The Linux-specific implementation lives in `linux.rs`. Non-Linux guests +//! currently return an explicit error instead of attempting a partial setup. + +use smolvm_protocol::guest_env; +use std::net::{Ipv4Addr, Ipv6Addr}; + +/// Configure the guest network interface from host-provided environment. +/// +/// Returns `Ok(false)` when virtio-net is not enabled for this boot. +/// +/// Environment contract +/// -------------------- +/// +/// The host launcher currently provides: +/// - `SMOLVM_NETWORK_BACKEND=virtio-net` +/// - `SMOLVM_NETWORK_GUEST_IP` +/// - `SMOLVM_NETWORK_GATEWAY` +/// - `SMOLVM_NETWORK_PREFIX_LEN` +/// - `SMOLVM_NETWORK_GUEST_MAC` +/// - `SMOLVM_NETWORK_DNS` +/// - `SMOLVM_NETWORK_GUEST_IP6` / `SMOLVM_NETWORK_GATEWAY6` / +/// `SMOLVM_NETWORK_PREFIX_LEN6` (optional trio — absent means IPv4-only) +/// +/// Example: +/// +/// ```text +/// SMOLVM_NETWORK_BACKEND=virtio-net +/// SMOLVM_NETWORK_GUEST_IP=10.0.2.15 +/// SMOLVM_NETWORK_GATEWAY=10.0.2.2 +/// SMOLVM_NETWORK_PREFIX_LEN=24 +/// SMOLVM_NETWORK_GUEST_MAC=02:53:4d:00:00:02 +/// SMOLVM_NETWORK_DNS=10.0.2.2 +/// SMOLVM_NETWORK_GUEST_IP6=fd53:4d00::2 +/// SMOLVM_NETWORK_GATEWAY6=fd53:4d00::1 +/// SMOLVM_NETWORK_PREFIX_LEN6=64 +/// ``` +/// +/// What this function does +/// ----------------------- +/// +/// 1. Decide whether the current boot even wants guest virtio networking. +/// 2. Parse the environment strings into typed values. +/// 3. Call the Linux backend to program `eth0`. +/// +/// Outcome +/// ------- +/// +/// - `Ok(false)`: no virtio-net request was present, so the agent leaves the +/// guest network untouched. +/// - `Ok(true)`: `eth0` was configured successfully. +/// - `Err(...)`: virtio-net was requested but the configuration was incomplete +/// or malformed, so boot should fail instead of continuing with a +/// half-configured NIC. +pub fn configure_from_env() -> Result { + let backend = match std::env::var(guest_env::BACKEND) { + Ok(value) if !value.is_empty() => value, + _ => return Ok(false), + }; + + if backend != guest_env::BACKEND_VIRTIO_NET { + return Err(format!( + "unsupported {} value: {}", + guest_env::BACKEND, + backend + )); + } + + let guest_ip = env_ipv4(guest_env::GUEST_IP)?; + let gateway = env_ipv4(guest_env::GATEWAY)?; + let prefix_len = env_u8(guest_env::PREFIX_LEN)?; + let guest_mac = env_mac(guest_env::GUEST_MAC)?; + let dns_server = env_ipv4(guest_env::DNS)?; + let ipv6 = env_ipv6_config()?; + + linux::configure_interface( + "eth0", guest_mac, 1500, guest_ip, prefix_len, gateway, ipv6, dns_server, + )?; + Ok(true) +} + +/// Parse the optional IPv6 trio. All three vars must be present together; a +/// partial set is a malformed contract and fails the boot rather than leaving a +/// half-configured stack. +fn env_ipv6_config() -> Result, String> { + let vars = [ + guest_env::GUEST_IP6, + guest_env::GATEWAY6, + guest_env::PREFIX_LEN6, + ]; + let present = vars + .iter() + .filter(|name| std::env::var(name).is_ok_and(|v| !v.is_empty())) + .count(); + match present { + 0 => Ok(None), + 3 => Ok(Some(( + env_ipv6(guest_env::GUEST_IP6)?, + env_u8(guest_env::PREFIX_LEN6)?, + env_ipv6(guest_env::GATEWAY6)?, + ))), + _ => Err(format!( + "incomplete IPv6 network config: {} / {} / {} must be set together", + guest_env::GUEST_IP6, + guest_env::GATEWAY6, + guest_env::PREFIX_LEN6 + )), + } +} + +fn env_ipv4(name: &str) -> Result { + let value = std::env::var(name).map_err(|_| format!("missing {}", name))?; + value + .parse::() + .map_err(|_| format!("invalid IPv4 address for {}: {}", name, value)) +} + +fn env_ipv6(name: &str) -> Result { + let value = std::env::var(name).map_err(|_| format!("missing {}", name))?; + value + .parse::() + .map_err(|_| format!("invalid IPv6 address for {}: {}", name, value)) +} + +fn env_u8(name: &str) -> Result { + let value = std::env::var(name).map_err(|_| format!("missing {}", name))?; + value + .parse::() + .map_err(|_| format!("invalid integer for {}: {}", name, value)) +} + +fn env_mac(name: &str) -> Result<[u8; 6], String> { + let value = std::env::var(name).map_err(|_| format!("missing {}", name))?; + parse_mac(&value) +} + +/// Parse a colon-separated MAC address into six raw octets. +/// +/// The guest kernel APIs do not consume the string form directly. They expect +/// the six raw Ethernet octets, so we translate: +/// +/// ```text +/// 02:53:4d:00:00:02 +/// -> [0x02, 0x53, 0x4d, 0x00, 0x00, 0x02] +/// ``` +/// +/// This parser is intentionally strict: exactly six hex octets separated by +/// `:` and nothing else. +fn parse_mac(value: &str) -> Result<[u8; 6], String> { + let mut mac = [0u8; 6]; + let mut count = 0usize; + for (index, part) in value.split(':').enumerate() { + if index >= 6 { + return Err(format!("invalid MAC address: {}", value)); + } + mac[index] = + u8::from_str_radix(part, 16).map_err(|_| format!("invalid MAC octet: {}", part))?; + count = index + 1; + } + if count != 6 { + return Err(format!("invalid MAC address: {}", value)); + } + Ok(mac) +} + +#[cfg(target_os = "linux")] +mod linux; + +#[cfg(not(target_os = "linux"))] +mod linux { + use std::net::{Ipv4Addr, Ipv6Addr}; + + #[allow(clippy::too_many_arguments)] + pub fn configure_interface( + _ifname: &str, + _mac: [u8; 6], + _mtu: u16, + _address: Ipv4Addr, + _prefix_len: u8, + _gateway: Ipv4Addr, + _ipv6: Option<(Ipv6Addr, u8, Ipv6Addr)>, + _dns_server: Ipv4Addr, + ) -> Result<(), String> { + Err("guest virtio networking is only supported on Linux".to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_mac_accepts_six_octets() { + assert_eq!( + parse_mac("02:53:4d:00:00:02").unwrap(), + [0x02, 0x53, 0x4d, 0x00, 0x00, 0x02] + ); + } + + #[test] + fn parse_mac_rejects_invalid_input() { + assert!(parse_mac("02:53:4d").is_err()); + assert!(parse_mac("zz:53:4d:00:00:02").is_err()); + } +} diff --git a/crates/smolvm-agent/src/oci.rs b/crates/smolvm-agent/src/oci.rs new file mode 100644 index 0000000..4de0182 --- /dev/null +++ b/crates/smolvm-agent/src/oci.rs @@ -0,0 +1,1520 @@ +//! OCI Runtime Specification generation for crun integration. +//! +//! This module provides types and functions for generating OCI-compliant +//! config.json files used by crun to execute containers. + +use serde::{Deserialize, Serialize}; +use std::fs; +use std::io::ErrorKind; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// OCI Runtime Specification (subset for container execution). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OciSpec { + #[serde(rename = "ociVersion")] + pub oci_version: String, + pub root: OciRoot, + pub process: OciProcess, + pub linux: OciLinux, + pub mounts: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub hostname: Option, +} + +/// Root filesystem configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OciRoot { + /// Path to the root filesystem (relative to bundle or absolute). + pub path: String, + /// Whether the root filesystem should be read-only. + #[serde(default)] + pub readonly: bool, +} + +/// Process configuration for the container. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OciProcess { + /// Whether to allocate a pseudo-terminal. + #[serde(default)] + pub terminal: bool, + /// Initial size of the container's pseudo-terminal (when `terminal: true`). + #[serde(rename = "consoleSize", skip_serializing_if = "Option::is_none")] + pub console_size: Option, + /// User and group IDs. + pub user: OciUser, + /// Command and arguments to execute. + pub args: Vec, + /// Environment variables in KEY=VALUE format. + #[serde(default)] + pub env: Vec, + /// Working directory inside the container. + pub cwd: String, + /// Linux capabilities (optional). + #[serde(skip_serializing_if = "Option::is_none")] + pub capabilities: Option, + /// Resource limits (optional). + #[serde(skip_serializing_if = "Option::is_none")] + pub rlimits: Option>, + /// Do not create a new session for the process. + #[serde(rename = "noNewPrivileges", default)] + pub no_new_privileges: bool, +} + +/// Initial size of a container's pseudo-terminal. +/// +/// Serialises as OCI `consoleSize: { height, width }`. The host sends a +/// `Resize` message with the real host terminal dimensions right after the +/// container starts, so this is just the size seen by the very first +/// frame the container draws. +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct OciConsoleSize { + pub height: u32, + pub width: u32, +} + +/// User configuration. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OciUser { + pub uid: u32, + pub gid: u32, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub additional_gids: Vec, +} + +/// Resolved process identity for container execution. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProcessIdentity { + pub user: OciUser, + pub home: Option, +} + +impl ProcessIdentity { + pub fn root() -> Self { + Self { + user: OciUser { + uid: 0, + gid: 0, + additional_gids: vec![], + }, + home: Some("/root".to_string()), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct PasswdEntry { + username: String, + uid: u32, + gid: u32, + home: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct GroupEntry { + name: String, + gid: u32, + members: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ResolvedUser { + uid: u32, + username: Option, + primary_gid: Option, + home: Option, +} + +pub fn resolve_process_identity( + rootfs: &Path, + user_spec: Option<&str>, +) -> Result { + let normalized = user_spec.map(str::trim).filter(|s| !s.is_empty()); + let passwd_entries = load_passwd_entries(rootfs)?; + let group_entries = load_group_entries(rootfs)?; + + if normalized.is_none() { + return Ok(resolve_default_root_identity( + &passwd_entries, + &group_entries, + )); + } + + let spec = normalized.expect("checked above"); + let (user_token, group_token) = parse_user_spec(spec)?; + + let resolved_user = resolve_user_token(user_token, &passwd_entries)?; + let primary_gid = match group_token { + Some(group) => resolve_group_token(group, &group_entries)?, + None => resolved_user.primary_gid.unwrap_or(0), + }; + + let additional_gids = supplemental_group_ids( + &group_entries, + resolved_user.username.as_deref(), + primary_gid, + ); + let home = resolved_user.home.or_else(|| { + if resolved_user.uid == 0 { + Some("/root".to_string()) + } else { + None + } + }); + + Ok(ProcessIdentity { + user: OciUser { + uid: resolved_user.uid, + gid: primary_gid, + additional_gids, + }, + home, + }) +} + +fn parse_user_spec(spec: &str) -> Result<(&str, Option<&str>), String> { + match spec.split_once(':') { + Some((user, group)) => { + if user.is_empty() { + return Err("invalid empty user in OCI image config".to_string()); + } + if group.is_empty() { + return Err("invalid empty group in OCI image config".to_string()); + } + Ok((user, Some(group))) + } + None => Ok((spec, None)), + } +} + +fn resolve_default_root_identity( + passwd_entries: &[PasswdEntry], + group_entries: &[GroupEntry], +) -> ProcessIdentity { + if let Some(root) = passwd_entries.iter().find(|entry| entry.username == "root") { + return ProcessIdentity { + user: OciUser { + uid: root.uid, + gid: root.gid, + additional_gids: supplemental_group_ids( + group_entries, + Some(root.username.as_str()), + root.gid, + ), + }, + home: Some(root.home.clone()), + }; + } + + ProcessIdentity::root() +} + +fn resolve_user_token(user: &str, passwd_entries: &[PasswdEntry]) -> Result { + if let Ok(uid) = user.parse::() { + if let Some(entry) = passwd_entries.iter().find(|entry| entry.uid == uid) { + return Ok(ResolvedUser { + uid, + username: Some(entry.username.clone()), + primary_gid: Some(entry.gid), + home: Some(entry.home.clone()), + }); + } + + return Ok(ResolvedUser { + uid, + username: None, + primary_gid: None, + home: None, + }); + } + + let entry = passwd_entries + .iter() + .find(|entry| entry.username == user) + .ok_or_else(|| format!("user '{}' not found in container rootfs", user))?; + + Ok(ResolvedUser { + uid: entry.uid, + username: Some(entry.username.clone()), + primary_gid: Some(entry.gid), + home: Some(entry.home.clone()), + }) +} + +fn resolve_group_token(group: &str, group_entries: &[GroupEntry]) -> Result { + if let Ok(gid) = group.parse::() { + return Ok(gid); + } + + group_entries + .iter() + .find(|entry| entry.name == group) + .map(|entry| entry.gid) + .ok_or_else(|| format!("group '{}' not found in container rootfs", group)) +} + +fn supplemental_group_ids( + group_entries: &[GroupEntry], + username: Option<&str>, + primary_gid: u32, +) -> Vec { + let Some(username) = username else { + return Vec::new(); + }; + + let mut gids = Vec::new(); + for entry in group_entries { + if entry.gid != primary_gid + && entry.members.iter().any(|member| member == username) + && !gids.contains(&entry.gid) + { + gids.push(entry.gid); + } + } + gids +} + +fn load_passwd_entries(rootfs: &Path) -> Result, String> { + load_entries(rootfs.join("etc/passwd"), parse_passwd_entries) +} + +fn load_group_entries(rootfs: &Path) -> Result, String> { + load_entries(rootfs.join("etc/group"), parse_group_entries) +} + +fn load_entries( + path: std::path::PathBuf, + parse: impl FnOnce(&str) -> Vec, +) -> Result, String> { + match fs::read_to_string(&path) { + Ok(contents) => Ok(parse(&contents)), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(Vec::new()), + Err(error) => Err(format!("failed to read {}: {}", path.display(), error)), + } +} + +fn parse_passwd_entries(contents: &str) -> Vec { + contents + .lines() + .filter_map(|line| { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + + let mut parts = line.splitn(7, ':'); + let username = parts.next()?; + let _password = parts.next()?; + let uid = parts.next()?.parse().ok()?; + let gid = parts.next()?.parse().ok()?; + let _gecos = parts.next()?; + let home = parts.next()?; + let _shell = parts.next()?; + + Some(PasswdEntry { + username: username.to_string(), + uid, + gid, + home: home.to_string(), + }) + }) + .collect() +} + +fn parse_group_entries(contents: &str) -> Vec { + contents + .lines() + .filter_map(|line| { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + + let mut parts = line.splitn(4, ':'); + let name = parts.next()?; + let _password = parts.next()?; + let gid = parts.next()?.parse().ok()?; + let members = parts + .next() + .map(|field| { + field + .split(',') + .filter(|member| !member.is_empty()) + .map(str::to_string) + .collect() + }) + .unwrap_or_default(); + + Some(GroupEntry { + name: name.to_string(), + gid, + members, + }) + }) + .collect() +} + +/// Linux capabilities configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OciCapabilities { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub bounding: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub effective: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub inheritable: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub permitted: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub ambient: Vec, +} + +/// Resource limit configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OciRlimit { + #[serde(rename = "type")] + pub rlimit_type: String, + pub hard: u64, + pub soft: u64, +} + +/// Linux-specific configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OciLinux { + /// Namespaces to create. + pub namespaces: Vec, + /// Device nodes to create in the container. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub devices: Vec, + /// Masked paths (paths that should appear empty). + #[serde(rename = "maskedPaths", default, skip_serializing_if = "Vec::is_empty")] + pub masked_paths: Vec, + /// Read-only paths. + #[serde( + rename = "readonlyPaths", + default, + skip_serializing_if = "Vec::is_empty" + )] + pub readonly_paths: Vec, +} + +/// Device node configuration for OCI runtime. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OciDevice { + /// Device type: "c" (char), "b" (block), "p" (pipe) + #[serde(rename = "type")] + pub device_type: String, + /// Path inside the container + pub path: String, + /// Major device number + pub major: u32, + /// Minor device number + pub minor: u32, + /// File mode/permissions + #[serde(rename = "fileMode", skip_serializing_if = "Option::is_none")] + pub file_mode: Option, + /// Owner UID + #[serde(skip_serializing_if = "Option::is_none")] + pub uid: Option, + /// Owner GID + #[serde(skip_serializing_if = "Option::is_none")] + pub gid: Option, +} + +/// Namespace configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OciNamespace { + /// Type of namespace (pid, network, mount, ipc, uts, user, cgroup). + #[serde(rename = "type")] + pub ns_type: String, + /// Path to an existing namespace to join (optional). + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, +} + +/// Mount configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OciMount { + /// Destination path inside the container. + pub destination: String, + /// Filesystem type (proc, sysfs, tmpfs, bind, etc.). + #[serde(rename = "type", skip_serializing_if = "Option::is_none")] + pub mount_type: Option, + /// Source path or device. + pub source: String, + /// Mount options. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub options: Vec, +} + +impl OciSpec { + /// Create a new OCI spec with sensible defaults for container execution. + /// + /// # Arguments + /// * `command` - Command and arguments to execute + /// * `env` - Environment variables as (key, value) pairs + /// * `workdir` - Working directory inside the container + /// * `tty` - Whether to allocate a pseudo-terminal + /// * `identity` - Resolved process uid/gid/home for the container + pub fn new( + command: &[String], + env: &[(String, String)], + workdir: &str, + tty: bool, + identity: &ProcessIdentity, + unprivileged: bool, + ) -> Self { + // Build environment variables + let mut env_strings = vec![ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(), + "TERM=xterm-256color".to_string(), + ]; + if !env.iter().any(|(key, _)| key == "HOME") { + if let Some(home) = &identity.home { + env_strings.push(format!("HOME={home}")); + } + } + env_strings.extend(env.iter().map(|(k, v)| format!("{}={}", k, v))); + + // Capabilities. Default is VM-grade (full set): the microVM is the security + // boundary, so the in-VM workload runs privileged — that's what lets init + // systems (systemd as PID 1) and tmpfs-mounting workloads boot (the "boot + // any image" promise). `unprivileged` opts into the restricted set for + // defense-in-depth with untrusted code. + let caps = if unprivileged { + default_capabilities() + } else { + full_capabilities() + }; + let capabilities = OciCapabilities { + bounding: caps.clone(), + effective: caps.clone(), + inheritable: vec![], + permitted: caps, + ambient: vec![], + }; + + Self { + oci_version: "1.0.2".to_string(), + root: OciRoot { + path: "rootfs".to_string(), + readonly: false, + }, + process: OciProcess { + terminal: tty, + console_size: None, + user: identity.user.clone(), + args: command.to_vec(), + env: env_strings, + cwd: workdir.to_string(), + capabilities: Some(capabilities), + rlimits: Some(vec![OciRlimit { + rlimit_type: "RLIMIT_NOFILE".to_string(), + hard: 1024, + soft: 1024, + }]), + no_new_privileges: false, + }, + linux: OciLinux { + namespaces: vec![ + OciNamespace { + ns_type: "pid".to_string(), + path: None, + }, + OciNamespace { + ns_type: "mount".to_string(), + path: None, + }, + OciNamespace { + ns_type: "ipc".to_string(), + path: None, + }, + OciNamespace { + ns_type: "uts".to_string(), + path: None, + }, + ], + devices: default_devices(), + // Mask/freeze the standard runc set of /proc and /sys paths only in + // unprivileged mode. In the default VM-grade mode the microVM is the + // security boundary (same rationale as the full capability set and the + // writable cgroup/tmpfs mounts above), so the workload gets a bare-VM + // /proc: an independent VM has all of /proc rw and unmasked, and + // freezing /proc/sys read-only here breaks in-VM init systems and + // docker-in-VM, which must write net.ipv4.ip_forward and friends. + masked_paths: if unprivileged { + vec![ + "/proc/asound".to_string(), + "/proc/acpi".to_string(), + "/proc/kcore".to_string(), + "/proc/keys".to_string(), + "/proc/latency_stats".to_string(), + "/proc/timer_list".to_string(), + "/proc/timer_stats".to_string(), + "/proc/sched_debug".to_string(), + "/proc/scsi".to_string(), + "/sys/firmware".to_string(), + ] + } else { + vec![] + }, + readonly_paths: if unprivileged { + vec![ + "/proc/bus".to_string(), + "/proc/fs".to_string(), + "/proc/irq".to_string(), + "/proc/sys".to_string(), + "/proc/sysrq-trigger".to_string(), + ] + } else { + vec![] + }, + }, + mounts: default_mounts(unprivileged), + hostname: Some("container".to_string()), + } + } + + /// Add a bind mount to the spec. + /// + /// # Arguments + /// * `source` - Source path on the host + /// * `destination` - Destination path inside the container + /// * `read_only` - Whether the mount should be read-only + pub fn add_bind_mount(&mut self, source: &str, destination: &str, read_only: bool) { + let mut options = vec!["bind".to_string(), "rprivate".to_string()]; + if read_only { + options.push("ro".to_string()); + } + self.mounts.push(OciMount { + destination: destination.to_string(), + mount_type: Some("bind".to_string()), + source: source.to_string(), + options, + }); + } + + /// Set or replace an environment variable on the container's process. + /// + /// If an entry with the same key already exists (e.g., inherited from + /// the image config), it is replaced — otherwise appended. This avoids + /// the container seeing two entries for the same variable, which would + /// leave the value shell-dependent. + pub fn add_env(&mut self, name: &str, value: &str) { + let prefix = format!("{}=", name); + self.process.env.retain(|entry| !entry.starts_with(&prefix)); + self.process.env.push(format!("{}{}", prefix, value)); + } + + /// Expose GPU render nodes to the container if /dev/dri exists in the VM. + /// + /// Detects virtio-gpu devices at runtime and adds a bind mount to the OCI + /// spec. This allows containers to use Vulkan (via Mesa Venus) without any + /// special configuration — if the VM was started with `--gpu`, the devices + /// appear automatically inside every container. + pub fn add_gpu_devices_if_available(&mut self) { + if self.mounts.iter().any(|m| m.destination == "/dev/dri") { + return; + } + + let dri_path = std::path::Path::new("/dev/dri"); + if !dri_path.exists() { + return; + } + + // Bind-mount /dev/dri from the VM's devtmpfs into the container. + // + // A bind mount is used rather than linux.devices entries because crun + // sets up a fresh tmpfs at /dev; the /dev/dri sub-directory does not + // exist in that tmpfs, so mknod silently fails for any device node + // whose parent directory is missing. A bind mount is equivalent to + // `--device /dev/dri` in Docker/Podman, and crun creates the + // destination directory automatically when it does not yet exist. + self.mounts.push(OciMount { + destination: "/dev/dri".to_string(), + mount_type: Some("bind".to_string()), + source: "/dev/dri".to_string(), + options: vec!["bind".to_string(), "rprivate".to_string()], + }); + } + + /// Write the OCI spec to a config.json file in the bundle directory. + pub fn write_to(&self, bundle_dir: &Path) -> std::io::Result<()> { + let config_path = bundle_dir.join("config.json"); + let json = serde_json::to_string_pretty(self) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + std::fs::write(config_path, json) + } +} + +/// Maximum allowed length for an image reference. +const MAX_IMAGE_REF_LENGTH: usize = 512; + +/// Validate an OCI image reference format. +/// +/// This validates that the image reference follows the expected format: +/// `[registry/][repository/]name[:tag][@digest]` +/// +/// # Arguments +/// * `image` - The image reference to validate +/// +/// # Returns +/// * `Ok(())` if valid +/// * `Err(message)` if invalid +/// +/// # Example valid references: +/// * `alpine` +/// * `alpine:latest` +/// * `alpine:3.18` +/// * `library/alpine` +/// * `docker.io/library/alpine:latest` +/// * `ghcr.io/owner/repo:tag` +/// * `alpine@sha256:abc123...` +pub fn validate_image_reference(image: &str) -> Result<(), String> { + // Check length + if image.is_empty() { + return Err("image reference cannot be empty".into()); + } + if image.len() > MAX_IMAGE_REF_LENGTH { + return Err(format!( + "image reference too long: {} bytes (max: {})", + image.len(), + MAX_IMAGE_REF_LENGTH + )); + } + + // Check for obviously dangerous characters that could enable injection + // These should never appear in valid OCI references + let forbidden_chars = ['$', '`', '|', ';', '&', '>', '<', '\n', '\r', '\0']; + for c in forbidden_chars { + if image.contains(c) { + return Err(format!( + "image reference contains forbidden character: {:?}", + c + )); + } + } + + // Check for shell metacharacters in sequence + if image.contains("..") && image.contains('/') { + // Path traversal attempt + return Err("image reference contains suspicious path traversal".into()); + } + + // Basic format validation: must have at least one valid character + // Valid characters: alphanumeric, '.', '-', '_', '/', ':', '@' + let valid_chars = |c: char| { + c.is_ascii_alphanumeric() + || c == '.' + || c == '-' + || c == '_' + || c == '/' + || c == ':' + || c == '@' + }; + + if !image.chars().all(valid_chars) { + return Err("image reference contains invalid characters".into()); + } + + // Must not start or end with special characters + // Note: We already checked for empty above, but use defensive programming + let first = match image.chars().next() { + Some(c) => c, + None => return Err("image reference is empty".into()), + }; + let last = match image.chars().last() { + Some(c) => c, + None => return Err("image reference is empty".into()), + }; + + if !first.is_ascii_alphanumeric() { + return Err("image reference must start with alphanumeric character".into()); + } + if !last.is_ascii_alphanumeric() { + return Err("image reference must end with alphanumeric character".into()); + } + + Ok(()) +} + +/// Validate environment variables. +/// +/// Environment variable keys must: +/// - Not be empty +/// - Start with a letter or underscore +/// - Contain only alphanumeric characters and underscores +/// - Not exceed 256 characters +/// +/// Values can be any string but must not exceed 32KB. +pub fn validate_env_vars(env: &[(String, String)]) -> Result<(), String> { + const MAX_KEY_LEN: usize = 256; + const MAX_VALUE_LEN: usize = 32 * 1024; // 32KB + + for (key, value) in env { + // Key validation + if key.is_empty() { + return Err("environment variable key cannot be empty".into()); + } + + if key.len() > MAX_KEY_LEN { + return Err(format!( + "environment variable key '{}...' exceeds {} character limit", + &key[..32.min(key.len())], + MAX_KEY_LEN + )); + } + + // Key must start with letter or underscore + // SAFETY: empty keys are rejected above + let first_char = key.chars().next().expect("key is non-empty"); + if !first_char.is_ascii_alphabetic() && first_char != '_' { + return Err(format!( + "environment variable key '{}' must start with a letter or underscore", + key + )); + } + + // Key must contain only alphanumeric and underscore + if !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err(format!( + "environment variable key '{}' contains invalid characters (only alphanumeric and underscore allowed)", + key + )); + } + + // Value length validation + if value.len() > MAX_VALUE_LEN { + return Err(format!( + "environment variable '{}' value exceeds {} byte limit", + key, MAX_VALUE_LEN + )); + } + } + + Ok(()) +} + +/// Generate a unique container ID. +/// +/// Uses a combination of timestamp and random bytes to ensure uniqueness +/// even when containers are created in rapid succession. +pub fn generate_container_id() -> String { + use std::fs::File; + use std::io::Read; + + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + + // Get 4 bytes of randomness from /dev/urandom + let random_bytes: u32 = File::open("/dev/urandom") + .and_then(|mut f| { + let mut buf = [0u8; 4]; + f.read_exact(&mut buf)?; + Ok(u32::from_ne_bytes(buf)) + }) + .unwrap_or_else(|_| { + // Fallback: use process ID and more timestamp bits if /dev/urandom fails + std::process::id() ^ ((timestamp >> 32) as u32) + }); + + // Combine lower 32 bits of timestamp with 32 bits of randomness + format!( + "smolvm-{:08x}{:08x}", + (timestamp & 0xFFFF_FFFF) as u32, + random_bytes + ) +} + +/// Default Linux capabilities for root containers. +// Retained as the fallback for a future opt-in unprivileged mode (`--unprivileged`). +#[allow(dead_code)] +fn default_capabilities() -> Vec { + vec![ + "CAP_CHOWN".to_string(), + "CAP_DAC_OVERRIDE".to_string(), + "CAP_FSETID".to_string(), + "CAP_FOWNER".to_string(), + "CAP_MKNOD".to_string(), + "CAP_NET_RAW".to_string(), + "CAP_SETGID".to_string(), + "CAP_SETUID".to_string(), + "CAP_SETFCAP".to_string(), + "CAP_SETPCAP".to_string(), + "CAP_NET_BIND_SERVICE".to_string(), + "CAP_SYS_CHROOT".to_string(), + "CAP_KILL".to_string(), + "CAP_AUDIT_WRITE".to_string(), + ] +} + +/// Full capability set for a "VM-grade" workload. smolvm runs one workload per +/// microVM, so the KVM boundary — not the in-VM container — is the security +/// boundary; dropping capabilities here protects nothing (an escape lands in the +/// VM's own single-tenant kernel) while breaking legitimate images that expect a +/// real machine (systemd/OpenRC as PID 1, or anything that mounts a tmpfs). So the +/// default container is effectively privileged. An opt-in unprivileged mode can +/// fall back to [`default_capabilities`] for defense-in-depth with untrusted code. +fn full_capabilities() -> Vec { + [ + "CAP_AUDIT_CONTROL", + "CAP_AUDIT_READ", + "CAP_AUDIT_WRITE", + "CAP_BLOCK_SUSPEND", + "CAP_BPF", + "CAP_CHECKPOINT_RESTORE", + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_DAC_READ_SEARCH", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_IPC_LOCK", + "CAP_IPC_OWNER", + "CAP_KILL", + "CAP_LEASE", + "CAP_LINUX_IMMUTABLE", + "CAP_MAC_ADMIN", + "CAP_MAC_OVERRIDE", + "CAP_MKNOD", + "CAP_NET_ADMIN", + "CAP_NET_BIND_SERVICE", + "CAP_NET_BROADCAST", + "CAP_NET_RAW", + "CAP_PERFMON", + "CAP_SETFCAP", + "CAP_SETGID", + "CAP_SETPCAP", + "CAP_SETUID", + "CAP_SYS_ADMIN", + "CAP_SYS_BOOT", + "CAP_SYS_CHROOT", + "CAP_SYS_MODULE", + "CAP_SYS_NICE", + "CAP_SYS_PACCT", + "CAP_SYS_PTRACE", + "CAP_SYS_RAWIO", + "CAP_SYS_RESOURCE", + "CAP_SYS_TIME", + "CAP_SYS_TTY_CONFIG", + "CAP_SYSLOG", + "CAP_WAKE_ALARM", + ] + .iter() + .map(|s| s.to_string()) + .collect() +} + +/// Default device nodes for container execution. +/// These are standard Linux devices that should exist in /dev. +fn default_devices() -> Vec { + vec![ + // /dev/null - discard all writes, reads return EOF + OciDevice { + device_type: "c".to_string(), + path: "/dev/null".to_string(), + major: 1, + minor: 3, + file_mode: Some(0o666), + uid: Some(0), + gid: Some(0), + }, + // /dev/zero - reads return null bytes + OciDevice { + device_type: "c".to_string(), + path: "/dev/zero".to_string(), + major: 1, + minor: 5, + file_mode: Some(0o666), + uid: Some(0), + gid: Some(0), + }, + // /dev/full - writes fail with ENOSPC + OciDevice { + device_type: "c".to_string(), + path: "/dev/full".to_string(), + major: 1, + minor: 7, + file_mode: Some(0o666), + uid: Some(0), + gid: Some(0), + }, + // /dev/random - random number generator (blocking) + OciDevice { + device_type: "c".to_string(), + path: "/dev/random".to_string(), + major: 1, + minor: 8, + file_mode: Some(0o666), + uid: Some(0), + gid: Some(0), + }, + // /dev/urandom - random number generator (non-blocking) + OciDevice { + device_type: "c".to_string(), + path: "/dev/urandom".to_string(), + major: 1, + minor: 9, + file_mode: Some(0o666), + uid: Some(0), + gid: Some(0), + }, + // /dev/tty - controlling terminal + OciDevice { + device_type: "c".to_string(), + path: "/dev/tty".to_string(), + major: 5, + minor: 0, + file_mode: Some(0o666), + uid: Some(0), + gid: Some(0), + }, + // /dev/kmsg - kernel log device. The kubelet (k3s/k3d, kind, most + // Kubernetes-in-Docker) hard-requires it and refuses to start without + // it ("failed to create kubelet: open /dev/kmsg: no such file or + // directory"). Bare VMs get it from devtmpfs, but the container /dev is + // built from this list, so add it here so nested Kubernetes works out + // of the box. Each machine is its own microVM, so exposing its kernel + // log to the container is not a cross-tenant concern. + OciDevice { + device_type: "c".to_string(), + path: "/dev/kmsg".to_string(), + major: 1, + minor: 11, + file_mode: Some(0o644), + uid: Some(0), + gid: Some(0), + }, + ] +} + +/// Default mounts for container execution. +fn default_mounts(unprivileged: bool) -> Vec { + let mut mounts = vec![ + // /proc - process information + OciMount { + destination: "/proc".to_string(), + mount_type: Some("proc".to_string()), + source: "proc".to_string(), + options: vec![ + "nosuid".to_string(), + "noexec".to_string(), + "nodev".to_string(), + ], + }, + // /dev - device nodes + OciMount { + destination: "/dev".to_string(), + mount_type: Some("tmpfs".to_string()), + source: "tmpfs".to_string(), + options: vec![ + "nosuid".to_string(), + "strictatime".to_string(), + "mode=755".to_string(), + "size=65536k".to_string(), + ], + }, + // /dev/pts - pseudo-terminal devices + OciMount { + destination: "/dev/pts".to_string(), + mount_type: Some("devpts".to_string()), + source: "devpts".to_string(), + options: vec![ + "nosuid".to_string(), + "noexec".to_string(), + "newinstance".to_string(), + "ptmxmode=0666".to_string(), + "mode=0620".to_string(), + ], + }, + // /dev/shm - shared memory + OciMount { + destination: "/dev/shm".to_string(), + mount_type: Some("tmpfs".to_string()), + source: "shm".to_string(), + options: vec![ + "nosuid".to_string(), + "noexec".to_string(), + "nodev".to_string(), + "mode=1777".to_string(), + "size=65536k".to_string(), + ], + }, + // /dev/mqueue - POSIX message queues + OciMount { + destination: "/dev/mqueue".to_string(), + mount_type: Some("mqueue".to_string()), + source: "mqueue".to_string(), + options: vec![ + "nosuid".to_string(), + "noexec".to_string(), + "nodev".to_string(), + ], + }, + // /sys - sysfs (read-only for security) + OciMount { + destination: "/sys".to_string(), + mount_type: Some("sysfs".to_string()), + source: "sysfs".to_string(), + options: vec![ + "nosuid".to_string(), + "noexec".to_string(), + "nodev".to_string(), + "ro".to_string(), + ], + }, + ]; + // /sys/fs/cgroup — cgroup2. Writable by default so an init system (systemd) can + // manage its own cgroup subtree; read-only in unprivileged mode. + mounts.push(OciMount { + destination: "/sys/fs/cgroup".to_string(), + mount_type: Some("cgroup2".to_string()), + source: "cgroup".to_string(), + options: vec![ + "nosuid".to_string(), + "noexec".to_string(), + "nodev".to_string(), + if unprivileged { "ro" } else { "rw" }.to_string(), + ], + }); + if !unprivileged { + // tmpfs that init systems and many services set up at boot. Pre-providing + // them means the workload doesn't have to mount them itself, and read-only + // rootfs images still get writable runtime dirs. Omitted when unprivileged + // (the workload can't mount, but also isn't expected to be an init system). + mounts.push(OciMount { + destination: "/run".to_string(), + mount_type: Some("tmpfs".to_string()), + source: "tmpfs".to_string(), + options: vec![ + "nosuid".to_string(), + "nodev".to_string(), + "mode=0755".to_string(), + "size=65536k".to_string(), + ], + }); + mounts.push(OciMount { + destination: "/run/lock".to_string(), + mount_type: Some("tmpfs".to_string()), + source: "tmpfs".to_string(), + options: vec![ + "nosuid".to_string(), + "nodev".to_string(), + "noexec".to_string(), + "mode=1777".to_string(), + "size=5120k".to_string(), + ], + }); + mounts.push(OciMount { + destination: "/tmp".to_string(), + mount_type: Some("tmpfs".to_string()), + source: "tmpfs".to_string(), + options: vec![ + "nosuid".to_string(), + "nodev".to_string(), + "mode=1777".to_string(), + ], + }); + } + + // Drop mounts whose filesystem the guest kernel doesn't provide. The WHP + // guest kernel may lack CONFIG_POSIX_MQUEUE; crun fails the entire container + // with "mount mqueue: No such device" if /dev/mqueue can't be mounted, and + // almost no workload needs POSIX message queues. /proc/filesystems lists the + // kernel's known filesystems (e.g. "nodev\tmqueue"). + if !proc_filesystems_has("mqueue") { + mounts.retain(|m| m.mount_type.as_deref() != Some("mqueue")); + } + mounts +} + +/// Whether the guest kernel registers `fstype` (per /proc/filesystems). Assumes +/// supported when the file can't be read, to avoid over-filtering. +fn proc_filesystems_has(fstype: &str) -> bool { + std::fs::read_to_string("/proc/filesystems") + .map(|s| s.split_whitespace().any(|w| w == fstype)) + .unwrap_or(true) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + #[test] + fn test_validate_image_reference_valid() { + // Valid references should pass + assert!(validate_image_reference("alpine").is_ok()); + assert!(validate_image_reference("alpine:latest").is_ok()); + assert!(validate_image_reference("alpine:3.18").is_ok()); + assert!(validate_image_reference("library/alpine").is_ok()); + assert!(validate_image_reference("docker.io/library/alpine").is_ok()); + assert!(validate_image_reference("ghcr.io/owner/repo:tag").is_ok()); + assert!(validate_image_reference("my-registry.com/my-image:v1.0.0").is_ok()); + assert!(validate_image_reference("alpine@sha256:abc123def456").is_ok()); + } + + #[test] + fn test_validate_image_reference_invalid() { + // Empty + assert!(validate_image_reference("").is_err()); + + // Forbidden characters (shell injection) + assert!(validate_image_reference("alpine; rm -rf /").is_err()); + assert!(validate_image_reference("alpine | cat /etc/passwd").is_err()); + assert!(validate_image_reference("alpine`whoami`").is_err()); + assert!(validate_image_reference("alpine$PATH").is_err()); + assert!(validate_image_reference("alpine > /tmp/x").is_err()); + assert!(validate_image_reference("alpine\nmalicious").is_err()); + + // Invalid characters + assert!(validate_image_reference("alpine image").is_err()); // space + assert!(validate_image_reference("alpine!").is_err()); + + // Must start/end with alphanumeric + assert!(validate_image_reference("/alpine").is_err()); + assert!(validate_image_reference("alpine:").is_err()); + assert!(validate_image_reference("-alpine").is_err()); + } + + #[test] + fn test_validate_image_reference_length() { + // Very long reference should fail + let long_ref = "a".repeat(600); + assert!(validate_image_reference(&long_ref).is_err()); + + // Just under limit should pass + let ok_ref = "a".repeat(500); + assert!(validate_image_reference(&ok_ref).is_ok()); + } + + #[test] + fn test_generate_container_id() { + let id1 = generate_container_id(); + let id2 = generate_container_id(); + + assert!(id1.starts_with("smolvm-")); + assert!(id2.starts_with("smolvm-")); + // ID format: smolvm-{8 hex}{8 hex} = "smolvm-" (7) + 16 hex chars = 23 total + assert_eq!(id1.len(), 23); + assert_eq!(id2.len(), 23); + // IDs should be unique (different timestamps + random bytes) + assert_ne!(id1, id2); + } + + #[test] + fn test_oci_spec_creation() { + let spec = OciSpec::new( + &["echo".to_string(), "hello".to_string()], + &[("FOO".to_string(), "bar".to_string())], + "/", + false, + &ProcessIdentity::root(), + false, + ); + + assert_eq!(spec.oci_version, "1.0.2"); + assert_eq!(spec.process.args, vec!["echo", "hello"]); + assert!(spec.process.env.contains(&"FOO=bar".to_string())); + assert!(spec.process.env.contains(&"HOME=/root".to_string())); + assert!(!spec.process.terminal); + assert!(spec.process.console_size.is_none()); + + // /dev/kmsg (1:11) must be in the container's device set — the kubelet + // (k3s/k3d, kind) refuses to start without it, so nested Kubernetes + // works out of the box. + let kmsg = spec + .linux + .devices + .iter() + .find(|d| d.path == "/dev/kmsg") + .expect("/dev/kmsg device present"); + assert_eq!( + (kmsg.device_type.as_str(), kmsg.major, kmsg.minor), + ("c", 1, 11) + ); + } + + #[test] + fn test_oci_console_size_serialises_as_camel_case() { + let mut spec = OciSpec::new( + &["sh".to_string()], + &[], + "/", + true, + &ProcessIdentity::root(), + false, + ); + spec.process.console_size = Some(OciConsoleSize { + height: 24, + width: 80, + }); + let json = serde_json::to_value(&spec).unwrap(); + let cs = json + .get("process") + .and_then(|p| p.get("consoleSize")) + .expect("consoleSize should serialise as camelCase under process"); + assert_eq!(cs.get("height").and_then(|v| v.as_u64()), Some(24)); + assert_eq!(cs.get("width").and_then(|v| v.as_u64()), Some(80)); + } + + #[test] + fn test_oci_console_size_omitted_when_none() { + let spec = OciSpec::new( + &["sh".to_string()], + &[], + "/", + true, + &ProcessIdentity::root(), + false, + ); + let json = serde_json::to_value(&spec).unwrap(); + assert!( + json.get("process") + .and_then(|p| p.get("consoleSize")) + .is_none(), + "consoleSize must be omitted when unset" + ); + } + + #[test] + fn test_add_bind_mount() { + let mut spec = OciSpec::new( + &["sh".to_string()], + &[], + "/", + false, + &ProcessIdentity::root(), + false, + ); + spec.add_bind_mount("/host/path", "/container/path", true); + + let mount = spec.mounts.last().unwrap(); + assert_eq!(mount.destination, "/container/path"); + assert_eq!(mount.source, "/host/path"); + assert!(mount.options.contains(&"ro".to_string())); + } + + #[test] + fn test_validate_env_vars_valid() { + // Valid env vars should pass + assert!(validate_env_vars(&[]).is_ok()); + assert!(validate_env_vars(&[("FOO".to_string(), "bar".to_string())]).is_ok()); + assert!(validate_env_vars(&[("_FOO".to_string(), "bar".to_string())]).is_ok()); + assert!(validate_env_vars(&[("FOO_BAR".to_string(), "baz".to_string())]).is_ok()); + assert!(validate_env_vars(&[("FOO123".to_string(), "value".to_string())]).is_ok()); + assert!(validate_env_vars(&[("PATH".to_string(), "/usr/bin:/bin".to_string())]).is_ok()); + // Empty values are allowed + assert!(validate_env_vars(&[("EMPTY".to_string(), "".to_string())]).is_ok()); + } + + #[test] + fn test_validate_env_vars_invalid_keys() { + // Empty key + assert!(validate_env_vars(&[("".to_string(), "value".to_string())]).is_err()); + + // Key starting with number + assert!(validate_env_vars(&[("1FOO".to_string(), "value".to_string())]).is_err()); + + // Key with invalid characters + assert!(validate_env_vars(&[("FOO-BAR".to_string(), "value".to_string())]).is_err()); + assert!(validate_env_vars(&[("FOO.BAR".to_string(), "value".to_string())]).is_err()); + assert!(validate_env_vars(&[("FOO BAR".to_string(), "value".to_string())]).is_err()); + assert!(validate_env_vars(&[("FOO=BAR".to_string(), "value".to_string())]).is_err()); + } + + #[test] + fn test_resolve_process_identity_named_user_uses_passwd_and_groups() { + let rootfs = tempdir().unwrap(); + let etc = rootfs.path().join("etc"); + std::fs::create_dir_all(&etc).unwrap(); + std::fs::write( + etc.join("passwd"), + "root:x:0:0:root:/root:/bin/sh\nsteam:x:1000:1000::/home/steam:/bin/sh\n", + ) + .unwrap(); + std::fs::write( + etc.join("group"), + "root:x:0:\nsteam:x:1000:\naudio:x:29:steam\nvideo:x:44:steam\n", + ) + .unwrap(); + + let identity = resolve_process_identity(rootfs.path(), Some("steam")).unwrap(); + + assert_eq!(identity.user.uid, 1000); + assert_eq!(identity.user.gid, 1000); + assert_eq!(identity.user.additional_gids, vec![29, 44]); + assert_eq!(identity.home.as_deref(), Some("/home/steam")); + } + + #[test] + fn test_resolve_process_identity_numeric_uid_gid_without_passwd() { + let rootfs = tempdir().unwrap(); + std::fs::create_dir_all(rootfs.path().join("etc")).unwrap(); + + let identity = resolve_process_identity(rootfs.path(), Some("1234:2345")).unwrap(); + + assert_eq!(identity.user.uid, 1234); + assert_eq!(identity.user.gid, 2345); + assert!(identity.user.additional_gids.is_empty()); + assert!(identity.home.is_none()); + } + + #[test] + fn test_parse_user_spec_rejects_empty_user_or_group() { + assert_eq!( + parse_user_spec(":1000"), + Err("invalid empty user in OCI image config".to_string()) + ); + assert_eq!( + parse_user_spec("steam:"), + Err("invalid empty group in OCI image config".to_string()) + ); + assert_eq!(parse_user_spec("steam"), Ok(("steam", None))); + assert_eq!(parse_user_spec("steam:audio"), Ok(("steam", Some("audio")))); + } + + #[test] + fn test_oci_spec_uses_identity_home_when_env_does_not_override_it() { + let spec = OciSpec::new( + &["id".to_string()], + &[("FOO".to_string(), "bar".to_string())], + "/home/steam", + false, + &ProcessIdentity { + user: OciUser { + uid: 1000, + gid: 1000, + additional_gids: vec![29], + }, + home: Some("/home/steam".to_string()), + }, + false, + ); + + assert!(spec.process.env.contains(&"HOME=/home/steam".to_string())); + assert_eq!(spec.process.user.uid, 1000); + assert_eq!(spec.process.user.gid, 1000); + assert_eq!(spec.process.user.additional_gids, vec![29]); + } + + #[test] + fn test_validate_env_vars_length_limits() { + // Key too long (> 256 chars) + let long_key = "A".repeat(300); + assert!(validate_env_vars(&[(long_key, "value".to_string())]).is_err()); + + // Value too long (> 32KB) + let long_value = "x".repeat(33 * 1024); + assert!(validate_env_vars(&[("KEY".to_string(), long_value)]).is_err()); + + // Values just under limit should pass + let ok_value = "x".repeat(32 * 1024); + assert!(validate_env_vars(&[("KEY".to_string(), ok_value)]).is_ok()); + } + + #[test] + fn privileged_spec_leaves_proc_unrestricted_unprivileged_keeps_runc_hardening() { + let identity = ProcessIdentity::root(); + + // VM-grade (default, unprivileged=false): the microVM is the boundary, so + // /proc is a bare-VM /proc — nothing masked, nothing frozen read-only. This + // is what lets docker-in-VM write /proc/sys/net/ipv4/ip_forward. + let privileged = OciSpec::new(&["sh".to_string()], &[], "/", false, &identity, false); + assert!( + privileged.linux.masked_paths.is_empty(), + "privileged spec must not mask any /proc paths" + ); + assert!( + privileged.linux.readonly_paths.is_empty(), + "privileged spec must not freeze /proc/sys read-only" + ); + + // Unprivileged opt-in (untrusted code): the standard runc masked/readonly + // set is preserved for defense-in-depth. + let unprivileged = OciSpec::new(&["sh".to_string()], &[], "/", false, &identity, true); + assert!( + unprivileged + .linux + .readonly_paths + .contains(&"/proc/sys".to_string()), + "unprivileged spec must keep /proc/sys read-only" + ); + assert!( + unprivileged + .linux + .masked_paths + .contains(&"/proc/kcore".to_string()), + "unprivileged spec must keep /proc/kcore masked" + ); + } + + #[test] + fn test_gpu_devices_added_when_dri_exists() { + // On a system with /dev/dri (GPU-enabled VM), a bind mount is added + let identity = ProcessIdentity::root(); + let mut spec = OciSpec::new(&["echo".to_string()], &[], "/", false, &identity, false); + let mounts_before = spec.mounts.len(); + spec.add_gpu_devices_if_available(); + + if std::path::Path::new("/dev/dri").exists() { + // GPU present — /dev/dri bind mount was added + assert!(spec.mounts.len() > mounts_before); + assert!(spec.mounts.iter().any(|m| m.destination == "/dev/dri")); + assert!(spec + .mounts + .iter() + .any(|m| m.destination == "/dev/dri" && m.options.contains(&"bind".to_string()))); + } else { + // No GPU — no extra mounts added (no-op) + assert_eq!(spec.mounts.len(), mounts_before); + } + } + + #[test] + fn test_gpu_devices_are_not_added_twice() { + let identity = ProcessIdentity::root(); + let mut spec = OciSpec::new(&["echo".to_string()], &[], "/", false, &identity, false); + spec.mounts.push(super::OciMount { + destination: "/dev/dri".to_string(), + mount_type: Some("bind".to_string()), + source: "/dev/dri".to_string(), + options: vec!["bind".to_string(), "rprivate".to_string()], + }); + + spec.add_gpu_devices_if_available(); + + assert_eq!( + spec.mounts + .iter() + .filter(|m| m.destination == "/dev/dri") + .count(), + 1 + ); + } + + #[test] + fn test_gpu_devices_correct_properties() { + let identity = ProcessIdentity::root(); + let mut spec = OciSpec::new(&["echo".to_string()], &[], "/", false, &identity, false); + // Manually push a GPU bind mount to verify expected properties + spec.mounts.push(super::OciMount { + destination: "/dev/dri".to_string(), + mount_type: Some("bind".to_string()), + source: "/dev/dri".to_string(), + options: vec!["bind".to_string(), "rprivate".to_string()], + }); + + let mount = spec + .mounts + .iter() + .find(|m| m.destination == "/dev/dri") + .unwrap(); + assert_eq!(mount.mount_type, Some("bind".to_string())); + assert_eq!(mount.source, "/dev/dri"); + // rprivate prevents the host from seeing any new mounts inside the + // container's /dev/dri subtree. + assert!(mount.options.contains(&"rprivate".to_string())); + } +} diff --git a/crates/smolvm-agent/src/paths.rs b/crates/smolvm-agent/src/paths.rs new file mode 100644 index 0000000..e887e5f --- /dev/null +++ b/crates/smolvm-agent/src/paths.rs @@ -0,0 +1,163 @@ +//! Centralized path constants and helpers for the smolvm agent. +//! +//! All filesystem paths used by the agent are defined here for consistency +//! and easy modification. + +use std::path::PathBuf; + +// ============================================================================= +// Binary Paths +// ============================================================================= + +/// Path to crun OCI runtime binary. +pub const CRUN_PATH: &str = "/usr/bin/crun"; + +/// crun state root directory. +/// Stored on the persistent storage disk instead of `/run/crun` because +/// `/run` may not be writable under the overlayfs rootfs. +pub const CRUN_ROOT_DIR: &str = "/storage/containers/crun"; + +/// crun cgroup manager setting. +/// Set to "disabled" because libkrun mounts cgroup2 as read-only. +/// Without this, crun create/start hang trying to create container cgroups. +pub const CRUN_CGROUP_MANAGER: &str = "disabled"; + +// ============================================================================= +// Mount Paths +// ============================================================================= + +/// Root directory for virtiofs mounts from the host. +pub const VIRTIOFS_MOUNT_ROOT: &str = "/mnt/virtiofs"; + +/// Guest path at which the shared workspace is exposed inside containers. +/// Used both when mounting /storage/workspace as the fallback workspace and +/// when checking whether a user-provided volume already claims this path. +pub const WORKSPACE_GUEST_PATH: &str = "/workspace"; + +// ============================================================================= +// Storage Paths +// ============================================================================= + +/// Root directory for all persistent storage. +pub const STORAGE_ROOT: &str = "/storage"; + +/// Directory for overlay filesystems. +pub const OVERLAYS_DIR: &str = "/storage/overlays"; + +// ============================================================================= +// Container Runtime Paths +// ============================================================================= + +/// Directory for per-container runtime state (pidfile, etc). +pub const CONTAINERS_RUN_DIR: &str = "/storage/containers/run"; + +/// Directory for container logs. +pub const CONTAINERS_LOGS_DIR: &str = "/storage/containers/logs"; + +/// Directory for container exit code files. +pub const CONTAINERS_EXIT_DIR: &str = "/storage/containers/exit"; + +// ============================================================================= +// Path Helper Functions +// ============================================================================= + +/// Get the runtime directory for a specific container. +pub fn container_run_dir(container_id: &str) -> PathBuf { + PathBuf::from(CONTAINERS_RUN_DIR).join(container_id) +} + +/// Get the log file path for a container. +pub fn container_log_path(container_id: &str) -> PathBuf { + PathBuf::from(CONTAINERS_LOGS_DIR).join(format!("{}.log", container_id)) +} + +/// Get the exit code file path for a container. +pub fn container_exit_path(container_id: &str) -> PathBuf { + PathBuf::from(CONTAINERS_EXIT_DIR).join(container_id) +} + +/// Get the overlay directory for a workload. +pub fn overlay_dir(workload_id: &str) -> PathBuf { + PathBuf::from(OVERLAYS_DIR).join(workload_id) +} + +/// Get the bundle directory for a workload. +pub fn bundle_dir(workload_id: &str) -> PathBuf { + overlay_dir(workload_id).join("bundle") +} + +/// Path to the file recording the main container ID for a persistent overlay. +/// Written when a detached container is started; read on every subsequent exec. +pub fn main_container_id_path(workload_id: &str) -> PathBuf { + overlay_dir(workload_id).join("main_container_id") +} + +// ============================================================================= +// Filesystem Helpers +// ============================================================================= + +/// Check if a path is a mountpoint by reading /proc/mounts. +/// +/// Returns true if the path appears as a mount destination in /proc/mounts. +#[cfg(target_os = "linux")] +pub fn is_mount_point(path: &std::path::Path) -> bool { + if let Ok(mounts) = std::fs::read_to_string("/proc/mounts") { + let path_str = path.to_string_lossy(); + for line in mounts.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() >= 2 && parts[1] == path_str { + return true; + } + } + } + false +} + +/// Stub for non-Linux platforms. +#[cfg(not(target_os = "linux"))] +pub fn is_mount_point(_path: &std::path::Path) -> bool { + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_container_paths() { + let id = "abc123"; + assert_eq!( + container_run_dir(id), + PathBuf::from("/storage/containers/run/abc123") + ); + assert_eq!( + container_log_path(id), + PathBuf::from("/storage/containers/logs/abc123.log") + ); + assert_eq!( + container_exit_path(id), + PathBuf::from("/storage/containers/exit/abc123") + ); + } + + #[test] + fn test_overlay_paths() { + let wl = "workload-123"; + assert_eq!( + overlay_dir(wl), + PathBuf::from("/storage/overlays/workload-123") + ); + assert_eq!( + bundle_dir(wl), + PathBuf::from("/storage/overlays/workload-123/bundle") + ); + } + + #[test] + fn test_main_container_id_path() { + assert_eq!( + main_container_id_path("persistent-myvm"), + PathBuf::from("/storage/overlays/persistent-myvm/main_container_id") + ); + } +} diff --git a/crates/smolvm-agent/src/process.rs b/crates/smolvm-agent/src/process.rs new file mode 100644 index 0000000..48494b3 --- /dev/null +++ b/crates/smolvm-agent/src/process.rs @@ -0,0 +1,645 @@ +//! Process execution utilities for the smolvm agent. +//! +//! This module provides common helpers for spawning and managing child processes, +//! including timeout handling and output capture. + +use std::io::Read; +use std::process::Child; +use std::time::{Duration, Instant}; + +/// Exit code used when a command is killed due to timeout. +pub const TIMEOUT_EXIT_CODE: i32 = 124; + +/// Map a process exit status to a numeric exit code. +/// +/// Normal exit → the process's exit code. Terminated by a signal (where +/// `ExitStatus::code()` is `None`) → `128 + signal`, matching shell +/// convention (SIGINT → 130, SIGTERM → 143), so callers and scripts can +/// distinguish a signal kill from a normal exit instead of seeing an +/// opaque 255. +pub fn exit_code_from_status(status: &std::process::ExitStatus) -> i32 { + use std::os::unix::process::ExitStatusExt; + if let Some(code) = status.code() { + code + } else if let Some(sig) = status.signal() { + 128 + sig + } else { + -1 + } +} + +/// Per-stream output cap for non-interactive exec. Vec is base64-encoded +/// in JSON frames (4/3 expansion). Two streams at this cap must fit within the +/// 32 MB frame limit with room for JSON overhead: +/// 11 MiB × 2 × 4/3 ≈ 29.3 MiB encoded + ~2.7 MiB JSON headroom. +pub const MAX_EXEC_OUTPUT: usize = 11 * 1024 * 1024; + +/// Maximum time to wait for reader threads to finish after the child is killed. +/// Guards against pathological cases where an inherited fd keeps a pipe open. +const READER_JOIN_TIMEOUT: Duration = Duration::from_secs(5); + +/// Captured output from a child process. +#[derive(Debug, Default)] +pub struct ChildOutput { + pub stdout: Vec, + pub stderr: Vec, +} + +/// Result of waiting for a child process. +#[derive(Debug)] +pub enum WaitResult { + /// Process completed with the given exit code. + Completed { exit_code: i32, output: ChildOutput }, + /// Process was killed due to timeout. + TimedOut { + output: ChildOutput, + timeout_ms: u64, + }, + /// Process was killed because the requesting client disconnected. + /// Used to free the accept loop when the client gives up mid-exec. + ClientDisconnected { output: ChildOutput }, +} + +/// Check whether the peer on `fd` has closed the connection. +/// +/// Uses `recv(MSG_PEEK | MSG_DONTWAIT)` which is more reliable than `poll()` +/// on vsock — vsock's poll implementation doesn't always propagate POLLHUP +/// when the peer closes, but a zero-length peek is the canonical way to +/// detect half-closed sockets. +/// +/// Returns `true` if the peer has closed OR the socket is in an error state. +/// Returns `false` if the socket is still alive OR we can't determine (fail +/// open — a bogus fd shouldn't cause us to kill a healthy child). +#[cfg(target_os = "linux")] +pub fn is_peer_closed(fd: std::os::unix::io::RawFd) -> bool { + if fd < 0 { + return false; + } + let mut buf = [0u8; 1]; + // SAFETY: buf is a valid write target, MSG_PEEK doesn't consume data. + let rc = unsafe { + libc::recv( + fd, + buf.as_mut_ptr() as *mut libc::c_void, + buf.len(), + libc::MSG_PEEK | libc::MSG_DONTWAIT, + ) + }; + if rc == 0 { + // Peer performed orderly shutdown (FIN received). + return true; + } + if rc < 0 { + let errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0); + // EAGAIN = no data but connection alive → peer still there. (EWOULDBLOCK + // is the same value as EAGAIN on Linux.) Any other error (ECONNRESET, + // ENOTCONN, EBADF, etc.) → peer gone. + return !matches!(errno, libc::EAGAIN); + } + // rc > 0: there's data in the buffer — connection is alive. + false +} + +#[cfg(not(target_os = "linux"))] +pub fn is_peer_closed(_fd: std::os::unix::io::RawFd) -> bool { + false +} + +/// Capture stdout and stderr from a child process. +/// +/// Reads raw bytes — preserves binary output (image bytes, tarballs, etc.) +/// that `read_to_string` would truncate at the first non-UTF-8 byte. +/// +/// # Safety note +/// +/// Call this only AFTER the process has already exited (or been killed). +/// If the process is still running and has filled the pipe buffer (~64 KB on +/// Linux), reading blocks until the process exits — which it cannot do while +/// the pipe is full. Prefer `spawn_pipe_drains` + `join_pipe_drains` when +/// the process is still running. +pub fn capture_child_output(child: &mut Child) -> ChildOutput { + let mut output = ChildOutput::default(); + + if let Some(mut stdout) = child.stdout.take() { + let _ = stdout.read_to_end(&mut output.stdout); + } + if let Some(mut stderr) = child.stderr.take() { + let _ = stderr.read_to_end(&mut output.stderr); + } + + output +} + +/// Start background threads that drain the child's stdout and stderr pipes. +/// +/// This must be called BEFORE the wait loop so the pipes are continuously +/// drained while the process runs. If a process fills the ~64 KB Linux pipe +/// buffer and nobody is reading, its next `write(2)` blocks and the process +/// can never exit — a classic pipe-deadlock. +/// +/// Returns `(stdout_drain, stderr_drain)` join handles. Call +/// `join_pipe_drains` to collect the output after the process has exited. +pub fn spawn_pipe_drains( + child: &mut Child, +) -> ( + Option>>, + Option>>, +) { + let stdout_handle = child.stdout.take().map(|mut pipe| { + std::thread::spawn(move || { + let mut buf = Vec::new(); + let _ = pipe.read_to_end(&mut buf); + buf + }) + }); + let stderr_handle = child.stderr.take().map(|mut pipe| { + std::thread::spawn(move || { + let mut buf = Vec::new(); + let _ = pipe.read_to_end(&mut buf); + buf + }) + }); + (stdout_handle, stderr_handle) +} + +/// Collect output from pipe-drain threads started by `spawn_pipe_drains`. +pub fn join_pipe_drains( + stdout_handle: Option>>, + stderr_handle: Option>>, +) -> ChildOutput { + ChildOutput { + stdout: stdout_handle + .and_then(|h| h.join().ok()) + .unwrap_or_default(), + stderr: stderr_handle + .and_then(|h| h.join().ok()) + .unwrap_or_default(), + } +} + +/// Wait for a child process with optional timeout. +/// +/// If timeout_ms is Some, the process will be killed after the timeout +/// and WaitResult::TimedOut will be returned. +/// +/// The poll_interval_ms parameter controls how often we check for completion +/// (default: 10ms). +/// +/// Handles EINTR (interrupted system call) by retrying the wait. +pub fn wait_with_timeout( + child: &mut Child, + timeout_ms: Option, + poll_interval_ms: Option, +) -> std::io::Result { + let poll_interval = Duration::from_millis(poll_interval_ms.unwrap_or(10)); + let deadline = timeout_ms.map(|ms| Instant::now() + Duration::from_millis(ms)); + + // Drain pipes in background threads so they never fill up and deadlock the child. + let (stdout_drain, stderr_drain) = spawn_pipe_drains(child); + + loop { + match try_wait_with_eintr(child) { + Ok(Some(status)) => { + // Process completed — join drains to get full output. + let output = join_pipe_drains(stdout_drain, stderr_drain); + let exit_code = exit_code_from_status(&status); + return Ok(WaitResult::Completed { exit_code, output }); + } + Ok(None) => { + // Still running - check timeout + if let Some(deadline) = deadline { + if Instant::now() >= deadline { + // Kill the process + let _ = child.kill(); + let _ = child.wait(); + + // Collect any partial output from the drain threads. + let output = join_pipe_drains(stdout_drain, stderr_drain); + + return Ok(WaitResult::TimedOut { + output, + timeout_ms: timeout_ms.unwrap_or(0), + }); + } + } + + // Sleep before checking again + std::thread::sleep(poll_interval); + } + Err(e) => return Err(e), + } + } +} + +/// Try to wait for a child process, handling EINTR by retrying. +/// +/// EINTR can occur when a signal is delivered during the wait syscall. +/// This is not a real error - we should just retry the wait. +fn try_wait_with_eintr(child: &mut Child) -> std::io::Result> { + loop { + match child.try_wait() { + Ok(status) => return Ok(status), + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => { + // EINTR - signal interrupted the syscall, retry + continue; + } + Err(e) => return Err(e), + } + } +} + +/// Wait for a child process with timeout and custom timeout handler. +/// +/// The on_timeout callback is called when the process times out, before +/// killing it. This allows for custom cleanup (e.g., killing containers). +/// +/// Handles EINTR (interrupted system call) by retrying the wait. +pub fn wait_with_timeout_and_cleanup( + child: &mut Child, + timeout_ms: Option, + on_timeout: F, +) -> std::io::Result +where + F: FnOnce(), +{ + wait_with_timeout_cleanup_and_liveness(child, timeout_ms, None, on_timeout) +} + +/// Wait for a child process, killing it if the timeout expires OR if the +/// requesting client disconnects (indicated by `client_fd`, which is polled +/// each iteration). +/// +/// Stdout and stderr are drained concurrently in background threads to prevent +/// pipe deadlock: if the child writes more than the OS pipe buffer (~64KB), +/// it blocks on write() while the agent blocks waiting for exit — neither side +/// makes progress. The background threads consume pipe data continuously, +/// preventing backpressure from stalling the child. +/// +/// The client-disconnect check is the short-term mitigation for BUG-12/20: +/// when the host-side exec client is SIGTERM'd or times out, the agent's +/// accept loop was left blocked on the still-running child. Now we kill the +/// child as soon as we detect the peer has closed the connection, freeing +/// the accept loop for the next request. +pub fn wait_with_timeout_cleanup_and_liveness( + child: &mut Child, + timeout_ms: Option, + client_fd: Option, + on_timeout: F, +) -> std::io::Result +where + F: FnOnce(), +{ + use std::sync::mpsc; + + const CHUNK_SIZE: usize = 64 * 1024; + + // Drain stdout/stderr in background threads BEFORE waiting for exit. + // Threads send chunks via channels so the parent accumulates data + // incrementally. On timeout/disconnect, already-received chunks are + // preserved even if the reader thread is still blocked on a pipe that + // hasn't reached EOF (e.g., background process inherited stdio). + let (stdout_tx, stdout_rx) = mpsc::channel::>(); + let (stderr_tx, stderr_rx) = mpsc::channel::>(); + + let stdout_handle = child.stdout.take().and_then(|mut out| { + std::thread::Builder::new() + .name("crun-stdout".into()) + .spawn(move || { + let mut total = 0usize; + loop { + let mut chunk = vec![0u8; CHUNK_SIZE]; + match out.read(&mut chunk) { + Ok(0) => break, // EOF + Ok(n) => { + total += n; + chunk.truncate(n); + if stdout_tx.send(chunk).is_err() { + break; // receiver dropped + } + if total >= MAX_EXEC_OUTPUT { + break; // cap reached + } + } + Err(_) => break, + } + } + }) + .ok() + }); + + let stderr_handle = child.stderr.take().and_then(|mut err| { + std::thread::Builder::new() + .name("crun-stderr".into()) + .spawn(move || { + let mut total = 0usize; + loop { + let mut chunk = vec![0u8; CHUNK_SIZE]; + match err.read(&mut chunk) { + Ok(0) => break, + Ok(n) => { + total += n; + chunk.truncate(n); + if stderr_tx.send(chunk).is_err() { + break; + } + if total >= MAX_EXEC_OUTPUT { + break; + } + } + Err(_) => break, + } + } + }) + .ok() + }); + + // Accumulated output — grows as reader threads send chunks. + let mut stdout_buf = Vec::new(); + let mut stderr_buf = Vec::new(); + + let poll_interval = Duration::from_millis(10); + let deadline = timeout_ms.map(|ms| Instant::now() + Duration::from_millis(ms)); + + // Drain any available chunks from the channels into local buffers. + let drain_channels = |stdout_rx: &mpsc::Receiver>, + stderr_rx: &mpsc::Receiver>, + stdout_buf: &mut Vec, + stderr_buf: &mut Vec| { + for chunk in stdout_rx.try_iter() { + stdout_buf.extend_from_slice(&chunk); + } + for chunk in stderr_rx.try_iter() { + stderr_buf.extend_from_slice(&chunk); + } + }; + + loop { + // Drain available chunks each iteration so local buffers stay current. + drain_channels(&stdout_rx, &stderr_rx, &mut stdout_buf, &mut stderr_buf); + + match try_wait_with_eintr(child) { + Ok(Some(status)) => { + // Child exited — give reader threads a bounded window to finish. + // After the child dies, pipe write ends close and readers see EOF. + // Use is_finished() on handles to detect completion without consuming + // chunks (try_recv as a probe races and can drop data). + let join_deadline = Instant::now() + READER_JOIN_TIMEOUT; + while Instant::now() < join_deadline { + drain_channels(&stdout_rx, &stderr_rx, &mut stdout_buf, &mut stderr_buf); + let stdout_done = stdout_handle.as_ref().map_or(true, |h| h.is_finished()); + let stderr_done = stderr_handle.as_ref().map_or(true, |h| h.is_finished()); + if stdout_done && stderr_done { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Final drain after threads are done (or timed out). + drain_channels(&stdout_rx, &stderr_rx, &mut stdout_buf, &mut stderr_buf); + let exit_code = exit_code_from_status(&status); + return Ok(WaitResult::Completed { + exit_code, + output: ChildOutput { + stdout: stdout_buf, + stderr: stderr_buf, + }, + }); + } + Ok(None) => { + if let Some(fd) = client_fd { + if is_peer_closed(fd) { + let _ = child.kill(); + let _ = child.wait(); + drain_channels(&stdout_rx, &stderr_rx, &mut stdout_buf, &mut stderr_buf); + return Ok(WaitResult::ClientDisconnected { + output: ChildOutput { + stdout: stdout_buf, + stderr: stderr_buf, + }, + }); + } + } + + if let Some(deadline) = deadline { + if Instant::now() >= deadline { + on_timeout(); + let _ = child.kill(); + let _ = child.wait(); + drain_channels(&stdout_rx, &stderr_rx, &mut stdout_buf, &mut stderr_buf); + return Ok(WaitResult::TimedOut { + output: ChildOutput { + stdout: stdout_buf, + stderr: stderr_buf, + }, + timeout_ms: timeout_ms.unwrap_or(0), + }); + } + } + + std::thread::sleep(poll_interval); + } + Err(e) => return Err(e), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::process::Command; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + #[test] + fn test_timeout_exit_code_value() { + // Matches the standard timeout command exit code + assert_eq!(TIMEOUT_EXIT_CODE, 124); + } + + #[test] + fn test_child_output_default() { + let output = ChildOutput::default(); + assert!(output.stdout.is_empty()); + assert!(output.stderr.is_empty()); + } + + #[test] + fn test_capture_child_output_stdout() { + let mut child = Command::new("echo") + .arg("hello world") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + child.wait().unwrap(); + let output = capture_child_output(&mut child); + + assert!(output + .stdout + .windows(b"hello world".len()) + .any(|w| w == b"hello world")); + assert!(output.stderr.is_empty()); + } + + #[test] + fn test_capture_child_output_stderr() { + let mut child = Command::new("sh") + .args(["-c", "echo error >&2"]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + child.wait().unwrap(); + let output = capture_child_output(&mut child); + + assert!(output.stdout.is_empty()); + assert!(output.stderr.windows(b"error".len()).any(|w| w == b"error")); + } + + #[test] + fn test_wait_completes_success() { + let mut child = Command::new("echo") + .arg("hello") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + let result = wait_with_timeout(&mut child, Some(5000), None).unwrap(); + + match result { + WaitResult::Completed { exit_code, output } => { + assert_eq!(exit_code, 0); + assert!(output.stdout.windows(b"hello".len()).any(|w| w == b"hello")); + } + WaitResult::TimedOut { .. } => panic!("unexpected timeout"), + WaitResult::ClientDisconnected { .. } => panic!("unexpected client disconnect"), + } + } + + #[test] + fn test_wait_completes_with_nonzero_exit() { + let mut child = Command::new("sh") + .args(["-c", "exit 42"]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + let result = wait_with_timeout(&mut child, Some(5000), None).unwrap(); + + match result { + WaitResult::Completed { exit_code, .. } => { + assert_eq!(exit_code, 42); + } + WaitResult::TimedOut { .. } => panic!("unexpected timeout"), + WaitResult::ClientDisconnected { .. } => panic!("unexpected client disconnect"), + } + } + + #[test] + fn test_wait_no_timeout() { + // With timeout_ms = None, should wait indefinitely (process completes quickly) + let mut child = Command::new("echo") + .arg("quick") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + let result = wait_with_timeout(&mut child, None, None).unwrap(); + + match result { + WaitResult::Completed { exit_code, output } => { + assert_eq!(exit_code, 0); + assert!(output.stdout.windows(b"quick".len()).any(|w| w == b"quick")); + } + WaitResult::TimedOut { .. } => panic!("unexpected timeout"), + WaitResult::ClientDisconnected { .. } => panic!("unexpected client disconnect"), + } + } + + #[test] + fn test_wait_timeout() { + let mut child = Command::new("sleep") + .arg("10") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + let result = wait_with_timeout(&mut child, Some(50), None).unwrap(); + + match result { + WaitResult::TimedOut { timeout_ms, .. } => { + assert_eq!(timeout_ms, 50); + } + WaitResult::Completed { .. } => panic!("expected timeout"), + WaitResult::ClientDisconnected { .. } => panic!("unexpected client disconnect"), + } + } + + #[test] + fn test_wait_custom_poll_interval() { + let mut child = Command::new("echo") + .arg("fast") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + // Use a custom poll interval of 1ms + let result = wait_with_timeout(&mut child, Some(5000), Some(1)).unwrap(); + + assert!(matches!(result, WaitResult::Completed { .. })); + } + + #[test] + fn test_wait_with_cleanup_calls_callback() { + let callback_called = Arc::new(AtomicBool::new(false)); + let callback_called_clone = callback_called.clone(); + + let mut child = Command::new("sleep") + .arg("10") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + let result = wait_with_timeout_and_cleanup(&mut child, Some(50), || { + callback_called_clone.store(true, Ordering::SeqCst); + }) + .unwrap(); + + assert!(matches!(result, WaitResult::TimedOut { .. })); + assert!( + callback_called.load(Ordering::SeqCst), + "cleanup callback should be called" + ); + } + + #[test] + fn test_wait_with_cleanup_no_callback_on_success() { + let callback_called = Arc::new(AtomicBool::new(false)); + let callback_called_clone = callback_called.clone(); + + let mut child = Command::new("echo") + .arg("done") + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + let result = wait_with_timeout_and_cleanup(&mut child, Some(5000), || { + callback_called_clone.store(true, Ordering::SeqCst); + }) + .unwrap(); + + assert!(matches!(result, WaitResult::Completed { .. })); + assert!( + !callback_called.load(Ordering::SeqCst), + "cleanup callback should not be called on success" + ); + } +} diff --git a/crates/smolvm-agent/src/pty.rs b/crates/smolvm-agent/src/pty.rs new file mode 100644 index 0000000..c77c4d2 --- /dev/null +++ b/crates/smolvm-agent/src/pty.rs @@ -0,0 +1,277 @@ +//! PTY (pseudo-terminal) helpers for interactive VM exec. +//! +//! Uses POSIX `openpty()` to allocate a PTY pair and provides RAII +//! management of the master fd. The slave side is attached to a child +//! process via `Command::pre_exec`. + +use std::io; +use std::os::unix::io::{AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::os::unix::net::UnixListener; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +/// Monotonic counter making each console-socket path unique, so concurrent +/// sessions that share a container id (e.g. multiple `crun exec` into one +/// running container) never collide on the same socket file. +static CONSOLE_SOCKET_SEQ: AtomicU64 = AtomicU64::new(0); + +/// RAII wrapper around the master side of a PTY pair. +/// +/// Closes the fd on drop. Provides read/write and window-size control. +pub struct PtyMaster { + fd: OwnedFd, +} + +impl PtyMaster { + /// Read bytes from the PTY master. + pub fn read(&self, buf: &mut [u8]) -> io::Result { + let n = unsafe { libc::read(self.fd.as_raw_fd(), buf.as_mut_ptr() as *mut _, buf.len()) }; + if n < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(n as usize) + } + } + + /// Write all bytes to the PTY master. + pub fn write_all(&self, mut data: &[u8]) -> io::Result<()> { + while !data.is_empty() { + let n = + unsafe { libc::write(self.fd.as_raw_fd(), data.as_ptr() as *const _, data.len()) }; + if n < 0 { + return Err(io::Error::last_os_error()); + } + data = &data[n as usize..]; + } + Ok(()) + } + + /// Set the terminal window size via `TIOCSWINSZ`. + pub fn set_window_size(&self, cols: u16, rows: u16) -> io::Result<()> { + let ws = libc::winsize { + ws_row: rows, + ws_col: cols, + ws_xpixel: 0, + ws_ypixel: 0, + }; + // SAFETY: TIOCSWINSZ with a valid winsize struct on our owned fd. + let ret = unsafe { libc::ioctl(self.fd.as_raw_fd(), libc::TIOCSWINSZ, &ws) }; + if ret < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } + } + + pub fn as_raw_fd(&self) -> RawFd { + self.fd.as_raw_fd() + } + + /// Wrap an already-open master fd — e.g. one received from crun over a + /// `--console-socket` — as a `PtyMaster`. + pub fn from_owned_fd(fd: OwnedFd) -> Self { + PtyMaster { fd } + } +} + +/// A listening AF_UNIX socket that an OCI runtime connects to (via +/// `--console-socket`) to hand back the master fd of the container's PTY. +/// +/// crun/runc, when `terminal: true`, create the console PTY themselves and — in +/// foreground mode — relay byte data between that PTY and their own stdio but do +/// NOT propagate window-size changes. Taking the master directly over this +/// socket means the agent owns the container's real console, so +/// [`PtyMaster::set_window_size`] (and resize events) reach the process's tty. +pub struct ConsoleSocket { + path: PathBuf, + listener: UnixListener, +} + +impl ConsoleSocket { + /// Bind a uniquely-named console socket. `tag` (e.g. the container id) is a + /// human-readable hint; a process-global sequence number guarantees + /// uniqueness so concurrent sessions sharing a container id (multiple + /// `crun exec` into one container) never collide. Only the tag's tail is + /// used, to stay within the ~108-byte AF_UNIX path limit. + pub fn bind(tag: &str) -> io::Result { + let tail: String = { + let s = tag.trim_matches(|c: char| !c.is_ascii_alphanumeric()); + let start = s.len().saturating_sub(16); + s[start..].to_string() + }; + let seq = CONSOLE_SOCKET_SEQ.fetch_add(1, Ordering::Relaxed); + // The guest root filesystem (and thus /tmp) is read-only, so bind the + // socket under the writable storage disk. crun shares this mount + // namespace, so it can connect to the same path. + let dir = Path::new(crate::paths::STORAGE_ROOT); + let path = dir.join(format!("smolvm-con-{}-{}.sock", tail, seq)); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path)?; + Ok(Self { path, listener }) + } + + pub fn path(&self) -> &Path { + &self.path + } + + /// Accept the runtime's connection and receive the PTY master fd via + /// `SCM_RIGHTS`. Bounded by `timeout` so a runtime that never connects (a + /// crun failure) can't hang the interactive session. + pub fn recv_master(&self, timeout: Duration) -> io::Result { + let mut pfd = libc::pollfd { + fd: self.listener.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + let ms = timeout.as_millis().min(i32::MAX as u128) as i32; + let pr = unsafe { libc::poll(&mut pfd, 1, ms) }; + if pr == 0 { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "console socket: runtime did not connect", + )); + } + if pr < 0 { + return Err(io::Error::last_os_error()); + } + let (conn, _) = self.listener.accept()?; + recv_fd(conn.as_raw_fd()).map(PtyMaster::from_owned_fd) + } +} + +impl Drop for ConsoleSocket { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} + +/// Receive a single fd from a connected unix socket via `SCM_RIGHTS`. +fn recv_fd(sock_fd: RawFd) -> io::Result { + // 8-byte-aligned control buffer (cmsghdr requires size_t alignment). + #[repr(C, align(8))] + struct AlignedCmsg([u8; 32]); + let mut cmsg = AlignedCmsg([0u8; 32]); + let mut data = [0u8; 1]; + let mut iov = libc::iovec { + iov_base: data.as_mut_ptr() as *mut libc::c_void, + iov_len: data.len(), + }; + // SAFETY: zeroed msghdr is valid; pointers below reference live local buffers. + let mut msg: libc::msghdr = unsafe { std::mem::zeroed() }; + msg.msg_iov = &mut iov; + msg.msg_iovlen = 1; + msg.msg_control = cmsg.0.as_mut_ptr() as *mut libc::c_void; + msg.msg_controllen = cmsg.0.len() as _; + + let n = unsafe { libc::recvmsg(sock_fd, &mut msg, 0) }; + if n < 0 { + return Err(io::Error::last_os_error()); + } + let cptr = unsafe { libc::CMSG_FIRSTHDR(&msg) }; + if cptr.is_null() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "console socket: no SCM_RIGHTS control message", + )); + } + // SAFETY: cptr is non-null and points into our cmsg buffer. + let chdr = unsafe { &*cptr }; + if chdr.cmsg_level != libc::SOL_SOCKET || chdr.cmsg_type != libc::SCM_RIGHTS { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "console socket: unexpected control message", + )); + } + let mut fd: RawFd = -1; + // SAFETY: CMSG_DATA points to at least size_of::() bytes here. + unsafe { + std::ptr::copy_nonoverlapping( + libc::CMSG_DATA(cptr), + &mut fd as *mut RawFd as *mut u8, + std::mem::size_of::(), + ); + } + if fd < 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "console socket: invalid fd received", + )); + } + // SAFETY: fd was just received from the kernel and is owned by us now. + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) +} + +/// Allocate a new PTY pair with the given initial window size. +/// +/// Returns `(master, slave_fd)`. The caller must close `slave_fd` in the +/// parent process after the child has been spawned. +pub fn open_pty(cols: u16, rows: u16) -> io::Result<(PtyMaster, OwnedFd)> { + let mut master_raw: RawFd = -1; + let mut slave_raw: RawFd = -1; + + // SAFETY: openpty writes into our pointers; we check the return value. + let ret = unsafe { + libc::openpty( + &mut master_raw, + &mut slave_raw, + std::ptr::null_mut(), + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + + // Wrap in OwnedFd immediately so they get closed on error paths. + // SAFETY: openpty returned successfully, so these are valid fds. + let master_fd = unsafe { OwnedFd::from_raw_fd(master_raw) }; + let slave_fd = unsafe { OwnedFd::from_raw_fd(slave_raw) }; + + let master = PtyMaster { fd: master_fd }; + + // Set initial window size. + master.set_window_size(cols, rows)?; + + Ok((master, slave_fd)) +} + +/// Build a `pre_exec` closure that makes the child the session leader +/// and attaches the slave PTY as its controlling terminal + stdio. +/// +/// # Safety +/// +/// The returned closure is meant for `Command::pre_exec` which runs +/// between `fork()` and `exec()`. It calls only async-signal-safe +/// functions (`setsid`, `ioctl`, `dup2`, `close`). +pub fn slave_pre_exec(slave_fd: RawFd) -> impl FnMut() -> io::Result<()> { + move || { + // Create a new session (detach from parent's controlling terminal). + if unsafe { libc::setsid() } < 0 { + return Err(io::Error::last_os_error()); + } + + // Make the slave our controlling terminal. + // SAFETY: TIOCSCTTY with arg 0 on a valid pty slave fd after setsid(). + if unsafe { libc::ioctl(slave_fd, libc::TIOCSCTTY as _, 0) } < 0 { + return Err(io::Error::last_os_error()); + } + + // Dup slave fd onto stdin/stdout/stderr. + for &target in &[0, 1, 2] { + if slave_fd != target { + if unsafe { libc::dup2(slave_fd, target) } < 0 { + return Err(io::Error::last_os_error()); + } + } + } + + // Close the original slave fd if it wasn't one of 0/1/2. + if slave_fd > 2 { + unsafe { libc::close(slave_fd) }; + } + + Ok(()) + } +} diff --git a/crates/smolvm-agent/src/retry.rs b/crates/smolvm-agent/src/retry.rs new file mode 100644 index 0000000..e677823 --- /dev/null +++ b/crates/smolvm-agent/src/retry.rs @@ -0,0 +1,8 @@ +//! Retry utilities for transient failure recovery. +//! +//! This module re-exports retry utilities from smolvm-protocol for use +//! in the agent. The shared implementation ensures consistent retry +//! behavior across host and guest. + +// Re-export everything from the protocol's retry module +pub use smolvm_protocol::retry::*; diff --git a/crates/smolvm-agent/src/rosetta.rs b/crates/smolvm-agent/src/rosetta.rs new file mode 100644 index 0000000..8b4dedb --- /dev/null +++ b/crates/smolvm-agent/src/rosetta.rs @@ -0,0 +1,253 @@ +//! Guest-side Rosetta 2 setup. +//! +//! When the host attaches the RosettaLinux runtime (virtiofs tag +//! [`smolvm_protocol::ROSETTA_TAG`]) and signals [`guest_env::ROSETTA`], this +//! module mounts that runtime at [`smolvm_protocol::ROSETTA_GUEST_PATH`] and +//! registers the ptrace wrapper (`/usr/bin/rosetta-wrapper`, shipped in the +//! agent rootfs) with `binfmt_misc` as the interpreter for x86_64 ELF binaries. +//! +//! libkrun runs under Hypervisor.framework, not Virtualization.framework, so +//! Rosetta's runtime-validation ioctl fails; the wrapper intercepts that ioctl +//! via ptrace, returns the expected magic, then detaches for full-speed +//! execution. The `F` (fix-binary) flag pins the wrapper fd at registration +//! time so it keeps working inside crun containers whose mount namespaces don't +//! include the agent rootfs. + +use smolvm_protocol::guest_env; + +/// Guest path of the ptrace wrapper (installed into the agent rootfs by +/// `scripts/build-agent-rootfs.sh`). +#[cfg(target_os = "linux")] +const WRAPPER_PATH: &str = "/usr/bin/rosetta-wrapper"; + +/// binfmt_misc control directory and its registration file. +#[cfg(target_os = "linux")] +const BINFMT_MISC_DIR: &str = "/proc/sys/fs/binfmt_misc"; +#[cfg(target_os = "linux")] +const BINFMT_REGISTER_FILE: &str = "/proc/sys/fs/binfmt_misc/register"; + +/// binfmt_misc registration line: match x86_64 ELF — both `ET_EXEC` and `ET_DYN` +/// (PIE), via the `\xfe` byte in the e_type mask — and route to the wrapper. The +/// magic/mask are literal `\xNN` escapes; the kernel's binfmt parser unescapes +/// them. Same magic/mask qemu-user uses for `qemu-x86_64`. +#[cfg(target_os = "linux")] +const BINFMT_REGISTER: &str = ":rosetta:M::\\x7fELF\\x02\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x3e\\x00:\\xff\\xff\\xff\\xff\\xff\\xfe\\xfe\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfe\\xff\\xff\\xff:/usr/bin/rosetta-wrapper:F"; + +/// Whether the host requested Rosetta translation for this VM. +pub fn is_enabled() -> bool { + std::env::var(guest_env::ROSETTA).as_deref() == Ok(guest_env::VALUE_ON) +} + +/// Bind-mount the Rosetta runtime into a workload container so the wrapper's +/// `execve("/mnt/rosetta/rosetta")` resolves inside the container's mount +/// namespace. binfmt_misc's `F` flag pins the wrapper fd itself, but the +/// translator it execs is an ordinary path lookup in the container's ns, so the +/// runtime must be visible there too. No-op unless Rosetta is enabled. +pub fn inject_into_container(spec: &mut crate::oci::OciSpec) { + inject_into_container_if(spec, is_enabled()); +} + +/// Testable core of [`inject_into_container`]: adds the read-only runtime +/// bind-mount when `enabled`. Split out so tests don't touch the process-wide +/// `SMOLVM_ROSETTA` env var. +fn inject_into_container_if(spec: &mut crate::oci::OciSpec, enabled: bool) { + if !enabled { + return; + } + // Read-only: the container only execs the translator, never writes it. Skip + // if the workload already mounts something at this path so a user mount wins. + if spec + .mounts + .iter() + .any(|m| m.destination == smolvm_protocol::ROSETTA_GUEST_PATH) + { + return; + } + spec.add_bind_mount( + smolvm_protocol::ROSETTA_GUEST_PATH, + smolvm_protocol::ROSETTA_GUEST_PATH, + true, + ); +} + +/// Mount the Rosetta runtime and register the `binfmt_misc` handler. +/// +/// Best-effort: every failure is logged and swallowed so a Rosetta problem never +/// blocks the VM from booting (x86_64 workloads simply won't translate). A no-op +/// on non-Linux (the agent builds a macOS stub for host-side unit tests). +pub fn setup() { + #[cfg(target_os = "linux")] + { + use smolvm_protocol::ROSETTA_GUEST_PATH; + + if let Err(e) = mount_runtime() { + tracing::warn!(error = %e, "rosetta: runtime mount failed; x86_64 translation unavailable"); + return; + } + if !std::path::Path::new(WRAPPER_PATH).exists() { + tracing::warn!( + wrapper = WRAPPER_PATH, + "rosetta: wrapper missing from rootfs; x86_64 translation unavailable" + ); + return; + } + if let Err(e) = register_binfmt() { + tracing::warn!(error = %e, "rosetta: binfmt_misc registration failed; x86_64 translation unavailable"); + return; + } + tracing::info!( + mount = ROSETTA_GUEST_PATH, + "rosetta: x86_64 translation enabled" + ); + } +} + +/// Mount virtiofs tag [`ROSETTA_TAG`] at [`ROSETTA_GUEST_PATH`]. Idempotent: if +/// the translator is already visible there, the mount is left as-is. +#[cfg(target_os = "linux")] +fn mount_runtime() -> std::io::Result<()> { + use smolvm_protocol::{ROSETTA_GUEST_PATH, ROSETTA_TAG}; + use std::ffi::CString; + + std::fs::create_dir_all(ROSETTA_GUEST_PATH)?; + + if std::path::Path::new(ROSETTA_GUEST_PATH) + .join("rosetta") + .exists() + { + return Ok(()); + } + + let src = CString::new(ROSETTA_TAG).expect("rosetta tag has no null byte"); + let dst = CString::new(ROSETTA_GUEST_PATH).expect("rosetta path has no null byte"); + let fstype = CString::new("virtiofs").expect("literal has no null byte"); + // SAFETY: all args are valid null-terminated C strings; virtiofs takes no + // mount data (matches storage::setup_packed_layers). + let rc = unsafe { + libc::mount( + src.as_ptr(), + dst.as_ptr(), + fstype.as_ptr(), + 0, + std::ptr::null(), + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +/// Mount `binfmt_misc` if needed and write the x86_64→wrapper registration. +/// Idempotent: skips if a `rosetta` handler is already registered on this kernel. +#[cfg(target_os = "linux")] +fn register_binfmt() -> std::io::Result<()> { + use std::ffi::CString; + + // The /register control file only exists once binfmt_misc is mounted. Some + // kernels auto-mount it; mount it ourselves otherwise. + if !std::path::Path::new(BINFMT_REGISTER_FILE).exists() { + std::fs::create_dir_all(BINFMT_MISC_DIR)?; + let src = CString::new("binfmt_misc").unwrap(); + let dst = CString::new(BINFMT_MISC_DIR).unwrap(); + let fstype = CString::new("binfmt_misc").unwrap(); + // SAFETY: valid C strings; binfmt_misc takes no mount data. + let rc = unsafe { + libc::mount( + src.as_ptr(), + dst.as_ptr(), + fstype.as_ptr(), + 0, + std::ptr::null(), + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + } + + if std::path::Path::new(BINFMT_MISC_DIR) + .join("rosetta") + .exists() + { + return Ok(()); + } + + std::fs::write(BINFMT_REGISTER_FILE, BINFMT_REGISTER) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::oci::{OciSpec, ProcessIdentity}; + + fn empty_spec() -> OciSpec { + OciSpec::new( + &["true".to_string()], + &[], + "/", + false, + &ProcessIdentity::root(), + false, + ) + } + + #[test] + fn inject_is_noop_when_disabled() { + let mut spec = empty_spec(); + let mounts_before = spec.mounts.len(); + inject_into_container_if(&mut spec, false); + assert_eq!(spec.mounts.len(), mounts_before); + } + + #[test] + fn inject_adds_readonly_runtime_mount_when_enabled() { + let mut spec = empty_spec(); + inject_into_container_if(&mut spec, true); + + let mount = spec + .mounts + .iter() + .find(|m| m.destination == smolvm_protocol::ROSETTA_GUEST_PATH) + .expect("rosetta runtime bind mount not found"); + assert_eq!(mount.source, smolvm_protocol::ROSETTA_GUEST_PATH); + assert_eq!(mount.mount_type.as_deref(), Some("bind")); + // Read-only: the container only execs the translator. + assert!(mount.options.iter().any(|o| o == "ro")); + assert!(mount.options.iter().any(|o| o == "bind")); + } + + #[test] + fn inject_does_not_duplicate_existing_mount() { + let mut spec = empty_spec(); + // A user mount already claims the path; injection must not add a second. + spec.add_bind_mount( + smolvm_protocol::ROSETTA_GUEST_PATH, + smolvm_protocol::ROSETTA_GUEST_PATH, + false, + ); + let mounts_before = spec.mounts.len(); + inject_into_container_if(&mut spec, true); + assert_eq!(spec.mounts.len(), mounts_before); + } + + #[test] + fn is_enabled_reflects_env_sentinel() { + // Save/restore to avoid cross-test env bleed. + let prev = std::env::var(guest_env::ROSETTA).ok(); + + std::env::remove_var(guest_env::ROSETTA); + assert!(!is_enabled(), "unset must be disabled"); + + std::env::set_var(guest_env::ROSETTA, guest_env::VALUE_ON); + assert!(is_enabled(), "sentinel value must enable"); + + std::env::set_var(guest_env::ROSETTA, "0"); + assert!(!is_enabled(), "non-sentinel value must be disabled"); + + match prev { + Some(v) => std::env::set_var(guest_env::ROSETTA, v), + None => std::env::remove_var(guest_env::ROSETTA), + } + } +} diff --git a/crates/smolvm-agent/src/ssh_agent.rs b/crates/smolvm-agent/src/ssh_agent.rs new file mode 100644 index 0000000..6e7305b --- /dev/null +++ b/crates/smolvm-agent/src/ssh_agent.rs @@ -0,0 +1,421 @@ +//! Guest-side SSH agent bridge. +//! +//! Listens on a Unix socket inside the VM and relays connections to the +//! host's SSH agent via vsock. Guest applications (git, ssh) connect to +//! this socket transparently via `SSH_AUTH_SOCK`. + +use smolvm_protocol::ports; +use std::io; +use std::os::unix::net::UnixListener; +use std::thread; + +/// Guest-side path for the SSH agent socket. +pub const GUEST_SSH_AUTH_SOCK: &str = "/tmp/ssh-agent.sock"; + +/// Start the guest-side SSH agent bridge in a background thread. +/// +/// Creates a Unix socket at [`GUEST_SSH_AUTH_SOCK`] and, for each incoming +/// connection, opens a vsock connection to the host-side bridge on +/// [`ports::SSH_AGENT`] and relays bytes bidirectionally. +pub fn start() { + thread::Builder::new() + .name("ssh-agent-guest".into()) + .spawn(|| { + if let Err(e) = run_bridge() { + tracing::warn!(error = %e, "guest SSH agent bridge stopped"); + } + }) + .ok(); +} + +/// Check if SSH agent forwarding is enabled via environment variable. +pub fn is_enabled() -> bool { + std::env::var("SMOLVM_SSH_AGENT").as_deref() == Ok("1") +} + +/// Inject SSH agent forwarding into an OCI container spec. +/// +/// When forwarding is enabled (`SMOLVM_SSH_AGENT=1`), bind-mount the +/// guest-side bridge socket into the container at the same path and set +/// `SSH_AUTH_SOCK` so tools inside the container can find it. No-op when +/// forwarding is disabled. +/// +/// Rationale: the container lives in its own mount namespace (so +/// `/tmp/ssh-agent.sock` from the VM rootfs is not visible) and gets env +/// from the image + request (not from the agent's own env), so both the +/// file and the variable have to be wired in explicitly. +pub fn inject_into_container(spec: &mut crate::oci::OciSpec) { + inject_into_container_if(spec, is_enabled()); +} + +/// Testable core of [`inject_into_container`]. Bind-mounts the bridge +/// socket and sets the env var when `enabled` is true; no-op otherwise. +/// Split out so tests can exercise the injection logic without mutating +/// the process-wide `SMOLVM_SSH_AGENT` env variable. +fn inject_into_container_if(spec: &mut crate::oci::OciSpec, enabled: bool) { + if !enabled { + return; + } + // Bind-mount the socket file; rw because SSH agent protocol is bidirectional. + spec.add_bind_mount(GUEST_SSH_AUTH_SOCK, GUEST_SSH_AUTH_SOCK, false); + spec.add_env("SSH_AUTH_SOCK", GUEST_SSH_AUTH_SOCK); +} + +/// Add `SSH_AUTH_SOCK` to a command's env list when forwarding is enabled. +/// +/// [`inject_into_container`] wires the socket + env into a *container spec*, but a +/// command run via `crun exec` into the persistent keep-alive container — the path +/// non-interactive exec + run take since #542 — gets a fresh process env, not the +/// container's, so the spec injection never reaches it. This wires the variable +/// into that exec/run env. No-op when disabled; never overrides an existing value +/// (e.g. a user-supplied `-e SSH_AUTH_SOCK`). +pub fn inject_into_env(env: &mut Vec<(String, String)>) { + inject_into_env_if(env, is_enabled()); +} + +/// Testable core of [`inject_into_env`]. +fn inject_into_env_if(env: &mut Vec<(String, String)>, enabled: bool) { + if enabled && !env.iter().any(|(k, _)| k == "SSH_AUTH_SOCK") { + env.push(("SSH_AUTH_SOCK".to_string(), GUEST_SSH_AUTH_SOCK.to_string())); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::oci::{OciSpec, ProcessIdentity}; + + // Regression guard for #542: non-interactive exec + run go through the + // keep-alive container's `crun exec`, whose process env is built fresh (not + // inherited from the container), so SSH_AUTH_SOCK must be injected into that + // env explicitly — the container-spec injection alone doesn't reach it. + #[test] + fn inject_into_env_adds_ssh_auth_sock_only_when_enabled() { + // Enabled → injected. + let mut env = vec![("PATH".to_string(), "/usr/bin".to_string())]; + inject_into_env_if(&mut env, true); + assert!( + env.iter() + .any(|(k, v)| k == "SSH_AUTH_SOCK" && v == GUEST_SSH_AUTH_SOCK), + "SSH_AUTH_SOCK must be injected into the exec/run env when forwarding is enabled" + ); + + // Disabled → no-op. + let mut env = vec![("PATH".to_string(), "/usr/bin".to_string())]; + inject_into_env_if(&mut env, false); + assert!(!env.iter().any(|(k, _)| k == "SSH_AUTH_SOCK")); + + // Never overrides a user-supplied value. + let mut env = vec![("SSH_AUTH_SOCK".to_string(), "/custom.sock".to_string())]; + inject_into_env_if(&mut env, true); + assert_eq!(env.iter().filter(|(k, _)| k == "SSH_AUTH_SOCK").count(), 1); + assert_eq!(env[0].1, "/custom.sock"); + } + + #[test] + fn inject_is_noop_when_disabled() { + let mut spec = OciSpec::new( + &["true".to_string()], + &[], + "/", + false, + &ProcessIdentity::root(), + false, + ); + let mounts_before = spec.mounts.len(); + let envs_before = spec.process.env.len(); + + inject_into_container_if(&mut spec, false); + + assert_eq!(spec.mounts.len(), mounts_before); + assert_eq!(spec.process.env.len(), envs_before); + assert!(!spec + .process + .env + .iter() + .any(|e| e.starts_with("SSH_AUTH_SOCK="))); + } + + #[test] + fn inject_adds_env_and_mount_when_enabled() { + let mut spec = OciSpec::new( + &["true".to_string()], + &[], + "/", + false, + &ProcessIdentity::root(), + false, + ); + + inject_into_container_if(&mut spec, true); + + // Env must point at the guest-side bridge socket. + assert!(spec + .process + .env + .iter() + .any(|e| e == &format!("SSH_AUTH_SOCK={}", GUEST_SSH_AUTH_SOCK))); + + // Mount must bind the socket at the same path inside the container. + let mount = spec + .mounts + .iter() + .find(|m| m.destination == GUEST_SSH_AUTH_SOCK) + .expect("bind mount for SSH agent socket not found"); + assert_eq!(mount.source, GUEST_SSH_AUTH_SOCK); + assert_eq!(mount.mount_type.as_deref(), Some("bind")); + // rw: the SSH agent protocol is bidirectional. + assert!(!mount.options.iter().any(|o| o == "ro")); + assert!(mount.options.iter().any(|o| o == "bind")); + } + + #[test] + fn inject_replaces_existing_ssh_auth_sock() { + // Simulates an image whose config already exports a stale + // SSH_AUTH_SOCK: we must replace it, not duplicate it — two entries + // for the same key leaves the effective value shell-dependent. + let mut spec = OciSpec::new( + &["true".to_string()], + &[], + "/", + false, + &ProcessIdentity::root(), + false, + ); + spec.process + .env + .push("SSH_AUTH_SOCK=/stale/path".to_string()); + + inject_into_container_if(&mut spec, true); + + let matches: Vec<_> = spec + .process + .env + .iter() + .filter(|e| e.starts_with("SSH_AUTH_SOCK=")) + .collect(); + assert_eq!(matches.len(), 1, "duplicate SSH_AUTH_SOCK entries"); + assert_eq!( + matches[0], + &format!("SSH_AUTH_SOCK={}", GUEST_SSH_AUTH_SOCK) + ); + } +} + +fn run_bridge() -> io::Result<()> { + let sock_path = std::path::Path::new(GUEST_SSH_AUTH_SOCK); + + // Clean up stale socket + let _ = std::fs::remove_file(sock_path); + + // Ensure parent directory exists + if let Some(parent) = sock_path.parent() { + std::fs::create_dir_all(parent)?; + } + + let listener = UnixListener::bind(sock_path)?; + + // Make socket accessible to all users in the VM + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(sock_path, std::fs::Permissions::from_mode(0o777))?; + } + + tracing::info!( + path = GUEST_SSH_AUTH_SOCK, + vsock_port = ports::SSH_AGENT, + "guest SSH agent bridge listening" + ); + + for stream in listener.incoming() { + match stream { + Ok(local_conn) => { + thread::Builder::new() + .name("ssh-agent-fwd".into()) + .spawn(move || { + if let Err(e) = relay_to_host(local_conn) { + tracing::debug!(error = %e, "SSH agent relay ended"); + } + }) + .ok(); + } + Err(e) => { + tracing::debug!(error = %e, "guest SSH agent accept error"); + if e.kind() == io::ErrorKind::InvalidInput { + break; + } + } + } + } + + Ok(()) +} + +/// Connect to the host SSH agent via vsock and relay bytes. +/// Bidirectional relay between a local Unix socket and a vsock connection. +/// +/// Uses `poll()` to multiplex reads on both sides, forwarding data in +/// whichever direction is ready. This handles fragmented messages and +/// concurrent I/O correctly — no assumptions about request/response ordering. +#[cfg(target_os = "linux")] +fn relay_to_host(local: std::os::unix::net::UnixStream) -> io::Result<()> { + use std::os::unix::io::AsRawFd; + + let mut vsock_conn = vsock_connect(ports::SSH_AGENT)?; + let mut local = local; + + let local_fd = local.as_raw_fd(); + let vsock_fd = vsock_conn.as_raw_fd(); + + let mut buf = [0u8; 16384]; + + loop { + let mut poll_fds = [ + libc::pollfd { + fd: local_fd, + events: libc::POLLIN, + revents: 0, + }, + libc::pollfd { + fd: vsock_fd, + events: libc::POLLIN, + revents: 0, + }, + ]; + + let ret = unsafe { libc::poll(poll_fds.as_mut_ptr(), 2, 30_000) }; + if ret < 0 { + return Err(io::Error::last_os_error()); + } + if ret == 0 { + // Timeout — SSH agent connections are short-lived, clean up + break; + } + + // local → vsock + if poll_fds[0].revents & (libc::POLLIN | libc::POLLHUP) != 0 { + let n = io::Read::read(&mut local, &mut buf)?; + if n == 0 { + break; + } + io::Write::write_all(&mut vsock_conn, &buf[..n])?; + } + + // vsock → local + if poll_fds[1].revents & (libc::POLLIN | libc::POLLHUP) != 0 { + let n = io::Read::read(&mut vsock_conn, &mut buf)?; + if n == 0 { + break; + } + io::Write::write_all(&mut local, &buf[..n])?; + } + } + + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn relay_to_host(_local: std::os::unix::net::UnixStream) -> io::Result<()> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "SSH agent forwarding only supported on Linux guests", + )) +} + +// ============================================================================ +// vsock client connect (guest → host) +// ============================================================================ + +/// Wrapper around a vsock file descriptor that implements Read + Write. +#[cfg(target_os = "linux")] +struct VsockStream { + fd: std::os::unix::io::OwnedFd, +} + +#[cfg(target_os = "linux")] +impl std::os::unix::io::AsRawFd for VsockStream { + fn as_raw_fd(&self) -> std::os::unix::io::RawFd { + self.fd.as_raw_fd() + } +} + +#[cfg(target_os = "linux")] +impl io::Read for VsockStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + use std::os::fd::AsRawFd; + unsafe { + let n = libc::read(self.fd.as_raw_fd(), buf.as_mut_ptr() as *mut _, buf.len()); + if n < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(n as usize) + } + } + } +} + +#[cfg(target_os = "linux")] +impl io::Write for VsockStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + use std::os::fd::AsRawFd; + unsafe { + let n = libc::write(self.fd.as_raw_fd(), buf.as_ptr() as *const _, buf.len()); + if n < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(n as usize) + } + } + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +/// Connect to a vsock port on the host (CID 2). +#[cfg(target_os = "linux")] +fn vsock_connect(port: u32) -> io::Result { + use std::mem; + use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + + const AF_VSOCK: libc::c_int = 40; + const HOST_CID: u32 = 2; + + #[repr(C)] + struct sockaddr_vm { + svm_family: libc::sa_family_t, + svm_reserved1: u16, + svm_port: u32, + svm_cid: u32, + svm_zero: [u8; 4], + } + + unsafe { + let fd = libc::socket(AF_VSOCK, libc::SOCK_STREAM, 0); + if fd < 0 { + return Err(io::Error::last_os_error()); + } + let fd = OwnedFd::from_raw_fd(fd); + + let addr = sockaddr_vm { + svm_family: AF_VSOCK as u16, + svm_reserved1: 0, + svm_port: port, + svm_cid: HOST_CID, + svm_zero: [0; 4], + }; + + if libc::connect( + fd.as_raw_fd(), + &addr as *const sockaddr_vm as *const libc::sockaddr, + mem::size_of::() as libc::socklen_t, + ) < 0 + { + return Err(io::Error::last_os_error()); + } + + Ok(VsockStream { fd }) + } +} diff --git a/crates/smolvm-agent/src/storage.rs b/crates/smolvm-agent/src/storage.rs new file mode 100644 index 0000000..a6c8fd3 --- /dev/null +++ b/crates/smolvm-agent/src/storage.rs @@ -0,0 +1,4367 @@ +//! Storage management for the helper daemon. +//! +//! This module handles: +//! - Storage disk initialization and formatting +//! - OCI image pulling via crane +//! - Layer extraction and deduplication +//! - Overlay filesystem management +//! - Container execution via crun OCI runtime +//! - Support for pre-packed OCI layers (smolvm pack) + +use crate::crun::CrunCommand; +use crate::oci::{generate_container_id, OciSpec}; +use crate::paths; +use crate::process::{WaitResult, TIMEOUT_EXIT_CODE}; +use smolvm_protocol::guest_env; +use smolvm_protocol::{ + image_repo, normalize_image_ref, ImageInfo, OverlayInfo, RegistryAuth, StorageStatus, +}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::OnceLock; +use tracing::{debug, info, warn}; + +/// Storage root path (where the ext4 disk is mounted). +const STORAGE_ROOT: &str = "/storage"; + +/// Directory structure within storage. +const LAYERS_DIR: &str = "layers"; +const CONFIGS_DIR: &str = "configs"; +const MANIFESTS_DIR: &str = "manifests"; +const OVERLAYS_DIR: &str = "overlays"; +const WORKSPACE_DIR: &str = "workspace"; +const DOCKER_HUB_AUTH_CONFIG_KEY: &str = "https://index.docker.io/v1/"; +const DOCKER_HUB_REGISTRY_ALIASES: &[&str] = &["docker.io", "index.docker.io"]; + +fn validate_storage_id(value: &str, context: &str) -> Result<()> { + if value.is_empty() { + return Err(StorageError::ValidationFailed { + context: context.to_string(), + reason: "cannot be empty".to_string(), + }); + } + + if value.len() > 128 { + return Err(StorageError::ValidationFailed { + context: context.to_string(), + reason: "too long (max 128 chars)".to_string(), + }); + } + + if value.contains('/') || value.contains('\\') { + return Err(StorageError::ValidationFailed { + context: context.to_string(), + reason: "path separators are not allowed".to_string(), + }); + } + + let path = Path::new(value); + for component in path.components() { + match component { + std::path::Component::ParentDir => { + return Err(StorageError::ValidationFailed { + context: context.to_string(), + reason: "parent traversal is not allowed".to_string(), + }); + } + std::path::Component::CurDir => { + return Err(StorageError::ValidationFailed { + context: context.to_string(), + reason: "dot segments are not allowed".to_string(), + }); + } + std::path::Component::RootDir | std::path::Component::Prefix(_) => { + return Err(StorageError::ValidationFailed { + context: context.to_string(), + reason: "path separators are not allowed".to_string(), + }); + } + std::path::Component::Normal(seg) => { + let seg = seg.to_string_lossy(); + if !seg + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.') + { + return Err(StorageError::ValidationFailed { + context: context.to_string(), + reason: format!("contains invalid character(s): {}", value), + }); + } + } + } + } + + Ok(()) +} + +fn overlay_root_for_workload(workload_id: &str) -> Result { + validate_storage_id(workload_id, "workload_id")?; + Ok(Path::new(STORAGE_ROOT).join(OVERLAYS_DIR).join(workload_id)) +} + +fn validate_container_destination_path(container_path: &str) -> Result { + if !container_path.starts_with('/') { + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: "must be an absolute path".to_string(), + }); + } + if container_path == "/" { + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: "mounting to '/' is not allowed".to_string(), + }); + } + + let mut relative = PathBuf::new(); + for component in Path::new(container_path).components() { + match component { + std::path::Component::RootDir => {} + std::path::Component::Normal(seg) => relative.push(seg), + std::path::Component::ParentDir => { + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: "parent traversal is not allowed".to_string(), + }); + } + std::path::Component::CurDir => { + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: "dot segments are not allowed".to_string(), + }); + } + std::path::Component::Prefix(_) => { + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: "path prefixes are not allowed".to_string(), + }); + } + } + } + + if relative.as_os_str().is_empty() { + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: "cannot resolve mount destination".to_string(), + }); + } + + Ok(relative) +} + +fn ensure_mount_target_under_root(rootfs: &Path, container_path: &str) -> Result { + let root_canon = rootfs.canonicalize().map_err(|e| StorageError::ReadFile { + path: rootfs.display().to_string(), + cause: format!("failed to canonicalize rootfs: {}", e), + })?; + + let relative = validate_container_destination_path(container_path)?; + let components: Vec<_> = relative.components().collect(); + let last_idx = components.len().saturating_sub(1); + let mut current = root_canon.clone(); + + for (idx, component) in components.into_iter().enumerate() { + let std::path::Component::Normal(seg) = component else { + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: "invalid destination component".to_string(), + }); + }; + + current.push(seg); + match std::fs::symlink_metadata(¤t) { + Ok(meta) => { + if meta.file_type().is_symlink() { + let canon = current.canonicalize().map_err(|e| StorageError::ReadFile { + path: current.display().to_string(), + cause: format!("failed to canonicalize symlink target: {}", e), + })?; + if !canon.starts_with(&root_canon) { + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: "resolved path escapes rootfs".to_string(), + }); + } + if idx == last_idx { + // The mount target itself is a symlink within the rootfs. + // Previous VM runs can leave such symlinks (e.g. /workspace → + // /storage/workspace) in the writable agent rootfs. Replace it + // with a real directory so the bind mount claims the path + // directly rather than following through to the symlink target. + std::fs::remove_file(¤t).map_err(|e| StorageError::ReadFile { + path: current.display().to_string(), + cause: format!("failed to remove symlink at mount target: {}", e), + })?; + std::fs::create_dir(¤t).map_err(|err| StorageError::CreateDir { + path: current.display().to_string(), + cause: err.to_string(), + })?; + } else if !current.is_dir() { + // Intermediate symlink must resolve to a directory. + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: format!( + "destination component is not a directory: {}", + current.display() + ), + }); + } + } else if !meta.is_dir() { + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: format!( + "destination component is not a directory: {}", + current.display() + ), + }); + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + std::fs::create_dir(¤t).map_err(|err| StorageError::CreateDir { + path: current.display().to_string(), + cause: err.to_string(), + })?; + } + Err(e) => { + return Err(StorageError::ReadFile { + path: current.display().to_string(), + cause: e.to_string(), + }); + } + } + } + + let final_canon = current.canonicalize().map_err(|e| StorageError::ReadFile { + path: current.display().to_string(), + cause: format!("failed to canonicalize mount destination: {}", e), + })?; + if !final_canon.starts_with(&root_canon) { + return Err(StorageError::ValidationFailed { + context: "mount destination".to_string(), + reason: "resolved path escapes rootfs".to_string(), + }); + } + + Ok(final_canon) +} + +/// Global state for packed layers support. +/// Set at startup if SMOLVM_PACKED_LAYERS env var is present. +static PACKED_LAYERS_DIR: OnceLock> = OnceLock::new(); + +/// Global state for boot-time volume mounts. +/// Set at startup if SMOLVM_MOUNT_COUNT env var is present. +static BOOT_VOLUME_MOUNTS: OnceLock> = OnceLock::new(); + +/// Initialize packed layers support by checking SMOLVM_PACKED_LAYERS env var. +/// Format: "virtiofs_tag:mount_point" (e.g., "smolvm_layers:/packed_layers") +/// Returns the mount point path if successfully mounted. +pub fn init_packed_layers() -> Option { + let env_val = match std::env::var("SMOLVM_PACKED_LAYERS") { + Ok(v) => v, + Err(_) => return None, + }; + + // Parse "tag:mount_point" + let parts: Vec<&str> = env_val.split(':').collect(); + if parts.len() != 2 { + warn!(env_val = %env_val, "invalid SMOLVM_PACKED_LAYERS format, expected 'tag:mount_point'"); + return None; + } + + let tag = parts[0]; + let mount_point = PathBuf::from(parts[1]); + + info!(tag = %tag, mount_point = %mount_point.display(), "setting up packed layers from virtiofs"); + + // Create mount point + if let Err(e) = std::fs::create_dir_all(&mount_point) { + warn!(error = %e, mount_point = %mount_point.display(), "failed to create packed layers mount point"); + return None; + } + + // Mount virtiofs using direct syscall (avoids ~3-5ms fork+exec overhead) + #[cfg(target_os = "linux")] + { + let src = std::ffi::CString::new(tag).ok()?; + let dst = std::ffi::CString::new(mount_point.to_str()?).ok()?; + let fstype = std::ffi::CString::new("virtiofs").unwrap(); + // SAFETY: mount virtiofs with valid CString arguments + let rc = unsafe { + libc::mount( + src.as_ptr(), + dst.as_ptr(), + fstype.as_ptr(), + 0, + std::ptr::null(), + ) + }; + + if rc != 0 { + let err = std::io::Error::last_os_error(); + warn!(error = %err, tag = %tag, "failed to mount packed layers virtiofs"); + return None; + } + info!(mount_point = %mount_point.display(), "packed layers mounted successfully"); + + // List contents for debugging (only at debug level to avoid boot overhead) + if let Ok(entries) = std::fs::read_dir(&mount_point) { + let layer_dirs: Vec<_> = entries + .filter_map(|e| e.ok()) + .filter(|e| e.path().is_dir()) + .map(|e| e.file_name().to_string_lossy().to_string()) + .collect(); + debug!(layer_count = layer_dirs.len(), layers = ?layer_dirs, "packed layers available"); + } + + Some(mount_point) + } + #[cfg(not(target_os = "linux"))] + { + warn!("packed layers mount not supported on non-Linux"); + None + } +} + +/// Get the packed layers directory if available. +pub fn get_packed_layers_dir() -> Option<&'static PathBuf> { + PACKED_LAYERS_DIR.get_or_init(init_packed_layers).as_ref() +} + +/// Initialize volume mounts at boot by reading SMOLVM_MOUNT_* env vars. +/// +/// The host launcher sets: +/// SMOLVM_MOUNT_COUNT=N +/// SMOLVM_MOUNT_0=smolvm0:/data:rw +/// SMOLVM_MOUNT_1=smolvm1:/config:ro +/// +/// This mounts each virtiofs device at its staging area and bind-mounts +/// to the guest target path, making volumes visible to all code paths +/// including VmExec. +pub fn init_volume_mounts() -> &'static [(String, String, bool)] { + BOOT_VOLUME_MOUNTS.get_or_init(|| { + let count: usize = match std::env::var("SMOLVM_MOUNT_COUNT") { + Ok(v) => match v.parse() { + Ok(n) => n, + Err(_) => { + warn!(value = %v, "invalid SMOLVM_MOUNT_COUNT"); + return Vec::new(); + } + }, + Err(_) => return Vec::new(), + }; + + let mut mounts = Vec::with_capacity(count); + for i in 0..count { + let env_key = format!("SMOLVM_MOUNT_{}", i); + let env_val = match std::env::var(&env_key) { + Ok(v) => v, + Err(_) => { + warn!(key = %env_key, "missing mount env var"); + continue; + } + }; + + // Parse "tag:guest_path:ro|rw" + let parts: Vec<&str> = env_val.splitn(3, ':').collect(); + if parts.len() != 3 { + warn!(key = %env_key, value = %env_val, "invalid mount format, expected tag:path:ro|rw"); + continue; + } + + let tag = parts[0].to_string(); + let guest_path = parts[1].to_string(); + let read_only = parts[2] == "ro"; + + info!(tag = %tag, guest_path = %guest_path, read_only = read_only, "boot volume mount"); + mounts.push((tag, guest_path, read_only)); + } + + // Mount using existing logic with empty rootfs prefix so bind mounts + // go to absolute guest paths (e.g., "/data"), visible to VmExec. + if !mounts.is_empty() { + if let Err(e) = setup_volume_mounts("/", &mounts) { + warn!(error = %e, "failed to setup boot volume mounts"); + } + } + + mounts + }) +} + +/// Add the /storage/workspace → /workspace fallback bind mount to an OCI spec, +/// unless a user-provided volume already claims /workspace. +/// +/// The fallback exposes the storage disk's workspace directory inside containers +/// so that persistent files written to /workspace survive across VM restarts. +/// It must be skipped when the user provides `-v host:/workspace` to avoid +/// silently overwriting their virtiofs mount (which comes earlier in the spec). +/// +/// Mount target comparison is slash-normalized to handle trailing slashes. +pub fn add_workspace_fallback(spec: &mut OciSpec, mounts: &[(String, String, bool)]) { + let workspace_src = Path::new(STORAGE_ROOT).join(WORKSPACE_DIR); + if !workspace_src.exists() { + return; + } + let user_owns_workspace = mounts + .iter() + .any(|(_, path, _)| path.trim_end_matches('/') == paths::WORKSPACE_GUEST_PATH); + if !user_owns_workspace { + spec.add_bind_mount( + &workspace_src.to_string_lossy(), + paths::WORKSPACE_GUEST_PATH, + false, + ); + } +} + +/// Expose the per-VM `/storage` disk inside privileged containers, so an +/// `--image` machine has the same filesystem topology as a bare VM: `/storage` +/// (and therefore `/storage/docker`, `/storage/workspace`, …) resolves to the +/// ext4 disk identically with or without `--image`, and bind-mounts the workload +/// makes against it — e.g. docker-in-VM binding `/storage/docker` → +/// `/var/lib/docker` so overlay2 lands on ext4, not the rootfs overlay — work +/// the same in a container as in a bare VM. +/// +/// Privileged-only: when `unprivileged` the container is a defense-in-depth +/// boundary for untrusted code, so it must NOT see the VM's storage disk (its +/// image archives and overlay plumbing). `/storage` is per-machine, so for a +/// privileged workload — where the microVM is the security boundary — exposing +/// it crosses no isolation boundary; it only mirrors what a bare VM already has. +/// +/// Skipped when the user already mounted something at `/storage`, and a no-op +/// when the disk isn't mounted (bare agent rootfs / no storage disk). +pub fn add_storage_fallback( + spec: &mut OciSpec, + mounts: &[(String, String, bool)], + unprivileged: bool, +) { + // Decide policy first (testable without a mounted disk), then gate on the + // disk actually being present (a no-op on a bare agent rootfs). + if should_expose_storage(mounts, unprivileged) && Path::new(STORAGE_ROOT).exists() { + spec.add_bind_mount(STORAGE_ROOT, STORAGE_ROOT, false); + } +} + +/// Policy for [`add_storage_fallback`]: a privileged workload that hasn't already +/// claimed `/storage` should see the VM's storage disk. Unprivileged containers +/// never do (defense-in-depth boundary for untrusted code). +fn should_expose_storage(mounts: &[(String, String, bool)], unprivileged: bool) -> bool { + if unprivileged { + return false; + } + !mounts + .iter() + .any(|(_, path, _)| path.trim_end_matches('/') == STORAGE_ROOT) +} + +/// Name of the optional index file (written into the packed-layers dir at +/// extraction time) recording the layers in OCI order, bottom-most first, one +/// short layer id per line. +/// +/// Packed layer subdirs are content-addressed (named by digest), so sorting +/// their names does NOT reproduce the manifest's stacking order. That is fine +/// for the common single-flattened-layer image, but a multi-layer pack (e.g. an +/// init-cache base + init-overlay layer from `pack create --from-vm`) gets +/// mis-stacked: the base can sort above the overlay, so overlayfs shadows the +/// overlay's in-place edits to base files (`/etc/ld.so.cache`, +/// `/var/lib/dpkg/status`) while keeping its new files — installed packages then +/// appear on disk but unregistered, and libs in multiarch dirs fail to load. +/// Honoring this index restores the true order; absent (older packs) we fall +/// back to a name sort. +const LAYER_ORDER_FILE: &str = "layer-order"; + +/// Packed layer directory names in OCI order, **bottom-most layer first**. +/// +/// Honors [`LAYER_ORDER_FILE`] when present and self-consistent; otherwise falls +/// back to a lexical name sort (correct for the single-flattened-layer case). +/// Only names backed by an existing subdirectory are returned, which also drops +/// stray non-layer dirs (e.g. macOS `.fseventsd`) when the index is present. +fn ordered_packed_layer_names(packed_dir: &Path) -> Result> { + // The layer subdirs actually present (excluding source `.tar` files and the + // order index itself, which is a plain file). + let mut present: std::collections::BTreeSet = std::collections::BTreeSet::new(); + let entries = std::fs::read_dir(packed_dir) + .map_err(|e| StorageError::read_error(packed_dir.display().to_string(), e))?; + for entry in entries { + let entry = entry?; + if entry.path().is_dir() { + let name = entry.file_name().to_string_lossy().to_string(); + if !name.ends_with(".tar") { + present.insert(name); + } + } + } + + // Prefer the explicit order index when it resolves to layers we actually have. + if let Ok(contents) = std::fs::read_to_string(packed_dir.join(LAYER_ORDER_FILE)) { + let ordered: Vec = contents + .lines() + .map(|l| l.trim().to_string()) + .filter(|l| present.contains(l)) + .collect(); + if !ordered.is_empty() { + return Ok(ordered); + } + } + + // Fallback: name sort (BTreeSet is already ascending = bottom→top by the + // legacy "stub creates layers in order" convention). + Ok(present.into_iter().collect()) +} + +/// Create a synthetic ImageInfo from packed layers. +/// This is used when running from a packed binary where layers are pre-extracted. +fn create_packed_image_info(image: &str, packed_dir: &Path) -> Result { + // Layer dirs in OCI order (bottom→top), as sha256:{short_digest} ids. + let layer_dirs: Vec = ordered_packed_layer_names(packed_dir)? + .into_iter() + .map(|name| format!("sha256:{}", name)) + .collect(); + + // Calculate approximate size + let mut total_size = 0u64; + for layer_digest in &layer_dirs { + let short_id = layer_digest.strip_prefix("sha256:").unwrap_or(layer_digest); + let layer_path = packed_dir.join(short_id); + if let Ok(size) = dir_size(&layer_path) { + total_size += size; + } + } + + // Determine architecture from environment or default + #[cfg(target_arch = "aarch64")] + let architecture = "arm64".to_string(); + #[cfg(target_arch = "x86_64")] + let architecture = "amd64".to_string(); + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + let architecture = "unknown".to_string(); + + // For a flattened local archive these come from the recovered image config; + // a .smolmachine has no config.json so they stay empty (its config lives in + // the PackManifest). + let (entrypoint, cmd, env, workdir, user) = read_packed_image_config(packed_dir); + + Ok(ImageInfo { + reference: image.to_string(), + digest: "packed".to_string(), // No real digest available for packed images + size: total_size, + created: None, + architecture, + os: "linux".to_string(), + layer_count: layer_dirs.len(), + layers: layer_dirs, + entrypoint, + cmd, + env, + workdir, + user, + }) +} + +// ============================================================================= +// Local image archives (`docker save` / `podman save`) +// ============================================================================= +// +// smolvm delegates turning a saved-image archive into a rootfs to the bundled +// `crane` (and `gunzip`/`tar`) rather than parsing OCI layers itself. The host +// stages `archive.tar` into a content-addressed dir mounted via virtiofs as the +// packed-layers dir; here we flatten it once into a rootfs on the writable +// storage disk, recover the image config, and present it as a single packed +// layer that the existing overlay path consumes. + +/// Filename the host stages the saved-image archive under. +const ARCHIVE_FILE_NAME: &str = "archive.tar"; +/// Subdir the flattened rootfs is written to (a single packed "layer"). +const ARCHIVE_ROOTFS_DIR: &str = "0000_rootfs"; +/// Recovered image config (`crane config` output) beside the rootfs. +const ARCHIVE_CONFIG_FILE: &str = "config.json"; +/// Marker written once a flatten completes, so restarts reuse it. +const ARCHIVE_EXTRACTED_MARKER: &str = ".extracted"; + +/// If `packed_dir` is a staged local image archive (it contains `archive.tar`), +/// flatten it once into a rootfs on the storage disk and return that directory +/// (holding `0000_rootfs/` + `config.json`). Returns `None` for an ordinary +/// packed-layers dir (a `.smolmachine`'s pre-extracted layers). +/// +/// The output is keyed by the virtiofs mount-point name (constant per VM, since +/// `/storage` is per-machine), and the completion marker stores the archive's +/// size+mtime signature. A start reuses the flatten only when that signature +/// still matches, so a machine re-created from a different image on a reused +/// disk re-flattens instead of booting the old rootfs. The marker is written +/// last, so the image-info and overlay paths share one flatten within a start. +fn ensure_archive_flattened(packed_dir: &Path) -> Result> { + let archive = packed_dir.join(ARCHIVE_FILE_NAME); + if !archive.exists() { + return Ok(None); + } + let key = packed_dir + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("archive"); + let out_base = Path::new(STORAGE_ROOT).join("image-archives").join(key); + let marker = out_base.join(ARCHIVE_EXTRACTED_MARKER); + let signature = archive_signature(&archive)?; + if std::fs::read_to_string(&marker).ok().as_deref() == Some(signature.as_str()) { + return Ok(Some(out_base)); + } + + // First flatten, or the archive changed under a reused disk: rebuild. + let _ = std::fs::remove_dir_all(&out_base); + let rootfs = out_base.join(ARCHIVE_ROOTFS_DIR); + std::fs::create_dir_all(&rootfs)?; + info!(archive = %archive.display(), rootfs = %rootfs.display(), "flattening local image archive"); + flatten_archive(&archive, &rootfs)?; + // Recover the image config before writing the marker, so a later reuse can + // rely on config.json being present. A docker/podman `save` always carries + // one. + recover_archive_config(&archive, &out_base.join(ARCHIVE_CONFIG_FILE))?; + std::fs::write(&marker, signature)?; + Ok(Some(out_base)) +} + +/// A cheap content signature for a staged archive (size + mtime), used to +/// invalidate a stale flatten when a reused disk's archive changed. +fn archive_signature(archive: &Path) -> Result { + let meta = std::fs::metadata(archive)?; + let mtime = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0); + Ok(format!("{}:{}", meta.len(), mtime)) +} + +/// Feed an archive to `cmd`'s stdin, transparently decompressing a gzip- OR +/// zstd-compressed outer archive first. A compressed archive is expanded to a +/// temp file whose handle is returned so the caller keeps it alive until `cmd` +/// has consumed it; a plain archive streams straight through. This handles zstd, +/// which the old `gunzip`-only path silently mangled. +fn pipe_archive_into(cmd: &mut Command, archive: &Path) -> Result> { + let file = std::fs::File::open(archive)?; + if !is_compressed(archive)? { + cmd.stdin(Stdio::from(file)); + return Ok(None); + } + // Expand to a temp file, then feed that. `docker save` archives are a local + // dev-import path, so the extra copy is cheap and avoids threading a + // streaming decompressor into a subprocess's stdin. The guest ships no zstd + // tool, so decompression is done in-process. + let mut reader = decompress_layer_reader(file)?; + let mut tmp = tempfile::NamedTempFile::new() + .map_err(|e| StorageError::new(format!("failed to create temp file: {e}")))?; + std::io::copy(&mut reader, tmp.as_file_mut()) + .map_err(|e| StorageError::new(format!("failed to decompress archive: {e}")))?; + let reopened = tmp + .reopen() + .map_err(|e| StorageError::new(format!("failed to reopen temp file: {e}")))?; + cmd.stdin(Stdio::from(reopened)); + Ok(Some(tmp)) +} + +/// Flatten a `docker save` archive into `rootfs`, delegating to the bundled +/// `crane export`. The flattened tar is a single layer with no whiteouts, so +/// plain `tar -x` is sufficient (no per-layer handling needed). +fn flatten_archive(archive: &Path, rootfs: &Path) -> Result<()> { + // crane export - - : read an image tarball from stdin, write a flat rootfs + // tar to stdout. + let mut crane = Command::new("crane"); + crane + .args(["export", "-", "-"]) + .stdout(Stdio::piped()) + // Capture (don't discard) crane's stderr so a failure reports the REAL + // reason — e.g. "file manifest.json not found in tar" for an empty or + // truncated archive — instead of a misleading guess. + .stderr(Stdio::piped()); + // Held alive until crane has consumed it (the decompressed input, if any). + let _archive_tmp = pipe_archive_into(&mut crane, archive)?; + + let mut crane_child = crane + .spawn() + .map_err(|e| StorageError::new(format!("failed to spawn crane export: {e}")))?; + let crane_out = crane_child + .stdout + .take() + .ok_or_else(|| StorageError::new("failed to capture crane stdout".to_string()))?; + let mut crane_err = crane_child + .stderr + .take() + .ok_or_else(|| StorageError::new("failed to capture crane stderr".to_string()))?; + + let tar_out = Command::new("tar") + .arg("-x") + .arg("-C") + .arg(rootfs) + .stdin(Stdio::from(crane_out)) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .output() + .map_err(|e| StorageError::new(format!("failed to run tar: {e}")))?; + + let crane_status = crane_child + .wait() + .map_err(|e| StorageError::new(format!("failed to wait for crane: {e}")))?; + + if !crane_status.success() { + // crane's stderr is a single short line; reading it after the process + // exits (its stdout was drained by `tar`) cannot deadlock. + let mut stderr = String::new(); + let _ = std::io::Read::read_to_string(&mut crane_err, &mut stderr); + let stderr = stderr.trim(); + return Err(StorageError::new(format!( + "crane export failed{} (is the image a valid `docker save` / OCI archive?)", + if stderr.is_empty() { + String::new() + } else { + format!(": {stderr}") + } + ))); + } + if !tar_out.status.success() { + return Err(StorageError::new(format!( + "extracting flattened rootfs failed: {}", + String::from_utf8_lossy(&tar_out.stderr) + ))); + } + Ok(()) +} + +/// Recover the image config (Entrypoint/Cmd/Env/…) from a `docker save` archive +/// and write it to `dest`. The archive's `manifest.json` names the config blob +/// under its `Config` key; both are small JSON members extracted with `tar` +/// (image metadata, not layer/rootfs assembly — that stays delegated to crane). +fn recover_archive_config(archive: &Path, dest: &Path) -> Result<()> { + let manifest_bytes = extract_tar_member(archive, "manifest.json")?; + let manifest: serde_json::Value = serde_json::from_slice(&manifest_bytes) + .map_err(|e| StorageError::new(format!("parse archive manifest.json: {e}")))?; + let config_path = manifest[0]["Config"].as_str().ok_or_else(|| { + StorageError::new("archive manifest.json has no Config entry".to_string()) + })?; + let config_bytes = extract_tar_member(archive, config_path)?; + std::fs::write(dest, &config_bytes)?; + Ok(()) +} + +/// Extract a single named member from an archive to memory via `tar -xO`, +/// transparently `gunzip`-ing a gzipped outer archive. +fn extract_tar_member(archive: &Path, member: &str) -> Result> { + let mut tar = Command::new("tar"); + tar.args(["-x", "-O", "-f", "-"]) + .arg(member) + .stderr(Stdio::null()); + // Held alive until tar has consumed it (the decompressed input, if any). + let _archive_tmp = pipe_archive_into(&mut tar, archive)?; + let out = tar + .output() + .map_err(|e| StorageError::new(format!("failed to run tar: {e}")))?; + if !out.status.success() || out.stdout.is_empty() { + return Err(StorageError::new(format!( + "could not read '{member}' from archive" + ))); + } + Ok(out.stdout) +} + +/// Whether a file begins with a supported compression magic — gzip (`1f 8b`) or +/// zstd (`28 b5 2f fd`). +fn is_compressed(path: &Path) -> Result { + use std::io::Read; + let mut magic = [0u8; 4]; + let n = std::fs::File::open(path)?.read(&mut magic).unwrap_or(0); + Ok((n >= 2 && magic[0] == 0x1f && magic[1] == 0x8b) + || (n >= 4 && magic[..4] == [0x28, 0xb5, 0x2f, 0xfd])) +} + +/// Read `Entrypoint`/`Cmd`/`Env`/`WorkingDir`/`User` from a recovered image +/// `config.json` (the `crane config` output) in `packed_dir`, defaulting to +/// empty when absent — a `.smolmachine` has no such file. +#[allow(clippy::type_complexity)] +fn read_packed_image_config( + packed_dir: &Path, +) -> ( + Vec, + Vec, + Vec, + Option, + Option, +) { + let empty = (Vec::new(), Vec::new(), Vec::new(), None, None); + let Ok(content) = std::fs::read_to_string(packed_dir.join(ARCHIVE_CONFIG_FILE)) else { + return empty; + }; + let Ok(json) = serde_json::from_str::(&content) else { + return empty; + }; + let cfg = &json["config"]; + let string_list = |key: &str| -> Vec { + cfg[key] + .as_array() + .map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default() + }; + let non_empty = |key: &str| -> Option { + cfg[key] + .as_str() + .filter(|s| !s.is_empty()) + .map(String::from) + }; + ( + string_list("Entrypoint"), + string_list("Cmd"), + string_list("Env"), + non_empty("WorkingDir"), + non_empty("User"), + ) +} + +/// Error type for storage operations. +#[derive(Debug)] +#[allow(dead_code)] // Some variants reserved for future use +pub enum StorageError { + // ======================================================================== + // I/O Errors + // ======================================================================== + /// Failed to create a directory. + CreateDir { path: String, cause: String }, + /// Failed to remove a directory. + RemoveDir { path: String, cause: String }, + /// Failed to read a file or directory. + ReadFile { path: String, cause: String }, + /// Failed to write a file. + WriteFile { path: String, cause: String }, + /// Failed to create a symlink. + Symlink { + source: String, + target: String, + cause: String, + }, + /// Path conversion error. + InvalidPath { path: String }, + + // ======================================================================== + // Image Errors + // ======================================================================== + /// Image not found locally. + ImageNotFound { image: String }, + /// Failed to pull image from registry. + ImagePullFailed { image: String, cause: String }, + /// Invalid image reference format. + InvalidImageReference { reference: String, reason: String }, + + // ======================================================================== + // Layer Errors + // ======================================================================== + /// Layer not found. + LayerNotFound { digest: String }, + /// Failed to extract layer. + LayerExtractionFailed { digest: String, cause: String }, + /// Layer index out of bounds. + LayerIndexOutOfBounds { + image: String, + index: usize, + total: usize, + }, + + // ======================================================================== + // Manifest/Config Errors + // ======================================================================== + /// Failed to parse manifest or config JSON. + ParseError { context: String, cause: String }, + /// Missing required field in manifest/config. + MissingField { context: String, field: String }, + /// Unsupported manifest format. + UnsupportedManifest { media_type: String }, + + // ======================================================================== + // Mount Errors + // ======================================================================== + /// Failed to mount overlay filesystem. + OverlayMountFailed { path: String, cause: String }, + /// Failed to unmount filesystem. + UnmountFailed { path: String, cause: String }, + + // ======================================================================== + // Command Execution Errors + // ======================================================================== + /// External command (crane, crun, etc.) failed. + CommandFailed { + command: String, + exit_code: Option, + stderr: String, + }, + /// Failed to spawn external command. + SpawnFailed { command: String, cause: String }, + + // ======================================================================== + // Validation Errors + // ======================================================================== + /// Input validation failed. + ValidationFailed { context: String, reason: String }, + + // ======================================================================== + // Storage State Errors + // ======================================================================== + /// Storage not formatted/initialized. + StorageNotReady { reason: String }, + /// No images found in storage. + NoImagesFound, + + // ======================================================================== + // Generic + // ======================================================================== + /// Internal error with message (fallback for complex cases). + Internal { message: String }, +} + +#[allow(dead_code)] // Some helpers reserved for future use +impl StorageError { + /// Create a new internal error with the given message. + /// Use this as a fallback when no specific variant fits. + pub fn new(message: impl Into) -> Self { + StorageError::Internal { + message: message.into(), + } + } + + /// Create an I/O read error. + pub fn read_error(path: impl Into, cause: impl std::fmt::Display) -> Self { + StorageError::ReadFile { + path: path.into(), + cause: cause.to_string(), + } + } + + /// Create an I/O write error. + pub fn write_error(path: impl Into, cause: impl std::fmt::Display) -> Self { + StorageError::WriteFile { + path: path.into(), + cause: cause.to_string(), + } + } + + /// Create a directory creation error. + pub fn create_dir_error(path: impl Into, cause: impl std::fmt::Display) -> Self { + StorageError::CreateDir { + path: path.into(), + cause: cause.to_string(), + } + } + + /// Create a parse error. + pub fn parse_error(context: impl Into, cause: impl std::fmt::Display) -> Self { + StorageError::ParseError { + context: context.into(), + cause: cause.to_string(), + } + } + + /// Create a command failed error. + pub fn command_failed( + command: impl Into, + exit_code: Option, + stderr: impl Into, + ) -> Self { + StorageError::CommandFailed { + command: command.into(), + exit_code, + stderr: stderr.into(), + } + } +} + +impl std::fmt::Display for StorageError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + // I/O errors + StorageError::CreateDir { path, cause } => { + write!(f, "failed to create directory '{}': {}", path, cause) + } + StorageError::RemoveDir { path, cause } => { + write!(f, "failed to remove directory '{}': {}", path, cause) + } + StorageError::ReadFile { path, cause } => { + write!(f, "failed to read '{}': {}", path, cause) + } + StorageError::WriteFile { path, cause } => { + write!(f, "failed to write '{}': {}", path, cause) + } + StorageError::Symlink { + source, + target, + cause, + } => { + write!( + f, + "failed to create symlink '{}' -> '{}': {}", + source, target, cause + ) + } + StorageError::InvalidPath { path } => { + write!(f, "invalid path: {}", path) + } + + // Image errors + StorageError::ImageNotFound { image } => { + write!(f, "image not found: {}", image) + } + StorageError::ImagePullFailed { image, cause } => { + write!(f, "failed to pull image '{}': {}", image, cause) + } + StorageError::InvalidImageReference { reference, reason } => { + write!(f, "invalid image reference '{}': {}", reference, reason) + } + + // Layer errors + StorageError::LayerNotFound { digest } => { + write!(f, "layer not found: {}", digest) + } + StorageError::LayerExtractionFailed { digest, cause } => { + write!(f, "failed to extract layer '{}': {}", digest, cause) + } + StorageError::LayerIndexOutOfBounds { + image, + index, + total, + } => { + write!( + f, + "layer index {} out of bounds for image '{}' (has {} layers)", + index, image, total + ) + } + + // Manifest/config errors + StorageError::ParseError { context, cause } => { + write!(f, "failed to parse {}: {}", context, cause) + } + StorageError::MissingField { context, field } => { + write!(f, "missing '{}' in {}", field, context) + } + StorageError::UnsupportedManifest { media_type } => { + write!(f, "unsupported manifest format: {}", media_type) + } + + // Mount errors + StorageError::OverlayMountFailed { path, cause } => { + write!(f, "overlay mount failed at '{}': {}", path, cause) + } + StorageError::UnmountFailed { path, cause } => { + write!(f, "failed to unmount '{}': {}", path, cause) + } + + // Command errors + StorageError::CommandFailed { + command, + exit_code, + stderr, + } => { + if let Some(code) = exit_code { + write!(f, "{} failed (exit {}): {}", command, code, stderr) + } else { + write!(f, "{} failed: {}", command, stderr) + } + } + StorageError::SpawnFailed { command, cause } => { + write!(f, "failed to spawn '{}': {}", command, cause) + } + + // Validation errors + StorageError::ValidationFailed { context, reason } => { + write!(f, "{}: {}", context, reason) + } + + // Storage state errors + StorageError::StorageNotReady { reason } => { + write!(f, "storage not ready: {}", reason) + } + StorageError::NoImagesFound => { + write!(f, "no images found") + } + + // Generic + StorageError::Internal { message } => { + write!(f, "{}", message) + } + } + } +} + +impl std::error::Error for StorageError {} + +impl From for StorageError { + fn from(e: std::io::Error) -> Self { + StorageError::Internal { + message: e.to_string(), + } + } +} + +type Result = std::result::Result; + +/// Check if a layer directory is properly cached (exists and has content). +/// +/// An empty layer directory indicates failed/incomplete extraction and should +/// be re-extracted. This prevents issues where layer_dir.exists() returns true +/// but the directory is empty due to interrupted extraction. +fn is_layer_cached(layer_dir: &Path) -> bool { + if !layer_dir.exists() { + return false; + } + // Check if the directory has any entries + match std::fs::read_dir(layer_dir) { + Ok(mut entries) => entries.next().is_some(), + Err(_) => false, + } +} + +/// Initialize storage directories. +/// +/// This function ensures all required storage directories exist and are accessible. +/// Returns early (successfully) if storage hasn't been formatted yet. +/// +/// Note: `mount_storage_disk()` already creates all directories, so this is +/// not called during boot. Kept for manual validation/repair use cases. +#[allow(dead_code)] +pub fn init() -> Result<()> { + let root = Path::new(STORAGE_ROOT); + + // Check if storage root exists or can be created + if !root.exists() { + info!(path = %root.display(), "creating storage root directory"); + std::fs::create_dir_all(root).map_err(|e| { + StorageError::new(format!( + "failed to create storage root '{}': {} (check permissions and disk space)", + root.display(), + e + )) + })?; + } + + // Verify storage root is accessible + if let Err(e) = std::fs::read_dir(root) { + return Err(StorageError::new(format!( + "storage root '{}' exists but is not accessible: {} (check permissions)", + root.display(), + e + ))); + } + + // Create container runtime directories unconditionally — these are needed + // as soon as containers are requested, regardless of storage format state. + let container_dirs = [ + (paths::CONTAINERS_RUN_DIR, "container runtime state"), + (paths::CONTAINERS_LOGS_DIR, "container logs"), + (paths::CONTAINERS_EXIT_DIR, "container exit codes"), + (paths::CRUN_ROOT_DIR, "crun state root"), + ]; + + let mut created_count = 0; + for (dir, description) in &container_dirs { + let path = Path::new(dir); + if !path.exists() { + std::fs::create_dir_all(path).map_err(|e| { + StorageError::new(format!( + "failed to create {} directory '{}': {}", + description, + path.display(), + e + )) + })?; + debug!(path = %path.display(), description = %description, "created directory"); + created_count += 1; + } + } + + // Check for marker file to see if formatted + let marker = root.join(".smolvm_formatted"); + if !marker.exists() { + info!(path = %root.display(), "storage not formatted, waiting for format request"); + return Ok(()); + } + + // Create OCI storage directory structure + let required_dirs = [ + (LAYERS_DIR, "OCI image layers"), + (CONFIGS_DIR, "image configurations"), + (MANIFESTS_DIR, "image manifests"), + (OVERLAYS_DIR, "overlay filesystems"), + ( + WORKSPACE_DIR, + "shared workspace (visible inside containers)", + ), + ]; + + for (dir, description) in &required_dirs { + let path = root.join(dir); + if !path.exists() { + std::fs::create_dir_all(&path).map_err(|e| { + StorageError::new(format!( + "failed to create {} directory '{}': {}", + description, + path.display(), + e + )) + })?; + debug!(path = %path.display(), description = %description, "created directory"); + created_count += 1; + } + } + + info!( + path = %root.display(), + dirs_created = created_count, + "storage initialized" + ); + Ok(()) +} + +/// Format the storage disk. +/// +/// Creates all required directories and writes the format marker file. +/// If directories already exist, they are left as-is. +pub fn format() -> Result<()> { + let root = Path::new(STORAGE_ROOT); + + // Ensure storage root exists + if !root.exists() { + std::fs::create_dir_all(root).map_err(|e| { + StorageError::new(format!( + "failed to create storage root '{}': {}", + root.display(), + e + )) + })?; + } + + // Create all storage directories + let all_dirs = [ + (root.join(LAYERS_DIR), "layers"), + (root.join(CONFIGS_DIR), "configs"), + (root.join(MANIFESTS_DIR), "manifests"), + (root.join(OVERLAYS_DIR), "overlays"), + (PathBuf::from(paths::CONTAINERS_RUN_DIR), "container run"), + (PathBuf::from(paths::CONTAINERS_LOGS_DIR), "container logs"), + (PathBuf::from(paths::CONTAINERS_EXIT_DIR), "container exit"), + (PathBuf::from(paths::CRUN_ROOT_DIR), "crun state root"), + ]; + + for (path, name) in &all_dirs { + std::fs::create_dir_all(path).map_err(|e| { + StorageError::new(format!( + "failed to create {} directory '{}': {}", + name, + path.display(), + e + )) + })?; + } + + // Create marker file + let marker = root.join(".smolvm_formatted"); + std::fs::write(&marker, "1").map_err(|e| { + StorageError::new(format!( + "failed to write format marker '{}': {}", + marker.display(), + e + )) + })?; + + info!(path = %root.display(), "storage formatted"); + Ok(()) +} + +/// Get storage status. +pub fn status() -> Result { + let root = Path::new(STORAGE_ROOT); + let marker = root.join(".smolvm_formatted"); + + let ready = marker.exists(); + + // Get disk usage (simplified) + let (total_bytes, used_bytes) = get_disk_usage(root)?; + + // Count layers and images + let layer_count = count_entries(&root.join(LAYERS_DIR))?; + let image_count = count_entries(&root.join(MANIFESTS_DIR))?; + + Ok(StorageStatus { + ready, + total_bytes, + used_bytes, + layer_count, + image_count, + }) +} + +/// Extract a JSON array of strings from a JSON value. +fn json_string_array(value: &serde_json::Value, key: &str) -> Vec { + value[key] + .as_array() + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default() +} + +/// OCI/AUFS whiteout marker that deletes a single name from lower layers +/// (`.wh.`). +const WHITEOUT_PREFIX: &str = ".wh."; +/// OCI/AUFS opaque-directory marker: the directory replaces, rather than merges +/// with, the same directory in lower layers (`.wh..wh..opq`). +const OPAQUE_WHITEOUT: &str = ".wh..wh..opq"; + +/// What an OCI layer tar entry means once its name is interpreted. +#[derive(Debug, PartialEq, Eq)] +enum LayerEntry<'a> { + /// `.wh..wh..opq`: mark the parent directory opaque. + OpaqueDir, + /// `.wh.`: delete `` from lower layers (carries ``). + Whiteout(&'a str), + /// An ordinary entry to extract as-is. + Normal, +} + +/// Classify an entry by its file name. The opaque marker must be checked before +/// the generic `.wh.` prefix, since `.wh..wh..opq` also starts with `.wh.`. +fn classify_layer_entry(file_name: &str) -> LayerEntry<'_> { + if file_name == OPAQUE_WHITEOUT { + return LayerEntry::OpaqueDir; + } + match file_name.strip_prefix(WHITEOUT_PREFIX) { + Some(name) if !name.is_empty() => LayerEntry::Whiteout(name), + _ => LayerEntry::Normal, + } +} + +/// Join `rel` under `base`, returning `None` if any component would escape the +/// base (`..`, an absolute path, or a Windows-style prefix). Mirrors the +/// containment guard tar extractors use to prevent path-traversal. +fn jailed_join(base: &Path, rel: &Path) -> Option { + use std::path::Component; + let mut out = base.to_path_buf(); + for component in rel.components() { + match component { + Component::Normal(part) => out.push(part), + Component::CurDir => {} + Component::ParentDir | Component::RootDir | Component::Prefix(_) => return None, + } + } + Some(out) +} + +/// Create an overlayfs whiteout (a `mknod` character device with device number +/// 0/0) at `path`, replacing any existing entry. This is how the kernel's +/// overlayfs records "this name is deleted from lower layers" — the on-disk +/// representation that OCI's `.wh.` marker must be translated into. +/// +/// Linux-only: overlayfs whiteouts are a Linux concept and the agent only runs +/// in the Linux guest. The non-Linux stub keeps the crate compiling on the +/// macOS host (where `mknod`/`makedev`/xattr signatures differ). +#[cfg(target_os = "linux")] +fn create_overlay_whiteout(path: &Path) -> std::io::Result<()> { + use std::os::unix::ffi::OsStrExt; + // Clear any entry the layer may already have written at this name so mknod + // doesn't fail with EEXIST. + let _ = std::fs::remove_file(path); + let _ = std::fs::remove_dir_all(path); + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + // SAFETY: `c_path` is a valid NUL-terminated path; mode and dev are scalars. + let rc = unsafe { + libc::mknod( + c_path.as_ptr(), + libc::S_IFCHR as libc::mode_t, + libc::makedev(0, 0), + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn create_overlay_whiteout(_path: &Path) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "overlayfs whiteouts are only created in the Linux guest", + )) +} + +/// Mark `dir` opaque for overlayfs via the `trusted.overlay.opaque` xattr — the +/// representation of OCI's `.wh..wh..opq` marker. Linux-only (see +/// [`create_overlay_whiteout`]). +#[cfg(target_os = "linux")] +fn set_overlay_opaque(dir: &Path) -> std::io::Result<()> { + use std::os::unix::ffi::OsStrExt; + let c_path = std::ffi::CString::new(dir.as_os_str().as_bytes()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + let name = std::ffi::CString::new("trusted.overlay.opaque").expect("static xattr name"); + let value = b"y"; + // SAFETY: path/name are NUL-terminated; value/len describe a valid buffer. + let rc = unsafe { + libc::setxattr( + c_path.as_ptr(), + name.as_ptr(), + value.as_ptr() as *const libc::c_void, + value.len(), + 0, + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn set_overlay_opaque(_dir: &Path) -> std::io::Result<()> { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "overlayfs opaque dirs are only set in the Linux guest", + )) +} + +/// Extract one decompressed OCI layer tar into `dest`, applying OCI whiteout +/// semantics so the overlayfs composition is correct. +/// +/// Each layer is extracted in isolation into its own directory (later stacked as +/// an overlayfs lowerdir). A plain `tar -x` does not give the kernel's overlayfs +/// what it needs, so this translates at unpack time — the same conversion +/// containerd / docker-overlay2 / containers-storage perform: +/// - `.wh.` (delete marker) → an overlayfs whiteout (`mknod c 0 0`), so +/// the stacked overlay hides `` from lower layers. (busybox `tar` left +/// it as a plain file overlayfs ignores, so deletions never applied; worse, +/// some images ship the marker as a hardlink to a lower-layer file, which +/// aborted the whole extraction — issue #397. Whiteouts are handled by name +/// here, before the hardlink path, so that case is just a normal whiteout.) +/// - `.wh..wh..opq` (opaque-dir marker) → the `trusted.overlay.opaque` xattr on +/// its parent directory. +/// - a hardlink whose target isn't in this layer (it lives in a lower layer) is +/// skipped rather than failing; overlayfs resolves the real file at runtime. +/// +/// Requires `CAP_MKNOD` + `CAP_SYS_ADMIN`, which the agent has as guest PID 1. +/// Wrap a layer stream so gzip- AND zstd-compressed layers are transparently +/// decompressed; an already-plain tar passes through. OCI layers are `+gzip` or +/// (what `skopeo` and `smolvm pack` emit by default) `+zstd`. Detection is by +/// magic bytes, so it's correct regardless of the manifest mediaType or the +/// source — registry pull, local pack, or docker-save import — and needs no +/// external tool in the guest rootfs. +fn decompress_layer_reader<'a>( + mut inner: impl std::io::Read + 'a, +) -> std::io::Result> { + use std::io::Read; + // Peek the compression magic without consuming it: read up to 4 bytes, then + // chain them back in front of the rest of the stream. + let mut magic = [0u8; 4]; + let mut n = 0; + while n < magic.len() { + match inner.read(&mut magic[n..])? { + 0 => break, + k => n += k, + } + } + let stream = std::io::Cursor::new(magic[..n].to_vec()).chain(inner); + if n >= 2 && magic[0] == 0x1f && magic[1] == 0x8b { + Ok(Box::new(flate2::read::GzDecoder::new(stream))) + } else if n >= 4 && magic[..4] == [0x28, 0xb5, 0x2f, 0xfd] { + // Pure-Rust zstd decoder — the C `zstd` crate can't link against musl + // (see the note in Cargo.toml). + let dec = ruzstd::StreamingDecoder::new(stream) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))?; + Ok(Box::new(dec)) + } else { + // Uncompressed tar (or unrecognized) — pass through untouched. + Ok(Box::new(stream)) + } +} + +fn extract_oci_layer(reader: R, dest: &Path) -> std::io::Result<()> { + use std::os::unix::fs::PermissionsExt; + + // Layers arrive gzip- or zstd-compressed (or, rarely, plain). Decompress + // transparently so a zstd layer no longer breaks extraction. + let reader = decompress_layer_reader(reader)?; + let mut archive = tar::Archive::new(reader); + archive.set_preserve_permissions(true); + archive.set_preserve_mtime(true); + // Preserve the archive's uid/gid. The agent runs as root (CAP_CHOWN) so this + // chowns each entry to the image's intended owner; without it every file is + // owned by root, breaking images that ship non-root-owned paths (e.g. a + // `node`/`postgres` user's home or data dir). The previous busybox `tar` + // preserved ownership by default — the Rust-tar rewrite dropped it. + archive.set_preserve_ownerships(true); + archive.set_overwrite(true); + + for entry in archive.entries()? { + let mut entry = entry?; + let path = entry.path()?.into_owned(); + + // Jail the on-disk path under the layer dir (defends against `..` and + // absolute paths embedded in the archive). + let Some(full_path) = jailed_join(dest, &path) else { + warn!(path = %path.display(), "skipping layer entry that escapes the layer dir"); + continue; + }; + + // Whiteout markers are interpreted by name, before normal extraction. + if let Some(file_name) = path.file_name().and_then(|n| n.to_str()) { + match classify_layer_entry(file_name) { + LayerEntry::OpaqueDir => { + if let Some(parent) = full_path.parent() { + std::fs::create_dir_all(parent)?; + set_overlay_opaque(parent)?; + } + continue; + } + LayerEntry::Whiteout(removed) => { + if let Some(parent) = full_path.parent() { + std::fs::create_dir_all(parent)?; + create_overlay_whiteout(&parent.join(removed))?; + } + continue; + } + LayerEntry::Normal => {} + } + } + + // A hardlink whose target wasn't extracted into this layer can't be + // created here; skip it (overlayfs resolves the lower-layer file). + if entry.header().entry_type() == tar::EntryType::Link { + let target = entry.link_name()?.and_then(|link| jailed_join(dest, &link)); + if target.is_none_or(|t| !t.exists()) { + continue; + } + } + + // Ensure the parent is writable before extracting children — OCI layers + // can set restrictive directory modes before their contents. + if let Some(parent) = full_path.parent() { + if parent.is_dir() { + let _ = std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o755)); + } + } + + if let Err(e) = entry.unpack_in(dest) { + // Regular files and directories failing is a real error; non-regular + // entries (symlinks, device nodes, fifos) can fail benignly — skip. + match entry.header().entry_type() { + tar::EntryType::Regular + | tar::EntryType::GNUSparse + | tar::EntryType::Continuous + | tar::EntryType::Directory => { + return Err(std::io::Error::new( + e.kind(), + format!("failed to unpack '{}': {}", path.display(), e), + )); + } + _ => { + warn!(path = %path.display(), error = %e, "skipping non-regular layer entry"); + } + } + } + } + Ok(()) +} + +/// Pull an OCI image with progress callback and optional authentication. +/// +/// The callback is called for each layer being pulled with (current, total, layer_id). +pub fn pull_image_with_progress_and_auth( + image: &str, + oci_platform: Option<&str>, + auth: Option<&RegistryAuth>, + proxy: Option<&str>, + no_proxy: Option<&str>, + mut progress: F, +) -> Result +where + F: FnMut(usize, usize, &str), +{ + // Validate image reference before any operations + crate::oci::validate_image_reference(image).map_err(|e| { + StorageError::InvalidImageReference { + reference: image.to_string(), + reason: e, + } + })?; + + // Canonicalize so all equivalent refs share the same on-disk cache key. + let image = normalize_image_ref(image); + let image = image.as_str(); + + // If packed layers are available, return synthetic image info + if let Some(packed_dir) = get_packed_layers_dir() { + info!(image = %image, "using packed layers, skipping network pull"); + // A local image archive is flattened into a rootfs first; an ordinary + // packed-layers dir is used as-is. + if let Some(flattened) = ensure_archive_flattened(packed_dir)? { + return create_packed_image_info(image, &flattened); + } + return create_packed_image_info(image, packed_dir); + } + + // Determine OCI platform - default to current architecture + // This must happen BEFORE the cache check so we can verify architecture + let oci_platform = oci_platform.or({ + #[cfg(target_arch = "aarch64")] + { + Some("linux/arm64") + } + #[cfg(target_arch = "x86_64")] + { + Some("linux/amd64") + } + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] + { + None + } + }); + + // Check if already cached with correct architecture + if let Ok(Some(info)) = query_image(image) { + // Verify cached image architecture matches requested OCI platform + let cached_arch = &info.architecture; + let requested_arch = oci_platform + .map(oci_platform_to_arch) + .unwrap_or_else(|| cached_arch.clone()); + + if cached_arch == &requested_arch { + debug!( + image = %image, + architecture = %cached_arch, + "image already cached with correct architecture, skipping pull" + ); + return Ok(info); + } else { + // Architecture mismatch - need to re-pull + info!( + image = %image, + cached_arch = %cached_arch, + requested_arch = %requested_arch, + "cached image has wrong architecture, will re-pull" + ); + // Clean up the mismatched cached manifest + let root = Path::new(STORAGE_ROOT); + let manifest_path = root + .join(MANIFESTS_DIR) + .join(sanitize_image_name(image) + ".json"); + let _ = std::fs::remove_file(&manifest_path); + } + } + + let root = Path::new(STORAGE_ROOT); + + // Get manifest with OCI platform specified + progress(0, 0, "fetching manifest"); + info!(image = %image, oci_platform = ?oci_platform, "fetching manifest"); + let manifest = crane_manifest(image, oci_platform, auth, proxy, no_proxy)?; + + // Parse manifest to get config and layers + let manifest_json: serde_json::Value = + serde_json::from_str(&manifest).map_err(|e| StorageError::parse_error("manifest", e))?; + + // Handle manifest list (multi-arch) + let config_digest = if manifest_json.get("config").is_some() { + manifest_json["config"]["digest"] + .as_str() + .ok_or_else(|| StorageError::MissingField { + context: "manifest".into(), + field: "config digest".into(), + })? + } else if manifest_json.get("manifests").is_some() { + return Err(StorageError::new(format!( + "got manifest list instead of image manifest - platform may not be available. \ + manifests: {:?}", + manifest_json["manifests"].as_array().map(|arr| arr + .iter() + .filter_map(|m| m["platform"]["architecture"].as_str()) + .collect::>()) + ))); + } else { + return Err(StorageError::UnsupportedManifest { + media_type: "unknown".into(), + }); + }; + + let layers: Vec = manifest_json["layers"] + .as_array() + .ok_or_else(|| StorageError::MissingField { + context: "manifest".into(), + field: "layers".into(), + })? + .iter() + .filter_map(|l| l["digest"].as_str().map(String::from)) + .collect(); + + let total_layers = layers.len(); + + // Save manifest + let manifest_path = root + .join(MANIFESTS_DIR) + .join(sanitize_image_name(image) + ".json"); + std::fs::write(&manifest_path, &manifest)?; + + // Fetch and save config + let config = crane_config(image, oci_platform, auth, proxy, no_proxy)?; + let config_id = config_digest + .strip_prefix("sha256:") + .unwrap_or(config_digest); + let config_path = root.join(CONFIGS_DIR).join(format!("{}.json", config_id)); + std::fs::write(&config_path, &config)?; + + // Parse config for metadata + let config_json: serde_json::Value = + serde_json::from_str(&config).map_err(|e| StorageError::parse_error("config", e))?; + + // Extract layers with progress updates + let mut total_size = 0u64; + for (i, layer_digest) in layers.iter().enumerate() { + let layer_id = layer_digest.strip_prefix("sha256:").unwrap_or(layer_digest); + let layer_dir = root.join(LAYERS_DIR).join(layer_id); + + if is_layer_cached(&layer_dir) { + info!(layer = %layer_id, "layer already cached"); + // Report progress after confirming cache hit + progress(i + 1, total_layers, layer_id); + continue; + } + + // Clean up empty/incomplete layer directory if it exists + if layer_dir.exists() { + warn!(layer = %layer_id, "removing empty/incomplete layer directory"); + if let Err(e) = std::fs::remove_dir_all(&layer_dir) { + warn!(layer = %layer_id, error = %e, "failed to remove incomplete layer directory"); + } + } + + info!( + layer = %layer_id, + progress = format!("{}/{}", i + 1, total_layers), + "extracting layer" + ); + + std::fs::create_dir_all(&layer_dir)?; + + // Stream layer directly to tar extraction using direct process piping + // (no shell to avoid injection risks) + + // Set up auth if provided (temp_dir must stay alive until command completes) + let temp_dir = setup_docker_auth(image, auth)?; + + // Build crane command + let mut crane_cmd = Command::new("crane"); + crane_cmd.arg("blob"); + crane_cmd.arg(format!("{}@{}", image_repo(image), layer_digest)); + if let Some(p) = oci_platform { + crane_cmd.arg("--platform").arg(p); + } + crane_cmd.stdout(Stdio::piped()); + // Capture crane stderr to a file (not a pipe — a file can't deadlock on a + // full buffer) so the real fetch failure (DNS, TLS, 4xx, redirect) is + // surfaced instead of a bare "crane blob failed". + let crane_stderr_path = layer_dir.join(".crane-stderr"); + match std::fs::File::create(&crane_stderr_path) { + Ok(f) => { + crane_cmd.stderr(Stdio::from(f)); + } + Err(_) => { + crane_cmd.stderr(Stdio::null()); + } + } + + if let Some(ref td) = temp_dir { + crane_cmd.env("DOCKER_CONFIG", td.path()); + } + + apply_proxy_env(&mut crane_cmd, proxy, no_proxy); + + // Spawn crane process + let mut crane = crane_cmd + .spawn() + .map_err(|e| StorageError::new(format!("failed to spawn crane: {}", e)))?; + + // Extract straight from crane's stdout. `extract_oci_layer` transparently + // decompresses gzip- OR zstd-compressed layers in-process (the guest + // ships no zstd tool, and the old external `gunzip` pipe silently failed + // on every zstd layer). Reading the stream to EOF also drives the crane + // fetch to completion. + let crane_stdout = crane + .stdout + .take() + .ok_or_else(|| StorageError::new("failed to capture crane stdout".to_string()))?; + + let extract_result = extract_oci_layer(crane_stdout, &layer_dir); + + let crane_status = crane + .wait() + .map_err(|e| StorageError::new(format!("failed to wait for crane: {}", e)))?; + + let crane_stderr = std::fs::read_to_string(&crane_stderr_path).unwrap_or_default(); + let _ = std::fs::remove_file(&crane_stderr_path); + let crane_stderr = crane_stderr.trim(); + + // Order matters. A genuine crane fetch failure (network/auth) prints a + // real message to its stderr, so surface that first. Otherwise, if + // extraction failed, THAT is the real cause — a crane that exited + // non-zero with empty stderr is just the SIGPIPE from us closing the pipe + // when extraction stopped reading (the exact trap that made every zstd + // layer look like "crane blob failed" when the real problem was that the + // old pipeline couldn't decompress it). + let layer_failure = if !crane_status.success() && !crane_stderr.is_empty() { + Some(format!( + "crane blob failed for layer {}: {}", + layer_digest, crane_stderr + )) + } else if let Err(e) = extract_result { + Some(format!( + "layer extraction failed for layer {}: {}", + layer_digest, e + )) + } else if !crane_status.success() { + Some(format!("crane blob failed for layer {}", layer_digest)) + } else { + None + }; + + if let Some(message) = layer_failure { + if let Err(e) = std::fs::remove_dir_all(&layer_dir) { + warn!(layer = %layer_id, error = %e, "failed to clean up layer directory after extraction failure"); + } + return Err(StorageError::new(message)); + } + + if let Ok(size) = dir_size(&layer_dir) { + total_size += size; + } + + // Report progress after successful extraction + progress(i + 1, total_layers, layer_id); + } + + // Signal that layers are done and we're syncing — this can take a while + // for large images (gigabytes flushed through virtio-blk). + progress(total_layers, total_layers, "syncing"); + + // Sync filesystem to ensure all layer data is persisted to the ext4 journal. + // Defense in depth: even though shutdown waits for acknowledgment (which also + // syncs), we sync here because: + // 1. Commands may complete and VM may exit before shutdown is called + // 2. Protects against ungraceful termination (SIGKILL, host crash) + // 3. Empty layer directories cause "executable not found" errors that are + // hard to diagnose - better to be safe than sorry + // SAFETY: sync() is always safe to call + unsafe { + libc::sync(); + } + + // Build ImageInfo + let architecture = config_json["architecture"] + .as_str() + .unwrap_or("unknown") + .to_string(); + let os = config_json["os"].as_str().unwrap_or("linux").to_string(); + let created = config_json["created"].as_str().map(String::from); + + // Extract OCI config fields (Entrypoint, Cmd, Env, WorkingDir, User) + let oci_config = &config_json["config"]; + let entrypoint = json_string_array(oci_config, "Entrypoint"); + let cmd = json_string_array(oci_config, "Cmd"); + let env = json_string_array(oci_config, "Env"); + let workdir = oci_config["WorkingDir"] + .as_str() + .filter(|s| !s.is_empty()) + .map(String::from); + let user = oci_config["User"] + .as_str() + .filter(|s| !s.is_empty()) + .map(String::from); + + Ok(ImageInfo { + reference: image.to_string(), + digest: config_digest.to_string(), + size: total_size, + created, + architecture, + os, + layer_count: layers.len(), + layers, + entrypoint, + cmd, + env, + workdir, + user, + }) +} + +/// Query if an image exists locally. +pub fn query_image(image: &str) -> Result> { + let image = normalize_image_ref(image); + let image = image.as_str(); + + // Packed layers (a `.smolmachine` or a staged local image archive/dir): + // synthesize image info without a registry manifest, mirroring the pull + // path. A local image archive is flattened into a rootfs first. + if let Some(packed_dir) = get_packed_layers_dir() { + let flattened = ensure_archive_flattened(packed_dir)?; + let effective = flattened.as_deref().unwrap_or(packed_dir); + return Ok(Some(create_packed_image_info(image, effective)?)); + } + + let root = Path::new(STORAGE_ROOT); + let manifest_path = root + .join(MANIFESTS_DIR) + .join(sanitize_image_name(image) + ".json"); + + if !manifest_path.exists() { + return Ok(None); + } + + // Read and parse manifest + let manifest = std::fs::read_to_string(&manifest_path)?; + let manifest_json: serde_json::Value = + serde_json::from_str(&manifest).map_err(|e| StorageError::parse_error("manifest", e))?; + + let config_digest = + manifest_json["config"]["digest"] + .as_str() + .ok_or_else(|| StorageError::MissingField { + context: "manifest".into(), + field: "config digest".into(), + })?; + + let layers: Vec = manifest_json["layers"] + .as_array() + .ok_or_else(|| StorageError::MissingField { + context: "manifest".into(), + field: "layers".into(), + })? + .iter() + .filter_map(|l| l["digest"].as_str().map(String::from)) + .collect(); + + // Read config + let config_id = config_digest + .strip_prefix("sha256:") + .unwrap_or(config_digest); + let config_path = root.join(CONFIGS_DIR).join(format!("{}.json", config_id)); + let config = std::fs::read_to_string(&config_path)?; + let config_json: serde_json::Value = + serde_json::from_str(&config).map_err(|e| StorageError::parse_error("config", e))?; + + let architecture = config_json["architecture"] + .as_str() + .unwrap_or("unknown") + .to_string(); + let os = config_json["os"].as_str().unwrap_or("linux").to_string(); + let created = config_json["created"].as_str().map(String::from); + + // Verify all layers exist and calculate total size + let mut total_size = 0u64; + for layer_digest in &layers { + let layer_id = layer_digest.strip_prefix("sha256:").unwrap_or(layer_digest); + let layer_dir = root.join(LAYERS_DIR).join(layer_id); + if !layer_dir.exists() { + // Layer missing - image is incomplete, needs re-pull + // Clean up corrupt manifest to avoid repeated failures + warn!(layer = %layer_id, image = %image, "cached image has missing layer, cleaning up and will re-pull"); + let _ = std::fs::remove_file(&manifest_path); + return Ok(None); + } + if let Ok(size) = dir_size(&layer_dir) { + total_size += size; + } + } + + // Extract OCI config fields + let oci_config = &config_json["config"]; + let entrypoint = json_string_array(oci_config, "Entrypoint"); + let cmd = json_string_array(oci_config, "Cmd"); + let env = json_string_array(oci_config, "Env"); + let workdir = oci_config["WorkingDir"] + .as_str() + .filter(|s| !s.is_empty()) + .map(String::from); + let user = oci_config["User"] + .as_str() + .filter(|s| !s.is_empty()) + .map(String::from); + + Ok(Some(ImageInfo { + reference: image.to_string(), + digest: config_digest.to_string(), + size: total_size, + created, + architecture, + os, + layer_count: layers.len(), + layers, + entrypoint, + cmd, + env, + workdir, + user, + })) +} + +/// List all cached images. +pub fn list_images() -> Result> { + let root = Path::new(STORAGE_ROOT); + let manifests_dir = root.join(MANIFESTS_DIR); + + if !manifests_dir.exists() { + return Ok(Vec::new()); + } + + let mut images = Vec::new(); + + for entry in std::fs::read_dir(&manifests_dir)? { + let entry: std::fs::DirEntry = entry?; + let path = entry.path(); + + if path.extension().map(|e| e == "json").unwrap_or(false) { + // Extract image name from filename + let name = path + .file_stem() + .and_then(|s| s.to_str()) + .map(unsanitize_image_name) + .unwrap_or_default(); + + if let Ok(Some(info)) = query_image(&name) { + images.push(info); + } + } + } + + Ok(images) +} + +/// Export a layer as a tar archive to a file. +/// +/// Used by `smolvm pack` to extract layers for packaging. +/// Returns the path to the created tar file. +/// Find the directory path for a specific layer of an image. +/// +/// Scans manifests to find the image by digest, then resolves the layer +/// directory. Used by the streaming export handler to pipe tar directly +/// without creating a temp file. +pub fn find_layer_path(image_digest: &str, layer_index: usize) -> Result { + let root = Path::new(STORAGE_ROOT); + + let manifests_dir = root.join(MANIFESTS_DIR); + if !manifests_dir.exists() { + return Err(StorageError::NoImagesFound); + } + + let mut layers: Option> = None; + + for entry in std::fs::read_dir(&manifests_dir)? { + let entry = entry?; + let content = std::fs::read_to_string(entry.path())?; + if let Ok(manifest) = serde_json::from_str::(&content) { + if let Some(config) = manifest.get("config") { + if let Some(digest) = config.get("digest").and_then(|d| d.as_str()) { + if digest == image_digest { + layers = manifest["layers"].as_array().map(|arr| { + arr.iter() + .filter_map(|l| l["digest"].as_str().map(String::from)) + .collect() + }); + break; + } + } + } + } + } + + let layers = layers.ok_or_else(|| { + StorageError::new(format!("image with digest {} not found", image_digest)) + })?; + + if layer_index >= layers.len() { + return Err(StorageError::new(format!( + "layer index {} out of bounds (image has {} layers)", + layer_index, + layers.len() + ))); + } + + let layer_digest = &layers[layer_index]; + let layer_id = layer_digest.strip_prefix("sha256:").unwrap_or(layer_digest); + let layer_dir = root.join(LAYERS_DIR).join(layer_id); + + if !layer_dir.exists() { + return Err(StorageError::new(format!( + "layer directory not found: {}", + layer_dir.display() + ))); + } + + Ok(layer_dir) +} + +/// Remove all image manifests and configs, making all layers unreferenced. +/// +/// Call this before `garbage_collect()` to implement `prune --all`. +pub fn purge_all_images() -> Result<()> { + let root = Path::new(STORAGE_ROOT); + let manifests_dir = root.join(MANIFESTS_DIR); + let configs_dir = root.join(CONFIGS_DIR); + + if manifests_dir.exists() { + std::fs::remove_dir_all(&manifests_dir)?; + } + if configs_dir.exists() { + std::fs::remove_dir_all(&configs_dir)?; + } + + Ok(()) +} + +/// Run garbage collection. +pub fn garbage_collect(dry_run: bool) -> Result { + let root = Path::new(STORAGE_ROOT); + let layers_dir = root.join(LAYERS_DIR); + let manifests_dir = root.join(MANIFESTS_DIR); + + // Collect all referenced layers + let mut referenced_layers = std::collections::HashSet::new(); + + if manifests_dir.exists() { + for entry in std::fs::read_dir(&manifests_dir)? { + let entry = entry?; + let content = std::fs::read_to_string(entry.path())?; + if let Ok(manifest) = serde_json::from_str::(&content) { + if let Some(layers) = manifest["layers"].as_array() { + for layer in layers { + if let Some(digest) = layer["digest"].as_str() { + let id = digest.strip_prefix("sha256:").unwrap_or(digest); + referenced_layers.insert(id.to_string()); + } + } + } + } + } + } + + // Find unreferenced layers + let mut freed = 0u64; + + if layers_dir.exists() { + for entry in std::fs::read_dir(&layers_dir)? { + let entry = entry?; + let layer_id = entry.file_name().to_string_lossy().to_string(); + + if !referenced_layers.contains(&layer_id) { + let size = dir_size(&entry.path()).unwrap_or(0); + info!(layer = %layer_id, size = size, dry_run = dry_run, "unreferenced layer"); + + if !dry_run { + std::fs::remove_dir_all(entry.path())?; + } + + freed += size; + } + } + } + + Ok(freed) +} + +// ============================================================================ +// Overlay Setup Helper +// ============================================================================ + +/// Helper for setting up overlay filesystems. +/// +/// Encapsulates the common logic for preparing overlay directories, +/// mounting layers, and creating OCI bundles. +struct OverlaySetup { + overlay_root: PathBuf, + upper_path: PathBuf, + work_path: PathBuf, + merged_path: PathBuf, + workload_id: String, +} + +impl OverlaySetup { + /// Create a new overlay setup for the given workload. + fn new(workload_id: &str) -> Result { + let overlay_root = overlay_root_for_workload(workload_id)?; + Ok(Self { + upper_path: overlay_root.join("upper"), + work_path: overlay_root.join("work"), + merged_path: overlay_root.join("merged"), + overlay_root, + workload_id: workload_id.to_string(), + }) + } + + /// Prepare overlay directories, cleaning up any previous state. + fn prepare_directories(&self) -> Result<()> { + // Clean up any previous overlay state - workdir must be empty for overlay mount + if self.overlay_root.exists() { + // Try to unmount if previously mounted + if let Err(e) = Command::new("umount").arg(&self.merged_path).output() { + debug!(path = %self.merged_path.display(), error = %e, "failed to unmount previous overlay (may not have been mounted)"); + } + // Remove old directories to ensure clean state + if let Err(e) = std::fs::remove_dir_all(&self.overlay_root) { + warn!(path = %self.overlay_root.display(), error = %e, "failed to remove old overlay directory"); + } + } + + std::fs::create_dir_all(&self.upper_path)?; + std::fs::create_dir_all(&self.work_path)?; + std::fs::create_dir_all(&self.merged_path)?; + + Ok(()) + } + + /// Set up the upper layer with DNS resolution and /dev directory. + fn setup_upper_layer(&self) -> Result<()> { + // Set up DNS resolution BEFORE mounting. Image-backed workloads read + // `/etc/resolv.conf` from the overlay upper layer, so this file must + // match the active networking mode rather than always hardcoding + // public resolvers. + let upper_etc = self.upper_path.join("etc"); + std::fs::create_dir_all(&upper_etc)?; + let resolv_path = upper_etc.join("resolv.conf"); + let resolv_contents = overlay_resolv_conf_contents(); + if let Err(e) = std::fs::write(&resolv_path, resolv_contents) { + warn!(error = %e, "failed to write resolv.conf to upper layer"); + } + + // Create /dev directory in upper layer - we'll bind mount the real /dev later + let upper_dev = self.upper_path.join("dev"); + std::fs::create_dir_all(&upper_dev)?; + + Ok(()) + } + + /// Verify that all layer paths exist and log warnings for empty layers. + fn verify_layers(&self, lowerdirs: &[String]) -> Result<()> { + for layer_path in lowerdirs { + let path = Path::new(layer_path); + if !path.exists() { + return Err(StorageError::new(format!( + "layer path does not exist: {}", + layer_path + ))); + } + // Check if layer has contents + let entry_count = std::fs::read_dir(path) + .map(|entries| entries.count()) + .unwrap_or(0); + if entry_count == 0 { + warn!(layer = %layer_path, "layer directory is empty"); + } + } + Ok(()) + } + + /// Mount the overlay filesystem with fallback from multi-lowerdir to sequential. + fn mount(&self, lowerdirs: &[String]) -> Result<()> { + // Try multi-lowerdir mount first (efficient) + let mount_result = try_mount_overlay_multi_lower( + lowerdirs, + &self.upper_path, + &self.work_path, + &self.merged_path, + ); + + if let Err(multi_err) = mount_result { + if lowerdirs.len() > 1 { + // Multi-lowerdir failed, try sequential approach + warn!( + layer_count = lowerdirs.len(), + error = %multi_err, + "multi-lowerdir mount failed, trying sequential overlay construction" + ); + + mount_overlay_sequential( + lowerdirs, + &self.upper_path, + &self.work_path, + &self.merged_path, + &self.overlay_root, + )?; + } else { + // Single layer, can't use sequential approach + return Err(multi_err); + } + } + + Ok(()) + } + + /// Verify that the mount succeeded by checking merged directory contents. + fn verify_mount(&self) -> usize { + let entry_count = std::fs::read_dir(&self.merged_path) + .map(|entries| entries.count()) + .unwrap_or(0); + + if entry_count == 0 { + warn!( + workload_id = %self.workload_id, + merged_path = %self.merged_path.display(), + "overlay mount returned success but merged directory is empty" + ); + // Try to get more info about the mount state + if let Ok(mounts) = std::fs::read_to_string("/proc/mounts") { + let merged_str = self.merged_path.to_string_lossy(); + let is_mounted = mounts.lines().any(|line| line.contains(&*merged_str)); + warn!(is_mounted = is_mounted, "mount point status"); + } + } + + entry_count + } + + /// Create OCI bundle directory structure. + fn create_bundle(&self) -> Result<()> { + let bundle_path = self.overlay_root.join("bundle"); + std::fs::create_dir_all(&bundle_path)?; + + // Create symlink: bundle/rootfs -> ../merged + let rootfs_link = bundle_path.join("rootfs"); + if !rootfs_link.exists() { + std::os::unix::fs::symlink("../merged", &rootfs_link).map_err(|e| { + StorageError::new(format!("failed to create rootfs symlink: {}", e)) + })?; + } + + debug!(bundle = %bundle_path.display(), "OCI bundle directory created"); + Ok(()) + } + + /// Convert to OverlayInfo result. + fn into_overlay_info(self) -> OverlayInfo { + OverlayInfo { + rootfs_path: self.merged_path.display().to_string(), + upper_path: self.upper_path.display().to_string(), + work_path: self.work_path.display().to_string(), + } + } + + /// Execute the full overlay setup pipeline with the given lower directories. + fn execute(self, lowerdirs: Vec) -> Result { + self.prepare_directories()?; + self.setup_upper_layer()?; + self.verify_layers(&lowerdirs)?; + self.mount(&lowerdirs)?; + + let entry_count = self.verify_mount(); + info!(workload_id = %self.workload_id, entry_count = entry_count, "overlay mounted"); + + self.create_bundle()?; + Ok(self.into_overlay_info()) + } + + /// Reuse an existing persistent overlay or create a new one. + /// + /// If the overlay is already mounted, returns it immediately. + /// If the overlay directory exists but is not mounted (e.g. after VM restart), + /// remounts it preserving the upper layer (which contains previous changes). + /// If the overlay does not exist at all, creates it fresh. + fn execute_or_remount(self, lowerdirs: Vec) -> Result { + // Already mounted — just reuse it + if self.merged_path.exists() && is_mountpoint(&self.merged_path) { + info!(workload_id = %self.workload_id, "reusing existing persistent overlay"); + self.create_bundle()?; + return Ok(self.into_overlay_info()); + } + + // Upper layer exists from a previous session — remount preserving it + if self.upper_path.exists() { + info!(workload_id = %self.workload_id, "remounting persistent overlay with existing upper layer"); + + // overlayfs requires an empty work directory at mount time + if self.work_path.exists() { + let _ = std::fs::remove_dir_all(&self.work_path); + } + std::fs::create_dir_all(&self.work_path)?; + std::fs::create_dir_all(&self.merged_path)?; + + self.verify_layers(&lowerdirs)?; + self.mount(&lowerdirs)?; + + let entry_count = self.verify_mount(); + info!(workload_id = %self.workload_id, entry_count = entry_count, "persistent overlay remounted"); + + self.create_bundle()?; + return Ok(self.into_overlay_info()); + } + + // First time — full setup + info!(workload_id = %self.workload_id, "creating new persistent overlay"); + self.execute(lowerdirs) + } +} + +fn overlay_resolv_conf_contents() -> String { + if std::env::var(guest_env::DNS_FILTER).as_deref() == Ok("1") { + return "nameserver 127.0.0.1\n".to_string(); + } + + // A nameserver supplied by the host (SMOLVM_NETWORK_DNS) wins for either + // backend: under virtio-net it's the gateway address, and under TSI it's a + // custom resolver (--dns) the guest queries directly. Only fall back to + // public resolvers when the host didn't specify one. + if let Ok(dns_server) = std::env::var(guest_env::DNS) { + if !dns_server.is_empty() { + return format!("nameserver {}\n", dns_server); + } + } + + "nameserver 8.8.8.8\nnameserver 1.1.1.1\n".to_string() +} + +/// Prepare an overlay filesystem for a workload. +/// +/// Reuses an existing overlay if already mounted, remounts if the upper +/// directory exists (preserving state from previous sessions), or creates +/// a fresh overlay. This idempotent behavior is critical for `machine cp` +/// which may call this before or after `machine exec`. +pub fn prepare_overlay(image: &str, workload_id: &str) -> Result { + // Check if we have packed layers available + if let Some(packed_dir) = get_packed_layers_dir() { + info!(image = %image, packed_dir = %packed_dir.display(), "using packed layers"); + // A local image archive is flattened into a rootfs (a single packed + // layer) first; an ordinary packed-layers dir is used as-is. + let flattened = ensure_archive_flattened(packed_dir)?; + let effective = flattened.as_deref().unwrap_or(packed_dir); + return prepare_overlay_from_packed(image, workload_id, effective); + } + + // Ensure image exists + let info = query_image(image)? + .ok_or_else(|| StorageError::new(format!("image not found: {}", image)))?; + + // Build lowerdir from layers (reversed for overlay order - top layer first) + let root = Path::new(STORAGE_ROOT); + let lowerdirs: Vec = info + .layers + .iter() + .rev() + .map(|digest| { + let id = digest.strip_prefix("sha256:").unwrap_or(digest); + root.join(LAYERS_DIR).join(id).display().to_string() + }) + .collect(); + + OverlaySetup::new(workload_id)?.execute_or_remount(lowerdirs) +} + +/// Prepare an overlay filesystem using pre-packed layers. +/// +/// Packed layers are stored as directories named by short digest (first 12 chars) +/// in the packed_dir. This function builds the overlay using these layers. +fn prepare_overlay_from_packed( + image: &str, + workload_id: &str, + packed_dir: &Path, +) -> Result { + // An unpacked-image directory IS the rootfs — one lowerdir, not its subdirs + // treated as separate layers. + if is_rootfs_dir(packed_dir) { + return OverlaySetup::new(workload_id)? + .execute_or_remount(vec![packed_dir.display().to_string()]); + } + + // Packed layers are named by short digest (first 12 chars of sha256). + // Order is taken from the layer-order index (manifest order, bottom→top), + // falling back to a name sort — see `ordered_packed_layer_names`. + let layer_names = ordered_packed_layer_names(packed_dir)?; + + if layer_names.is_empty() { + return Err(StorageError::new(format!( + "no layer directories found in {}", + packed_dir.display() + ))); + } + + info!( + image = %image, + layer_count = layer_names.len(), + layers = ?layer_names, + "found packed layers" + ); + + // Build lowerdir from layers (reversed so the top-most layer is leftmost, + // as overlayfs gives leftmost lowerdir the highest priority). + let lowerdirs: Vec = layer_names + .iter() + .rev() + .map(|name| packed_dir.join(name).display().to_string()) + .collect(); + + // Use shared overlay setup logic — execute_or_remount preserves the + // upper layer from a previous session (e.g., packages installed via exec) + // instead of recreating the overlay from scratch. + OverlaySetup::new(workload_id)?.execute_or_remount(lowerdirs) +} + +/// Build lowerdir list from a pulled OCI image's layers. +fn get_image_lowerdirs(image: &str) -> Result> { + let info = query_image(image)? + .ok_or_else(|| StorageError::new(format!("image not found: {}", image)))?; + + let root = Path::new(STORAGE_ROOT); + Ok(info + .layers + .iter() + .rev() + .map(|digest| { + let id = digest.strip_prefix("sha256:").unwrap_or(digest); + root.join(LAYERS_DIR).join(id).display().to_string() + }) + .collect()) +} + +/// Whether `dir` is itself a root filesystem (an unpacked-image directory, +/// `--image ./rootfs/`) rather than a set of layer subdirs — detected by the +/// presence of standard top-level rootfs directories. A `.smolmachine`'s +/// packed-layers dir holds per-layer subdirs, not these, so it reads as false. +fn is_rootfs_dir(dir: &Path) -> bool { + ["bin", "usr", "etc", "sbin"] + .iter() + .any(|d| dir.join(d).is_dir()) +} + +/// Build lowerdir list from pre-packed layer directories. +fn get_packed_lowerdirs(packed_dir: &Path) -> Result> { + // An unpacked-image directory IS the rootfs — one lowerdir, not its subdirs + // treated as separate layers. + if is_rootfs_dir(packed_dir) { + return Ok(vec![packed_dir.display().to_string()]); + } + + // Order from the layer-order index (manifest order, bottom→top), falling + // back to a name sort — see `ordered_packed_layer_names`. + let layer_names = ordered_packed_layer_names(packed_dir)?; + + if layer_names.is_empty() { + return Err(StorageError::new(format!( + "no layer directories found in {}", + packed_dir.display() + ))); + } + + // Reversed so the top-most layer is leftmost (overlayfs priority order). + Ok(layer_names + .iter() + .rev() + .map(|name| packed_dir.join(name).display().to_string()) + .collect()) +} + +/// Clean up an overlay filesystem. +/// Log the error inside this function to skip the repetitive Err matching when unnecessary. +pub fn cleanup_overlay(workload_id: &str) -> Result<()> { + let overlay_root = overlay_root_for_workload(workload_id)?; + let merged_path = overlay_root.join("merged"); + + // Unmount nested bind mounts inside the overlay rootfs first. Volume mounts + // like /workspace are bind-mounted under merged/, and they keep the overlay + // rootfs busy if we try to unmount merged directly. + if let Ok(mounts) = std::fs::read_to_string("/proc/mounts") { + let merged_prefix = format!("{}/", merged_path.display()); + let mut nested_mounts: Vec = mounts + .lines() + .filter_map(|line| { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 2 { + return None; + } + + let mount_point = PathBuf::from(parts[1]); + let mount_point_str = mount_point.to_string_lossy(); + if mount_point_str.starts_with(&merged_prefix) { + Some(mount_point) + } else { + None + } + }) + .collect(); + + nested_mounts.sort_by_key(|path| std::cmp::Reverse(path.components().count())); + + for mount_point in nested_mounts { + if let Err(e) = Command::new("umount").arg(&mount_point).status() { + debug!( + workload_id = %workload_id, + path = %mount_point.display(), + error = %e, + "failed to unmount nested overlay mount" + ); + } + } + } + + // Unmount main merged path if mounted + if merged_path.exists() { + if let Err(e) = Command::new("umount").arg(&merged_path).status() { + debug!( + workload_id = %workload_id, + path = %merged_path.display(), + error = %e, + "failed to unmount overlay (may not have been mounted)" + ); + } + } + + // Remove overlay directories (includes merged_layers, upper, work, etc.) + if overlay_root.exists() { + if let Err(cleanup_err) = std::fs::remove_dir_all(&overlay_root) { + warn!( + workload_id = %workload_id, + error = %cleanup_err, + "failed to clean up overlay." + ); + return Err(cleanup_err.into()); + } + } + + info!(workload_id = %workload_id, "overlay cleaned up"); + Ok(()) +} + +/// Result of running a command. +/// +/// Uses `Vec` so binary output is preserved end-to-end. +pub struct RunResult { + pub exit_code: i32, + pub stdout: Vec, + pub stderr: Vec, +} + +/// Prepared rootfs info for a single ephemeral run. +pub struct PreparedOverlayRootfs { + pub workload_id: String, + pub rootfs_path: String, +} + +fn prepare_rootfs_for_ephemeral_run(image: &str) -> Result { + let workload_id = format!( + "run-{}-{}", + sanitize_image_name(image), + generate_container_id() + ); + let overlay = prepare_overlay(image, &workload_id)?; + debug!( + workload_id = %workload_id, + rootfs = %overlay.rootfs_path, + "prepared ephemeral overlay for command execution" + ); + Ok(PreparedOverlayRootfs { + workload_id, + rootfs_path: overlay.rootfs_path, + }) +} + +/// Run a command in an image's overlay rootfs using crun OCI runtime. +/// +/// When `persistent_overlay_id` is `Some`, the overlay persists across runs +/// (filesystem changes accumulate). When `None`, an ephemeral overlay is +/// created and destroyed after the run. +pub fn run_command( + image: &str, + command: &[String], + env: &[(String, String)], + workdir: Option<&str>, + user: Option<&str>, + mounts: &[(String, String, bool)], + timeout_ms: Option, + persistent_overlay_id: Option<&str>, + stdin_data: Option<&str>, + client_fd: Option, + unprivileged: bool, +) -> Result { + // Validate inputs + crate::oci::validate_image_reference(image).map_err(StorageError::new)?; + crate::oci::validate_env_vars(env).map_err(StorageError::new)?; + + let prepared = match persistent_overlay_id { + Some(id) => prepare_for_run_persistent(image, id)?, + None => prepare_rootfs_for_ephemeral_run(image)?, + }; + debug!(rootfs = %prepared.rootfs_path, persistent = persistent_overlay_id.is_some(), "using overlay for command execution"); + + // Gather all steps to run a command in a single anon function + let result = (|| { + // Setup volume mounts (mount virtiofs to staging area) + let mounted_paths = setup_volume_mounts(&prepared.rootfs_path, mounts)?; + + // Get bundle path + let overlay_root = Path::new(STORAGE_ROOT) + .join(OVERLAYS_DIR) + .join(&prepared.workload_id); + let bundle_path = overlay_root.join("bundle"); + + // Create OCI spec + let workdir_str = workdir.unwrap_or("/"); + let identity = crate::oci::resolve_process_identity(Path::new(&prepared.rootfs_path), user) + .map_err(StorageError::new)?; + let mut spec = OciSpec::new(command, env, workdir_str, false, &identity, unprivileged); + spec.add_gpu_devices_if_available(); + + // Add virtiofs bind mounts to OCI spec + for (tag, container_path, read_only) in mounts { + let virtiofs_mount = Path::new(paths::VIRTIOFS_MOUNT_ROOT).join(tag); + spec.add_bind_mount( + &virtiofs_mount.to_string_lossy(), + container_path, + *read_only, + ); + } + + add_workspace_fallback(&mut spec, mounts); + add_storage_fallback(&mut spec, mounts, unprivileged); + + // Forward SSH agent into the container if enabled at boot. + crate::ssh_agent::inject_into_container(&mut spec); + crate::cuda::inject_into_container(&mut spec, Path::new(&prepared.rootfs_path)); + + // Write config.json to bundle + spec.write_to(&bundle_path) + .map_err(|e| StorageError::new(format!("failed to write OCI spec: {}", e)))?; + + // If a main workload container is running for this overlay, join it + // via crun exec instead of creating a fresh isolated container. + if let Some(cid) = crate::resolve_main_container(persistent_overlay_id) { + let result = run_exec_in_container(&cid, command, env, workdir, timeout_ms, client_fd); + let _ = mounted_paths; + return result; + } + + // Generate unique container ID for this execution + let container_id = generate_container_id(); + + // Run with crun + let result = run_with_crun( + &bundle_path, + &container_id, + timeout_ms, + stdin_data, + client_fd, + ); + + // Note: virtiofs mounts are left in place for reuse + // They will be cleaned up when the overlay is cleaned up or the VM shuts down + let _ = mounted_paths; // Suppress unused warning + + result + })(); + + // Only clean up ephemeral overlays; persistent ones survive across runs + if persistent_overlay_id.is_none() { + let _ = cleanup_overlay(&prepared.workload_id); + } + result +} + +/// Spawn a command in an image's overlay rootfs and return the crun PID. +/// +/// Unlike `run_command`, this does not wait for the container to exit. The +/// container runs detached under crun with stdout/stderr redirected to +/// /dev/null; the returned PID is the crun process, which stays alive as +/// long as the container init runs. +/// +/// Requires a persistent overlay ID — ephemeral overlays would leak their +/// upper/work/merged directories because nothing is waiting to clean them +/// up after the container exits. +pub fn spawn_in_overlay( + image: &str, + command: &[String], + env: &[(String, String)], + workdir: Option<&str>, + user: Option<&str>, + mounts: &[(String, String, bool)], + persistent_overlay_id: &str, + unprivileged: bool, +) -> Result { + crate::oci::validate_image_reference(image).map_err(StorageError::new)?; + crate::oci::validate_env_vars(env).map_err(StorageError::new)?; + + let prepared = prepare_for_run_persistent(image, persistent_overlay_id)?; + debug!(rootfs = %prepared.rootfs_path, "using persistent overlay for background command"); + + let mounted_paths = setup_volume_mounts(&prepared.rootfs_path, mounts)?; + + let overlay_root = Path::new(STORAGE_ROOT) + .join(OVERLAYS_DIR) + .join(&prepared.workload_id); + let bundle_path = overlay_root.join("bundle"); + + let workdir_str = workdir.unwrap_or("/"); + let identity = crate::oci::resolve_process_identity(Path::new(&prepared.rootfs_path), user) + .map_err(StorageError::new)?; + let mut spec = OciSpec::new(command, env, workdir_str, false, &identity, unprivileged); + + for (tag, container_path, read_only) in mounts { + let virtiofs_mount = Path::new(paths::VIRTIOFS_MOUNT_ROOT).join(tag); + spec.add_bind_mount( + &virtiofs_mount.to_string_lossy(), + container_path, + *read_only, + ); + } + + add_workspace_fallback(&mut spec, mounts); + add_storage_fallback(&mut spec, mounts, unprivileged); + + crate::ssh_agent::inject_into_container(&mut spec); + crate::cuda::inject_into_container(&mut spec, Path::new(&prepared.rootfs_path)); + spec.add_gpu_devices_if_available(); + + spec.write_to(&bundle_path) + .map_err(|e| StorageError::new(format!("failed to write OCI spec: {}", e)))?; + + let container_id = generate_container_id(); + + let child = CrunCommand::run(&bundle_path, &container_id) + .stdin_null() + .discard_output() + .spawn() + .map_err(|e| { + StorageError::new(format!( + "failed to spawn crun: {}. Is crun installed at {}?", + e, + paths::CRUN_PATH + )) + })?; + + let pid = child.id(); + // Don't wait on the child; it reaps itself when the container exits. + // reap_background_children() in the agent's accept loop collects the + // eventual zombie. + std::mem::forget(child); + + let _ = mounted_paths; // suppress unused warning; mounts persist with the overlay + info!(container_id = %container_id, pid = pid, "background container started"); + Ok(pid) +} + +/// Prepare for running a command - returns the rootfs path. +/// This is used by interactive mode which spawns the command separately. +pub fn prepare_for_run(image: &str) -> Result { + prepare_rootfs_for_ephemeral_run(image) +} + +/// Prepare a persistent overlay that survives across exec sessions. +/// +/// Uses a deterministic workload ID derived from `overlay_id` (typically the +/// machine name). If the overlay already exists and is mounted, reuses it. +/// If it exists but is unmounted (e.g. after VM restart), remounts preserving +/// the upper layer that contains previous changes. +pub fn prepare_for_run_persistent(image: &str, overlay_id: &str) -> Result { + validate_storage_id(overlay_id, "persistent overlay id")?; + let workload_id = format!("persistent-{}", overlay_id); + + // Resolve image layers (same logic as prepare_overlay). A local image + // archive is flattened into a rootfs first; a packed-layers dir is used + // as-is. + let lowerdirs = if let Some(packed_dir) = get_packed_layers_dir() { + let flattened = ensure_archive_flattened(packed_dir)?; + let effective = flattened.as_deref().unwrap_or(packed_dir); + get_packed_lowerdirs(effective)? + } else { + get_image_lowerdirs(image)? + }; + + let setup = OverlaySetup::new(&workload_id)?; + let overlay = setup.execute_or_remount(lowerdirs)?; + + debug!( + workload_id = %workload_id, + rootfs = %overlay.rootfs_path, + "prepared persistent overlay for command execution" + ); + Ok(PreparedOverlayRootfs { + workload_id, + rootfs_path: overlay.rootfs_path, + }) +} + +/// Setup volume mounts for a rootfs (public wrapper). +pub fn setup_mounts(rootfs: &str, mounts: &[(String, String, bool)]) -> Result<()> { + let _mounted_paths = setup_volume_mounts(rootfs, mounts)?; + Ok(()) +} + +/// Setup volume mounts by mounting virtiofs and bind-mounting into the rootfs. +#[cfg(target_os = "linux")] +fn setup_volume_mounts(rootfs: &str, mounts: &[(String, String, bool)]) -> Result> { + let mut mounted_paths = Vec::new(); + let rootfs_path = Path::new(rootfs); + + for (tag, container_path, read_only) in mounts { + validate_storage_id(tag, "mount tag")?; + debug!(tag = %tag, container_path = %container_path, read_only = %read_only, "setting up volume mount"); + + // First, mount the virtiofs device at a staging location + let virtiofs_mount = Path::new(paths::VIRTIOFS_MOUNT_ROOT).join(tag); + std::fs::create_dir_all(&virtiofs_mount)?; + + // Check if already mounted + if !is_mountpoint(&virtiofs_mount) { + info!(tag = %tag, mount_point = %virtiofs_mount.display(), "mounting virtiofs"); + + // Mount virtiofs using direct syscall (avoids ~3-5ms fork+exec overhead). + // Use sync option to ensure writes are persisted immediately. + let src = std::ffi::CString::new(tag.as_str()).map_err(|e| StorageError::Internal { + message: format!("invalid tag: {}", e), + })?; + let dst = + std::ffi::CString::new(virtiofs_mount.to_string_lossy().as_ref()).map_err(|e| { + StorageError::Internal { + message: format!("invalid mount point: {}", e), + } + })?; + let fstype = std::ffi::CString::new("virtiofs").unwrap(); + let opts = std::ffi::CString::new("sync").unwrap(); + // SAFETY: mount virtiofs with valid CString arguments + let rc = unsafe { + libc::mount( + src.as_ptr(), + dst.as_ptr(), + fstype.as_ptr(), + 0, + opts.as_ptr() as *const libc::c_void, + ) + }; + if rc != 0 { + let err = std::io::Error::last_os_error(); + warn!(error = %err, tag = %tag, "failed to mount virtiofs device"); + continue; + } + } + + // Now bind-mount into the container rootfs + let target_path = ensure_mount_target_under_root(rootfs_path, container_path)?; + + // Check if already bind-mounted + if !is_mountpoint(&target_path) { + info!( + source = %virtiofs_mount.display(), + target = %target_path.display(), + read_only = %read_only, + "bind-mounting into container" + ); + + // Bind mount using direct syscall + let bind_src = std::ffi::CString::new(virtiofs_mount.to_string_lossy().as_ref()) + .map_err(|e| StorageError::Internal { + message: format!("invalid source: {}", e), + })?; + let bind_dst = + std::ffi::CString::new(target_path.to_string_lossy().as_ref()).map_err(|e| { + StorageError::Internal { + message: format!("invalid target: {}", e), + } + })?; + // SAFETY: bind mount with MS_BIND flag + let rc = unsafe { + libc::mount( + bind_src.as_ptr(), + bind_dst.as_ptr(), + std::ptr::null(), + libc::MS_BIND, + std::ptr::null(), + ) + }; + if rc != 0 { + let err = std::io::Error::last_os_error(); + warn!(error = %err, target = %target_path.display(), "failed to bind-mount"); + continue; + } + + // Remount read-only if requested + if *read_only { + // SAFETY: remount with MS_BIND|MS_RDONLY|MS_REMOUNT + unsafe { + libc::mount( + std::ptr::null(), + bind_dst.as_ptr(), + std::ptr::null(), + libc::MS_BIND | libc::MS_REMOUNT | libc::MS_RDONLY, + std::ptr::null(), + ); + } + } + } + + mounted_paths.push(target_path); + } + + Ok(mounted_paths) +} + +/// Stub for non-Linux platforms. +#[cfg(not(target_os = "linux"))] +fn setup_volume_mounts(_rootfs: &str, _mounts: &[(String, String, bool)]) -> Result> { + Ok(Vec::new()) +} + +/// Check if a path is a mountpoint (delegates to paths::is_mount_point). +fn is_mountpoint(path: &Path) -> bool { + paths::is_mount_point(path) +} + +/// Run a command using crun OCI runtime (one-shot execution). +/// +/// This uses `crun run` which creates, starts, waits, and deletes the container +/// in a single operation. Stdout and stderr are captured. +/// Join a running container via `crun exec` (non-interactive). +fn run_exec_in_container( + container_id: &str, + command: &[String], + env: &[(String, String)], + workdir: Option<&str>, + timeout_ms: Option, + client_fd: Option, +) -> Result { + info!(container_id = %container_id, command = ?command, "joining container via crun exec"); + + let mut child = CrunCommand::exec(container_id, env, command, workdir, false) + .stdin_null() + .capture_output() + .spawn() + .map_err(|e| StorageError::new(format!("crun exec failed: {}", e)))?; + + // On timeout/disconnect, kill only the exec'd process — NOT the main + // container. The main container hosts the shared namespace for all execs; + // a timed-out `exec -- sleep 10` must not destroy the workload. + let exec_pid = child.id(); + let result = crate::process::wait_with_timeout_cleanup_and_liveness( + &mut child, + timeout_ms, + client_fd, + || unsafe { + libc::kill(exec_pid as libc::pid_t, libc::SIGKILL); + }, + )?; + + match result { + WaitResult::Completed { exit_code, output } => Ok(RunResult { + exit_code, + stdout: output.stdout, + stderr: output.stderr, + }), + WaitResult::TimedOut { output, timeout_ms } => { + let mut stderr = output.stderr; + stderr.extend_from_slice( + format!("\ncommand timed out after {}ms", timeout_ms).as_bytes(), + ); + Ok(RunResult { + exit_code: 124, + stdout: output.stdout, + stderr, + }) + } + WaitResult::ClientDisconnected { output } => { + let mut stderr = output.stderr; + stderr.extend_from_slice(b"\nclient disconnected"); + Ok(RunResult { + exit_code: 137, + stdout: output.stdout, + stderr, + }) + } + } +} + +fn run_with_crun( + bundle_dir: &Path, + container_id: &str, + timeout_ms: Option, + stdin_data: Option<&str>, + client_fd: Option, +) -> Result { + info!( + container_id = %container_id, + bundle = %bundle_dir.display(), + timeout_ms = ?timeout_ms, + "running container with crun" + ); + + // Spawn the container using CrunCommand. + // stdin_null() is critical when no input is supplied: without it, crun + // inherits the agent's vsock stdin, and /bin/sh reads protocol bytes + // instead of user input, hanging. With input, pipe it in and close the + // pipe so the command sees EOF (same contract as the bare-VM exec path). + let builder = CrunCommand::run(bundle_dir, container_id); + let builder = if stdin_data.is_some() { + builder.stdin_piped() + } else { + builder.stdin_null() + }; + let mut child = builder.capture_output().spawn().map_err(|e| { + StorageError::new(format!( + "failed to spawn crun: {}. Is crun installed at {}?", + e, + paths::CRUN_PATH + )) + })?; + + // Write stdin on a separate thread so the wait/timeout loop stays live + // even if the child never reads and the pipe buffer fills. Dropping the + // handle closes the pipe → EOF. + let _stdin_writer = stdin_data.and_then(|data| { + child.stdin.take().map(|mut child_stdin| { + let data = data.to_owned(); + std::thread::Builder::new() + .name("run-stdin".into()) + .spawn(move || { + use std::io::Write; + let _ = child_stdin.write_all(data.as_bytes()); + }) + }) + }); + + // Capture container_id for the cleanup closure + let cid = container_id.to_string(); + + // Wait with timeout + client liveness, cleaning up container on timeout. + // If the client disconnects mid-exec, we kill the container so the agent's + // accept loop is free to serve the next request. + let result = crate::process::wait_with_timeout_cleanup_and_liveness( + &mut child, + timeout_ms, + client_fd, + || { + // Kill and delete the container on timeout + let _ = CrunCommand::kill(&cid, "SIGKILL").status(); + let _ = CrunCommand::delete(&cid, true).status(); + }, + )?; + + // Convert WaitResult to RunResult + match result { + WaitResult::Completed { exit_code, output } => { + info!( + container_id = %container_id, + exit_code = exit_code, + stdout_len = output.stdout.len(), + stderr_len = output.stderr.len(), + "container finished" + ); + Ok(RunResult { + exit_code, + stdout: output.stdout, + stderr: output.stderr, + }) + } + WaitResult::TimedOut { output, timeout_ms } => { + warn!( + container_id = %container_id, + timeout_ms = timeout_ms, + "container timed out" + ); + let mut stderr = output.stderr; + stderr.extend_from_slice( + format!("\ncontainer timed out after {}ms", timeout_ms).as_bytes(), + ); + Ok(RunResult { + exit_code: TIMEOUT_EXIT_CODE, + stdout: output.stdout, + stderr, + }) + } + WaitResult::ClientDisconnected { output } => { + // Client gave up before the container finished. Also clean up the + // crun container state so the next exec starts fresh. + let _ = CrunCommand::kill(container_id, "SIGKILL").status(); + let _ = CrunCommand::delete(container_id, true).status(); + warn!( + container_id = %container_id, + "container killed — client disconnected" + ); + let mut stderr = output.stderr; + stderr.extend_from_slice(b"\ncontainer killed: client disconnected"); + Ok(RunResult { + exit_code: 129, // SIGHUP convention for disconnect + stdout: output.stdout, + stderr, + }) + } + } +} + +// ============================================================================ +// Overlay mounting helper functions +// ============================================================================ + +/// Mount an overlay with multiple lower layers, appending each layer via the new +/// mount API (`fsopen`/`fsconfig`/`fsmount`/`move_mount`) instead of shelling out +/// to `mount(8)`. +/// +/// Why not `mount -o lowerdir=…`: the `mount(8)` command rejects a `lowerdir=` +/// value longer than ~255 bytes, so any image with ≥4 layers (each OCI layer path +/// `/storage/layers/<64-hex>` is ~79 bytes) failed this fast path and fell back to +/// a slow physical layer-merge on *every* (re)mount. The kernel has no such limit +/// — verified on the guest kernel (6.12) that a raw `mount(2)` AND this `fsconfig` +/// path both mount a 599-byte / 8-layer overlay. Passing each layer as its own +/// `lowerdir+` also sidesteps the classic `mount(2)` PAGE_SIZE option ceiling. +fn try_mount_overlay_multi_lower( + lowerdirs: &[String], + upper_path: &Path, + work_path: &Path, + merged_path: &Path, +) -> Result<()> { + info!( + layer_count = lowerdirs.len(), + merged_path = %merged_path.display(), + "attempting multi-lowerdir overlay mount (fsconfig API)" + ); + mount_overlay_fsconfig(lowerdirs, upper_path, work_path, merged_path) +} + +/// Linux implementation of the overlay mount via the new mount API. Requires +/// `lowerdir+` support (Linux ≥ 6.7); returns `Err` on older kernels (or any other +/// failure) so the caller falls back to the physical merge. +#[cfg(target_os = "linux")] +fn mount_overlay_fsconfig( + lowerdirs: &[String], + upper_path: &Path, + work_path: &Path, + merged_path: &Path, +) -> Result<()> { + use rustix::fd::AsFd; + use rustix::mount::{ + fsconfig_create, fsconfig_set_string, fsmount, fsopen, move_mount, FsMountFlags, + FsOpenFlags, MountAttrFlags, MoveMountFlags, + }; + + let upper = upper_path + .to_str() + .ok_or_else(|| StorageError::new("upperdir path is not valid UTF-8".to_string()))?; + let work = work_path + .to_str() + .ok_or_else(|| StorageError::new("workdir path is not valid UTF-8".to_string()))?; + + let fs = fsopen("overlay", FsOpenFlags::FSOPEN_CLOEXEC) + .map_err(|e| StorageError::new(format!("fsopen(overlay) failed: {e}")))?; + + // Append each lower layer individually — no single long option string, so + // neither the `mount(8)` ~255-byte limit nor the `mount(2)` page limit applies. + for lower in lowerdirs { + fsconfig_set_string(fs.as_fd(), "lowerdir+", lower.as_str()).map_err(|e| { + StorageError::new(format!( + "fsconfig lowerdir+={lower} failed (kernel may lack lowerdir+ (<6.7)): {e}" + )) + })?; + } + fsconfig_set_string(fs.as_fd(), "upperdir", upper) + .map_err(|e| StorageError::new(format!("fsconfig upperdir failed: {e}")))?; + fsconfig_set_string(fs.as_fd(), "workdir", work) + .map_err(|e| StorageError::new(format!("fsconfig workdir failed: {e}")))?; + // Preserve prior semantics: index=off disables the inode-index feature. + fsconfig_set_string(fs.as_fd(), "index", "off") + .map_err(|e| StorageError::new(format!("fsconfig index=off failed: {e}")))?; + + fsconfig_create(fs.as_fd()) + .map_err(|e| StorageError::new(format!("fsconfig create (overlay) failed: {e}")))?; + + let mnt = fsmount( + fs.as_fd(), + FsMountFlags::FSMOUNT_CLOEXEC, + MountAttrFlags::empty(), + ) + .map_err(|e| StorageError::new(format!("fsmount(overlay) failed: {e}")))?; + + // Attach the freshly-created mount at merged_path (the mount fd itself is the + // source, via MOVE_MOUNT_F_EMPTY_PATH). + move_mount( + mnt.as_fd(), + "", + rustix::fs::CWD, + merged_path, + MoveMountFlags::MOVE_MOUNT_F_EMPTY_PATH, + ) + .map_err(|e| { + StorageError::new(format!( + "move_mount to {} failed: {e}", + merged_path.display() + )) + })?; + + Ok(()) +} + +/// Non-Linux stub: overlayfs is Linux-only, so error and let callers fall back. +#[cfg(not(target_os = "linux"))] +fn mount_overlay_fsconfig( + _lowerdirs: &[String], + _upper_path: &Path, + _work_path: &Path, + _merged_path: &Path, +) -> Result<()> { + Err(StorageError::new( + "overlay mount is only supported on Linux".to_string(), + )) +} + +/// Mount overlay by merging layers into a single directory (most compatible). +/// +/// This approach physically copies all layers into a single merged directory, +/// then creates a simple overlay on top of it. This works on all kernels with +/// basic overlay support, but uses more disk space and is slower for initial setup. +/// +/// This is the fallback when multi-lowerdir overlay mounts fail. +fn mount_overlay_sequential( + lowerdirs: &[String], + upper_path: &Path, + work_path: &Path, + merged_path: &Path, + overlay_root: &Path, +) -> Result<()> { + info!( + layer_count = lowerdirs.len(), + "building overlay by merging layers" + ); + + // If only one layer, mount directly + if lowerdirs.len() == 1 { + let mount_opts = format!( + "lowerdir={},upperdir={},workdir={},index=off", + lowerdirs[0], + upper_path.display(), + work_path.display() + ); + + let output = Command::new("mount") + .args(["-t", "overlay", "overlay", "-o", &mount_opts]) + .arg(merged_path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(StorageError::new(format!( + "overlay mount failed: {}", + stderr + ))); + } + return Ok(()); + } + + // Create a directory to hold the physically merged layers + let merged_layers_dir = overlay_root.join("merged_layers"); + std::fs::create_dir_all(&merged_layers_dir)?; + + // lowerdirs is in overlay order (topmost first) + // We need to copy from bottom up so top layers overwrite bottom layers + let layers: Vec<&String> = lowerdirs.iter().rev().collect(); + + info!( + layer_count = layers.len(), + merged_dir = %merged_layers_dir.display(), + "physically merging layers" + ); + + for (i, layer_path) in layers.iter().enumerate() { + debug!( + layer_index = i, + layer_path = %layer_path, + "copying layer to merged directory" + ); + + // Copy layer contents preserving all attributes. + // cp -a preserves symlinks, permissions, etc. + // Uses explicit args instead of shell to avoid injection risks. + let layer_src = format!("{}/.", layer_path); + let output = Command::new("cp") + .arg("-a") + .arg(&layer_src) + .arg(merged_layers_dir.as_os_str()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output()?; + + // Don't fail on cp errors - some layers might have special files + // that can't be copied, but the overlay should still work + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + if !stderr.is_empty() { + debug!( + layer_index = i, + stderr = %stderr, + "layer copy had warnings (non-fatal)" + ); + } + } + } + + info!( + merged_dir = %merged_layers_dir.display(), + "layer merge complete, mounting overlay" + ); + + // Now mount a simple overlay with just the merged directory as lowerdir + let mount_opts = format!( + "lowerdir={},upperdir={},workdir={},index=off", + merged_layers_dir.display(), + upper_path.display(), + work_path.display() + ); + + let output = Command::new("mount") + .args(["-t", "overlay", "overlay", "-o", &mount_opts]) + .arg(merged_path) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(StorageError::new(format!( + "overlay mount on merged layers failed: {}", + stderr + ))); + } + + info!( + layer_count = lowerdirs.len(), + "overlay construction complete (merged layers approach)" + ); + + Ok(()) +} + +// ============================================================================ +// Helper functions +// ============================================================================ + +/// Extract the registry hostname from an image reference. +/// e.g., "alpine:latest" -> "https://index.docker.io/v1/" +/// e.g., "ghcr.io/owner/repo" -> "ghcr.io" +fn extract_registry_from_image(image: &str) -> String { + if let Some(slash_pos) = image.find('/') { + let potential_registry = &image[..slash_pos]; + if potential_registry.contains('.') || potential_registry.contains(':') { + return docker_config_registry_key(potential_registry).to_string(); + } + } + // Docker Hub uses this URL in config.json + DOCKER_HUB_AUTH_CONFIG_KEY.to_string() +} + +fn docker_config_registry_key(registry: &str) -> &str { + if DOCKER_HUB_REGISTRY_ALIASES.contains(®istry) { + DOCKER_HUB_AUTH_CONFIG_KEY + } else { + registry + } +} + +/// Simple base64 encoding for auth string. +fn base64_encode(input: &str) -> String { + const ALPHABET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let bytes = input.as_bytes(); + let mut result = String::new(); + + for chunk in bytes.chunks(3) { + let b0 = chunk[0] as usize; + let b1 = chunk.get(1).copied().unwrap_or(0) as usize; + let b2 = chunk.get(2).copied().unwrap_or(0) as usize; + + result.push(ALPHABET[b0 >> 2] as char); + result.push(ALPHABET[((b0 & 0x03) << 4) | (b1 >> 4)] as char); + + if chunk.len() > 1 { + result.push(ALPHABET[((b1 & 0x0f) << 2) | (b2 >> 6)] as char); + } else { + result.push('='); + } + + if chunk.len() > 2 { + result.push(ALPHABET[b2 & 0x3f] as char); + } else { + result.push('='); + } + } + + result +} + +/// Set up Docker auth configuration for crane commands. +/// +/// Creates a temporary directory with a Docker config.json file containing +/// registry credentials. The returned TempDir must be kept alive for the +/// duration of the command execution. +/// +/// Returns `Ok(None)` if no auth is provided. +fn setup_docker_auth( + image: &str, + auth: Option<&RegistryAuth>, +) -> Result> { + let Some(a) = auth else { + return Ok(None); + }; + + let registry = extract_registry_from_image(image); + + // The guest root filesystem (and thus the default temp dir, /tmp) is + // read-only, so create the auth config under the writable storage disk. + let temp_dir = tempfile::Builder::new() + .prefix("smolauth") + .tempdir_in(STORAGE_ROOT) + .map_err(|e| { + StorageError::new(format!("failed to create temp directory for auth: {}", e)) + })?; + + let auth_b64 = base64_encode(&format!("{}:{}", a.username, a.password)); + let config_json = format!( + r#"{{"auths":{{"{}":{{"auth":"{}"}}}}}}"#, + registry, auth_b64 + ); + + let config_path = temp_dir.path().join("config.json"); + std::fs::write(&config_path, &config_json) + .map_err(|e| StorageError::new(format!("failed to write docker auth config: {}", e)))?; + + debug!( + registry = %registry, + username = %a.username, + "using registry credentials via docker config" + ); + + Ok(Some(temp_dir)) +} + +/// Set HTTP_PROXY / HTTPS_PROXY / NO_PROXY on a crane subprocess so the +/// in-VM registry client can reach the registry through a corporate proxy. +fn apply_proxy_env(cmd: &mut Command, proxy: Option<&str>, no_proxy: Option<&str>) { + if let Some(p) = proxy { + cmd.env("HTTP_PROXY", p); + cmd.env("HTTPS_PROXY", p); + } + if let Some(np) = no_proxy { + cmd.env("NO_PROXY", np); + } +} + +/// Run a crane command with the given operation. +/// +/// If auth is provided, creates a temporary Docker config for crane to use. +/// Includes retry logic for transient network failures. +fn run_crane( + operation: &str, + image: &str, + oci_platform: Option<&str>, + auth: Option<&RegistryAuth>, + proxy: Option<&str>, + no_proxy: Option<&str>, +) -> Result { + use crate::retry::{ + is_permanent_error, is_transient_network_error, retry_with_backoff, RetryConfig, + }; + + let op_name = format!("crane {}", operation); + + retry_with_backoff( + RetryConfig::for_network(), + &op_name, + || run_crane_once(operation, image, oci_platform, auth, proxy, no_proxy), + |e| { + let error_msg = e.to_string(); + // Don't retry permanent errors + if is_permanent_error(&error_msg) { + return false; + } + // Retry transient network errors + is_transient_network_error(&error_msg) + }, + ) +} + +/// Execute a single crane command attempt. +fn run_crane_once( + operation: &str, + image: &str, + oci_platform: Option<&str>, + auth: Option<&RegistryAuth>, + proxy: Option<&str>, + no_proxy: Option<&str>, +) -> Result { + let mut cmd = Command::new("crane"); + cmd.arg(operation).arg(image); + + if let Some(p) = oci_platform { + cmd.arg("--platform").arg(p); + } + + // Set up auth if provided (temp_dir must stay alive until command completes) + let _temp_dir = setup_docker_auth(image, auth)?; + if let Some(ref td) = _temp_dir { + cmd.env("DOCKER_CONFIG", td.path()); + } + + apply_proxy_env(&mut cmd, proxy, no_proxy); + + let output = cmd.output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(StorageError::new(format!( + "crane {} failed: {}", + operation, stderr + ))); + } + + Ok(String::from_utf8_lossy(&output.stdout).to_string()) +} + +/// Run crane manifest command. +fn crane_manifest( + image: &str, + oci_platform: Option<&str>, + auth: Option<&RegistryAuth>, + proxy: Option<&str>, + no_proxy: Option<&str>, +) -> Result { + run_crane("manifest", image, oci_platform, auth, proxy, no_proxy) +} + +/// Run crane config command. +fn crane_config( + image: &str, + oci_platform: Option<&str>, + auth: Option<&RegistryAuth>, + proxy: Option<&str>, + no_proxy: Option<&str>, +) -> Result { + run_crane("config", image, oci_platform, auth, proxy, no_proxy) +} + +/// Sanitize image name for use as filename. +fn sanitize_image_name(image: &str) -> String { + image.replace(['/', ':', '@'], "_") +} + +/// Reverse sanitization of a canonical image filename back to an image reference. +/// +/// Because we now always store under the canonical form the mapping is +/// deterministic: +/// - The last `_`-delimited segment is the tag (or digest hex), except when +/// the penultimate segment is `sha256`, in which case `sha256_` is the +/// digest. +/// - Everything else is the `registry/path` portion, with `_` reversed to `/`. +/// +/// The result is passed to `query_image`, which normalizes it before +/// computing the cache key. +fn unsanitize_image_name(name: &str) -> String { + let parts: Vec<&str> = name.split('_').collect(); + if parts.len() < 2 { + return name.to_string(); + } + + // Detect sha256 digest: penultimate segment is "sha256", last is 64 hex chars. + let n = parts.len(); + if n >= 2 + && parts[n - 2] == "sha256" + && parts[n - 1].len() == 64 + && parts[n - 1].chars().all(|c| c.is_ascii_hexdigit()) + { + let name_part = parts[..n - 2].join("/"); + return format!("{name_part}@sha256:{}", parts[n - 1]); + } + + // Normal case: last segment is the tag. + let name_part = parts[..n - 1].join("/"); + format!("{name_part}:{}", parts[n - 1]) +} + +/// Get disk usage for a path. +#[allow(unused_variables)] // path is used only on Linux +fn get_disk_usage(path: &Path) -> Result<(u64, u64)> { + // Use statvfs on Linux + #[cfg(target_os = "linux")] + { + use std::ffi::CString; + use std::mem::MaybeUninit; + + let path_cstr = CString::new(path.to_string_lossy().as_bytes()).map_err(|_| { + StorageError::InvalidPath { + path: "overlay path".into(), + } + })?; + + unsafe { + let mut stat: MaybeUninit = MaybeUninit::uninit(); + if libc::statvfs(path_cstr.as_ptr(), stat.as_mut_ptr()) != 0 { + return Err(std::io::Error::last_os_error().into()); + } + + let stat = stat.assume_init(); + let total = stat.f_blocks * stat.f_frsize; + let free = stat.f_bfree * stat.f_frsize; + let used = total - free; + + Ok((total as u64, used as u64)) + } + } + + #[cfg(not(target_os = "linux"))] + { + Ok((0, 0)) + } +} + +/// Count entries in a directory. +fn count_entries(path: &Path) -> Result { + if !path.exists() { + return Ok(0); + } + + Ok(std::fs::read_dir(path)?.count()) +} + +/// Convert an OCI platform string to its architecture component. +/// +/// # Examples +/// - "linux/arm64" -> "arm64" +/// - "linux/amd64" -> "amd64" +/// - "linux/arm64/v8" -> "arm64" +fn oci_platform_to_arch(oci_platform: &str) -> String { + // OCI platform format is "os/arch" or "os/arch/variant" + // We want just the arch part + let parts: Vec<&str> = oci_platform.split('/').collect(); + if parts.len() >= 2 { + parts[1].to_string() + } else { + // Fallback: return as-is if not in expected format + oci_platform.to_string() + } +} + +/// Calculate directory size recursively. +fn dir_size(path: &Path) -> Result { + let mut size = 0; + + if path.is_file() { + return Ok(std::fs::metadata(path)?.len()); + } + + for entry in std::fs::read_dir(path)? { + let entry: std::fs::DirEntry = entry?; + let path = entry.path(); + + if path.is_file() { + size += std::fs::metadata(&path)?.len(); + } else if path.is_dir() { + size += dir_size(&path)?; + } + } + + Ok(size) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Mutex, OnceLock}; + + fn env_lock() -> &'static Mutex<()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())) + } + + #[test] + fn storage_exposed_only_to_privileged_workloads_without_a_user_storage_mount() { + // Privileged (default) and no user mount at /storage → expose it, so an + // --image VM sees /storage like a bare VM (docker-in-VM bind targets work). + assert!(should_expose_storage(&[], false)); + + // Unprivileged (untrusted code) never sees the VM's storage disk. + assert!(!should_expose_storage(&[], true)); + + // The user already claimed /storage (e.g. -v host:/storage) → don't clobber + // it, even when privileged. Trailing slash is normalized. + let user_mount = vec![("tag".to_string(), "/storage/".to_string(), false)]; + assert!(!should_expose_storage(&user_mount, false)); + + // A user mount elsewhere doesn't suppress the /storage fallback. + let other_mount = vec![("tag".to_string(), "/data".to_string(), false)]; + assert!(should_expose_storage(&other_mount, false)); + } + + #[test] + fn test_oci_platform_to_arch_linux_arm64() { + assert_eq!(oci_platform_to_arch("linux/arm64"), "arm64"); + } + + #[test] + fn classifies_oci_whiteout_markers() { + // Opaque marker must win over the generic `.wh.` prefix. + assert_eq!(classify_layer_entry(".wh..wh..opq"), LayerEntry::OpaqueDir); + // `.wh.` carries the name to delete. + assert_eq!( + classify_layer_entry(".wh.RPM-GPG-KEY-kojiv2"), + LayerEntry::Whiteout("RPM-GPG-KEY-kojiv2") + ); + // A bare `.wh.` (no name) and ordinary files are normal entries. + assert_eq!(classify_layer_entry(".wh."), LayerEntry::Normal); + assert_eq!(classify_layer_entry("CERN.repo"), LayerEntry::Normal); + assert_eq!(classify_layer_entry(".wherever"), LayerEntry::Normal); + } + + #[test] + fn jailed_join_blocks_escapes() { + let base = Path::new("/layer"); + assert_eq!( + jailed_join(base, Path::new("tmp/CERN.repo")), + Some(PathBuf::from("/layer/tmp/CERN.repo")) + ); + assert_eq!( + jailed_join(base, Path::new("./tmp/./x")), + Some(PathBuf::from("/layer/tmp/x")) + ); + // `..` and absolute paths escape the layer dir — rejected. + assert!(jailed_join(base, Path::new("../etc/passwd")).is_none()); + assert!(jailed_join(base, Path::new("tmp/../../etc")).is_none()); + assert!(jailed_join(base, Path::new("/etc/passwd")).is_none()); + } + + /// End-to-end extraction with whiteout conversion. mknod/setxattr(trusted.*) + /// need root, so skip when not privileged (the live guest is PID 1 root). + /// Linux-only: the syscalls and overlayfs semantics don't exist on macOS. + #[test] + fn extract_oci_layer_decompresses_gzip_and_zstd() { + use std::io::Write; + + // A minimal single-file tar, owned by the current uid/gid so extraction + // (which preserves ownership) succeeds without root. + let uid = unsafe { libc::getuid() } as u64; + let gid = unsafe { libc::getgid() } as u64; + let mut tar_buf = Vec::new(); + { + let mut builder = tar::Builder::new(&mut tar_buf); + let body = b"hello from a layer"; + let mut header = tar::Header::new_gnu(); + header.set_path("greeting.txt").unwrap(); + header.set_size(body.len() as u64); + header.set_entry_type(tar::EntryType::Regular); + header.set_mode(0o644); + header.set_uid(uid); + header.set_gid(gid); + header.set_cksum(); + builder.append(&header, &body[..]).unwrap(); + builder.finish().unwrap(); + } + + let extract = |bytes: &[u8]| -> Vec { + let dir = tempfile::tempdir().unwrap(); + extract_oci_layer(bytes, dir.path()).expect("extraction should succeed"); + std::fs::read(dir.path().join("greeting.txt")).unwrap() + }; + + // Plain tar passes through unchanged. + assert_eq!(extract(&tar_buf), b"hello from a layer"); + + // gzip-compressed layer. + let gz = { + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(&tar_buf).unwrap(); + enc.finish().unwrap() + }; + assert_eq!(&gz[..2], &[0x1f, 0x8b], "gzip magic"); + assert_eq!(extract(&gz), b"hello from a layer"); + + // zstd-compressed layer — the format that broke every library-image pull. + let zst = zstd::stream::encode_all(&tar_buf[..], 0).unwrap(); + assert_eq!(&zst[..4], &[0x28, 0xb5, 0x2f, 0xfd], "zstd magic"); + assert_eq!(extract(&zst), b"hello from a layer"); + } + + #[cfg(target_os = "linux")] + #[test] + fn extract_oci_layer_applies_whiteouts() { + // SAFETY: geteuid is always safe. + if unsafe { libc::geteuid() } != 0 { + eprintln!("skipping: extract_oci_layer whiteout test needs root (mknod/setxattr)"); + return; + } + use std::os::unix::fs::FileTypeExt; + + let dir = tempfile::tempdir().unwrap(); + let dest = dir.path(); + + // Build a layer tar: a real file, a `.wh.` delete marker shipped as a + // hardlink to that file (the issue #397 shape), and an opaque dir. + let mut buf = Vec::new(); + { + let mut builder = tar::Builder::new(&mut buf); + let mut header = tar::Header::new_gnu(); + let body = b"repo-contents"; + header.set_path("tmp/CERN.repo").unwrap(); + header.set_size(body.len() as u64); + header.set_entry_type(tar::EntryType::Regular); + header.set_mode(0o644); + header.set_cksum(); + builder.append(&header, &body[..]).unwrap(); + + // Whiteout as a hardlink to the sibling file (would crash busybox tar). + let mut wh = tar::Header::new_gnu(); + wh.set_entry_type(tar::EntryType::Link); + wh.set_size(0); + wh.set_mode(0o644); + wh.set_path("tmp/.wh.RPM-GPG-KEY-kojiv2").unwrap(); + wh.set_link_name("tmp/CERN.repo").unwrap(); + wh.set_cksum(); + builder.append(&wh, std::io::empty()).unwrap(); + + // Opaque marker for an `etc` directory. + let mut opq = tar::Header::new_gnu(); + opq.set_entry_type(tar::EntryType::Regular); + opq.set_size(0); + opq.set_mode(0o644); + opq.set_path("etc/.wh..wh..opq").unwrap(); + opq.set_cksum(); + builder.append(&opq, std::io::empty()).unwrap(); + + builder.finish().unwrap(); + } + + extract_oci_layer(&buf[..], dest).expect("extraction should succeed"); + + // The real file extracted. + assert_eq!( + std::fs::read(dest.join("tmp/CERN.repo")).unwrap(), + b"repo-contents" + ); + // The whiteout became an overlayfs char-device whiteout (0/0). + let wh = dest.join("tmp/RPM-GPG-KEY-kojiv2"); + let meta = std::fs::symlink_metadata(&wh).expect("whiteout node exists"); + assert!( + meta.file_type().is_char_device(), + "whiteout is a char device" + ); + use std::os::unix::fs::MetadataExt; + assert_eq!(meta.rdev(), 0, "whiteout device number is 0/0"); + // The `.wh.` marker file itself is gone. + assert!(!dest.join("tmp/.wh.RPM-GPG-KEY-kojiv2").exists()); + // The opaque xattr is set on the directory, and the marker file is gone. + assert!(dest.join("etc").is_dir()); + assert_eq!(read_opaque_xattr(&dest.join("etc")), Some(b"y".to_vec())); + assert!(!dest.join("etc/.wh..wh..opq").exists()); + } + + /// Read `trusted.overlay.opaque` for the extraction test (root-only). + #[cfg(target_os = "linux")] + fn read_opaque_xattr(path: &Path) -> Option> { + use std::os::unix::ffi::OsStrExt; + let c_path = std::ffi::CString::new(path.as_os_str().as_bytes()).ok()?; + let name = std::ffi::CString::new("trusted.overlay.opaque").ok()?; + let mut buf = [0u8; 16]; + // SAFETY: path/name are NUL-terminated; buf/len describe a valid buffer. + let len = unsafe { + libc::getxattr( + c_path.as_ptr(), + name.as_ptr(), + buf.as_mut_ptr() as *mut libc::c_void, + buf.len(), + ) + }; + if len < 0 { + return None; + } + Some(buf[..len as usize].to_vec()) + } + + #[test] + fn test_oci_platform_to_arch_linux_amd64() { + assert_eq!(oci_platform_to_arch("linux/amd64"), "amd64"); + } + + #[test] + fn test_oci_platform_to_arch_with_variant() { + assert_eq!(oci_platform_to_arch("linux/arm64/v8"), "arm64"); + assert_eq!(oci_platform_to_arch("linux/arm/v7"), "arm"); + } + + #[test] + fn test_oci_platform_to_arch_fallback() { + // If not in expected format, return as-is + assert_eq!(oci_platform_to_arch("arm64"), "arm64"); + assert_eq!(oci_platform_to_arch("unknown"), "unknown"); + } + + /// Collect (name, value) for env vars explicitly set on a Command, with + /// inherited vars filtered out. `Command::get_envs()` yields a tuple per + /// explicit `.env()` / `.env_remove()` call: the value is `None` for + /// removals and `Some(_)` for sets. We only care about sets here. + fn explicit_envs(cmd: &Command) -> Vec<(String, String)> { + cmd.get_envs() + .filter_map(|(k, v)| { + v.map(|val| { + ( + k.to_string_lossy().into_owned(), + val.to_string_lossy().into_owned(), + ) + }) + }) + .collect() + } + + #[test] + fn apply_proxy_env_sets_http_and_https_when_proxy_present() { + let mut cmd = Command::new("crane"); + apply_proxy_env(&mut cmd, Some("http://proxy.example.com:3128"), None); + + let envs = explicit_envs(&cmd); + assert!(envs.contains(&( + "HTTP_PROXY".to_string(), + "http://proxy.example.com:3128".to_string() + ))); + assert!(envs.contains(&( + "HTTPS_PROXY".to_string(), + "http://proxy.example.com:3128".to_string() + ))); + // No NO_PROXY when not asked for — silent overreach would be a bug. + assert!(!envs.iter().any(|(k, _)| k == "NO_PROXY")); + } + + #[test] + fn apply_proxy_env_sets_no_proxy_when_present() { + let mut cmd = Command::new("crane"); + apply_proxy_env(&mut cmd, None, Some("127.0.0.1,.internal")); + + let envs = explicit_envs(&cmd); + assert!(envs.contains(&("NO_PROXY".to_string(), "127.0.0.1,.internal".to_string()))); + // proxy=None must not set HTTP_PROXY / HTTPS_PROXY. + assert!(!envs.iter().any(|(k, _)| k == "HTTP_PROXY")); + assert!(!envs.iter().any(|(k, _)| k == "HTTPS_PROXY")); + } + + #[test] + fn apply_proxy_env_with_both_sets_all_three() { + let mut cmd = Command::new("crane"); + apply_proxy_env( + &mut cmd, + Some("http://192.168.127.254:3128"), + Some("127.0.0.1,localhost"), + ); + + let envs = explicit_envs(&cmd); + assert_eq!( + envs.len(), + 3, + "expected exactly HTTP_PROXY, HTTPS_PROXY, NO_PROXY" + ); + let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); + assert_eq!( + map.get("HTTP_PROXY").map(String::as_str), + Some("http://192.168.127.254:3128") + ); + assert_eq!( + map.get("HTTPS_PROXY").map(String::as_str), + Some("http://192.168.127.254:3128") + ); + assert_eq!( + map.get("NO_PROXY").map(String::as_str), + Some("127.0.0.1,localhost") + ); + } + + #[test] + fn apply_proxy_env_with_none_is_noop() { + let mut cmd = Command::new("crane"); + apply_proxy_env(&mut cmd, None, None); + + // Without explicit envs the iterator is empty — no accidental fallbacks. + assert_eq!(explicit_envs(&cmd).len(), 0); + } + + #[test] + fn test_sanitize_image_name() { + // sanitize_image_name operates on already-canonical refs + assert_eq!( + sanitize_image_name("docker.io/library/alpine:latest"), + "docker.io_library_alpine_latest" + ); + assert_eq!( + sanitize_image_name("docker.io/library/alpine:3.18"), + "docker.io_library_alpine_3.18" + ); + assert_eq!( + sanitize_image_name("ghcr.io/owner/repo@sha256:abc123"), + "ghcr.io_owner_repo_sha256_abc123" + ); + } + + #[test] + fn test_unsanitize_image_name() { + // Normal tag case + assert_eq!( + unsanitize_image_name("docker.io_library_alpine_3.20"), + "docker.io/library/alpine:3.20" + ); + assert_eq!( + unsanitize_image_name("ghcr.io_owner_repo_v1"), + "ghcr.io/owner/repo:v1" + ); + // Digest case + let hex = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + assert_eq!( + unsanitize_image_name(&format!("docker.io_library_alpine_sha256_{hex}")), + format!("docker.io/library/alpine@sha256:{hex}") + ); + } + + #[test] + fn test_extract_registry_from_image_normalizes_docker_hub() { + assert_eq!( + extract_registry_from_image("alpine:latest"), + DOCKER_HUB_AUTH_CONFIG_KEY + ); + assert_eq!( + extract_registry_from_image("library/alpine:latest"), + DOCKER_HUB_AUTH_CONFIG_KEY + ); + assert_eq!( + extract_registry_from_image("docker.io/nginxinc/nginx-unprivileged:stable-alpine"), + DOCKER_HUB_AUTH_CONFIG_KEY + ); + assert_eq!( + extract_registry_from_image("index.docker.io/library/alpine:latest"), + DOCKER_HUB_AUTH_CONFIG_KEY + ); + } + + #[test] + fn test_extract_registry_from_image_preserves_non_docker_hub_registry() { + assert_eq!( + extract_registry_from_image("ghcr.io/owner/repo:tag"), + "ghcr.io" + ); + assert_eq!( + extract_registry_from_image("registry.example.com:5000/image:tag"), + "registry.example.com:5000" + ); + } + + #[test] + fn overlay_resolv_conf_uses_localhost_when_dns_filter_enabled() { + let _guard = env_lock().lock().unwrap(); + std::env::set_var(guest_env::DNS_FILTER, "1"); + std::env::remove_var(guest_env::BACKEND); + std::env::remove_var(guest_env::DNS); + + assert_eq!(overlay_resolv_conf_contents(), "nameserver 127.0.0.1\n"); + + std::env::remove_var(guest_env::DNS_FILTER); + } + + #[test] + fn overlay_resolv_conf_uses_virtio_dns_server() { + let _guard = env_lock().lock().unwrap(); + std::env::remove_var(guest_env::DNS_FILTER); + std::env::set_var(guest_env::BACKEND, guest_env::BACKEND_VIRTIO_NET); + std::env::set_var(guest_env::DNS, "100.96.0.1"); + + assert_eq!(overlay_resolv_conf_contents(), "nameserver 100.96.0.1\n"); + + std::env::remove_var(guest_env::BACKEND); + std::env::remove_var(guest_env::DNS); + } + + #[test] + fn overlay_resolv_conf_uses_custom_dns_under_tsi() { + // TSI sets SMOLVM_NETWORK_DNS without SMOLVM_NETWORK_BACKEND. The guest + // must honor the custom resolver (--dns) rather than the public default. + let _guard = env_lock().lock().unwrap(); + std::env::remove_var(guest_env::DNS_FILTER); + std::env::remove_var(guest_env::BACKEND); + std::env::set_var(guest_env::DNS, "100.100.100.100"); + + assert_eq!( + overlay_resolv_conf_contents(), + "nameserver 100.100.100.100\n" + ); + + std::env::remove_var(guest_env::DNS); + } + + #[test] + fn overlay_resolv_conf_defaults_to_public_resolvers() { + let _guard = env_lock().lock().unwrap(); + std::env::remove_var(guest_env::DNS_FILTER); + std::env::remove_var(guest_env::BACKEND); + std::env::remove_var(guest_env::DNS); + + assert_eq!( + overlay_resolv_conf_contents(), + "nameserver 8.8.8.8\nnameserver 1.1.1.1\n" + ); + } + + #[test] + fn test_validate_storage_id_rejects_traversal() { + assert!(validate_storage_id("../escape", "workload_id").is_err()); + assert!(validate_storage_id("foo/bar", "workload_id").is_err()); + } + + #[test] + fn test_validate_container_destination_path_requires_absolute() { + assert!(validate_container_destination_path("var/data").is_err()); + assert!(validate_container_destination_path("/").is_err()); + assert!(validate_container_destination_path("/var/data").is_ok()); + } + + #[test] + fn test_ensure_mount_target_under_root_rejects_parent_traversal() { + let root = tempfile::tempdir().unwrap(); + let rootfs = root.path().join("rootfs"); + std::fs::create_dir_all(&rootfs).unwrap(); + + assert!(ensure_mount_target_under_root(&rootfs, "/../../escape").is_err()); + } + + #[cfg(unix)] + #[test] + fn test_ensure_mount_target_under_root_rejects_symlink_escape() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let rootfs = root.path().join("rootfs"); + std::fs::create_dir_all(&rootfs).unwrap(); + + symlink(outside.path(), rootfs.join("link-out")).unwrap(); + assert!(ensure_mount_target_under_root(&rootfs, "/link-out/dir").is_err()); + } + + #[cfg(unix)] + #[test] + fn test_ensure_mount_target_under_root_replaces_intra_rootfs_symlink_with_dir() { + use std::os::unix::fs::symlink; + + // Simulates the agent-rootfs having a pre-baked /workspace symlink from + // a previous VM run (via virtiofs write-through). The function must + // replace it with a real directory so the bind mount can claim the path. + let root = tempfile::tempdir().unwrap(); + let rootfs = root.path().join("rootfs"); + let target_dir = rootfs.join("storage").join("workspace"); + std::fs::create_dir_all(&target_dir).unwrap(); + + // /workspace → /storage/workspace (relative to rootfs) — symlink within rootfs + let workspace_link = rootfs.join("workspace"); + symlink(&target_dir, &workspace_link).unwrap(); + assert!(workspace_link.is_symlink()); + + let result = ensure_mount_target_under_root(&rootfs, "/workspace"); + assert!(result.is_ok(), "expected Ok, got {:?}", result); + + // The symlink must have been replaced with a real directory. + assert!( + !workspace_link.is_symlink(), + "/workspace should no longer be a symlink" + ); + assert!( + workspace_link.is_dir(), + "/workspace should now be a directory" + ); + } + + #[test] + fn ordered_packed_layers_honor_index_over_name_sort() { + // Two layers whose digest-named dirs sort base-above-overlay (the bug): + // base "fff…" sorts after overlay "4c8…", so a plain name sort + rev + // would stack the base on top and shadow the overlay's modified files. + let dir = tempfile::tempdir().unwrap(); + for name in ["fff3795b4371", "4c857248e0e2"] { + std::fs::create_dir_all(dir.path().join(name)).unwrap(); + } + // A stray non-layer dir (e.g. macOS .fseventsd) must be excluded when an + // index is present. + std::fs::create_dir_all(dir.path().join(".fseventsd")).unwrap(); + + // Index records true OCI order, bottom→top: base then overlay. + std::fs::write( + dir.path().join(LAYER_ORDER_FILE), + "fff3795b4371\n4c857248e0e2\n", + ) + .unwrap(); + + let ordered = ordered_packed_layer_names(dir.path()).unwrap(); + assert_eq!( + ordered, + vec!["fff3795b4371".to_string(), "4c857248e0e2".to_string()], + "must follow the index (base→overlay), not the name sort, and drop .fseventsd" + ); + } + + #[test] + fn ordered_packed_layers_fall_back_to_name_sort_without_index() { + // No index → legacy behavior: ascending name sort (correct for the + // single-flattened-layer common case). + let dir = tempfile::tempdir().unwrap(); + for name in ["bbb", "aaa"] { + std::fs::create_dir_all(dir.path().join(name)).unwrap(); + } + let ordered = ordered_packed_layer_names(dir.path()).unwrap(); + assert_eq!(ordered, vec!["aaa".to_string(), "bbb".to_string()]); + } + + #[test] + fn ordered_packed_layers_ignore_index_entries_without_a_dir() { + // An index naming a missing layer falls back to the name sort rather + // than silently dropping real layers. + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("aaa")).unwrap(); + std::fs::write(dir.path().join(LAYER_ORDER_FILE), "does-not-exist\n").unwrap(); + let ordered = ordered_packed_layer_names(dir.path()).unwrap(); + assert_eq!(ordered, vec!["aaa".to_string()]); + } + + /// Regression for the >255-byte `lowerdir` bug: the fsconfig mount path must + /// mount a multi-layer overlay whose joined lower paths far exceed 255 bytes — + /// the length the old `mount(8)` shell-out rejected. Needs root + Linux + /// overlayfs, so it's `#[ignore]`d; run on a Linux host with: + /// cargo test -p smolvm-agent -- --ignored overlay_fsconfig_mounts_long_lowerdir + #[test] + #[ignore = "requires root + Linux overlayfs"] + #[cfg(target_os = "linux")] + fn overlay_fsconfig_mounts_long_lowerdir() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + // 8 layers with 64-char names => joined lowerdir ~600 bytes (>255). + let mut lowerdirs = Vec::new(); + for i in 0..8u32 { + let d = root.join("layers").join(format!("{i:064}")); + std::fs::create_dir_all(&d).unwrap(); + std::fs::write(d.join(format!("f{i}")), b"x").unwrap(); + lowerdirs.push(d.to_string_lossy().into_owned()); + } + let joined_len: usize = lowerdirs.iter().map(|s| s.len() + 1).sum(); + assert!( + joined_len > 255, + "test must exceed the old limit, got {joined_len}" + ); + + let upper = root.join("upper"); + let work = root.join("work"); + let merged = root.join("merged"); + for p in [&upper, &work, &merged] { + std::fs::create_dir_all(p).unwrap(); + } + + mount_overlay_fsconfig(&lowerdirs, &upper, &work, &merged) + .expect("fsconfig overlay mount with a >255B lowerdir should succeed"); + // The merged view exposes files from every layer. + assert!(merged.join("f0").exists()); + assert!(merged.join("f7").exists()); + let _ = std::process::Command::new("umount").arg(&merged).status(); + } +} diff --git a/crates/smolvm-agent/src/timesync.rs b/crates/smolvm-agent/src/timesync.rs new file mode 100644 index 0000000..fb10510 --- /dev/null +++ b/crates/smolvm-agent/src/timesync.rs @@ -0,0 +1,100 @@ +//! Guest wall-clock sync from the host over a vsock DGRAM (port 123). +//! +//! On hypervisors without a guest-readable paravirt clock (macOS/HVF and +//! Windows/WHP), libkrun's host-side `TimesyncThread` pushes the host's +//! `CLOCK_REALTIME` to the guest — at boot, every 60s, and (critically) as soon +//! as it detects the host was suspended (lid closed / sleep). During host sleep +//! the VM is frozen, so the guest clock stops; without a receiver it falls +//! behind by the accumulated sleep and eventually breaks TLS / signed package +//! indexes inside the guest (issue #521). The agent runs as PID-1 with +//! `CAP_SYS_TIME`, so — unlike the unprivileged container — it can apply the fix. +//! +//! Each host datagram is an 8-byte little-endian u64: host `CLOCK_REALTIME` in +//! nanoseconds (see libkrun `vsock/timesync.rs::write_time_sync`). Mirrors +//! libkrun's own guest reader (`init/src/timesync.rs`), but as a thread since +//! the agent is long-lived (it never `exec`s away, so it needs no `fork`). + +use std::mem; + +const AF_VSOCK: libc::c_int = 40; +const VMADDR_CID_ANY: u32 = u32::MAX; +const TSYNC_PORT: u32 = 123; +const NANOS_IN_SECOND: u64 = 1_000_000_000; +/// Ignore sub-100ms drift — matches libkrun's guest reader. +const DELTA_SYNC_NS: u64 = 100_000_000; + +#[repr(C)] +struct sockaddr_vm { + svm_family: libc::sa_family_t, + svm_reserved1: u16, + svm_port: u32, + svm_cid: u32, + svm_zero: [u8; 4], +} + +/// Spawn the host→guest clock-sync receiver as a background thread. Best-effort: +/// on any setup error the thread exits and the boot-seeded clock stands. +pub fn spawn() { + let _ = std::thread::Builder::new() + .name("timesync".into()) + .spawn(run); +} + +fn run() { + // AF_VSOCK DGRAM bound to port 123 — the port libkrun's TimesyncThread sends + // to. VMADDR_CID_ANY: accept from the host CID. + let fd = unsafe { libc::socket(AF_VSOCK, libc::SOCK_DGRAM, 0) }; + if fd < 0 { + return; + } + let addr = sockaddr_vm { + svm_family: AF_VSOCK as libc::sa_family_t, + svm_reserved1: 0, + svm_port: TSYNC_PORT, + svm_cid: VMADDR_CID_ANY, + svm_zero: [0; 4], + }; + let rc = unsafe { + libc::bind( + fd, + &addr as *const sockaddr_vm as *const libc::sockaddr, + mem::size_of::() as libc::socklen_t, + ) + }; + if rc < 0 { + unsafe { libc::close(fd) }; + return; + } + + let mut buf = [0u8; 8]; + loop { + let n = unsafe { libc::recv(fd, buf.as_mut_ptr() as *mut libc::c_void, buf.len(), 0) }; + if n < 0 { + break; // socket error — give up (the boot clock stands) + } + if n != 8 { + continue; // malformed datagram — ignore + } + + let host_ns = u64::from_le_bytes(buf); + if host_ns.abs_diff(current_realtime_ns()) > DELTA_SYNC_NS { + let ts = libc::timespec { + tv_sec: (host_ns / NANOS_IN_SECOND) as libc::time_t, + tv_nsec: (host_ns % NANOS_IN_SECOND) as libc::c_long, + }; + // SAFETY: ts is a valid timespec; the agent is PID-1 with CAP_SYS_TIME. + unsafe { libc::clock_settime(libc::CLOCK_REALTIME, &ts) }; + } + } + unsafe { libc::close(fd) }; +} + +fn current_realtime_ns() -> u64 { + let mut ts = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + // SAFETY: ts is a valid, writable timespec. + unsafe { libc::clock_gettime(libc::CLOCK_REALTIME, &mut ts) }; + ts.tv_sec as u64 * NANOS_IN_SECOND + ts.tv_nsec as u64 +} diff --git a/crates/smolvm-agent/src/vsock.rs b/crates/smolvm-agent/src/vsock.rs new file mode 100644 index 0000000..3dfbb3b --- /dev/null +++ b/crates/smolvm-agent/src/vsock.rs @@ -0,0 +1,240 @@ +//! vsock support for the helper daemon. +//! +//! This module provides vsock server functionality for Linux guests. + +use std::io::{Read, Write}; +#[cfg(target_os = "linux")] +use std::os::fd::FromRawFd; +use std::os::fd::OwnedFd; +use std::os::unix::io::AsRawFd; + +/// vsock listener. +pub struct VsockListener { + #[allow(dead_code)] // Accessed via AsRawFd trait + fd: OwnedFd, +} + +/// vsock stream (connection). +pub struct VsockStream { + #[allow(dead_code)] // Accessed via AsRawFd trait + fd: OwnedFd, +} + +#[cfg(target_os = "linux")] +mod linux { + use super::*; + use std::mem; + + // vsock constants + const AF_VSOCK: libc::c_int = 40; + const VMADDR_CID_ANY: u32 = u32::MAX; + + #[repr(C)] + struct sockaddr_vm { + svm_family: libc::sa_family_t, + svm_reserved1: u16, + svm_port: u32, + svm_cid: u32, + svm_zero: [u8; 4], + } + + impl VsockListener { + /// Create a new vsock listener on the given port. + pub fn bind(port: u32) -> std::io::Result { + unsafe { + // Create socket + let fd = libc::socket(AF_VSOCK, libc::SOCK_STREAM, 0); + if fd < 0 { + return Err(std::io::Error::last_os_error()); + } + + let fd = OwnedFd::from_raw_fd(fd); + + // Bind to port + let addr = sockaddr_vm { + svm_family: AF_VSOCK as u16, + svm_reserved1: 0, + svm_port: port, + svm_cid: VMADDR_CID_ANY, + svm_zero: [0; 4], + }; + + if libc::bind( + fd.as_raw_fd(), + &addr as *const sockaddr_vm as *const libc::sockaddr, + mem::size_of::() as libc::socklen_t, + ) < 0 + { + return Err(std::io::Error::last_os_error()); + } + + // Listen + if libc::listen(fd.as_raw_fd(), 1) < 0 { + return Err(std::io::Error::last_os_error()); + } + + Ok(Self { fd }) + } + } + + /// Accept a new connection. + pub fn accept(&self) -> std::io::Result { + unsafe { + let fd = libc::accept( + self.fd.as_raw_fd(), + std::ptr::null_mut(), + std::ptr::null_mut(), + ); + if fd < 0 { + return Err(std::io::Error::last_os_error()); + } + + Ok(VsockStream { + fd: OwnedFd::from_raw_fd(fd), + }) + } + } + } +} + +#[cfg(not(target_os = "linux"))] +mod stub { + use super::*; + + impl VsockListener { + pub fn bind(_port: u32) -> std::io::Result { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "vsock only supported on Linux", + )) + } + + pub fn accept(&self) -> std::io::Result { + unreachable!() + } + } +} + +#[cfg(target_os = "linux")] +impl AsRawFd for VsockStream { + fn as_raw_fd(&self) -> std::os::unix::io::RawFd { + self.fd.as_raw_fd() + } +} + +#[cfg(target_os = "linux")] +impl Read for VsockStream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + unsafe { + let n = libc::read(self.fd.as_raw_fd(), buf.as_mut_ptr() as *mut _, buf.len()); + if n < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(n as usize) + } + } + } +} + +#[cfg(target_os = "linux")] +impl Write for VsockStream { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + unsafe { + let n = libc::write(self.fd.as_raw_fd(), buf.as_ptr() as *const _, buf.len()); + if n < 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(n as usize) + } + } + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[cfg(not(target_os = "linux"))] +impl AsRawFd for VsockStream { + fn as_raw_fd(&self) -> std::os::unix::io::RawFd { + unreachable!("vsock only supported on Linux") + } +} + +#[cfg(not(target_os = "linux"))] +impl Read for VsockStream { + fn read(&mut self, _buf: &mut [u8]) -> std::io::Result { + unreachable!("vsock only supported on Linux") + } +} + +#[cfg(not(target_os = "linux"))] +impl Write for VsockStream { + fn write(&mut self, _buf: &[u8]) -> std::io::Result { + unreachable!("vsock only supported on Linux") + } + + fn flush(&mut self) -> std::io::Result<()> { + unreachable!("vsock only supported on Linux") + } +} + +/// Listen on a vsock port. +pub fn listen(port: u32) -> std::io::Result { + VsockListener::bind(port) +} + +/// Connect to a vsock port on the host (CID 2). +#[cfg(target_os = "linux")] +pub fn connect(port: u32) -> std::io::Result { + use std::mem; + use std::os::fd::FromRawFd; + + const AF_VSOCK: libc::c_int = 40; + const HOST_CID: u32 = 2; + + #[repr(C)] + struct sockaddr_vm { + svm_family: libc::sa_family_t, + svm_reserved1: u16, + svm_port: u32, + svm_cid: u32, + svm_zero: [u8; 4], + } + + unsafe { + let fd = libc::socket(AF_VSOCK, libc::SOCK_STREAM, 0); + if fd < 0 { + return Err(std::io::Error::last_os_error()); + } + let fd = OwnedFd::from_raw_fd(fd); + + let addr = sockaddr_vm { + svm_family: AF_VSOCK as u16, + svm_reserved1: 0, + svm_port: port, + svm_cid: HOST_CID, + svm_zero: [0; 4], + }; + + if libc::connect( + fd.as_raw_fd(), + &addr as *const sockaddr_vm as *const libc::sockaddr, + mem::size_of::() as libc::socklen_t, + ) < 0 + { + return Err(std::io::Error::last_os_error()); + } + + Ok(VsockStream { fd }) + } +} + +/// Connect stub for non-Linux platforms. +#[cfg(not(target_os = "linux"))] +pub fn connect(_port: u32) -> std::io::Result { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "vsock only supported on Linux", + )) +} diff --git a/crates/smolvm-cuda-codegen/Cargo.toml b/crates/smolvm-cuda-codegen/Cargo.toml new file mode 100644 index 0000000..1a8ce3f --- /dev/null +++ b/crates/smolvm-cuda-codegen/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "smolvm-cuda-codegen" +version = "1.5.2" +edition = "2021" +description = "Generates forward-to-host-lib marshaling (guest stubs + host dispatch) from a function spec" +license = "Apache-2.0" + +[[bin]] +name = "smolvm-cuda-codegen" +path = "src/main.rs" diff --git a/crates/smolvm-cuda-codegen/src/main.rs b/crates/smolvm-cuda-codegen/src/main.rs new file mode 100644 index 0000000..1c1b0b8 --- /dev/null +++ b/crates/smolvm-cuda-codegen/src/main.rs @@ -0,0 +1,839 @@ +//! Forward-to-host-lib marshaling generator. +//! +//! CUDA's library surface is hundreds of functions across cudart, cuBLAS, +//! cuDNN, … Hand-writing the guest stub + host dispatch + (de)serialization for +//! each is unmaintainable. This tool takes a compact per-function spec and emits +//! both sides of the marshaling over the generic `LibCall` transport +//! (`smolvm_cuda::proto::Op::LibCall`), so adding a function is a few lines of +//! spec, not code across five files. +//! +//! Parameter kinds capture the one thing a C signature can't: whether a pointer +//! is an opaque host **Handle**, a **DevPtr** (real device address), or a +//! **HostIn** scalar to read and ship by value. Everything else follows. +//! +//! Emits two files: +//! - `/_guest.rs` — `extern "C"` stubs, `include!`d by the guest shim. +//! - `/_host.rs` — dlsym table + `dispatch(func, args)`, `include!`d by +//! the host backend. +//! +//! This is a spec-driven generator; the specs below are hand-authored today, but +//! nothing stops a future front-end from emitting them from the CUDA headers. + +use std::fmt::Write as _; + +/// How one parameter is marshaled. +#[derive(Clone, Copy)] +enum Kind { + /// A machine scalar passed by value. `.0` is the Rust scalar type. + Scalar(&'static str), + /// An opaque host handle (cuBLAS/cuDNN handle, …) — pass the pointer + /// value by reference; the guest never dereferences it. + Handle, + /// A `cudaStream_t`. Streams are session-mapped small ids on the wire (the + /// host mints them at StreamCreate), so the host must translate through the + /// session stream table — NOT vh_resolve — or the raw id reaches the real + /// library as a pointer and crashes it. 0 and the legacy constants + /// (0x1/0x2) pass through. + Stream, + /// A real device address — pass by value. + DevPtr, + /// A host input scalar behind a pointer (cuBLAS alpha/beta) — read `*p` on + /// the guest, ship the value, rebuild a pointer to a local on the host. + HostInScalar(&'static str), + /// A host output scalar behind a pointer (cuDNN workspace size) — the host + /// fills a local and ships it back; the guest writes it through `*p`. `.0` is + /// the wire scalar type; the pointee type comes from the param's `cty`. + HostOutScalar(&'static str), +} + +/// The pointee of a `*mut T` C type (for building a local of the right type). +fn pointee(cty: &str) -> &str { + cty.strip_prefix("*mut ").unwrap_or(cty) +} + +/// Wire size in bytes of a scalar type name. +fn wire_size(t: &str) -> usize { + match t { + "u16" => 2, // __half forwarded bitwise + "i32" | "u32" | "f32" => 4, + _ => 8, + } +} + +/// One parameter: its name, the Rust type in the `extern "C"` signature, and how +/// to marshal it. +struct P { + name: &'static str, + cty: &'static str, + kind: Kind, +} +const fn p(name: &'static str, cty: &'static str, kind: Kind) -> P { + P { name, cty, kind } +} + +/// One function to generate. +struct Fun { + /// Exported symbol name (what the guest program links). + sym: &'static str, + /// Real host symbol to dlsym (usually the same). + real: &'static str, + params: Vec

, +} + +/// A library: an id (matches the guest shim's `LIB_*` constant) and its +/// functions, indexed in order (the index is the `func` id on the wire). +struct Lib { + name: &'static str, + id: u8, + /// Default soname to dlopen on the host if the env override is unset. + /// Sonames to probe via `open_host_lib`, most-specific last. + default_sos: &'static [&'static str], + funcs: Vec, +} + +fn cublas_spec() -> Lib { + use Kind::*; + let handle = || p("handle", "*mut c_void", Handle); + let i = |n| p(n, "c_int", Scalar("i32")); + // Column-major GEMM shared shape for S/D, differing only in element type. + let gemm = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + i("transa"), + i("transb"), + i("m"), + i("n"), + i("k"), + p("alpha", leak(format!("*const {ety}")), HostInScalar(ety)), + p("A", leak(format!("*const {ety}")), DevPtr), + i("lda"), + p("B", leak(format!("*const {ety}")), DevPtr), + i("ldb"), + p("beta", leak(format!("*const {ety}")), HostInScalar(ety)), + p("C", leak(format!("*mut {ety}")), DevPtr), + i("ldc"), + ], + }; + // Strided-batched GEMM (PyTorch's `bmm`): same as gemm plus per-operand + // strides (i64) and a batch count. + let s64 = |n| p(n, "i64", Scalar("i64")); + let gemm_strided = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + i("transa"), + i("transb"), + i("m"), + i("n"), + i("k"), + p("alpha", leak(format!("*const {ety}")), HostInScalar(ety)), + p("A", leak(format!("*const {ety}")), DevPtr), + i("lda"), + s64("strideA"), + p("B", leak(format!("*const {ety}")), DevPtr), + i("ldb"), + s64("strideB"), + p("beta", leak(format!("*const {ety}")), HostInScalar(ety)), + p("C", leak(format!("*mut {ety}")), DevPtr), + i("ldc"), + s64("strideC"), + i("batchCount"), + ], + }; + // ---- BLAS Level 1/2/3 (device-pointer ops with host scalars) ---- + // Reduction ops (dot/nrm2/asum/amax) are omitted deliberately: their + // result pointer is host- or device-resident depending on the handle's + // pointer mode, which this uniform marshaling can't disambiguate — they + // stay honest NOT_SUPPORTED stubs rather than risk a silently-wrong result. + let ic = i; // alias for closures below + let dp = |n, ety: &'static str, m: bool| { + p( + n, + leak(format!("*{} {ety}", if m { "mut" } else { "const" })), + DevPtr, + ) + }; + let hin = |n, ety: &'static str| p(n, leak(format!("*const {ety}")), HostInScalar(ety)); + let axpy = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + ic("n"), + hin("alpha", ety), + dp("x", ety, false), + ic("incx"), + dp("y", ety, true), + ic("incy"), + ], + }; + let scal = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + ic("n"), + hin("alpha", ety), + dp("x", ety, true), + ic("incx"), + ], + }; + let copyswap = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + ic("n"), + dp("x", ety, true), + ic("incx"), + dp("y", ety, true), + ic("incy"), + ], + }; + let gemv = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + ic("trans"), + ic("m"), + ic("n"), + hin("alpha", ety), + dp("A", ety, false), + ic("lda"), + dp("x", ety, false), + ic("incx"), + hin("beta", ety), + dp("y", ety, true), + ic("incy"), + ], + }; + let ger = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + ic("m"), + ic("n"), + hin("alpha", ety), + dp("x", ety, false), + ic("incx"), + dp("y", ety, false), + ic("incy"), + dp("A", ety, true), + ic("lda"), + ], + }; + let geam = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + ic("transa"), + ic("transb"), + ic("m"), + ic("n"), + hin("alpha", ety), + dp("A", ety, false), + ic("lda"), + hin("beta", ety), + dp("B", ety, false), + ic("ldb"), + dp("C", ety, true), + ic("ldc"), + ], + }; + let syrk = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + ic("uplo"), + ic("trans"), + ic("n"), + ic("k"), + hin("alpha", ety), + dp("A", ety, false), + ic("lda"), + hin("beta", ety), + dp("C", ety, true), + ic("ldc"), + ], + }; + let symm = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + ic("side"), + ic("uplo"), + ic("m"), + ic("n"), + hin("alpha", ety), + dp("A", ety, false), + ic("lda"), + dp("B", ety, false), + ic("ldb"), + hin("beta", ety), + dp("C", ety, true), + ic("ldc"), + ], + }; + let trsm = |sym, real, ety: &'static str| Fun { + sym, + real, + params: vec![ + handle(), + ic("side"), + ic("uplo"), + ic("trans"), + ic("diag"), + ic("m"), + ic("n"), + hin("alpha", ety), + dp("A", ety, false), + ic("lda"), + dp("B", ety, true), + ic("ldb"), + ], + }; + Lib { + name: "cublas", + id: 1, + default_sos: &["libcublas.so", "libcublas.so.13", "libcublas.so.12"], + funcs: vec![ + Fun { + sym: "cublasCreate_v2", + real: "cublasCreate_v2", + params: vec![p("handle_out", "*mut *mut c_void", Handle)], // out-handle: special-cased below + }, + Fun { + sym: "cublasDestroy_v2", + real: "cublasDestroy_v2", + params: vec![handle()], + }, + gemm("cublasSgemm_v2", "cublasSgemm_v2", "f32"), + // __half forwarded bitwise as u16 (the value is never interpreted + // guest-side; the host passes a pointer to the same 2 bytes). + gemm("cublasHgemm", "cublasHgemm", "u16"), + gemm("cublasDgemm_v2", "cublasDgemm_v2", "f64"), + gemm_strided( + "cublasSgemmStridedBatched", + "cublasSgemmStridedBatched", + "f32", + ), + gemm_strided( + "cublasDgemmStridedBatched", + "cublasDgemmStridedBatched", + "f64", + ), + // Stream + config that PyTorch sets up before a matmul. + Fun { + sym: "cublasSetStream_v2", + real: "cublasSetStream_v2", + params: vec![handle(), p("stream", "*mut c_void", Stream)], + }, + Fun { + sym: "cublasSetWorkspace_v2", + real: "cublasSetWorkspace_v2", + params: vec![ + handle(), + p("workspace", "*mut c_void", DevPtr), + p("size", "usize", Scalar("u64")), + ], + }, + Fun { + sym: "cublasSetMathMode", + real: "cublasSetMathMode", + params: vec![handle(), i("mode")], + }, + Fun { + sym: "cublasGetMathMode", + real: "cublasGetMathMode", + params: vec![handle(), p("mode", "*mut c_int", HostOutScalar("i32"))], + }, + Fun { + sym: "cublasGetProperty", + real: "cublasGetProperty", + params: vec![ + i("prop_type"), + p("value", "*mut c_int", HostOutScalar("i32")), + ], + }, + // cublasGemmEx — PyTorch's general GEMM. alpha/beta are the compute + // type (f32 for the common CUBLAS_COMPUTE_32F path). + Fun { + sym: "cublasGemmEx", + real: "cublasGemmEx", + params: vec![ + handle(), + i("transa"), + i("transb"), + i("m"), + i("n"), + i("k"), + p("alpha", "*const f32", HostInScalar("f32")), + p("A", "*const c_void", DevPtr), + i("Atype"), + i("lda"), + p("B", "*const c_void", DevPtr), + i("Btype"), + i("ldb"), + p("beta", "*const f32", HostInScalar("f32")), + p("C", "*mut c_void", DevPtr), + i("Ctype"), + i("ldc"), + i("computeType"), + i("algo"), + ], + }, + // cublasGemmStridedBatchedEx — PyTorch's mixed-precision batched GEMM + // (fp16/bf16 `bmm`). alpha/beta are the compute type (f32 common path). + Fun { + sym: "cublasGemmStridedBatchedEx", + real: "cublasGemmStridedBatchedEx", + params: vec![ + handle(), + i("transa"), + i("transb"), + i("m"), + i("n"), + i("k"), + p("alpha", "*const f32", HostInScalar("f32")), + p("A", "*const c_void", DevPtr), + i("Atype"), + i("lda"), + s64("strideA"), + p("B", "*const c_void", DevPtr), + i("Btype"), + i("ldb"), + s64("strideB"), + p("beta", "*const f32", HostInScalar("f32")), + p("C", "*mut c_void", DevPtr), + i("Ctype"), + i("ldc"), + s64("strideC"), + i("batchCount"), + i("computeType"), + i("algo"), + ], + }, + // cublasGemmBatchedEx — pointer-array batched GEMM (llama.cpp's + // ggml-cuda attention batching). The A/B/C arrays are arrays OF + // device pointers living in device memory, so they forward as + // plain device pointers. + Fun { + sym: "cublasGemmBatchedEx", + real: "cublasGemmBatchedEx", + params: vec![ + handle(), + i("transa"), + i("transb"), + i("m"), + i("n"), + i("k"), + p("alpha", "*const f32", HostInScalar("f32")), + p("Aarray", "*const c_void", DevPtr), + i("Atype"), + i("lda"), + p("Barray", "*const c_void", DevPtr), + i("Btype"), + i("ldb"), + p("beta", "*const f32", HostInScalar("f32")), + p("Carray", "*mut c_void", DevPtr), + i("Ctype"), + i("ldc"), + i("batchCount"), + i("computeType"), + i("algo"), + ], + }, + // BLAS Level 1/2/3 — appended (indices grow; existing ones stable). + axpy("cublasSaxpy_v2", "cublasSaxpy_v2", "f32"), + axpy("cublasDaxpy_v2", "cublasDaxpy_v2", "f64"), + scal("cublasSscal_v2", "cublasSscal_v2", "f32"), + scal("cublasDscal_v2", "cublasDscal_v2", "f64"), + copyswap("cublasScopy_v2", "cublasScopy_v2", "f32"), + copyswap("cublasDcopy_v2", "cublasDcopy_v2", "f64"), + copyswap("cublasSswap_v2", "cublasSswap_v2", "f32"), + copyswap("cublasDswap_v2", "cublasDswap_v2", "f64"), + gemv("cublasSgemv_v2", "cublasSgemv_v2", "f32"), + gemv("cublasDgemv_v2", "cublasDgemv_v2", "f64"), + ger("cublasSger_v2", "cublasSger_v2", "f32"), + ger("cublasDger_v2", "cublasDger_v2", "f64"), + geam("cublasSgeam", "cublasSgeam", "f32"), + geam("cublasDgeam", "cublasDgeam", "f64"), + syrk("cublasSsyrk_v2", "cublasSsyrk_v2", "f32"), + syrk("cublasDsyrk_v2", "cublasDsyrk_v2", "f64"), + symm("cublasSsymm_v2", "cublasSsymm_v2", "f32"), + symm("cublasDsymm_v2", "cublasDsymm_v2", "f64"), + trsm("cublasStrsm_v2", "cublasStrsm_v2", "f32"), + trsm("cublasDtrsm_v2", "cublasDtrsm_v2", "f64"), + ], + } +} + +/// An out-handle constructor (`fn(T* out)`), e.g. `cudnnCreateTensorDescriptor`. +fn create(name: &'static str) -> Fun { + Fun { + sym: name, + real: name, + params: vec![p("out", "*mut *mut c_void", Kind::Handle)], + } +} + +/// Core cuDNN legacy convolution path — the surface a basic conv forward needs. +/// Descriptors are opaque handles; tensors are device pointers; alpha/beta are +/// host scalars; the workspace size is a host output. +fn cudnn_spec() -> Lib { + use Kind::*; + let h = || p("handle", "*mut c_void", Handle); + let d = |n| p(n, "*mut c_void", Handle); + let dev = |n| p(n, "*mut c_void", DevPtr); + let i = |n| p(n, "c_int", Scalar("i32")); + let f = |sym, params| Fun { + sym, + real: sym, + params, + }; + Lib { + name: "cudnn", + id: 2, + default_sos: &["libcudnn.so", "libcudnn.so.9", "libcudnn.so.8"], + funcs: vec![ + create("cudnnCreate"), + f("cudnnDestroy", vec![h()]), + f( + "cudnnSetStream", + vec![h(), p("stream", "*mut c_void", Stream)], + ), + create("cudnnCreateTensorDescriptor"), + f( + "cudnnSetTensor4dDescriptor", + vec![d("t"), i("fmt"), i("dtype"), i("n"), i("c"), i("h"), i("w")], + ), + f("cudnnDestroyTensorDescriptor", vec![d("t")]), + create("cudnnCreateFilterDescriptor"), + f( + "cudnnSetFilter4dDescriptor", + vec![d("f"), i("dtype"), i("fmt"), i("k"), i("c"), i("h"), i("w")], + ), + f("cudnnDestroyFilterDescriptor", vec![d("f")]), + create("cudnnCreateConvolutionDescriptor"), + f( + "cudnnSetConvolution2dDescriptor", + vec![ + d("cv"), + i("pad_h"), + i("pad_w"), + i("u"), + i("v"), + i("dil_h"), + i("dil_w"), + i("mode"), + i("ctype"), + ], + ), + f("cudnnDestroyConvolutionDescriptor", vec![d("cv")]), + f( + "cudnnGetConvolutionForwardWorkspaceSize", + vec![ + h(), + d("x"), + d("w"), + d("cv"), + d("y"), + i("algo"), + p("size", "*mut usize", HostOutScalar("u64")), + ], + ), + f( + "cudnnConvolutionForward", + vec![ + h(), + p("alpha", "*const f32", HostInScalar("f32")), + d("xDesc"), + dev("x"), + d("wDesc"), + dev("w"), + d("convDesc"), + i("algo"), + dev("workspace"), + p("wsSize", "usize", Scalar("u64")), + p("beta", "*const f32", HostInScalar("f32")), + d("yDesc"), + dev("y"), + ], + ), + ], + } +} + +/// Leak a String to `&'static str` (generator is short-lived). +fn leak(s: String) -> &'static str { + Box::leak(s.into_boxed_str()) +} + +fn main() { + let out = std::env::args().nth(1).unwrap_or_else(|| ".".into()); + for lib in [cublas_spec(), cudnn_spec()] { + std::fs::write(format!("{out}/{}_guest.rs", lib.name), gen_guest(&lib)).unwrap(); + std::fs::write(format!("{out}/{}_host.rs", lib.name), gen_host(&lib)).unwrap(); + eprintln!( + "generated {} functions for lib '{}' (id {})", + lib.funcs.len(), + lib.name, + lib.id + ); + } +} + +/// Is this the create/out-handle special case (first param is `*mut *mut`)? +fn is_create(f: &Fun) -> bool { + f.params.len() == 1 && f.params[0].cty.starts_with("*mut *mut") +} + +fn gen_guest(lib: &Lib) -> String { + let mut s = String::new(); + let _ = writeln!( + s, + "// @generated by smolvm-cuda-codegen — do not edit. lib='{}' id={}", + lib.name, lib.id + ); + let _ = writeln!(s, "const LIB_ID: u8 = {};", lib.id); + for (idx, f) in lib.funcs.iter().enumerate() { + if is_create(f) { + // out-handle create: no args in; returns a handle (u64) in `out`. + let _ = writeln!( + s, + "#[no_mangle]\npub extern \"C\" fn {}(handle_out: *mut *mut c_void) -> c_int {{\n \ + if handle_out.is_null() {{ return 1; }}\n \ + // Fire-and-forget create: return a guest-assigned virtual id\n \ + // immediately; the host materializes the real descriptor and\n \ + // maps the id (see vh_resolve on the host side).\n \ + let vh = super::alloc_vhandle();\n \ + match with_client(|c| c.lib_call_deferred(LIB_ID, {idx}, vh.to_le_bytes().to_vec())) {{\n \ + Ok(()) => {{ unsafe {{ *handle_out = vh as *mut c_void }}; 0 }}\n \ + Err(_) => 1,\n }}\n}}", + f.sym + ); + continue; + } + // signature + let sig = f + .params + .iter() + .map(|p| format!("{}: {}", p.name, p.cty)) + .collect::>() + .join(", "); + let _ = writeln!( + s, + "#[no_mangle]\npub extern \"C\" fn {}({sig}) -> c_int {{", + f.sym + ); + let _ = writeln!(s, " let mut a: Vec = Vec::new();"); + for p in &f.params { + match p.kind { + Kind::Scalar(t) => { + let _ = writeln!( + s, + " a.extend_from_slice(&({} as {t}).to_le_bytes());", + p.name + ); + } + Kind::Handle | Kind::Stream | Kind::DevPtr => { + let _ = writeln!( + s, + " a.extend_from_slice(&({} as u64).to_le_bytes());", + p.name + ); + } + Kind::HostInScalar(t) => { + let _ = writeln!( + s, + " if {n}.is_null() {{ return 1; }}\n a.extend_from_slice(&(unsafe {{ *{n} }} as {t}).to_le_bytes());", + n = p.name + ); + } + Kind::HostOutScalar(_) => {} // output-only: nothing to send + } + } + // Output params (in declaration order) come back in `out`. + let outs: Vec<(&str, &str)> = f + .params + .iter() + .filter_map(|p| match p.kind { + Kind::HostOutScalar(t) => Some((p.name, t)), + _ => None, + }) + .collect(); + if outs.is_empty() { + let _ = writeln!( + s, + " // No output params: fire-and-forget (failures surface as sticky async errors).\n match with_client(|c| c.lib_call_deferred(LIB_ID, {idx}, a)) {{ Ok(()) => 0, Err(_) => 1 }}\n}}" + ); + } else { + let _ = writeln!( + s, + " match with_client(|c| c.lib_call(LIB_ID, {idx}, a)) {{\n Ok((st, out)) => {{ let mut o = 0usize;" + ); + for (name, t) in &outs { + let sz = wire_size(t); + let _ = writeln!( + s, + " if !{name}.is_null() && out.len() >= o + {sz} {{ unsafe {{ *{name} = {t}::from_le_bytes(out[o..o + {sz}].try_into().unwrap()) as _ }}; }} o += {sz};" + ); + } + let _ = writeln!( + s, + " let _ = o; st }}\n Err(_) => 1,\n }}\n}}" + ); + } + } + s +} + +fn gen_host(lib: &Lib) -> String { + let mut s = String::new(); + let _ = writeln!( + s, + "// @generated by smolvm-cuda-codegen — do not edit. lib='{}' id={}", + lib.name, lib.id + ); + // The resolved-symbols struct. + let _ = writeln!(s, "pub struct GenLib {{ _lib: Library,"); + for f in &lib.funcs { + let cargs = f + .params + .iter() + .map(|p| p.cty) + .collect::>() + .join(", "); + let _ = writeln!( + s, + " f_{}: unsafe extern \"C\" fn({cargs}) -> c_int,", + f.real + ); + } + let _ = writeln!(s, "}}"); + // Loader. + let _ = writeln!( + s, + "impl GenLib {{\n pub fn load() -> Result {{\n unsafe {{\n let lib = super::open_host_lib(\"SMOLVM_{}_LIB\", &[{}])?;\n Ok(GenLib {{", + lib.name.to_uppercase(), + lib.default_sos + .iter() + .map(|s| format!("\"{s}\"")) + .collect::>() + .join(", ") + ); + for f in &lib.funcs { + let _ = writeln!( + s, + " f_{r}: sym(&lib, b\"{r}\\0\")?,", + r = f.real + ); + } + let _ = writeln!( + s, + " _lib: lib,\n }})\n }}\n }}" + ); + // Dispatch. + let _ = writeln!( + s, + " pub fn dispatch(&self, func: u16, args: &[u8], __vh: &mut std::collections::HashMap, __streams: &std::collections::HashMap) -> (i32, Vec) {{\n let mut __c = GenCur {{ b: args, p: 0 }};\n let _ = __streams;\n match func {{" + ); + for (idx, f) in lib.funcs.iter().enumerate() { + if is_create(f) { + let _ = writeln!( + s, + " {idx} => {{ let mut h: *mut c_void = std::ptr::null_mut(); let st = unsafe {{ (self.f_{})(&mut h) }}; if st == 0 && args.len() >= 8 {{ let id = u64::from_le_bytes(args[..8].try_into().unwrap()); if id & super::VHANDLE_TAG != 0 {{ __vh.insert(id, h as u64); }} }} (st, (h as u64).to_le_bytes().to_vec()) }}", + f.real + ); + continue; + } + let mut binds = String::new(); + let mut call = Vec::new(); + let mut outs = String::new(); // append out-params after the call + for p in &f.params { + match p.kind { + Kind::Scalar(t) => { + let _ = writeln!(binds, " let {} = __c.{t}();", p.name); + call.push(format!("{} as {}", p.name, p.cty)); + } + Kind::Handle => { + // May be a guest-assigned virtual id — resolve to the real + // pointer (untagged values pass through). Destroy calls + // also retire the mapping so the table doesn't grow. + if f.sym.contains("Destroy") { + let _ = writeln!( + binds, + " let __raw = __c.u64();\n let {} = super::vh_resolve(__vh, __raw) as {};\n if __raw & super::VHANDLE_TAG != 0 {{ __vh.remove(&__raw); }}", + p.name, p.cty + ); + } else { + let _ = writeln!( + binds, + " let {} = super::vh_resolve(__vh, __c.u64()) as {};", + p.name, p.cty + ); + } + call.push(p.name.to_string()); + } + Kind::Stream => { + let _ = writeln!( + binds, + " let {} = super::stream_resolve(__streams, __c.u64()) as {};", + p.name, p.cty + ); + call.push(p.name.to_string()); + } + Kind::DevPtr => { + let _ = writeln!( + binds, + " let {} = __c.u64() as {};", + p.name, p.cty + ); + call.push(p.name.to_string()); + } + Kind::HostInScalar(t) => { + let _ = writeln!(binds, " let {}_v = __c.{t}();", p.name); + call.push(format!("&{}_v", p.name)); + } + Kind::HostOutScalar(t) => { + let pt = pointee(p.cty); + let _ = writeln!( + binds, + " let mut {}_v: {pt} = 0 as {pt};", + p.name + ); + call.push(format!("&mut {}_v", p.name)); + let _ = writeln!( + outs, + " out.extend_from_slice(&({}_v as {t}).to_le_bytes());", + p.name + ); + } + } + } + let _ = writeln!( + s, + " {idx} => {{\n{binds} let mut out = Vec::new();\n let st = unsafe {{ (self.f_{})({}) }};\n{outs} (st, out)\n }}", + f.real, + call.join(", ") + ); + } + let _ = writeln!( + s, + " _ => (super::super::CUDA_ERROR_NOT_FOUND, Vec::new()),\n }}\n }}\n}}" + ); + // A tiny cursor with the scalar readers the generated code uses. + let _ = writeln!( + s, + "struct GenCur<'a> {{ b: &'a [u8], p: usize }}\nimpl GenCur<'_> {{\n fn take(&mut self, n: usize) -> [u8; 8] {{ let mut o = [0u8; 8]; let end = (self.p + n).min(self.b.len()); o[..end - self.p].copy_from_slice(&self.b[self.p..end]); self.p = end; o }}\n fn u16(&mut self) -> u16 {{ u16::from_le_bytes(self.take(2)[..2].try_into().unwrap()) }}\n fn i32(&mut self) -> i32 {{ i32::from_le_bytes(self.take(4)[..4].try_into().unwrap()) }}\n fn i64(&mut self) -> i64 {{ i64::from_le_bytes(self.take(8)) }}\n fn u64(&mut self) -> u64 {{ u64::from_le_bytes(self.take(8)) }}\n fn f32(&mut self) -> f32 {{ f32::from_le_bytes(self.take(4)[..4].try_into().unwrap()) }}\n fn f64(&mut self) -> f64 {{ f64::from_le_bytes(self.take(8)) }}\n}}" + ); + s +} diff --git a/crates/smolvm-cuda-guest/Cargo.toml b/crates/smolvm-cuda-guest/Cargo.toml new file mode 100644 index 0000000..9dacb71 --- /dev/null +++ b/crates/smolvm-cuda-guest/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "smolvm-cuda-guest" +version = "1.5.2" +edition = "2021" +description = "Guest-side CUDA-over-vsock runner for smolvm microVMs (Linux)" +license = "Apache-2.0" + +[[bin]] +name = "smolvm-cuda-run" +path = "src/main.rs" + +[dependencies] +smolvm-cuda = { path = "../smolvm-cuda", default-features = false } + +# vsock is Linux-only; the guest binary is meaningful only inside a Linux +# microVM. Off Linux the binary compiles as a stub so the workspace still builds. +[target.'cfg(target_os = "linux")'.dependencies] +vsock = "0.5" diff --git a/crates/smolvm-cuda-guest/src/main.rs b/crates/smolvm-cuda-guest/src/main.rs new file mode 100644 index 0000000..4d123f7 --- /dev/null +++ b/crates/smolvm-cuda-guest/src/main.rs @@ -0,0 +1,125 @@ +//! Guest-side CUDA-over-vsock runner. +//! +//! Connects out to the host CUDA server over `AF_VSOCK` (host CID 2, the smolvm +//! CUDA port) and drives a real Driver-API workload through the RPC client: +//! load an arbitrary PTX module, look up a named kernel, allocate device +//! buffers, copy in, launch, synchronize, copy out, and verify. This exercises +//! the *generalized* protocol (arbitrary module + named launch), not a baked-in +//! single operation — the same path a real driver-API program takes. +//! +//! On success it prints `SMOLVM-CUDA-OK`; on any failure it prints the error and +//! exits non-zero so a `machine exec` caller sees the result. + +/// smolvm's reserved guest→host CUDA vsock port (mirrors `smolvm_protocol::ports::CUDA`). +#[cfg(target_os = "linux")] +const CUDA_PORT: u32 = 7000; +/// `VMADDR_CID_HOST` — the host end of the vsock. +#[cfg(target_os = "linux")] +const HOST_CID: u32 = 2; + +/// A hand-written vector-add kernel. `.target sm_52` JITs forward-compatibly to +/// any newer GPU (e.g. the RTX 3070's sm_86). No nvcc required. +#[cfg(target_os = "linux")] +const VECADD_PTX: &str = r#".version 7.0 +.target sm_52 +.address_size 64 +.visible .entry vecadd(.param .u64 a, .param .u64 b, .param .u64 c, .param .u32 n) +{ .reg .pred %p<2>; .reg .f32 %f<4>; .reg .b32 %r<6>; .reg .b64 %rd<11>; + ld.param.u64 %rd1,[a]; ld.param.u64 %rd2,[b]; ld.param.u64 %rd3,[c]; ld.param.u32 %r2,[n]; + mov.u32 %r3,%ntid.x; mov.u32 %r4,%ctaid.x; mov.u32 %r5,%tid.x; mad.lo.s32 %r1,%r4,%r3,%r5; + setp.ge.u32 %p1,%r1,%r2; @%p1 bra $E; + cvta.to.global.u64 %rd4,%rd1; cvta.to.global.u64 %rd5,%rd2; cvta.to.global.u64 %rd6,%rd3; + mul.wide.u32 %rd7,%r1,4; add.s64 %rd8,%rd4,%rd7; add.s64 %rd9,%rd5,%rd7; add.s64 %rd10,%rd6,%rd7; + ld.global.f32 %f1,[%rd8]; ld.global.f32 %f2,[%rd9]; add.f32 %f3,%f1,%f2; st.global.f32 [%rd10],%f3; +$E: ret; } +"#; + +#[cfg(target_os = "linux")] +fn run() -> Result<(), Box> { + use smolvm_cuda::client::Client; + use vsock::VsockStream; + + let stream = VsockStream::connect_with_cid_port(HOST_CID, CUDA_PORT) + .map_err(|e| format!("connect vsock {HOST_CID}:{CUDA_PORT}: {e}"))?; + let mut cu = Client::new(stream); + + cu.init()?; + let count = cu.device_get_count()?; + if count < 1 { + return Err("no CUDA devices reported by host".into()); + } + let name = cu.device_get_name(0)?; + let _ctx = cu.ctx_create(0)?; + println!("smolvm-cuda: host device 0 = {name}"); + + // Load an arbitrary module + look up the kernel by name (the general path). + let module = cu.module_load_data(VECADD_PTX.as_bytes())?; + let func = cu.module_get_function(module, "vecadd")?; + + let n: usize = 1024; + let bytes = (n * 4) as u64; + let a: Vec = (0..n).map(|i| i as f32).collect(); + let b: Vec = (0..n).map(|i| (2 * i) as f32).collect(); + + let da = cu.mem_alloc(bytes)?; + let db = cu.mem_alloc(bytes)?; + let dc = cu.mem_alloc(bytes)?; + cu.memcpy_htod(da, as_bytes(&a), 0)?; + cu.memcpy_htod(db, as_bytes(&b), 0)?; + + // kernelParams: one little-endian blob per arg, in declaration order. + let params = vec![ + da.to_le_bytes().to_vec(), + db.to_le_bytes().to_vec(), + dc.to_le_bytes().to_vec(), + (n as u32).to_le_bytes().to_vec(), + ]; + let block = 256u32; + let grid = (n as u32).div_ceil(block); + cu.launch_kernel(func, [grid, 1, 1], [block, 1, 1], 0, 0, ¶ms)?; + cu.ctx_synchronize()?; + + let out = cu.memcpy_dtoh(dc, bytes, 0)?; + let c: Vec = out + .chunks_exact(4) + .map(|p| f32::from_le_bytes(p.try_into().unwrap())) + .collect(); + + for i in 0..n { + let expect = a[i] + b[i]; + if (c[i] - expect).abs() > 1e-3 { + return Err(format!("mismatch at {i}: got {} want {expect}", c[i]).into()); + } + } + cu.mem_free(da)?; + cu.mem_free(db)?; + cu.mem_free(dc)?; + println!( + "smolvm-cuda: vecadd n={n} verified (c[1]={}, c[{}]={})", + c[1], + n - 1, + c[n - 1] + ); + println!("SMOLVM-CUDA-OK"); + Ok(()) +} + +#[cfg(target_os = "linux")] +fn as_bytes(v: &[f32]) -> &[u8] { + // SAFETY: f32 has no padding/invalid bit patterns; reading its bytes is sound. + unsafe { std::slice::from_raw_parts(v.as_ptr() as *const u8, std::mem::size_of_val(v)) } +} + +#[cfg(target_os = "linux")] +fn main() { + if let Err(e) = run() { + eprintln!("smolvm-cuda: ERROR: {e}"); + std::process::exit(1); + } +} + +#[cfg(not(target_os = "linux"))] +fn main() { + eprintln!("smolvm-cuda-run runs only inside a Linux guest microVM"); + std::process::exit(1); +} diff --git a/crates/smolvm-cuda-shim/Cargo.toml b/crates/smolvm-cuda-shim/Cargo.toml new file mode 100644 index 0000000..b846076 --- /dev/null +++ b/crates/smolvm-cuda-shim/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "smolvm-cuda-shim" +version = "1.5.2" +edition = "2021" +description = "Guest-side libcuda.so.1 drop-in: CUDA Driver API over smolvm's vsock RPC" +license = "Apache-2.0" + +[lib] +name = "cuda" +crate-type = ["cdylib"] + +[dependencies] +smolvm-cuda = { path = "../smolvm-cuda", default-features = false } + +# vsock is the production transport (guest → host CID 2). Off Linux the crate +# still compiles (tcp/unix transports only) so the workspace builds everywhere. +[target.'cfg(target_os = "linux")'.dependencies] +vsock = "0.5" diff --git a/crates/smolvm-cuda-shim/build.rs b/crates/smolvm-cuda-shim/build.rs new file mode 100644 index 0000000..20380ee --- /dev/null +++ b/crates/smolvm-cuda-shim/build.rs @@ -0,0 +1,10 @@ +fn main() { + // The library must announce itself as libcuda.so.1 so the dynamic linker + // satisfies programs linked against (or dlopen-ing) the real driver's + // soname. The build artifact is libcuda.so; installers symlink/rename to + // libcuda.so.1. + let target = std::env::var("TARGET").unwrap_or_default(); + if target.contains("linux") { + println!("cargo:rustc-cdylib-link-arg=-Wl,-soname,libcuda.so.1"); + } +} diff --git a/crates/smolvm-cuda-shim/src/driver_stubs.rs b/crates/smolvm-cuda-shim/src/driver_stubs.rs new file mode 100644 index 0000000..f7d51e5 --- /dev/null +++ b/crates/smolvm-cuda-shim/src/driver_stubs.rs @@ -0,0 +1,2555 @@ +// @generated by smolvm nm-based stub generation — do not edit. +//! Auto-generated: every real libcuda.so.1 driver function we don't yet +//! forward, exported as a NOT_SUPPORTED stub so the full driver ABI resolves +//! at load (PyTorch's NVIDIA math libs reference many; none of these are on +//! the basic init/compute path — those are the hand-written forwarders). +#![allow(non_snake_case)] +use std::os::raw::c_int; +#[no_mangle] +pub extern "C" fn cuArray3DCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArray3DCreate_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArray3DGetDescriptor() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArray3DGetDescriptor_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArrayCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArrayCreate_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArrayDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArrayGetDescriptor() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArrayGetDescriptor_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArrayGetMemoryRequirements() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArrayGetPlane() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuArrayGetSparseProperties() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCheckpointProcessCheckpoint() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCheckpointProcessGetRestoreThreadId() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCheckpointProcessGetState() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCheckpointProcessLock() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCheckpointProcessRestore() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCheckpointProcessUnlock() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCoredumpDeregisterCompleteCallback() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCoredumpDeregisterStartCallback() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCoredumpGetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCoredumpGetAttributeGlobal() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCoredumpRegisterCompleteCallback() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCoredumpRegisterStartCallback() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCoredumpSetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCoredumpSetAttributeGlobal() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxAttach() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxCreate_v3() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxCreate_v4() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxDetach() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxDisablePeerAccess() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxEnablePeerAccess() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxFromGreenCtx() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxGetApiVersion() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxGetCacheConfig() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxGetDevice_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxGetDevResource() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxGetExecAffinity() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxGetFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxGetId() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxGetSharedMemConfig() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxGetStreamPriorityRange() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxPopCurrent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxPushCurrent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxRecordEvent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxResetPersistingL2Cache() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxSetCacheConfig() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxSetFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxSetSharedMemConfig() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxSynchronize_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuCtxWaitEvent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDestroyExternalMemory() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDestroyExternalSemaphore() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceCanAccessPeer() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetByPCIBusId() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetDefaultMemPool() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetDevResource() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetExecAffinitySupport() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetGraphMemAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetHostAtomicCapabilities() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetLuid() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetMemPool() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetNvSciSyncAttributes() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetP2PAtomicCapabilities() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetP2PAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetPCIBusId() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetProperties() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGetTexture1DLinearMaxWidth() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceGraphMemTrim() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDevicePrimaryCtxReset() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDevicePrimaryCtxReset_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDevicePrimaryCtxSetFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceRegisterAsyncNotification() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceSetGraphMemAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceSetMemPool() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceTotalMem() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDeviceUnregisterAsyncNotification() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDevResourceGenerateDesc() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDevSmResourceSplit() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDevSmResourceSplitByCount() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuDriverGetGpuCodeIsaVersion() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEGLApiInit() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEGLStreamConsumerAcquireFrame() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEGLStreamConsumerConnect() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEGLStreamConsumerConnectWithFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEGLStreamConsumerDisconnect() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEGLStreamConsumerReleaseFrame() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEGLStreamProducerConnect() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEGLStreamProducerDisconnect() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEGLStreamProducerPresentFrame() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEGLStreamProducerReturnFrame() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEventDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEventElapsedTime_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEventRecord_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEventRecordWithFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuEventRecordWithFlags_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuExternalMemoryGetMappedBuffer() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuExternalMemoryGetMappedMipmappedArray() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuFlushGPUDirectRDMAWrites() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuFuncGetModule() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuFuncGetName() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuFuncGetParamCount() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuFuncGetParamInfo() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuFuncIsLoaded() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuFuncLoad() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuFuncSetBlockShape() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuFuncSetSharedMemConfig() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuFuncSetSharedSize() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLCtxCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLCtxCreate_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLGetDevices() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLGetDevices_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLInit() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLMapBufferObject() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLMapBufferObjectAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLMapBufferObjectAsync_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLMapBufferObjectAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLMapBufferObject_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLMapBufferObject_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLRegisterBufferObject() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLSetBufferObjectMapFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLUnmapBufferObject() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLUnmapBufferObjectAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGLUnregisterBufferObject() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddBatchMemOpNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddChildGraphNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddDependencies() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddDependencies_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddEmptyNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddEventRecordNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddEventWaitNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddExternalSemaphoresSignalNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddExternalSemaphoresWaitNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddHostNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddKernelNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddKernelNode_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddMemAllocNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddMemcpyNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddMemFreeNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddMemsetNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphAddNode_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphBatchMemOpNodeGetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphBatchMemOpNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphChildGraphNodeGetGraph() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphClone() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphConditionalHandleCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphDebugDotPrint() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphDestroyNode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphEventRecordNodeGetEvent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphEventRecordNodeSetEvent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphEventWaitNodeGetEvent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphEventWaitNodeSetEvent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecBatchMemOpNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecChildGraphNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecEventRecordNodeSetEvent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecEventWaitNodeSetEvent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecExternalSemaphoresSignalNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecExternalSemaphoresWaitNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecGetFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecGetId() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecHostNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecKernelNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecKernelNodeSetParams_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecMemcpyNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecMemsetNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecUpdate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExecUpdate_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExternalSemaphoresSignalNodeGetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExternalSemaphoresSignalNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExternalSemaphoresWaitNodeGetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphExternalSemaphoresWaitNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphGetEdges() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphGetEdges_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphGetId() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphGetNodes() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphGetRootNodes() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphHostNodeGetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphHostNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsEGLRegisterImage() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsGLRegisterBuffer() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsGLRegisterImage() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsMapResources() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsMapResources_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsResourceGetMappedEglFrame() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsResourceGetMappedMipmappedArray() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsResourceGetMappedPointer() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsResourceGetMappedPointer_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsResourceSetMapFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsResourceSetMapFlags_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsSubResourceGetMappedArray() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsUnmapResources() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsUnmapResources_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsUnregisterResource() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsVDPAURegisterOutputSurface() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphicsVDPAURegisterVideoSurface() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphInstantiate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphInstantiate_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphInstantiateWithFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphInstantiateWithParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphInstantiateWithParams_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphKernelNodeCopyAttributes() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphKernelNodeGetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphKernelNodeGetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphKernelNodeGetParams_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphKernelNodeSetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphKernelNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphKernelNodeSetParams_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphLaunch() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphLaunch_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphMemAllocNodeGetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphMemcpyNodeGetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphMemcpyNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphMemFreeNodeGetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphMemsetNodeGetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphMemsetNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeFindInClone() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeGetContainingGraph() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeGetDependencies() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeGetDependencies_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeGetDependentNodes() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeGetDependentNodes_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeGetEnabled() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeGetLocalId() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeGetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeGetToolsId() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeGetType() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeSetEnabled() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphNodeSetParams() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphReleaseUserObject() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphRemoveDependencies() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphRemoveDependencies_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphRetainUserObject() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphUpload() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGraphUpload_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGreenCtxCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGreenCtxDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGreenCtxGetDevResource() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGreenCtxGetId() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGreenCtxRecordEvent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGreenCtxStreamCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuGreenCtxWaitEvent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuImportExternalMemory() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuImportExternalSemaphore() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuIpcCloseMemHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuIpcGetEventHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuIpcGetMemHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuIpcOpenEventHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuIpcOpenMemHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuIpcOpenMemHandle_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuKernelGetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuKernelGetFunction() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuKernelGetLibrary() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuKernelGetName() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuKernelGetParamCount() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuKernelGetParamInfo() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuKernelSetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuKernelSetCacheConfig() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLaunch() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLaunchCooperativeKernelMultiDevice() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLaunchCooperativeKernel_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLaunchGrid() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLaunchGridAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLaunchHostFunc() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLaunchHostFunc_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLaunchHostFunc_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLaunchHostFunc_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLaunchKernel_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLibraryEnumerateKernels() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLibraryGetGlobal() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLibraryGetKernel() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLibraryGetKernelCount() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLibraryGetManaged() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLibraryGetModule() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLibraryGetUnifiedFunction() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLibraryLoadData() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLibraryLoadFromFile() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLibraryUnload() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLinkAddData() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLinkAddFile() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLinkAddFile_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLinkCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLinkDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointAddDevice() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointBindAddr() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointBindMem() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointExport() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointGetLimits() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointIdRelease() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointIdReserve() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointImport() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointQuery() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogicalEndpointUnbind() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogsCurrent() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogsDumpToFile() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogsDumpToMemory() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogsRegisterCallback() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuLogsUnregisterCallback() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAdvise() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAdvise_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAlloc() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAllocAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAllocAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAllocFromPoolAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAllocFromPoolAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAllocHost() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAllocHost_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAllocManaged() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAllocPitch() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemAllocPitch_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemBatchDecompressAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemBatchDecompressAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy2D() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy2DAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy2DAsync_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy2DAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy2DUnaligned() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy2DUnaligned_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy2DUnaligned_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy2D_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy2D_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3D() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DAsync_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DBatchAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DBatchAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DBatchAsync_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DBatchAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DPeer() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DPeerAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DPeerAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DPeer_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3D_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3D_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DWithAttributesAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy3DWithAttributesAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoA() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoA_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoA_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoD() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoD_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoD_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoH() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoHAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoHAsync_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoHAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoH_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyAtoH_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyBatchAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyBatchAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyBatchAsync_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyBatchAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoA() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoA_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoA_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoD() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoDAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoDAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoD_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoH() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoHAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoHAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyDtoH_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyHtoA() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyHtoAAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyHtoAAsync_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyHtoAAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyHtoA_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyHtoA_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyHtoD() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyHtoDAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyHtoDAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyHtoD_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyPeer() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyPeerAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyPeerAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyPeer_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpy_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyWithAttributesAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemcpyWithAttributesAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemDiscardAndPrefetchBatchAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemDiscardAndPrefetchBatchAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemDiscardBatchAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemDiscardBatchAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemExportToShareableHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemFree() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemFreeAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemFreeAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemFreeHost() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemGetAccess() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemGetAddressRange() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemGetAddressRange_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemGetAllocationPropertiesFromHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemGetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemGetAttribute_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemGetDefaultMemPool() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemGetHandleForAddressRange() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemGetInfo() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemGetMemPool() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemHostAlloc() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemHostGetDevicePointer() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemHostGetDevicePointer_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemHostGetFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemHostRegister() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemHostRegister_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemHostUnregister() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemImportFromShareableHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemMapArrayAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemMapArrayAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolExportPointer() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolExportToShareableHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolGetAccess() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolGetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolImportFromShareableHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolImportPointer() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolSetAccess() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolSetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPoolTrimTo() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPrefetchAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPrefetchAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPrefetchAsync_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPrefetchAsync_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPrefetchBatchAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemPrefetchBatchAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemRangeGetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemRangeGetAttributes() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemRetainAllocationHandle() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD16() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD16Async() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD16Async_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD16_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD16_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D16() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D16Async() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D16Async_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D16_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D16_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D32() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D32Async() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D32Async_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D32_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D32_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D8() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D8Async() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D8Async_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D8_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD2D8_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD32() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD32Async() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD32Async_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD32_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD32_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD8() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD8Async() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD8Async_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemsetD8_v2_ptds() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMemSetMemPool() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMipmappedArrayCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMipmappedArrayDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMipmappedArrayGetLevel() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMipmappedArrayGetMemoryRequirements() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMipmappedArrayGetSparseProperties() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuModuleEnumerateFunctions() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuModuleGetFunctionCount() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuModuleGetGlobal() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuModuleGetGlobal_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuModuleGetLoadingMode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuModuleGetSurfRef() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuModuleGetTexRef() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMulticastAddDevice() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMulticastBindAddr() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMulticastBindAddr_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMulticastBindMem() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMulticastBindMem_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMulticastCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMulticastGetGranularity() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuMulticastUnbind() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuOccupancyAvailableDynamicSMemPerBlock() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuOccupancyMaxPotentialBlockSize() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuOccupancyMaxPotentialBlockSizeWithFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuOccupancyMaxPotentialClusterSize() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuParamSetf() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuParamSeti() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuParamSetSize() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuParamSetTexRef() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuParamSetv() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuPointerSetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuProfilerInitialize() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuProfilerStart() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuProfilerStop() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuSignalExternalSemaphoresAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuSignalExternalSemaphoresAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamAddCallback() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamAddCallback_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamAttachMemAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamAttachMemAsync_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBatchMemOp() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBatchMemOp_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBatchMemOp_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBatchMemOp_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBeginCapture() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBeginCapture_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBeginCaptureToCig() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBeginCaptureToCig_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBeginCaptureToGraph() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBeginCaptureToGraph_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBeginCapture_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBeginCapture_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBeginRecaptureToGraph() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamBeginRecaptureToGraph_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamCopyAttributes() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamCopyAttributes_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamEndCapture() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamEndCapture_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamEndCaptureToCig() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamEndCaptureToCig_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetAttribute_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetCaptureInfo() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetCaptureInfo_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetCaptureInfo_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetCaptureInfo_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetCaptureInfo_v3() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetCaptureInfo_v3_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetCtx() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetCtx_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetCtx_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetCtx_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetDevice() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetDevice_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetDevResource() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetDevResource_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetFlags_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetGreenCtx() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetId() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetId_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetPriority() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamGetPriority_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamIsCapturing() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamIsCapturing_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamQuery_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamSetAttribute() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamSetAttribute_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamSynchronize_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamUpdateCaptureDependencies() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamUpdateCaptureDependencies_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamUpdateCaptureDependencies_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamUpdateCaptureDependencies_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWaitEvent_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWaitValue32() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWaitValue32_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWaitValue32_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWaitValue32_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWaitValue64() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWaitValue64_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWaitValue64_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWaitValue64_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWriteValue32() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWriteValue32_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWriteValue32_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWriteValue32_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWriteValue64() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWriteValue64_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWriteValue64_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuStreamWriteValue64_v2_ptsz() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuSubgridCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuSubgridWorkerGridCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuSubgridWorksetCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuSurfObjectCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuSurfObjectDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuSurfObjectGetResourceDesc() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuSurfRefGetArray() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuSurfRefSetArray() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTensorMapEncodeIm2col() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTensorMapEncodeIm2colWide() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTensorMapReplaceAddress() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexObjectCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexObjectDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexObjectGetResourceDesc() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexObjectGetResourceViewDesc() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexObjectGetTextureDesc() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefDestroy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetAddress() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetAddressMode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetAddress_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetArray() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetBorderColor() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetFilterMode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetFormat() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetMaxAnisotropy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetMipmapFilterMode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetMipmapLevelBias() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetMipmapLevelClamp() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefGetMipmappedArray() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetAddress() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetAddress2D() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetAddress2D_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetAddress2D_v3() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetAddressMode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetAddress_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetArray() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetBorderColor() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetFilterMode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetFlags() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetFormat() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetMaxAnisotropy() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetMipmapFilterMode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetMipmapLevelBias() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetMipmapLevelClamp() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuTexRefSetMipmappedArray() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuThreadExchangeStreamCaptureMode() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuUserObjectCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuUserObjectRelease() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuUserObjectRetain() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuVDPAUCtxCreate() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuVDPAUCtxCreate_v2() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuVDPAUGetDevice() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuWaitExternalSemaphoresAsync() -> c_int { + 801 +} +#[no_mangle] +pub extern "C" fn cuWaitExternalSemaphoresAsync_ptsz() -> c_int { + 801 +} diff --git a/crates/smolvm-cuda-shim/src/lib.rs b/crates/smolvm-cuda-shim/src/lib.rs new file mode 100644 index 0000000..b719dd0 --- /dev/null +++ b/crates/smolvm-cuda-shim/src/lib.rs @@ -0,0 +1,1491 @@ +//! Drop-in `libcuda.so.1` for smolvm guests: the CUDA Driver API C ABI, +//! implemented by marshaling every call over smolvm's CUDA RPC to the host. +//! +//! Install this library at `/usr/lib/libcuda.so.1` inside a guest (or point +//! `LD_LIBRARY_PATH` at it) and unmodified Driver-API programs — including the +//! CUDA runtime's dlopen of the driver — run their GPU work on the host GPU +//! with no code changes. +//! +//! Transport is chosen by `SMOLVM_CUDA_RPC`: +//! * unset / `vsock` — AF_VSOCK to host CID 2, port 7000 (production, in-guest) +//! * `tcp:HOST:PORT` — TCP (host-side testing against a loopback server) +//! * `unix:/path` — AF_UNIX (host-side testing against the real cuda.sock) +//! +//! Semantics notes: +//! * Handles returned to the app (contexts, modules, functions, streams, +//! events) are the server's opaque ids cast to pointers — never host +//! addresses. +//! * Everything executes synchronously on the host; the `*Async` entry points +//! complete before returning, which the CUDA API permits (an implementation +//! may be more synchronous than requested). Stream/event query entry points +//! therefore always report "complete". +//! * The context-current APIs are process-global, not per-thread. Programs +//! that juggle distinct contexts on different threads concurrently are out +//! of scope for this shim. +//! * `cuLaunchKernel` serializes `kernelParams` using per-parameter sizes the +//! host extracts from the loaded module (`cuFuncGetParamInfo`, CUDA 12.4+). + +// These are C ABI entry points: the caller is C code holding the CUDA Driver +// API contract (valid pointers or NULL, checked where the API allows NULL). +// Marking them `unsafe fn` would not change what C callers can do. +#![allow(clippy::not_unsafe_ptr_arg_deref)] + +use smolvm_cuda::client::{Client, CudaRpcError}; +mod driver_stubs; +use std::collections::HashMap; +use std::ffi::{c_char, c_int, c_uint, c_void, CStr}; +use std::io::{Read, Write}; +use std::sync::Mutex; + +// ---- CUresult codes the shim produces locally ------------------------------- + +const CUDA_SUCCESS: c_int = 0; +const CUDA_ERROR_INVALID_VALUE: c_int = 1; +const CUDA_ERROR_NOT_INITIALIZED: c_int = 3; +const CUDA_ERROR_NO_DEVICE: c_int = 100; +const CUDA_ERROR_INVALID_CONTEXT: c_int = 201; +const CUDA_ERROR_NOT_FOUND: c_int = 500; +const CUDA_ERROR_NOT_SUPPORTED: c_int = 801; +const CUDA_ERROR_UNKNOWN: c_int = 999; + +/// The CUDA version this shim reports for its own API surface. +const SHIM_CUDA_VERSION: c_int = 12040; + +// ---- transport --------------------------------------------------------------- + +/// One concrete byte stream to the host CUDA server. `Bridged` owns no +/// socket: the client routes through the runtime shim's connection instead +/// and never touches its stream. +enum Stream { + #[cfg(target_os = "linux")] + Vsock(vsock::VsockStream), + Tcp(std::net::TcpStream), + #[cfg(unix)] + Unix(std::os::unix::net::UnixStream), + Bridged, +} + +impl Read for Stream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + #[cfg(target_os = "linux")] + Stream::Vsock(s) => s.read(buf), + Stream::Tcp(s) => s.read(buf), + #[cfg(unix)] + Stream::Unix(s) => s.read(buf), + Stream::Bridged => Err(std::io::Error::other("bridged client has no stream")), + } + } +} +impl Write for Stream { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + #[cfg(target_os = "linux")] + Stream::Vsock(s) => s.write(buf), + Stream::Tcp(s) => s.write(buf), + #[cfg(unix)] + Stream::Unix(s) => s.write(buf), + Stream::Bridged => Err(std::io::Error::other("bridged client has no stream")), + } + } + fn flush(&mut self) -> std::io::Result<()> { + match self { + #[cfg(target_os = "linux")] + Stream::Vsock(s) => s.flush(), + Stream::Tcp(s) => s.flush(), + #[cfg(unix)] + Stream::Unix(s) => s.flush(), + Stream::Bridged => Ok(()), + } + } +} + +fn connect() -> Result { + let spec = std::env::var("SMOLVM_CUDA_RPC").unwrap_or_default(); + if let Some(addr) = spec.strip_prefix("tcp:") { + return std::net::TcpStream::connect(addr) + .map(|s| { + let _ = s.set_nodelay(true); // low-latency request/response + Stream::Tcp(s) + }) + .map_err(|_| CUDA_ERROR_NO_DEVICE); + } + #[cfg(unix)] + if let Some(path) = spec.strip_prefix("unix:") { + return std::os::unix::net::UnixStream::connect(path) + .map(Stream::Unix) + .map_err(|_| CUDA_ERROR_NO_DEVICE); + } + #[cfg(target_os = "linux")] + { + // smolvm's reserved CUDA port on the host CID (smolvm_protocol::ports::CUDA). + const HOST_CID: u32 = 2; + const CUDA_PORT: u32 = 7000; + vsock::VsockStream::connect_with_cid_port(HOST_CID, CUDA_PORT) + .map(Stream::Vsock) + .map_err(|_| CUDA_ERROR_NO_DEVICE) + } + #[cfg(not(target_os = "linux"))] + Err(CUDA_ERROR_NO_DEVICE) +} + +// ---- global state ------------------------------------------------------------- + +struct ShimState { + client: Client, + /// Kernel-argument byte sizes per function handle, fetched once per function. + param_sizes: HashMap>, + /// Primary-context handle per device (retain is refcounted host-side; the + /// app-visible handle stays stable per device as the real driver's does). + primary_ctx: HashMap, + /// Process-global "current context" stack (bottom = cuCtxSetCurrent slot). + ctx_stack: Vec, +} + +static STATE: Mutex> = Mutex::new(None); + +/// Establish the server connection + primary context if not already done. +/// Idempotent. Called from `cuInit` and lazily from any driver call — the real +/// driver treats `cuInit` as process-global, but this shim is a separate library +/// (and separate connection) from our `libcudart` shim, so a consumer that only +/// initialized CUDA through the runtime API never called *our* `cuInit`. Lazily +/// connecting on first use makes any driver entry point self-initializing. +fn ensure_connected(guard: &mut Option) -> Result<(), c_int> { + if guard.is_some() { + return Ok(()); + } + // Preferred: ride the runtime shim's connection (both shims loaded, one + // program-ordered pipeline, full deferral). Fallback: own connection — + // then this traffic shares one guest program-order stream with the + // runtime shim's connection, and two independently flushed deferred + // queues would let the host execute work out of order (recorded misorder + // in a CUDA graph replays wrong), so run every op sync and fence the + // runtime connection before each one (see fence_runtime). + let mut client = match resolve_bridge() { + Some(bridge) => Client::new_bridged(Stream::Bridged, bridge), + None => { + let mut c = Client::new(connect()?); + c.set_defer_enabled(false); + c + } + }; + if let Err(e) = client.init() { + return Err(match e { + CudaRpcError::Cuda(code) => code as c_int, + _ => CUDA_ERROR_NO_DEVICE, + }); + } + BRIDGED.store(client.is_bridged(), std::sync::atomic::Ordering::Relaxed); + *guard = Some(ShimState { + client, + param_sizes: HashMap::new(), + primary_ctx: HashMap::new(), + ctx_stack: Vec::new(), + }); + Ok(()) +} + +/// Set once at connect: this shim's client rides the runtime shim's +/// connection, so the per-op runtime fence is unnecessary. +static BRIDGED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// The runtime shim's bridge exports, if it's loaded in this process. +/// All three must resolve or none are used. `SMOLVM_CUDA_BRIDGE=0` forces the +/// standalone fallback (kill-switch, mirrors `SMOLVM_CUDA_ASYNC=0`). +fn resolve_bridge() -> Option { + if std::env::var("SMOLVM_CUDA_BRIDGE").as_deref() == Ok("0") { + return None; + } + extern "C" { + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + } + // glibc: RTLD_DEFAULT == NULL (this shim is glibc-only). + unsafe { + let quiet = dlsym(std::ptr::null_mut(), c"smolvm_cudart_bridge_quiet".as_ptr()); + let call = dlsym(std::ptr::null_mut(), c"smolvm_cudart_bridge_call".as_ptr()); + let drain = dlsym(std::ptr::null_mut(), c"smolvm_cudart_bridge_drain".as_ptr()); + if quiet.is_null() || call.is_null() || drain.is_null() { + return None; + } + Some(smolvm_cuda::client::Bridge { + quiet: std::mem::transmute::<*mut c_void, unsafe extern "C" fn(*const u8, usize) -> i32>( + quiet, + ), + call: std::mem::transmute::< + *mut c_void, + unsafe extern "C" fn(*const u8, usize, *mut u8, usize) -> isize, + >(call), + drain: std::mem::transmute::<*mut c_void, unsafe extern "C" fn() -> i32>(drain), + }) + } +} + +/// Run `f` against the connected client, translating errors to `CUresult`. +/// Auto-connects on first use (see [`ensure_connected`]). +/// Settle the runtime shim's deferred pipeline before running one of our ops, +/// so guest program order holds across the two connections. The hook is +/// exported by the runtime shim (`smolvm_cudart_fence` in libcudart); resolved +/// lazily because either shim can load first. dlsym cost is paid only until +/// the symbol appears (a process without the runtime shim keeps probing, but +/// such a process has no cross-connection ordering to preserve anyway). +fn fence_runtime() { + use std::sync::atomic::{AtomicUsize, Ordering}; + extern "C" { + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + } + // glibc: RTLD_DEFAULT == NULL (this shim is glibc-only). + static HOOK: AtomicUsize = AtomicUsize::new(0); + let mut p = HOOK.load(Ordering::Relaxed); + if p == 0 { + p = unsafe { dlsym(std::ptr::null_mut(), c"smolvm_cudart_fence".as_ptr()) } as usize; + if p != 0 { + HOOK.store(p, Ordering::Relaxed); + } + } + if p != 0 { + let f: extern "C" fn() = unsafe { std::mem::transmute::(p) }; + f(); + } +} + +fn with_state(f: impl FnOnce(&mut ShimState) -> Result) -> Result { + // Bridged: program order is inherent (one pipeline), no fence needed. + if !BRIDGED.load(std::sync::atomic::Ordering::Relaxed) { + fence_runtime(); + } + let mut guard = match STATE.lock() { + Ok(g) => g, + Err(_) => return Err(CUDA_ERROR_UNKNOWN), + }; + ensure_connected(&mut guard)?; + let state = guard.as_mut().expect("connected"); + f(state).map_err(|e| match e { + CudaRpcError::Cuda(code) => code as c_int, + CudaRpcError::Io(_) | CudaRpcError::Protocol(_) => CUDA_ERROR_UNKNOWN, + }) +} + +/// Fold a `Result` into the C return convention. +fn ret(r: Result<(), c_int>) -> c_int { + match r { + Ok(()) => CUDA_SUCCESS, + Err(code) => code, + } +} + +/// Write `v` through an out-pointer, guarding NULL. +unsafe fn out(p: *mut T, v: T) -> Result<(), c_int> { + if p.is_null() { + return Err(CUDA_ERROR_INVALID_VALUE); + } + unsafe { p.write(v) }; + Ok(()) +} + +// ---- init / device ------------------------------------------------------------ + +#[no_mangle] +pub extern "C" fn cuInit(_flags: c_uint) -> c_int { + let mut guard = match STATE.lock() { + Ok(g) => g, + Err(_) => return CUDA_ERROR_UNKNOWN, + }; + ret(ensure_connected(&mut guard)) +} + +/// Undocumented internal driver interface tables, keyed by a 16-byte UUID. +/// NVIDIA's CUDA runtime (`libcudart`) requires several of these to complete +/// its context bootstrap; their layout and function ABIs are private and not +/// part of the public Driver API this shim implements. Returning "not +/// supported" (and logging the UUID under trace) makes the boundary explicit: +/// a pure `libcuda` shim cannot host NVIDIA's runtime — that needs remoting at +/// the `libcudart` level instead. See docs/cuda-support-plan.md (Phase 4). +#[no_mangle] +pub extern "C" fn cuGetExportTable(_table: *mut *const c_void, uuid: *const c_void) -> c_int { + if std::env::var_os("SMOLVM_CUDA_SHIM_TRACE").is_some() && !uuid.is_null() { + let b = unsafe { std::slice::from_raw_parts(uuid as *const u8, 16) }; + let hex: String = b.iter().map(|x| format!("{x:02x}")).collect(); + eprintln!("[shim] cuGetExportTable uuid={hex} -> NOT_SUPPORTED"); + } + CUDA_ERROR_NOT_SUPPORTED +} + +#[no_mangle] +pub extern "C" fn cuDriverGetVersion(version: *mut c_int) -> c_int { + // Queryable before cuInit, per the driver's contract. + if STATE.lock().map(|g| g.is_some()).unwrap_or(false) { + ret(with_state(|s| s.client.driver_get_version()).and_then(|v| unsafe { out(version, v) })) + } else { + ret(unsafe { out(version, SHIM_CUDA_VERSION) }) + } +} + +#[no_mangle] +pub extern "C" fn cuDeviceGetCount(count: *mut c_int) -> c_int { + ret(with_state(|s| s.client.device_get_count()).and_then(|v| unsafe { out(count, v) })) +} + +#[no_mangle] +pub extern "C" fn cuDeviceGet(device: *mut c_int, ordinal: c_int) -> c_int { + // CUdevice is the ordinal itself in this ABI. + match with_state(|s| s.client.device_get_count()) { + Ok(n) if ordinal >= 0 && ordinal < n => ret(unsafe { out(device, ordinal) }), + Ok(_) => CUDA_ERROR_INVALID_VALUE, + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuDeviceGetName(name: *mut c_char, len: c_int, device: c_int) -> c_int { + if name.is_null() || len <= 0 { + return CUDA_ERROR_INVALID_VALUE; + } + match with_state(|s| s.client.device_get_name(device)) { + Ok(n) => { + let bytes = n.as_bytes(); + let copy = bytes.len().min(len as usize - 1); + unsafe { + std::ptr::copy_nonoverlapping(bytes.as_ptr(), name as *mut u8, copy); + *name.add(copy) = 0; + } + CUDA_SUCCESS + } + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuDeviceTotalMem_v2(bytes: *mut usize, device: c_int) -> c_int { + ret(with_state(|s| s.client.device_total_mem(device)) + .and_then(|v| unsafe { out(bytes, v as usize) })) +} + +#[no_mangle] +pub extern "C" fn cuDeviceGetAttribute(pi: *mut c_int, attrib: c_int, device: c_int) -> c_int { + // Immutable — memoize to spare a host round-trip per repeat. + static DEV_ATTRS: Mutex>> = Mutex::new(None); + if let Ok(mut g) = DEV_ATTRS.lock() { + if let Some(&v) = g.get_or_insert_with(HashMap::new).get(&(device, attrib)) { + return ret(unsafe { out(pi, v) }); + } + } + ret( + with_state(|s| s.client.device_get_attribute(attrib, device)).and_then(|v| { + if let Ok(mut g) = DEV_ATTRS.lock() { + g.get_or_insert_with(HashMap::new) + .insert((device, attrib), v); + } + unsafe { out(pi, v) } + }), + ) +} + +/// Deprecated capability query PyTorch still calls at init. Forwards the two +/// compute-capability attributes (MAJOR=75, MINOR=76). +#[no_mangle] +pub extern "C" fn cuDeviceComputeCapability( + major: *mut c_int, + minor: *mut c_int, + device: c_int, +) -> c_int { + let r = with_state(|s| { + let maj = s.client.device_get_attribute(75, device)?; + let min = s.client.device_get_attribute(76, device)?; + Ok((maj, min)) + }); + ret(r.and_then(|(maj, min)| unsafe { + out(major, maj)?; + out(minor, min) + })) +} + +#[no_mangle] +pub extern "C" fn cuDeviceGetUuid(uuid: *mut u8, device: c_int) -> c_int { + if uuid.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + match with_state(|s| s.client.device_get_uuid(device)) { + Ok(u) => { + unsafe { std::ptr::copy_nonoverlapping(u.as_ptr(), uuid, 16) }; + CUDA_SUCCESS + } + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuDeviceGetUuid_v2(uuid: *mut u8, device: c_int) -> c_int { + cuDeviceGetUuid(uuid, device) +} + +// ---- contexts ------------------------------------------------------------------- + +#[no_mangle] +pub extern "C" fn cuCtxCreate_v2(pctx: *mut *mut c_void, _flags: c_uint, device: c_int) -> c_int { + match with_state(|s| s.client.ctx_create(device)) { + Ok(h) => { + let r = ret(unsafe { out(pctx, h as *mut c_void) }); + if r == CUDA_SUCCESS { + if let Ok(mut g) = STATE.lock() { + if let Some(s) = g.as_mut() { + // cuCtxCreate makes the new context current (pushes it). + s.ctx_stack.push(h); + } + } + } + r + } + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuCtxDestroy_v2(ctx: *mut c_void) -> c_int { + let h = ctx as u64; + let r = ret(with_state(|s| s.client.ctx_destroy(h))); + if r == CUDA_SUCCESS { + if let Ok(mut g) = STATE.lock() { + if let Some(s) = g.as_mut() { + s.ctx_stack.retain(|&c| c != h); + } + } + } + r +} + +#[no_mangle] +pub extern "C" fn cuCtxSetCurrent(ctx: *mut c_void) -> c_int { + let mut g = match STATE.lock() { + Ok(g) => g, + Err(_) => return CUDA_ERROR_UNKNOWN, + }; + if let Err(e) = ensure_connected(&mut g) { + return e; + } + let s = g.as_mut().expect("connected"); + s.ctx_stack.pop(); + if !ctx.is_null() { + s.ctx_stack.push(ctx as u64); + } + CUDA_SUCCESS +} + +#[no_mangle] +pub extern "C" fn cuCtxGetCurrent(pctx: *mut *mut c_void) -> c_int { + let mut g = match STATE.lock() { + Ok(g) => g, + Err(_) => return CUDA_ERROR_UNKNOWN, + }; + if let Err(e) = ensure_connected(&mut g) { + return e; + } + let cur = g + .as_ref() + .expect("connected") + .ctx_stack + .last() + .copied() + .unwrap_or(0); + ret(unsafe { out(pctx, cur as *mut c_void) }) +} + +#[no_mangle] +pub extern "C" fn cuCtxPushCurrent_v2(ctx: *mut c_void) -> c_int { + if ctx.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + let mut g = match STATE.lock() { + Ok(g) => g, + Err(_) => return CUDA_ERROR_UNKNOWN, + }; + if let Err(e) = ensure_connected(&mut g) { + return e; + } + g.as_mut().expect("connected").ctx_stack.push(ctx as u64); + CUDA_SUCCESS +} + +#[no_mangle] +pub extern "C" fn cuCtxPopCurrent_v2(pctx: *mut *mut c_void) -> c_int { + let mut g = match STATE.lock() { + Ok(g) => g, + Err(_) => return CUDA_ERROR_UNKNOWN, + }; + if let Err(e) = ensure_connected(&mut g) { + return e; + } + match g.as_mut().expect("connected").ctx_stack.pop() { + Some(h) => { + if !pctx.is_null() { + unsafe { *pctx = h as *mut c_void }; + } + CUDA_SUCCESS + } + None => CUDA_ERROR_INVALID_CONTEXT, + } +} + +#[no_mangle] +pub extern "C" fn cuCtxGetDevice(device: *mut c_int) -> c_int { + // Single-device model: the current context always belongs to device 0. + let mut g = match STATE.lock() { + Ok(g) => g, + Err(_) => return CUDA_ERROR_UNKNOWN, + }; + if let Err(e) = ensure_connected(&mut g) { + return e; + } + ret(unsafe { out(device, 0) }) +} + +#[no_mangle] +pub extern "C" fn cuCtxSynchronize() -> c_int { + ret(with_state(|s| s.client.ctx_synchronize())) +} + +#[no_mangle] +pub extern "C" fn cuDevicePrimaryCtxRetain(pctx: *mut *mut c_void, device: c_int) -> c_int { + let r = with_state(|s| { + if let Some(&h) = s.primary_ctx.get(&device) { + return Ok(h); + } + let h = s.client.primary_ctx_retain(device)?; + s.primary_ctx.insert(device, h); + Ok(h) + }); + match r { + Ok(h) => { + let rc = ret(unsafe { out(pctx, h as *mut c_void) }); + if rc == CUDA_SUCCESS { + if let Ok(mut g) = STATE.lock() { + if let Some(s) = g.as_mut() { + // The host binds the retained primary context current on + // its serving thread; mirror that in the guest's + // process-global current-context view. + s.ctx_stack.push(h); + } + } + } + rc + } + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuDevicePrimaryCtxRelease_v2(device: c_int) -> c_int { + ret(with_state(|s| s.client.primary_ctx_release(device))) +} + +#[no_mangle] +pub extern "C" fn cuDevicePrimaryCtxRelease(device: c_int) -> c_int { + cuDevicePrimaryCtxRelease_v2(device) +} + +#[no_mangle] +pub extern "C" fn cuDevicePrimaryCtxSetFlags_v2(_device: c_int, _flags: c_uint) -> c_int { + CUDA_SUCCESS // flags are advisory; the host context uses defaults +} + +#[no_mangle] +pub extern "C" fn cuDevicePrimaryCtxGetState( + device: c_int, + flags: *mut c_uint, + active: *mut c_int, +) -> c_int { + let mut g = match STATE.lock() { + Ok(g) => g, + Err(_) => return CUDA_ERROR_UNKNOWN, + }; + if let Err(e) = ensure_connected(&mut g) { + return e; + } + let is_active = g + .as_ref() + .expect("connected") + .primary_ctx + .contains_key(&device); + if !flags.is_null() { + unsafe { *flags = 0 }; + } + if !active.is_null() { + unsafe { *active = is_active as c_int }; + } + CUDA_SUCCESS +} + +// ---- modules -------------------------------------------------------------------- + +/// Byte length of a module image the app handed us as a bare pointer. +/// +/// The Driver API infers the length from the image itself; the RPC needs it +/// explicit. PTX is NUL-terminated text; cubins are ELF; nvcc embeddings are +/// fatbin containers with a size field in the header. +unsafe fn module_image_len(image: *const c_void) -> Result { + if image.is_null() { + return Err(CUDA_ERROR_INVALID_VALUE); + } + let p = image as *const u8; + let magic = unsafe { std::slice::from_raw_parts(p, 4) }; + // Fatbin container: u32 magic, u16 version, u16 headerSize, u64 fatSize. + if magic == [0x50, 0xED, 0x55, 0xBA] { + let header_size = u16::from_le_bytes(unsafe { *(p.add(6) as *const [u8; 2]) }) as usize; + let fat_size = u64::from_le_bytes(unsafe { *(p.add(8) as *const [u8; 8]) }) as usize; + return Ok(header_size + fat_size); + } + // ELF (cubin): total = e_shoff + e_shnum * e_shentsize (sections are last). + if magic == [0x7F, b'E', b'L', b'F'] { + let e_shoff = u64::from_le_bytes(unsafe { *(p.add(0x28) as *const [u8; 8]) }) as usize; + let e_shentsize = u16::from_le_bytes(unsafe { *(p.add(0x3A) as *const [u8; 2]) }) as usize; + let e_shnum = u16::from_le_bytes(unsafe { *(p.add(0x3C) as *const [u8; 2]) }) as usize; + return Ok(e_shoff + e_shentsize * e_shnum); + } + // PTX text: NUL-terminated. + Ok(unsafe { CStr::from_ptr(image as *const c_char) } + .to_bytes() + .len() + + 1) +} + +// ---- VMM (torch expandable-segments allocator) -------------------------------- +// Prop/desc structs are read at fixed offsets: CUmemAllocationProp.location.id +// sits at byte 12, CUmemAccessDesc.location.id at byte 4. + +#[no_mangle] +pub extern "C" fn cuMemAddressReserve( + ptr: *mut u64, + size: usize, + align: usize, + _addr_hint: u64, + _flags: u64, +) -> c_int { + match with_state(|s| s.client.mem_address_reserve(size as u64, align as u64)) { + Ok(va) => ret(unsafe { out(ptr, va) }), + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuMemCreate( + handle: *mut u64, + size: usize, + prop: *const c_void, + _flags: u64, +) -> c_int { + let device = if prop.is_null() { + 0 + } else { + unsafe { ((prop as *const u8).add(12) as *const c_int).read_unaligned() } + }; + match with_state(|s| s.client.mem_create(size as u64, device)) { + Ok(h) => ret(unsafe { out(handle, h) }), + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuMemMap( + ptr: u64, + size: usize, + offset: usize, + handle: u64, + _flags: u64, +) -> c_int { + ret(with_state(|s| { + s.client.mem_map(ptr, size as u64, offset as u64, handle) + })) +} + +#[no_mangle] +pub extern "C" fn cuMemSetAccess( + ptr: u64, + size: usize, + desc: *const c_void, + _count: usize, +) -> c_int { + let device = if desc.is_null() { + 0 + } else { + unsafe { ((desc as *const u8).add(4) as *const c_int).read_unaligned() } + }; + ret(with_state(|s| { + s.client.mem_set_access(ptr, size as u64, device) + })) +} + +#[no_mangle] +pub extern "C" fn cuMemUnmap(ptr: u64, size: usize) -> c_int { + ret(with_state(|s| s.client.mem_unmap(ptr, size as u64))) +} + +#[no_mangle] +pub extern "C" fn cuMemRelease(handle: u64) -> c_int { + ret(with_state(|s| s.client.mem_release(handle))) +} + +#[no_mangle] +pub extern "C" fn cuMemAddressFree(ptr: u64, size: usize) -> c_int { + ret(with_state(|s| s.client.mem_address_free(ptr, size as u64))) +} + +#[no_mangle] +pub extern "C" fn cuMemGetAllocationGranularity( + granularity: *mut usize, + prop: *const c_void, + flags: c_uint, +) -> c_int { + let device = if prop.is_null() { + 0 + } else { + unsafe { ((prop as *const u8).add(12) as *const c_int).read_unaligned() } + }; + match with_state(|s| s.client.mem_get_allocation_granularity(device, flags)) { + Ok(g) => ret(unsafe { out(granularity, g as usize) }), + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuModuleLoadData(module: *mut *mut c_void, image: *const c_void) -> c_int { + let len = match unsafe { module_image_len(image) } { + Ok(l) => l, + Err(code) => return code, + }; + let bytes = unsafe { std::slice::from_raw_parts(image as *const u8, len) }; + match with_state(|s| s.client.module_load_data(bytes)) { + Ok(h) => ret(unsafe { out(module, h as *mut c_void) }), + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuModuleLoadDataEx( + module: *mut *mut c_void, + image: *const c_void, + _num_options: c_uint, + _options: *mut c_void, + _option_values: *mut *mut c_void, +) -> c_int { + // JIT options are tuning hints; the host driver applies its defaults. + cuModuleLoadData(module, image) +} + +#[no_mangle] +pub extern "C" fn cuModuleLoadFatBinary(module: *mut *mut c_void, fatbin: *const c_void) -> c_int { + cuModuleLoadData(module, fatbin) +} + +#[no_mangle] +pub extern "C" fn cuModuleLoad(module: *mut *mut c_void, fname: *const c_char) -> c_int { + if fname.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + let path = match unsafe { CStr::from_ptr(fname) }.to_str() { + Ok(p) => p, + Err(_) => return CUDA_ERROR_INVALID_VALUE, + }; + let bytes = match std::fs::read(path) { + Ok(b) => b, + Err(_) => return CUDA_ERROR_NOT_FOUND, + }; + match with_state(|s| s.client.module_load_data(&bytes)) { + Ok(h) => ret(unsafe { out(module, h as *mut c_void) }), + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuModuleGetFunction( + func: *mut *mut c_void, + module: *mut c_void, + name: *const c_char, +) -> c_int { + if name.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + let fn_name = match unsafe { CStr::from_ptr(name) }.to_str() { + Ok(n) => n.to_string(), + Err(_) => return CUDA_ERROR_INVALID_VALUE, + }; + match with_state(|s| s.client.module_get_function(module as u64, &fn_name)) { + Ok(h) => ret(unsafe { out(func, h as *mut c_void) }), + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuModuleUnload(module: *mut c_void) -> c_int { + ret(with_state(|s| s.client.module_unload(module as u64))) +} + +// ---- memory --------------------------------------------------------------------- + +#[no_mangle] +pub extern "C" fn cuMemAlloc_v2(dptr: *mut u64, bytes: usize) -> c_int { + match with_state(|s| s.client.mem_alloc(bytes as u64)) { + Ok(d) => ret(unsafe { out(dptr, d) }), + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuMemFree_v2(dptr: u64) -> c_int { + ret(with_state(|s| s.client.mem_free(dptr))) +} + +#[no_mangle] +pub extern "C" fn cuMemcpyHtoD_v2(dptr: u64, src: *const c_void, bytes: usize) -> c_int { + if src.is_null() && bytes > 0 { + return CUDA_ERROR_INVALID_VALUE; + } + let data = unsafe { std::slice::from_raw_parts(src as *const u8, bytes) }; + ret(with_state(|s| s.client.memcpy_htod(dptr, data, 0))) +} + +#[no_mangle] +pub extern "C" fn cuMemcpyDtoH_v2(dst: *mut c_void, dptr: u64, bytes: usize) -> c_int { + if dst.is_null() && bytes > 0 { + return CUDA_ERROR_INVALID_VALUE; + } + match with_state(|s| s.client.memcpy_dtoh(dptr, bytes as u64, 0)) { + Ok(data) => { + if data.len() != bytes { + return CUDA_ERROR_UNKNOWN; + } + unsafe { std::ptr::copy_nonoverlapping(data.as_ptr(), dst as *mut u8, bytes) }; + CUDA_SUCCESS + } + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuMemcpyDtoD_v2(dst: u64, src: u64, bytes: usize) -> c_int { + ret(with_state(|s| s.client.memcpy_dtod(dst, src, bytes as u64))) +} + +#[no_mangle] +pub extern "C" fn cuMemsetD8_v2(dptr: u64, value: u8, n: usize) -> c_int { + ret(with_state(|s| s.client.memset_d8(dptr, value, n as u64))) +} + +#[no_mangle] +pub extern "C" fn cuMemGetInfo_v2(free: *mut usize, total: *mut usize) -> c_int { + match with_state(|s| s.client.mem_get_info()) { + Ok((f, t)) => { + if !free.is_null() { + unsafe { *free = f as usize }; + } + if !total.is_null() { + unsafe { *total = t as usize }; + } + CUDA_SUCCESS + } + Err(code) => code, + } +} + +// The *Async variants complete synchronously — permitted by the API contract +// (an implementation may be more synchronous than requested). + +#[no_mangle] +pub extern "C" fn cuMemcpyHtoDAsync_v2( + dptr: u64, + src: *const c_void, + bytes: usize, + _stream: *mut c_void, +) -> c_int { + cuMemcpyHtoD_v2(dptr, src, bytes) +} + +#[no_mangle] +pub extern "C" fn cuMemcpyDtoHAsync_v2( + dst: *mut c_void, + dptr: u64, + bytes: usize, + _stream: *mut c_void, +) -> c_int { + cuMemcpyDtoH_v2(dst, dptr, bytes) +} + +#[no_mangle] +pub extern "C" fn cuMemcpyDtoDAsync_v2( + dst: u64, + src: u64, + bytes: usize, + _stream: *mut c_void, +) -> c_int { + cuMemcpyDtoD_v2(dst, src, bytes) +} + +// ---- kernel launch ---------------------------------------------------------------- + +/// `CUlaunchConfig` (CUDA 12): dims + shared bytes + stream + an attribute +/// array we don't forward (clusters/programmatic completion — not used by the +/// Triton kernels that launch through this entry point). +#[repr(C)] +struct CuLaunchConfig { + grid_x: c_uint, + grid_y: c_uint, + grid_z: c_uint, + block_x: c_uint, + block_y: c_uint, + block_z: c_uint, + shared_bytes: c_uint, + stream: *mut c_void, + attrs: *mut c_void, + num_attrs: c_uint, +} + +/// Attribute-carrying launch (Triton/torch.compile's launcher). The attributes +/// are ignored; everything else lowers onto the plain launch path. +#[no_mangle] +pub extern "C" fn cuLaunchKernelEx( + config: *const c_void, + func: *mut c_void, + kernel_params: *mut *mut c_void, + extra: *mut *mut c_void, +) -> c_int { + if config.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + // SAFETY: caller passes a valid CUlaunchConfig. + let c = unsafe { &*(config as *const CuLaunchConfig) }; + cuLaunchKernel( + func, + c.grid_x, + c.grid_y, + c.grid_z, + c.block_x, + c.block_y, + c.block_z, + c.shared_bytes, + c.stream, + kernel_params, + extra, + ) +} + +#[no_mangle] +pub extern "C" fn cuLaunchKernelEx_ptsz( + config: *const c_void, + func: *mut c_void, + kernel_params: *mut *mut c_void, + extra: *mut *mut c_void, +) -> c_int { + cuLaunchKernelEx(config, func, kernel_params, extra) +} + +/// Pointer attributes. Every device pointer we hand out is the host's real +/// `CUdeviceptr`, so a pointer the guest holds is a device pointer: report +/// memory-type Device, device-pointer = the value itself, ordinal 0, not +/// managed. Range/host-pointer queries return the pointer / null respectively. +/// vLLM's allocator and Triton link these; a stub error broke import. +#[no_mangle] +pub extern "C" fn cuPointerGetAttribute(data: *mut c_void, attribute: c_int, ptr: u64) -> c_int { + if data.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + // CUpointer_attribute: 1=CONTEXT, 2=MEMORY_TYPE, 3=DEVICE_POINTER, + // 4=HOST_POINTER, 6=BUFFER_ID, 7=IS_MANAGED, 9=DEVICE_ORDINAL, + // 11=RANGE_START_ADDR, 12=RANGE_SIZE, 13=MAPPED. + unsafe { + match attribute { + 2 => *(data as *mut c_uint) = 2, // CU_MEMORYTYPE_DEVICE + 3 | 11 => *(data as *mut u64) = ptr, + 12 => *(data as *mut usize) = 0, + 4 => *(data as *mut u64) = 0, // no host mapping + 7 => *(data as *mut c_int) = 0, + 9 => *(data as *mut c_int) = 0, + 13 => *(data as *mut c_int) = 1, + _ => *(data as *mut u64) = 0, + } + } + CUDA_SUCCESS +} + +/// Batched pointer-attribute query: fill each requested attribute per the +/// single-attribute logic above. +#[no_mangle] +pub extern "C" fn cuPointerGetAttributes( + num_attributes: c_uint, + attributes: *const c_int, + data: *const *mut c_void, + ptr: u64, +) -> c_int { + if attributes.is_null() || data.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + for i in 0..num_attributes as isize { + let attr = unsafe { *attributes.offset(i) }; + let out = unsafe { *data.offset(i) }; + if !out.is_null() { + cuPointerGetAttribute(out, attr, ptr); + } + } + CUDA_SUCCESS +} + +/// Cache-config is a scheduling hint the host driver applies at launch; a +/// no-op here is correct (work runs on the host's real function). +#[no_mangle] +pub extern "C" fn cuFuncSetCacheConfig(_func: *mut c_void, _config: c_int) -> c_int { + CUDA_SUCCESS +} + +/// Context limits (stack size, printf FIFO, malloc heap): report a generous +/// value on get, accept any set. Triton reads the stack-size limit during +/// launcher setup; a stub error there aborted the launch. +#[no_mangle] +pub extern "C" fn cuCtxGetLimit(pvalue: *mut usize, _limit: c_int) -> c_int { + if pvalue.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + unsafe { *pvalue = 8 * 1024 * 1024 }; + CUDA_SUCCESS +} +#[no_mangle] +pub extern "C" fn cuCtxSetLimit(_limit: c_int, _value: usize) -> c_int { + CUDA_SUCCESS +} + +/// No cluster support (clusters are an sm_90+ feature we don't forward); +/// reporting 0 max clusters keeps Triton on the standard, non-cluster launch. +#[no_mangle] +pub extern "C" fn cuOccupancyMaxActiveClusters( + num_clusters: *mut c_int, + _func: *mut c_void, + _config: *const c_void, +) -> c_int { + if num_clusters.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + unsafe { *num_clusters = 0 }; + CUDA_SUCCESS +} + +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn cuLaunchKernel( + func: *mut c_void, + grid_x: c_uint, + grid_y: c_uint, + grid_z: c_uint, + block_x: c_uint, + block_y: c_uint, + block_z: c_uint, + shared_bytes: c_uint, + stream: *mut c_void, + kernel_params: *mut *mut c_void, + extra: *mut *mut c_void, +) -> c_int { + let fh = func as u64; + // Argument sizes come from the host's view of the loaded module; fetch once + // per function and cache. + let sizes = match with_state(|s| { + if let Some(sz) = s.param_sizes.get(&fh) { + return Ok(sz.clone()); + } + let sz = s.client.func_get_param_info(fh)?; + s.param_sizes.insert(fh, sz.clone()); + Ok(sz) + }) { + Ok(s) => s, + Err(code) => return code, + }; + let params: Vec> = if sizes.is_empty() { + Vec::new() + } else { + if kernel_params.is_null() { + // The `extra` buffer-pointer convention is not implemented. + return if extra.is_null() { + CUDA_ERROR_INVALID_VALUE + } else { + CUDA_ERROR_NOT_SUPPORTED + }; + } + let ptrs = unsafe { std::slice::from_raw_parts(kernel_params, sizes.len()) }; + sizes + .iter() + .zip(ptrs) + .map(|(&sz, &p)| { + unsafe { std::slice::from_raw_parts(p as *const u8, sz as usize) }.to_vec() + }) + .collect() + }; + ret(with_state(|s| { + s.client.launch_kernel( + fh, + [grid_x, grid_y, grid_z], + [block_x, block_y, block_z], + shared_bytes, + stream as u64, + ¶ms, + ) + })) +} + +// ---- streams / events ---------------------------------------------------------------- + +#[no_mangle] +pub extern "C" fn cuStreamCreate(stream: *mut *mut c_void, flags: c_uint) -> c_int { + match with_state(|s| s.client.stream_create(flags)) { + Ok(h) => ret(unsafe { out(stream, h as *mut c_void) }), + Err(code) => code, + } +} + +// PyTorch's stream pool (used for CUDA-graph capture) creates streams here. +// Priority is advisory scheduling only; forward as a plain stream. A stub that +// left `stream` unwritten fed callers an uninitialized handle → a bad-pointer +// crash inside cuBLAS during graph capture. +#[no_mangle] +pub extern "C" fn cuStreamCreateWithPriority( + stream: *mut *mut c_void, + flags: c_uint, + _priority: c_int, +) -> c_int { + match with_state(|s| s.client.stream_create(flags)) { + Ok(h) => ret(unsafe { out(stream, h as *mut c_void) }), + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuStreamDestroy_v2(stream: *mut c_void) -> c_int { + ret(with_state(|s| s.client.stream_destroy(stream as u64))) +} + +#[no_mangle] +pub extern "C" fn cuStreamSynchronize(stream: *mut c_void) -> c_int { + ret(with_state(|s| s.client.stream_synchronize(stream as u64))) +} + +#[no_mangle] +pub extern "C" fn cuStreamQuery(stream: *mut c_void) -> c_int { + // Honest completion status (0 or 600-NotReady) now that work runs on real + // side streams. + match with_state(|s| s.client.stream_query(stream as u64)) { + Ok(code) => code, + Err(e) => e, + } +} + +#[no_mangle] +pub extern "C" fn cuStreamWaitEvent( + stream: *mut c_void, + event: *mut c_void, + flags: c_uint, +) -> c_int { + // Real cross-stream ordering edge (a graph dependency during capture). + ret(with_state(|s| { + s.client + .stream_wait_event(stream as u64, event as u64, flags) + })) +} + +#[no_mangle] +pub extern "C" fn cuEventCreate(event: *mut *mut c_void, flags: c_uint) -> c_int { + match with_state(|s| s.client.event_create(flags)) { + Ok(h) => ret(unsafe { out(event, h as *mut c_void) }), + Err(code) => code, + } +} + +#[no_mangle] +pub extern "C" fn cuEventDestroy_v2(event: *mut c_void) -> c_int { + ret(with_state(|s| s.client.event_destroy(event as u64))) +} + +#[no_mangle] +pub extern "C" fn cuEventRecord(event: *mut c_void, stream: *mut c_void) -> c_int { + ret(with_state(|s| { + s.client.event_record(event as u64, stream as u64) + })) +} + +#[no_mangle] +pub extern "C" fn cuEventSynchronize(event: *mut c_void) -> c_int { + ret(with_state(|s| s.client.event_synchronize(event as u64))) +} + +#[no_mangle] +pub extern "C" fn cuEventQuery(_event: *mut c_void) -> c_int { + CUDA_SUCCESS +} + +#[no_mangle] +pub extern "C" fn cuEventElapsedTime(ms: *mut f32, start: *mut c_void, end: *mut c_void) -> c_int { + match with_state(|s| s.client.event_elapsed_time(start as u64, end as u64)) { + Ok(v) => ret(unsafe { out(ms, v) }), + Err(code) => code, + } +} + +// ---- error strings ------------------------------------------------------------------- + +fn error_name(code: c_int) -> &'static CStr { + match code { + CUDA_SUCCESS => c"CUDA_SUCCESS", + CUDA_ERROR_INVALID_VALUE => c"CUDA_ERROR_INVALID_VALUE", + CUDA_ERROR_NOT_INITIALIZED => c"CUDA_ERROR_NOT_INITIALIZED", + CUDA_ERROR_NO_DEVICE => c"CUDA_ERROR_NO_DEVICE", + CUDA_ERROR_INVALID_CONTEXT => c"CUDA_ERROR_INVALID_CONTEXT", + 200 => c"CUDA_ERROR_INVALID_IMAGE", + 218 => c"CUDA_ERROR_INVALID_PTX", + 400 => c"CUDA_ERROR_INVALID_HANDLE", + CUDA_ERROR_NOT_FOUND => c"CUDA_ERROR_NOT_FOUND", + 700 => c"CUDA_ERROR_ILLEGAL_ADDRESS", + 701 => c"CUDA_ERROR_LAUNCH_OUT_OF_RESOURCES", + 719 => c"CUDA_ERROR_LAUNCH_FAILED", + CUDA_ERROR_NOT_SUPPORTED => c"CUDA_ERROR_NOT_SUPPORTED", + _ => c"CUDA_ERROR_UNKNOWN", + } +} + +#[no_mangle] +pub extern "C" fn cuGetErrorName(code: c_int, pstr: *mut *const c_char) -> c_int { + ret(unsafe { out(pstr, error_name(code).as_ptr()) }) +} + +#[no_mangle] +pub extern "C" fn cuGetErrorString(code: c_int, pstr: *mut *const c_char) -> c_int { + // Name doubles as description; apps only ever print it. + cuGetErrorName(code, pstr) +} + +// ---- cuGetProcAddress ------------------------------------------------------------------ + +/// The modern CUDA runtime resolves nearly every driver entry point through +/// this table rather than dlsym, so it must cover everything we export. +fn proc_table(name: &str) -> Option<*mut c_void> { + macro_rules! table { + ($($sym:literal => $f:expr),+ $(,)?) => { + match name { + $($sym => Some($f as *mut c_void),)+ + _ => None, + } + }; + } + table! { + "cuInit" => cuInit, + "cuDriverGetVersion" => cuDriverGetVersion, + "cuDeviceGetCount" => cuDeviceGetCount, + "cuDeviceGet" => cuDeviceGet, + "cuDeviceGetName" => cuDeviceGetName, + "cuDeviceTotalMem" => cuDeviceTotalMem_v2, + "cuDeviceGetAttribute" => cuDeviceGetAttribute, + "cuDeviceGetUuid" => cuDeviceGetUuid, + "cuCtxCreate" => cuCtxCreate_v2, + "cuCtxDestroy" => cuCtxDestroy_v2, + "cuCtxSetCurrent" => cuCtxSetCurrent, + "cuCtxGetCurrent" => cuCtxGetCurrent, + "cuCtxPushCurrent" => cuCtxPushCurrent_v2, + "cuCtxPopCurrent" => cuCtxPopCurrent_v2, + "cuCtxGetDevice" => cuCtxGetDevice, + "cuCtxSynchronize" => cuCtxSynchronize, + "cuDevicePrimaryCtxRetain" => cuDevicePrimaryCtxRetain, + "cuDevicePrimaryCtxRelease" => cuDevicePrimaryCtxRelease_v2, + "cuDevicePrimaryCtxSetFlags" => cuDevicePrimaryCtxSetFlags_v2, + "cuDevicePrimaryCtxGetState" => cuDevicePrimaryCtxGetState, + "cuModuleLoad" => cuModuleLoad, + "cuModuleLoadData" => cuModuleLoadData, + "cuModuleLoadDataEx" => cuModuleLoadDataEx, + "cuModuleLoadFatBinary" => cuModuleLoadFatBinary, + "cuModuleGetFunction" => cuModuleGetFunction, + "cuModuleUnload" => cuModuleUnload, + "cuMemAlloc" => cuMemAlloc_v2, + "cuMemFree" => cuMemFree_v2, + "cuMemcpyHtoD" => cuMemcpyHtoD_v2, + "cuMemcpyDtoH" => cuMemcpyDtoH_v2, + "cuMemcpyDtoD" => cuMemcpyDtoD_v2, + "cuMemcpyHtoDAsync" => cuMemcpyHtoDAsync_v2, + "cuMemcpyDtoHAsync" => cuMemcpyDtoHAsync_v2, + "cuMemcpyDtoDAsync" => cuMemcpyDtoDAsync_v2, + "cuMemsetD8" => cuMemsetD8_v2, + "cuMemGetInfo" => cuMemGetInfo_v2, + "cuLaunchKernel" => cuLaunchKernel, + "cuStreamCreate" => cuStreamCreate, + "cuStreamCreateWithPriority" => cuStreamCreateWithPriority, + "cuStreamDestroy" => cuStreamDestroy_v2, + "cuStreamSynchronize" => cuStreamSynchronize, + "cuStreamQuery" => cuStreamQuery, + "cuStreamWaitEvent" => cuStreamWaitEvent, + "cuEventCreate" => cuEventCreate, + "cuEventDestroy" => cuEventDestroy_v2, + "cuEventRecord" => cuEventRecord, + "cuEventSynchronize" => cuEventSynchronize, + "cuEventQuery" => cuEventQuery, + "cuEventElapsedTime" => cuEventElapsedTime, + "cuGetErrorName" => cuGetErrorName, + "cuGetErrorString" => cuGetErrorString, + "cuGetExportTable" => cuGetExportTable, + // NB: cuGetProcAddress is resolved by `resolve_proc` (version-dependent + // v1 vs v2 ABI), never through this table — see the note there. + } +} + +/// Resolve a driver symbol to a function pointer, honoring the caller's CUDA +/// version for entry points whose C ABI changed across revisions. +/// +/// `cuGetProcAddress` is the load-bearing case: its own signature gained a +/// fifth `symbolStatus` parameter in CUDA 12.0 (the `_v2` form). A CUDA 11.x +/// runtime resolves "cuGetProcAddress" and then *calls the result with the +/// 4-argument v1 convention*. Handing back the v2 pointer there makes it read +/// an uninitialized fifth argument and write through it — a crash on the very +/// first bootstrap call. So serve v1 below 12000 and v2 at/above it. +fn resolve_proc(name: &str, cuda_version: c_int) -> Option<*mut c_void> { + let base = name.trim_end_matches("_v3").trim_end_matches("_v2"); + if base == "cuGetProcAddress" { + // An explicit "_v2" request always wants v2; a bare request follows the + // caller's version. + return Some(if name.ends_with("_v2") || cuda_version >= 12000 { + cuGetProcAddress_v2 as *mut c_void + } else { + cuGetProcAddress as *mut c_void + }); + } + proc_table(base) +} + +#[no_mangle] +pub extern "C" fn cuGetProcAddress_v2( + symbol: *const c_char, + pfn: *mut *mut c_void, + cuda_version: c_int, + _flags: u64, + symbol_status: *mut c_int, +) -> c_int { + if symbol.is_null() || pfn.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + let name = match unsafe { CStr::from_ptr(symbol) }.to_str() { + Ok(n) => n, + Err(_) => return CUDA_ERROR_INVALID_VALUE, + }; + let resolved = resolve_proc(name, cuda_version); + trace_proc(name, resolved.is_some()); + match resolved { + Some(f) => { + unsafe { *pfn = f }; + if !symbol_status.is_null() { + unsafe { *symbol_status = 0 }; // CU_GET_PROC_ADDRESS_SUCCESS + } + CUDA_SUCCESS + } + None => { + unsafe { *pfn = std::ptr::null_mut() }; + if !symbol_status.is_null() { + unsafe { *symbol_status = 1 }; // CU_GET_PROC_ADDRESS_SYMBOL_NOT_FOUND + } + CUDA_ERROR_NOT_FOUND + } + } +} + +/// When `SMOLVM_CUDA_SHIM_TRACE` is set, log each cuGetProcAddress lookup and +/// whether the shim served it — the fastest way to enumerate the surface a +/// given CUDA runtime needs. No-op otherwise. +fn trace_proc(name: &str, hit: bool) { + use std::sync::atomic::{AtomicU8, Ordering}; + static ENABLED: AtomicU8 = AtomicU8::new(0); // 0=unknown, 1=on, 2=off + let on = match ENABLED.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => { + let on = std::env::var_os("SMOLVM_CUDA_SHIM_TRACE").is_some(); + ENABLED.store(if on { 1 } else { 2 }, Ordering::Relaxed); + on + } + }; + if on { + eprintln!( + "[shim] cuGetProcAddress {name} -> {}", + if hit { "hit" } else { "MISS" } + ); + } +} + +#[no_mangle] +pub extern "C" fn cuGetProcAddress( + symbol: *const c_char, + pfn: *mut *mut c_void, + cuda_version: c_int, + flags: u64, +) -> c_int { + cuGetProcAddress_v2(symbol, pfn, cuda_version, flags, std::ptr::null_mut()) +} + +// ---- driver symbols PyTorch's stack links (exported so the shim loads) ------- +// None are called during basic CUDA init; JIT-link (cuLink*), cooperative +// launch, and Hopper TMA (cuTensorMapEncodeTiled) are unused on sm_86 forwarding. +const CU_ERROR_NOT_SUPPORTED: c_int = 801; +#[no_mangle] +pub extern "C" fn cuLinkCreate_v2() -> c_int { + CU_ERROR_NOT_SUPPORTED +} +#[no_mangle] +pub extern "C" fn cuLinkAddData_v2() -> c_int { + CU_ERROR_NOT_SUPPORTED +} +#[no_mangle] +pub extern "C" fn cuLinkComplete() -> c_int { + CU_ERROR_NOT_SUPPORTED +} +// These write out-params that callers (PyTorch's launch config) divide by, so a +// bare NOT_SUPPORTED stub (leaving the out uninitialized) causes a divide-by-zero +// crash. Return plausible values instead. +#[no_mangle] +pub extern "C" fn cuFuncGetAttribute(pi: *mut c_int, attrib: c_int, _func: *mut c_void) -> c_int { + // CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK=0 → 1024; else 0. + if !pi.is_null() { + unsafe { *pi = if attrib == 0 { 1024 } else { 0 } }; + } + CUDA_SUCCESS +} +// Triton/vLLM kernels needing >48 KiB shared memory must raise +// CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES here before launching, or the +// launch fails with INVALID_VALUE. Forward it so the host function's real limit +// is raised; Triton checks the return to decide whether the kernel can run. +#[no_mangle] +pub extern "C" fn cuFuncSetAttribute(func: *mut c_void, attrib: c_int, value: c_int) -> c_int { + ret(with_state(|s| { + s.client.func_set_attribute(func as u64, attrib, value) + })) +} +#[no_mangle] +pub extern "C" fn cuLaunchCooperativeKernel() -> c_int { + CU_ERROR_NOT_SUPPORTED +} +#[no_mangle] +pub extern "C" fn cuOccupancyMaxActiveBlocksPerMultiprocessor( + num_blocks: *mut c_int, + _func: *mut c_void, + block_size: c_int, + _dynamic_smem: usize, +) -> c_int { + let bs = block_size.max(1); + if !num_blocks.is_null() { + unsafe { *num_blocks = (2048 / bs).clamp(1, 32) }; + } + CUDA_SUCCESS +} +#[no_mangle] +pub extern "C" fn cuTensorMapEncodeTiled() -> c_int { + CU_ERROR_NOT_SUPPORTED +} + +#[no_mangle] +pub extern "C" fn cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( + num_blocks: *mut c_int, + _func: *mut c_void, + block_size: c_int, + _dyn_smem: usize, + _flags: c_uint, +) -> c_int { + let bs = block_size.max(1); + if !num_blocks.is_null() { + unsafe { *num_blocks = (2048 / bs).clamp(1, 32) } + } + CUDA_SUCCESS +} diff --git a/crates/smolvm-cuda-shim/tests/vecadd.c b/crates/smolvm-cuda-shim/tests/vecadd.c new file mode 100644 index 0000000..b162c00 --- /dev/null +++ b/crates/smolvm-cuda-shim/tests/vecadd.c @@ -0,0 +1,220 @@ +/* Unmodified CUDA Driver-API program — the acceptance test for the smolvm + * libcuda.so.1 shim. Nothing here is smolvm-specific: it is written exactly as + * a program against the real driver would be (hand-declared prototypes stand + * in for cuda.h so no CUDA toolkit is needed to compile it). + * + * Exercises: init, device queries, primary context (the cudart pattern), + * module load from PTX, kernel launch with kernelParams, memcpys, memset, + * device-to-device copy, streams, events, mem info, error strings — and + * resolves a second round of entry points through cuGetProcAddress the way + * the CUDA runtime does. + * + * Build: gcc vecadd.c -o vecadd -L -lcuda -Wl,-rpath, + * Run: ./vecadd (prints SHIM-CUDA-OK on success, exit 0) + */ +#include +#include +#include + +typedef int CUresult; +typedef int CUdevice; +typedef void *CUcontext; +typedef void *CUmodule; +typedef void *CUfunction; +typedef void *CUstream; +typedef void *CUevent; +typedef unsigned long long CUdeviceptr; + +extern CUresult cuInit(unsigned int flags); +extern CUresult cuDriverGetVersion(int *version); +extern CUresult cuDeviceGetCount(int *count); +extern CUresult cuDeviceGet(CUdevice *dev, int ordinal); +extern CUresult cuDeviceGetName(char *name, int len, CUdevice dev); +extern CUresult cuDeviceTotalMem_v2(size_t *bytes, CUdevice dev); +extern CUresult cuDeviceGetAttribute(int *pi, int attrib, CUdevice dev); +extern CUresult cuDevicePrimaryCtxRetain(CUcontext *pctx, CUdevice dev); +extern CUresult cuDevicePrimaryCtxRelease_v2(CUdevice dev); +extern CUresult cuCtxSetCurrent(CUcontext ctx); +extern CUresult cuCtxGetCurrent(CUcontext *pctx); +extern CUresult cuCtxSynchronize(void); +extern CUresult cuModuleLoadData(CUmodule *module, const void *image); +extern CUresult cuModuleGetFunction(CUfunction *fn, CUmodule module, const char *name); +extern CUresult cuModuleUnload(CUmodule module); +extern CUresult cuMemAlloc_v2(CUdeviceptr *dptr, size_t bytes); +extern CUresult cuMemFree_v2(CUdeviceptr dptr); +extern CUresult cuMemcpyHtoD_v2(CUdeviceptr dst, const void *src, size_t bytes); +extern CUresult cuMemcpyDtoH_v2(void *dst, CUdeviceptr src, size_t bytes); +extern CUresult cuMemcpyDtoD_v2(CUdeviceptr dst, CUdeviceptr src, size_t bytes); +extern CUresult cuMemsetD8_v2(CUdeviceptr dptr, unsigned char value, size_t n); +extern CUresult cuMemGetInfo_v2(size_t *free_b, size_t *total_b); +extern CUresult cuLaunchKernel(CUfunction fn, unsigned gx, unsigned gy, unsigned gz, + unsigned bx, unsigned by, unsigned bz, + unsigned shared, CUstream stream, + void **params, void **extra); +extern CUresult cuStreamCreate(CUstream *stream, unsigned flags); +extern CUresult cuStreamSynchronize(CUstream stream); +extern CUresult cuStreamDestroy_v2(CUstream stream); +extern CUresult cuEventCreate(CUevent *ev, unsigned flags); +extern CUresult cuEventRecord(CUevent ev, CUstream stream); +extern CUresult cuEventSynchronize(CUevent ev); +extern CUresult cuEventElapsedTime(float *ms, CUevent start, CUevent end); +extern CUresult cuEventDestroy_v2(CUevent ev); +extern CUresult cuGetErrorName(CUresult code, const char **pstr); +extern CUresult cuGetProcAddress_v2(const char *symbol, void **pfn, int version, + unsigned long long flags, int *status); + +#define CHECK(call) \ + do { \ + CUresult _rc = (call); \ + if (_rc != 0) { \ + const char *_name = "?"; \ + cuGetErrorName(_rc, &_name); \ + fprintf(stderr, "FAIL %s -> %d (%s) at line %d\n", #call, _rc, \ + _name, __LINE__); \ + exit(1); \ + } \ + } while (0) + +static const char *VECADD_PTX = + ".version 7.0\n" + ".target sm_52\n" + ".address_size 64\n" + ".visible .entry vecadd(.param .u64 a, .param .u64 b, .param .u64 c, .param .u32 n)\n" + "{ .reg .pred %p<2>; .reg .f32 %f<4>; .reg .b32 %r<6>; .reg .b64 %rd<11>;\n" + " ld.param.u64 %rd1,[a]; ld.param.u64 %rd2,[b]; ld.param.u64 %rd3,[c]; ld.param.u32 %r2,[n];\n" + " mov.u32 %r3,%ntid.x; mov.u32 %r4,%ctaid.x; mov.u32 %r5,%tid.x; mad.lo.s32 %r1,%r4,%r3,%r5;\n" + " setp.ge.u32 %p1,%r1,%r2; @%p1 bra $E;\n" + " cvta.to.global.u64 %rd4,%rd1; cvta.to.global.u64 %rd5,%rd2; cvta.to.global.u64 %rd6,%rd3;\n" + " mul.wide.u32 %rd7,%r1,4; add.s64 %rd8,%rd4,%rd7; add.s64 %rd9,%rd5,%rd7; add.s64 %rd10,%rd6,%rd7;\n" + " ld.global.f32 %f1,[%rd8]; ld.global.f32 %f2,[%rd9]; add.f32 %f3,%f1,%f2; st.global.f32 [%rd10],%f3;\n" + "$E: ret; }\n"; + +enum { N = 4096 }; + +int main(void) { + CHECK(cuInit(0)); + + int version = 0, count = 0; + CHECK(cuDriverGetVersion(&version)); + CHECK(cuDeviceGetCount(&count)); + if (count < 1) { + fprintf(stderr, "FAIL no devices\n"); + return 1; + } + CUdevice dev; + CHECK(cuDeviceGet(&dev, 0)); + char name[256]; + CHECK(cuDeviceGetName(name, sizeof name, dev)); + size_t total = 0; + CHECK(cuDeviceTotalMem_v2(&total, dev)); + int cc_major = 0; + CHECK(cuDeviceGetAttribute(&cc_major, /*COMPUTE_CAPABILITY_MAJOR*/ 75, dev)); + printf("device 0: %s (%zu MiB, cc %d.x, driver %d)\n", name, + total >> 20, cc_major, version); + + /* Primary context, exactly as the CUDA runtime does it. */ + CUcontext ctx = NULL; + CHECK(cuDevicePrimaryCtxRetain(&ctx, dev)); + CHECK(cuCtxSetCurrent(ctx)); + CUcontext cur = NULL; + CHECK(cuCtxGetCurrent(&cur)); + if (cur != ctx) { + fprintf(stderr, "FAIL current ctx mismatch\n"); + return 1; + } + + CUmodule module; + CUfunction vecadd; + CHECK(cuModuleLoadData(&module, VECADD_PTX)); + CHECK(cuModuleGetFunction(&vecadd, module, "vecadd")); + + static float a[N], b[N], c[N]; + for (int i = 0; i < N; i++) { + a[i] = (float)i; + b[i] = 2.0f * (float)i; + } + CUdeviceptr da, db, dc, dscratch; + CHECK(cuMemAlloc_v2(&da, sizeof a)); + CHECK(cuMemAlloc_v2(&db, sizeof b)); + CHECK(cuMemAlloc_v2(&dc, sizeof c)); + CHECK(cuMemAlloc_v2(&dscratch, sizeof c)); + CHECK(cuMemcpyHtoD_v2(da, a, sizeof a)); + CHECK(cuMemcpyHtoD_v2(db, b, sizeof b)); + CHECK(cuMemsetD8_v2(dc, 0, sizeof c)); + + size_t mem_free = 0, mem_total = 0; + CHECK(cuMemGetInfo_v2(&mem_free, &mem_total)); + + /* Launch on a created stream, timed with events. */ + CUstream stream; + CUevent ev_start, ev_end; + CHECK(cuStreamCreate(&stream, 0)); + CHECK(cuEventCreate(&ev_start, 0)); + CHECK(cuEventCreate(&ev_end, 0)); + + unsigned block = 256, grid = (N + block - 1) / block; + unsigned n = N; + void *params[] = { &da, &db, &dc, &n }; + CHECK(cuEventRecord(ev_start, stream)); + CHECK(cuLaunchKernel(vecadd, grid, 1, 1, block, 1, 1, 0, stream, params, NULL)); + CHECK(cuEventRecord(ev_end, stream)); + CHECK(cuStreamSynchronize(stream)); + CHECK(cuEventSynchronize(ev_end)); + float ms = -1.0f; + CHECK(cuEventElapsedTime(&ms, ev_start, ev_end)); + + /* Round-trip the result through a device-to-device copy. */ + CHECK(cuMemcpyDtoD_v2(dscratch, dc, sizeof c)); + CHECK(cuMemcpyDtoH_v2(c, dscratch, sizeof c)); + for (int i = 0; i < N; i++) { + float want = a[i] + b[i]; + if (c[i] < want - 1e-2f || c[i] > want + 1e-2f) { + fprintf(stderr, "FAIL mismatch at %d: got %f want %f\n", i, c[i], want); + return 1; + } + } + printf("vecadd n=%d verified (kernel %.3f ms, %zu/%zu MiB free)\n", N, ms, + mem_free >> 20, mem_total >> 20); + + /* Second pass resolving entry points via cuGetProcAddress — the path the + * CUDA runtime takes for every driver call since 11.3. */ + CUresult (*p_cuCtxSynchronize)(void) = NULL; + CUresult (*p_cuMemcpyDtoH)(void *, CUdeviceptr, size_t) = NULL; + int status = -1; + CHECK(cuGetProcAddress_v2("cuCtxSynchronize", (void **)&p_cuCtxSynchronize, + 12000, 0, &status)); + CHECK(cuGetProcAddress_v2("cuMemcpyDtoH", (void **)&p_cuMemcpyDtoH, + 12000, 0, &status)); + if (!p_cuCtxSynchronize || !p_cuMemcpyDtoH) { + fprintf(stderr, "FAIL cuGetProcAddress returned NULL fn\n"); + return 1; + } + CHECK(p_cuCtxSynchronize()); + memset(c, 0, sizeof c); + CHECK(p_cuMemcpyDtoH(c, dc, sizeof c)); + if (c[N - 1] != a[N - 1] + b[N - 1]) { + fprintf(stderr, "FAIL proc-address path mismatch\n"); + return 1; + } + + /* Unknown symbols must be reported, not crash. */ + void *bogus = (void *)1; + if (cuGetProcAddress_v2("cuDefinitelyNotReal", &bogus, 12000, 0, &status) == 0 || + bogus != NULL) { + fprintf(stderr, "FAIL unknown symbol not rejected\n"); + return 1; + } + + CHECK(cuEventDestroy_v2(ev_start)); + CHECK(cuEventDestroy_v2(ev_end)); + CHECK(cuStreamDestroy_v2(stream)); + CHECK(cuMemFree_v2(da)); + CHECK(cuMemFree_v2(db)); + CHECK(cuMemFree_v2(dc)); + CHECK(cuMemFree_v2(dscratch)); + CHECK(cuModuleUnload(module)); + CHECK(cuDevicePrimaryCtxRelease_v2(dev)); + + printf("SHIM-CUDA-OK: %s\n", name); + return 0; +} diff --git a/crates/smolvm-cuda/Cargo.toml b/crates/smolvm-cuda/Cargo.toml new file mode 100644 index 0000000..e234233 --- /dev/null +++ b/crates/smolvm-cuda/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "smolvm-cuda" +version = "1.5.2" +edition = "2021" +description = "CUDA Driver-API remoting over vsock for smolvm guests" +license = "Apache-2.0" + +[features] +# host: server-side dispatch + CPU emulation backend (std only). +# gpu: real driver backend via libloading (implies host). +default = ["host", "gpu"] +host = [] +gpu = ["host", "dep:libloading"] + +[dependencies] +libloading = { version = "0.8", optional = true } +libc = "0.2" diff --git a/crates/smolvm-cuda/build.rs b/crates/smolvm-cuda/build.rs new file mode 100644 index 0000000..ae87bbc --- /dev/null +++ b/crates/smolvm-cuda/build.rs @@ -0,0 +1,37 @@ +//! Emit `SMOLVM_PROTO_HASH`: a fingerprint of the wire-defining source, so a +//! shim and a server built from different source (the classic "rebuilt one +//! binary, not the other" stale-binary trap that silently corrupts results) +//! fail the connect handshake loudly instead. The hash covers only files both +//! the client (shim) and the host (server) compile, i.e. the wire contract — +//! not host-only backend code, which doesn't change the byte protocol. +//! +//! FNV-1a (not `DefaultHasher`) so the value is reproducible across toolchains +//! and platforms: a guest-built shim and a host-built server must agree. + +fn fnv1a(seed: u64, bytes: &[u8]) -> u64 { + let mut h = seed; + for &b in bytes { + h ^= b as u64; + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + +fn main() { + let files = [ + "src/proto.rs", + "src/client.rs", + "src/ring.rs", + "src/generated/cublas_guest.rs", + "src/generated/cudnn_guest.rs", + ]; + // A manual epoch to force a bump on wire changes the file set misses. + const PROTO_EPOCH: &[u8] = b"epoch-1"; + let mut h = fnv1a(0xcbf2_9ce4_8422_2325, PROTO_EPOCH); + for f in files { + println!("cargo:rerun-if-changed={f}"); + let bytes = std::fs::read(f).unwrap_or_default(); + h = fnv1a(h, &bytes); + } + println!("cargo:rustc-env=SMOLVM_PROTO_HASH={h:016x}"); +} diff --git a/crates/smolvm-cuda/examples/gpu_loopback.rs b/crates/smolvm-cuda/examples/gpu_loopback.rs new file mode 100644 index 0000000..484d9d6 --- /dev/null +++ b/crates/smolvm-cuda/examples/gpu_loopback.rs @@ -0,0 +1,129 @@ +//! GPU live-verification harness. +//! +//! Exercises the *real* CUDA stack — `GpuBackend` (driver via `nvcuda.dll` / +//! `libcuda.so.1`), the `host::serve` dispatch, the wire `proto`, and the guest +//! `Client` — over a local TCP loopback socket, running an arbitrary-module + +//! named-kernel workload on the host GPU. This is the same code path production +//! uses; only the vsock/microVM transport is swapped for loopback so it can run +//! directly on a GPU host (e.g. the RTX 3070) without booting a VM. +//! +//! Run on a machine with an NVIDIA driver: +//! cargo run --release --example gpu_loopback --features gpu +//! +//! Prints the device name and `GPU-VERIFY-OK` on success; exits 2 if no GPU +//! driver loads, 3 on a result mismatch. + +use smolvm_cuda::client::Client; +use smolvm_cuda::host::{serve, Backend, GpuBackend}; +use std::net::{TcpListener, TcpStream}; + +const VECADD_PTX: &str = r#".version 7.0 +.target sm_52 +.address_size 64 +.visible .entry vecadd(.param .u64 a, .param .u64 b, .param .u64 c, .param .u32 n) +{ .reg .pred %p<2>; .reg .f32 %f<4>; .reg .b32 %r<6>; .reg .b64 %rd<11>; + ld.param.u64 %rd1,[a]; ld.param.u64 %rd2,[b]; ld.param.u64 %rd3,[c]; ld.param.u32 %r2,[n]; + mov.u32 %r3,%ntid.x; mov.u32 %r4,%ctaid.x; mov.u32 %r5,%tid.x; mad.lo.s32 %r1,%r4,%r3,%r5; + setp.ge.u32 %p1,%r1,%r2; @%p1 bra $E; + cvta.to.global.u64 %rd4,%rd1; cvta.to.global.u64 %rd5,%rd2; cvta.to.global.u64 %rd6,%rd3; + mul.wide.u32 %rd7,%r1,4; add.s64 %rd8,%rd4,%rd7; add.s64 %rd9,%rd5,%rd7; add.s64 %rd10,%rd6,%rd7; + ld.global.f32 %f1,[%rd8]; ld.global.f32 %f2,[%rd9]; add.f32 %f3,%f1,%f2; st.global.f32 [%rd10],%f3; +$E: ret; } +"#; + +fn main() { + let listener = TcpListener::bind("127.0.0.1:0").expect("bind loopback"); + let addr = listener.local_addr().unwrap(); + + // Host side: real driver backend, served on the accepting thread (the CUDA + // context is created and stays current on this one thread). + let server = std::thread::spawn(move || { + let (stream, _) = listener.accept().expect("accept"); + let mut backend: Box = match GpuBackend::load() { + Ok(b) => Box::new(b), + Err(e) => { + eprintln!("gpu_loopback: no CUDA driver: {e}"); + std::process::exit(2); + } + }; + let _ = serve(stream, backend.as_mut()); + }); + + let mut cu = Client::new(TcpStream::connect(addr).expect("connect")); + cu.init().expect("cuInit"); + let count = cu.device_get_count().expect("device count"); + let name = cu.device_get_name(0).expect("device name"); + let vram = cu.device_total_mem(0).expect("total mem"); + println!( + "gpu_loopback: {count} device(s); device 0 = {name} ({} MiB)", + vram / (1024 * 1024) + ); + let _ctx = cu.ctx_create(0).expect("ctx create"); + + // Arbitrary module + named kernel — the general path, not a baked op. + let module = cu + .module_load_data(VECADD_PTX.as_bytes()) + .expect("module load"); + let func = cu + .module_get_function(module, "vecadd") + .expect("get function"); + + let n: usize = 1 << 16; // 65536 elements + let bytes = (n * 4) as u64; + let a: Vec = (0..n).map(|i| i as f32).collect(); + let b: Vec = (0..n).map(|i| (3 * i) as f32).collect(); + let da = cu.mem_alloc(bytes).expect("alloc a"); + let db = cu.mem_alloc(bytes).expect("alloc b"); + let dc = cu.mem_alloc(bytes).expect("alloc c"); + cu.memcpy_htod(da, bytemuck(&a), 0).expect("h2d a"); + cu.memcpy_htod(db, bytemuck(&b), 0).expect("h2d b"); + + let block = 256u32; + let grid = (n as u32).div_ceil(block); + cu.launch_kernel( + func, + [grid, 1, 1], + [block, 1, 1], + 0, + 0, + &[ + da.to_le_bytes().to_vec(), + db.to_le_bytes().to_vec(), + dc.to_le_bytes().to_vec(), + (n as u32).to_le_bytes().to_vec(), + ], + ) + .expect("launch"); + cu.ctx_synchronize().expect("sync"); + + let out = cu.memcpy_dtoh(dc, bytes, 0).expect("d2h"); + let c: Vec = out + .chunks_exact(4) + .map(|p| f32::from_le_bytes(p.try_into().unwrap())) + .collect(); + for i in 0..n { + let expect = a[i] + b[i]; // i + 3i = 4i + if (c[i] - expect).abs() > 1e-2 { + eprintln!("gpu_loopback: MISMATCH at {i}: got {} want {expect}", c[i]); + std::process::exit(3); + } + } + cu.mem_free(da).ok(); + cu.mem_free(db).ok(); + cu.mem_free(dc).ok(); + drop(cu); + let _ = server.join(); + println!( + "gpu_loopback: vecadd n={n} verified on GPU (c[1]={}, c[{}]={})", + c[1], + n - 1, + c[n - 1] + ); + println!("GPU-VERIFY-OK: {name}"); +} + +/// Reinterpret `&[f32]` as bytes (f32 has no invalid bit patterns). +fn bytemuck(v: &[f32]) -> &[u8] { + // SAFETY: f32 is plain-old-data; reading its bytes is sound. + unsafe { std::slice::from_raw_parts(v.as_ptr() as *const u8, std::mem::size_of_val(v)) } +} diff --git a/crates/smolvm-cuda/examples/shim_server.rs b/crates/smolvm-cuda/examples/shim_server.rs new file mode 100644 index 0000000..d4f6643 --- /dev/null +++ b/crates/smolvm-cuda/examples/shim_server.rs @@ -0,0 +1,40 @@ +//! Host-side CUDA RPC server on TCP loopback, for exercising the +//! `smolvm-cuda-shim` ABI without a VM. +//! +//! Serves the real GPU backend when the driver loads, else CPU emulation, one +//! connection at a time (each on its own thread — the CUDA context binds to +//! the serving thread, mirroring the production per-connection model). +//! +//! cargo run --release --example shim_server -p smolvm-cuda --features gpu -- [addr] +//! +//! Default addr 127.0.0.1:7901. Prints `SHIM-SERVER-READY ` once +//! listening, so scripts can wait for the line then point a shim-linked binary +//! at it via `SMOLVM_CUDA_RPC=tcp:`. + +use smolvm_cuda::host::{serve, Backend, CpuBackend, GpuBackend}; +use std::net::TcpListener; + +fn main() { + let addr = std::env::args() + .nth(1) + .unwrap_or_else(|| "127.0.0.1:7901".into()); + let listener = TcpListener::bind(&addr).expect("bind"); + let backend_kind = match GpuBackend::load() { + Ok(_) => "gpu", + Err(_) => "cpu-emulation", + }; + println!("SHIM-SERVER-READY {addr} {backend_kind}"); + for stream in listener.incoming() { + let Ok(stream) = stream else { continue }; + let __r = stream.set_nodelay(true); // low-latency request/response + std::thread::spawn(move || { + let mut backend: Box = match GpuBackend::load() { + Ok(b) => Box::new(b), + Err(_) => Box::::default(), + }; + if let Err(e) = serve(stream, backend.as_mut()) { + eprintln!("[conn-err] {e}"); + } + }); + } +} diff --git a/crates/smolvm-cuda/src/client.rs b/crates/smolvm-cuda/src/client.rs new file mode 100644 index 0000000..9cee7fd --- /dev/null +++ b/crates/smolvm-cuda/src/client.rs @@ -0,0 +1,1464 @@ +//! Guest-side CUDA-RPC client: marshal `cu*` calls over a byte stream. +//! +//! Transport-agnostic — it takes any [`Read`]/[`Write`], so the guest binary +//! supplies an `AF_VSOCK` stream while tests supply an in-memory pipe. Each +//! method does one request→response round-trip and surfaces a non-zero +//! `CUresult` as [`CudaRpcError::Cuda`]. + +use crate::proto::{decode_response, encode_request, read_msg, write_msg, Op, Request, Response}; +use std::io::{self, Read, Write}; + +/// A client-side failure: transport error, a CUDA error code from the host, or +/// a protocol mismatch (host returned the wrong response shape). +#[derive(Debug)] +pub enum CudaRpcError { + Io(io::Error), + /// Non-zero `CUresult` returned by the host driver. + Cuda(i32), + Protocol(&'static str), +} + +impl std::fmt::Display for CudaRpcError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CudaRpcError::Io(e) => write!(f, "cuda-rpc io: {e}"), + CudaRpcError::Cuda(c) => write!(f, "CUDA error {c}"), + CudaRpcError::Protocol(m) => write!(f, "cuda-rpc protocol: {m}"), + } + } +} +impl std::error::Error for CudaRpcError {} +impl From for CudaRpcError { + fn from(e: io::Error) -> Self { + CudaRpcError::Io(e) + } +} + +pub type Result = std::result::Result; + +/// Debug profiling (`SMOLVM_CUDA_COUNT_SYNC=1`): tally synchronous round-trips +/// per op, dumped to stderr every 4096 calls, to show what still serializes an +/// asynchronously-pipelined workload. For `LibCall` the tally key includes the +/// library id and function index. +fn count_sync(req: &Request, op: Op) { + use std::collections::HashMap; + use std::sync::Mutex; + static COUNTS: Mutex>> = Mutex::new(None); + let key = match req { + Request::LibCall { lib, func, .. } => format!("LibCall(lib={lib},func={func})"), + Request::ModuleGetFunction { name, .. } => format!("ModuleGetFunction({name})"), + Request::FuncGetParamInfo { function } => format!("FuncGetParamInfo(fid={function})"), + Request::ModuleLoadData { image } => format!("ModuleLoadData(len={})", image.len()), + _ => format!("{op:?}"), + }; + let mut g = COUNTS.lock().unwrap(); + let m = g.get_or_insert_with(HashMap::new); + *m.entry(key).or_insert(0) += 1; + let total: u64 = m.values().sum(); + if total.is_multiple_of(4096) { + let mut v: Vec<_> = m.iter().collect(); + v.sort_by(|a, b| b.1.cmp(a.1)); + eprintln!("[sync-counts after {total}]"); + for (k, n) in v.iter().take(12) { + eprintln!(" {n:>8} {k}"); + } + } +} + +/// C-ABI hooks into another shim's connection in the same process, resolved +/// via `dlsym`. The driver shim (`libcuda.so.1`) routes its traffic through +/// the runtime shim's connection with these, so the host sees ONE +/// program-ordered pipeline instead of two independently flushed queues (two +/// queues let the host execute work out of guest program order — fatal once +/// CUDA-graph capture records the misorder). +#[derive(Clone, Copy)] +pub struct Bridge { + /// Append one encoded request to the shared pipeline, fire-and-forget. + /// Nonzero return = transport failure (op-level failures surface later as + /// sticky asynchronous errors on the owning connection). + pub quiet: unsafe extern "C" fn(req: *const u8, len: usize) -> i32, + /// Send one encoded request, receive its response payload (status still + /// in-band). Returns the response length; -1 = transport failure; any + /// other negative = `cap` too small, retry with `-ret` bytes and an empty + /// request to fetch the stashed response. + pub call: unsafe extern "C" fn(req: *const u8, len: usize, resp: *mut u8, cap: usize) -> isize, + /// Settle the shared pipeline (fence); returns the first collected + /// quiet-failure status, 0 if none. + pub drain: unsafe extern "C" fn() -> i32, +} + +/// Debug bisection (`SMOLVM_CUDA_SYNC_OPS=LaunchKernel,LibCall1,LibCall4:10`): +/// force the named op kinds — optionally narrowed to a library id and +/// function index for `LibCall` — to synchronous round-trips while the rest +/// stay deferred, to isolate which deferred class corrupts a failing workload. +fn sync_forced(req: &Request, op: Op) -> bool { + use std::sync::OnceLock; + static SET: OnceLock> = OnceLock::new(); + let set = SET.get_or_init(|| { + std::env::var("SMOLVM_CUDA_SYNC_OPS") + .unwrap_or_default() + .split(',') + .filter(|s| !s.is_empty()) + .map(|s| s.trim().to_ascii_lowercase()) + .collect() + }); + if set.is_empty() { + return false; + } + let kind = format!("{op:?}").to_ascii_lowercase(); + if set.contains(&kind) { + return true; + } + if let Request::LibCall { lib, func, .. } = req { + return set.contains(&format!("libcall{lib}")) + || set.contains(&format!("libcall{lib}:{func}")); + } + false +} + +/// Bench/diagnostic: `SMOLVM_CUDA_RTT_DELAY_US` sleeps this many microseconds +/// per host round-trip, modeling a remote server's network RTT. Batched +/// deferred work pays it once per fence (as a real network would), so the +/// resulting throughput-vs-latency curve shows how tolerant each mode is of +/// distance. Read once (env lookups aren't free on the hot path). +fn rtt_tax() { + use std::sync::atomic::{AtomicI64, Ordering}; + static US: AtomicI64 = AtomicI64::new(-1); + let mut v = US.load(Ordering::Relaxed); + if v == -1 { + v = std::env::var("SMOLVM_CUDA_RTT_DELAY_US") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + US.store(v, Ordering::Relaxed); + } + if v > 0 { + std::thread::sleep(std::time::Duration::from_micros(v as u64)); + } +} + +/// Round-trip one encoded request over a [`Bridge`], growing the response +/// buffer when the callee reports it too small (the callee stashes the +/// response; an empty request fetches the stash). +fn bridge_call_vec(b: &Bridge, req: &[u8]) -> Result> { + let mut buf = vec![0u8; 4096]; + let mut ret = unsafe { (b.call)(req.as_ptr(), req.len(), buf.as_mut_ptr(), buf.len()) }; + if ret < -1 { + buf = vec![0u8; (-ret) as usize]; + ret = unsafe { (b.call)(std::ptr::null(), 0, buf.as_mut_ptr(), buf.len()) }; + } + if ret >= 0 { + buf.truncate(ret as usize); + Ok(buf) + } else { + Err(CudaRpcError::Protocol("bridge call failed")) + } +} + +/// Shared-memory ring transport state (in-VM fast path; see `crate::ring`). +pub struct ClientRing { + /// Guest→host requests (this side produces). + req: crate::ring::Ring, + /// Host→guest completions (this side consumes). + resp: crate::ring::Ring, + /// Guest VAs of the bounce pages: staging for oversized traffic in both + /// directions (requests chunk guest→host, responses spill host→guest — + /// never simultaneously, the queue is strictly request-then-response). + bounce: Vec<*mut u8>, + page_size: usize, +} + +// SAFETY: raw pointers reference process-owned pinned pages; the shim holds +// the client behind a mutex. +unsafe impl Send for ClientRing {} + +/// A CUDA Driver-API client over one connection to the host server. +pub struct Client { + stream: S, + /// Shared-memory ring transport, when negotiated (in-VM fast path). + ring: Option, + /// When set, this client owns no connection: every request rides another + /// shim's connection through these hooks (see [`Bridge`]). + bridge: Option, + /// Count of quiet (fire-and-forget) requests since the last fence. Quiet + /// requests produce no responses; a fence settles them all at once. + deferred: usize, + /// First non-zero status collected from a deferred response — surfaced at + /// the next launch/synchronize, mirroring CUDA's asynchronous ("sticky") + /// error reporting. + sticky: i32, + /// Kill-switch: `SMOLVM_CUDA_ASYNC=0` restores strict per-call round-trips. + defer_enabled: bool, + /// Framed-but-unsent deferred requests. Batching them into one write turns + /// a launch storm's thousand syscalls into a handful; flushed before any + /// read (a response can only exist for a request the host has seen). + wbuf: Vec, +} + +/// Outstanding-response cap. Responses are ~8 bytes, so even the smallest +/// socket buffer holds far more than this; the cap just bounds how much work +/// can race ahead of an error being noticed. +const MAX_DEFERRED: usize = 512; +/// Flush the deferred-write buffer beyond this size even without a sync point, +/// so bulk H2D byte-shipping doesn't accumulate unbounded copies in memory. +const WBUF_FLUSH: usize = 256 * 1024; + +impl Client { + pub fn new(stream: S) -> Self { + Client { + stream, + ring: None, + bridge: None, + deferred: 0, + sticky: 0, + defer_enabled: std::env::var("SMOLVM_CUDA_ASYNC").as_deref() != Ok("0"), + wbuf: Vec::new(), + } + } + + /// A client that owns no connection: every request rides another shim's + /// connection via `bridge`. `stream` is never read or written. + pub fn new_bridged(stream: S, bridge: Bridge) -> Self { + let mut c = Self::new(stream); + c.bridge = Some(bridge); + c + } + + pub fn is_bridged(&self) -> bool { + self.bridge.is_some() + } + + /// Negotiate the shared-memory ring transport (in-VM fast path). `req`, + /// `resp` and `bounce` are (page VAs, page GPAs) of zeroed, mlocked, + /// page-aligned guest allocations. On Ok the connection switches: the + /// socket carries only doorbell bytes from here on. + #[allow(clippy::type_complexity)] + pub fn ring_setup( + &mut self, + page_size: usize, + req: (Vec<*mut u8>, Vec), + resp: (Vec<*mut u8>, Vec), + bounce: (Vec<*mut u8>, Vec), + ) -> Result<()> { + self.call( + &Request::RingSetup { + page_size: page_size as u32, + req_pages: req.1, + resp_pages: resp.1, + bounce_pages: bounce.1, + }, + Op::RingSetup, + )?; + // SAFETY: pages are owned by the shim and stay mapped + resident for + // the process lifetime (mlocked, never freed). + self.ring = Some(ClientRing { + req: unsafe { crate::ring::Ring::from_pages(req.0, page_size) }, + resp: unsafe { crate::ring::Ring::from_pages(resp.0, page_size) }, + bounce: bounce.0, + page_size, + }); + Ok(()) + } + + pub fn is_ring(&self) -> bool { + self.ring.is_some() + } + + /// Push one frame (socket-payload bytes: QUIET/FENCE prefix included) to + /// the request ring. Oversized frames chunk through the bounce pages: + /// each chunk record is acked by the host before the pages are reused, + /// except the last — the caller's own response wait covers it, so every + /// oversized frame MUST be a sync op (quiet callers upgrade to sync). + fn ring_push(&mut self, frame: &[u8]) -> Result<()> { + use crate::ring::{INLINE_MAX, LEN_INDIRECT}; + if frame.len() <= INLINE_MAX { + let ring = self.ring.as_ref().expect("ring transport"); + while !ring.req.try_push(frame, 0) { + std::hint::spin_loop(); // host drains continuously + } + if ring.req.take_parked() { + self.stream.write_all(&[1u8])?; + self.stream.flush()?; + } + return Ok(()); + } + let (bounce_cap, page_size) = { + let ring = self.ring.as_ref().expect("ring transport"); + (ring.bounce.len() * ring.page_size, ring.page_size) + }; + let total = frame.len(); + let mut off = 0; + while off < total { + let chunk = (total - off).min(bounce_cap); + { + let ring = self.ring.as_ref().expect("ring transport"); + for (i, piece) in frame[off..off + chunk].chunks(page_size).enumerate() { + // SAFETY: bounce pages are live shim allocations of + // page_size bytes each; piece fits by construction. + unsafe { + std::ptr::copy_nonoverlapping(piece.as_ptr(), ring.bounce[i], piece.len()); + } + } + let mut rec = [0u8; 16]; + rec[..8].copy_from_slice(&(total as u64).to_le_bytes()); + rec[8..].copy_from_slice(&(chunk as u64).to_le_bytes()); + while !ring.req.try_push(&rec, LEN_INDIRECT) { + std::hint::spin_loop(); + } + if ring.req.take_parked() { + self.stream.write_all(&[1u8])?; + self.stream.flush()?; + } + } + off += chunk; + if off < total { + // Host acks each non-final chunk once copied out; the final + // chunk is covered by the operation's own response. + let _ack = self.ring_pop_response()?; + } + } + Ok(()) + } + + /// Await exactly one completion record: spin briefly, then park and block + /// on a doorbell byte. + fn ring_pop_response(&mut self) -> Result> { + use crate::ring::LEN_INDIRECT; + loop { + { + let ring = self.ring.as_ref().expect("ring transport"); + let mut popped = ring.resp.try_pop(); + if popped.is_none() { + for _ in 0..20_000 { + popped = ring.resp.try_pop(); + if popped.is_some() { + break; + } + std::hint::spin_loop(); + } + } + if let Some((payload, flags)) = popped { + if flags & LEN_INDIRECT == 0 { + return Ok(payload); + } + // Oversized response: [total][chunk] records, each chunk + // staged in the bounce pages; we post a continue record + // after copying each non-final chunk out. + let mut hdr = payload; + let mut buf: Vec = Vec::new(); + loop { + if hdr.len() < 16 { + return Err(CudaRpcError::Protocol("ring: short indirect")); + } + let total = u64::from_le_bytes(hdr[..8].try_into().unwrap()) as usize; + let chunk = u64::from_le_bytes(hdr[8..16].try_into().unwrap()) as usize; + if chunk > ring.bounce.len() * ring.page_size { + return Err(CudaRpcError::Protocol("ring: bounce overrun")); + } + let mut left = chunk; + for &page in &ring.bounce { + if left == 0 { + break; + } + let take = left.min(ring.page_size); + // SAFETY: bounce pages are live shim allocations. + unsafe { + buf.extend_from_slice(std::slice::from_raw_parts( + page as *const u8, + take, + )); + } + left -= take; + } + if buf.len() >= total { + return Ok(buf); + } + // More chunks: tell the host the pages are free. + while !ring.req.try_push(&[0xFF; 16], LEN_INDIRECT) { + std::hint::spin_loop(); + } + hdr = loop { + if let Some((p, f)) = ring.resp.try_pop() { + if f & LEN_INDIRECT == 0 { + return Err(CudaRpcError::Protocol("ring: expected chunk")); + } + break p; + } + std::hint::spin_loop(); + }; + } + } + if ring.resp.park() { + continue; // record landed while parking + } + } + // Blocked: wait for the host's doorbell byte on the socket. + let mut byte = [0u8; 1]; + match self.stream.read(&mut byte) { + Ok(0) => return Err(CudaRpcError::Protocol("host closed ring connection")), + Ok(_) => {} + Err(e) => return Err(e.into()), + } + self.ring.as_ref().expect("ring transport").resp.unpark(); + } + } + + /// One sync round-trip over the rings. + fn ring_roundtrip(&mut self, frame: &[u8]) -> Result> { + self.ring_push(frame)?; + rtt_tax(); + self.ring_pop_response() + } + + /// Send everything buffered by deferred calls. + fn flush_wbuf(&mut self) -> Result<()> { + if !self.wbuf.is_empty() { + self.stream.write_all(&self.wbuf)?; + self.stream.flush()?; + self.wbuf.clear(); + } + Ok(()) + } + + /// Force every op to a synchronous round-trip (disable the deferred + /// pipeline). Used by the driver shim: it shares one guest program-order + /// stream with the runtime shim's connection, and two independently + /// flushed deferred queues would let the host execute their work out of + /// program order (fatal once CUDA-graph capture records the misorder). + pub fn set_defer_enabled(&mut self, on: bool) { + self.defer_enabled = on; + } + + /// Settle all fire-and-forget work with a single fence round-trip: quiet + /// requests produce no per-op responses (each response read costs a guest + /// wake-up on vsock), so one fence reply carries the first failure among + /// them. + pub fn drain(&mut self) -> Result<()> { + if self.ring.is_some() { + if self.deferred == 0 { + return Ok(()); + } + self.deferred = 0; + let resp = self.ring_roundtrip(&[crate::proto::FENCE_OP])?; + if resp.len() >= 4 { + let status = i32::from_le_bytes(resp[..4].try_into().unwrap()); + if status != 0 && self.sticky == 0 { + self.sticky = status; + } + } + return Ok(()); + } + if let Some(b) = self.bridge { + let st = unsafe { (b.drain)() }; + if st != 0 && self.sticky == 0 { + self.sticky = st; + } + return Ok(()); + } + if self.deferred == 0 { + return self.flush_wbuf(); + } + self.deferred = 0; + self.wbuf.extend_from_slice(&1u32.to_le_bytes()); + self.wbuf.push(crate::proto::FENCE_OP); + self.flush_wbuf()?; + rtt_tax(); + let payload = + read_msg(&mut self.stream)?.ok_or(CudaRpcError::Protocol("host closed mid-fence"))?; + if payload.len() >= 4 { + let status = i32::from_le_bytes(payload[..4].try_into().unwrap()); + if status != 0 && self.sticky == 0 { + if std::env::var_os("SMOLVM_CUDA_TRACE_STICKY").is_some() { + eprintln!("[sticky] fence collected {status}"); + } + self.sticky = status; + } + } + Ok(()) + } + + /// Take (and clear) the sticky asynchronous error, if any. Non-blocking: + /// reports failures already collected by a past drain, the way + /// `cudaGetLastError` reports asynchronous errors observed so far. + pub fn take_sticky(&mut self) -> i32 { + std::mem::take(&mut self.sticky) + } + + /// Send `req` without waiting for its (status-only) response. Ordering is + /// preserved — the host serves one request at a time in arrival order — so + /// the deferred work is complete by the time any later round-trip returns. + /// Fails fast if an earlier deferred op already failed (sticky). + fn call_deferred(&mut self, req: &Request, op: Op) -> Result<()> { + self.call_deferred_kind(req, op, false) + } + + fn call_deferred_kind(&mut self, req: &Request, op: Op, _is_libcall: bool) -> Result<()> { + if !self.defer_enabled || sync_forced(req, op) { + return self.call(req, op).map(|_| ()); + } + if let Some(b) = self.bridge { + // Push into the owning connection's pipeline NOW — buffering here + // would re-create the two-queue reorder the bridge exists to kill. + if self.sticky != 0 { + return Err(CudaRpcError::Cuda(std::mem::take(&mut self.sticky))); + } + let payload = encode_request(req); + let rc = unsafe { (b.quiet)(payload.as_ptr(), payload.len()) }; + if rc != 0 { + return Err(CudaRpcError::Cuda(rc)); + } + return Ok(()); + } + if self.ring.is_some() { + if self.deferred >= MAX_DEFERRED { + self.drain()?; + } + if self.sticky != 0 { + return Err(CudaRpcError::Cuda(std::mem::take(&mut self.sticky))); + } + let payload = encode_request(req); + if payload.len() < crate::ring::INLINE_MAX { + let mut frame = Vec::with_capacity(payload.len() + 1); + frame.push(crate::proto::QUIET_PREFIX); + frame.extend_from_slice(&payload); + self.ring_push(&frame)?; + self.deferred += 1; + } else { + // Oversized quiet frames go indirect, which needs the staging + // buffer alive until consumption — round-trip instead and + // fold a failure into the sticky slot. + let resp = self.ring_roundtrip(&payload)?; + if resp.len() >= 4 { + let st = i32::from_le_bytes(resp[..4].try_into().unwrap()); + if st != 0 && self.sticky == 0 { + self.sticky = st; + } + } + } + return Ok(()); + } + if self.deferred >= MAX_DEFERRED { + self.drain()?; + } + if self.sticky != 0 { + return Err(CudaRpcError::Cuda(std::mem::take(&mut self.sticky))); + } + // Frame into the batch buffer as a QUIET request (no response) — one + // write syscall per sync point (or per WBUF_FLUSH bytes), and one + // fence reply per drain instead of one reply per request. + let payload = encode_request(req); + self.wbuf + .extend_from_slice(&((payload.len() + 1) as u32).to_le_bytes()); + self.wbuf.push(crate::proto::QUIET_PREFIX); + self.wbuf.extend_from_slice(&payload); + self.deferred += 1; + if self.wbuf.len() >= WBUF_FLUSH { + self.flush_wbuf()?; + } + Ok(()) + } + + /// Fire-and-forget `LibCall` for library functions with no output + /// parameters (GEMMs, conv/batch-norm executes, stream setters): the call + /// reports optimistic success and a real failure surfaces as a sticky + /// asynchronous error, like a failed kernel launch. + pub fn lib_call_deferred(&mut self, lib: u8, func: u16, args: Vec) -> Result<()> { + self.call_deferred_kind(&Request::LibCall { lib, func, args }, Op::LibCall, true) + } + + fn call(&mut self, req: &Request, op: Op) -> Result { + if std::env::var_os("SMOLVM_CUDA_COUNT_SYNC").is_some() { + count_sync(req, op); + } + let payload = if self.ring.is_some() { + self.ring_roundtrip(&encode_request(req))? + } else if let Some(b) = self.bridge { + bridge_call_vec(&b, &encode_request(req))? + } else { + // Flush, don't fence: the host serves in arrival order, so this + // request's response already proves every deferred request before + // it was consumed. A fence here would double the RTT of every + // sync op under deferred load just to surface quiet failures a + // little earlier — they still surface at explicit sync points + // (drain) and the MAX_DEFERRED backstop. + self.flush_wbuf()?; + write_msg(&mut self.stream, &encode_request(req))?; + rtt_tax(); + read_msg(&mut self.stream)?.ok_or(CudaRpcError::Protocol("host closed mid-call"))? + }; + let (status, resp) = decode_response(op, &payload)?; + if status != 0 { + return Err(CudaRpcError::Cuda(status)); + } + Ok(resp) + } + + /// Serve a bridged peer: append one pre-encoded request to this + /// connection's deferred pipeline, preserving arrival order. In strict + /// mode (`SMOLVM_CUDA_ASYNC=0`) the request round-trips instead and a + /// failure status is collected as this connection's sticky error. + pub fn raw_quiet(&mut self, payload: &[u8]) -> Result<()> { + if self.ring.is_some() && self.defer_enabled { + if self.deferred >= MAX_DEFERRED { + self.drain()?; + } + if self.sticky != 0 { + return Err(CudaRpcError::Cuda(std::mem::take(&mut self.sticky))); + } + if payload.len() < crate::ring::INLINE_MAX { + let mut frame = Vec::with_capacity(payload.len() + 1); + frame.push(crate::proto::QUIET_PREFIX); + frame.extend_from_slice(payload); + self.ring_push(&frame)?; + self.deferred += 1; + } else { + let resp = self.ring_roundtrip(payload)?; + if resp.len() >= 4 { + let st = i32::from_le_bytes(resp[..4].try_into().unwrap()); + if st != 0 && self.sticky == 0 { + self.sticky = st; + } + } + } + return Ok(()); + } + if !self.defer_enabled { + let resp = self.raw_call(payload)?; + if resp.len() >= 4 { + let st = i32::from_le_bytes(resp[..4].try_into().unwrap()); + if st != 0 && self.sticky == 0 { + self.sticky = st; + } + } + return Ok(()); + } + if self.deferred >= MAX_DEFERRED { + self.drain()?; + } + if self.sticky != 0 { + return Err(CudaRpcError::Cuda(std::mem::take(&mut self.sticky))); + } + self.wbuf + .extend_from_slice(&((payload.len() + 1) as u32).to_le_bytes()); + self.wbuf.push(crate::proto::QUIET_PREFIX); + self.wbuf.extend_from_slice(payload); + self.deferred += 1; + if self.wbuf.len() >= WBUF_FLUSH { + self.flush_wbuf()?; + } + Ok(()) + } + + /// Serve a bridged peer: one pre-encoded synchronous round-trip. Returns + /// the raw response payload — the status stays in-band for the peer to + /// decode, so nothing is lost to error mapping. + pub fn raw_call(&mut self, payload: &[u8]) -> Result> { + if self.ring.is_some() { + return self.ring_roundtrip(payload); + } + // Flush, don't fence — see `call`. + self.flush_wbuf()?; + write_msg(&mut self.stream, payload)?; + rtt_tax(); + read_msg(&mut self.stream)?.ok_or(CudaRpcError::Protocol("host closed mid-call")) + } + + pub fn init(&mut self) -> Result<()> { + self.call( + &Request::Init { + proto_hash: crate::PROTO_HASH, + }, + Op::Init, + ) + .map(|_| ()) + } + + pub fn device_get_count(&mut self) -> Result { + match self.call(&Request::DeviceGetCount, Op::DeviceGetCount)? { + Response::Count(n) => Ok(n), + _ => Err(CudaRpcError::Protocol("expected Count")), + } + } + + pub fn device_get_name(&mut self, device: i32) -> Result { + match self.call(&Request::DeviceGetName { device }, Op::DeviceGetName)? { + Response::Name(s) => Ok(s), + _ => Err(CudaRpcError::Protocol("expected Name")), + } + } + + pub fn device_total_mem(&mut self, device: i32) -> Result { + match self.call(&Request::DeviceTotalMem { device }, Op::DeviceTotalMem)? { + Response::Bytes(v) => Ok(v), + _ => Err(CudaRpcError::Protocol("expected Bytes")), + } + } + + pub fn ctx_create(&mut self, device: i32) -> Result { + match self.call(&Request::CtxCreate { device }, Op::CtxCreate)? { + Response::Handle(h) => Ok(h), + _ => Err(CudaRpcError::Protocol("expected Handle")), + } + } + + pub fn ctx_destroy(&mut self, ctx: u64) -> Result<()> { + self.call(&Request::CtxDestroy { ctx }, Op::CtxDestroy) + .map(|_| ()) + } + + pub fn module_load_data(&mut self, image: &[u8]) -> Result { + match self.call( + &Request::ModuleLoadData { + image: image.to_vec(), + }, + Op::ModuleLoadData, + )? { + Response::Handle(h) => Ok(h), + _ => Err(CudaRpcError::Protocol("expected Handle")), + } + } + + pub fn module_get_function(&mut self, module: u64, name: &str) -> Result { + match self.call( + &Request::ModuleGetFunction { + module, + name: name.to_string(), + }, + Op::ModuleGetFunction, + )? { + Response::Handle(h) => Ok(h), + _ => Err(CudaRpcError::Protocol("expected Handle")), + } + } + + pub fn mem_alloc(&mut self, bytes: u64) -> Result { + match self.call(&Request::MemAlloc { bytes }, Op::MemAlloc)? { + Response::Dptr(d) => Ok(d), + _ => Err(CudaRpcError::Protocol("expected Dptr")), + } + } + + pub fn mem_free(&mut self, dptr: u64) -> Result<()> { + // Deferred: status-only, and callers ignore free failures anyway. + self.call_deferred(&Request::MemFree { dptr }, Op::MemFree) + } + + pub fn memcpy_htod(&mut self, dptr: u64, data: &[u8], stream: u64) -> Result<()> { + // Deferred: the bytes are copied into the request, so the caller may + // reuse its buffer immediately — synchronous-memcpy semantics hold. + self.call_deferred( + &Request::MemcpyHtoD { + dptr, + stream, + data: data.to_vec(), + }, + Op::MemcpyHtoD, + ) + } + + pub fn memcpy_dtoh(&mut self, dptr: u64, bytes: u64, stream: u64) -> Result> { + match self.call( + &Request::MemcpyDtoH { + dptr, + bytes, + stream, + }, + Op::MemcpyDtoH, + )? { + Response::Data(d) => Ok(d), + _ => Err(CudaRpcError::Protocol("expected Data")), + } + } + + #[allow(clippy::too_many_arguments)] + pub fn launch_kernel( + &mut self, + function: u64, + grid: [u32; 3], + block: [u32; 3], + shared_bytes: u32, + stream: u64, + params: &[Vec], + ) -> Result<()> { + // Deferred: kernel launches are asynchronous by CUDA contract; launch + // failures surface at the next synchronize (or as a sticky error), + // exactly like a real asynchronous launch error. + self.call_deferred( + &Request::LaunchKernel { + function, + grid, + block, + shared_bytes, + stream, + params: params.to_vec(), + }, + Op::LaunchKernel, + ) + } + + /// Exchange the serving thread's stream-capture interaction mode; returns + /// the previous mode. Sync: the caller's next op must see the new mode. + pub fn thread_exchange_capture_mode(&mut self, mode: i32) -> Result { + match self.call( + &Request::ThreadExchangeCaptureMode { mode }, + Op::ThreadExchangeCaptureMode, + )? { + Response::Count(old) => Ok(old), + _ => Err(CudaRpcError::Protocol("expected Count")), + } + } + + pub fn stream_begin_capture(&mut self, stream: u64, mode: i32) -> Result<()> { + self.call( + &Request::StreamBeginCapture { stream, mode }, + Op::StreamBeginCapture, + ) + .map(|_| ()) + } + + /// Fire-and-forget begin-capture: the host starts capture when this drains, + /// and subsequent (also-deferred) launches record in order. Saves a host + /// round-trip per captured graph (coldstart over a network). + pub fn stream_begin_capture_deferred(&mut self, stream: u64, mode: i32) -> Result<()> { + self.call_deferred( + &Request::StreamBeginCapture { stream, mode }, + Op::StreamBeginCapture, + ) + } + + /// Fire-and-forget end-capture: the guest supplies a virtual graph handle + /// it minted; the host maps it to the real captured graph when it drains. + pub fn stream_end_capture_deferred(&mut self, stream: u64, graph_vh: u64) -> Result<()> { + self.call_deferred( + &Request::StreamEndCapture { stream, graph_vh }, + Op::StreamEndCapture, + ) + } + + /// `(capture_status, capture_id)` straight from the host driver. + pub fn stream_capture_info(&mut self, stream: u64) -> Result<(u64, u64)> { + match self.call( + &Request::StreamCaptureInfo { stream }, + Op::StreamCaptureInfo, + )? { + Response::Pair(a, b) => Ok((a, b)), + _ => Err(CudaRpcError::Protocol("expected Pair")), + } + } + + /// Fire-and-forget instantiate: `graph` is a virtual graph handle; the + /// guest supplies a virtual exec handle the host maps to the real exec. + pub fn graph_instantiate_deferred(&mut self, graph: u64, exec_vh: u64) -> Result<()> { + self.call_deferred( + &Request::GraphInstantiate { graph, exec_vh }, + Op::GraphInstantiate, + ) + } + + /// Replay an instantiated graph — the hot path (one message replays every + /// captured kernel), so it pipelines like a kernel launch. + pub fn graph_launch(&mut self, graph_exec: u64, stream: u64) -> Result<()> { + self.call_deferred( + &Request::GraphLaunch { graph_exec, stream }, + Op::GraphLaunch, + ) + } + + /// Node count of a captured graph (count-only query; PyTorch uses it to + /// warn about empty captures). + pub fn graph_get_node_count(&mut self, graph: u64) -> Result { + match self.call(&Request::GraphGetNodes { graph }, Op::GraphGetNodes)? { + Response::Bytes(n) => Ok(n), + _ => Err(CudaRpcError::Protocol("expected Bytes")), + } + } + + pub fn graph_exec_destroy(&mut self, graph_exec: u64) -> Result<()> { + self.call_deferred( + &Request::GraphExecDestroy { graph_exec }, + Op::GraphExecDestroy, + ) + } + + pub fn graph_destroy(&mut self, graph: u64) -> Result<()> { + self.call_deferred(&Request::GraphDestroy { graph }, Op::GraphDestroy) + } + + /// Stream-ordered memset: capture-safe (recorded into an active graph). + pub fn memset_d8_async(&mut self, dptr: u64, value: u8, bytes: u64, stream: u64) -> Result<()> { + self.call_deferred( + &Request::MemsetD8Async { + dptr, + value, + bytes, + stream, + }, + Op::MemsetD8Async, + ) + } + + /// Stream-ordered device copy: capture-safe. + pub fn memcpy_dtod_async(&mut self, dst: u64, src: u64, bytes: u64, stream: u64) -> Result<()> { + self.call_deferred( + &Request::MemcpyDtoDAsync { + dst, + src, + bytes, + stream, + }, + Op::MemcpyDtoDAsync, + ) + } + + pub fn ctx_synchronize(&mut self) -> Result<()> { + self.call(&Request::CtxSynchronize, Op::CtxSynchronize)?; + // Surface any asynchronous failure collected while draining, the way + // cudaDeviceSynchronize reports errors from earlier async work. + match self.take_sticky() { + 0 => Ok(()), + code => Err(CudaRpcError::Cuda(code)), + } + } + + pub fn driver_get_version(&mut self) -> Result { + match self.call(&Request::DriverGetVersion, Op::DriverGetVersion)? { + Response::Count(v) => Ok(v), + _ => Err(CudaRpcError::Protocol("expected Count")), + } + } + + pub fn device_get_attribute(&mut self, attrib: i32, device: i32) -> Result { + match self.call( + &Request::DeviceGetAttribute { attrib, device }, + Op::DeviceGetAttribute, + )? { + Response::Count(v) => Ok(v), + _ => Err(CudaRpcError::Protocol("expected Count")), + } + } + + pub fn device_get_uuid(&mut self, device: i32) -> Result<[u8; 16]> { + match self.call(&Request::DeviceGetUuid { device }, Op::DeviceGetUuid)? { + Response::Data(d) if d.len() == 16 => { + let mut uuid = [0u8; 16]; + uuid.copy_from_slice(&d); + Ok(uuid) + } + _ => Err(CudaRpcError::Protocol("expected 16-byte Data")), + } + } + + pub fn primary_ctx_retain(&mut self, device: i32) -> Result { + match self.call(&Request::PrimaryCtxRetain { device }, Op::PrimaryCtxRetain)? { + Response::Handle(h) => Ok(h), + _ => Err(CudaRpcError::Protocol("expected Handle")), + } + } + + pub fn primary_ctx_release(&mut self, device: i32) -> Result<()> { + self.call( + &Request::PrimaryCtxRelease { device }, + Op::PrimaryCtxRelease, + ) + .map(|_| ()) + } + + pub fn module_unload(&mut self, module: u64) -> Result<()> { + self.call(&Request::ModuleUnload { module }, Op::ModuleUnload) + .map(|_| ()) + } + + /// Per-parameter byte sizes of the kernel's arguments, in declaration order. + /// Set a `CUfunction_attribute` on the host function (round-trip: the caller + /// — Triton — checks the status to decide whether the kernel can run). + pub fn func_get_attribute(&mut self, function: u64, attrib: i32) -> Result { + match self.call( + &Request::FuncGetAttribute { function, attrib }, + Op::FuncGetAttribute, + )? { + Response::Count(v) => Ok(v), + _ => Err(CudaRpcError::Protocol("expected Count")), + } + } + + pub fn func_set_attribute(&mut self, function: u64, attrib: i32, value: i32) -> Result<()> { + self.call( + &Request::FuncSetAttribute { + function, + attrib, + value, + }, + Op::FuncSetAttribute, + ) + .map(|_| ()) + } + + pub fn func_get_param_info(&mut self, function: u64) -> Result> { + match self.call( + &Request::FuncGetParamInfo { function }, + Op::FuncGetParamInfo, + )? { + Response::Data(d) if d.len() % 4 == 0 => Ok(d + .chunks_exact(4) + .map(|c| u32::from_le_bytes(c.try_into().unwrap())) + .collect()), + _ => Err(CudaRpcError::Protocol("expected u32-array Data")), + } + } + + pub fn memcpy_dtod(&mut self, dst: u64, src: u64, bytes: u64) -> Result<()> { + self.call(&Request::MemcpyDtoD { dst, src, bytes }, Op::MemcpyDtoD) + .map(|_| ()) + } + + pub fn memset_d8(&mut self, dptr: u64, value: u8, bytes: u64) -> Result<()> { + // Deferred: status-only device-side work, same contract as a launch. + self.call_deferred(&Request::MemsetD8 { dptr, value, bytes }, Op::MemsetD8) + } + + /// Returns `(free, total)` device memory in bytes. + pub fn mem_get_info(&mut self) -> Result<(u64, u64)> { + match self.call(&Request::MemGetInfo, Op::MemGetInfo)? { + Response::Pair(free, total) => Ok((free, total)), + _ => Err(CudaRpcError::Protocol("expected Pair")), + } + } + + pub fn stream_create(&mut self, flags: u32) -> Result { + match self.call(&Request::StreamCreate { flags }, Op::StreamCreate)? { + Response::Handle(h) => Ok(h), + _ => Err(CudaRpcError::Protocol("expected Handle")), + } + } + + pub fn stream_destroy(&mut self, stream: u64) -> Result<()> { + self.call(&Request::StreamDestroy { stream }, Op::StreamDestroy) + .map(|_| ()) + } + + pub fn stream_synchronize(&mut self, stream: u64) -> Result<()> { + self.call( + &Request::StreamSynchronize { stream }, + Op::StreamSynchronize, + ) + .map(|_| ()) + } + + /// Raw `cuStreamQuery` code: 0 complete, 600 not ready. + pub fn stream_query(&mut self, stream: u64) -> Result { + match self.call(&Request::StreamQuery { stream }, Op::StreamQuery)? { + Response::Count(code) => Ok(code), + _ => Err(CudaRpcError::Protocol("expected Count")), + } + } + + /// Deferred like a launch: a stream-ordered dependency edge whose failure + /// surfaces at the next fence, exactly like an async launch error. + pub fn stream_wait_event(&mut self, stream: u64, event: u64, flags: u32) -> Result<()> { + self.call_deferred( + &Request::StreamWaitEvent { + stream, + event, + flags, + }, + Op::StreamWaitEvent, + ) + } + + /// Raw `cuEventQuery` code: 0 complete, 600 not ready. + pub fn event_query(&mut self, event: u64) -> Result { + match self.call(&Request::EventQuery { event }, Op::EventQuery)? { + Response::Count(code) => Ok(code), + _ => Err(CudaRpcError::Protocol("expected Count")), + } + } + + pub fn event_create(&mut self, flags: u32) -> Result { + match self.call(&Request::EventCreate { flags }, Op::EventCreate)? { + Response::Handle(h) => Ok(h), + _ => Err(CudaRpcError::Protocol("expected Handle")), + } + } + + pub fn event_destroy(&mut self, event: u64) -> Result<()> { + self.call(&Request::EventDestroy { event }, Op::EventDestroy) + .map(|_| ()) + } + + pub fn event_record(&mut self, event: u64, stream: u64) -> Result<()> { + self.call(&Request::EventRecord { event, stream }, Op::EventRecord) + .map(|_| ()) + } + + pub fn event_synchronize(&mut self, event: u64) -> Result<()> { + self.call(&Request::EventSynchronize { event }, Op::EventSynchronize) + .map(|_| ()) + } + + pub fn event_elapsed_time(&mut self, start: u64, end: u64) -> Result { + match self.call( + &Request::EventElapsedTime { start, end }, + Op::EventElapsedTime, + )? { + Response::Millis(ms) => Ok(ms), + _ => Err(CudaRpcError::Protocol("expected Millis")), + } + } + + /// `(nvcomp_status, temp_bytes)` — nvcomp status is the library's own code. + pub fn nvcomp_deflate_temp_size( + &mut self, + num_chunks: u64, + max_uncompressed_chunk_bytes: u64, + max_total_uncompressed_bytes: u64, + ) -> Result<(i32, u64)> { + match self.call( + &Request::NvcompDeflateTempSize { + num_chunks, + max_uncompressed_chunk_bytes, + max_total_uncompressed_bytes, + }, + Op::NvcompDeflateTempSize, + )? { + Response::Pair(st, tb) => Ok((st as i32, tb)), + _ => Err(CudaRpcError::Protocol("expected Pair")), + } + } + + /// Returns the nvcomp status code. + #[allow(clippy::too_many_arguments)] + pub fn nvcomp_deflate_decompress( + &mut self, + device_compressed_ptrs: u64, + device_compressed_bytes: u64, + device_uncompressed_bytes: u64, + device_actual_uncompressed_bytes: u64, + batch_size: u64, + device_temp: u64, + temp_bytes: u64, + device_uncompressed_ptrs: u64, + device_statuses: u64, + stream: u64, + ) -> Result { + match self.call( + &Request::NvcompDeflateDecompress { + device_compressed_ptrs, + device_compressed_bytes, + device_uncompressed_bytes, + device_actual_uncompressed_bytes, + batch_size, + device_temp, + temp_bytes, + device_uncompressed_ptrs, + device_statuses, + stream, + }, + Op::NvcompDeflateDecompress, + )? { + Response::Count(st) => Ok(st), + _ => Err(CudaRpcError::Protocol("expected Count")), + } + } + + pub fn cublas_create(&mut self) -> Result { + match self.call(&Request::CublasCreate, Op::CublasCreate)? { + Response::Handle(h) => Ok(h), + _ => Err(CudaRpcError::Protocol("expected Handle")), + } + } + + pub fn cublas_destroy(&mut self, handle: u64) -> Result<()> { + self.call(&Request::CublasDestroy { handle }, Op::CublasDestroy) + .map(|_| ()) + } + + pub fn cublas_set_stream(&mut self, handle: u64, stream: u64) -> Result<()> { + self.call( + &Request::CublasSetStream { handle, stream }, + Op::CublasSetStream, + ) + .map(|_| ()) + } + + #[allow(clippy::too_many_arguments)] + pub fn cublas_sgemm( + &mut self, + handle: u64, + transa: u32, + transb: u32, + m: i32, + n: i32, + k: i32, + alpha: f32, + a: u64, + lda: i32, + b: u64, + ldb: i32, + beta: f32, + c: u64, + ldc: i32, + ) -> Result<()> { + self.call( + &Request::CublasSgemm { + handle, + transa, + transb, + m, + n, + k, + alpha_bits: alpha.to_bits(), + a, + lda, + b, + ldb, + beta_bits: beta.to_bits(), + c, + ldc, + }, + Op::CublasSgemm, + ) + .map(|_| ()) + } + + /// Generic forward-to-host-lib call. Returns `(library_status, outputs)`. + pub fn lib_call(&mut self, lib: u8, func: u16, args: Vec) -> Result<(i32, Vec)> { + match self.call(&Request::LibCall { lib, func, args }, Op::LibCall)? { + Response::LibResult(status, out) => Ok((status, out)), + _ => Err(CudaRpcError::Protocol("expected LibResult")), + } + } + + // ---- VMM (torch expandable-segments) — sync control-plane ops ---- + pub fn mem_address_reserve(&mut self, size: u64, align: u64) -> Result { + match self.call( + &Request::MemAddressReserve { size, align }, + Op::MemAddressReserve, + )? { + Response::Dptr(va) => Ok(va), + _ => Err(CudaRpcError::Protocol("expected Dptr")), + } + } + pub fn mem_create(&mut self, size: u64, device: i32) -> Result { + match self.call(&Request::MemCreate { size, device }, Op::MemCreate)? { + Response::Handle(h) => Ok(h), + _ => Err(CudaRpcError::Protocol("expected Handle")), + } + } + pub fn mem_map(&mut self, va: u64, size: u64, offset: u64, handle: u64) -> Result<()> { + self.call( + &Request::MemMap { + va, + size, + offset, + handle, + }, + Op::MemMap, + ) + .map(|_| ()) + } + pub fn mem_set_access(&mut self, va: u64, size: u64, device: i32) -> Result<()> { + self.call( + &Request::MemSetAccess { va, size, device }, + Op::MemSetAccess, + ) + .map(|_| ()) + } + pub fn mem_unmap(&mut self, va: u64, size: u64) -> Result<()> { + self.call(&Request::MemUnmap { va, size }, Op::MemUnmap) + .map(|_| ()) + } + pub fn mem_release(&mut self, handle: u64) -> Result<()> { + self.call(&Request::MemRelease { handle }, Op::MemRelease) + .map(|_| ()) + } + pub fn mem_address_free(&mut self, va: u64, size: u64) -> Result<()> { + self.call(&Request::MemAddressFree { va, size }, Op::MemAddressFree) + .map(|_| ()) + } + pub fn mem_get_allocation_granularity(&mut self, device: i32, flags: u32) -> Result { + match self.call( + &Request::MemGetAllocationGranularity { device, flags }, + Op::MemGetAllocationGranularity, + )? { + Response::Bytes(g) => Ok(g), + _ => Err(CudaRpcError::Protocol("expected Bytes")), + } + } + + /// Zero-copy H2D via the shared region (data already written at `offset`). + pub fn memcpy_shm_htod( + &mut self, + dptr: u64, + offset: u64, + size: u64, + stream: u64, + ) -> Result<()> { + self.call( + &Request::MemcpyShmHtoD { + dptr, + offset, + size, + stream, + }, + Op::MemcpyShmHtoD, + ) + .map(|_| ()) + } + + /// Zero-copy D2H via the shared region (host writes into `offset`). + pub fn memcpy_shm_dtoh( + &mut self, + offset: u64, + dptr: u64, + size: u64, + stream: u64, + ) -> Result<()> { + self.call( + &Request::MemcpyShmDtoH { + offset, + dptr, + size, + stream, + }, + Op::MemcpyShmDtoH, + ) + .map(|_| ()) + } + + /// Zero-copy H2D from guest RAM: the host gathers `segments` (guest-physical) + /// and DMAs to `dptr`. + pub fn memcpy_gpa_htod( + &mut self, + dptr: u64, + segments: Vec<(u64, u64)>, + stream: u64, + ) -> Result<()> { + self.call( + &Request::MemcpyGpaHtoD { + dptr, + stream, + segments, + }, + Op::MemcpyGpaHtoD, + ) + .map(|_| ()) + } + + /// Zero-copy D2H to guest RAM: the host DMAs from `dptr` and scatters into + /// `segments` (guest-physical). + pub fn memcpy_gpa_dtoh( + &mut self, + dptr: u64, + segments: Vec<(u64, u64)>, + stream: u64, + ) -> Result<()> { + self.call( + &Request::MemcpyGpaDtoH { + dptr, + stream, + segments, + }, + Op::MemcpyGpaDtoH, + ) + .map(|_| ()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::proto::QUIET_PREFIX; + use std::sync::Mutex; + + /// What the fake bridge callee observed / will serve. + static QUIET_LOG: Mutex>> = Mutex::new(Vec::new()); + static PENDING: Mutex>> = Mutex::new(None); + static RESPONSE: Mutex> = Mutex::new(Vec::new()); + + unsafe extern "C" fn fake_quiet(req: *const u8, len: usize) -> i32 { + let bytes = unsafe { std::slice::from_raw_parts(req, len) }.to_vec(); + QUIET_LOG.lock().unwrap().push(bytes); + 0 + } + unsafe extern "C" fn fake_call( + req: *const u8, + _len: usize, + resp: *mut u8, + cap: usize, + ) -> isize { + let payload = if req.is_null() { + PENDING.lock().unwrap().take().expect("no stashed response") + } else { + RESPONSE.lock().unwrap().clone() + }; + if payload.len() > cap { + let n = payload.len() as isize; + *PENDING.lock().unwrap() = Some(payload); + return -n; + } + unsafe { std::ptr::copy_nonoverlapping(payload.as_ptr(), resp, payload.len()) }; + payload.len() as isize + } + unsafe extern "C" fn fake_drain() -> i32 { + 7 // pretend a quiet failure was collected + } + + fn bridge() -> Bridge { + Bridge { + quiet: fake_quiet, + call: fake_call, + drain: fake_drain, + } + } + + /// io::Empty satisfies Client's bounds; a bridged client never touches it. + fn client() -> Client>> { + Client::new_bridged(std::io::Cursor::new(Vec::new()), bridge()) + } + + #[test] + fn bridged_deferred_ops_push_immediately() { + QUIET_LOG.lock().unwrap().clear(); + let mut c = client(); + c.set_defer_enabled(true); + c.mem_free(0xAB).unwrap(); + let log = QUIET_LOG.lock().unwrap(); + assert_eq!(log.len(), 1, "quiet op must reach the bridge at call time"); + assert_eq!(log[0], encode_request(&Request::MemFree { dptr: 0xAB })); + } + + #[test] + fn bridged_call_retries_oversized_response() { + // Encoded Count response for DeviceGetCount, padded past the caller's + // first 4 KiB buffer to force the stash-and-retry path. + let mut resp = 0i32.to_le_bytes().to_vec(); // status 0 + resp.extend_from_slice(&3i32.to_le_bytes()); // count 3 + resp.resize(8192, 0); + *RESPONSE.lock().unwrap() = resp; + let mut c = client(); + assert_eq!(c.device_get_count().unwrap(), 3); + assert!(PENDING.lock().unwrap().is_none(), "stash must be consumed"); + } + + #[test] + fn bridged_drain_collects_sticky() { + let mut c = client(); + c.drain().unwrap(); + assert_eq!(c.take_sticky(), 7); + } + + #[test] + fn raw_quiet_frames_like_call_deferred() { + // Serving side: raw_quiet must produce the same wire framing as a + // native deferred call, so bridged traffic is indistinguishable. + let mut direct: Client>> = + Client::new(std::io::Cursor::new(Vec::new())); + direct.set_defer_enabled(true); + let payload = encode_request(&Request::MemFree { dptr: 0xCD }); + direct.raw_quiet(&payload).unwrap(); + let mut expect = ((payload.len() + 1) as u32).to_le_bytes().to_vec(); + expect.push(QUIET_PREFIX); + expect.extend_from_slice(&payload); + assert_eq!(direct.wbuf, expect); + } +} diff --git a/crates/smolvm-cuda/src/generated/cublas_guest.rs b/crates/smolvm-cuda/src/generated/cublas_guest.rs new file mode 100644 index 0000000..da80273 --- /dev/null +++ b/crates/smolvm-cuda/src/generated/cublas_guest.rs @@ -0,0 +1,606 @@ +// @generated by smolvm-cuda-codegen — do not edit. lib='cublas' id=1 +const LIB_ID: u8 = 1; +#[no_mangle] +pub extern "C" fn cublasCreate_v2(handle_out: *mut *mut c_void) -> c_int { + if handle_out.is_null() { return 1; } + // Fire-and-forget create: return a guest-assigned virtual id + // immediately; the host materializes the real descriptor and + // maps the id (see vh_resolve on the host side). + let vh = super::alloc_vhandle(); + match with_client(|c| c.lib_call_deferred(LIB_ID, 0, vh.to_le_bytes().to_vec())) { + Ok(()) => { unsafe { *handle_out = vh as *mut c_void }; 0 } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cublasDestroy_v2(handle: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 1, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSgemm_v2(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f32, A: *const f32, lda: c_int, B: *const f32, ldb: c_int, beta: *const f32, C: *mut f32, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 2, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasHgemm(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const u16, A: *const u16, lda: c_int, B: *const u16, ldb: c_int, beta: *const u16, C: *mut u16, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as u16).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as u16).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 3, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDgemm_v2(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f64, A: *const f64, lda: c_int, B: *const f64, ldb: c_int, beta: *const f64, C: *mut f64, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 4, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSgemmStridedBatched(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f32, A: *const f32, lda: c_int, strideA: i64, B: *const f32, ldb: c_int, strideB: i64, beta: *const f32, C: *mut f32, ldc: c_int, strideC: i64, batchCount: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(strideA as i64).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + a.extend_from_slice(&(strideB as i64).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + a.extend_from_slice(&(strideC as i64).to_le_bytes()); + a.extend_from_slice(&(batchCount as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 5, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDgemmStridedBatched(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f64, A: *const f64, lda: c_int, strideA: i64, B: *const f64, ldb: c_int, strideB: i64, beta: *const f64, C: *mut f64, ldc: c_int, strideC: i64, batchCount: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(strideA as i64).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + a.extend_from_slice(&(strideB as i64).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + a.extend_from_slice(&(strideC as i64).to_le_bytes()); + a.extend_from_slice(&(batchCount as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 6, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSetStream_v2(handle: *mut c_void, stream: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(stream as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 7, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSetWorkspace_v2(handle: *mut c_void, workspace: *mut c_void, size: usize) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(workspace as u64).to_le_bytes()); + a.extend_from_slice(&(size as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 8, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSetMathMode(handle: *mut c_void, mode: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(mode as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 9, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasGetMathMode(handle: *mut c_void, mode: *mut c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + match with_client(|c| c.lib_call(LIB_ID, 10, a)) { + Ok((st, out)) => { let mut o = 0usize; + if !mode.is_null() && out.len() >= o + 4 { unsafe { *mode = i32::from_le_bytes(out[o..o + 4].try_into().unwrap()) as _ }; } o += 4; + let _ = o; st } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cublasGetProperty(prop_type: c_int, value: *mut c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(prop_type as i32).to_le_bytes()); + match with_client(|c| c.lib_call(LIB_ID, 11, a)) { + Ok((st, out)) => { let mut o = 0usize; + if !value.is_null() && out.len() >= o + 4 { unsafe { *value = i32::from_le_bytes(out[o..o + 4].try_into().unwrap()) as _ }; } o += 4; + let _ = o; st } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cublasGemmEx(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f32, A: *const c_void, Atype: c_int, lda: c_int, B: *const c_void, Btype: c_int, ldb: c_int, beta: *const f32, C: *mut c_void, Ctype: c_int, ldc: c_int, computeType: c_int, algo: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(Atype as i32).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(Btype as i32).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(Ctype as i32).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + a.extend_from_slice(&(computeType as i32).to_le_bytes()); + a.extend_from_slice(&(algo as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 12, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasGemmStridedBatchedEx(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f32, A: *const c_void, Atype: c_int, lda: c_int, strideA: i64, B: *const c_void, Btype: c_int, ldb: c_int, strideB: i64, beta: *const f32, C: *mut c_void, Ctype: c_int, ldc: c_int, strideC: i64, batchCount: c_int, computeType: c_int, algo: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(Atype as i32).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(strideA as i64).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(Btype as i32).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + a.extend_from_slice(&(strideB as i64).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(Ctype as i32).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + a.extend_from_slice(&(strideC as i64).to_le_bytes()); + a.extend_from_slice(&(batchCount as i32).to_le_bytes()); + a.extend_from_slice(&(computeType as i32).to_le_bytes()); + a.extend_from_slice(&(algo as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 13, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasGemmBatchedEx(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f32, Aarray: *const c_void, Atype: c_int, lda: c_int, Barray: *const c_void, Btype: c_int, ldb: c_int, beta: *const f32, Carray: *mut c_void, Ctype: c_int, ldc: c_int, batchCount: c_int, computeType: c_int, algo: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(Aarray as u64).to_le_bytes()); + a.extend_from_slice(&(Atype as i32).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(Barray as u64).to_le_bytes()); + a.extend_from_slice(&(Btype as i32).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(Carray as u64).to_le_bytes()); + a.extend_from_slice(&(Ctype as i32).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + a.extend_from_slice(&(batchCount as i32).to_le_bytes()); + a.extend_from_slice(&(computeType as i32).to_le_bytes()); + a.extend_from_slice(&(algo as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 14, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSaxpy_v2(handle: *mut c_void, n: c_int, alpha: *const f32, x: *const f32, incx: c_int, y: *mut f32, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 15, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDaxpy_v2(handle: *mut c_void, n: c_int, alpha: *const f64, x: *const f64, incx: c_int, y: *mut f64, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 16, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSscal_v2(handle: *mut c_void, n: c_int, alpha: *const f32, x: *mut f32, incx: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 17, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDscal_v2(handle: *mut c_void, n: c_int, alpha: *const f64, x: *mut f64, incx: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 18, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasScopy_v2(handle: *mut c_void, n: c_int, x: *mut f32, incx: c_int, y: *mut f32, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 19, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDcopy_v2(handle: *mut c_void, n: c_int, x: *mut f64, incx: c_int, y: *mut f64, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 20, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSswap_v2(handle: *mut c_void, n: c_int, x: *mut f32, incx: c_int, y: *mut f32, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 21, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDswap_v2(handle: *mut c_void, n: c_int, x: *mut f64, incx: c_int, y: *mut f64, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 22, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSgemv_v2(handle: *mut c_void, trans: c_int, m: c_int, n: c_int, alpha: *const f32, A: *const f32, lda: c_int, x: *const f32, incx: c_int, beta: *const f32, y: *mut f32, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 23, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDgemv_v2(handle: *mut c_void, trans: c_int, m: c_int, n: c_int, alpha: *const f64, A: *const f64, lda: c_int, x: *const f64, incx: c_int, beta: *const f64, y: *mut f64, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 24, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSger_v2(handle: *mut c_void, m: c_int, n: c_int, alpha: *const f32, x: *const f32, incx: c_int, y: *const f32, incy: c_int, A: *mut f32, lda: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 25, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDger_v2(handle: *mut c_void, m: c_int, n: c_int, alpha: *const f64, x: *const f64, incx: c_int, y: *const f64, incy: c_int, A: *mut f64, lda: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 26, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSgeam(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, alpha: *const f32, A: *const f32, lda: c_int, beta: *const f32, B: *const f32, ldb: c_int, C: *mut f32, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 27, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDgeam(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, alpha: *const f64, A: *const f64, lda: c_int, beta: *const f64, B: *const f64, ldb: c_int, C: *mut f64, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 28, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSsyrk_v2(handle: *mut c_void, uplo: c_int, trans: c_int, n: c_int, k: c_int, alpha: *const f32, A: *const f32, lda: c_int, beta: *const f32, C: *mut f32, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 29, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDsyrk_v2(handle: *mut c_void, uplo: c_int, trans: c_int, n: c_int, k: c_int, alpha: *const f64, A: *const f64, lda: c_int, beta: *const f64, C: *mut f64, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 30, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSsymm_v2(handle: *mut c_void, side: c_int, uplo: c_int, m: c_int, n: c_int, alpha: *const f32, A: *const f32, lda: c_int, B: *const f32, ldb: c_int, beta: *const f32, C: *mut f32, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(side as i32).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 31, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDsymm_v2(handle: *mut c_void, side: c_int, uplo: c_int, m: c_int, n: c_int, alpha: *const f64, A: *const f64, lda: c_int, B: *const f64, ldb: c_int, beta: *const f64, C: *mut f64, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(side as i32).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 32, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasStrsm_v2(handle: *mut c_void, side: c_int, uplo: c_int, trans: c_int, diag: c_int, m: c_int, n: c_int, alpha: *const f32, A: *const f32, lda: c_int, B: *mut f32, ldb: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(side as i32).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(diag as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 33, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDtrsm_v2(handle: *mut c_void, side: c_int, uplo: c_int, trans: c_int, diag: c_int, m: c_int, n: c_int, alpha: *const f64, A: *const f64, lda: c_int, B: *mut f64, ldb: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(side as i32).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(diag as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 34, a)) { Ok(()) => 0, Err(_) => 1 } +} diff --git a/crates/smolvm-cuda/src/generated/cublas_host.rs b/crates/smolvm-cuda/src/generated/cublas_host.rs new file mode 100644 index 0000000..cf48323 --- /dev/null +++ b/crates/smolvm-cuda/src/generated/cublas_host.rs @@ -0,0 +1,617 @@ +// @generated by smolvm-cuda-codegen — do not edit. lib='cublas' id=1 +pub struct GenLib { _lib: Library, + f_cublasCreate_v2: unsafe extern "C" fn(*mut *mut c_void) -> c_int, + f_cublasDestroy_v2: unsafe extern "C" fn(*mut c_void) -> c_int, + f_cublasSgemm_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, *const f32, *const f32, c_int, *const f32, c_int, *const f32, *mut f32, c_int) -> c_int, + f_cublasHgemm: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, *const u16, *const u16, c_int, *const u16, c_int, *const u16, *mut u16, c_int) -> c_int, + f_cublasDgemm_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, *const f64, *const f64, c_int, *const f64, c_int, *const f64, *mut f64, c_int) -> c_int, + f_cublasSgemmStridedBatched: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, *const f32, *const f32, c_int, i64, *const f32, c_int, i64, *const f32, *mut f32, c_int, i64, c_int) -> c_int, + f_cublasDgemmStridedBatched: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, *const f64, *const f64, c_int, i64, *const f64, c_int, i64, *const f64, *mut f64, c_int, i64, c_int) -> c_int, + f_cublasSetStream_v2: unsafe extern "C" fn(*mut c_void, *mut c_void) -> c_int, + f_cublasSetWorkspace_v2: unsafe extern "C" fn(*mut c_void, *mut c_void, usize) -> c_int, + f_cublasSetMathMode: unsafe extern "C" fn(*mut c_void, c_int) -> c_int, + f_cublasGetMathMode: unsafe extern "C" fn(*mut c_void, *mut c_int) -> c_int, + f_cublasGetProperty: unsafe extern "C" fn(c_int, *mut c_int) -> c_int, + f_cublasGemmEx: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, *const f32, *const c_void, c_int, c_int, *const c_void, c_int, c_int, *const f32, *mut c_void, c_int, c_int, c_int, c_int) -> c_int, + f_cublasGemmStridedBatchedEx: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, *const f32, *const c_void, c_int, c_int, i64, *const c_void, c_int, c_int, i64, *const f32, *mut c_void, c_int, c_int, i64, c_int, c_int, c_int) -> c_int, + f_cublasGemmBatchedEx: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, *const f32, *const c_void, c_int, c_int, *const c_void, c_int, c_int, *const f32, *mut c_void, c_int, c_int, c_int, c_int, c_int) -> c_int, + f_cublasSaxpy_v2: unsafe extern "C" fn(*mut c_void, c_int, *const f32, *const f32, c_int, *mut f32, c_int) -> c_int, + f_cublasDaxpy_v2: unsafe extern "C" fn(*mut c_void, c_int, *const f64, *const f64, c_int, *mut f64, c_int) -> c_int, + f_cublasSscal_v2: unsafe extern "C" fn(*mut c_void, c_int, *const f32, *mut f32, c_int) -> c_int, + f_cublasDscal_v2: unsafe extern "C" fn(*mut c_void, c_int, *const f64, *mut f64, c_int) -> c_int, + f_cublasScopy_v2: unsafe extern "C" fn(*mut c_void, c_int, *mut f32, c_int, *mut f32, c_int) -> c_int, + f_cublasDcopy_v2: unsafe extern "C" fn(*mut c_void, c_int, *mut f64, c_int, *mut f64, c_int) -> c_int, + f_cublasSswap_v2: unsafe extern "C" fn(*mut c_void, c_int, *mut f32, c_int, *mut f32, c_int) -> c_int, + f_cublasDswap_v2: unsafe extern "C" fn(*mut c_void, c_int, *mut f64, c_int, *mut f64, c_int) -> c_int, + f_cublasSgemv_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, *const f32, *const f32, c_int, *const f32, c_int, *const f32, *mut f32, c_int) -> c_int, + f_cublasDgemv_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, *const f64, *const f64, c_int, *const f64, c_int, *const f64, *mut f64, c_int) -> c_int, + f_cublasSger_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, *const f32, *const f32, c_int, *const f32, c_int, *mut f32, c_int) -> c_int, + f_cublasDger_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, *const f64, *const f64, c_int, *const f64, c_int, *mut f64, c_int) -> c_int, + f_cublasSgeam: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, *const f32, *const f32, c_int, *const f32, *const f32, c_int, *mut f32, c_int) -> c_int, + f_cublasDgeam: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, *const f64, *const f64, c_int, *const f64, *const f64, c_int, *mut f64, c_int) -> c_int, + f_cublasSsyrk_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, *const f32, *const f32, c_int, *const f32, *mut f32, c_int) -> c_int, + f_cublasDsyrk_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, *const f64, *const f64, c_int, *const f64, *mut f64, c_int) -> c_int, + f_cublasSsymm_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, *const f32, *const f32, c_int, *const f32, c_int, *const f32, *mut f32, c_int) -> c_int, + f_cublasDsymm_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, *const f64, *const f64, c_int, *const f64, c_int, *const f64, *mut f64, c_int) -> c_int, + f_cublasStrsm_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, c_int, *const f32, *const f32, c_int, *mut f32, c_int) -> c_int, + f_cublasDtrsm_v2: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, c_int, *const f64, *const f64, c_int, *mut f64, c_int) -> c_int, +} +impl GenLib { + pub fn load() -> Result { + unsafe { + let lib = super::open_host_lib("SMOLVM_CUBLAS_LIB", &["libcublas.so", "libcublas.so.13", "libcublas.so.12"])?; + Ok(GenLib { + f_cublasCreate_v2: sym(&lib, b"cublasCreate_v2\0")?, + f_cublasDestroy_v2: sym(&lib, b"cublasDestroy_v2\0")?, + f_cublasSgemm_v2: sym(&lib, b"cublasSgemm_v2\0")?, + f_cublasHgemm: sym(&lib, b"cublasHgemm\0")?, + f_cublasDgemm_v2: sym(&lib, b"cublasDgemm_v2\0")?, + f_cublasSgemmStridedBatched: sym(&lib, b"cublasSgemmStridedBatched\0")?, + f_cublasDgemmStridedBatched: sym(&lib, b"cublasDgemmStridedBatched\0")?, + f_cublasSetStream_v2: sym(&lib, b"cublasSetStream_v2\0")?, + f_cublasSetWorkspace_v2: sym(&lib, b"cublasSetWorkspace_v2\0")?, + f_cublasSetMathMode: sym(&lib, b"cublasSetMathMode\0")?, + f_cublasGetMathMode: sym(&lib, b"cublasGetMathMode\0")?, + f_cublasGetProperty: sym(&lib, b"cublasGetProperty\0")?, + f_cublasGemmEx: sym(&lib, b"cublasGemmEx\0")?, + f_cublasGemmStridedBatchedEx: sym(&lib, b"cublasGemmStridedBatchedEx\0")?, + f_cublasGemmBatchedEx: sym(&lib, b"cublasGemmBatchedEx\0")?, + f_cublasSaxpy_v2: sym(&lib, b"cublasSaxpy_v2\0")?, + f_cublasDaxpy_v2: sym(&lib, b"cublasDaxpy_v2\0")?, + f_cublasSscal_v2: sym(&lib, b"cublasSscal_v2\0")?, + f_cublasDscal_v2: sym(&lib, b"cublasDscal_v2\0")?, + f_cublasScopy_v2: sym(&lib, b"cublasScopy_v2\0")?, + f_cublasDcopy_v2: sym(&lib, b"cublasDcopy_v2\0")?, + f_cublasSswap_v2: sym(&lib, b"cublasSswap_v2\0")?, + f_cublasDswap_v2: sym(&lib, b"cublasDswap_v2\0")?, + f_cublasSgemv_v2: sym(&lib, b"cublasSgemv_v2\0")?, + f_cublasDgemv_v2: sym(&lib, b"cublasDgemv_v2\0")?, + f_cublasSger_v2: sym(&lib, b"cublasSger_v2\0")?, + f_cublasDger_v2: sym(&lib, b"cublasDger_v2\0")?, + f_cublasSgeam: sym(&lib, b"cublasSgeam\0")?, + f_cublasDgeam: sym(&lib, b"cublasDgeam\0")?, + f_cublasSsyrk_v2: sym(&lib, b"cublasSsyrk_v2\0")?, + f_cublasDsyrk_v2: sym(&lib, b"cublasDsyrk_v2\0")?, + f_cublasSsymm_v2: sym(&lib, b"cublasSsymm_v2\0")?, + f_cublasDsymm_v2: sym(&lib, b"cublasDsymm_v2\0")?, + f_cublasStrsm_v2: sym(&lib, b"cublasStrsm_v2\0")?, + f_cublasDtrsm_v2: sym(&lib, b"cublasDtrsm_v2\0")?, + _lib: lib, + }) + } + } + pub fn dispatch(&self, func: u16, args: &[u8], __vh: &mut std::collections::HashMap, __streams: &std::collections::HashMap) -> (i32, Vec) { + let mut __c = GenCur { b: args, p: 0 }; + let _ = __streams; + match func { + 0 => { let mut h: *mut c_void = std::ptr::null_mut(); let st = unsafe { (self.f_cublasCreate_v2)(&mut h) }; if st == 0 && args.len() >= 8 { let id = u64::from_le_bytes(args[..8].try_into().unwrap()); if id & super::VHANDLE_TAG != 0 { __vh.insert(id, h as u64); } } (st, (h as u64).to_le_bytes().to_vec()) } + 1 => { + let __raw = __c.u64(); + let handle = super::vh_resolve(__vh, __raw) as *mut c_void; + if __raw & super::VHANDLE_TAG != 0 { __vh.remove(&__raw); } + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDestroy_v2)(handle) }; + (st, out) + } + 2 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let transa = __c.i32(); + let transb = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let k = __c.i32(); + let alpha_v = __c.f32(); + let A = __c.u64() as *const f32; + let lda = __c.i32(); + let B = __c.u64() as *const f32; + let ldb = __c.i32(); + let beta_v = __c.f32(); + let C = __c.u64() as *mut f32; + let ldc = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSgemm_v2)(handle, transa as c_int, transb as c_int, m as c_int, n as c_int, k as c_int, &alpha_v, A, lda as c_int, B, ldb as c_int, &beta_v, C, ldc as c_int) }; + (st, out) + } + 3 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let transa = __c.i32(); + let transb = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let k = __c.i32(); + let alpha_v = __c.u16(); + let A = __c.u64() as *const u16; + let lda = __c.i32(); + let B = __c.u64() as *const u16; + let ldb = __c.i32(); + let beta_v = __c.u16(); + let C = __c.u64() as *mut u16; + let ldc = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasHgemm)(handle, transa as c_int, transb as c_int, m as c_int, n as c_int, k as c_int, &alpha_v, A, lda as c_int, B, ldb as c_int, &beta_v, C, ldc as c_int) }; + (st, out) + } + 4 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let transa = __c.i32(); + let transb = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let k = __c.i32(); + let alpha_v = __c.f64(); + let A = __c.u64() as *const f64; + let lda = __c.i32(); + let B = __c.u64() as *const f64; + let ldb = __c.i32(); + let beta_v = __c.f64(); + let C = __c.u64() as *mut f64; + let ldc = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDgemm_v2)(handle, transa as c_int, transb as c_int, m as c_int, n as c_int, k as c_int, &alpha_v, A, lda as c_int, B, ldb as c_int, &beta_v, C, ldc as c_int) }; + (st, out) + } + 5 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let transa = __c.i32(); + let transb = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let k = __c.i32(); + let alpha_v = __c.f32(); + let A = __c.u64() as *const f32; + let lda = __c.i32(); + let strideA = __c.i64(); + let B = __c.u64() as *const f32; + let ldb = __c.i32(); + let strideB = __c.i64(); + let beta_v = __c.f32(); + let C = __c.u64() as *mut f32; + let ldc = __c.i32(); + let strideC = __c.i64(); + let batchCount = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSgemmStridedBatched)(handle, transa as c_int, transb as c_int, m as c_int, n as c_int, k as c_int, &alpha_v, A, lda as c_int, strideA as i64, B, ldb as c_int, strideB as i64, &beta_v, C, ldc as c_int, strideC as i64, batchCount as c_int) }; + (st, out) + } + 6 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let transa = __c.i32(); + let transb = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let k = __c.i32(); + let alpha_v = __c.f64(); + let A = __c.u64() as *const f64; + let lda = __c.i32(); + let strideA = __c.i64(); + let B = __c.u64() as *const f64; + let ldb = __c.i32(); + let strideB = __c.i64(); + let beta_v = __c.f64(); + let C = __c.u64() as *mut f64; + let ldc = __c.i32(); + let strideC = __c.i64(); + let batchCount = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDgemmStridedBatched)(handle, transa as c_int, transb as c_int, m as c_int, n as c_int, k as c_int, &alpha_v, A, lda as c_int, strideA as i64, B, ldb as c_int, strideB as i64, &beta_v, C, ldc as c_int, strideC as i64, batchCount as c_int) }; + (st, out) + } + 7 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let stream = super::stream_resolve(__streams, __c.u64()) as *mut c_void; + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSetStream_v2)(handle, stream) }; + (st, out) + } + 8 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let workspace = __c.u64() as *mut c_void; + let size = __c.u64(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSetWorkspace_v2)(handle, workspace, size as usize) }; + (st, out) + } + 9 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let mode = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSetMathMode)(handle, mode as c_int) }; + (st, out) + } + 10 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let mut mode_v: c_int = 0 as c_int; + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasGetMathMode)(handle, &mut mode_v) }; + out.extend_from_slice(&(mode_v as i32).to_le_bytes()); + (st, out) + } + 11 => { + let prop_type = __c.i32(); + let mut value_v: c_int = 0 as c_int; + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasGetProperty)(prop_type as c_int, &mut value_v) }; + out.extend_from_slice(&(value_v as i32).to_le_bytes()); + (st, out) + } + 12 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let transa = __c.i32(); + let transb = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let k = __c.i32(); + let alpha_v = __c.f32(); + let A = __c.u64() as *const c_void; + let Atype = __c.i32(); + let lda = __c.i32(); + let B = __c.u64() as *const c_void; + let Btype = __c.i32(); + let ldb = __c.i32(); + let beta_v = __c.f32(); + let C = __c.u64() as *mut c_void; + let Ctype = __c.i32(); + let ldc = __c.i32(); + let computeType = __c.i32(); + let algo = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasGemmEx)(handle, transa as c_int, transb as c_int, m as c_int, n as c_int, k as c_int, &alpha_v, A, Atype as c_int, lda as c_int, B, Btype as c_int, ldb as c_int, &beta_v, C, Ctype as c_int, ldc as c_int, computeType as c_int, algo as c_int) }; + (st, out) + } + 13 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let transa = __c.i32(); + let transb = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let k = __c.i32(); + let alpha_v = __c.f32(); + let A = __c.u64() as *const c_void; + let Atype = __c.i32(); + let lda = __c.i32(); + let strideA = __c.i64(); + let B = __c.u64() as *const c_void; + let Btype = __c.i32(); + let ldb = __c.i32(); + let strideB = __c.i64(); + let beta_v = __c.f32(); + let C = __c.u64() as *mut c_void; + let Ctype = __c.i32(); + let ldc = __c.i32(); + let strideC = __c.i64(); + let batchCount = __c.i32(); + let computeType = __c.i32(); + let algo = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasGemmStridedBatchedEx)(handle, transa as c_int, transb as c_int, m as c_int, n as c_int, k as c_int, &alpha_v, A, Atype as c_int, lda as c_int, strideA as i64, B, Btype as c_int, ldb as c_int, strideB as i64, &beta_v, C, Ctype as c_int, ldc as c_int, strideC as i64, batchCount as c_int, computeType as c_int, algo as c_int) }; + (st, out) + } + 14 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let transa = __c.i32(); + let transb = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let k = __c.i32(); + let alpha_v = __c.f32(); + let Aarray = __c.u64() as *const c_void; + let Atype = __c.i32(); + let lda = __c.i32(); + let Barray = __c.u64() as *const c_void; + let Btype = __c.i32(); + let ldb = __c.i32(); + let beta_v = __c.f32(); + let Carray = __c.u64() as *mut c_void; + let Ctype = __c.i32(); + let ldc = __c.i32(); + let batchCount = __c.i32(); + let computeType = __c.i32(); + let algo = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasGemmBatchedEx)(handle, transa as c_int, transb as c_int, m as c_int, n as c_int, k as c_int, &alpha_v, Aarray, Atype as c_int, lda as c_int, Barray, Btype as c_int, ldb as c_int, &beta_v, Carray, Ctype as c_int, ldc as c_int, batchCount as c_int, computeType as c_int, algo as c_int) }; + (st, out) + } + 15 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let n = __c.i32(); + let alpha_v = __c.f32(); + let x = __c.u64() as *const f32; + let incx = __c.i32(); + let y = __c.u64() as *mut f32; + let incy = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSaxpy_v2)(handle, n as c_int, &alpha_v, x, incx as c_int, y, incy as c_int) }; + (st, out) + } + 16 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let n = __c.i32(); + let alpha_v = __c.f64(); + let x = __c.u64() as *const f64; + let incx = __c.i32(); + let y = __c.u64() as *mut f64; + let incy = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDaxpy_v2)(handle, n as c_int, &alpha_v, x, incx as c_int, y, incy as c_int) }; + (st, out) + } + 17 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let n = __c.i32(); + let alpha_v = __c.f32(); + let x = __c.u64() as *mut f32; + let incx = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSscal_v2)(handle, n as c_int, &alpha_v, x, incx as c_int) }; + (st, out) + } + 18 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let n = __c.i32(); + let alpha_v = __c.f64(); + let x = __c.u64() as *mut f64; + let incx = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDscal_v2)(handle, n as c_int, &alpha_v, x, incx as c_int) }; + (st, out) + } + 19 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let n = __c.i32(); + let x = __c.u64() as *mut f32; + let incx = __c.i32(); + let y = __c.u64() as *mut f32; + let incy = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasScopy_v2)(handle, n as c_int, x, incx as c_int, y, incy as c_int) }; + (st, out) + } + 20 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let n = __c.i32(); + let x = __c.u64() as *mut f64; + let incx = __c.i32(); + let y = __c.u64() as *mut f64; + let incy = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDcopy_v2)(handle, n as c_int, x, incx as c_int, y, incy as c_int) }; + (st, out) + } + 21 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let n = __c.i32(); + let x = __c.u64() as *mut f32; + let incx = __c.i32(); + let y = __c.u64() as *mut f32; + let incy = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSswap_v2)(handle, n as c_int, x, incx as c_int, y, incy as c_int) }; + (st, out) + } + 22 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let n = __c.i32(); + let x = __c.u64() as *mut f64; + let incx = __c.i32(); + let y = __c.u64() as *mut f64; + let incy = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDswap_v2)(handle, n as c_int, x, incx as c_int, y, incy as c_int) }; + (st, out) + } + 23 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let trans = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let alpha_v = __c.f32(); + let A = __c.u64() as *const f32; + let lda = __c.i32(); + let x = __c.u64() as *const f32; + let incx = __c.i32(); + let beta_v = __c.f32(); + let y = __c.u64() as *mut f32; + let incy = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSgemv_v2)(handle, trans as c_int, m as c_int, n as c_int, &alpha_v, A, lda as c_int, x, incx as c_int, &beta_v, y, incy as c_int) }; + (st, out) + } + 24 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let trans = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let alpha_v = __c.f64(); + let A = __c.u64() as *const f64; + let lda = __c.i32(); + let x = __c.u64() as *const f64; + let incx = __c.i32(); + let beta_v = __c.f64(); + let y = __c.u64() as *mut f64; + let incy = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDgemv_v2)(handle, trans as c_int, m as c_int, n as c_int, &alpha_v, A, lda as c_int, x, incx as c_int, &beta_v, y, incy as c_int) }; + (st, out) + } + 25 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let m = __c.i32(); + let n = __c.i32(); + let alpha_v = __c.f32(); + let x = __c.u64() as *const f32; + let incx = __c.i32(); + let y = __c.u64() as *const f32; + let incy = __c.i32(); + let A = __c.u64() as *mut f32; + let lda = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSger_v2)(handle, m as c_int, n as c_int, &alpha_v, x, incx as c_int, y, incy as c_int, A, lda as c_int) }; + (st, out) + } + 26 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let m = __c.i32(); + let n = __c.i32(); + let alpha_v = __c.f64(); + let x = __c.u64() as *const f64; + let incx = __c.i32(); + let y = __c.u64() as *const f64; + let incy = __c.i32(); + let A = __c.u64() as *mut f64; + let lda = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDger_v2)(handle, m as c_int, n as c_int, &alpha_v, x, incx as c_int, y, incy as c_int, A, lda as c_int) }; + (st, out) + } + 27 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let transa = __c.i32(); + let transb = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let alpha_v = __c.f32(); + let A = __c.u64() as *const f32; + let lda = __c.i32(); + let beta_v = __c.f32(); + let B = __c.u64() as *const f32; + let ldb = __c.i32(); + let C = __c.u64() as *mut f32; + let ldc = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSgeam)(handle, transa as c_int, transb as c_int, m as c_int, n as c_int, &alpha_v, A, lda as c_int, &beta_v, B, ldb as c_int, C, ldc as c_int) }; + (st, out) + } + 28 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let transa = __c.i32(); + let transb = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let alpha_v = __c.f64(); + let A = __c.u64() as *const f64; + let lda = __c.i32(); + let beta_v = __c.f64(); + let B = __c.u64() as *const f64; + let ldb = __c.i32(); + let C = __c.u64() as *mut f64; + let ldc = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDgeam)(handle, transa as c_int, transb as c_int, m as c_int, n as c_int, &alpha_v, A, lda as c_int, &beta_v, B, ldb as c_int, C, ldc as c_int) }; + (st, out) + } + 29 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let uplo = __c.i32(); + let trans = __c.i32(); + let n = __c.i32(); + let k = __c.i32(); + let alpha_v = __c.f32(); + let A = __c.u64() as *const f32; + let lda = __c.i32(); + let beta_v = __c.f32(); + let C = __c.u64() as *mut f32; + let ldc = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSsyrk_v2)(handle, uplo as c_int, trans as c_int, n as c_int, k as c_int, &alpha_v, A, lda as c_int, &beta_v, C, ldc as c_int) }; + (st, out) + } + 30 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let uplo = __c.i32(); + let trans = __c.i32(); + let n = __c.i32(); + let k = __c.i32(); + let alpha_v = __c.f64(); + let A = __c.u64() as *const f64; + let lda = __c.i32(); + let beta_v = __c.f64(); + let C = __c.u64() as *mut f64; + let ldc = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDsyrk_v2)(handle, uplo as c_int, trans as c_int, n as c_int, k as c_int, &alpha_v, A, lda as c_int, &beta_v, C, ldc as c_int) }; + (st, out) + } + 31 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let side = __c.i32(); + let uplo = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let alpha_v = __c.f32(); + let A = __c.u64() as *const f32; + let lda = __c.i32(); + let B = __c.u64() as *const f32; + let ldb = __c.i32(); + let beta_v = __c.f32(); + let C = __c.u64() as *mut f32; + let ldc = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasSsymm_v2)(handle, side as c_int, uplo as c_int, m as c_int, n as c_int, &alpha_v, A, lda as c_int, B, ldb as c_int, &beta_v, C, ldc as c_int) }; + (st, out) + } + 32 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let side = __c.i32(); + let uplo = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let alpha_v = __c.f64(); + let A = __c.u64() as *const f64; + let lda = __c.i32(); + let B = __c.u64() as *const f64; + let ldb = __c.i32(); + let beta_v = __c.f64(); + let C = __c.u64() as *mut f64; + let ldc = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDsymm_v2)(handle, side as c_int, uplo as c_int, m as c_int, n as c_int, &alpha_v, A, lda as c_int, B, ldb as c_int, &beta_v, C, ldc as c_int) }; + (st, out) + } + 33 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let side = __c.i32(); + let uplo = __c.i32(); + let trans = __c.i32(); + let diag = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let alpha_v = __c.f32(); + let A = __c.u64() as *const f32; + let lda = __c.i32(); + let B = __c.u64() as *mut f32; + let ldb = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasStrsm_v2)(handle, side as c_int, uplo as c_int, trans as c_int, diag as c_int, m as c_int, n as c_int, &alpha_v, A, lda as c_int, B, ldb as c_int) }; + (st, out) + } + 34 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let side = __c.i32(); + let uplo = __c.i32(); + let trans = __c.i32(); + let diag = __c.i32(); + let m = __c.i32(); + let n = __c.i32(); + let alpha_v = __c.f64(); + let A = __c.u64() as *const f64; + let lda = __c.i32(); + let B = __c.u64() as *mut f64; + let ldb = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cublasDtrsm_v2)(handle, side as c_int, uplo as c_int, trans as c_int, diag as c_int, m as c_int, n as c_int, &alpha_v, A, lda as c_int, B, ldb as c_int) }; + (st, out) + } + _ => (super::super::CUDA_ERROR_NOT_FOUND, Vec::new()), + } + } +} +struct GenCur<'a> { b: &'a [u8], p: usize } +impl GenCur<'_> { + fn take(&mut self, n: usize) -> [u8; 8] { let mut o = [0u8; 8]; let end = (self.p + n).min(self.b.len()); o[..end - self.p].copy_from_slice(&self.b[self.p..end]); self.p = end; o } + fn u16(&mut self) -> u16 { u16::from_le_bytes(self.take(2)[..2].try_into().unwrap()) } + fn i32(&mut self) -> i32 { i32::from_le_bytes(self.take(4)[..4].try_into().unwrap()) } + fn i64(&mut self) -> i64 { i64::from_le_bytes(self.take(8)) } + fn u64(&mut self) -> u64 { u64::from_le_bytes(self.take(8)) } + fn f32(&mut self) -> f32 { f32::from_le_bytes(self.take(4)[..4].try_into().unwrap()) } + fn f64(&mut self) -> f64 { f64::from_le_bytes(self.take(8)) } +} diff --git a/crates/smolvm-cuda/src/generated/cudnn_guest.rs b/crates/smolvm-cuda/src/generated/cudnn_guest.rs new file mode 100644 index 0000000..c14e363 --- /dev/null +++ b/crates/smolvm-cuda/src/generated/cudnn_guest.rs @@ -0,0 +1,164 @@ +// @generated by smolvm-cuda-codegen — do not edit. lib='cudnn' id=2 +const LIB_ID: u8 = 2; +#[no_mangle] +pub extern "C" fn cudnnCreate(handle_out: *mut *mut c_void) -> c_int { + if handle_out.is_null() { return 1; } + // Fire-and-forget create: return a guest-assigned virtual id + // immediately; the host materializes the real descriptor and + // maps the id (see vh_resolve on the host side). + let vh = super::alloc_vhandle(); + match with_client(|c| c.lib_call_deferred(LIB_ID, 0, vh.to_le_bytes().to_vec())) { + Ok(()) => { unsafe { *handle_out = vh as *mut c_void }; 0 } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cudnnDestroy(handle: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 1, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnSetStream(handle: *mut c_void, stream: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(stream as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 2, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnCreateTensorDescriptor(handle_out: *mut *mut c_void) -> c_int { + if handle_out.is_null() { return 1; } + // Fire-and-forget create: return a guest-assigned virtual id + // immediately; the host materializes the real descriptor and + // maps the id (see vh_resolve on the host side). + let vh = super::alloc_vhandle(); + match with_client(|c| c.lib_call_deferred(LIB_ID, 3, vh.to_le_bytes().to_vec())) { + Ok(()) => { unsafe { *handle_out = vh as *mut c_void }; 0 } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cudnnSetTensor4dDescriptor(t: *mut c_void, fmt: c_int, dtype: c_int, n: c_int, c: c_int, h: c_int, w: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(t as u64).to_le_bytes()); + a.extend_from_slice(&(fmt as i32).to_le_bytes()); + a.extend_from_slice(&(dtype as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(c as i32).to_le_bytes()); + a.extend_from_slice(&(h as i32).to_le_bytes()); + a.extend_from_slice(&(w as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 4, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnDestroyTensorDescriptor(t: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(t as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 5, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnCreateFilterDescriptor(handle_out: *mut *mut c_void) -> c_int { + if handle_out.is_null() { return 1; } + // Fire-and-forget create: return a guest-assigned virtual id + // immediately; the host materializes the real descriptor and + // maps the id (see vh_resolve on the host side). + let vh = super::alloc_vhandle(); + match with_client(|c| c.lib_call_deferred(LIB_ID, 6, vh.to_le_bytes().to_vec())) { + Ok(()) => { unsafe { *handle_out = vh as *mut c_void }; 0 } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cudnnSetFilter4dDescriptor(f: *mut c_void, dtype: c_int, fmt: c_int, k: c_int, c: c_int, h: c_int, w: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(f as u64).to_le_bytes()); + a.extend_from_slice(&(dtype as i32).to_le_bytes()); + a.extend_from_slice(&(fmt as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + a.extend_from_slice(&(c as i32).to_le_bytes()); + a.extend_from_slice(&(h as i32).to_le_bytes()); + a.extend_from_slice(&(w as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 7, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnDestroyFilterDescriptor(f: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(f as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 8, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnCreateConvolutionDescriptor(handle_out: *mut *mut c_void) -> c_int { + if handle_out.is_null() { return 1; } + // Fire-and-forget create: return a guest-assigned virtual id + // immediately; the host materializes the real descriptor and + // maps the id (see vh_resolve on the host side). + let vh = super::alloc_vhandle(); + match with_client(|c| c.lib_call_deferred(LIB_ID, 9, vh.to_le_bytes().to_vec())) { + Ok(()) => { unsafe { *handle_out = vh as *mut c_void }; 0 } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cudnnSetConvolution2dDescriptor(cv: *mut c_void, pad_h: c_int, pad_w: c_int, u: c_int, v: c_int, dil_h: c_int, dil_w: c_int, mode: c_int, ctype: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(cv as u64).to_le_bytes()); + a.extend_from_slice(&(pad_h as i32).to_le_bytes()); + a.extend_from_slice(&(pad_w as i32).to_le_bytes()); + a.extend_from_slice(&(u as i32).to_le_bytes()); + a.extend_from_slice(&(v as i32).to_le_bytes()); + a.extend_from_slice(&(dil_h as i32).to_le_bytes()); + a.extend_from_slice(&(dil_w as i32).to_le_bytes()); + a.extend_from_slice(&(mode as i32).to_le_bytes()); + a.extend_from_slice(&(ctype as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 10, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnDestroyConvolutionDescriptor(cv: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(cv as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 11, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnGetConvolutionForwardWorkspaceSize(handle: *mut c_void, x: *mut c_void, w: *mut c_void, cv: *mut c_void, y: *mut c_void, algo: c_int, size: *mut usize) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(w as u64).to_le_bytes()); + a.extend_from_slice(&(cv as u64).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(algo as i32).to_le_bytes()); + match with_client(|c| c.lib_call(LIB_ID, 12, a)) { + Ok((st, out)) => { let mut o = 0usize; + if !size.is_null() && out.len() >= o + 8 { unsafe { *size = u64::from_le_bytes(out[o..o + 8].try_into().unwrap()) as _ }; } o += 8; + let _ = o; st } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cudnnConvolutionForward(handle: *mut c_void, alpha: *const f32, xDesc: *mut c_void, x: *mut c_void, wDesc: *mut c_void, w: *mut c_void, convDesc: *mut c_void, algo: c_int, workspace: *mut c_void, wsSize: usize, beta: *const f32, yDesc: *mut c_void, y: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(xDesc as u64).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(wDesc as u64).to_le_bytes()); + a.extend_from_slice(&(w as u64).to_le_bytes()); + a.extend_from_slice(&(convDesc as u64).to_le_bytes()); + a.extend_from_slice(&(algo as i32).to_le_bytes()); + a.extend_from_slice(&(workspace as u64).to_le_bytes()); + a.extend_from_slice(&(wsSize as u64).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(yDesc as u64).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 13, a)) { Ok(()) => 0, Err(_) => 1 } +} diff --git a/crates/smolvm-cuda/src/generated/cudnn_host.rs b/crates/smolvm-cuda/src/generated/cudnn_host.rs new file mode 100644 index 0000000..b211ae6 --- /dev/null +++ b/crates/smolvm-cuda/src/generated/cudnn_host.rs @@ -0,0 +1,170 @@ +// @generated by smolvm-cuda-codegen — do not edit. lib='cudnn' id=2 +pub struct GenLib { _lib: Library, + f_cudnnCreate: unsafe extern "C" fn(*mut *mut c_void) -> c_int, + f_cudnnDestroy: unsafe extern "C" fn(*mut c_void) -> c_int, + f_cudnnSetStream: unsafe extern "C" fn(*mut c_void, *mut c_void) -> c_int, + f_cudnnCreateTensorDescriptor: unsafe extern "C" fn(*mut *mut c_void) -> c_int, + f_cudnnSetTensor4dDescriptor: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, c_int) -> c_int, + f_cudnnDestroyTensorDescriptor: unsafe extern "C" fn(*mut c_void) -> c_int, + f_cudnnCreateFilterDescriptor: unsafe extern "C" fn(*mut *mut c_void) -> c_int, + f_cudnnSetFilter4dDescriptor: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, c_int) -> c_int, + f_cudnnDestroyFilterDescriptor: unsafe extern "C" fn(*mut c_void) -> c_int, + f_cudnnCreateConvolutionDescriptor: unsafe extern "C" fn(*mut *mut c_void) -> c_int, + f_cudnnSetConvolution2dDescriptor: unsafe extern "C" fn(*mut c_void, c_int, c_int, c_int, c_int, c_int, c_int, c_int, c_int) -> c_int, + f_cudnnDestroyConvolutionDescriptor: unsafe extern "C" fn(*mut c_void) -> c_int, + f_cudnnGetConvolutionForwardWorkspaceSize: unsafe extern "C" fn(*mut c_void, *mut c_void, *mut c_void, *mut c_void, *mut c_void, c_int, *mut usize) -> c_int, + f_cudnnConvolutionForward: unsafe extern "C" fn(*mut c_void, *const f32, *mut c_void, *mut c_void, *mut c_void, *mut c_void, *mut c_void, c_int, *mut c_void, usize, *const f32, *mut c_void, *mut c_void) -> c_int, +} +impl GenLib { + pub fn load() -> Result { + unsafe { + let lib = super::open_host_lib("SMOLVM_CUDNN_LIB", &["libcudnn.so", "libcudnn.so.9", "libcudnn.so.8"])?; + Ok(GenLib { + f_cudnnCreate: sym(&lib, b"cudnnCreate\0")?, + f_cudnnDestroy: sym(&lib, b"cudnnDestroy\0")?, + f_cudnnSetStream: sym(&lib, b"cudnnSetStream\0")?, + f_cudnnCreateTensorDescriptor: sym(&lib, b"cudnnCreateTensorDescriptor\0")?, + f_cudnnSetTensor4dDescriptor: sym(&lib, b"cudnnSetTensor4dDescriptor\0")?, + f_cudnnDestroyTensorDescriptor: sym(&lib, b"cudnnDestroyTensorDescriptor\0")?, + f_cudnnCreateFilterDescriptor: sym(&lib, b"cudnnCreateFilterDescriptor\0")?, + f_cudnnSetFilter4dDescriptor: sym(&lib, b"cudnnSetFilter4dDescriptor\0")?, + f_cudnnDestroyFilterDescriptor: sym(&lib, b"cudnnDestroyFilterDescriptor\0")?, + f_cudnnCreateConvolutionDescriptor: sym(&lib, b"cudnnCreateConvolutionDescriptor\0")?, + f_cudnnSetConvolution2dDescriptor: sym(&lib, b"cudnnSetConvolution2dDescriptor\0")?, + f_cudnnDestroyConvolutionDescriptor: sym(&lib, b"cudnnDestroyConvolutionDescriptor\0")?, + f_cudnnGetConvolutionForwardWorkspaceSize: sym(&lib, b"cudnnGetConvolutionForwardWorkspaceSize\0")?, + f_cudnnConvolutionForward: sym(&lib, b"cudnnConvolutionForward\0")?, + _lib: lib, + }) + } + } + pub fn dispatch(&self, func: u16, args: &[u8], __vh: &mut std::collections::HashMap, __streams: &std::collections::HashMap) -> (i32, Vec) { + let mut __c = GenCur { b: args, p: 0 }; + let _ = __streams; + match func { + 0 => { let mut h: *mut c_void = std::ptr::null_mut(); let st = unsafe { (self.f_cudnnCreate)(&mut h) }; if st == 0 && args.len() >= 8 { let id = u64::from_le_bytes(args[..8].try_into().unwrap()); if id & super::VHANDLE_TAG != 0 { __vh.insert(id, h as u64); } } (st, (h as u64).to_le_bytes().to_vec()) } + 1 => { + let __raw = __c.u64(); + let handle = super::vh_resolve(__vh, __raw) as *mut c_void; + if __raw & super::VHANDLE_TAG != 0 { __vh.remove(&__raw); } + let mut out = Vec::new(); + let st = unsafe { (self.f_cudnnDestroy)(handle) }; + (st, out) + } + 2 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let stream = super::stream_resolve(__streams, __c.u64()) as *mut c_void; + let mut out = Vec::new(); + let st = unsafe { (self.f_cudnnSetStream)(handle, stream) }; + (st, out) + } + 3 => { let mut h: *mut c_void = std::ptr::null_mut(); let st = unsafe { (self.f_cudnnCreateTensorDescriptor)(&mut h) }; if st == 0 && args.len() >= 8 { let id = u64::from_le_bytes(args[..8].try_into().unwrap()); if id & super::VHANDLE_TAG != 0 { __vh.insert(id, h as u64); } } (st, (h as u64).to_le_bytes().to_vec()) } + 4 => { + let t = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let fmt = __c.i32(); + let dtype = __c.i32(); + let n = __c.i32(); + let c = __c.i32(); + let h = __c.i32(); + let w = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cudnnSetTensor4dDescriptor)(t, fmt as c_int, dtype as c_int, n as c_int, c as c_int, h as c_int, w as c_int) }; + (st, out) + } + 5 => { + let __raw = __c.u64(); + let t = super::vh_resolve(__vh, __raw) as *mut c_void; + if __raw & super::VHANDLE_TAG != 0 { __vh.remove(&__raw); } + let mut out = Vec::new(); + let st = unsafe { (self.f_cudnnDestroyTensorDescriptor)(t) }; + (st, out) + } + 6 => { let mut h: *mut c_void = std::ptr::null_mut(); let st = unsafe { (self.f_cudnnCreateFilterDescriptor)(&mut h) }; if st == 0 && args.len() >= 8 { let id = u64::from_le_bytes(args[..8].try_into().unwrap()); if id & super::VHANDLE_TAG != 0 { __vh.insert(id, h as u64); } } (st, (h as u64).to_le_bytes().to_vec()) } + 7 => { + let f = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let dtype = __c.i32(); + let fmt = __c.i32(); + let k = __c.i32(); + let c = __c.i32(); + let h = __c.i32(); + let w = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cudnnSetFilter4dDescriptor)(f, dtype as c_int, fmt as c_int, k as c_int, c as c_int, h as c_int, w as c_int) }; + (st, out) + } + 8 => { + let __raw = __c.u64(); + let f = super::vh_resolve(__vh, __raw) as *mut c_void; + if __raw & super::VHANDLE_TAG != 0 { __vh.remove(&__raw); } + let mut out = Vec::new(); + let st = unsafe { (self.f_cudnnDestroyFilterDescriptor)(f) }; + (st, out) + } + 9 => { let mut h: *mut c_void = std::ptr::null_mut(); let st = unsafe { (self.f_cudnnCreateConvolutionDescriptor)(&mut h) }; if st == 0 && args.len() >= 8 { let id = u64::from_le_bytes(args[..8].try_into().unwrap()); if id & super::VHANDLE_TAG != 0 { __vh.insert(id, h as u64); } } (st, (h as u64).to_le_bytes().to_vec()) } + 10 => { + let cv = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let pad_h = __c.i32(); + let pad_w = __c.i32(); + let u = __c.i32(); + let v = __c.i32(); + let dil_h = __c.i32(); + let dil_w = __c.i32(); + let mode = __c.i32(); + let ctype = __c.i32(); + let mut out = Vec::new(); + let st = unsafe { (self.f_cudnnSetConvolution2dDescriptor)(cv, pad_h as c_int, pad_w as c_int, u as c_int, v as c_int, dil_h as c_int, dil_w as c_int, mode as c_int, ctype as c_int) }; + (st, out) + } + 11 => { + let __raw = __c.u64(); + let cv = super::vh_resolve(__vh, __raw) as *mut c_void; + if __raw & super::VHANDLE_TAG != 0 { __vh.remove(&__raw); } + let mut out = Vec::new(); + let st = unsafe { (self.f_cudnnDestroyConvolutionDescriptor)(cv) }; + (st, out) + } + 12 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let x = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let w = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let cv = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let y = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let algo = __c.i32(); + let mut size_v: usize = 0 as usize; + let mut out = Vec::new(); + let st = unsafe { (self.f_cudnnGetConvolutionForwardWorkspaceSize)(handle, x, w, cv, y, algo as c_int, &mut size_v) }; + out.extend_from_slice(&(size_v as u64).to_le_bytes()); + (st, out) + } + 13 => { + let handle = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let alpha_v = __c.f32(); + let xDesc = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let x = __c.u64() as *mut c_void; + let wDesc = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let w = __c.u64() as *mut c_void; + let convDesc = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let algo = __c.i32(); + let workspace = __c.u64() as *mut c_void; + let wsSize = __c.u64(); + let beta_v = __c.f32(); + let yDesc = super::vh_resolve(__vh, __c.u64()) as *mut c_void; + let y = __c.u64() as *mut c_void; + let mut out = Vec::new(); + let st = unsafe { (self.f_cudnnConvolutionForward)(handle, &alpha_v, xDesc, x, wDesc, w, convDesc, algo as c_int, workspace, wsSize as usize, &beta_v, yDesc, y) }; + (st, out) + } + _ => (super::super::CUDA_ERROR_NOT_FOUND, Vec::new()), + } + } +} +struct GenCur<'a> { b: &'a [u8], p: usize } +impl GenCur<'_> { + fn take(&mut self, n: usize) -> [u8; 8] { let mut o = [0u8; 8]; let end = (self.p + n).min(self.b.len()); o[..end - self.p].copy_from_slice(&self.b[self.p..end]); self.p = end; o } + fn u16(&mut self) -> u16 { u16::from_le_bytes(self.take(2)[..2].try_into().unwrap()) } + fn i32(&mut self) -> i32 { i32::from_le_bytes(self.take(4)[..4].try_into().unwrap()) } + fn i64(&mut self) -> i64 { i64::from_le_bytes(self.take(8)) } + fn u64(&mut self) -> u64 { u64::from_le_bytes(self.take(8)) } + fn f32(&mut self) -> f32 { f32::from_le_bytes(self.take(4)[..4].try_into().unwrap()) } + fn f64(&mut self) -> f64 { f64::from_le_bytes(self.take(8)) } +} diff --git a/crates/smolvm-cuda/src/host.rs b/crates/smolvm-cuda/src/host.rs new file mode 100644 index 0000000..c8851b9 --- /dev/null +++ b/crates/smolvm-cuda/src/host.rs @@ -0,0 +1,1893 @@ +//! Host-side CUDA-RPC server: dispatch decoded [`Request`]s to a [`Backend`]. +//! +//! The dispatch layer ([`serve`]) owns opaque→raw handle tables for modules, +//! functions and contexts, so the guest only ever sees host-minted ids and +//! cannot forge a host pointer; an unknown id is rejected with +//! `CUDA_ERROR_INVALID_HANDLE`. Device pointers (`CUdeviceptr`) pass through by +//! value because kernel launch parameters embed them — exactly as the Driver +//! API works. +//! +//! Two backends ship: [`GpuBackend`] (the real driver via `nvcuda.dll` / +//! `libcuda.so.1`) and [`CpuBackend`] (emulates a known test kernel so the full +//! protocol + transport can be exercised on a host with no NVIDIA GPU). + +use crate::proto::{decode_request, encode_response, read_msg, write_msg, Request, Response}; +use std::collections::HashMap; +use std::io::{Read, Write}; + +/// `Ok(value)` or `Err(CUresult)` — a non-zero CUDA error code. +pub type CuResult = Result; + +/// `CUDA_ERROR_INVALID_HANDLE`. +pub const CUDA_ERROR_INVALID_HANDLE: i32 = 400; +/// Bit-63 marks a guest-minted virtual handle (see `raw_graph`, cublas vh). +const VHANDLE_TAG: u64 = 1 << 63; +/// `CUDA_ERROR_NOT_FOUND`. +pub const CUDA_ERROR_NOT_FOUND: i32 = 500; +pub const CUDA_ERROR_NOT_SUPPORTED: i32 = 801; + +/// A CUDA Driver-API implementation. Handles returned here are the backend's +/// own raw values (e.g. real `CUmodule` pointers); [`serve`] hides them behind +/// opaque ids before they reach the guest. +pub trait Backend: Send { + fn init(&mut self) -> CuResult<()>; + fn device_get_count(&mut self) -> CuResult; + fn device_get_name(&mut self, device: i32) -> CuResult; + fn device_total_mem(&mut self, device: i32) -> CuResult; + fn driver_get_version(&mut self) -> CuResult; + fn device_get_attribute(&mut self, attrib: i32, device: i32) -> CuResult; + fn device_get_uuid(&mut self, device: i32) -> CuResult<[u8; 16]>; + fn ctx_create(&mut self, device: i32) -> CuResult; + fn ctx_destroy(&mut self, ctx: u64) -> CuResult<()>; + fn primary_ctx_retain(&mut self, device: i32) -> CuResult; + fn primary_ctx_release(&mut self, device: i32) -> CuResult<()>; + fn module_load_data(&mut self, image: &[u8]) -> CuResult; + fn module_get_function(&mut self, module: u64, name: &str) -> CuResult; + fn module_unload(&mut self, module: u64) -> CuResult<()>; + /// Per-parameter byte sizes of the kernel's arguments, in declaration order. + fn func_get_param_info(&mut self, function: u64) -> CuResult>; + /// Set a `CUfunction_attribute` (e.g. raise max dynamic shared memory). + fn func_set_attribute(&mut self, function: u64, attrib: i32, value: i32) -> CuResult<()>; + /// Read a `CUfunction_attribute`; default 0 (CPU backend has no kernels). + fn func_get_attribute(&mut self, _function: u64, _attrib: i32) -> CuResult { + Ok(0) + } + fn mem_alloc(&mut self, bytes: u64) -> CuResult; + fn mem_free(&mut self, dptr: u64) -> CuResult<()>; + /// Copy with stream ordering: prior work on `stream` (0 = legacy default) + /// must complete first. Torch's pool streams are created non-blocking, so + /// a NULL-stream copy does NOT order against them — dropping `stream` + /// makes a `cudaMemcpyAsync` overwrite buffers still-running kernels read. + fn memcpy_htod(&mut self, dptr: u64, data: &[u8], stream: u64) -> CuResult<()>; + /// See `memcpy_htod` for the `stream` contract. + fn memcpy_dtoh(&mut self, dptr: u64, bytes: u64, stream: u64) -> CuResult>; + fn memcpy_dtod(&mut self, dst: u64, src: u64, bytes: u64) -> CuResult<()>; + fn memset_d8(&mut self, dptr: u64, value: u8, bytes: u64) -> CuResult<()>; + fn mem_get_info(&mut self) -> CuResult<(u64, u64)>; + fn launch_kernel( + &mut self, + function: u64, + grid: [u32; 3], + block: [u32; 3], + shared_bytes: u32, + stream: u64, + params: &[Vec], + ) -> CuResult<()>; + fn ctx_synchronize(&mut self) -> CuResult<()>; + fn stream_create(&mut self, flags: u32) -> CuResult; + /// Begin capturing `stream`'s work into a CUDA graph (mode per + /// `cudaStreamCaptureMode`). + fn stream_begin_capture(&mut self, stream: u64, mode: i32) -> CuResult<()>; + /// Exchange this (serving) thread's capture interaction mode; returns the + /// previous mode. Lets a guest run capture-unsafe calls (allocator growth) + /// under `cudaStreamCaptureModeRelaxed` exactly like PyTorch does natively. + fn thread_exchange_capture_mode(&mut self, mode: i32) -> CuResult; + /// Raw `cuStreamQuery` code as a value (0 complete, 600 not ready) — not + /// an error, so it rides the response body. + fn stream_query(&mut self, stream: u64) -> CuResult; + /// `cuStreamWaitEvent`: make `stream` wait for `event`. + fn stream_wait_event(&mut self, stream: u64, event: u64, flags: u32) -> CuResult<()>; + /// Raw `cuEventQuery` code as a value (0 complete, 600 not ready). + fn event_query(&mut self, event: u64) -> CuResult; + /// End capture; returns the raw `cudaGraph_t`. + fn stream_end_capture(&mut self, stream: u64) -> CuResult; + /// `(capture_status, capture_id)` for `stream`. + fn stream_capture_info(&mut self, stream: u64) -> CuResult<(u64, u64)>; + /// Instantiate a captured graph; returns the raw `cudaGraphExec_t`. + fn graph_instantiate(&mut self, graph: u64) -> CuResult; + /// Replay an instantiated graph on `stream`. + fn graph_launch(&mut self, graph_exec: u64, stream: u64) -> CuResult<()>; + fn graph_exec_destroy(&mut self, graph_exec: u64) -> CuResult<()>; + fn graph_destroy(&mut self, graph: u64) -> CuResult<()>; + /// Number of nodes in a captured graph (PyTorch warns on empty graphs). + fn graph_get_node_count(&mut self, graph: u64) -> CuResult; + /// Stream-ordered memset (capture-safe; the sync form would invalidate an + /// active capture). + fn memset_d8_async(&mut self, dptr: u64, value: u8, bytes: u64, stream: u64) -> CuResult<()>; + /// Stream-ordered device-to-device copy (capture-safe). + fn memcpy_dtod_async(&mut self, dst: u64, src: u64, bytes: u64, stream: u64) -> CuResult<()>; + fn stream_destroy(&mut self, stream: u64) -> CuResult<()>; + fn stream_synchronize(&mut self, stream: u64) -> CuResult<()>; + fn event_create(&mut self, flags: u32) -> CuResult; + fn event_destroy(&mut self, event: u64) -> CuResult<()>; + fn event_record(&mut self, event: u64, stream: u64) -> CuResult<()>; + fn event_synchronize(&mut self, event: u64) -> CuResult<()>; + fn event_elapsed_time(&mut self, start: u64, end: u64) -> CuResult; + + // ---- forward-to-host-lib: nvcomp batched Deflate ---- + // Return the library's own `nvcompStatus_t` (0 = success) rather than a + // CUresult; the transport status stays 0 and the shim surfaces this verbatim. + /// `(nvcomp_status, temp_bytes)`. + fn nvcomp_deflate_temp_size( + &mut self, + num_chunks: u64, + max_uncompressed_chunk_bytes: u64, + max_total_uncompressed_bytes: u64, + ) -> CuResult<(i32, u64)>; + #[allow(clippy::too_many_arguments)] + fn nvcomp_deflate_decompress( + &mut self, + device_compressed_ptrs: u64, + device_compressed_bytes: u64, + device_uncompressed_bytes: u64, + device_actual_uncompressed_bytes: u64, + batch_size: u64, + device_temp: u64, + temp_bytes: u64, + device_uncompressed_ptrs: u64, + device_statuses: u64, + stream: u64, + ) -> CuResult; + + // ---- forward-to-host-lib: cuBLAS ---- + /// Returns the backend's raw `cublasHandle_t`. + fn cublas_create(&mut self) -> CuResult; + fn cublas_destroy(&mut self, handle: u64) -> CuResult<()>; + fn cublas_set_stream(&mut self, handle: u64, stream: u64) -> CuResult<()>; + #[allow(clippy::too_many_arguments)] + fn cublas_sgemm( + &mut self, + handle: u64, + transa: u32, + transb: u32, + m: i32, + n: i32, + k: i32, + alpha: f32, + a: u64, + lda: i32, + b: u64, + ldb: i32, + beta: f32, + c: u64, + ldc: i32, + ) -> CuResult<()>; + + /// Generic forward-to-host-lib dispatch (code-generated per function). + /// `streams` is the session's stream table (wire id → raw host stream) for + /// resolving `cudaStream_t` parameters. Returns `(library_status, + /// serialized_outputs)`. Default: unsupported. + fn lib_call( + &mut self, + _lib: u8, + _func: u16, + _args: &[u8], + _streams: &HashMap, + ) -> CuResult<(i32, Vec)> { + Err(CUDA_ERROR_NOT_FOUND) + } + + /// Zero-copy H2D: DMA `size` bytes from shared-region `offset` to `dptr`. + /// Default: no shared region → caller must fall back to byte-shipping. + fn memcpy_shm_htod( + &mut self, + _dptr: u64, + _offset: u64, + _size: u64, + _stream: u64, + ) -> CuResult<()> { + Err(CUDA_ERROR_NOT_FOUND) + } + /// Zero-copy D2H: DMA `size` bytes from `dptr` to shared-region `offset`. + fn memcpy_shm_dtoh( + &mut self, + _offset: u64, + _dptr: u64, + _size: u64, + _stream: u64, + ) -> CuResult<()> { + Err(CUDA_ERROR_NOT_FOUND) + } + + /// Provide the host mappings of guest RAM (`gpa_start, host_va, len` triples) + /// so `memcpy_gpa_*` can read guest memory directly. Set once per connection + /// by the embedder. Guest RAM is usually split around the 4 GiB PCI hole. + fn set_guest_ram(&mut self, _regions: Vec<(u64, u64, u64)>) {} + /// Zero-copy H2D from guest RAM: gather `segments` and DMA to `dptr`. + fn memcpy_gpa_htod( + &mut self, + _dptr: u64, + _segments: &[(u64, u64)], + _stream: u64, + ) -> CuResult<()> { + Err(CUDA_ERROR_NOT_FOUND) + } + /// Zero-copy D2H to guest RAM: DMA from `dptr` and scatter into `segments`. + fn memcpy_gpa_dtoh( + &mut self, + _dptr: u64, + _segments: &[(u64, u64)], + _stream: u64, + ) -> CuResult<()> { + Err(CUDA_ERROR_NOT_FOUND) + } + /// Host virtual address of `len` bytes at guest-physical `gpa`, when the + /// embedder mapped guest RAM (in-VM only). Rings need long-lived page + /// mappings, unlike the per-call `memcpy_gpa_*` segment reads. + fn gpa_to_hva(&mut self, _gpa: u64, _len: u64) -> Option { + None + } + // VMM (torch expandable-segments allocator). Defaults: unsupported. + fn mem_address_reserve(&mut self, _size: u64, _align: u64) -> CuResult { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn mem_create(&mut self, _size: u64, _device: i32) -> CuResult { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn mem_map(&mut self, _va: u64, _size: u64, _offset: u64, _handle: u64) -> CuResult<()> { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn mem_set_access(&mut self, _va: u64, _size: u64, _device: i32) -> CuResult<()> { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn mem_unmap(&mut self, _va: u64, _size: u64) -> CuResult<()> { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn mem_release(&mut self, _handle: u64) -> CuResult<()> { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn mem_address_free(&mut self, _va: u64, _size: u64) -> CuResult<()> { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn mem_get_allocation_granularity(&mut self, _device: i32, _flags: u32) -> CuResult { + Err(CUDA_ERROR_NOT_SUPPORTED) + } +} + +/// Per-connection opaque→raw handle translation. Ids are dense and monotonic so +/// a stale/forged id from the guest never aliases a live resource. +#[derive(Default)] +struct Session { + next_id: u64, + modules: HashMap, + functions: HashMap, + contexts: HashMap, + streams: HashMap, + events: HashMap, + cublas_handles: HashMap, + /// Guest-minted virtual graph/exec handles → real (bit-63 tagged). Lets + /// EndCapture / GraphInstantiate be fire-and-forget: the guest invents the + /// handle, the host maps it when it materializes the real one. + graph_vhandles: HashMap, + /// Live resources this connection created, reclaimed when it ends — + /// a guest that dies mid-run must not leak GPU memory (device + /// allocations are the multi-GB hazard; modules/streams/events are + /// hygiene). Raw backend handles. + owned_dptrs: HashMap, // dptr → size (also the quota ledger) + /// VMM state for reclaim: physical handles → size (quota ledger too), + /// live mappings (va → size), and address reservations (va → size). + owned_vmm_handles: HashMap, + owned_vmm_maps: HashMap, + owned_vmm_reservations: HashMap, + owned_modules: std::collections::HashSet, + owned_streams: std::collections::HashSet, + owned_events: std::collections::HashSet, + primary_retains: u32, +} + +/// Free everything a finished connection still owns. Failures are ignored — +/// the driver may already have reclaimed (context destruction, device reset). +fn reclaim_session(sess: &mut Session, b: &mut dyn Backend) { + for (d, _size) in std::mem::take(&mut sess.owned_dptrs) { + let _ = b.mem_free(d); + } + // VMM teardown order: unmap, release physical, free reservations. + for (va, size) in std::mem::take(&mut sess.owned_vmm_maps) { + let _ = b.mem_unmap(va, size); + } + for (h, _size) in std::mem::take(&mut sess.owned_vmm_handles) { + let _ = b.mem_release(h); + } + for (va, size) in std::mem::take(&mut sess.owned_vmm_reservations) { + let _ = b.mem_address_free(va, size); + } + for m in std::mem::take(&mut sess.owned_modules) { + let _ = b.module_unload(m); + } + for st in std::mem::take(&mut sess.owned_streams) { + let _ = b.stream_destroy(st); + } + for e in std::mem::take(&mut sess.owned_events) { + let _ = b.event_destroy(e); + } + for (vh, real) in std::mem::take(&mut sess.graph_vhandles) { + // Exec handles (used by GraphLaunch) vs graphs — try exec destroy + // first, then graph destroy; the wrong one errors harmlessly. + let _ = if vh != 0 { + b.graph_exec_destroy(real) + .or_else(|_| b.graph_destroy(real)) + } else { + Ok(()) + }; + } + for _ in 0..std::mem::take(&mut sess.primary_retains) { + let _ = b.primary_ctx_release(0); + } +} + +impl Session { + fn mint(&mut self) -> u64 { + self.next_id += 1; + self.next_id + } +} + +/// Serve one CUDA-RPC connection to completion (until the peer closes). Each +/// request is dispatched to `backend`; returns on clean EOF. +pub fn serve(stream: S, backend: &mut dyn Backend) -> std::io::Result<()> { + let mut sess = Session::default(); + let r = serve_inner(stream, backend, &mut sess); + // The connection is over (guest exit, crash, or transport error): free + // everything it still owns so a dead client can't hold GPU memory. + reclaim_session(&mut sess, backend); + r +} + +fn serve_inner( + mut stream: S, + backend: &mut dyn Backend, + sess: &mut Session, +) -> std::io::Result<()> { + // First failure among quiet (fire-and-forget) requests since the last + // fence. Quiet requests get no response at all — the client collects + // failures with one Fence round-trip instead of reading N per-op replies, + // which on vsock cost a guest wake-up each. + let mut quiet_sticky: i32 = 0; + while let Some(payload) = read_msg(&mut stream)? { + match payload.first() { + // Quiet wrapper: execute the inner request, reply with nothing. + Some(&crate::proto::QUIET_PREFIX) => { + let req = decode_request(&payload[1..])?; + if std::env::var_os("SMOLVM_CUDA_HOST_OPLOG").is_some() { + eprintln!("[op~] 0x{:02x} len={}", payload[1], payload.len()); + } + let (status, _) = dispatch(sess, backend, req); + if status != 0 && std::env::var_os("SMOLVM_CUDA_HOST_OPLOG").is_some() { + eprintln!("[op~!] status={status}"); + } + if status != 0 && quiet_sticky == 0 { + quiet_sticky = status; + } + } + // Fence: report (and clear) the sticky quiet failure. + Some(&crate::proto::FENCE_OP) if payload.len() == 1 => { + let st = std::mem::take(&mut quiet_sticky); + write_msg(&mut stream, &encode_response(st, &Response::Ok))?; + } + _ => { + let req = decode_request(&payload)?; + if std::env::var_os("SMOLVM_CUDA_HOST_OPLOG").is_some() { + eprintln!("[op] 0x{:02x} len={}", payload[0], payload.len()); + } + // Transport upgrade: switch this connection to shared-memory + // rings and never return to socket framing (the socket then + // carries only doorbell bytes). + if let Request::RingSetup { + page_size, + req_pages, + resp_pages, + bounce_pages, + } = req + { + match HostRings::map(backend, page_size, &req_pages, &resp_pages, &bounce_pages) + { + Ok(rings) => { + write_msg(&mut stream, &encode_response(0, &Response::Ok))?; + return serve_rings(stream, backend, sess, quiet_sticky, rings); + // (session reclaimed by `serve` on return) + } + Err(code) => { + write_msg(&mut stream, &encode_response(code, &Response::Ok))?; + continue; + } + } + } + let (status, resp) = dispatch(sess, backend, req); + if status != 0 && std::env::var_os("SMOLVM_CUDA_HOST_OPLOG").is_some() { + eprintln!("[op!] status={status}"); + } + let out = encode_response(status, &resp); + write_msg(&mut stream, &out)?; + } + } + } + Ok(()) +} + +/// Host mappings of the guest's rings + bounce buffer. +struct HostRings { + req: crate::ring::Ring, + resp: crate::ring::Ring, + /// Bounce pages (host VAs) for responses too large for an inline record. + bounce: Vec<*mut u8>, + page_size: usize, +} + +impl HostRings { + fn map( + backend: &mut dyn Backend, + page_size: u32, + req_pages: &[u64], + resp_pages: &[u64], + bounce_pages: &[u64], + ) -> Result { + let ps = page_size as usize; + if ps < crate::ring::HEADER_SIZE + crate::ring::RECORD_SIZE + || req_pages.is_empty() + || resp_pages.is_empty() + { + return Err(1); // CUDA_ERROR_INVALID_VALUE + } + let mut map_all = |gpas: &[u64]| -> Result, i32> { + gpas.iter() + .map(|&gpa| { + backend + .gpa_to_hva(gpa, page_size as u64) + .map(|hva| hva as *mut u8) + .ok_or(CUDA_ERROR_NOT_SUPPORTED) + }) + .collect() + }; + let req = map_all(req_pages)?; + let resp = map_all(resp_pages)?; + let bounce = map_all(bounce_pages)?; + // SAFETY: pages are backed by mapped guest RAM for the VM's lifetime. + Ok(HostRings { + req: unsafe { crate::ring::Ring::from_pages(req, ps) }, + resp: unsafe { crate::ring::Ring::from_pages(resp, ps) }, + bounce, + page_size: ps, + }) + } + + /// Copy an oversized response into the bounce buffer. Returns false when + /// it doesn't fit (protocol error surfaced to the guest as a status). + fn write_bounce(&self, bytes: &[u8]) -> bool { + if bytes.len() > self.bounce.len() * self.page_size { + return false; + } + for (i, chunk) in bytes.chunks(self.page_size).enumerate() { + // SAFETY: chunk fits within bounce page i (checked above). + unsafe { + std::ptr::copy_nonoverlapping(chunk.as_ptr(), self.bounce[i], chunk.len()); + } + } + true + } +} + +/// Ring-mode serve loop: requests pop from the guest's request ring, +/// responses push to the completion ring; the socket carries doorbells only. +fn serve_rings( + mut stream: S, + backend: &mut dyn Backend, + sess: &mut Session, + mut quiet_sticky: i32, + rings: HostRings, +) -> std::io::Result<()> { + use crate::ring::LEN_INDIRECT; + let oplog = std::env::var_os("SMOLVM_CUDA_HOST_OPLOG").is_some(); + // Reassembly buffer for oversized frames arriving as bounce chunks. + let mut pending: Vec = Vec::new(); + // Push one response. Oversized ones chunk through the bounce pages: the + // guest posts a continue record (16×0xFF, LEN_INDIRECT) on the request + // ring after copying each non-final chunk out — unambiguous because a + // blocked guest can't send anything else. Doorbell on park either way. + fn respond( + rings: &HostRings, + stream: &mut S, + bytes: &[u8], + ) -> std::io::Result<()> { + let kick = |stream: &mut S| -> std::io::Result<()> { + if rings_take_parked(rings) { + stream.write_all(&[1u8])?; + stream.flush()?; + } + Ok(()) + }; + fn rings_take_parked(rings: &HostRings) -> bool { + rings.resp.take_parked() + } + if bytes.len() <= crate::ring::INLINE_MAX { + while !rings.resp.try_push(bytes, 0) { + std::hint::spin_loop(); // guest drains sync responses promptly + } + return kick(stream); + } + let cap = rings.bounce.len() * rings.page_size; + let total = bytes.len(); + let mut off = 0; + while off < total { + let chunk = (total - off).min(cap); + if !rings.write_bounce(&bytes[off..off + chunk]) { + return Err(std::io::Error::other("ring: bounce write failed")); + } + let mut hdr = [0u8; 16]; + hdr[..8].copy_from_slice(&(total as u64).to_le_bytes()); + hdr[8..].copy_from_slice(&(chunk as u64).to_le_bytes()); + while !rings.resp.try_push(&hdr, LEN_INDIRECT) { + std::hint::spin_loop(); + } + kick(stream)?; + off += chunk; + if off < total { + // Await the guest's continue record before refilling. + loop { + if let Some((p, f)) = rings.req.try_pop() { + if f & LEN_INDIRECT != 0 && p == [0xFF; 16] { + break; + } + return Err(std::io::Error::other("ring: expected continue")); + } + std::hint::spin_loop(); + } + } + } + Ok(()) + } + loop { + let (payload, flags) = match rings.req.try_pop() { + Some(rec) => rec, + None => { + // Adaptive wait: spin briefly, then park and block on a + // doorbell byte from the guest. + let mut found = None; + for _ in 0..20_000 { + if let Some(rec) = rings.req.try_pop() { + found = Some(rec); + break; + } + std::hint::spin_loop(); + } + match found { + Some(rec) => rec, + None => { + if !rings.req.park() { + let mut byte = [0u8; 1]; + match stream.read(&mut byte) { + Ok(0) => return Ok(()), // guest closed + Ok(_) => {} + Err(e) => return Err(e), + } + rings.req.unpark(); + } + continue; + } + } + } + }; + // Oversized frame: chunks arrive through the bounce pages. Every + // non-final chunk is acked (so the guest can refill the pages); the + // final chunk completes the frame, which dispatches below and whose + // own response closes the exchange. + let frame: Vec = if flags & LEN_INDIRECT != 0 { + if payload.len() < 16 { + return Err(std::io::Error::other("ring: short chunk record")); + } + let total = u64::from_le_bytes(payload[..8].try_into().unwrap()) as usize; + let chunk = u64::from_le_bytes(payload[8..16].try_into().unwrap()) as usize; + if total > crate::proto::MAX_MSG || chunk > rings.bounce.len() * rings.page_size { + return Err(std::io::Error::other("ring: oversized chunk")); + } + let mut left = chunk; + for &page in &rings.bounce { + if left == 0 { + break; + } + let take = left.min(rings.page_size); + // SAFETY: bounce pages are mapped guest RAM. + unsafe { + pending.extend_from_slice(std::slice::from_raw_parts(page, take)); + } + left -= take; + } + if pending.len() < total { + // Ack the chunk so the guest may refill the bounce pages. + respond(&rings, &mut stream, &encode_response(0, &Response::Ok))?; + continue; + } + std::mem::take(&mut pending) + } else { + payload + }; + match frame.first() { + Some(&crate::proto::QUIET_PREFIX) => { + let req = match decode_request(&frame[1..]) { + Ok(r) => r, + Err(e) => { + eprintln!( + "[ring-dbg] malformed QUIET frame len={} head={:02x?}", + frame.len(), + &frame[..frame.len().min(12)] + ); + return Err(e); + } + }; + if oplog { + eprintln!("[op~] 0x{:02x} len={}", frame[1], frame.len()); + } + let (status, _) = dispatch(sess, backend, req); + if status != 0 { + if oplog { + eprintln!("[op~!] status={status}"); + } + if quiet_sticky == 0 { + quiet_sticky = status; + } + } + } + Some(&crate::proto::FENCE_OP) if frame.len() == 1 => { + let st = std::mem::take(&mut quiet_sticky); + respond(&rings, &mut stream, &encode_response(st, &Response::Ok))?; + } + _ => { + let req = match decode_request(&frame) { + Ok(r) => r, + Err(e) => { + eprintln!( + "[ring-dbg] malformed frame len={} head={:02x?}", + frame.len(), + &frame[..frame.len().min(12)] + ); + return Err(e); + } + }; + if oplog { + eprintln!("[op] 0x{:02x} len={}", frame[0], frame.len()); + } + let (status, resp) = dispatch(sess, backend, req); + if status != 0 && oplog { + eprintln!("[op!] status={status}"); + } + respond(&rings, &mut stream, &encode_response(status, &resp))?; + } + } + } +} + +/// Per-connection VRAM budget (`SMOLVM_CUDA_VRAM_LIMIT_MB`), read per +/// allocation: rare calls, and staying uncached keeps it adjustable and +/// test-deterministic. +fn vram_limit() -> u64 { + std::env::var("SMOLVM_CUDA_VRAM_LIMIT_MB") + .ok() + .and_then(|v| v.parse::().ok()) + .map(|mb| mb * 1024 * 1024) + .unwrap_or(u64::MAX) +} + +fn dispatch(sess: &mut Session, b: &mut dyn Backend, req: Request) -> (i32, Response) { + // Translate an opaque id to the backend's raw handle, or error. + fn raw(map: &HashMap, id: u64) -> CuResult { + map.get(&id).copied().ok_or(CUDA_ERROR_INVALID_HANDLE) + } + // Translate an opaque stream id: 0 is the default stream (passes through), + // anything else must be a live minted id. + fn raw_stream(sess: &Session, stream: u64) -> CuResult { + // Streams are raw host pointers on the wire (see StreamCreate below); + // the table only translates ids minted by pre-raw-stream guests. + if stream == 0 { + Ok(0) + } else { + Ok(sess.streams.get(&stream).copied().unwrap_or(stream)) + } + } + fn raw_graph(sess: &Session, h: u64) -> u64 { + // Virtual graph/exec handle → real; untagged values pass through. + if h & VHANDLE_TAG != 0 { + sess.graph_vhandles.get(&h).copied().unwrap_or(0) + } else { + h + } + } + fn raw_event(sess: &Session, event: u64) -> CuResult { + // Same raw-on-the-wire convention as streams. + Ok(sess.events.get(&event).copied().unwrap_or(event)) + } + let r: CuResult = (|| match req { + Request::Init { proto_hash } => { + if proto_hash != crate::PROTO_HASH { + eprintln!( + "[smolvm-cuda] PROTOCOL MISMATCH: client wire hash {:016x} != server {:016x} \ + — the guest shim and host server were built from different source. \ + Rebuild and restage both. Refusing the connection to avoid corruption.", + proto_hash, + crate::PROTO_HASH + ); + // CUDA_ERROR_NOT_SUPPORTED — surfaced at cuInit, loud and early. + Err(CUDA_ERROR_NOT_SUPPORTED) + } else { + b.init().map(|_| Response::Ok) + } + } + Request::DeviceGetCount => b.device_get_count().map(Response::Count), + Request::DeviceGetName { device } => b.device_get_name(device).map(Response::Name), + Request::DeviceTotalMem { device } => b.device_total_mem(device).map(Response::Bytes), + Request::DriverGetVersion => b.driver_get_version().map(Response::Count), + Request::DeviceGetAttribute { attrib, device } => { + b.device_get_attribute(attrib, device).map(Response::Count) + } + Request::DeviceGetUuid { device } => b + .device_get_uuid(device) + .map(|u| Response::Data(u.to_vec())), + Request::CtxCreate { device } => { + let raw = b.ctx_create(device)?; + let id = sess.mint(); + sess.contexts.insert(id, raw); + Ok(Response::Handle(id)) + } + Request::CtxDestroy { ctx } => { + let raw = raw(&sess.contexts, ctx)?; + b.ctx_destroy(raw)?; + sess.contexts.remove(&ctx); + Ok(Response::Ok) + } + Request::PrimaryCtxRetain { device } => { + let raw = b.primary_ctx_retain(device)?; + sess.primary_retains += 1; + let id = sess.mint(); + sess.contexts.insert(id, raw); + Ok(Response::Handle(id)) + } + Request::PrimaryCtxRelease { device } => { + sess.primary_retains = sess.primary_retains.saturating_sub(1); + b.primary_ctx_release(device).map(|_| Response::Ok) + } + Request::ModuleLoadData { image } => { + let raw = b.module_load_data(&image)?; + sess.owned_modules.insert(raw); + let id = sess.mint(); + sess.modules.insert(id, raw); + Ok(Response::Handle(id)) + } + Request::ModuleGetFunction { module, name } => { + let raw_mod = raw(&sess.modules, module)?; + let raw_fn = b.module_get_function(raw_mod, &name)?; + let id = sess.mint(); + sess.functions.insert(id, raw_fn); + Ok(Response::Handle(id)) + } + Request::ModuleUnload { module } => { + let raw_mod = raw(&sess.modules, module)?; + b.module_unload(raw_mod)?; + sess.owned_modules.remove(&raw_mod); + sess.modules.remove(&module); + Ok(Response::Ok) + } + Request::FuncGetParamInfo { function } => { + let raw_fn = raw(&sess.functions, function)?; + let sizes = b.func_get_param_info(raw_fn)?; + let mut out = Vec::with_capacity(sizes.len() * 4); + for s in sizes { + out.extend_from_slice(&s.to_le_bytes()); + } + Ok(Response::Data(out)) + } + Request::FuncSetAttribute { + function, + attrib, + value, + } => { + let raw_fn = raw(&sess.functions, function)?; + b.func_set_attribute(raw_fn, attrib, value) + .map(|_| Response::Ok) + } + Request::FuncGetAttribute { function, attrib } => { + let raw_fn = raw(&sess.functions, function)?; + b.func_get_attribute(raw_fn, attrib).map(Response::Count) + } + Request::MemAlloc { bytes } => { + // Per-connection VRAM quota (SMOLVM_CUDA_VRAM_LIMIT_MB on the + // host): a guest may not allocate past its budget — the CUDA- + // native failure (out of memory) surfaces to the app. + let limit = vram_limit(); + let used: u64 = sess.owned_dptrs.values().sum::() + + sess.owned_vmm_handles.values().sum::(); + if used.saturating_add(bytes) > limit { + return Err(2); // CUDA_ERROR_OUT_OF_MEMORY + } + b.mem_alloc(bytes).map(|d| { + sess.owned_dptrs.insert(d, bytes); + Response::Dptr(d) + }) + } + Request::MemFree { dptr } => { + sess.owned_dptrs.remove(&dptr); + // (removal returns the freed size to the quota ledger) + b.mem_free(dptr).map(|_| Response::Ok) + } + Request::MemcpyHtoD { dptr, stream, data } => b + .memcpy_htod(dptr, &data, raw_stream(sess, stream)?) + .map(|_| Response::Ok), + Request::MemcpyDtoH { + dptr, + bytes, + stream, + } => b + .memcpy_dtoh(dptr, bytes, raw_stream(sess, stream)?) + .map(Response::Data), + Request::MemcpyDtoD { dst, src, bytes } => { + b.memcpy_dtod(dst, src, bytes).map(|_| Response::Ok) + } + Request::MemsetD8 { dptr, value, bytes } => { + b.memset_d8(dptr, value, bytes).map(|_| Response::Ok) + } + Request::MemGetInfo => b.mem_get_info().map(|(f, t)| Response::Pair(f, t)), + Request::LaunchKernel { + function, + grid, + block, + shared_bytes, + stream, + params, + } => { + let raw_fn = raw(&sess.functions, function)?; + let raw_str = raw_stream(sess, stream)?; + b.launch_kernel(raw_fn, grid, block, shared_bytes, raw_str, ¶ms) + .map(|_| Response::Ok) + } + Request::CtxSynchronize => b.ctx_synchronize().map(|_| Response::Ok), + // Streams hand back the RAW host pointer, not a session-minted id. + // Streams are context-scoped and every connection retains the same + // device primary context — but one guest process holds SEPARATE + // connections (runtime shim + driver shim), so a per-session id minted + // on one is garbage on the other: torch's capture stream (runtime) + // fed to a Triton kernel launch (driver) came back INVALID_HANDLE. + // Raw pointers are valid on every connection, like device pointers. + Request::StreamCreate { flags } => b.stream_create(flags).map(|st| { + sess.owned_streams.insert(st); + Response::Handle(st) + }), + Request::StreamBeginCapture { stream, mode } => { + let raw = raw_stream(sess, stream)?; + b.stream_begin_capture(raw, mode).map(|_| Response::Ok) + } + Request::ThreadExchangeCaptureMode { mode } => { + b.thread_exchange_capture_mode(mode).map(Response::Count) + } + Request::StreamEndCapture { stream, graph_vh } => { + let raw = raw_stream(sess, stream)?; + let g = b.stream_end_capture(raw)?; + if graph_vh & VHANDLE_TAG != 0 { + sess.graph_vhandles.insert(graph_vh, g); + } + Ok(Response::Handle(g)) + } + Request::StreamCaptureInfo { stream } => { + let raw = raw_stream(sess, stream)?; + b.stream_capture_info(raw) + .map(|(st, id)| Response::Pair(st, id)) + } + Request::GraphInstantiate { graph, exec_vh } => { + let real_graph = raw_graph(sess, graph); + let e = b.graph_instantiate(real_graph)?; + if exec_vh & VHANDLE_TAG != 0 { + sess.graph_vhandles.insert(exec_vh, e); + } + Ok(Response::Handle(e)) + } + Request::GraphLaunch { graph_exec, stream } => { + let raw = raw_stream(sess, stream)?; + b.graph_launch(raw_graph(sess, graph_exec), raw) + .map(|_| Response::Ok) + } + Request::GraphExecDestroy { graph_exec } => { + let real = raw_graph(sess, graph_exec); + sess.graph_vhandles.remove(&graph_exec); + b.graph_exec_destroy(real).map(|_| Response::Ok) + } + Request::GraphDestroy { graph } => { + let real = raw_graph(sess, graph); + sess.graph_vhandles.remove(&graph); + b.graph_destroy(real).map(|_| Response::Ok) + } + Request::GraphGetNodes { graph } => b.graph_get_node_count(graph).map(Response::Bytes), + Request::MemsetD8Async { + dptr, + value, + bytes, + stream, + } => { + let raw = raw_stream(sess, stream)?; + b.memset_d8_async(dptr, value, bytes, raw) + .map(|_| Response::Ok) + } + Request::MemcpyDtoDAsync { + dst, + src, + bytes, + stream, + } => { + let raw = raw_stream(sess, stream)?; + b.memcpy_dtod_async(dst, src, bytes, raw) + .map(|_| Response::Ok) + } + Request::StreamDestroy { stream } => { + let raw = raw_stream(sess, stream)?; + b.stream_destroy(raw)?; + sess.owned_streams.remove(&raw); + sess.streams.remove(&stream); + Ok(Response::Ok) + } + Request::StreamSynchronize { stream } => { + let raw = raw_stream(sess, stream)?; + b.stream_synchronize(raw).map(|_| Response::Ok) + } + Request::StreamQuery { stream } => { + let raw = raw_stream(sess, stream)?; + b.stream_query(raw).map(Response::Count) + } + Request::StreamWaitEvent { + stream, + event, + flags, + } => { + let raw_s = raw_stream(sess, stream)?; + let raw_e = raw_event(sess, event)?; + b.stream_wait_event(raw_s, raw_e, flags) + .map(|_| Response::Ok) + } + // Events are raw host pointers on the wire, same as streams (see + // StreamCreate): context-scoped, and one guest process talks over + // several connections that must all understand the same handle. + Request::EventCreate { flags } => b.event_create(flags).map(|e| { + sess.owned_events.insert(e); + Response::Handle(e) + }), + Request::EventDestroy { event } => { + let raw = raw_event(sess, event)?; + b.event_destroy(raw)?; + sess.owned_events.remove(&raw); + sess.events.remove(&event); + Ok(Response::Ok) + } + Request::EventRecord { event, stream } => { + let raw_ev = raw_event(sess, event)?; + let raw_str = raw_stream(sess, stream)?; + b.event_record(raw_ev, raw_str).map(|_| Response::Ok) + } + Request::EventSynchronize { event } => { + let raw_ev = raw_event(sess, event)?; + b.event_synchronize(raw_ev).map(|_| Response::Ok) + } + Request::EventQuery { event } => { + let raw_ev = raw_event(sess, event)?; + b.event_query(raw_ev).map(Response::Count) + } + Request::EventElapsedTime { start, end } => { + let raw_start = raw_event(sess, start)?; + let raw_end = raw_event(sess, end)?; + b.event_elapsed_time(raw_start, raw_end) + .map(Response::Millis) + } + Request::NvcompDeflateTempSize { + num_chunks, + max_uncompressed_chunk_bytes, + max_total_uncompressed_bytes, + } => b + .nvcomp_deflate_temp_size( + num_chunks, + max_uncompressed_chunk_bytes, + max_total_uncompressed_bytes, + ) + .map(|(st, tb)| Response::Pair(st as u64, tb)), + Request::NvcompDeflateDecompress { + device_compressed_ptrs, + device_compressed_bytes, + device_uncompressed_bytes, + device_actual_uncompressed_bytes, + batch_size, + device_temp, + temp_bytes, + device_uncompressed_ptrs, + device_statuses, + stream, + } => { + let raw_str = raw_stream(sess, stream)?; + b.nvcomp_deflate_decompress( + device_compressed_ptrs, + device_compressed_bytes, + device_uncompressed_bytes, + device_actual_uncompressed_bytes, + batch_size, + device_temp, + temp_bytes, + device_uncompressed_ptrs, + device_statuses, + raw_str, + ) + .map(Response::Count) + } + Request::CublasCreate => { + let raw = b.cublas_create()?; + let id = sess.mint(); + sess.cublas_handles.insert(id, raw); + Ok(Response::Handle(id)) + } + Request::CublasDestroy { handle } => { + let raw = raw(&sess.cublas_handles, handle)?; + b.cublas_destroy(raw)?; + sess.cublas_handles.remove(&handle); + Ok(Response::Ok) + } + Request::CublasSetStream { handle, stream } => { + let raw_h = raw(&sess.cublas_handles, handle)?; + let raw_s = raw_stream(sess, stream)?; + b.cublas_set_stream(raw_h, raw_s).map(|_| Response::Ok) + } + Request::CublasSgemm { + handle, + transa, + transb, + m, + n, + k, + alpha_bits, + a, + lda, + b: bmat, + ldb, + beta_bits, + c, + ldc, + } => { + let raw_h = raw(&sess.cublas_handles, handle)?; + b.cublas_sgemm( + raw_h, + transa, + transb, + m, + n, + k, + f32::from_bits(alpha_bits), + a, + lda, + bmat, + ldb, + f32::from_bits(beta_bits), + c, + ldc, + ) + .map(|_| Response::Ok) + } + Request::LibCall { lib, func, args } => b + .lib_call(lib, func, &args, &sess.streams) + .map(|(status, out)| Response::LibResult(status, out)), + Request::MemcpyShmHtoD { + dptr, + offset, + size, + stream, + } => b + .memcpy_shm_htod(dptr, offset, size, raw_stream(sess, stream)?) + .map(|_| Response::Ok), + Request::MemcpyShmDtoH { + offset, + dptr, + size, + stream, + } => b + .memcpy_shm_dtoh(offset, dptr, size, raw_stream(sess, stream)?) + .map(|_| Response::Ok), + Request::MemcpyGpaHtoD { + dptr, + stream, + segments, + } => b + .memcpy_gpa_htod(dptr, &segments, raw_stream(sess, stream)?) + .map(|_| Response::Ok), + Request::MemcpyGpaDtoH { + dptr, + stream, + segments, + } => b + .memcpy_gpa_dtoh(dptr, &segments, raw_stream(sess, stream)?) + .map(|_| Response::Ok), + // Handled by the serve loop (transport concern, not a backend op); + // reaching dispatch means the transport doesn't support rings. + Request::RingSetup { .. } => Err(CUDA_ERROR_NOT_SUPPORTED), + Request::MemAddressReserve { size, align } => { + b.mem_address_reserve(size, align).map(|va| { + sess.owned_vmm_reservations.insert(va, size); + Response::Dptr(va) + }) + } + Request::MemCreate { size, device } => { + let limit = vram_limit(); + let used: u64 = sess.owned_dptrs.values().sum::() + + sess.owned_vmm_handles.values().sum::(); + if used.saturating_add(size) > limit { + return Err(2); // CUDA_ERROR_OUT_OF_MEMORY + } + b.mem_create(size, device).map(|h| { + sess.owned_vmm_handles.insert(h, size); + Response::Handle(h) + }) + } + Request::MemMap { + va, + size, + offset, + handle, + } => b.mem_map(va, size, offset, handle).map(|_| { + sess.owned_vmm_maps.insert(va, size); + Response::Ok + }), + Request::MemSetAccess { va, size, device } => { + b.mem_set_access(va, size, device).map(|_| Response::Ok) + } + Request::MemUnmap { va, size } => { + sess.owned_vmm_maps.remove(&va); + b.mem_unmap(va, size).map(|_| Response::Ok) + } + Request::MemRelease { handle } => { + sess.owned_vmm_handles.remove(&handle); + b.mem_release(handle).map(|_| Response::Ok) + } + Request::MemAddressFree { va, size } => { + sess.owned_vmm_reservations.remove(&va); + b.mem_address_free(va, size).map(|_| Response::Ok) + } + Request::MemGetAllocationGranularity { device, flags } => b + .mem_get_allocation_granularity(device, flags) + .map(Response::Bytes), + })(); + match r { + Ok(resp) => (0, resp), + Err(code) => (code, Response::Ok), + } +} + +// =========================================================================== +// CPU emulation backend (for GPU-less verification of the full RPC path) +// =========================================================================== + +/// Emulates a small, known set of kernels so the protocol + transport can be +/// exercised end-to-end without an NVIDIA GPU. Device memory is a bump-allocated +/// host buffer keyed by fake `CUdeviceptr`. Recognizes the `vecadd(a,b,c,n)` +/// test kernel by name; unknown kernels return `CUDA_ERROR_NOT_FOUND`. +pub struct CpuBackend { + next_dptr: u64, + mem: HashMap>, + fn_names: HashMap, + next_handle: u64, + /// Guest-RAM mappings, same shape as the GPU backend's — lets the ring + /// transport (and its tests) run without a GPU. + guest_ram: Vec<(u64, u64, u64)>, +} + +impl Default for CpuBackend { + fn default() -> Self { + CpuBackend { + next_dptr: 0x1_0000_0000, // distinct from small handle ids + mem: HashMap::new(), + fn_names: HashMap::new(), + next_handle: 1, + guest_ram: Vec::new(), + } + } +} + +impl CpuBackend { + fn handle(&mut self) -> u64 { + let h = self.next_handle; + self.next_handle += 1; + h + } +} + +fn read_u64(p: &[u8]) -> Option { + Some(u64::from_le_bytes(p.get(..8)?.try_into().ok()?)) +} +fn read_u32(p: &[u8]) -> Option { + Some(u32::from_le_bytes(p.get(..4)?.try_into().ok()?)) +} + +impl Backend for CpuBackend { + fn init(&mut self) -> CuResult<()> { + Ok(()) + } + fn set_guest_ram(&mut self, regions: Vec<(u64, u64, u64)>) { + self.guest_ram = regions; + } + fn gpa_to_hva(&mut self, gpa: u64, len: u64) -> Option { + self.guest_ram.iter().find_map(|&(gs, hva, rlen)| { + (gpa >= gs && gpa.checked_add(len)? <= gs + rlen).then(|| hva + (gpa - gs)) + }) + } + fn device_get_count(&mut self) -> CuResult { + Ok(1) + } + fn device_get_name(&mut self, _device: i32) -> CuResult { + Ok("smolvm CPU emulation device".into()) + } + fn device_total_mem(&mut self, _device: i32) -> CuResult { + Ok(1 << 30) + } + fn driver_get_version(&mut self) -> CuResult { + Ok(13000) + } + fn device_get_attribute(&mut self, attrib: i32, _device: i32) -> CuResult { + // Plausible values for the attributes real programs commonly probe + // (CUdevice_attribute numeric ids from cuda.h). + Ok(match attrib { + 1 => 1024, // MAX_THREADS_PER_BLOCK + 2..=4 => 1024, // MAX_BLOCK_DIM_X/Y/Z + 5 => 2147483647, // MAX_GRID_DIM_X + 6 | 7 => 65535, // MAX_GRID_DIM_Y/Z + 8 => 49152, // MAX_SHARED_MEMORY_PER_BLOCK + 10 => 32, // WARP_SIZE + 16 => 1, // MULTIPROCESSOR_COUNT + 75 => 8, // COMPUTE_CAPABILITY_MAJOR + 76 => 6, // COMPUTE_CAPABILITY_MINOR + _ => 1, + }) + } + fn device_get_uuid(&mut self, _device: i32) -> CuResult<[u8; 16]> { + Ok(*b"smolvm-cpu-emul\0") + } + fn ctx_create(&mut self, _device: i32) -> CuResult { + Ok(self.handle()) + } + fn ctx_destroy(&mut self, _ctx: u64) -> CuResult<()> { + Ok(()) + } + fn primary_ctx_retain(&mut self, _device: i32) -> CuResult { + Ok(self.handle()) + } + fn primary_ctx_release(&mut self, _device: i32) -> CuResult<()> { + Ok(()) + } + fn module_load_data(&mut self, _image: &[u8]) -> CuResult { + Ok(self.handle()) + } + fn module_get_function(&mut self, _module: u64, name: &str) -> CuResult { + let h = self.handle(); + self.fn_names.insert(h, name.to_string()); + Ok(h) + } + fn module_unload(&mut self, _module: u64) -> CuResult<()> { + Ok(()) + } + fn func_get_param_info(&mut self, function: u64) -> CuResult> { + let name = self + .fn_names + .get(&function) + .ok_or(CUDA_ERROR_INVALID_HANDLE)?; + match name.as_str() { + // vecadd(const float* a, const float* b, float* c, int n) + "vecadd" => Ok(vec![8, 8, 8, 4]), + _ => Err(CUDA_ERROR_NOT_FOUND), + } + } + fn func_set_attribute(&mut self, _function: u64, _attrib: i32, _value: i32) -> CuResult<()> { + // CPU emulation has no shared-memory limits to raise. + Ok(()) + } + fn mem_alloc(&mut self, bytes: u64) -> CuResult { + let dptr = self.next_dptr; + self.next_dptr += bytes.max(1); + self.mem.insert(dptr, vec![0u8; bytes as usize]); + Ok(dptr) + } + fn mem_free(&mut self, dptr: u64) -> CuResult<()> { + self.mem + .remove(&dptr) + .map(|_| ()) + .ok_or(CUDA_ERROR_INVALID_HANDLE) + } + fn memcpy_htod(&mut self, dptr: u64, data: &[u8], _stream: u64) -> CuResult<()> { + let buf = self.mem.get_mut(&dptr).ok_or(CUDA_ERROR_INVALID_HANDLE)?; + if data.len() > buf.len() { + return Err(CUDA_ERROR_INVALID_HANDLE); + } + buf[..data.len()].copy_from_slice(data); + Ok(()) + } + fn memcpy_dtoh(&mut self, dptr: u64, bytes: u64, _stream: u64) -> CuResult> { + let buf = self.mem.get(&dptr).ok_or(CUDA_ERROR_INVALID_HANDLE)?; + let n = bytes as usize; + if n > buf.len() { + return Err(CUDA_ERROR_INVALID_HANDLE); + } + Ok(buf[..n].to_vec()) + } + fn memcpy_dtod(&mut self, dst: u64, src: u64, bytes: u64) -> CuResult<()> { + let n = bytes as usize; + let data = { + let s = self.mem.get(&src).ok_or(CUDA_ERROR_INVALID_HANDLE)?; + if n > s.len() { + return Err(CUDA_ERROR_INVALID_HANDLE); + } + s[..n].to_vec() + }; + let d = self.mem.get_mut(&dst).ok_or(CUDA_ERROR_INVALID_HANDLE)?; + if n > d.len() { + return Err(CUDA_ERROR_INVALID_HANDLE); + } + d[..n].copy_from_slice(&data); + Ok(()) + } + fn memset_d8(&mut self, dptr: u64, value: u8, bytes: u64) -> CuResult<()> { + let buf = self.mem.get_mut(&dptr).ok_or(CUDA_ERROR_INVALID_HANDLE)?; + let n = bytes as usize; + if n > buf.len() { + return Err(CUDA_ERROR_INVALID_HANDLE); + } + buf[..n].fill(value); + Ok(()) + } + fn mem_get_info(&mut self) -> CuResult<(u64, u64)> { + Ok((1 << 29, 1 << 30)) + } + fn launch_kernel( + &mut self, + function: u64, + _grid: [u32; 3], + _block: [u32; 3], + _shared_bytes: u32, + _stream: u64, + params: &[Vec], + ) -> CuResult<()> { + let name = self + .fn_names + .get(&function) + .cloned() + .ok_or(CUDA_ERROR_INVALID_HANDLE)?; + match name.as_str() { + // vecadd(const float* a, const float* b, float* c, int n): c[i]=a[i]+b[i] + "vecadd" => { + if params.len() != 4 { + return Err(CUDA_ERROR_NOT_FOUND); + } + let a = read_u64(¶ms[0]).ok_or(CUDA_ERROR_NOT_FOUND)?; + let b = read_u64(¶ms[1]).ok_or(CUDA_ERROR_NOT_FOUND)?; + let c = read_u64(¶ms[2]).ok_or(CUDA_ERROR_NOT_FOUND)?; + let n = read_u32(¶ms[3]).ok_or(CUDA_ERROR_NOT_FOUND)? as usize; + let av = self.mem.get(&a).ok_or(CUDA_ERROR_INVALID_HANDLE)?.clone(); + let bv = self.mem.get(&b).ok_or(CUDA_ERROR_INVALID_HANDLE)?.clone(); + let out = self.mem.get_mut(&c).ok_or(CUDA_ERROR_INVALID_HANDLE)?; + for i in 0..n { + let x = f32::from_le_bytes(av[i * 4..i * 4 + 4].try_into().unwrap()); + let y = f32::from_le_bytes(bv[i * 4..i * 4 + 4].try_into().unwrap()); + out[i * 4..i * 4 + 4].copy_from_slice(&(x + y).to_le_bytes()); + } + Ok(()) + } + _ => Err(CUDA_ERROR_NOT_FOUND), + } + } + fn ctx_synchronize(&mut self) -> CuResult<()> { + Ok(()) + } + // Streams and events are inert in emulation: every operation completes + // synchronously, so create/destroy mint handles and the rest are no-ops. + fn stream_create(&mut self, _flags: u32) -> CuResult { + Ok(self.handle()) + } + fn stream_begin_capture(&mut self, _stream: u64, _mode: i32) -> CuResult<()> { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn thread_exchange_capture_mode(&mut self, mode: i32) -> CuResult { + // No capture machinery in CPU emulation; echo the mode back. + Ok(mode) + } + fn stream_query(&mut self, _stream: u64) -> CuResult { + Ok(0) // everything executes synchronously + } + fn stream_wait_event(&mut self, _stream: u64, _event: u64, _flags: u32) -> CuResult<()> { + Ok(()) // in-order execution: the dependency already holds + } + fn event_query(&mut self, _event: u64) -> CuResult { + Ok(0) + } + fn stream_end_capture(&mut self, _stream: u64) -> CuResult { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn stream_capture_info(&mut self, _stream: u64) -> CuResult<(u64, u64)> { + Ok((0, 0)) // cudaStreamCaptureStatusNone + } + fn graph_instantiate(&mut self, _graph: u64) -> CuResult { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn graph_launch(&mut self, _graph_exec: u64, _stream: u64) -> CuResult<()> { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn graph_exec_destroy(&mut self, _graph_exec: u64) -> CuResult<()> { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn graph_destroy(&mut self, _graph: u64) -> CuResult<()> { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn graph_get_node_count(&mut self, _graph: u64) -> CuResult { + Err(CUDA_ERROR_NOT_SUPPORTED) + } + fn memset_d8_async(&mut self, dptr: u64, value: u8, bytes: u64, _stream: u64) -> CuResult<()> { + self.memset_d8(dptr, value, bytes) + } + fn memcpy_dtod_async(&mut self, dst: u64, src: u64, bytes: u64, _stream: u64) -> CuResult<()> { + self.memcpy_dtod(dst, src, bytes) + } + fn stream_destroy(&mut self, _stream: u64) -> CuResult<()> { + Ok(()) + } + fn stream_synchronize(&mut self, _stream: u64) -> CuResult<()> { + Ok(()) + } + fn event_create(&mut self, _flags: u32) -> CuResult { + Ok(self.handle()) + } + fn event_destroy(&mut self, _event: u64) -> CuResult<()> { + Ok(()) + } + fn event_record(&mut self, _event: u64, _stream: u64) -> CuResult<()> { + Ok(()) + } + fn event_synchronize(&mut self, _event: u64) -> CuResult<()> { + Ok(()) + } + fn event_elapsed_time(&mut self, _start: u64, _end: u64) -> CuResult { + Ok(0.0) + } + fn nvcomp_deflate_temp_size(&mut self, _n: u64, _m: u64, _t: u64) -> CuResult<(i32, u64)> { + Err(CUDA_ERROR_NOT_FOUND) // no nvcomp under CPU emulation + } + fn nvcomp_deflate_decompress( + &mut self, + _a: u64, + _b: u64, + _c: u64, + _d: u64, + _e: u64, + _f: u64, + _g: u64, + _h: u64, + _i: u64, + _j: u64, + ) -> CuResult { + Err(CUDA_ERROR_NOT_FOUND) + } + fn cublas_create(&mut self) -> CuResult { + Err(CUDA_ERROR_NOT_FOUND) + } + fn cublas_destroy(&mut self, _handle: u64) -> CuResult<()> { + Err(CUDA_ERROR_NOT_FOUND) + } + fn cublas_set_stream(&mut self, _handle: u64, _stream: u64) -> CuResult<()> { + Err(CUDA_ERROR_NOT_FOUND) + } + fn cublas_sgemm( + &mut self, + _handle: u64, + _transa: u32, + _transb: u32, + _m: i32, + _n: i32, + _k: i32, + _alpha: f32, + _a: u64, + _lda: i32, + _b: u64, + _ldb: i32, + _beta: f32, + _c: u64, + _ldc: i32, + ) -> CuResult<()> { + Err(CUDA_ERROR_NOT_FOUND) + } +} + +#[cfg(feature = "gpu")] +mod gpu; +#[cfg(feature = "gpu")] +pub use gpu::GpuBackend; + +#[cfg(test)] +mod tests { + use super::*; + use crate::client::Client; + use std::io::{Read, Write}; + + // Drive the CPU backend through the full encode→dispatch→decode path. + #[test] + fn cpu_backend_vecadd_via_dispatch() { + let mut sess = Session::default(); + let mut b = CpuBackend::default(); + // module + function + let (_, m) = dispatch(&mut sess, &mut b, Request::ModuleLoadData { image: vec![] }); + let module = match m { + Response::Handle(h) => h, + _ => panic!(), + }; + let (_, f) = dispatch( + &mut sess, + &mut b, + Request::ModuleGetFunction { + module, + name: "vecadd".into(), + }, + ); + let function = match f { + Response::Handle(h) => h, + _ => panic!(), + }; + // alloc a, b, c (4 floats) + let alloc = |sess: &mut Session, b: &mut CpuBackend| -> u64 { + match dispatch(sess, b, Request::MemAlloc { bytes: 16 }).1 { + Response::Dptr(d) => d, + _ => panic!(), + } + }; + let da = alloc(&mut sess, &mut b); + let db = alloc(&mut sess, &mut b); + let dc = alloc(&mut sess, &mut b); + let a: Vec = [1f32, 2., 3., 4.] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect(); + let bb: Vec = [10f32, 20., 30., 40.] + .iter() + .flat_map(|v| v.to_le_bytes()) + .collect(); + dispatch( + &mut sess, + &mut b, + Request::MemcpyHtoD { + dptr: da, + stream: 0, + data: a, + }, + ); + dispatch( + &mut sess, + &mut b, + Request::MemcpyHtoD { + dptr: db, + stream: 0, + data: bb, + }, + ); + let params = vec![ + da.to_le_bytes().to_vec(), + db.to_le_bytes().to_vec(), + dc.to_le_bytes().to_vec(), + 4u32.to_le_bytes().to_vec(), + ]; + let (st, _) = dispatch( + &mut sess, + &mut b, + Request::LaunchKernel { + function, + grid: [1, 1, 1], + block: [4, 1, 1], + shared_bytes: 0, + stream: 0, + params, + }, + ); + assert_eq!(st, 0); + let out = match dispatch( + &mut sess, + &mut b, + Request::MemcpyDtoH { + dptr: dc, + bytes: 16, + stream: 0, + }, + ) + .1 + { + Response::Data(d) => d, + _ => panic!(), + }; + let c: Vec = out + .chunks(4) + .map(|p| f32::from_le_bytes(p.try_into().unwrap())) + .collect(); + assert_eq!(c, vec![11., 22., 33., 44.]); + } + + #[test] + fn unknown_handle_rejected() { + let mut sess = Session::default(); + let mut b = CpuBackend::default(); + // function id 999 was never minted + let (st, _) = dispatch( + &mut sess, + &mut b, + Request::LaunchKernel { + function: 999, + grid: [1, 1, 1], + block: [1, 1, 1], + shared_bytes: 0, + stream: 0, + params: vec![], + }, + ); + assert_eq!(st, CUDA_ERROR_INVALID_HANDLE); + } + + // Full client↔serve round-trip over an in-process socketpair-like channel. + #[test] + fn client_serve_roundtrip() { + use std::sync::mpsc::{Receiver, Sender}; + // A duplex stream built from two mpsc byte channels. + struct Chan { + tx: Sender, + rx: Receiver, + } + impl Write for Chan { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + for &x in buf { + self.tx + .send(x) + .map_err(|_| std::io::ErrorKind::BrokenPipe)?; + } + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + impl Read for Chan { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + // Block for at least one byte, then drain what's buffered. + if buf.is_empty() { + return Ok(0); + } + let mut n = match self.rx.recv() { + Ok(b) => { + buf[0] = b; + 1 + } + Err(_) => return Ok(0), // EOF + }; + while n < buf.len() { + match self.rx.try_recv() { + Ok(b) => { + buf[n] = b; + n += 1; + } + Err(_) => break, + } + } + Ok(n) + } + } + let (c2s_tx, c2s_rx) = std::sync::mpsc::channel(); + let (s2c_tx, s2c_rx) = std::sync::mpsc::channel(); + let server_side = Chan { + tx: s2c_tx, + rx: c2s_rx, + }; + let client_side = Chan { + tx: c2s_tx, + rx: s2c_rx, + }; + + let server = std::thread::spawn(move || { + let mut b = CpuBackend::default(); + let _ = serve(server_side, &mut b); + }); + + let mut cli = Client::new(client_side); + cli.init().unwrap(); + assert_eq!(cli.device_get_count().unwrap(), 1); + let module = cli.module_load_data(b"").unwrap(); + let func = cli.module_get_function(module, "vecadd").unwrap(); + let n = 8usize; + let da = cli.mem_alloc((n * 4) as u64).unwrap(); + let db = cli.mem_alloc((n * 4) as u64).unwrap(); + let dc = cli.mem_alloc((n * 4) as u64).unwrap(); + let a: Vec = (0..n).flat_map(|i| (i as f32).to_le_bytes()).collect(); + let bb: Vec = (0..n) + .flat_map(|i| ((2 * i) as f32).to_le_bytes()) + .collect(); + cli.memcpy_htod(da, &a, 0).unwrap(); + cli.memcpy_htod(db, &bb, 0).unwrap(); + cli.launch_kernel( + func, + [1, 1, 1], + [n as u32, 1, 1], + 0, + 0, + &[ + da.to_le_bytes().to_vec(), + db.to_le_bytes().to_vec(), + dc.to_le_bytes().to_vec(), + (n as u32).to_le_bytes().to_vec(), + ], + ) + .unwrap(); + cli.ctx_synchronize().unwrap(); + let out = cli.memcpy_dtoh(dc, (n * 4) as u64, 0).unwrap(); + let c: Vec = out + .chunks(4) + .map(|p| f32::from_le_bytes(p.try_into().unwrap())) + .collect(); + let expect: Vec = (0..n).map(|i| (3 * i) as f32).collect(); + assert_eq!(c, expect); + drop(cli); // closes client_side → server sees EOF + server.join().unwrap(); + } + // Ring transport end-to-end: handshake over the socket, then requests + // (inline quiet, indirect oversized, fence) through shared memory with + // bounce-buffer responses — no GPU, no VM. + #[test] + fn ring_transport_end_to_end() { + use std::sync::atomic::AtomicUsize; + use std::sync::mpsc::{Receiver, Sender}; + struct Chan { + tx: Sender, + rx: Receiver, + } + impl Write for Chan { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + for &x in buf { + self.tx + .send(x) + .map_err(|_| std::io::ErrorKind::BrokenPipe)?; + } + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + impl Read for Chan { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if buf.is_empty() { + return Ok(0); + } + match self.rx.recv() { + Ok(b) => { + buf[0] = b; + Ok(1) + } + Err(_) => Ok(0), // EOF + } + } + } + + // In-process fake guest RAM: identity-mapped (GPA == host VA), so + // any heap address — ring pages and indirect staging buffers alike — + // resolves. The aligned ring pages are leaked so both threads may + // hold pointers. + const PAGES: usize = 64; + const PAGE: usize = 4096; + let _ = AtomicUsize::new(0); // (kept import happy on older toolchains) + let layout = std::alloc::Layout::from_size_align(PAGES * PAGE, PAGE).unwrap(); + // SAFETY: fresh allocation, zeroed, intentionally leaked. + let hva = unsafe { + let p = std::alloc::alloc_zeroed(layout); + assert!(!p.is_null()); + p as usize + }; + + let (c2s_tx, c2s_rx) = std::sync::mpsc::channel(); + let (s2c_tx, s2c_rx) = std::sync::mpsc::channel(); + let server_side = Chan { + tx: s2c_tx, + rx: c2s_rx, + }; + let client_side = Chan { + tx: c2s_tx, + rx: s2c_rx, + }; + let server = std::thread::spawn(move || { + let mut b = CpuBackend::default(); + b.set_guest_ram(vec![(0, 0, u64::MAX / 2)]); // identity: hva == gpa + let r = serve(server_side, &mut b); + eprintln!("[test] serve exited: {r:?}"); + }); + + let mut cli = Client::new(client_side); + cli.init().unwrap(); + let vas = |r: std::ops::Range| -> Vec<*mut u8> { + r.map(|i| (hva + i * PAGE) as *mut u8).collect() + }; + let gpas = |r: std::ops::Range| -> Vec { + r.map(|i| (hva + i * PAGE) as u64).collect() + }; + cli.ring_setup( + PAGE, + (vas(0..8), gpas(0..8)), + (vas(8..16), gpas(8..16)), + (vas(16..24), gpas(16..24)), + ) + .unwrap(); + assert!(cli.is_ring()); + + // Sync op over the ring (inline both ways). + assert_eq!(cli.device_get_count().unwrap(), 1); + let d = cli.mem_alloc(8192).unwrap(); + // Deferred quiet op, inline record. + cli.memcpy_htod(d, &[9u8; 64], 0).unwrap(); + // Fence over the ring settles it. + cli.drain().unwrap(); + assert_eq!(cli.take_sticky(), 0); + // Oversized write: multi-chunk through the 32 KiB bounce staging. + let big: Vec = (0..100_000u32).map(|i| (i % 251) as u8).collect(); + let d2 = cli.mem_alloc(big.len() as u64).unwrap(); + cli.memcpy_htod(d2, &big, 0).unwrap(); + // Oversized read: response spills through the same bounce pages. + let back = cli.memcpy_dtoh(d2, big.len() as u64, 0).unwrap(); + assert_eq!(back, big); + drop(cli); + server.join().unwrap(); + } + // A connection may not allocate past SMOLVM_CUDA_VRAM_LIMIT_MB; freeing + // returns budget. (Env is process-global; restored at the end so + // parallel tests never see the 1 MB cap on their own allocations.) + #[test] + fn vram_quota_enforced() { + std::env::set_var("SMOLVM_CUDA_VRAM_LIMIT_MB", "1"); + let mut sess = Session::default(); + let mut b = CpuBackend::default(); + let mb = 1024 * 1024; + let (st, r) = dispatch(&mut sess, &mut b, Request::MemAlloc { bytes: mb / 2 }); + assert_eq!(st, 0); + let d1 = match r { + Response::Dptr(d) => d, + _ => unreachable!(), + }; + // Second half-MB fits exactly; a byte more must fail with OOM(2). + let (st, _) = dispatch(&mut sess, &mut b, Request::MemAlloc { bytes: mb / 2 }); + assert_eq!(st, 0); + let (st, _) = dispatch(&mut sess, &mut b, Request::MemAlloc { bytes: 1 }); + assert_eq!(st, 2, "over-quota alloc must report OUT_OF_MEMORY"); + // Freeing restores budget. + let (st, _) = dispatch(&mut sess, &mut b, Request::MemFree { dptr: d1 }); + assert_eq!(st, 0); + let (st, _) = dispatch(&mut sess, &mut b, Request::MemAlloc { bytes: mb / 4 }); + assert_eq!(st, 0); + std::env::remove_var("SMOLVM_CUDA_VRAM_LIMIT_MB"); + } + // The connect handshake rejects a client whose wire fingerprint differs + // (stale shim/server), so protocol skew fails loudly instead of decoding + // the wrong bytes and corrupting silently. + #[test] + fn init_rejects_proto_mismatch() { + let mut sess = Session::default(); + let mut b = CpuBackend::default(); + let (st, _) = dispatch( + &mut sess, + &mut b, + Request::Init { + proto_hash: crate::PROTO_HASH, + }, + ); + assert_eq!(st, 0, "matching proto hash must connect"); + let (st, _) = dispatch( + &mut sess, + &mut b, + Request::Init { + proto_hash: crate::PROTO_HASH ^ 0x1, + }, + ); + assert_eq!( + st, CUDA_ERROR_NOT_SUPPORTED, + "mismatched proto hash must be rejected" + ); + } +} diff --git a/crates/smolvm-cuda/src/host/gpu.rs b/crates/smolvm-cuda/src/host/gpu.rs new file mode 100644 index 0000000..f644ca2 --- /dev/null +++ b/crates/smolvm-cuda/src/host/gpu.rs @@ -0,0 +1,2314 @@ +//! Real CUDA Driver-API backend via `dlopen` of the driver library +//! (`nvcuda.dll` on Windows, `libcuda.so.1` on Linux). No CUDA toolkit needed — +//! the `cu*` signatures are declared by hand. The same calls were proven on an +//! RTX 3070; this wraps them behind the [`Backend`] trait. +//! +//! Context affinity: the CUDA context becomes current on the thread that calls +//! `cuCtxCreate`. The host runs one [`serve`](super::serve) loop per connection +//! on a single thread, and `ctx_create` is invoked on that thread, so all later +//! calls on the connection see the right current context. + +use super::{Backend, CuResult}; +use libloading::{Library, Symbol}; +use std::ffi::{c_void, CStr, CString}; +use std::os::raw::{c_char, c_int, c_uint}; + +#[cfg(windows)] +const CUDA_LIB: &str = "nvcuda.dll"; +#[cfg(target_os = "linux")] +const CUDA_LIB: &str = "libcuda.so.1"; +#[cfg(target_os = "macos")] +const CUDA_LIB: &str = "libcuda.dylib"; // not expected to exist; macOS has no CUDA + +type CuResultCode = c_int; + +/// Hand-declared driver entry points. Stored as `'static` fn pointers; `_lib` +/// keeps the library mapped for as long as the backend lives. +pub struct GpuBackend { + _lib: Library, + init: unsafe extern "C" fn(c_uint) -> CuResultCode, + device_get_count: unsafe extern "C" fn(*mut c_int) -> CuResultCode, + device_get_name: unsafe extern "C" fn(*mut c_char, c_int, c_int) -> CuResultCode, + device_total_mem: unsafe extern "C" fn(*mut usize, c_int) -> CuResultCode, + driver_get_version: unsafe extern "C" fn(*mut c_int) -> CuResultCode, + device_get_attribute: unsafe extern "C" fn(*mut c_int, c_int, c_int) -> CuResultCode, + device_get_uuid: unsafe extern "C" fn(*mut u8, c_int) -> CuResultCode, + ctx_create: unsafe extern "C" fn(*mut *mut c_void, c_uint, c_int) -> CuResultCode, + ctx_destroy: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + ctx_set_current: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + primary_ctx_retain: unsafe extern "C" fn(*mut *mut c_void, c_int) -> CuResultCode, + primary_ctx_release: unsafe extern "C" fn(c_int) -> CuResultCode, + module_load_data: unsafe extern "C" fn(*mut *mut c_void, *const c_void) -> CuResultCode, + module_get_function: + unsafe extern "C" fn(*mut *mut c_void, *mut c_void, *const c_char) -> CuResultCode, + module_unload: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + /// `cuFuncGetParamInfo` — CUDA 12.4+. `None` on older drivers, where + /// [`Backend::func_get_param_info`] reports `CUDA_ERROR_NOT_SUPPORTED`. + func_get_param_info: + Option CuResultCode>, + func_set_attribute: unsafe extern "C" fn(*mut c_void, c_int, c_int) -> CuResultCode, + func_get_attribute: unsafe extern "C" fn(*mut c_int, c_int, *mut c_void) -> CuResultCode, + mem_alloc: unsafe extern "C" fn(*mut u64, usize) -> CuResultCode, + mem_free: unsafe extern "C" fn(u64) -> CuResultCode, + memcpy_htod: unsafe extern "C" fn(u64, *const c_void, usize) -> CuResultCode, + memcpy_dtoh: unsafe extern "C" fn(*mut c_void, u64, usize) -> CuResultCode, + memcpy_dtod: unsafe extern "C" fn(u64, u64, usize) -> CuResultCode, + memset_d8: unsafe extern "C" fn(u64, u8, usize) -> CuResultCode, + mem_get_info: unsafe extern "C" fn(*mut usize, *mut usize) -> CuResultCode, + /// `cuMemHostRegister_v2` / `cuMemHostUnregister` — pin a host range into the + /// context so DMAs from it run at full (pinned) bandwidth instead of the + /// ~3 GB/s pageable path. `None` on very old drivers. Used to pin guest RAM + /// for zero-copy. Registration needs a current context, so it happens lazily + /// on the first zero-copy transfer, not at `set_guest_ram`. + mem_host_register: Option CuResultCode>, + mem_host_unregister: Option CuResultCode>, + /// `cuMemAllocHost_v2` / `cuMemFreeHost` — allocate pinned host memory, used + /// for the gather/scatter staging buffer on heavily-fragmented zero-copy + /// transfers. `None` on very old drivers (then the fragmented path DMAs each + /// segment directly). + mem_host_alloc: Option CuResultCode>, + mem_free_host: Option CuResultCode>, + #[allow(clippy::type_complexity)] + launch_kernel: unsafe extern "C" fn( + *mut c_void, + c_uint, + c_uint, + c_uint, + c_uint, + c_uint, + c_uint, + c_uint, + *mut c_void, + *mut *mut c_void, + *mut *mut c_void, + ) -> CuResultCode, + ctx_synchronize: unsafe extern "C" fn() -> CuResultCode, + stream_create: unsafe extern "C" fn(*mut *mut c_void, c_uint) -> CuResultCode, + // CUDA graphs (capture on the real driver; replay = one launch). + stream_begin_capture: unsafe extern "C" fn(*mut c_void, c_int) -> CuResultCode, + thread_exchange_capture_mode: unsafe extern "C" fn(*mut c_int) -> CuResultCode, + stream_end_capture: unsafe extern "C" fn(*mut c_void, *mut *mut c_void) -> CuResultCode, + stream_get_capture_info: + unsafe extern "C" fn(*mut c_void, *mut c_int, *mut u64) -> CuResultCode, + graph_instantiate_with_flags: + unsafe extern "C" fn(*mut *mut c_void, *mut c_void, u64) -> CuResultCode, + graph_launch: unsafe extern "C" fn(*mut c_void, *mut c_void) -> CuResultCode, + graph_exec_destroy: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + graph_destroy: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + graph_get_nodes: + unsafe extern "C" fn(*mut c_void, *mut *mut c_void, *mut usize) -> CuResultCode, + memset_d8_async: unsafe extern "C" fn(u64, u8, usize, *mut c_void) -> CuResultCode, + memcpy_dtod_async: unsafe extern "C" fn(u64, u64, usize, *mut c_void) -> CuResultCode, + stream_destroy: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + stream_synchronize: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + stream_query: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + stream_wait_event: unsafe extern "C" fn(*mut c_void, *mut c_void, c_uint) -> CuResultCode, + // VMM (all optional: pre-Pascal drivers lack them; ops report NOT_SUPPORTED) + vmm_address_reserve: + Option CuResultCode>, + vmm_create: Option CuResultCode>, + vmm_map: Option CuResultCode>, + vmm_set_access: + Option CuResultCode>, + vmm_unmap: Option CuResultCode>, + vmm_release: Option CuResultCode>, + vmm_address_free: Option CuResultCode>, + vmm_granularity: + Option CuResultCode>, + event_query: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + event_create: unsafe extern "C" fn(*mut *mut c_void, c_uint) -> CuResultCode, + event_destroy: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + event_record: unsafe extern "C" fn(*mut c_void, *mut c_void) -> CuResultCode, + event_synchronize: unsafe extern "C" fn(*mut c_void) -> CuResultCode, + event_elapsed_time: unsafe extern "C" fn(*mut f32, *mut c_void, *mut c_void) -> CuResultCode, + /// Real nvcomp, dlopened on demand from `SMOLVM_NVCOMP_LIB`. Kept in an + /// `Option` because most workloads never touch it. Runs on the same primary + /// context as everything else, so forwarded device pointers are valid. + nvcomp: Option, + /// Real cuBLAS, dlopened on demand from `SMOLVM_CUBLAS_LIB` (default + /// `libcublas.so`). Runs on the same primary context. + cublas: Option, + /// Code-generated dispatch tables (library id → handlers), loaded on first use. + cublas_gen: Option, + cudnn_gen: Option, + /// cuDNN v8 backend (graph) API forwarder — PyTorch's convolution path. + cudnn_backend: Option, + /// cuBLASLt matmul forwarder — PyTorch's `cublasLtMatmul` linear-layer path. + cublaslt: Option, + /// Legacy cuDNN batch-norm (Ex) forwarder — PyTorch's `BatchNorm2d` path. + cudnn_bn: Option, + /// Guest-assigned virtual handle (bit 63 set) → real host descriptor + /// pointer. Lets the guest fire-and-forget descriptor creation: it invents + /// the id, the host materializes and maps it, and every later reference is + /// translated here. Real pointers (bit 63 clear) pass through untouched. + vhandles: std::collections::HashMap, + /// Host mappings of guest RAM, each `(gpa_start, host_va, len)`, for + /// zero-copy `memcpy_gpa_*` (via `krun_get_guest_ram`). Empty outside a + /// microVM / on older libkrun. Guest RAM is usually split into a low and a + /// high region around the 4 GiB PCI hole. + guest_ram: Vec<(u64, u64, u64)>, + /// Guest-RAM host ranges `(host_va, len)` this backend pinned via + /// `cuMemHostRegister` (to unregister on drop). Excludes ranges another + /// connection already owns. Empty until the first zero-copy transfer. + registered: Vec<(u64, u64)>, + /// Whether lazy guest-RAM pinning has been attempted (once per backend). + guest_ram_pin_tried: bool, + /// Reusable pinned host staging buffer `(addr, capacity)` for the gather / + /// scatter path on fragmented zero-copy transfers. Held as an address (not a + /// pointer) so the backend stays `Send`; 0 until first needed, grown on + /// demand, freed on drop. + staging: (usize, usize), +} + +/// Above this segment count, a fragmented zero-copy transfer gathers into one +/// pinned staging buffer and issues a single DMA instead of one DMA per segment +/// — past here the per-DMA launch overhead of thousands of tiny copies costs +/// more than a single host-side gather plus one big transfer. +const ZC_DIRECT_MAX_SEGMENTS: usize = 16; + +/// Code-generated forward-to-host-lib dispatch (see `smolvm-cuda-codegen`). +/// Each `include!`d module exposes a `GenLib` with `load()` + `dispatch()`. +mod gen_cublas { + #![allow(non_snake_case, clippy::unnecessary_cast, unused_mut, dead_code)] + use super::*; + include!("../generated/cublas_host.rs"); +} +mod gen_cudnn { + #![allow(non_snake_case, clippy::unnecessary_cast, unused_mut, dead_code)] + use super::*; + include!("../generated/cudnn_host.rs"); +} + +/// Library ids on the `LibCall` wire (must match the generated `LIB_ID`). +const LIB_CUBLAS: u8 = 1; +const LIB_CUDNN: u8 = 2; +/// cuDNN v8 backend (graph) API — hand-marshaled, not codegen (opaque +/// descriptors + typed attribute arrays). +const LIB_CUDNN_BACKEND: u8 = 3; +const LIB_CUBLASLT: u8 = 4; +const LIB_CUDNN_BN: u8 = 5; + +/// Hand-declared cuBLAS entry points (subset). `cublasStatus_t` is an enum (i32). +struct Cublas { + _lib: Library, + create: unsafe extern "C" fn(*mut *mut c_void) -> c_int, + destroy: unsafe extern "C" fn(*mut c_void) -> c_int, + set_stream: unsafe extern "C" fn(*mut c_void, *mut c_void) -> c_int, + #[allow(clippy::type_complexity)] + sgemm: unsafe extern "C" fn( + *mut c_void, // handle + c_int, // transa + c_int, // transb + c_int, + c_int, + c_int, // m,n,k + *const f32, + *const f32, + c_int, // alpha, A, lda + *const f32, + c_int, // B, ldb + *const f32, + *mut f32, + c_int, // beta, C, ldc + ) -> c_int, +} + +/// Open a host CUDA library with auto-discovery: an env override wins, then +/// the plain soname (dlopen's ldconfig search), then the usual CUDA install +/// prefixes. Removes the need to export `SMOLVM_*_LIB` on typical hosts, where +/// the toolkit lives under `/opt/cuda` or `/usr/local/cuda` without ldconfig +/// entries. +pub(super) fn open_host_lib(env_var: &str, sonames: &[&str]) -> Result { + if let Ok(path) = std::env::var(env_var) { + // SAFETY: loading a user-designated shared library. + return unsafe { Library::new(&path).map_err(|e| format!("load {path}: {e}")) }; + } + const PREFIXES: &[&str] = &[ + "", // plain soname: dlopen's own ldconfig search + "/opt/cuda/lib64/", + "/usr/local/cuda/lib64/", + "/usr/lib/x86_64-linux-gnu/", + "/usr/lib64/", + "/usr/lib/", + ]; + let mut last_err = String::new(); + for soname in sonames { + for prefix in PREFIXES { + let cand = format!("{prefix}{soname}"); + // SAFETY: probing well-known system library locations. + match unsafe { Library::new(&cand) } { + Ok(lib) => return Ok(lib), + Err(e) => last_err = format!("load {cand}: {e}"), + } + } + } + Err(format!( + "{last_err} (set {env_var} to the library path if it lives elsewhere)" + )) +} + +impl Cublas { + fn load() -> Result { + unsafe { + let lib = open_host_lib( + "SMOLVM_CUBLAS_LIB", + &["libcublas.so", "libcublas.so.13", "libcublas.so.12"], + )?; + let c = Cublas { + create: sym(&lib, b"cublasCreate_v2\0")?, + destroy: sym(&lib, b"cublasDestroy_v2\0")?, + set_stream: sym(&lib, b"cublasSetStream_v2\0")?, + sgemm: sym(&lib, b"cublasSgemm_v2\0")?, + _lib: lib, + }; + Ok(c) + } + } +} + +/// Hand-declared nvcomp batched Deflate entry points (uniform batched ABI). +struct Nvcomp { + _lib: Library, + temp_size: unsafe extern "C" fn(usize, usize, *mut usize, usize) -> c_int, + #[allow(clippy::type_complexity)] + decompress: unsafe extern "C" fn( + *const *const c_void, // device_compressed_ptrs + *const usize, // device_compressed_bytes + *const usize, // device_uncompressed_bytes + *mut usize, // device_actual_uncompressed_bytes (nullable) + usize, // batch_size + *mut c_void, // device_temp + usize, // temp_bytes + *const *mut c_void, // device_uncompressed_ptrs + *mut c_int, // device_statuses (nullable) + *mut c_void, // stream (CUstream) + ) -> c_int, +} + +impl Nvcomp { + fn load() -> Result { + let path = std::env::var("SMOLVM_NVCOMP_LIB") + .map_err(|_| "SMOLVM_NVCOMP_LIB not set".to_string())?; + unsafe { + let lib = Library::new(&path).map_err(|e| format!("load {path}: {e}"))?; + let n = Nvcomp { + temp_size: sym(&lib, b"nvcompBatchedDeflateDecompressGetTempSizeEx\0")?, + decompress: sym(&lib, b"nvcompBatchedDeflateDecompressAsync\0")?, + _lib: lib, + }; + Ok(n) + } + } +} + +unsafe fn sym(lib: &Library, name: &[u8]) -> Result { + let s: Symbol = lib + .get(name) + .map_err(|e| format!("symbol {}: {e}", String::from_utf8_lossy(name)))?; + // Transmute the borrowed Symbol into a bare fn pointer; `_lib` in the + // struct keeps the library loaded for the pointer's whole lifetime. + Ok(std::ptr::read(&s as *const Symbol as *const T)) +} + +/// Resolve `primary`, falling back to `fallback` — for entry points whose +/// canonical name gained a `_v2` suffix in a later CUDA release. +unsafe fn sym2(lib: &Library, primary: &[u8], fallback: &[u8]) -> Result { + sym(lib, primary).or_else(|_| sym(lib, fallback)) +} + +impl GpuBackend { + /// Load the driver and resolve every entry point. Returns the library name + /// + error string on failure so the caller can fall back (e.g. to CPU). + pub fn load() -> Result { + unsafe { + let lib = Library::new(CUDA_LIB).map_err(|e| format!("load {CUDA_LIB}: {e}"))?; + let b = GpuBackend { + init: sym(&lib, b"cuInit\0")?, + device_get_count: sym(&lib, b"cuDeviceGetCount\0")?, + device_get_name: sym(&lib, b"cuDeviceGetName\0")?, + device_total_mem: sym(&lib, b"cuDeviceTotalMem_v2\0")?, + driver_get_version: sym(&lib, b"cuDriverGetVersion\0")?, + device_get_attribute: sym(&lib, b"cuDeviceGetAttribute\0")?, + device_get_uuid: sym2(&lib, b"cuDeviceGetUuid_v2\0", b"cuDeviceGetUuid\0")?, + ctx_create: sym(&lib, b"cuCtxCreate_v2\0")?, + ctx_destroy: sym(&lib, b"cuCtxDestroy_v2\0")?, + ctx_set_current: sym(&lib, b"cuCtxSetCurrent\0")?, + primary_ctx_retain: sym(&lib, b"cuDevicePrimaryCtxRetain\0")?, + primary_ctx_release: sym2( + &lib, + b"cuDevicePrimaryCtxRelease_v2\0", + b"cuDevicePrimaryCtxRelease\0", + )?, + module_load_data: sym(&lib, b"cuModuleLoadData\0")?, + module_get_function: sym(&lib, b"cuModuleGetFunction\0")?, + module_unload: sym(&lib, b"cuModuleUnload\0")?, + func_get_param_info: sym(&lib, b"cuFuncGetParamInfo\0").ok(), + func_set_attribute: sym(&lib, b"cuFuncSetAttribute\0")?, + func_get_attribute: sym(&lib, b"cuFuncGetAttribute\0")?, + mem_alloc: sym(&lib, b"cuMemAlloc_v2\0")?, + mem_free: sym(&lib, b"cuMemFree_v2\0")?, + memcpy_htod: sym(&lib, b"cuMemcpyHtoD_v2\0")?, + memcpy_dtoh: sym(&lib, b"cuMemcpyDtoH_v2\0")?, + memcpy_dtod: sym(&lib, b"cuMemcpyDtoD_v2\0")?, + memset_d8: sym(&lib, b"cuMemsetD8_v2\0")?, + mem_get_info: sym(&lib, b"cuMemGetInfo_v2\0")?, + mem_host_register: sym(&lib, b"cuMemHostRegister_v2\0").ok(), + mem_host_unregister: sym(&lib, b"cuMemHostUnregister\0").ok(), + mem_host_alloc: sym(&lib, b"cuMemAllocHost_v2\0").ok(), + mem_free_host: sym(&lib, b"cuMemFreeHost\0").ok(), + launch_kernel: sym(&lib, b"cuLaunchKernel\0")?, + ctx_synchronize: sym(&lib, b"cuCtxSynchronize\0")?, + stream_create: sym(&lib, b"cuStreamCreate\0")?, + stream_begin_capture: sym2( + &lib, + b"cuStreamBeginCapture_v2\0", + b"cuStreamBeginCapture\0", + )?, + thread_exchange_capture_mode: sym(&lib, b"cuThreadExchangeStreamCaptureMode\0")?, + stream_end_capture: sym(&lib, b"cuStreamEndCapture\0")?, + stream_get_capture_info: sym2( + &lib, + b"cuStreamGetCaptureInfo\0", + b"cuStreamGetCaptureInfo_v2\0", + )?, + graph_instantiate_with_flags: sym(&lib, b"cuGraphInstantiateWithFlags\0")?, + graph_launch: sym(&lib, b"cuGraphLaunch\0")?, + graph_exec_destroy: sym(&lib, b"cuGraphExecDestroy\0")?, + graph_destroy: sym(&lib, b"cuGraphDestroy\0")?, + graph_get_nodes: sym(&lib, b"cuGraphGetNodes\0")?, + memset_d8_async: sym(&lib, b"cuMemsetD8Async\0")?, + memcpy_dtod_async: sym2(&lib, b"cuMemcpyDtoDAsync_v2\0", b"cuMemcpyDtoDAsync\0")?, + stream_destroy: sym2(&lib, b"cuStreamDestroy_v2\0", b"cuStreamDestroy\0")?, + stream_synchronize: sym(&lib, b"cuStreamSynchronize\0")?, + stream_query: sym(&lib, b"cuStreamQuery\0")?, + stream_wait_event: sym(&lib, b"cuStreamWaitEvent\0")?, + vmm_address_reserve: sym(&lib, b"cuMemAddressReserve\0").ok(), + vmm_create: sym(&lib, b"cuMemCreate\0").ok(), + vmm_map: sym(&lib, b"cuMemMap\0").ok(), + vmm_set_access: sym(&lib, b"cuMemSetAccess\0").ok(), + vmm_unmap: sym(&lib, b"cuMemUnmap\0").ok(), + vmm_release: sym(&lib, b"cuMemRelease\0").ok(), + vmm_address_free: sym(&lib, b"cuMemAddressFree\0").ok(), + vmm_granularity: sym(&lib, b"cuMemGetAllocationGranularity\0").ok(), + event_query: sym(&lib, b"cuEventQuery\0")?, + event_create: sym(&lib, b"cuEventCreate\0")?, + event_destroy: sym2(&lib, b"cuEventDestroy_v2\0", b"cuEventDestroy\0")?, + event_record: sym(&lib, b"cuEventRecord\0")?, + event_synchronize: sym(&lib, b"cuEventSynchronize\0")?, + event_elapsed_time: sym(&lib, b"cuEventElapsedTime\0")?, + nvcomp: None, + cublas: None, + cublas_gen: None, + cudnn_gen: None, + cudnn_backend: None, + vhandles: std::collections::HashMap::new(), + cublaslt: None, + cudnn_bn: None, + guest_ram: Vec::new(), + registered: Vec::new(), + guest_ram_pin_tried: false, + staging: (0, 0), + _lib: lib, + }; + Ok(b) + } + } +} + +/// Map a raw `CUresult`: 0 → `Ok`, else the code as `Err`. +fn chk(code: CuResultCode) -> CuResult<()> { + if code == 0 { + Ok(()) + } else { + Err(code) + } +} + +impl Backend for GpuBackend { + fn init(&mut self) -> CuResult<()> { + unsafe { chk((self.init)(0)) } + } + fn device_get_count(&mut self) -> CuResult { + let mut n = 0; + unsafe { chk((self.device_get_count)(&mut n))? }; + Ok(n) + } + fn device_get_name(&mut self, device: i32) -> CuResult { + let mut buf = [0i8; 256]; + unsafe { + chk((self.device_get_name)( + buf.as_mut_ptr() as *mut c_char, + 256, + device, + ))? + }; + let name = unsafe { CStr::from_ptr(buf.as_ptr() as *const c_char) } + .to_string_lossy() + .into_owned(); + Ok(name) + } + fn device_total_mem(&mut self, device: i32) -> CuResult { + let mut bytes: usize = 0; + unsafe { chk((self.device_total_mem)(&mut bytes, device))? }; + Ok(bytes as u64) + } + fn driver_get_version(&mut self) -> CuResult { + let mut v = 0; + unsafe { chk((self.driver_get_version)(&mut v))? }; + Ok(v) + } + fn device_get_attribute(&mut self, attrib: i32, device: i32) -> CuResult { + let mut v = 0; + unsafe { chk((self.device_get_attribute)(&mut v, attrib, device))? }; + Ok(v) + } + fn device_get_uuid(&mut self, device: i32) -> CuResult<[u8; 16]> { + let mut uuid = [0u8; 16]; + unsafe { chk((self.device_get_uuid)(uuid.as_mut_ptr(), device))? }; + Ok(uuid) + } + fn ctx_create(&mut self, device: i32) -> CuResult { + let mut ctx: *mut c_void = std::ptr::null_mut(); + unsafe { chk((self.ctx_create)(&mut ctx, 0, device))? }; + Ok(ctx as u64) + } + fn ctx_destroy(&mut self, ctx: u64) -> CuResult<()> { + unsafe { chk((self.ctx_destroy)(ctx as *mut c_void)) } + } + fn primary_ctx_retain(&mut self, device: i32) -> CuResult { + let mut ctx: *mut c_void = std::ptr::null_mut(); + unsafe { chk((self.primary_ctx_retain)(&mut ctx, device))? }; + // Unlike cuCtxCreate, retain does not bind the context to the calling + // thread — bind it here so every later call on this connection's + // serving thread (module load, alloc, launch) has a current context. + unsafe { chk((self.ctx_set_current)(ctx))? }; + Ok(ctx as u64) + } + fn primary_ctx_release(&mut self, device: i32) -> CuResult<()> { + unsafe { chk((self.primary_ctx_release)(device)) } + } + fn module_load_data(&mut self, image: &[u8]) -> CuResult { + // cuModuleLoadData reads a NUL-terminated PTX string or a cubin blob. + // Ensure a trailing NUL so PTX text is well-formed for the JIT. + let mut buf = image.to_vec(); + if !buf.ends_with(&[0]) { + buf.push(0); + } + let mut module: *mut c_void = std::ptr::null_mut(); + let code = unsafe { (self.module_load_data)(&mut module, buf.as_ptr() as *const c_void) }; + if code != 0 { + // Debug: keep failing images for cuobjdump post-mortem. + if let Some(dir) = std::env::var_os("SMOLVM_CUDA_DUMP_BAD_MODULES") { + let n = buf.len(); + let path = std::path::Path::new(&dir).join(format!("badmod-{code}-{n}.bin")); + let _ = std::fs::write(path, &buf); + } + return Err(code); + } + Ok(module as u64) + } + fn module_get_function(&mut self, module: u64, name: &str) -> CuResult { + let cname = CString::new(name).map_err(|_| super::CUDA_ERROR_NOT_FOUND)?; + let mut func: *mut c_void = std::ptr::null_mut(); + unsafe { + chk((self.module_get_function)( + &mut func, + module as *mut c_void, + cname.as_ptr(), + ))? + }; + Ok(func as u64) + } + fn module_unload(&mut self, module: u64) -> CuResult<()> { + unsafe { chk((self.module_unload)(module as *mut c_void)) } + } + fn func_get_param_info(&mut self, function: u64) -> CuResult> { + // CUDA 12.4+. Walk parameter indices until INVALID_VALUE marks the end. + const CUDA_ERROR_INVALID_VALUE: i32 = 1; + const CUDA_ERROR_NOT_SUPPORTED: i32 = 801; + let f = self.func_get_param_info.ok_or(CUDA_ERROR_NOT_SUPPORTED)?; + let mut sizes = Vec::new(); + for i in 0.. { + let (mut offset, mut size): (usize, usize) = (0, 0); + let code = unsafe { f(function as *mut c_void, i, &mut offset, &mut size) }; + match code { + 0 => sizes.push(size as u32), + CUDA_ERROR_INVALID_VALUE => break, + other => return Err(other), + } + } + Ok(sizes) + } + fn func_set_attribute(&mut self, function: u64, attrib: i32, value: i32) -> CuResult<()> { + unsafe { + chk((self.func_set_attribute)( + function as *mut c_void, + attrib, + value, + )) + } + } + fn func_get_attribute(&mut self, function: u64, attrib: i32) -> CuResult { + let mut v = 0; + unsafe { + chk((self.func_get_attribute)( + &mut v, + attrib, + function as *mut c_void, + ))? + }; + Ok(v) + } + fn mem_alloc(&mut self, bytes: u64) -> CuResult { + let mut dptr: u64 = 0; + unsafe { chk((self.mem_alloc)(&mut dptr, bytes as usize))? }; + Ok(dptr) + } + fn mem_free(&mut self, dptr: u64) -> CuResult<()> { + unsafe { chk((self.mem_free)(dptr)) } + } + fn memcpy_htod(&mut self, dptr: u64, data: &[u8], stream: u64) -> CuResult<()> { + self.wait_stream(stream)?; + unsafe { + chk((self.memcpy_htod)( + dptr, + data.as_ptr() as *const c_void, + data.len(), + )) + } + } + fn memcpy_dtoh(&mut self, dptr: u64, bytes: u64, stream: u64) -> CuResult> { + self.wait_stream(stream)?; + let mut out = vec![0u8; bytes as usize]; + unsafe { + chk((self.memcpy_dtoh)( + out.as_mut_ptr() as *mut c_void, + dptr, + bytes as usize, + ))? + }; + Ok(out) + } + fn memcpy_dtod(&mut self, dst: u64, src: u64, bytes: u64) -> CuResult<()> { + unsafe { chk((self.memcpy_dtod)(dst, src, bytes as usize)) } + } + fn memset_d8(&mut self, dptr: u64, value: u8, bytes: u64) -> CuResult<()> { + unsafe { chk((self.memset_d8)(dptr, value, bytes as usize)) } + } + fn mem_get_info(&mut self) -> CuResult<(u64, u64)> { + let (mut free, mut total): (usize, usize) = (0, 0); + unsafe { chk((self.mem_get_info)(&mut free, &mut total))? }; + Ok((free as u64, total as u64)) + } + + // The shared-memory data channel is Linux-only (`crate::shm`); on other + // platforms these fall back to the trait's not-supported default. + #[cfg(target_os = "linux")] + fn memcpy_shm_htod(&mut self, dptr: u64, offset: u64, size: u64, stream: u64) -> CuResult<()> { + // Read straight from the shared region (no bytes over the socket) and + // DMA to the GPU — one copy instead of three. + self.wait_stream(stream)?; + let region = crate::shm::get_or_create().ok_or(super::CUDA_ERROR_NOT_FOUND)?; + let src = region + .checked(offset, size) + .ok_or(super::CUDA_ERROR_INVALID_HANDLE)?; + unsafe { + chk((self.memcpy_htod)( + dptr, + src as *const c_void, + size as usize, + )) + } + } + #[cfg(target_os = "linux")] + fn memcpy_shm_dtoh(&mut self, offset: u64, dptr: u64, size: u64, stream: u64) -> CuResult<()> { + self.wait_stream(stream)?; + let region = crate::shm::get_or_create().ok_or(super::CUDA_ERROR_NOT_FOUND)?; + let dst = region + .checked(offset, size) + .ok_or(super::CUDA_ERROR_INVALID_HANDLE)?; + unsafe { chk((self.memcpy_dtoh)(dst as *mut c_void, dptr, size as usize)) } + } + + fn set_guest_ram(&mut self, regions: Vec<(u64, u64, u64)>) { + self.guest_ram = regions; + } + + fn gpa_to_hva(&mut self, gpa: u64, len: u64) -> Option { + self.guest_ram.iter().find_map(|&(gs, hva, rlen)| { + (gpa >= gs && gpa.checked_add(len)? <= gs + rlen).then(|| hva + (gpa - gs)) + }) + } + + fn mem_address_reserve(&mut self, size: u64, align: u64) -> CuResult { + let f = self + .vmm_address_reserve + .ok_or(super::CUDA_ERROR_NOT_SUPPORTED)?; + let mut va = 0u64; + unsafe { chk(f(&mut va, size as usize, align as usize, 0, 0))? }; + Ok(va) + } + fn mem_create(&mut self, size: u64, device: i32) -> CuResult { + let f = self.vmm_create.ok_or(super::CUDA_ERROR_NOT_SUPPORTED)?; + let prop = vmm_prop(device); + let mut h = 0u64; + unsafe { chk(f(&mut h, size as usize, &prop, 0))? }; + Ok(h) + } + fn mem_map(&mut self, va: u64, size: u64, offset: u64, handle: u64) -> CuResult<()> { + let f = self.vmm_map.ok_or(super::CUDA_ERROR_NOT_SUPPORTED)?; + unsafe { chk(f(va, size as usize, offset as usize, handle, 0)) } + } + fn mem_set_access(&mut self, va: u64, size: u64, device: i32) -> CuResult<()> { + let f = self.vmm_set_access.ok_or(super::CUDA_ERROR_NOT_SUPPORTED)?; + let desc = VmmAccessDesc { + location_type: 1, + location_id: device, + flags: 3, + }; + unsafe { chk(f(va, size as usize, &desc, 1)) } + } + fn mem_unmap(&mut self, va: u64, size: u64) -> CuResult<()> { + let f = self.vmm_unmap.ok_or(super::CUDA_ERROR_NOT_SUPPORTED)?; + unsafe { chk(f(va, size as usize)) } + } + fn mem_release(&mut self, handle: u64) -> CuResult<()> { + let f = self.vmm_release.ok_or(super::CUDA_ERROR_NOT_SUPPORTED)?; + unsafe { chk(f(handle)) } + } + fn mem_address_free(&mut self, va: u64, size: u64) -> CuResult<()> { + let f = self + .vmm_address_free + .ok_or(super::CUDA_ERROR_NOT_SUPPORTED)?; + unsafe { chk(f(va, size as usize)) } + } + fn mem_get_allocation_granularity(&mut self, device: i32, flags: u32) -> CuResult { + let f = self + .vmm_granularity + .ok_or(super::CUDA_ERROR_NOT_SUPPORTED)?; + let prop = vmm_prop(device); + let mut g = 0usize; + unsafe { chk(f(&mut g, &prop, flags))? }; + Ok(g as u64) + } + + fn memcpy_gpa_htod(&mut self, dptr: u64, segments: &[(u64, u64)], stream: u64) -> CuResult<()> { + if self.guest_ram.is_empty() { + return Err(super::CUDA_ERROR_NOT_FOUND); + } + self.wait_stream(stream)?; + self.ensure_guest_ram_pinned(); + zc_trace_segments("H2D", segments); + // Few segments: DMA each straight from its guest-RAM host mapping to the + // right device offset — no staging copy. The common contiguous case is a + // single big DMA at full pinned bandwidth. + if segments.len() <= ZC_DIRECT_MAX_SEGMENTS { + let mut off = 0u64; + for &(gpa, len) in segments { + let src = self.gpa_to_host(gpa, len)?; + unsafe { + chk((self.memcpy_htod)( + dptr + off, + src as *const c_void, + len as usize, + ))? + }; + off += len; + } + return Ok(()); + } + // Heavily fragmented: gather into one pinned staging buffer and issue a + // single DMA, avoiding thousands of tiny per-segment transfers. + let total: u64 = segments.iter().map(|(_, l)| *l).sum(); + if let Some(stg) = self.ensure_staging(total as usize) { + let mut off = 0usize; + for &(gpa, len) in segments { + let src = self.gpa_to_host(gpa, len)?; + unsafe { std::ptr::copy_nonoverlapping(src, stg.add(off), len as usize) }; + off += len as usize; + } + return unsafe { + chk((self.memcpy_htod)( + dptr, + stg as *const c_void, + total as usize, + )) + }; + } + // No staging available → fall back to direct per-segment DMA. + let mut off = 0u64; + for &(gpa, len) in segments { + let src = self.gpa_to_host(gpa, len)?; + unsafe { + chk((self.memcpy_htod)( + dptr + off, + src as *const c_void, + len as usize, + ))? + }; + off += len; + } + Ok(()) + } + + fn memcpy_gpa_dtoh(&mut self, dptr: u64, segments: &[(u64, u64)], stream: u64) -> CuResult<()> { + if self.guest_ram.is_empty() { + return Err(super::CUDA_ERROR_NOT_FOUND); + } + self.wait_stream(stream)?; + self.ensure_guest_ram_pinned(); + zc_trace_segments("D2H", segments); + if segments.len() <= ZC_DIRECT_MAX_SEGMENTS { + let mut off = 0u64; + for &(gpa, len) in segments { + let dst = self.gpa_to_host(gpa, len)?; + unsafe { + chk((self.memcpy_dtoh)( + dst as *mut c_void, + dptr + off, + len as usize, + ))? + }; + off += len; + } + return Ok(()); + } + // Heavily fragmented: one DMA into pinned staging, then scatter to the + // guest segments. + let total: u64 = segments.iter().map(|(_, l)| *l).sum(); + if let Some(stg) = self.ensure_staging(total as usize) { + unsafe { chk((self.memcpy_dtoh)(stg as *mut c_void, dptr, total as usize))? }; + let mut off = 0usize; + for &(gpa, len) in segments { + let dst = self.gpa_to_host(gpa, len)?; + unsafe { + std::ptr::copy_nonoverlapping(stg.add(off) as *const u8, dst, len as usize) + }; + off += len as usize; + } + return Ok(()); + } + let mut off = 0u64; + for &(gpa, len) in segments { + let dst = self.gpa_to_host(gpa, len)?; + unsafe { + chk((self.memcpy_dtoh)( + dst as *mut c_void, + dptr + off, + len as usize, + ))? + }; + off += len; + } + Ok(()) + } + fn launch_kernel( + &mut self, + function: u64, + grid: [u32; 3], + block: [u32; 3], + shared_bytes: u32, + stream: u64, + params: &[Vec], + ) -> CuResult<()> { + // The Driver API wants `void* kernelParams[]`, each pointing at one + // argument's value. Point at each param blob in place (CUDA only reads). + let mut ptrs: Vec<*mut c_void> = params.iter().map(|p| p.as_ptr() as *mut c_void).collect(); + let params_ptr = if ptrs.is_empty() { + std::ptr::null_mut() + } else { + ptrs.as_mut_ptr() + }; + unsafe { + chk((self.launch_kernel)( + function as *mut c_void, + grid[0], + grid[1], + grid[2], + block[0], + block[1], + block[2], + shared_bytes, + stream as *mut c_void, + params_ptr, + std::ptr::null_mut(), + )) + } + } + fn ctx_synchronize(&mut self) -> CuResult<()> { + unsafe { chk((self.ctx_synchronize)()) } + } + fn stream_create(&mut self, flags: u32) -> CuResult { + let mut s: *mut c_void = std::ptr::null_mut(); + unsafe { chk((self.stream_create)(&mut s, flags))? }; + if std::env::var_os("SMOLVM_CUDA_HOST_OPLOG").is_some() { + eprintln!("[strm] host created {:#x}", s as u64); + } + Ok(s as u64) + } + fn thread_exchange_capture_mode(&mut self, mode: i32) -> CuResult { + let mut m: c_int = mode; + unsafe { chk((self.thread_exchange_capture_mode)(&mut m))? }; + Ok(m) + } + fn stream_begin_capture(&mut self, stream: u64, mode: i32) -> CuResult<()> { + unsafe { chk((self.stream_begin_capture)(stream as *mut c_void, mode)) } + } + fn stream_end_capture(&mut self, stream: u64) -> CuResult { + let mut g: *mut c_void = std::ptr::null_mut(); + unsafe { chk((self.stream_end_capture)(stream as *mut c_void, &mut g))? }; + Ok(g as u64) + } + fn stream_capture_info(&mut self, stream: u64) -> CuResult<(u64, u64)> { + let mut status: c_int = 0; + let mut id: u64 = 0; + unsafe { + chk((self.stream_get_capture_info)( + stream as *mut c_void, + &mut status, + &mut id, + ))? + }; + Ok((status as u64, id)) + } + fn graph_instantiate(&mut self, graph: u64) -> CuResult { + let mut e: *mut c_void = std::ptr::null_mut(); + unsafe { + chk((self.graph_instantiate_with_flags)( + &mut e, + graph as *mut c_void, + 0, + ))? + }; + Ok(e as u64) + } + fn graph_launch(&mut self, graph_exec: u64, stream: u64) -> CuResult<()> { + unsafe { + chk((self.graph_launch)( + graph_exec as *mut c_void, + stream as *mut c_void, + )) + } + } + fn graph_exec_destroy(&mut self, graph_exec: u64) -> CuResult<()> { + unsafe { chk((self.graph_exec_destroy)(graph_exec as *mut c_void)) } + } + fn graph_destroy(&mut self, graph: u64) -> CuResult<()> { + unsafe { chk((self.graph_destroy)(graph as *mut c_void)) } + } + fn graph_get_node_count(&mut self, graph: u64) -> CuResult { + let mut n: usize = 0; + unsafe { + chk((self.graph_get_nodes)( + graph as *mut c_void, + std::ptr::null_mut(), + &mut n, + ))? + }; + Ok(n as u64) + } + fn memset_d8_async(&mut self, dptr: u64, value: u8, bytes: u64, stream: u64) -> CuResult<()> { + unsafe { + chk((self.memset_d8_async)( + dptr, + value, + bytes as usize, + stream as *mut c_void, + )) + } + } + fn memcpy_dtod_async(&mut self, dst: u64, src: u64, bytes: u64, stream: u64) -> CuResult<()> { + unsafe { + chk((self.memcpy_dtod_async)( + dst, + src, + bytes as usize, + stream as *mut c_void, + )) + } + } + fn stream_destroy(&mut self, stream: u64) -> CuResult<()> { + unsafe { chk((self.stream_destroy)(stream as *mut c_void)) } + } + fn stream_query(&mut self, stream: u64) -> CuResult { + // 600 (NOT_READY) is a status, not an error — return the raw code. + Ok(unsafe { (self.stream_query)(stream as *mut c_void) }) + } + fn stream_wait_event(&mut self, stream: u64, event: u64, flags: u32) -> CuResult<()> { + unsafe { + chk((self.stream_wait_event)( + stream as *mut c_void, + event as *mut c_void, + flags, + )) + } + } + fn event_query(&mut self, event: u64) -> CuResult { + Ok(unsafe { (self.event_query)(event as *mut c_void) }) + } + fn stream_synchronize(&mut self, stream: u64) -> CuResult<()> { + unsafe { chk((self.stream_synchronize)(stream as *mut c_void)) } + } + fn event_create(&mut self, flags: u32) -> CuResult { + let mut e: *mut c_void = std::ptr::null_mut(); + unsafe { chk((self.event_create)(&mut e, flags))? }; + Ok(e as u64) + } + fn event_destroy(&mut self, event: u64) -> CuResult<()> { + unsafe { chk((self.event_destroy)(event as *mut c_void)) } + } + fn event_record(&mut self, event: u64, stream: u64) -> CuResult<()> { + unsafe { + chk((self.event_record)( + event as *mut c_void, + stream as *mut c_void, + )) + } + } + fn event_synchronize(&mut self, event: u64) -> CuResult<()> { + unsafe { chk((self.event_synchronize)(event as *mut c_void)) } + } + fn event_elapsed_time(&mut self, start: u64, end: u64) -> CuResult { + let mut ms: f32 = 0.0; + unsafe { + chk((self.event_elapsed_time)( + &mut ms, + start as *mut c_void, + end as *mut c_void, + ))? + }; + Ok(ms) + } + + fn nvcomp_deflate_temp_size( + &mut self, + num_chunks: u64, + max_uncompressed_chunk_bytes: u64, + max_total_uncompressed_bytes: u64, + ) -> CuResult<(i32, u64)> { + let nv = self.ensure_nvcomp()?; + let mut temp: usize = 0; + let st = unsafe { + (nv.temp_size)( + num_chunks as usize, + max_uncompressed_chunk_bytes as usize, + &mut temp, + max_total_uncompressed_bytes as usize, + ) + }; + Ok((st, temp as u64)) + } + + fn nvcomp_deflate_decompress( + &mut self, + device_compressed_ptrs: u64, + device_compressed_bytes: u64, + device_uncompressed_bytes: u64, + device_actual_uncompressed_bytes: u64, + batch_size: u64, + device_temp: u64, + temp_bytes: u64, + device_uncompressed_ptrs: u64, + device_statuses: u64, + stream: u64, + ) -> CuResult { + let nv = self.ensure_nvcomp()?; + if std::env::var_os("SMOLVM_CUDA_HOST_TRACE").is_some() { + eprintln!( + "cuda-host: nvcompDecompress batch={batch_size} temp_bytes={temp_bytes} \ + comp_ptrs={device_compressed_ptrs:#x} uncomp_ptrs={device_uncompressed_ptrs:#x} \ + stream={stream:#x} — calling real nvcomp" + ); + } + // Every pointer is already a real host device address in this process's + // primary context; pass them straight to real nvcomp. + let st = unsafe { + (nv.decompress)( + device_compressed_ptrs as *const *const c_void, + device_compressed_bytes as *const usize, + device_uncompressed_bytes as *const usize, + device_actual_uncompressed_bytes as *mut usize, + batch_size as usize, + device_temp as *mut c_void, + temp_bytes as usize, + device_uncompressed_ptrs as *const *mut c_void, + device_statuses as *mut c_int, + stream as *mut c_void, + ) + }; + if std::env::var_os("SMOLVM_CUDA_HOST_TRACE").is_some() { + eprintln!("cuda-host: nvcompDecompress returned status={st}"); + } + Ok(st) + } + + fn cublas_create(&mut self) -> CuResult { + let cb = self.ensure_cublas()?; + let mut h: *mut c_void = std::ptr::null_mut(); + let st = unsafe { (cb.create)(&mut h) }; + if st != 0 { + return Err(st); + } + Ok(h as u64) + } + fn cublas_destroy(&mut self, handle: u64) -> CuResult<()> { + let cb = self.ensure_cublas()?; + chk_cublas(unsafe { (cb.destroy)(handle as *mut c_void) }) + } + fn cublas_set_stream(&mut self, handle: u64, stream: u64) -> CuResult<()> { + let cb = self.ensure_cublas()?; + chk_cublas(unsafe { (cb.set_stream)(handle as *mut c_void, stream as *mut c_void) }) + } + #[allow(clippy::too_many_arguments)] + fn cublas_sgemm( + &mut self, + handle: u64, + transa: u32, + transb: u32, + m: i32, + n: i32, + k: i32, + alpha: f32, + a: u64, + lda: i32, + b: u64, + ldb: i32, + beta: f32, + c: u64, + ldc: i32, + ) -> CuResult<()> { + let cb = self.ensure_cublas()?; + // alpha/beta are host scalars (default cuBLAS pointer mode); the matrix + // pointers are real device addresses in this process's primary context. + chk_cublas(unsafe { + (cb.sgemm)( + handle as *mut c_void, + transa as c_int, + transb as c_int, + m, + n, + k, + &alpha, + a as *const f32, + lda, + b as *const f32, + ldb, + &beta, + c as *mut f32, + ldc, + ) + }) + } + + fn lib_call( + &mut self, + lib: u8, + func: u16, + args: &[u8], + streams: &std::collections::HashMap, + ) -> CuResult<(i32, Vec)> { + self.gen_lib_call(lib, func, args, streams) + } +} + +/// cuBLAS status: 0 = `CUBLAS_STATUS_SUCCESS`, else the code as `Err`. +fn chk_cublas(st: c_int) -> CuResult<()> { + if st == 0 { + Ok(()) + } else { + Err(st) + } +} + +/// Byte size of one element of a `cudnnBackendAttributeType_t`. PyTorch's conv +/// graph uses handles/pointers (8), enums/int32/float (4), int64/double (8), +/// bool/char (1). Everything else defaults to a 4-byte enum. +fn cudnn_elem_size(attr_type: i32) -> usize { + match attr_type { + 0 | 3 | 5 | 6 | 15 => 8, // HANDLE, INT64, DOUBLE, VOID_PTR, BACKEND_DESCRIPTOR + 2 | 24 => 1, // BOOLEAN, CHAR + 26 => 16, // FRACTION + _ => 4, // DATA_TYPE / enums / FLOAT / INT32 / ... + } +} + +/// The 6-function cuDNN v8 **backend (graph) API** PyTorch uses for convolution. +/// Descriptors and device pointers passing through are the server's real host +/// pointers (opaque to the guest), so attribute arrays forward as raw bytes. +struct CudnnBackend { + _lib: Library, + create: unsafe extern "C" fn(c_int, *mut *mut c_void) -> c_int, + destroy: unsafe extern "C" fn(*mut c_void) -> c_int, + set_attr: unsafe extern "C" fn(*mut c_void, c_int, c_int, i64, *const c_void) -> c_int, + get_attr: unsafe extern "C" fn(*mut c_void, c_int, c_int, i64, *mut i64, *mut c_void) -> c_int, + finalize: unsafe extern "C" fn(*mut c_void) -> c_int, + execute: unsafe extern "C" fn(*mut c_void, *mut c_void, *mut c_void) -> c_int, +} + +impl CudnnBackend { + fn load() -> Result { + unsafe { + let lib = open_host_lib( + "SMOLVM_CUDNN_LIB", + &["libcudnn.so", "libcudnn.so.9", "libcudnn.so.8"], + )?; + let b = CudnnBackend { + create: sym(&lib, b"cudnnBackendCreateDescriptor\0")?, + destroy: sym(&lib, b"cudnnBackendDestroyDescriptor\0")?, + set_attr: sym(&lib, b"cudnnBackendSetAttribute\0")?, + get_attr: sym(&lib, b"cudnnBackendGetAttribute\0")?, + finalize: sym(&lib, b"cudnnBackendFinalize\0")?, + execute: sym(&lib, b"cudnnBackendExecute\0")?, + _lib: lib, + }; + Ok(b) + } + } + + /// `func` selects the entry point; `args` is the hand-packed little-endian + /// argument blob (see the guest shim). Returns `(cudnnStatus, out_bytes)`. + /// Descriptor handles may be guest-assigned virtual ids (`vh`); attribute + /// arrays of type `CUDNN_TYPE_BACKEND_DESCRIPTOR` embed handles and are + /// translated element-wise. + fn dispatch( + &self, + func: u16, + args: &[u8], + vh: &mut std::collections::HashMap, + ) -> (i32, Vec) { + /// `cudnnBackendAttributeType_t` values whose 8-byte elements are + /// handles the guest may know only by virtual id: HANDLE(0) — the + /// cudnnHandle_t itself — and BACKEND_DESCRIPTOR(15). VOID_PTR(6) + /// elements are device pointers (untagged), translated harmlessly. + fn handle_bearing(ty: i32) -> bool { + matches!(ty, 0 | 6 | 15) + } + let rd_u64 = |a: &[u8], o: usize| u64::from_le_bytes(a[o..o + 8].try_into().unwrap()); + let rd_i32 = |a: &[u8], o: usize| i32::from_le_bytes(a[o..o + 4].try_into().unwrap()); + let rd_i64 = |a: &[u8], o: usize| i64::from_le_bytes(a[o..o + 8].try_into().unwrap()); + match func { + 0 => { + // create(type[, virtual-id]) -> handle. With a virtual id the + // guest fired-and-forgot: map it and reply with the real + // pointer (ignored by a pipelining guest, used by an old one). + let ty = rd_i32(args, 0); + let mut desc: *mut c_void = std::ptr::null_mut(); + let st = unsafe { (self.create)(ty, &mut desc) }; + if st == 0 && args.len() >= 12 { + let id = rd_u64(args, 4); + if id & VHANDLE_TAG != 0 { + vh.insert(id, desc as u64); + } + } + (st, (desc as u64).to_le_bytes().to_vec()) + } + 1 => { + let id = rd_u64(args, 0); + let desc = vh_resolve(vh, id) as *mut c_void; + if id & VHANDLE_TAG != 0 { + vh.remove(&id); + } + (unsafe { (self.destroy)(desc) }, Vec::new()) + } + 2 => { + // set_attr(desc, name, type, count, elements...) + let desc = vh_resolve(vh, rd_u64(args, 0)) as *mut c_void; + let name = rd_i32(args, 8); + let ty = rd_i32(args, 12); + let count = rd_i64(args, 16); + let mut elems = args[24..].to_vec(); + if handle_bearing(ty) { + for chunk in elems.chunks_exact_mut(8) { + let h = u64::from_le_bytes(chunk.try_into().unwrap()); + chunk.copy_from_slice(&vh_resolve(vh, h).to_le_bytes()); + } + } + let st = unsafe { (self.set_attr)(desc, name, ty, count, elems.as_ptr().cast()) }; + (st, Vec::new()) + } + 3 => { + // get_attr(desc, name, type, requestedCount, input_bytes) -> count + bytes. + // For descriptor-array attributes the caller pre-creates the + // descriptors and passes their handles in; cuDNN populates them + // in place. So seed the buffer with the forwarded input + // (translated — the guest may pass virtual ids). + let desc = vh_resolve(vh, rd_u64(args, 0)) as *mut c_void; + let name = rd_i32(args, 8); + let ty = rd_i32(args, 12); + let req = rd_i64(args, 16); + let cap = (req.max(0) as usize) * cudnn_elem_size(ty); + let input = &args[24..]; + let mut buf = vec![0u8; cap]; + let seed = input.len().min(cap); + buf[..seed].copy_from_slice(&input[..seed]); + if handle_bearing(ty) { + for chunk in buf[..seed].chunks_exact_mut(8) { + let h = u64::from_le_bytes(chunk.try_into().unwrap()); + chunk.copy_from_slice(&vh_resolve(vh, h).to_le_bytes()); + } + } + let mut count: i64 = 0; + let ptr = if cap == 0 { + std::ptr::null_mut() + } else { + buf.as_mut_ptr().cast() + }; + let st = unsafe { (self.get_attr)(desc, name, ty, req, &mut count, ptr) }; + let n = (count.max(0) as usize) * cudnn_elem_size(ty); + let mut out = count.to_le_bytes().to_vec(); + out.extend_from_slice(&buf[..n.min(buf.len())]); + (st, out) + } + 4 => { + let desc = vh_resolve(vh, rd_u64(args, 0)) as *mut c_void; + (unsafe { (self.finalize)(desc) }, Vec::new()) + } + 5 => { + // execute(handle, plan, variantPack) + let handle = vh_resolve(vh, rd_u64(args, 0)) as *mut c_void; + let plan = vh_resolve(vh, rd_u64(args, 8)) as *mut c_void; + let vpack = vh_resolve(vh, rd_u64(args, 16)) as *mut c_void; + (unsafe { (self.execute)(handle, plan, vpack) }, Vec::new()) + } + _ => (super::CUDA_ERROR_NOT_FOUND, Vec::new()), + } + } +} + +/// The 11-function cuBLASLt matmul API PyTorch uses for linear layers. +/// Descriptors, layouts, preferences and device pointers passing through are the +/// server's real host pointers (opaque to the guest); scalars, attribute buffers +/// and the opaque algo blob forward as raw little-endian bytes. The "light +/// handle" is the connection's real cuBLAS handle (torch reuses it), forwarded as +/// a handle just like every other pointer. +struct CublasLt { + _lib: Library, + matmul: unsafe extern "C" fn( + *mut c_void, + *mut c_void, + *const c_void, + *const c_void, + *mut c_void, + *const c_void, + *mut c_void, + *const c_void, + *const c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *const c_void, + *mut c_void, + usize, + *mut c_void, + ) -> c_int, + algo_get_heuristic: unsafe extern "C" fn( + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + c_int, + *mut c_void, + *mut c_int, + ) -> c_int, + desc_create: unsafe extern "C" fn(*mut *mut c_void, c_int, c_int) -> c_int, + desc_destroy: unsafe extern "C" fn(*mut c_void) -> c_int, + desc_set_attr: unsafe extern "C" fn(*mut c_void, c_int, *const c_void, usize) -> c_int, + layout_create: unsafe extern "C" fn(*mut *mut c_void, c_int, u64, u64, i64) -> c_int, + layout_destroy: unsafe extern "C" fn(*mut c_void) -> c_int, + layout_set_attr: unsafe extern "C" fn(*mut c_void, c_int, *const c_void, usize) -> c_int, + pref_create: unsafe extern "C" fn(*mut *mut c_void) -> c_int, + pref_destroy: unsafe extern "C" fn(*mut c_void) -> c_int, + pref_set_attr: unsafe extern "C" fn(*mut c_void, c_int, *const c_void, usize) -> c_int, + // Standalone callers (not PyTorch, which reuses the cuBLAS handle) create a + // dedicated cuBLASLt handle. + lt_create: unsafe extern "C" fn(*mut *mut c_void) -> c_int, + lt_destroy: unsafe extern "C" fn(*mut c_void) -> c_int, +} + +impl CublasLt { + fn load() -> Result { + // cuBLASLt ships in its own soname; fall back to the cuBLAS path's dir. + unsafe { + let lib = open_host_lib( + "SMOLVM_CUBLASLT_LIB", + &["libcublasLt.so", "libcublasLt.so.13", "libcublasLt.so.12"], + )?; + let b = CublasLt { + matmul: sym(&lib, b"cublasLtMatmul\0")?, + algo_get_heuristic: sym(&lib, b"cublasLtMatmulAlgoGetHeuristic\0")?, + desc_create: sym(&lib, b"cublasLtMatmulDescCreate\0")?, + desc_destroy: sym(&lib, b"cublasLtMatmulDescDestroy\0")?, + desc_set_attr: sym(&lib, b"cublasLtMatmulDescSetAttribute\0")?, + layout_create: sym(&lib, b"cublasLtMatrixLayoutCreate\0")?, + layout_destroy: sym(&lib, b"cublasLtMatrixLayoutDestroy\0")?, + layout_set_attr: sym(&lib, b"cublasLtMatrixLayoutSetAttribute\0")?, + pref_create: sym(&lib, b"cublasLtMatmulPreferenceCreate\0")?, + pref_destroy: sym(&lib, b"cublasLtMatmulPreferenceDestroy\0")?, + pref_set_attr: sym(&lib, b"cublasLtMatmulPreferenceSetAttribute\0")?, + lt_create: sym(&lib, b"cublasLtCreate\0")?, + lt_destroy: sym(&lib, b"cublasLtDestroy\0")?, + _lib: lib, + }; + Ok(b) + } + } + + /// `func` selects the entry point; `args` is the hand-packed little-endian + /// argument blob (see the guest shim). Returns `(cublasStatus, out_bytes)`. + fn dispatch( + &self, + func: u16, + args: &[u8], + vh: &mut std::collections::HashMap, + streams: &std::collections::HashMap, + ) -> (i32, Vec) { + // Fire-and-forget creates append a guest-minted virtual id after the + // regular args; register it so later (deferred) calls resolve. + fn register_vh( + vh: &mut std::collections::HashMap, + args: &[u8], + fixed: usize, + real: u64, + ) { + if args.len() >= fixed + 8 { + let id = u64::from_le_bytes(args[fixed..fixed + 8].try_into().unwrap()); + if id & VHANDLE_TAG != 0 { + vh.insert(id, real); + } + } + } + // sizeof(cublasLtMatmulHeuristicResult_t): algo[64] + workspaceSize(8) + // + state(4) + wavesCount(4) + reserved[4](16) = 96 bytes. + const HEUR_RESULT_SZ: usize = 96; + let rd_u64 = |o: usize| u64::from_le_bytes(args[o..o + 8].try_into().unwrap()); + let rd_i64 = |o: usize| i64::from_le_bytes(args[o..o + 8].try_into().unwrap()); + let rd_i32 = |o: usize| i32::from_le_bytes(args[o..o + 4].try_into().unwrap()); + match func { + 0 => { + // desc_create(computeType, scaleType) -> handle + let compute = rd_i32(0); + let scale = rd_i32(4); + let mut d: *mut c_void = std::ptr::null_mut(); + let st = unsafe { (self.desc_create)(&mut d, compute, scale) }; + register_vh(vh, args, 8, d as u64); + (st, (d as u64).to_le_bytes().to_vec()) + } + 1 => { + let id = rd_u64(0); + let real = vh_resolve(vh, id); + vh.remove(&id); + ( + unsafe { (self.desc_destroy)(real as *mut c_void) }, + Vec::new(), + ) + } + 2 => { + // desc_set_attr(desc, attr, buf, size) + let desc = vh_resolve(vh, rd_u64(0)) as *mut c_void; + let attr = rd_i32(8); + let buf = &args[12..]; + let st = + unsafe { (self.desc_set_attr)(desc, attr, buf.as_ptr().cast(), buf.len()) }; + (st, Vec::new()) + } + 3 => { + // layout_create(type, rows, cols, ld) -> handle + let ty = rd_i32(0); + let rows = rd_u64(4); + let cols = rd_u64(12); + let ld = rd_i64(20); + let mut l: *mut c_void = std::ptr::null_mut(); + let st = unsafe { (self.layout_create)(&mut l, ty, rows, cols, ld) }; + register_vh(vh, args, 28, l as u64); + (st, (l as u64).to_le_bytes().to_vec()) + } + 4 => { + let id = rd_u64(0); + let real = vh_resolve(vh, id); + vh.remove(&id); + ( + unsafe { (self.layout_destroy)(real as *mut c_void) }, + Vec::new(), + ) + } + 5 => { + let layout = vh_resolve(vh, rd_u64(0)) as *mut c_void; + let attr = rd_i32(8); + let buf = &args[12..]; + let st = + unsafe { (self.layout_set_attr)(layout, attr, buf.as_ptr().cast(), buf.len()) }; + (st, Vec::new()) + } + 6 => { + // pref_create() -> handle + let mut p: *mut c_void = std::ptr::null_mut(); + let st = unsafe { (self.pref_create)(&mut p) }; + register_vh(vh, args, 0, p as u64); + (st, (p as u64).to_le_bytes().to_vec()) + } + 7 => { + let id = rd_u64(0); + let real = vh_resolve(vh, id); + vh.remove(&id); + ( + unsafe { (self.pref_destroy)(real as *mut c_void) }, + Vec::new(), + ) + } + 8 => { + let pref = vh_resolve(vh, rd_u64(0)) as *mut c_void; + let attr = rd_i32(8); + let buf = &args[12..]; + let st = + unsafe { (self.pref_set_attr)(pref, attr, buf.as_ptr().cast(), buf.len()) }; + (st, Vec::new()) + } + 9 => { + // algo_get_heuristic(light, opDesc, A, B, C, D, pref, requested) + // -> returnCount + returnCount * heuristicResult structs. + let light = vh_resolve(vh, rd_u64(0)) as *mut c_void; + let op = vh_resolve(vh, rd_u64(8)) as *mut c_void; + let a = vh_resolve(vh, rd_u64(16)) as *mut c_void; + let b = vh_resolve(vh, rd_u64(24)) as *mut c_void; + let c = vh_resolve(vh, rd_u64(32)) as *mut c_void; + let d = vh_resolve(vh, rd_u64(40)) as *mut c_void; + let pref = vh_resolve(vh, rd_u64(48)) as *mut c_void; + let requested = rd_i32(56).max(0); + let mut results = vec![0u8; requested as usize * HEUR_RESULT_SZ]; + let mut count: c_int = 0; + let st = unsafe { + (self.algo_get_heuristic)( + light, + op, + a, + b, + c, + d, + pref, + requested, + results.as_mut_ptr().cast(), + &mut count, + ) + }; + let n = (count.max(0) as usize) * HEUR_RESULT_SZ; + let mut out = (count as i64).to_le_bytes().to_vec(); + out.extend_from_slice(&results[..n.min(results.len())]); + (st, out) + } + 10 => { + // matmul(light, desc, alpha[16], A, Adesc, B, Bdesc, beta[16], + // C, Cdesc, D, Ddesc, algo[64], workspace, wsSize, stream) + let light = vh_resolve(vh, rd_u64(0)) as *mut c_void; + let desc = vh_resolve(vh, rd_u64(8)) as *mut c_void; + let alpha = &args[16..32]; + let a = rd_u64(32) as *mut c_void; + let adesc = vh_resolve(vh, rd_u64(40)) as *mut c_void; + let b = rd_u64(48) as *mut c_void; + let bdesc = vh_resolve(vh, rd_u64(56)) as *mut c_void; + let beta = &args[64..80]; + let c = rd_u64(80) as *mut c_void; + let cdesc = vh_resolve(vh, rd_u64(88)) as *mut c_void; + let dd = rd_u64(96) as *mut c_void; + let ddesc = vh_resolve(vh, rd_u64(104)) as *mut c_void; + let algo = &args[112..176]; + let algo_present = rd_u64(176) != 0; + let workspace = rd_u64(184) as *mut c_void; + let ws_size = rd_u64(192) as usize; + // Session-minted stream id → real host stream (see stream_resolve). + let stream = stream_resolve(streams, rd_u64(200)) as *mut c_void; + let algo_ptr = if algo_present { + algo.as_ptr().cast() + } else { + std::ptr::null() + }; + let st = unsafe { + (self.matmul)( + light, + desc, + alpha.as_ptr().cast(), + a, + adesc, + b, + bdesc, + beta.as_ptr().cast(), + c, + cdesc, + dd, + ddesc, + algo_ptr, + workspace, + ws_size, + stream, + ) + }; + (st, Vec::new()) + } + 11 => { + let mut h: *mut c_void = std::ptr::null_mut(); + let st = unsafe { (self.lt_create)(&mut h) }; + (st, (h as u64).to_le_bytes().to_vec()) + } + 12 => ( + unsafe { (self.lt_destroy)(rd_u64(0) as *mut c_void) }, + Vec::new(), + ), + _ => (super::CUDA_ERROR_NOT_FOUND, Vec::new()), + } + } +} + +/// The tag bit marking a guest-assigned virtual handle. Host userspace +/// pointers and CUDA device VAs never have bit 63 set, so tagged values are +/// unambiguous on the wire. +const VHANDLE_TAG: u64 = 1 << 63; + +/// Resolve a possibly-virtual handle against the map: tagged values must be +/// mapped (0 → guaranteed invalid-handle error from the library), real +/// pointers pass through. +fn vh_resolve(map: &std::collections::HashMap, h: u64) -> u64 { + if h & VHANDLE_TAG != 0 { + map.get(&h).copied().unwrap_or(0) + } else { + h + } +} + +/// Resolve a wire `cudaStream_t` against the session stream table. Streams are +/// session-minted small ids (see `StreamCreate` in host.rs), so passing one +/// straight to a real library would be read as a pointer and crash it. 0 (the +/// default stream) and unmapped values (the legacy 0x1/0x2 stream constants) +/// pass through. +fn stream_resolve(map: &std::collections::HashMap, s: u64) -> u64 { + if s == 0 { + 0 + } else { + map.get(&s).copied().unwrap_or(s) + } +} + +/// Little-endian cursor over a hand-packed argument blob (host side of the +/// batchnorm forwarder). Mirrors the guest shim's packing exactly. +struct Cur<'a> { + b: &'a [u8], + p: usize, + /// Virtual-handle map for resolving guest-assigned descriptor ids in + /// [`Cur::ptr`]. Device pointers and real host pointers pass through + /// (bit 63 clear). + vh: &'a std::collections::HashMap, +} +impl Cur<'_> { + fn take(&mut self, n: usize) -> &[u8] { + let s = &self.b[self.p..self.p + n]; + self.p += n; + s + } + fn i32(&mut self) -> i32 { + i32::from_le_bytes(self.take(4).try_into().unwrap()) + } + fn u64(&mut self) -> u64 { + u64::from_le_bytes(self.take(8).try_into().unwrap()) + } + fn f32(&mut self) -> f32 { + f32::from_le_bytes(self.take(4).try_into().unwrap()) + } + fn f64(&mut self) -> f64 { + f64::from_le_bytes(self.take(8).try_into().unwrap()) + } + fn ptr(&mut self) -> *mut c_void { + vh_resolve(self.vh, self.u64()) as *mut c_void + } +} + +/// The legacy cuDNN **batch-norm (Ex) + N-D tensor descriptor** API PyTorch uses +/// for `BatchNorm2d`. Tensor/activation descriptors and device pointers are the +/// server's real host pointers; alpha/beta are float scalars, epsilon/factor are +/// doubles, and the N-D descriptor's dim/stride arrays forward as raw i32s. +#[allow(clippy::type_complexity)] +struct CudnnBn { + _lib: Library, + set_nd: unsafe extern "C" fn(*mut c_void, c_int, c_int, *const c_int, *const c_int) -> c_int, + derive_bn: unsafe extern "C" fn(*mut c_void, *mut c_void, c_int) -> c_int, + fwd_infer: unsafe extern "C" fn( + *mut c_void, + c_int, + *const c_void, + *const c_void, + *mut c_void, + *const c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *const c_void, + *const c_void, + *const c_void, + *const c_void, + f64, + ) -> c_int, + fwd_train_ex: unsafe extern "C" fn( + *mut c_void, + c_int, + c_int, + *const c_void, + *const c_void, + *mut c_void, + *const c_void, + *mut c_void, + *const c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *const c_void, + *const c_void, + f64, + *mut c_void, + *mut c_void, + f64, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + usize, + *mut c_void, + usize, + ) -> c_int, + bwd_ex: unsafe extern "C" fn( + *mut c_void, + c_int, + c_int, + *const c_void, + *const c_void, + *const c_void, + *const c_void, + *mut c_void, + *const c_void, + *mut c_void, + *const c_void, + *mut c_void, + *const c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *const c_void, + *const c_void, + *const c_void, + *mut c_void, + *mut c_void, + f64, + *const c_void, + *const c_void, + *mut c_void, + *mut c_void, + usize, + *mut c_void, + usize, + ) -> c_int, + ws_train: unsafe extern "C" fn( + *mut c_void, + c_int, + c_int, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *mut usize, + ) -> c_int, + ws_bwd: unsafe extern "C" fn( + *mut c_void, + c_int, + c_int, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + *mut usize, + ) -> c_int, + reserve: unsafe extern "C" fn( + *mut c_void, + c_int, + c_int, + *mut c_void, + *mut c_void, + *mut usize, + ) -> c_int, +} + +impl CudnnBn { + fn load() -> Result { + unsafe { + let lib = open_host_lib( + "SMOLVM_CUDNN_LIB", + &["libcudnn.so", "libcudnn.so.9", "libcudnn.so.8"], + )?; + let b = CudnnBn { + set_nd: sym(&lib, b"cudnnSetTensorNdDescriptor\0")?, + derive_bn: sym(&lib, b"cudnnDeriveBNTensorDescriptor\0")?, + fwd_infer: sym(&lib, b"cudnnBatchNormalizationForwardInference\0")?, + fwd_train_ex: sym(&lib, b"cudnnBatchNormalizationForwardTrainingEx\0")?, + bwd_ex: sym(&lib, b"cudnnBatchNormalizationBackwardEx\0")?, + ws_train: sym( + &lib, + b"cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize\0", + )?, + ws_bwd: sym(&lib, b"cudnnGetBatchNormalizationBackwardExWorkspaceSize\0")?, + reserve: sym( + &lib, + b"cudnnGetBatchNormalizationTrainingExReserveSpaceSize\0", + )?, + _lib: lib, + }; + Ok(b) + } + } + + fn dispatch( + &self, + func: u16, + args: &[u8], + vh: &std::collections::HashMap, + ) -> (i32, Vec) { + let mut c = Cur { b: args, p: 0, vh }; + match func { + 0 => { + // set_nd(desc, dataType, nbDims, dimA[], strideA[]) + let desc = c.ptr(); + let dtype = c.i32(); + let nb = c.i32(); + let n = nb.max(0) as usize; + let dims: Vec = (0..n).map(|_| c.i32()).collect(); + let strides: Vec = (0..n).map(|_| c.i32()).collect(); + let st = unsafe { (self.set_nd)(desc, dtype, nb, dims.as_ptr(), strides.as_ptr()) }; + (st, Vec::new()) + } + 1 => { + let derived = c.ptr(); + let xdesc = c.ptr(); + let mode = c.i32(); + ( + unsafe { (self.derive_bn)(derived, xdesc, mode) }, + Vec::new(), + ) + } + 2 => { + let handle = c.ptr(); + let mode = c.i32(); + let alpha = c.f32(); + let beta = c.f32(); + let xdesc = c.ptr(); + let x = c.ptr(); + let ydesc = c.ptr(); + let y = c.ptr(); + let bndesc = c.ptr(); + let scale = c.ptr(); + let bias = c.ptr(); + let mean = c.ptr(); + let var = c.ptr(); + let eps = c.f64(); + let st = unsafe { + (self.fwd_infer)( + handle, + mode, + (&alpha as *const f32).cast(), + (&beta as *const f32).cast(), + xdesc, + x.cast_const(), + ydesc, + y, + bndesc, + scale.cast_const(), + bias.cast_const(), + mean.cast_const(), + var.cast_const(), + eps, + ) + }; + (st, Vec::new()) + } + 3 => { + let handle = c.ptr(); + let mode = c.i32(); + let bn_ops = c.i32(); + let alpha = c.f32(); + let beta = c.f32(); + let xdesc = c.ptr(); + let x = c.ptr(); + let zdesc = c.ptr(); + let z = c.ptr(); + let ydesc = c.ptr(); + let y = c.ptr(); + let bndesc = c.ptr(); + let scale = c.ptr(); + let bias = c.ptr(); + let factor = c.f64(); + let run_mean = c.ptr(); + let run_var = c.ptr(); + let eps = c.f64(); + let save_mean = c.ptr(); + let save_ivar = c.ptr(); + let act = c.ptr(); + let ws = c.ptr(); + let ws_sz = c.u64() as usize; + let rs = c.ptr(); + let rs_sz = c.u64() as usize; + let st = unsafe { + (self.fwd_train_ex)( + handle, + mode, + bn_ops, + (&alpha as *const f32).cast(), + (&beta as *const f32).cast(), + xdesc, + x.cast_const(), + zdesc, + z.cast_const(), + ydesc, + y, + bndesc, + scale.cast_const(), + bias.cast_const(), + factor, + run_mean, + run_var, + eps, + save_mean, + save_ivar, + act, + ws, + ws_sz, + rs, + rs_sz, + ) + }; + (st, Vec::new()) + } + 4 => { + let handle = c.ptr(); + let mode = c.i32(); + let bn_ops = c.i32(); + let alpha_d = c.f32(); + let beta_d = c.f32(); + let alpha_p = c.f32(); + let beta_p = c.f32(); + let xdesc = c.ptr(); + let x = c.ptr(); + let ydesc = c.ptr(); + let y = c.ptr(); + let dydesc = c.ptr(); + let dy = c.ptr(); + let dzdesc = c.ptr(); + let dz = c.ptr(); + let dxdesc = c.ptr(); + let dx = c.ptr(); + let dbndesc = c.ptr(); + let scale = c.ptr(); + let bias = c.ptr(); + let dscale = c.ptr(); + let dbias = c.ptr(); + let eps = c.f64(); + let save_mean = c.ptr(); + let save_ivar = c.ptr(); + let act = c.ptr(); + let ws = c.ptr(); + let ws_sz = c.u64() as usize; + let rs = c.ptr(); + let rs_sz = c.u64() as usize; + let st = unsafe { + (self.bwd_ex)( + handle, + mode, + bn_ops, + (&alpha_d as *const f32).cast(), + (&beta_d as *const f32).cast(), + (&alpha_p as *const f32).cast(), + (&beta_p as *const f32).cast(), + xdesc, + x.cast_const(), + ydesc, + y.cast_const(), + dydesc, + dy.cast_const(), + dzdesc, + dz, + dxdesc, + dx, + dbndesc, + scale.cast_const(), + bias.cast_const(), + dscale, + dbias, + eps, + save_mean.cast_const(), + save_ivar.cast_const(), + act, + ws, + ws_sz, + rs, + rs_sz, + ) + }; + (st, Vec::new()) + } + 5 => { + let handle = c.ptr(); + let mode = c.i32(); + let bn_ops = c.i32(); + let xdesc = c.ptr(); + let zdesc = c.ptr(); + let ydesc = c.ptr(); + let bndesc = c.ptr(); + let act = c.ptr(); + let mut size: usize = 0; + let st = unsafe { + (self.ws_train)( + handle, mode, bn_ops, xdesc, zdesc, ydesc, bndesc, act, &mut size, + ) + }; + (st, (size as u64).to_le_bytes().to_vec()) + } + 6 => { + let handle = c.ptr(); + let mode = c.i32(); + let bn_ops = c.i32(); + let xdesc = c.ptr(); + let ydesc = c.ptr(); + let dydesc = c.ptr(); + let dzdesc = c.ptr(); + let dxdesc = c.ptr(); + let bndesc = c.ptr(); + let act = c.ptr(); + let mut size: usize = 0; + let st = unsafe { + (self.ws_bwd)( + handle, mode, bn_ops, xdesc, ydesc, dydesc, dzdesc, dxdesc, bndesc, act, + &mut size, + ) + }; + (st, (size as u64).to_le_bytes().to_vec()) + } + 7 => { + let handle = c.ptr(); + let mode = c.i32(); + let bn_ops = c.i32(); + let act = c.ptr(); + let xdesc = c.ptr(); + let mut size: usize = 0; + let st = unsafe { (self.reserve)(handle, mode, bn_ops, act, xdesc, &mut size) }; + (st, (size as u64).to_le_bytes().to_vec()) + } + _ => (super::CUDA_ERROR_NOT_FOUND, Vec::new()), + } + } +} + +impl Drop for GpuBackend { + fn drop(&mut self) { + // Release any guest-RAM pins this backend took so the next connection can + // re-register them. Safe to call at drop: the context is still alive. + if let Some(unreg) = self.mem_host_unregister { + for &(hva, _) in &self.registered { + // SAFETY: we registered exactly this host range earlier. + unsafe { + unreg(hva as *mut c_void); + } + } + } + let (addr, _) = self.staging; + if addr != 0 { + if let Some(free) = self.mem_free_host { + // SAFETY: `addr` came from cuMemAllocHost and is freed once. + unsafe { free(addr as *mut c_void) }; + } + } + } +} + +/// `CUmemAllocationProp` prefix we populate: pinned device memory on one +/// device, no export handles. Zeroed tail keeps future fields defined. +#[repr(C)] +pub struct VmmProp { + type_: c_int, // CU_MEM_ALLOCATION_TYPE_PINNED = 1 + requested_handle_types: c_int, + location_type: c_int, // CU_MEM_LOCATION_TYPE_DEVICE = 1 + location_id: c_int, + win32_handle_meta: *mut c_void, + alloc_flags: [u8; 8], +} + +/// `CUmemAccessDesc`: device location + RW flags. +#[repr(C)] +pub struct VmmAccessDesc { + location_type: c_int, + location_id: c_int, + flags: c_int, // CU_MEM_ACCESS_FLAGS_PROT_READWRITE = 3 +} + +fn vmm_prop(device: i32) -> VmmProp { + VmmProp { + type_: 1, + requested_handle_types: 0, + location_type: 1, + location_id: device, + win32_handle_meta: std::ptr::null_mut(), + alloc_flags: [0; 8], + } +} + +impl GpuBackend { + /// Order a blocking copy after prior work on `stream`. Torch creates its + /// pool streams non-blocking, so the NULL-stream blocking `cuMemcpy*` our + /// copies use does NOT wait for them — without this wait, a stream-ordered + /// `cudaMemcpyAsync` from the guest could overwrite (or read) memory that + /// kernels still running on `stream` are using. + fn wait_stream(&mut self, stream: u64) -> CuResult<()> { + if stream == 0 { + return Ok(()); // legacy default stream: blocking copies already order + } + unsafe { chk((self.stream_synchronize)(stream as *mut c_void)) } + } + + /// Pin the guest-RAM host mappings into the current CUDA context on the first + /// zero-copy transfer, so subsequent DMAs from guest RAM run at full pinned + /// bandwidth (~9 GB/s) instead of the ~3 GB/s pageable path. Best-effort and + /// one-shot: a driver without the API, memory that can't be pinned, or a + /// range another connection already owns just leaves that region pageable. + /// Requires a current context, which exists by the time a transfer happens. + fn ensure_guest_ram_pinned(&mut self) { + if self.guest_ram_pin_tried { + return; + } + self.guest_ram_pin_tried = true; + let Some(reg) = self.mem_host_register else { + zc_trace("[zc-host] driver lacks cuMemHostRegister — pageable DMA"); + return; + }; + for &(_, hva, len) in &self.guest_ram { + // SAFETY: [hva, hva+len) is a live host mapping of guest RAM. + let rc = unsafe { reg(hva as *mut c_void, len as usize, 0) }; + match rc { + 0 => { + self.registered.push((hva, len)); + zc_trace(&format!( + "[zc-host] pinned guest RAM hva={hva:#x} len={len}" + )); + } + // CUDA_ERROR_HOST_MEMORY_ALREADY_REGISTERED: another connection + // owns this pin — leave it, but don't unregister it on drop. + 712 => zc_trace(&format!("[zc-host] guest RAM hva={hva:#x} already pinned")), + _ => zc_trace(&format!( + "[zc-host] pin guest RAM hva={hva:#x} len={len} failed rc={rc} (pageable)" + )), + } + } + } + + /// A pinned host staging buffer of at least `size` bytes, grown on demand and + /// reused across transfers. `None` if the driver lacks `cuMemAllocHost` or the + /// allocation fails (the caller then DMAs each segment directly). + fn ensure_staging(&mut self, size: usize) -> Option<*mut u8> { + let (addr, cap) = self.staging; + if addr != 0 && cap >= size { + return Some(addr as *mut u8); + } + let alloc = self.mem_host_alloc?; + if addr != 0 { + if let Some(free) = self.mem_free_host { + // SAFETY: `addr` came from a prior cuMemAllocHost. + unsafe { free(addr as *mut c_void) }; + } + self.staging = (0, 0); + } + let mut p: *mut c_void = std::ptr::null_mut(); + // SAFETY: valid out-pointer; a current context exists during a transfer. + let rc = unsafe { alloc(&mut p, size) }; + if rc != 0 || p.is_null() { + return None; + } + self.staging = (p as usize, size); + Some(p as *mut u8) + } + + /// Translate a guest-physical range `[gpa, gpa+len)` to a host pointer, + /// finding the RAM region that contains it. The whole range must lie in one + /// region (guest buffers are page-pinned and never straddle the PCI hole). + fn gpa_to_host(&self, gpa: u64, len: u64) -> CuResult<*mut u8> { + let end = gpa + .checked_add(len) + .ok_or(super::CUDA_ERROR_INVALID_HANDLE)?; + for &(start, hva, rlen) in &self.guest_ram { + if gpa >= start && end <= start + rlen { + return Ok((hva + (gpa - start)) as *mut u8); + } + } + Err(super::CUDA_ERROR_INVALID_HANDLE) + } + + /// Load real nvcomp on first use (from `SMOLVM_NVCOMP_LIB`). + fn ensure_nvcomp(&mut self) -> CuResult<&Nvcomp> { + if self.nvcomp.is_none() { + match Nvcomp::load() { + Ok(n) => self.nvcomp = Some(n), + Err(e) => { + tracing_note(&e); + return Err(super::CUDA_ERROR_NOT_FOUND); + } + } + } + Ok(self.nvcomp.as_ref().unwrap()) + } + /// Load real cuBLAS on first use (from `SMOLVM_CUBLAS_LIB`). + fn ensure_cublas(&mut self) -> CuResult<&Cublas> { + if self.cublas.is_none() { + match Cublas::load() { + Ok(c) => self.cublas = Some(c), + Err(e) => { + tracing_note(&e); + return Err(super::CUDA_ERROR_NOT_FOUND); + } + } + } + Ok(self.cublas.as_ref().unwrap()) + } + + /// Dispatch a generic `LibCall` to the code-generated handler for `lib`. + fn gen_lib_call( + &mut self, + lib: u8, + func: u16, + args: &[u8], + streams: &std::collections::HashMap, + ) -> CuResult<(i32, Vec)> { + match lib { + LIB_CUBLAS => { + if self.cublas_gen.is_none() { + match gen_cublas::GenLib::load() { + Ok(g) => self.cublas_gen = Some(g), + Err(e) => { + tracing_note(&e); + return Err(super::CUDA_ERROR_NOT_FOUND); + } + } + } + Ok(self.cublas_gen.as_ref().unwrap().dispatch( + func, + args, + &mut self.vhandles, + streams, + )) + } + LIB_CUDNN => { + if self.cudnn_gen.is_none() { + match gen_cudnn::GenLib::load() { + Ok(g) => self.cudnn_gen = Some(g), + Err(e) => { + tracing_note(&e); + return Err(super::CUDA_ERROR_NOT_FOUND); + } + } + } + Ok(self.cudnn_gen.as_ref().unwrap().dispatch( + func, + args, + &mut self.vhandles, + streams, + )) + } + LIB_CUDNN_BACKEND => { + if self.cudnn_backend.is_none() { + match CudnnBackend::load() { + Ok(b) => self.cudnn_backend = Some(b), + Err(e) => { + tracing_note(&e); + return Err(super::CUDA_ERROR_NOT_FOUND); + } + } + } + Ok(self + .cudnn_backend + .as_ref() + .unwrap() + .dispatch(func, args, &mut self.vhandles)) + } + LIB_CUBLASLT => { + if self.cublaslt.is_none() { + match CublasLt::load() { + Ok(b) => self.cublaslt = Some(b), + Err(e) => { + tracing_note(&e); + return Err(super::CUDA_ERROR_NOT_FOUND); + } + } + } + Ok(self.cublaslt.as_ref().unwrap().dispatch( + func, + args, + &mut self.vhandles, + streams, + )) + } + LIB_CUDNN_BN => { + if self.cudnn_bn.is_none() { + match CudnnBn::load() { + Ok(b) => self.cudnn_bn = Some(b), + Err(e) => { + tracing_note(&e); + return Err(super::CUDA_ERROR_NOT_FOUND); + } + } + } + Ok(self + .cudnn_bn + .as_ref() + .unwrap() + .dispatch(func, args, &self.vhandles)) + } + _ => Err(super::CUDA_ERROR_NOT_FOUND), + } + } +} + +fn tracing_note(msg: &str) { + eprintln!("cuda-host: nvcomp unavailable: {msg}"); +} + +/// Emit a host-side zero-copy trace line. Goes to the file named by +/// `SMOLVM_CUDA_HOST_TRACE_FILE` if set (the boot subprocess silences stderr), +/// else to stderr when `SMOLVM_CUDA_HOST_TRACE` is set. +pub(crate) fn zc_trace(msg: &str) { + if let Some(path) = std::env::var_os("SMOLVM_CUDA_HOST_TRACE_FILE") { + use std::io::Write; + if let Ok(mut f) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&path) + { + let _ = writeln!(f, "{msg}"); + } + } else if std::env::var_os("SMOLVM_CUDA_HOST_TRACE").is_some() { + eprintln!("{msg}"); + } +} + +fn zc_trace_enabled() -> bool { + std::env::var_os("SMOLVM_CUDA_HOST_TRACE_FILE").is_some() + || std::env::var_os("SMOLVM_CUDA_HOST_TRACE").is_some() +} + +/// Trace guest-RAM zero-copy transfers: direction, total bytes, and how many +/// physical segments the buffer coalesced to (1 = physically contiguous → a +/// single DMA). +fn zc_trace_segments(dir: &str, segments: &[(u64, u64)]) { + if zc_trace_enabled() { + let total: u64 = segments.iter().map(|(_, l)| *l).sum(); + zc_trace(&format!( + "[zc-host] {dir} gpa memcpy: {total} bytes in {} segment(s)", + segments.len() + )); + } +} diff --git a/crates/smolvm-cuda/src/lib.rs b/crates/smolvm-cuda/src/lib.rs new file mode 100644 index 0000000..57a134c --- /dev/null +++ b/crates/smolvm-cuda/src/lib.rs @@ -0,0 +1,43 @@ +//! CUDA Driver-API remoting for smolvm: a guest microVM forwards `cu*` calls +//! over vsock to a host server that runs them on the host's NVIDIA GPU. +//! +//! - `proto` — the wire protocol (framing, request/response codec). No deps. +//! - `client` — guest-side marshalling over any `Read`/`Write` stream. +//! - `host` — host-side dispatch with a `Backend` trait; ships a real GPU +//! backend (`GpuBackend`, `gpu` feature) and a CPU emulation backend +//! (`CpuBackend`) for GPU-less verification. +//! +//! The guest binary depends on this crate with `default-features = false` to +//! pull only `proto` + `client` (no `libloading`), keeping the musl build lean. + +pub mod client; +pub mod proto; +/// Shared-memory command/completion rings (low-latency in-VM transport). +pub mod ring; + +/// Fingerprint of the wire-defining source (see `build.rs`). The client sends +/// it in the `Init` handshake; the host rejects a mismatch, turning a stale +/// shim/server pairing into a loud error instead of silent data corruption. +pub const PROTO_HASH: u64 = { + // env! gives the hex string from build.rs; parse it at compile time. + let s = env!("SMOLVM_PROTO_HASH").as_bytes(); + let mut v = 0u64; + let mut i = 0; + while i < s.len() { + let d = match s[i] { + b'0'..=b'9' => s[i] - b'0', + b'a'..=b'f' => s[i] - b'a' + 10, + _ => 0, + }; + v = v * 16 + d as u64; + i += 1; + } + v +}; + +/// Shared-memory bulk-data channel (zero-copy memcpy). Linux-only. +#[cfg(target_os = "linux")] +pub mod shm; + +#[cfg(feature = "host")] +pub mod host; diff --git a/crates/smolvm-cuda/src/proto.rs b/crates/smolvm-cuda/src/proto.rs new file mode 100644 index 0000000..4ca97d7 --- /dev/null +++ b/crates/smolvm-cuda/src/proto.rs @@ -0,0 +1,1691 @@ +//! Wire protocol for CUDA Driver-API remoting over a byte stream (vsock). +//! +//! Framing: every message is a `u32` little-endian length followed by the +//! payload. A request payload is a `u8` opcode then its args; a response payload +//! is an `i32` status (`CUresult`, LE) then return data (present only when +//! `status == 0`). The codec is zero-dependency and transport-agnostic: it +//! operates on any [`Read`]/[`Write`], so the host (AF_UNIX) and guest +//! (AF_VSOCK) share one definition. +//! +//! Handle model: modules, functions and contexts are referred to by opaque +//! `u64` ids minted by the host (so the guest can never forge a host pointer). +//! Device pointers (`CUdeviceptr`) are passed by their real value, because a +//! kernel's launch parameters embed the device address by value — that is how +//! the CUDA Driver API itself works. + +use std::io::{self, Read, Write}; + +/// Maximum accepted message payload (256 MiB) — bounds a hostile/length field. +pub const MAX_MSG: usize = 256 * 1024 * 1024; + +/// First byte of a *quiet* request frame: the server executes the wrapped +/// request (encoded normally after this byte) and sends **no response**; the +/// first failing status is held until the next [`FENCE_OP`]. Chosen outside +/// the [`Op`] value space. +pub const QUIET_PREFIX: u8 = 0x7F; +/// Single-byte fence request: replies with (and clears) the first failing +/// status among quiet requests since the previous fence. +pub const FENCE_OP: u8 = 0x7E; + +/// Request opcodes. Stable wire values — append only. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum Op { + Init = 0x01, + DeviceGetCount = 0x02, + DeviceGetName = 0x03, + DeviceTotalMem = 0x04, + DriverGetVersion = 0x05, + DeviceGetAttribute = 0x06, + DeviceGetUuid = 0x07, + CtxCreate = 0x10, + CtxDestroy = 0x11, + PrimaryCtxRetain = 0x12, + PrimaryCtxRelease = 0x13, + ModuleLoadData = 0x20, + ModuleGetFunction = 0x21, + ModuleUnload = 0x22, + FuncGetParamInfo = 0x23, + FuncSetAttribute = 0x24, + FuncGetAttribute = 0x25, + MemAlloc = 0x30, + MemFree = 0x31, + MemcpyHtoD = 0x32, + MemcpyDtoH = 0x33, + MemcpyDtoD = 0x34, + MemsetD8 = 0x35, + MemGetInfo = 0x36, + LaunchKernel = 0x40, + CtxSynchronize = 0x50, + StreamCreate = 0x60, + StreamDestroy = 0x61, + StreamSynchronize = 0x62, + StreamQuery = 0x63, + EventCreate = 0x70, + EventDestroy = 0x71, + EventRecord = 0x72, + EventSynchronize = 0x73, + EventElapsedTime = 0x74, + StreamWaitEvent = 0x75, + EventQuery = 0x76, + // CUDA graphs: capture forwarded to the host driver (which records the + // stream's work into a graph), replayed with a single GraphLaunch. + StreamBeginCapture = 0xC0, + StreamEndCapture = 0xC1, + GraphInstantiate = 0xC2, + GraphLaunch = 0xC3, + GraphExecDestroy = 0xC4, + GraphDestroy = 0xC5, + StreamCaptureInfo = 0xC6, + // Stream-ordered variants (capture-safe: the sync forms would invalidate + // an active capture and cannot be recorded into a graph). + MemsetD8Async = 0xC7, + MemcpyDtoDAsync = 0xC8, + GraphGetNodes = 0xC9, + ThreadExchangeCaptureMode = 0xCA, + // nvcomp (forward-to-host-lib): batched Deflate decompression. Device-pointer + // args are real host device addresses, forwarded by value. + NvcompDeflateTempSize = 0x80, + NvcompDeflateDecompress = 0x81, + // cuBLAS (forward-to-host-lib). Handles are host-minted opaque ids; device + // pointers pass by value. + CublasCreate = 0x90, + CublasDestroy = 0x91, + CublasSetStream = 0x92, + CublasSgemm = 0x93, + /// Generic forward-to-host-lib call: `(lib, func, args)` → `(status, out)`. + /// The wire is library-agnostic; per-function (de)serialization is + /// code-generated (see `smolvm-cuda-codegen`). This is how the cuBLAS/cuDNN + /// surface scales without a new opcode per function. + LibCall = 0xA0, + /// Zero-copy memcpy via the shared-memory channel: the payload is an + /// `(offset, size)` descriptor into the shared region, not the bytes. + MemcpyShmHtoD = 0xB0, + MemcpyShmDtoH = 0xB1, + /// Zero-copy memcpy via guest RAM the host maps: the payload is a list of + /// `(guest_physical_addr, len)` segments, not the bytes. The host reads + /// guest memory directly (one DMA if contiguous, gather otherwise). + MemcpyGpaHtoD = 0xB2, + MemcpyGpaDtoH = 0xB3, + /// Switch this connection to shared-memory rings (see `crate::ring`): + /// the payload names the guest-physical pages of the request ring, the + /// completion ring, and a bounce buffer for oversized responses. After + /// the host acks, the socket carries only doorbell bytes. + RingSetup = 0xD0, + // CUDA VMM (virtual memory management) — torch's expandable-segments + // allocator. Control-plane ops, all synchronous. + MemAddressReserve = 0xE0, + MemCreate = 0xE1, + MemMap = 0xE2, + MemSetAccess = 0xE3, + MemUnmap = 0xE4, + MemRelease = 0xE5, + MemAddressFree = 0xE6, + MemGetAllocationGranularity = 0xE7, +} + +impl Op { + pub fn from_u8(v: u8) -> Option { + Some(match v { + 0x01 => Op::Init, + 0x02 => Op::DeviceGetCount, + 0x03 => Op::DeviceGetName, + 0x04 => Op::DeviceTotalMem, + 0x05 => Op::DriverGetVersion, + 0x06 => Op::DeviceGetAttribute, + 0x07 => Op::DeviceGetUuid, + 0x10 => Op::CtxCreate, + 0x11 => Op::CtxDestroy, + 0x12 => Op::PrimaryCtxRetain, + 0x13 => Op::PrimaryCtxRelease, + 0x20 => Op::ModuleLoadData, + 0x21 => Op::ModuleGetFunction, + 0x22 => Op::ModuleUnload, + 0x23 => Op::FuncGetParamInfo, + 0x24 => Op::FuncSetAttribute, + 0x25 => Op::FuncGetAttribute, + 0x30 => Op::MemAlloc, + 0x31 => Op::MemFree, + 0x32 => Op::MemcpyHtoD, + 0x33 => Op::MemcpyDtoH, + 0x34 => Op::MemcpyDtoD, + 0x35 => Op::MemsetD8, + 0x36 => Op::MemGetInfo, + 0x40 => Op::LaunchKernel, + 0x50 => Op::CtxSynchronize, + 0xC0 => Op::StreamBeginCapture, + 0xC1 => Op::StreamEndCapture, + 0xC2 => Op::GraphInstantiate, + 0xC3 => Op::GraphLaunch, + 0xC4 => Op::GraphExecDestroy, + 0xC5 => Op::GraphDestroy, + 0xC6 => Op::StreamCaptureInfo, + 0xC7 => Op::MemsetD8Async, + 0xC8 => Op::MemcpyDtoDAsync, + 0xC9 => Op::GraphGetNodes, + 0xCA => Op::ThreadExchangeCaptureMode, + 0x60 => Op::StreamCreate, + 0x61 => Op::StreamDestroy, + 0x62 => Op::StreamSynchronize, + 0x63 => Op::StreamQuery, + 0x70 => Op::EventCreate, + 0x71 => Op::EventDestroy, + 0x72 => Op::EventRecord, + 0x73 => Op::EventSynchronize, + 0x74 => Op::EventElapsedTime, + 0x75 => Op::StreamWaitEvent, + 0x76 => Op::EventQuery, + 0x80 => Op::NvcompDeflateTempSize, + 0x81 => Op::NvcompDeflateDecompress, + 0x90 => Op::CublasCreate, + 0x91 => Op::CublasDestroy, + 0x92 => Op::CublasSetStream, + 0x93 => Op::CublasSgemm, + 0xA0 => Op::LibCall, + 0xB0 => Op::MemcpyShmHtoD, + 0xB1 => Op::MemcpyShmDtoH, + 0xB2 => Op::MemcpyGpaHtoD, + 0xB3 => Op::MemcpyGpaDtoH, + 0xD0 => Op::RingSetup, + 0xE0 => Op::MemAddressReserve, + 0xE1 => Op::MemCreate, + 0xE2 => Op::MemMap, + 0xE3 => Op::MemSetAccess, + 0xE4 => Op::MemUnmap, + 0xE5 => Op::MemRelease, + 0xE6 => Op::MemAddressFree, + 0xE7 => Op::MemGetAllocationGranularity, + _ => return None, + }) + } +} + +/// A decoded request. Handles are opaque ids; device pointers are real values. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Request { + /// Connect handshake. `proto_hash` is the client's wire fingerprint + /// (`crate::PROTO_HASH`); the host rejects a mismatch (stale binary). + Init { + proto_hash: u64, + }, + DeviceGetCount, + DeviceGetName { + device: i32, + }, + DeviceTotalMem { + device: i32, + }, + DriverGetVersion, + DeviceGetAttribute { + attrib: i32, + device: i32, + }, + DeviceGetUuid { + device: i32, + }, + CtxCreate { + device: i32, + }, + CtxDestroy { + ctx: u64, + }, + PrimaryCtxRetain { + device: i32, + }, + PrimaryCtxRelease { + device: i32, + }, + ModuleLoadData { + image: Vec, + }, + ModuleGetFunction { + module: u64, + name: String, + }, + ModuleUnload { + module: u64, + }, + /// Per-parameter byte sizes of `function`'s kernel arguments, in declaration + /// order — what a generic client needs to serialize `kernelParams` blobs. + FuncGetParamInfo { + function: u64, + }, + /// Raise/set a `CUfunction_attribute` on the host function — chiefly + /// `CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES`, which Triton/vLLM + /// kernels needing >48 KiB shared memory must opt into before launching, or + /// the launch fails with `INVALID_VALUE`. + FuncSetAttribute { + function: u64, + attrib: i32, + value: i32, + }, + /// Read a `CUfunction_attribute` (max-threads, num-regs, shared/local + /// bytes, ptx/binary version) — forwarded so `cudaFuncGetAttributes` + /// returns real values instead of fakes (num_regs=0 divides by zero in + /// occupancy math). + FuncGetAttribute { + function: u64, + attrib: i32, + }, + MemAlloc { + bytes: u64, + }, + MemFree { + dptr: u64, + }, + MemcpyHtoD { + dptr: u64, + /// Stream whose prior work must complete before the copy (0 = legacy + /// default). Guest `cudaMemcpyAsync` semantics: the copy is ordered + /// within this stream, and torch's pool streams are non-blocking, so + /// a NULL-stream copy would NOT wait for them. + stream: u64, + data: Vec, + }, + MemcpyDtoH { + dptr: u64, + bytes: u64, + /// See `MemcpyHtoD::stream`. + stream: u64, + }, + MemcpyDtoD { + dst: u64, + src: u64, + bytes: u64, + }, + MemsetD8 { + dptr: u64, + value: u8, + bytes: u64, + }, + MemGetInfo, + /// Launch `function` with the given geometry. `params` is one byte-blob per + /// kernel argument, in order — the host rebuilds the `void*[]` the Driver + /// API expects by pointing at local copies of each blob. `stream` is an + /// opaque stream id (0 = the default stream). + LaunchKernel { + function: u64, + grid: [u32; 3], + block: [u32; 3], + shared_bytes: u32, + stream: u64, + params: Vec>, + }, + CtxSynchronize, + StreamBeginCapture { + stream: u64, + mode: i32, + }, + StreamEndCapture { + stream: u64, + /// Guest-minted virtual graph handle (bit-63 tagged). The host maps it + /// to the real captured graph so EndCapture can be fire-and-forget. + graph_vh: u64, + }, + GraphInstantiate { + graph: u64, + /// Guest-minted virtual exec handle; host maps it to the real + /// instantiated exec so GraphInstantiate can be fire-and-forget. + exec_vh: u64, + }, + GraphLaunch { + graph_exec: u64, + stream: u64, + }, + GraphExecDestroy { + graph_exec: u64, + }, + GraphDestroy { + graph: u64, + }, + StreamCaptureInfo { + stream: u64, + }, + GraphGetNodes { + graph: u64, + }, + /// `cuThreadExchangeStreamCaptureMode` on the serving thread. PyTorch's + /// allocator wraps its capture-time `cudaMalloc` in a thread-local + /// relaxed-mode guard; each connection is served by one host thread, so + /// forwarding the exchange preserves the per-thread semantics. Returns the + /// previous mode. + ThreadExchangeCaptureMode { + mode: i32, + }, + MemsetD8Async { + dptr: u64, + value: u8, + bytes: u64, + stream: u64, + }, + MemcpyDtoDAsync { + dst: u64, + src: u64, + bytes: u64, + stream: u64, + }, + StreamCreate { + flags: u32, + }, + StreamDestroy { + stream: u64, + }, + StreamSynchronize { + stream: u64, + }, + /// `cuStreamQuery`: 0 = all work complete, 600 = not ready (raw code in + /// the Count response; the guest surfaces it as its own return). + StreamQuery { + stream: u64, + }, + /// Make `stream` wait for `event` — the cross-stream ordering edge. Once + /// work really runs on side streams (and graph capture records these as + /// graph dependencies), dropping it means racy replays. + StreamWaitEvent { + stream: u64, + event: u64, + flags: u32, + }, + /// `cuEventQuery`: 0 = complete, 600 = not ready. PyTorch's allocator + /// polls this to decide when freed blocks are safe to reuse. + EventQuery { + event: u64, + }, + EventCreate { + flags: u32, + }, + EventDestroy { + event: u64, + }, + /// Record `event` on `stream` (0 = the default stream). + EventRecord { + event: u64, + stream: u64, + }, + EventSynchronize { + event: u64, + }, + EventElapsedTime { + start: u64, + end: u64, + }, + /// `nvcompBatchedDeflateDecompressGetTempSizeEx` — all host scalars. + NvcompDeflateTempSize { + num_chunks: u64, + max_uncompressed_chunk_bytes: u64, + max_total_uncompressed_bytes: u64, + }, + /// `nvcompBatchedDeflateDecompressAsync` — every pointer is a host device + /// address (or 0 for the optional out-params), forwarded by value. + NvcompDeflateDecompress { + device_compressed_ptrs: u64, + device_compressed_bytes: u64, + device_uncompressed_bytes: u64, + device_actual_uncompressed_bytes: u64, + batch_size: u64, + device_temp: u64, + temp_bytes: u64, + device_uncompressed_ptrs: u64, + device_statuses: u64, + stream: u64, + }, + CublasCreate, + CublasDestroy { + handle: u64, + }, + CublasSetStream { + handle: u64, + stream: u64, + }, + /// `cublasSgemm` (column-major). `alpha`/`beta` carried as f32 bits. + CublasSgemm { + handle: u64, + transa: u32, + transb: u32, + m: i32, + n: i32, + k: i32, + alpha_bits: u32, + a: u64, + lda: i32, + b: u64, + ldb: i32, + beta_bits: u32, + c: u64, + ldc: i32, + }, + /// Generic library call; `args` is the code-generated arg blob. + LibCall { + lib: u8, + func: u16, + args: Vec, + }, + /// Zero-copy H2D: copy `size` bytes from shared-region `offset` to `dptr`. + MemcpyShmHtoD { + dptr: u64, + offset: u64, + size: u64, + /// See `MemcpyHtoD::stream`. + stream: u64, + }, + /// Zero-copy D2H: copy `size` bytes from `dptr` to shared-region `offset`. + MemcpyShmDtoH { + offset: u64, + dptr: u64, + size: u64, + /// See `MemcpyHtoD::stream`. + stream: u64, + }, + /// Zero-copy H2D from guest RAM: gather `segments` (each `(gpa, len)`, in + /// buffer order) and copy to `dptr`. + MemcpyGpaHtoD { + dptr: u64, + /// See `MemcpyHtoD::stream`. + stream: u64, + segments: Vec<(u64, u64)>, + }, + /// Zero-copy D2H to guest RAM: copy from `dptr` and scatter into `segments`. + MemcpyGpaDtoH { + dptr: u64, + /// See `MemcpyHtoD::stream`. + stream: u64, + segments: Vec<(u64, u64)>, + }, + /// Switch to shared-memory rings. Page lists are guest-physical addresses + /// of `page_size`-sized pages; `bounce` receives responses too large for + /// an inline completion record. + RingSetup { + page_size: u32, + req_pages: Vec, + resp_pages: Vec, + bounce_pages: Vec, + }, + /// VMM: reserve a virtual address range (no backing). + MemAddressReserve { + size: u64, + align: u64, + }, + /// VMM: create a physical allocation on `device`. + MemCreate { + size: u64, + device: i32, + }, + /// VMM: back `va` with `handle` at `offset`. + MemMap { + va: u64, + size: u64, + offset: u64, + handle: u64, + }, + /// VMM: grant `device` read/write access to the mapped range. + MemSetAccess { + va: u64, + size: u64, + device: i32, + }, + MemUnmap { + va: u64, + size: u64, + }, + MemRelease { + handle: u64, + }, + MemAddressFree { + va: u64, + size: u64, + }, + MemGetAllocationGranularity { + device: i32, + flags: u32, + }, +} + +/// A decoded successful response body (the `status == 0` payload). +#[derive(Debug, Clone, PartialEq)] +pub enum Response { + /// No return value beyond status (Init, CtxDestroy, MemFree, Memcpy*, Launch, Sync). + Ok, + Count(i32), + Name(String), + Bytes(u64), + Handle(u64), + Dptr(u64), + Data(Vec), + /// Two u64s (MemGetInfo: free, total). + Pair(u64, u64), + /// Milliseconds (EventElapsedTime). f32 bits on the wire. + Millis(f32), + /// Generic library-call result: library status + serialized output params. + LibResult(i32, Vec), +} + +// ---- low-level primitives ------------------------------------------------- + +fn w_u8(b: &mut Vec, v: u8) { + b.push(v); +} +fn w_i32(b: &mut Vec, v: i32) { + b.extend_from_slice(&v.to_le_bytes()); +} +fn w_u32(b: &mut Vec, v: u32) { + b.extend_from_slice(&v.to_le_bytes()); +} +fn w_u64(b: &mut Vec, v: u64) { + b.extend_from_slice(&v.to_le_bytes()); +} +fn w_bytes(b: &mut Vec, v: &[u8]) { + w_u64(b, v.len() as u64); + b.extend_from_slice(v); +} +fn w_str(b: &mut Vec, v: &str) { + w_bytes(b, v.as_bytes()); +} + +/// Cursor-based reader over an in-memory payload. Every accessor is +/// bounds-checked so a malformed/hostile message yields `InvalidData`, never a +/// panic. +pub(crate) struct Cur<'a> { + b: &'a [u8], + p: usize, +} +impl<'a> Cur<'a> { + pub(crate) fn new(b: &'a [u8]) -> Self { + Cur { b, p: 0 } + } + fn take(&mut self, n: usize) -> io::Result<&'a [u8]> { + let end = self.p.checked_add(n).ok_or_else(bad)?; + if end > self.b.len() { + return Err(bad()); + } + let s = &self.b[self.p..end]; + self.p = end; + Ok(s) + } + fn u8(&mut self) -> io::Result { + Ok(self.take(1)?[0]) + } + fn i32(&mut self) -> io::Result { + Ok(i32::from_le_bytes(self.take(4)?.try_into().unwrap())) + } + pub(crate) fn u32(&mut self) -> io::Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().unwrap())) + } + pub(crate) fn u64(&mut self) -> io::Result { + Ok(u64::from_le_bytes(self.take(8)?.try_into().unwrap())) + } + fn bytes(&mut self) -> io::Result> { + let n = self.u64()? as usize; + Ok(self.take(n)?.to_vec()) + } + fn string(&mut self) -> io::Result { + let v = self.bytes()?; + String::from_utf8(v).map_err(|_| bad()) + } +} + +fn bad() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, "malformed cuda-rpc message") +} + +// ---- framing -------------------------------------------------------------- + +/// Write a length-prefixed payload as a **single** write. +/// +/// Framing the length and payload into one buffer (rather than two `write_all` +/// calls) matters for latency: on a request/response protocol, a length-then- +/// payload pair triggers the classic write-write-read Nagle + delayed-ACK stall +/// (~40 ms/round-trip on TCP), and even on vsock it halves the syscall count. +/// One write per message is the single biggest per-call latency win. +pub fn write_msg(w: &mut W, payload: &[u8]) -> io::Result<()> { + if payload.len() > MAX_MSG { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "message too large", + )); + } + let len = (payload.len() as u32).to_le_bytes(); + // Small messages (control calls, launches): coalesce header+payload into one + // write so the write-write-read Nagle/delayed-ACK stall can't happen; the + // copy is negligible. Large messages (bulk memcpy): write the 4-byte header + // then the payload directly — a large payload already fills segments (no + // stall), and skipping the copy is a real bandwidth win for H2D/D2H. + const COALESCE_MAX: usize = 64 * 1024; + if payload.len() <= COALESCE_MAX { + let mut framed = Vec::with_capacity(4 + payload.len()); + framed.extend_from_slice(&len); + framed.extend_from_slice(payload); + w.write_all(&framed)?; + } else { + w.write_all(&len)?; + w.write_all(payload)?; + } + w.flush() +} + +/// Read a length-prefixed payload. Returns `None` on a clean EOF at a frame +/// boundary (peer closed), `Err` on a truncated/oversized frame. +pub fn read_msg(r: &mut R) -> io::Result>> { + let mut len = [0u8; 4]; + match r.read_exact(&mut len) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None), + Err(e) => return Err(e), + } + let n = u32::from_le_bytes(len) as usize; + if n > MAX_MSG { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "message too large", + )); + } + let mut buf = vec![0u8; n]; + r.read_exact(&mut buf)?; + Ok(Some(buf)) +} + +// ---- request encode/decode ------------------------------------------------ + +pub fn encode_request(req: &Request) -> Vec { + let mut b = Vec::new(); + match req { + Request::Init { proto_hash } => { + w_u8(&mut b, Op::Init as u8); + w_u64(&mut b, *proto_hash); + } + Request::DeviceGetCount => w_u8(&mut b, Op::DeviceGetCount as u8), + Request::DeviceGetName { device } => { + w_u8(&mut b, Op::DeviceGetName as u8); + w_i32(&mut b, *device); + } + Request::DeviceTotalMem { device } => { + w_u8(&mut b, Op::DeviceTotalMem as u8); + w_i32(&mut b, *device); + } + Request::DriverGetVersion => w_u8(&mut b, Op::DriverGetVersion as u8), + Request::DeviceGetAttribute { attrib, device } => { + w_u8(&mut b, Op::DeviceGetAttribute as u8); + w_i32(&mut b, *attrib); + w_i32(&mut b, *device); + } + Request::DeviceGetUuid { device } => { + w_u8(&mut b, Op::DeviceGetUuid as u8); + w_i32(&mut b, *device); + } + Request::CtxCreate { device } => { + w_u8(&mut b, Op::CtxCreate as u8); + w_i32(&mut b, *device); + } + Request::CtxDestroy { ctx } => { + w_u8(&mut b, Op::CtxDestroy as u8); + w_u64(&mut b, *ctx); + } + Request::PrimaryCtxRetain { device } => { + w_u8(&mut b, Op::PrimaryCtxRetain as u8); + w_i32(&mut b, *device); + } + Request::PrimaryCtxRelease { device } => { + w_u8(&mut b, Op::PrimaryCtxRelease as u8); + w_i32(&mut b, *device); + } + Request::ModuleLoadData { image } => { + w_u8(&mut b, Op::ModuleLoadData as u8); + w_bytes(&mut b, image); + } + Request::ModuleGetFunction { module, name } => { + w_u8(&mut b, Op::ModuleGetFunction as u8); + w_u64(&mut b, *module); + w_str(&mut b, name); + } + Request::ModuleUnload { module } => { + w_u8(&mut b, Op::ModuleUnload as u8); + w_u64(&mut b, *module); + } + Request::FuncGetParamInfo { function } => { + w_u8(&mut b, Op::FuncGetParamInfo as u8); + w_u64(&mut b, *function); + } + Request::FuncGetAttribute { function, attrib } => { + w_u8(&mut b, Op::FuncGetAttribute as u8); + w_u64(&mut b, *function); + w_i32(&mut b, *attrib); + } + Request::FuncSetAttribute { + function, + attrib, + value, + } => { + w_u8(&mut b, Op::FuncSetAttribute as u8); + w_u64(&mut b, *function); + w_i32(&mut b, *attrib); + w_i32(&mut b, *value); + } + Request::MemAlloc { bytes } => { + w_u8(&mut b, Op::MemAlloc as u8); + w_u64(&mut b, *bytes); + } + Request::MemFree { dptr } => { + w_u8(&mut b, Op::MemFree as u8); + w_u64(&mut b, *dptr); + } + Request::MemcpyHtoD { dptr, stream, data } => { + w_u8(&mut b, Op::MemcpyHtoD as u8); + w_u64(&mut b, *dptr); + w_u64(&mut b, *stream); + w_bytes(&mut b, data); + } + Request::MemcpyDtoH { + dptr, + bytes, + stream, + } => { + w_u8(&mut b, Op::MemcpyDtoH as u8); + w_u64(&mut b, *dptr); + w_u64(&mut b, *bytes); + w_u64(&mut b, *stream); + } + Request::MemcpyDtoD { dst, src, bytes } => { + w_u8(&mut b, Op::MemcpyDtoD as u8); + w_u64(&mut b, *dst); + w_u64(&mut b, *src); + w_u64(&mut b, *bytes); + } + Request::MemsetD8 { dptr, value, bytes } => { + w_u8(&mut b, Op::MemsetD8 as u8); + w_u64(&mut b, *dptr); + w_u8(&mut b, *value); + w_u64(&mut b, *bytes); + } + Request::MemGetInfo => w_u8(&mut b, Op::MemGetInfo as u8), + Request::LaunchKernel { + function, + grid, + block, + shared_bytes, + stream, + params, + } => { + w_u8(&mut b, Op::LaunchKernel as u8); + w_u64(&mut b, *function); + for v in grid { + w_u32(&mut b, *v); + } + for v in block { + w_u32(&mut b, *v); + } + w_u32(&mut b, *shared_bytes); + w_u64(&mut b, *stream); + w_u32(&mut b, params.len() as u32); + for p in params { + w_bytes(&mut b, p); + } + } + Request::CtxSynchronize => w_u8(&mut b, Op::CtxSynchronize as u8), + Request::StreamBeginCapture { stream, mode } => { + w_u8(&mut b, Op::StreamBeginCapture as u8); + w_u64(&mut b, *stream); + w_i32(&mut b, *mode); + } + Request::StreamEndCapture { stream, graph_vh } => { + w_u8(&mut b, Op::StreamEndCapture as u8); + w_u64(&mut b, *stream); + w_u64(&mut b, *graph_vh); + } + Request::GraphInstantiate { graph, exec_vh } => { + w_u8(&mut b, Op::GraphInstantiate as u8); + w_u64(&mut b, *graph); + w_u64(&mut b, *exec_vh); + } + Request::GraphLaunch { graph_exec, stream } => { + w_u8(&mut b, Op::GraphLaunch as u8); + w_u64(&mut b, *graph_exec); + w_u64(&mut b, *stream); + } + Request::GraphExecDestroy { graph_exec } => { + w_u8(&mut b, Op::GraphExecDestroy as u8); + w_u64(&mut b, *graph_exec); + } + Request::GraphDestroy { graph } => { + w_u8(&mut b, Op::GraphDestroy as u8); + w_u64(&mut b, *graph); + } + Request::StreamCaptureInfo { stream } => { + w_u8(&mut b, Op::StreamCaptureInfo as u8); + w_u64(&mut b, *stream); + } + Request::GraphGetNodes { graph } => { + w_u8(&mut b, Op::GraphGetNodes as u8); + w_u64(&mut b, *graph); + } + Request::ThreadExchangeCaptureMode { mode } => { + w_u8(&mut b, Op::ThreadExchangeCaptureMode as u8); + w_i32(&mut b, *mode); + } + Request::MemsetD8Async { + dptr, + value, + bytes, + stream, + } => { + w_u8(&mut b, Op::MemsetD8Async as u8); + w_u64(&mut b, *dptr); + w_u8(&mut b, *value); + w_u64(&mut b, *bytes); + w_u64(&mut b, *stream); + } + Request::MemcpyDtoDAsync { + dst, + src, + bytes, + stream, + } => { + w_u8(&mut b, Op::MemcpyDtoDAsync as u8); + w_u64(&mut b, *dst); + w_u64(&mut b, *src); + w_u64(&mut b, *bytes); + w_u64(&mut b, *stream); + } + Request::StreamCreate { flags } => { + w_u8(&mut b, Op::StreamCreate as u8); + w_u32(&mut b, *flags); + } + Request::StreamDestroy { stream } => { + w_u8(&mut b, Op::StreamDestroy as u8); + w_u64(&mut b, *stream); + } + Request::StreamSynchronize { stream } => { + w_u8(&mut b, Op::StreamSynchronize as u8); + w_u64(&mut b, *stream); + } + Request::StreamQuery { stream } => { + w_u8(&mut b, Op::StreamQuery as u8); + w_u64(&mut b, *stream); + } + Request::StreamWaitEvent { + stream, + event, + flags, + } => { + w_u8(&mut b, Op::StreamWaitEvent as u8); + w_u64(&mut b, *stream); + w_u64(&mut b, *event); + w_u32(&mut b, *flags); + } + Request::EventQuery { event } => { + w_u8(&mut b, Op::EventQuery as u8); + w_u64(&mut b, *event); + } + Request::EventCreate { flags } => { + w_u8(&mut b, Op::EventCreate as u8); + w_u32(&mut b, *flags); + } + Request::EventDestroy { event } => { + w_u8(&mut b, Op::EventDestroy as u8); + w_u64(&mut b, *event); + } + Request::EventRecord { event, stream } => { + w_u8(&mut b, Op::EventRecord as u8); + w_u64(&mut b, *event); + w_u64(&mut b, *stream); + } + Request::EventSynchronize { event } => { + w_u8(&mut b, Op::EventSynchronize as u8); + w_u64(&mut b, *event); + } + Request::EventElapsedTime { start, end } => { + w_u8(&mut b, Op::EventElapsedTime as u8); + w_u64(&mut b, *start); + w_u64(&mut b, *end); + } + Request::NvcompDeflateTempSize { + num_chunks, + max_uncompressed_chunk_bytes, + max_total_uncompressed_bytes, + } => { + w_u8(&mut b, Op::NvcompDeflateTempSize as u8); + w_u64(&mut b, *num_chunks); + w_u64(&mut b, *max_uncompressed_chunk_bytes); + w_u64(&mut b, *max_total_uncompressed_bytes); + } + Request::NvcompDeflateDecompress { + device_compressed_ptrs, + device_compressed_bytes, + device_uncompressed_bytes, + device_actual_uncompressed_bytes, + batch_size, + device_temp, + temp_bytes, + device_uncompressed_ptrs, + device_statuses, + stream, + } => { + w_u8(&mut b, Op::NvcompDeflateDecompress as u8); + for v in [ + device_compressed_ptrs, + device_compressed_bytes, + device_uncompressed_bytes, + device_actual_uncompressed_bytes, + batch_size, + device_temp, + temp_bytes, + device_uncompressed_ptrs, + device_statuses, + stream, + ] { + w_u64(&mut b, *v); + } + } + Request::CublasCreate => w_u8(&mut b, Op::CublasCreate as u8), + Request::CublasDestroy { handle } => { + w_u8(&mut b, Op::CublasDestroy as u8); + w_u64(&mut b, *handle); + } + Request::CublasSetStream { handle, stream } => { + w_u8(&mut b, Op::CublasSetStream as u8); + w_u64(&mut b, *handle); + w_u64(&mut b, *stream); + } + Request::CublasSgemm { + handle, + transa, + transb, + m, + n, + k, + alpha_bits, + a, + lda, + b: bmat, + ldb, + beta_bits, + c, + ldc, + } => { + w_u8(&mut b, Op::CublasSgemm as u8); + w_u64(&mut b, *handle); + w_u32(&mut b, *transa); + w_u32(&mut b, *transb); + w_i32(&mut b, *m); + w_i32(&mut b, *n); + w_i32(&mut b, *k); + w_u32(&mut b, *alpha_bits); + w_u64(&mut b, *a); + w_i32(&mut b, *lda); + w_u64(&mut b, *bmat); + w_i32(&mut b, *ldb); + w_u32(&mut b, *beta_bits); + w_u64(&mut b, *c); + w_i32(&mut b, *ldc); + } + Request::LibCall { lib, func, args } => { + w_u8(&mut b, Op::LibCall as u8); + w_u8(&mut b, *lib); + w_u32(&mut b, *func as u32); + w_bytes(&mut b, args); + } + Request::MemcpyShmHtoD { + dptr, + offset, + size, + stream, + } => { + w_u8(&mut b, Op::MemcpyShmHtoD as u8); + w_u64(&mut b, *dptr); + w_u64(&mut b, *offset); + w_u64(&mut b, *size); + w_u64(&mut b, *stream); + } + Request::MemcpyShmDtoH { + offset, + dptr, + size, + stream, + } => { + w_u8(&mut b, Op::MemcpyShmDtoH as u8); + w_u64(&mut b, *offset); + w_u64(&mut b, *dptr); + w_u64(&mut b, *size); + w_u64(&mut b, *stream); + } + Request::MemcpyGpaHtoD { + dptr, + stream, + segments, + } => { + w_u8(&mut b, Op::MemcpyGpaHtoD as u8); + w_u64(&mut b, *dptr); + w_u64(&mut b, *stream); + w_gpa_segments(&mut b, segments); + } + Request::MemcpyGpaDtoH { + dptr, + stream, + segments, + } => { + w_u8(&mut b, Op::MemcpyGpaDtoH as u8); + w_u64(&mut b, *dptr); + w_u64(&mut b, *stream); + w_gpa_segments(&mut b, segments); + } + Request::RingSetup { + page_size, + req_pages, + resp_pages, + bounce_pages, + } => { + w_u8(&mut b, Op::RingSetup as u8); + w_u32(&mut b, *page_size); + for pages in [req_pages, resp_pages, bounce_pages] { + w_u32(&mut b, pages.len() as u32); + for gpa in pages { + w_u64(&mut b, *gpa); + } + } + } + Request::MemAddressReserve { size, align } => { + w_u8(&mut b, Op::MemAddressReserve as u8); + w_u64(&mut b, *size); + w_u64(&mut b, *align); + } + Request::MemCreate { size, device } => { + w_u8(&mut b, Op::MemCreate as u8); + w_u64(&mut b, *size); + w_i32(&mut b, *device); + } + Request::MemMap { + va, + size, + offset, + handle, + } => { + w_u8(&mut b, Op::MemMap as u8); + w_u64(&mut b, *va); + w_u64(&mut b, *size); + w_u64(&mut b, *offset); + w_u64(&mut b, *handle); + } + Request::MemSetAccess { va, size, device } => { + w_u8(&mut b, Op::MemSetAccess as u8); + w_u64(&mut b, *va); + w_u64(&mut b, *size); + w_i32(&mut b, *device); + } + Request::MemUnmap { va, size } => { + w_u8(&mut b, Op::MemUnmap as u8); + w_u64(&mut b, *va); + w_u64(&mut b, *size); + } + Request::MemRelease { handle } => { + w_u8(&mut b, Op::MemRelease as u8); + w_u64(&mut b, *handle); + } + Request::MemAddressFree { va, size } => { + w_u8(&mut b, Op::MemAddressFree as u8); + w_u64(&mut b, *va); + w_u64(&mut b, *size); + } + Request::MemGetAllocationGranularity { device, flags } => { + w_u8(&mut b, Op::MemGetAllocationGranularity as u8); + w_i32(&mut b, *device); + w_u32(&mut b, *flags); + } + } + b +} + +fn w_gpa_segments(b: &mut Vec, segs: &[(u64, u64)]) { + w_u32(b, segs.len() as u32); + for (gpa, len) in segs { + w_u64(b, *gpa); + w_u64(b, *len); + } +} + +pub fn decode_request(payload: &[u8]) -> io::Result { + let mut c = Cur::new(payload); + let op = Op::from_u8(c.u8()?).ok_or_else(bad)?; + Ok(match op { + Op::Init => Request::Init { + proto_hash: c.u64()?, + }, + Op::DeviceGetCount => Request::DeviceGetCount, + Op::DeviceGetName => Request::DeviceGetName { device: c.i32()? }, + Op::DeviceTotalMem => Request::DeviceTotalMem { device: c.i32()? }, + Op::DriverGetVersion => Request::DriverGetVersion, + Op::DeviceGetAttribute => Request::DeviceGetAttribute { + attrib: c.i32()?, + device: c.i32()?, + }, + Op::DeviceGetUuid => Request::DeviceGetUuid { device: c.i32()? }, + Op::CtxCreate => Request::CtxCreate { device: c.i32()? }, + Op::CtxDestroy => Request::CtxDestroy { ctx: c.u64()? }, + Op::PrimaryCtxRetain => Request::PrimaryCtxRetain { device: c.i32()? }, + Op::PrimaryCtxRelease => Request::PrimaryCtxRelease { device: c.i32()? }, + Op::ModuleLoadData => Request::ModuleLoadData { image: c.bytes()? }, + Op::ModuleGetFunction => Request::ModuleGetFunction { + module: c.u64()?, + name: c.string()?, + }, + Op::ModuleUnload => Request::ModuleUnload { module: c.u64()? }, + Op::FuncGetParamInfo => Request::FuncGetParamInfo { function: c.u64()? }, + Op::FuncSetAttribute => Request::FuncSetAttribute { + function: c.u64()?, + attrib: c.i32()?, + value: c.i32()?, + }, + Op::FuncGetAttribute => Request::FuncGetAttribute { + function: c.u64()?, + attrib: c.i32()?, + }, + Op::MemAlloc => Request::MemAlloc { bytes: c.u64()? }, + Op::MemFree => Request::MemFree { dptr: c.u64()? }, + Op::MemcpyHtoD => Request::MemcpyHtoD { + dptr: c.u64()?, + stream: c.u64()?, + data: c.bytes()?, + }, + Op::MemcpyDtoH => Request::MemcpyDtoH { + dptr: c.u64()?, + bytes: c.u64()?, + stream: c.u64()?, + }, + Op::MemcpyDtoD => Request::MemcpyDtoD { + dst: c.u64()?, + src: c.u64()?, + bytes: c.u64()?, + }, + Op::MemsetD8 => Request::MemsetD8 { + dptr: c.u64()?, + value: c.u8()?, + bytes: c.u64()?, + }, + Op::MemGetInfo => Request::MemGetInfo, + Op::LaunchKernel => { + let function = c.u64()?; + let grid = [c.u32()?, c.u32()?, c.u32()?]; + let block = [c.u32()?, c.u32()?, c.u32()?]; + let shared_bytes = c.u32()?; + let stream = c.u64()?; + let n = c.u32()? as usize; + let mut params = Vec::with_capacity(n); + for _ in 0..n { + params.push(c.bytes()?); + } + Request::LaunchKernel { + function, + grid, + block, + shared_bytes, + stream, + params, + } + } + Op::CtxSynchronize => Request::CtxSynchronize, + Op::StreamBeginCapture => Request::StreamBeginCapture { + stream: c.u64()?, + mode: c.i32()?, + }, + Op::StreamEndCapture => Request::StreamEndCapture { + stream: c.u64()?, + graph_vh: c.u64()?, + }, + Op::GraphInstantiate => Request::GraphInstantiate { + graph: c.u64()?, + exec_vh: c.u64()?, + }, + Op::GraphLaunch => Request::GraphLaunch { + graph_exec: c.u64()?, + stream: c.u64()?, + }, + Op::GraphExecDestroy => Request::GraphExecDestroy { + graph_exec: c.u64()?, + }, + Op::GraphDestroy => Request::GraphDestroy { graph: c.u64()? }, + Op::StreamCaptureInfo => Request::StreamCaptureInfo { stream: c.u64()? }, + Op::GraphGetNodes => Request::GraphGetNodes { graph: c.u64()? }, + Op::ThreadExchangeCaptureMode => Request::ThreadExchangeCaptureMode { mode: c.i32()? }, + Op::MemsetD8Async => Request::MemsetD8Async { + dptr: c.u64()?, + value: c.u8()?, + bytes: c.u64()?, + stream: c.u64()?, + }, + Op::MemcpyDtoDAsync => Request::MemcpyDtoDAsync { + dst: c.u64()?, + src: c.u64()?, + bytes: c.u64()?, + stream: c.u64()?, + }, + Op::StreamCreate => Request::StreamCreate { flags: c.u32()? }, + Op::StreamDestroy => Request::StreamDestroy { stream: c.u64()? }, + Op::StreamSynchronize => Request::StreamSynchronize { stream: c.u64()? }, + Op::StreamQuery => Request::StreamQuery { stream: c.u64()? }, + Op::StreamWaitEvent => Request::StreamWaitEvent { + stream: c.u64()?, + event: c.u64()?, + flags: c.u32()?, + }, + Op::EventQuery => Request::EventQuery { event: c.u64()? }, + Op::EventCreate => Request::EventCreate { flags: c.u32()? }, + Op::EventDestroy => Request::EventDestroy { event: c.u64()? }, + Op::EventRecord => Request::EventRecord { + event: c.u64()?, + stream: c.u64()?, + }, + Op::EventSynchronize => Request::EventSynchronize { event: c.u64()? }, + Op::EventElapsedTime => Request::EventElapsedTime { + start: c.u64()?, + end: c.u64()?, + }, + Op::NvcompDeflateTempSize => Request::NvcompDeflateTempSize { + num_chunks: c.u64()?, + max_uncompressed_chunk_bytes: c.u64()?, + max_total_uncompressed_bytes: c.u64()?, + }, + Op::NvcompDeflateDecompress => Request::NvcompDeflateDecompress { + device_compressed_ptrs: c.u64()?, + device_compressed_bytes: c.u64()?, + device_uncompressed_bytes: c.u64()?, + device_actual_uncompressed_bytes: c.u64()?, + batch_size: c.u64()?, + device_temp: c.u64()?, + temp_bytes: c.u64()?, + device_uncompressed_ptrs: c.u64()?, + device_statuses: c.u64()?, + stream: c.u64()?, + }, + Op::CublasCreate => Request::CublasCreate, + Op::CublasDestroy => Request::CublasDestroy { handle: c.u64()? }, + Op::CublasSetStream => Request::CublasSetStream { + handle: c.u64()?, + stream: c.u64()?, + }, + Op::CublasSgemm => Request::CublasSgemm { + handle: c.u64()?, + transa: c.u32()?, + transb: c.u32()?, + m: c.i32()?, + n: c.i32()?, + k: c.i32()?, + alpha_bits: c.u32()?, + a: c.u64()?, + lda: c.i32()?, + b: c.u64()?, + ldb: c.i32()?, + beta_bits: c.u32()?, + c: c.u64()?, + ldc: c.i32()?, + }, + Op::LibCall => Request::LibCall { + lib: c.u8()?, + func: c.u32()? as u16, + args: c.bytes()?, + }, + Op::MemcpyShmHtoD => Request::MemcpyShmHtoD { + dptr: c.u64()?, + offset: c.u64()?, + size: c.u64()?, + stream: c.u64()?, + }, + Op::MemcpyShmDtoH => Request::MemcpyShmDtoH { + offset: c.u64()?, + dptr: c.u64()?, + size: c.u64()?, + stream: c.u64()?, + }, + Op::MemcpyGpaHtoD => Request::MemcpyGpaHtoD { + dptr: c.u64()?, + stream: c.u64()?, + segments: r_gpa_segments(&mut c)?, + }, + Op::MemcpyGpaDtoH => Request::MemcpyGpaDtoH { + dptr: c.u64()?, + stream: c.u64()?, + segments: r_gpa_segments(&mut c)?, + }, + Op::RingSetup => { + let page_size = c.u32()?; + let mut lists = [const { Vec::new() }; 3]; + for list in lists.iter_mut() { + let n = c.u32()? as usize; + list.reserve(n.min(1 << 16)); + for _ in 0..n { + list.push(c.u64()?); + } + } + let [req_pages, resp_pages, bounce_pages] = lists; + Request::RingSetup { + page_size, + req_pages, + resp_pages, + bounce_pages, + } + } + Op::MemAddressReserve => Request::MemAddressReserve { + size: c.u64()?, + align: c.u64()?, + }, + Op::MemCreate => Request::MemCreate { + size: c.u64()?, + device: c.i32()?, + }, + Op::MemMap => Request::MemMap { + va: c.u64()?, + size: c.u64()?, + offset: c.u64()?, + handle: c.u64()?, + }, + Op::MemSetAccess => Request::MemSetAccess { + va: c.u64()?, + size: c.u64()?, + device: c.i32()?, + }, + Op::MemUnmap => Request::MemUnmap { + va: c.u64()?, + size: c.u64()?, + }, + Op::MemRelease => Request::MemRelease { handle: c.u64()? }, + Op::MemAddressFree => Request::MemAddressFree { + va: c.u64()?, + size: c.u64()?, + }, + Op::MemGetAllocationGranularity => Request::MemGetAllocationGranularity { + device: c.i32()?, + flags: c.u32()?, + }, + }) +} + +fn r_gpa_segments(c: &mut Cur) -> io::Result> { + let n = c.u32()? as usize; + let mut segs = Vec::with_capacity(n.min(1 << 20)); + for _ in 0..n { + segs.push((c.u64()?, c.u64()?)); + } + Ok(segs) +} + +// ---- response encode/decode ----------------------------------------------- + +/// Encode a response: `i32 status` then, only when `status == 0`, the body. +pub fn encode_response(status: i32, resp: &Response) -> Vec { + let mut b = Vec::new(); + w_i32(&mut b, status); + if status == 0 { + match resp { + Response::Ok => {} + Response::Count(v) => w_i32(&mut b, *v), + Response::Name(s) => w_str(&mut b, s), + Response::Bytes(v) | Response::Handle(v) | Response::Dptr(v) => w_u64(&mut b, *v), + Response::Data(d) => w_bytes(&mut b, d), + Response::Pair(a, z) => { + w_u64(&mut b, *a); + w_u64(&mut b, *z); + } + Response::Millis(ms) => w_u32(&mut b, ms.to_bits()), + Response::LibResult(status, out) => { + w_i32(&mut b, *status); + w_bytes(&mut b, out); + } + } + } + b +} + +/// Decode a response for `op`. Returns `(status, body)`; `body` is `Response::Ok` +/// when status != 0 (error — no body on the wire). +pub fn decode_response(op: Op, payload: &[u8]) -> io::Result<(i32, Response)> { + let mut c = Cur::new(payload); + let status = c.i32()?; + if status != 0 { + return Ok((status, Response::Ok)); + } + let body = match op { + Op::DeviceGetCount + | Op::DriverGetVersion + | Op::DeviceGetAttribute + | Op::ThreadExchangeCaptureMode + | Op::StreamQuery + | Op::EventQuery + | Op::FuncGetAttribute => Response::Count(c.i32()?), + Op::DeviceGetName => Response::Name(c.string()?), + Op::DeviceTotalMem => Response::Bytes(c.u64()?), + Op::CtxCreate | Op::PrimaryCtxRetain => Response::Handle(c.u64()?), + Op::ModuleLoadData | Op::ModuleGetFunction => Response::Handle(c.u64()?), + Op::StreamCreate | Op::EventCreate => Response::Handle(c.u64()?), + Op::StreamEndCapture | Op::GraphInstantiate => Response::Handle(c.u64()?), + Op::GraphGetNodes => Response::Bytes(c.u64()?), + Op::StreamCaptureInfo => Response::Pair(c.u64()?, c.u64()?), + Op::MemAlloc => Response::Dptr(c.u64()?), + Op::MemcpyDtoH | Op::DeviceGetUuid | Op::FuncGetParamInfo => Response::Data(c.bytes()?), + Op::MemGetInfo => Response::Pair(c.u64()?, c.u64()?), + // nvcomp calls carry their own nvcompStatus in the body (transport + // status stays 0): TempSize -> (status, temp_bytes); Decompress -> status. + Op::NvcompDeflateTempSize => Response::Pair(c.u64()?, c.u64()?), + Op::NvcompDeflateDecompress => Response::Count(c.i32()?), + Op::CublasCreate => Response::Handle(c.u64()?), + Op::LibCall => Response::LibResult(c.i32()?, c.bytes()?), + Op::EventElapsedTime => Response::Millis(f32::from_bits(c.u32()?)), + Op::Init + | Op::StreamBeginCapture + | Op::GraphLaunch + | Op::GraphExecDestroy + | Op::GraphDestroy + | Op::MemsetD8Async + | Op::MemcpyDtoDAsync + | Op::CtxDestroy + | Op::PrimaryCtxRelease + | Op::ModuleUnload + | Op::FuncSetAttribute + | Op::MemFree + | Op::MemcpyHtoD + | Op::MemcpyDtoD + | Op::MemsetD8 + | Op::LaunchKernel + | Op::CtxSynchronize + | Op::StreamDestroy + | Op::StreamSynchronize + | Op::StreamWaitEvent + | Op::EventDestroy + | Op::EventRecord + | Op::EventSynchronize + | Op::CublasDestroy + | Op::CublasSetStream + | Op::CublasSgemm + | Op::MemcpyShmHtoD + | Op::MemcpyShmDtoH + | Op::MemcpyGpaHtoD + | Op::MemcpyGpaDtoH + | Op::RingSetup + | Op::MemMap + | Op::MemSetAccess + | Op::MemUnmap + | Op::MemRelease + | Op::MemAddressFree => Response::Ok, + Op::MemAddressReserve => Response::Dptr(c.u64()?), + Op::MemCreate => Response::Handle(c.u64()?), + Op::MemGetAllocationGranularity => Response::Bytes(c.u64()?), + }; + Ok((status, body)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn roundtrip(req: Request) { + let enc = encode_request(&req); + let dec = decode_request(&enc).expect("decode"); + assert_eq!(req, dec); + } + + #[test] + fn request_roundtrips() { + roundtrip(Request::Init { + proto_hash: 0xdeadbeef, + }); + roundtrip(Request::DeviceGetCount); + roundtrip(Request::DeviceGetName { device: 3 }); + roundtrip(Request::DeviceTotalMem { device: 0 }); + roundtrip(Request::CtxCreate { device: 1 }); + roundtrip(Request::CtxDestroy { ctx: 0xdead_beef }); + roundtrip(Request::ModuleLoadData { + image: b".version 7.0\n".to_vec(), + }); + roundtrip(Request::ModuleGetFunction { + module: 42, + name: "vecadd".into(), + }); + roundtrip(Request::MemAlloc { bytes: 4096 }); + roundtrip(Request::MemFree { dptr: 0x7f00_0000 }); + roundtrip(Request::MemcpyHtoD { + dptr: 0x7f00_0000, + stream: 0x7001, + data: vec![1, 2, 3, 4], + }); + roundtrip(Request::MemcpyDtoH { + dptr: 0x7f00_0000, + bytes: 16, + stream: 0x7001, + }); + roundtrip(Request::LaunchKernel { + function: 7, + grid: [4, 1, 1], + block: [256, 1, 1], + shared_bytes: 0, + stream: 0, + params: vec![ + 0x1000u64.to_le_bytes().to_vec(), + 0x2000u64.to_le_bytes().to_vec(), + 1024u32.to_le_bytes().to_vec(), + ], + }); + roundtrip(Request::CtxSynchronize); + } + + #[test] + fn extended_request_roundtrips() { + roundtrip(Request::DriverGetVersion); + roundtrip(Request::DeviceGetAttribute { + attrib: 75, + device: 0, + }); + roundtrip(Request::DeviceGetUuid { device: 0 }); + roundtrip(Request::PrimaryCtxRetain { device: 0 }); + roundtrip(Request::PrimaryCtxRelease { device: 0 }); + roundtrip(Request::ModuleUnload { module: 7 }); + roundtrip(Request::FuncGetParamInfo { function: 9 }); + roundtrip(Request::FuncSetAttribute { + function: 9, + attrib: 8, + value: 73728, + }); + roundtrip(Request::MemcpyDtoD { + dst: 0x2000, + src: 0x1000, + bytes: 64, + }); + roundtrip(Request::MemsetD8 { + dptr: 0x1000, + value: 0xAB, + bytes: 128, + }); + roundtrip(Request::MemGetInfo); + roundtrip(Request::StreamCreate { flags: 1 }); + roundtrip(Request::StreamDestroy { stream: 3 }); + roundtrip(Request::StreamSynchronize { stream: 3 }); + roundtrip(Request::EventCreate { flags: 0 }); + roundtrip(Request::EventDestroy { event: 4 }); + roundtrip(Request::EventRecord { + event: 4, + stream: 0, + }); + roundtrip(Request::EventSynchronize { event: 4 }); + roundtrip(Request::EventElapsedTime { start: 4, end: 5 }); + roundtrip(Request::StreamQuery { stream: 3 }); + roundtrip(Request::StreamWaitEvent { + stream: 3, + event: 4, + flags: 0, + }); + roundtrip(Request::EventQuery { event: 4 }); + roundtrip(Request::ThreadExchangeCaptureMode { mode: 2 }); + } + + #[test] + fn extended_response_roundtrips() { + for (op, resp) in [ + (Op::DriverGetVersion, Response::Count(13000)), + (Op::DeviceGetAttribute, Response::Count(1024)), + (Op::DeviceGetUuid, Response::Data(vec![0u8; 16])), + (Op::PrimaryCtxRetain, Response::Handle(11)), + ( + Op::FuncGetParamInfo, + Response::Data(vec![8, 0, 0, 0, 4, 0, 0, 0]), + ), + (Op::MemGetInfo, Response::Pair(6 << 30, 8 << 30)), + (Op::StreamCreate, Response::Handle(21)), + (Op::EventCreate, Response::Handle(22)), + (Op::EventElapsedTime, Response::Millis(1.25)), + (Op::ModuleUnload, Response::Ok), + (Op::MemsetD8, Response::Ok), + ] { + let enc = encode_response(0, &resp); + let (status, dec) = decode_response(op, &enc).expect("decode"); + assert_eq!(status, 0); + assert_eq!(dec, resp); + } + } + + #[test] + fn response_roundtrips() { + for (op, resp) in [ + (Op::DeviceGetCount, Response::Count(2)), + ( + Op::DeviceGetName, + Response::Name("NVIDIA GeForce RTX 3070".into()), + ), + (Op::DeviceTotalMem, Response::Bytes(8 << 30)), + (Op::CtxCreate, Response::Handle(99)), + (Op::ModuleLoadData, Response::Handle(1)), + (Op::MemAlloc, Response::Dptr(0x7f00_0000)), + (Op::MemcpyDtoH, Response::Data(vec![9, 8, 7])), + (Op::CtxSynchronize, Response::Ok), + ] { + let enc = encode_response(0, &resp); + let (status, dec) = decode_response(op, &enc).expect("decode"); + assert_eq!(status, 0); + assert_eq!(dec, resp); + } + } + + #[test] + fn error_response_has_no_body() { + let enc = encode_response(700, &Response::Handle(123)); // CUDA_ERROR_* + let (status, body) = decode_response(Op::ModuleLoadData, &enc).unwrap(); + assert_eq!(status, 700); + assert_eq!(body, Response::Ok); // body omitted on error + assert_eq!(enc.len(), 4); // status only + } + + #[test] + fn framing_roundtrip_and_eof() { + let mut buf = Vec::new(); + let payload = encode_request(&Request::DeviceGetCount); + write_msg(&mut buf, &payload).unwrap(); + let mut cur = std::io::Cursor::new(buf); + let got = read_msg(&mut cur).unwrap().expect("frame"); + assert_eq!(got, payload); + // clean EOF at boundary + assert!(read_msg(&mut cur).unwrap().is_none()); + } + + #[test] + fn truncated_message_is_error_not_panic() { + // opcode says ModuleGetFunction but payload is truncated mid-string + let mut b = vec![Op::ModuleGetFunction as u8]; + b.extend_from_slice(&7u64.to_le_bytes()); // module + b.extend_from_slice(&100u64.to_le_bytes()); // claims 100-byte name… + b.extend_from_slice(b"short"); // …but only 5 bytes follow + assert!(decode_request(&b).is_err()); + } +} diff --git a/crates/smolvm-cuda/src/ring.rs b/crates/smolvm-cuda/src/ring.rs new file mode 100644 index 0000000..2bd9e8f --- /dev/null +++ b/crates/smolvm-cuda/src/ring.rs @@ -0,0 +1,303 @@ +//! Shared-memory command/completion rings: the low-latency transport between +//! a guest shim and the host CUDA service. +//! +//! The guest allocates two rings in its own RAM (request: guest→host, +//! completion: host→guest) and hands their guest-physical page lists to the +//! host over the bootstrap vsock connection (`Request::RingSetup`). The host +//! already maps guest RAM for zero-copy transfers, so both sides then share +//! the rings at memory speed: a sync round-trip costs ~1-2µs of polling +//! instead of a vsock wakeup (~50-70µs). +//! +//! Layout (one ring): a 64-byte header page-prefix followed by `capacity` +//! fixed-size records. Records never straddle a page boundary +//! (`RECORD_SIZE` divides the page size), because guest pages are physically +//! discontiguous — each side addresses record `i` through a per-page mapping +//! table, not one flat pointer. +//! +//! Record publish protocol: the producer writes payload then `len`, then +//! stores the slot's wrapping sequence number with release ordering; the +//! consumer reads the sequence with acquire ordering and only then the +//! payload — a torn read is impossible because a slot's sequence only +//! becomes valid after its bytes are complete. +//! +//! Doorbells: pure polling burns a core, so each side may park. A consumer +//! that decides to sleep sets `PARKED` in its ring's header *then re-checks +//! for new records* (the producer's publish-then-check mirrors it) and blocks +//! on the bootstrap vsock socket; a producer that sees `PARKED` after +//! publishing clears it and writes one byte to the socket. The socket carries +//! only doorbells after ring setup — every request and response rides the +//! rings, preserving the single program-ordered queue. + +use std::sync::atomic::{AtomicU32, Ordering}; + +/// Bytes per record. Divides 4096 so records never straddle a guest page. +/// Sized so common frames stay inline: a Triton launch with ~30 pointer args +/// serializes to ~300 bytes, cudnn attribute blobs run to ~1 KiB. +pub const RECORD_SIZE: usize = 1024; +/// Per-record header: sequence (u32) + payload length (u32). +pub const RECORD_HDR: usize = 8; +/// Largest payload that fits inline in one record. +pub const INLINE_MAX: usize = RECORD_SIZE - RECORD_HDR; +/// Ring header size (one cache line, at the start of the first page). +pub const HEADER_SIZE: usize = 64; + +/// Consumer-parked flag in the header `flags` word: the producer must clear +/// it and kick the doorbell after publishing. +pub const FLAG_PARKED: u32 = 1; + +/// Payload `len` flag bit: the record body is not the payload itself but a +/// list of `(gpa: u64, len: u64)` segments the consumer must gather from +/// guest RAM (requests too large for a record — H2D byte-ship, fatbins). +pub const LEN_INDIRECT: u32 = 1 << 31; + +/// Offsets within the 64-byte header. +const OFF_HEAD: usize = 0; // producer: next sequence to publish (u32) +const OFF_TAIL: usize = 8; // consumer: next sequence to consume (u32) +const OFF_FLAGS: usize = 16; // FLAG_PARKED etc. + +/// One side of a ring: raw per-page pointers into the shared memory. The +/// guest builds this over its own allocation; the host over its mapping of +/// the same guest pages. `pages[0]` begins with the header; records start at +/// `HEADER_SIZE` and continue across pages (each page after the first is +/// wall-to-wall records). +pub struct Ring { + pages: Vec<*mut u8>, + page_size: usize, + capacity: u32, +} + +// SAFETY: the ring is shared memory by design; all cross-thread access goes +// through atomics with acquire/release ordering per the publish protocol. +unsafe impl Send for Ring {} +unsafe impl Sync for Ring {} + +impl Ring { + /// Records that fit in `pages` (first page loses `HEADER_SIZE` bytes). + pub fn capacity_for(num_pages: usize, page_size: usize) -> u32 { + let first = (page_size - HEADER_SIZE) / RECORD_SIZE; + let rest = (num_pages - 1) * (page_size / RECORD_SIZE); + (first + rest) as u32 + } + + /// Wrap `pages` (each `page_size` bytes, zeroed by the creator). + /// + /// # Safety + /// Every pointer must be valid for `page_size` bytes for the ring's + /// lifetime, and the memory must not be moved or unmapped. + pub unsafe fn from_pages(pages: Vec<*mut u8>, page_size: usize) -> Ring { + let capacity = Self::capacity_for(pages.len(), page_size); + Ring { + pages, + page_size, + capacity, + } + } + + pub fn capacity(&self) -> u32 { + self.capacity + } + + fn header_atomic(&self, off: usize) -> &AtomicU32 { + // SAFETY: header lives in the first page, within HEADER_SIZE. + unsafe { AtomicU32::from_ptr(self.pages[0].add(off) as *mut u32) } + } + + /// (page pointer, offset) of record `slot`. + fn record_ptr(&self, slot: u32) -> *mut u8 { + let first_cap = (self.page_size - HEADER_SIZE) / RECORD_SIZE; + let slot = slot as usize; + if slot < first_cap { + // SAFETY: slot bounds-checked against the first page's capacity. + unsafe { self.pages[0].add(HEADER_SIZE + slot * RECORD_SIZE) } + } else { + let per_page = self.page_size / RECORD_SIZE; + let rest = slot - first_cap; + let page = 1 + rest / per_page; + // SAFETY: capacity_for bounds slots to the page list. + unsafe { self.pages[page].add((rest % per_page) * RECORD_SIZE) } + } + } + + /// Producer: try to publish `payload` (with `flags` OR-ed into its + /// length). Returns false when the ring is full. + pub fn try_push(&self, payload: &[u8], flags: u32) -> bool { + debug_assert!(payload.len() <= INLINE_MAX); + let head = self.header_atomic(OFF_HEAD); + let tail = self.header_atomic(OFF_TAIL); + let seq = head.load(Ordering::Relaxed); + if seq.wrapping_sub(tail.load(Ordering::Acquire)) >= self.capacity { + return false; // full + } + let slot = seq % self.capacity; + let rec = self.record_ptr(slot); + // SAFETY: rec points at a full RECORD_SIZE record inside our pages; + // the slot is ours until we bump `head` (single producer). + unsafe { + std::ptr::copy_nonoverlapping(payload.as_ptr(), rec.add(RECORD_HDR), payload.len()); + (rec.add(4) as *mut u32).write_volatile(payload.len() as u32 | flags); + // Publish: sequence write is the release gate for the payload. + AtomicU32::from_ptr(rec as *mut u32).store(seq.wrapping_add(1), Ordering::Release); + } + head.store(seq.wrapping_add(1), Ordering::Release); + true + } + + /// Consumer: read the next record if one is published. Returns + /// `(payload, flags)`. + pub fn try_pop(&self) -> Option<(Vec, u32)> { + let tail = self.header_atomic(OFF_TAIL); + let seq = tail.load(Ordering::Relaxed); + let slot = seq % self.capacity; + let rec = self.record_ptr(slot); + // SAFETY: reading the slot's sequence atomically; payload only after + // the acquire load observes the publish. + let published = unsafe { AtomicU32::from_ptr(rec as *mut u32).load(Ordering::Acquire) }; + if published != seq.wrapping_add(1) { + return None; + } + let (len_flags, payload) = unsafe { + let lf = (rec.add(4) as *const u32).read_volatile(); + let len = (lf & !LEN_INDIRECT) as usize; + let mut buf = vec![0u8; len.min(INLINE_MAX)]; + std::ptr::copy_nonoverlapping(rec.add(RECORD_HDR), buf.as_mut_ptr(), buf.len()); + (lf, buf) + }; + tail.store(seq.wrapping_add(1), Ordering::Release); + Some((payload, len_flags & LEN_INDIRECT)) + } + + /// Consumer: about to block — set the parked flag. Returns true if new + /// records were published in the meantime (caller must NOT sleep). + pub fn park(&self) -> bool { + self.header_atomic(OFF_FLAGS) + .fetch_or(FLAG_PARKED, Ordering::SeqCst); + // Re-check after the flag is visible: a producer that published + // before seeing the flag would not kick. + let head = self.header_atomic(OFF_HEAD).load(Ordering::SeqCst); + let tail = self.header_atomic(OFF_TAIL).load(Ordering::Relaxed); + if head != tail { + self.unpark(); + return true; + } + false + } + + pub fn unpark(&self) { + self.header_atomic(OFF_FLAGS) + .fetch_and(!FLAG_PARKED, Ordering::SeqCst); + } + + /// Producer: does the consumer need a doorbell kick? (Checked after + /// publishing; clears the flag when set so exactly one kick is sent.) + pub fn take_parked(&self) -> bool { + let flags = self.header_atomic(OFF_FLAGS); + if flags.load(Ordering::SeqCst) & FLAG_PARKED != 0 { + flags.fetch_and(!FLAG_PARKED, Ordering::SeqCst) & FLAG_PARKED != 0 + } else { + false + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_ring(num_pages: usize) -> (Ring, Vec>) { + let page = 4096; + let mut backing: Vec> = (0..num_pages).map(|_| vec![0u8; page]).collect(); + let pages: Vec<*mut u8> = backing.iter_mut().map(|p| p.as_mut_ptr()).collect(); + // SAFETY: backing outlives the ring within each test. + (unsafe { Ring::from_pages(pages, page) }, backing) + } + + #[test] + fn push_pop_roundtrip() { + let (ring, _keep) = make_ring(2); + assert!(ring.try_push(b"hello", 0)); + assert!(ring.try_push(b"world", LEN_INDIRECT)); + let (a, f_a) = ring.try_pop().unwrap(); + let (b, f_b) = ring.try_pop().unwrap(); + assert_eq!((a.as_slice(), f_a), (b"hello".as_slice(), 0)); + assert_eq!((b.as_slice(), f_b), (b"world".as_slice(), LEN_INDIRECT)); + assert!(ring.try_pop().is_none()); + } + + #[test] + fn fills_and_wraps() { + let (ring, _keep) = make_ring(1); + let cap = ring.capacity(); + for i in 0..cap { + assert!(ring.try_push(&[i as u8], 0), "push {i}"); + } + assert!(!ring.try_push(b"x", 0), "must report full"); + for i in 0..cap { + assert_eq!(ring.try_pop().unwrap().0, vec![i as u8]); + } + // Wrapped sequences keep working past capacity. + for round in 0..3u32 { + assert!(ring.try_push(&round.to_le_bytes(), 0)); + assert_eq!(ring.try_pop().unwrap().0, round.to_le_bytes()); + } + } + + #[test] + fn record_slots_never_straddle_pages() { + let (ring, keep) = make_ring(3); + let page = 4096; + for slot in 0..ring.capacity() { + let p = ring.record_ptr(slot) as usize; + let off_in_page = keep + .iter() + .map(|b| b.as_ptr() as usize) + .filter(|&base| p >= base && p < base + page) + .map(|base| p - base) + .next() + .expect("record outside all pages"); + assert!(off_in_page + RECORD_SIZE <= page, "slot {slot} straddles"); + } + } + + #[test] + fn park_sees_concurrent_publish() { + let (ring, _keep) = make_ring(1); + assert!(!ring.park(), "empty ring parks"); + ring.unpark(); + assert!(ring.try_push(b"x", 0)); + assert!(ring.park(), "publish before park must cancel the sleep"); + // Producer-side doorbell handoff. + assert!(ring.try_pop().is_some()); + assert!(!ring.park()); + assert!(ring.try_push(b"y", 0)); + assert!(ring.take_parked(), "producer collects exactly one kick"); + assert!(!ring.take_parked()); + } + + #[test] + fn cross_thread_stream() { + use std::sync::Arc; + let (ring, keep) = make_ring(4); + let ring = Arc::new(ring); + let producer = { + let ring = Arc::clone(&ring); + std::thread::spawn(move || { + for i in 0..10_000u32 { + while !ring.try_push(&i.to_le_bytes(), 0) { + std::hint::spin_loop(); + } + } + }) + }; + let mut expected = 0u32; + while expected < 10_000 { + if let Some((payload, _)) = ring.try_pop() { + assert_eq!(payload, expected.to_le_bytes()); + expected += 1; + } else { + std::hint::spin_loop(); + } + } + producer.join().unwrap(); + drop(keep); + } +} diff --git a/crates/smolvm-cuda/src/shm.rs b/crates/smolvm-cuda/src/shm.rs new file mode 100644 index 0000000..61fea30 --- /dev/null +++ b/crates/smolvm-cuda/src/shm.rs @@ -0,0 +1,130 @@ +//! A named POSIX shared-memory region, mapped by both the guest shim and the +//! host CUDA server, used as a **zero-copy bulk-data channel**. +//! +//! Bulk `cudaMemcpy` is the bandwidth bottleneck of socket-based remoting: the +//! bytes are copied guest→socket→host→GPU. With a region both sides map, a +//! host-pinned/`cudaMallocHost` buffer lives *in* the region, and a memcpy +//! ships only an `(offset, size)` descriptor — the host reads the bytes +//! straight from the shared mapping and DMAs them to the GPU. No bulk bytes +//! cross the socket. +//! +//! Same-host today (guest shim + host server share `/dev/shm/`). The +//! microVM case backs the *same protocol* with guest RAM the host already maps +//! (a libkrun API to expose the guest-memory base; see docs). +//! +//! Selected by `SMOLVM_CUDA_SHM=` (+ `SMOLVM_CUDA_SHM_SIZE`, default +//! 512 MiB). Unset → the region is absent and callers fall back to byte-shipping. + +use std::ffi::CString; +use std::sync::OnceLock; + +/// A mapped shared region: base pointer + length. `Send`/`Sync` because it is a +/// process-lifetime mapping accessed under the server's per-connection ordering. +pub struct ShmRegion { + base: *mut u8, + len: usize, +} +// SAFETY: the mapping lives for the whole process; concurrent access is +// serialized by the CUDA server's single-threaded-per-connection model and, on +// the guest, by the client mutex. +unsafe impl Send for ShmRegion {} +unsafe impl Sync for ShmRegion {} + +impl ShmRegion { + pub fn base(&self) -> *mut u8 { + self.base + } + pub fn len(&self) -> usize { + self.len + } + pub fn is_empty(&self) -> bool { + self.len == 0 + } + /// A checked slice `[offset, offset+size)` into the region, or `None` if out + /// of bounds (guards a hostile/buggy offset). + pub fn checked(&self, offset: u64, size: u64) -> Option<*mut u8> { + let end = offset.checked_add(size)?; + if end > self.len as u64 { + return None; + } + Some(unsafe { self.base.add(offset as usize) }) + } +} + +fn shm_size() -> usize { + std::env::var("SMOLVM_CUDA_SHM_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(512 * 1024 * 1024) +} + +/// Open (server: `create=true`) or attach (`create=false`) the region named by +/// `SMOLVM_CUDA_SHM`. Returns `None` if the env var is unset (no shm channel). +pub fn open(create: bool) -> Option { + let trace = std::env::var_os("SMOLVM_CUDA_SHM_TRACE").is_some(); + let name = match std::env::var("SMOLVM_CUDA_SHM") { + Ok(n) => n, + Err(_) => { + if trace { + eprintln!("shm: SMOLVM_CUDA_SHM not set (create={create})"); + } + return None; + } + }; + if trace { + eprintln!("shm: open name={name:?} create={create}"); + } + let len = shm_size(); + let cname = CString::new(name).ok()?; + unsafe { + let mut flags = libc::O_RDWR; + if create { + flags |= libc::O_CREAT; + } + let fd = libc::shm_open(cname.as_ptr(), flags, 0o600); + if fd < 0 { + if trace { + eprintln!( + "shm: shm_open failed errno={}", + std::io::Error::last_os_error() + ); + } + return None; + } + if create && libc::ftruncate(fd, len as libc::off_t) != 0 { + if trace { + eprintln!("shm: ftruncate failed"); + } + libc::close(fd); + return None; + } + let base = libc::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + fd, + 0, + ); + libc::close(fd); // the mapping keeps the region alive + if base == libc::MAP_FAILED { + return None; + } + Some(ShmRegion { + base: base as *mut u8, + len, + }) + } +} + +/// Process-wide handle, opened once on first use (attach mode). +pub fn get() -> Option<&'static ShmRegion> { + static REGION: OnceLock> = OnceLock::new(); + REGION.get_or_init(|| open(false)).as_ref() +} + +/// Server-side: create-or-attach the region once. +pub fn get_or_create() -> Option<&'static ShmRegion> { + static REGION: OnceLock> = OnceLock::new(); + REGION.get_or_init(|| open(true)).as_ref() +} diff --git a/crates/smolvm-cudart-shim/Cargo.toml b/crates/smolvm-cudart-shim/Cargo.toml new file mode 100644 index 0000000..bac1c7f --- /dev/null +++ b/crates/smolvm-cudart-shim/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "smolvm-cudart-shim" +version = "1.5.2" +edition = "2021" +description = "Guest-side libcudart.so drop-in: CUDA Runtime API lowered to smolvm's Driver-API vsock RPC" +license = "Apache-2.0" + +[lib] +name = "cudart" +crate-type = ["cdylib"] + +[dependencies] +smolvm-cuda = { path = "../smolvm-cuda", default-features = false } + +[target.'cfg(target_os = "linux")'.dependencies] +vsock = "0.5" +libc = "0.2" diff --git a/crates/smolvm-cudart-shim/build.rs b/crates/smolvm-cudart-shim/build.rs new file mode 100644 index 0000000..caa246f --- /dev/null +++ b/crates/smolvm-cudart-shim/build.rs @@ -0,0 +1,9 @@ +fn main() { + // Announce the soname a CUDA 11.x program links against, so the dynamic + // linker satisfies `libcudart.so.11.0`. Build artifact is libcudart.so; + // installers stage it as libcudart.so.11.0 (or LD_PRELOAD it). + let target = std::env::var("TARGET").unwrap_or_default(); + if target.contains("linux") { + println!("cargo:rustc-cdylib-link-arg=-Wl,-soname,libcudart.so.11.0"); + } +} diff --git a/crates/smolvm-cudart-shim/src/cublas_stubs.rs b/crates/smolvm-cudart-shim/src/cublas_stubs.rs new file mode 100644 index 0000000..ad134bf --- /dev/null +++ b/crates/smolvm-cudart-shim/src/cublas_stubs.rs @@ -0,0 +1,912 @@ +// @generated by smolvm nm-based stub generation — do not edit. +//! Auto-generated stub exports for cuBLAS/cuBLASLt symbols PyTorch links but +//! that aren't yet forwarded. Each logs under SMOLVM_CUDA_SHIM_TRACE and +//! returns CUBLAS_STATUS_NOT_SUPPORTED so we can trace what torch actually calls. +#![allow(non_snake_case)] +use std::os::raw::c_int; +fn stub(name: &str) -> c_int { + if std::env::var_os("SMOLVM_CUDA_SHIM_TRACE").is_some() { + eprintln!("[cublas-stub] CALLED (unimplemented): {name}"); + } + 15 +} +// cublasLt* are forwarded for real in lib.rs (LIB_CUBLASLT). +#[no_mangle] +pub extern "C" fn cublasCdotc_v2() -> c_int { + stub("cublasCdotc_v2") +} +#[no_mangle] +pub extern "C" fn cublasCdotu_v2() -> c_int { + stub("cublasCdotu_v2") +} +#[no_mangle] +pub extern "C" fn cublasCgelsBatched() -> c_int { + stub("cublasCgelsBatched") +} +#[no_mangle] +pub extern "C" fn cublasCgemmStridedBatched() -> c_int { + stub("cublasCgemmStridedBatched") +} +#[no_mangle] +pub extern "C" fn cublasCgemm_v2() -> c_int { + stub("cublasCgemm_v2") +} +#[no_mangle] +pub extern "C" fn cublasCgemv_v2() -> c_int { + stub("cublasCgemv_v2") +} +#[no_mangle] +pub extern "C" fn cublasCgeqrfBatched() -> c_int { + stub("cublasCgeqrfBatched") +} +#[no_mangle] +pub extern "C" fn cublasCgetrfBatched() -> c_int { + stub("cublasCgetrfBatched") +} +#[no_mangle] +pub extern "C" fn cublasCgetrsBatched() -> c_int { + stub("cublasCgetrsBatched") +} +#[no_mangle] +pub extern "C" fn cublasCtrsmBatched() -> c_int { + stub("cublasCtrsmBatched") +} +#[no_mangle] +pub extern "C" fn cublasCtrsm_v2() -> c_int { + stub("cublasCtrsm_v2") +} +#[no_mangle] +pub extern "C" fn cublasDdot_v2() -> c_int { + stub("cublasDdot_v2") +} +#[no_mangle] +pub extern "C" fn cublasDgelsBatched() -> c_int { + stub("cublasDgelsBatched") +} +#[no_mangle] +pub extern "C" fn cublasDgeqrfBatched() -> c_int { + stub("cublasDgeqrfBatched") +} +#[no_mangle] +pub extern "C" fn cublasDgetrfBatched() -> c_int { + stub("cublasDgetrfBatched") +} +#[no_mangle] +pub extern "C" fn cublasDgetrsBatched() -> c_int { + stub("cublasDgetrsBatched") +} +#[no_mangle] +pub extern "C" fn cublasDotEx() -> c_int { + stub("cublasDotEx") +} +#[no_mangle] +pub extern "C" fn cublasDtrsmBatched() -> c_int { + stub("cublasDtrsmBatched") +} +#[no_mangle] +pub extern "C" fn cublasGetPointerMode_v2() -> c_int { + stub("cublasGetPointerMode_v2") +} +#[no_mangle] +pub extern "C" fn cublasSdot_v2() -> c_int { + stub("cublasSdot_v2") +} +#[no_mangle] +pub extern "C" fn cublasSetPointerMode_v2() -> c_int { + stub("cublasSetPointerMode_v2") +} +#[no_mangle] +pub extern "C" fn cublasSgelsBatched() -> c_int { + stub("cublasSgelsBatched") +} +#[no_mangle] +pub extern "C" fn cublasSgemmEx() -> c_int { + stub("cublasSgemmEx") +} +#[no_mangle] +pub extern "C" fn cublasSgeqrfBatched() -> c_int { + stub("cublasSgeqrfBatched") +} +#[no_mangle] +pub extern "C" fn cublasSgetrfBatched() -> c_int { + stub("cublasSgetrfBatched") +} +#[no_mangle] +pub extern "C" fn cublasSgetrsBatched() -> c_int { + stub("cublasSgetrsBatched") +} +#[no_mangle] +pub extern "C" fn cublasStrsmBatched() -> c_int { + stub("cublasStrsmBatched") +} +#[no_mangle] +pub extern "C" fn cublasZdotc_v2() -> c_int { + stub("cublasZdotc_v2") +} +#[no_mangle] +pub extern "C" fn cublasZdotu_v2() -> c_int { + stub("cublasZdotu_v2") +} +#[no_mangle] +pub extern "C" fn cublasZgelsBatched() -> c_int { + stub("cublasZgelsBatched") +} +#[no_mangle] +pub extern "C" fn cublasZgemmStridedBatched() -> c_int { + stub("cublasZgemmStridedBatched") +} +#[no_mangle] +pub extern "C" fn cublasZgemm_v2() -> c_int { + stub("cublasZgemm_v2") +} +#[no_mangle] +pub extern "C" fn cublasZgemv_v2() -> c_int { + stub("cublasZgemv_v2") +} +#[no_mangle] +pub extern "C" fn cublasZgeqrfBatched() -> c_int { + stub("cublasZgeqrfBatched") +} +#[no_mangle] +pub extern "C" fn cublasZgetrfBatched() -> c_int { + stub("cublasZgetrfBatched") +} +#[no_mangle] +pub extern "C" fn cublasZgetrsBatched() -> c_int { + stub("cublasZgetrsBatched") +} +#[no_mangle] +pub extern "C" fn cublasZtrsmBatched() -> c_int { + stub("cublasZtrsmBatched") +} +#[no_mangle] +pub extern "C" fn cublasZtrsm_v2() -> c_int { + stub("cublasZtrsm_v2") +} +#[no_mangle] +pub extern "C" fn cudnn_batch_norm4callERKNS_6TensorES4_RKSt8optionalIS2_ES8_S8_bdd() -> c_int { + stub("cudnn_batch_norm4callERKNS_6TensorES4_RKSt8optionalIS2_ES8_S8_bdd") +} +#[no_mangle] +pub extern "C" fn cudnn_batch_norm_backward4callERKNS_6TensorES4_S4_RKSt8optionalIS2_ES8_S8_S8_dS4_( +) -> c_int { + stub("cudnn_batch_norm_backward4callERKNS_6TensorES4_S4_RKSt8optionalIS2_ES8_S8_S8_dS4_") +} +#[no_mangle] +pub extern "C" fn cudnn_batch_norm_out4callERKNS_6TensorES4_RKSt8optionalIS2_ES8_S8_bddRS2_S9_S9_S9_( +) -> c_int { + stub("cudnn_batch_norm_out4callERKNS_6TensorES4_RKSt8optionalIS2_ES8_S8_bddRS2_S9_S9_S9_") +} +#[no_mangle] +pub extern "C" fn cudnnConvolutionBackwardData() -> c_int { + stub("cudnnConvolutionBackwardData") +} +#[no_mangle] +pub extern "C" fn cudnnConvolutionBackwardFilter() -> c_int { + stub("cudnnConvolutionBackwardFilter") +} +#[no_mangle] +pub extern "C" fn cudnn_convolution_backward_stubE() -> c_int { + stub("cudnn_convolution_backward_stubE") +} +#[no_mangle] +pub extern "C" fn cudnnConvolutionBiasActivationForward() -> c_int { + stub("cudnnConvolutionBiasActivationForward") +} +#[no_mangle] +pub extern "C" fn cudnn_convolution_transpose_backward_stubE() -> c_int { + stub("cudnn_convolution_transpose_backward_stubE") +} +#[no_mangle] +pub extern "C" fn cudnnCreateActivationDescriptor() -> c_int { + stub("cudnnCreateActivationDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnCreateCTCLossDescriptor() -> c_int { + stub("cudnnCreateCTCLossDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnCreateDropoutDescriptor() -> c_int { + stub("cudnnCreateDropoutDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnCreatePoolingDescriptor() -> c_int { + stub("cudnnCreatePoolingDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnCreateRNNDataDescriptor() -> c_int { + stub("cudnnCreateRNNDataDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnCreateRNNDescriptor() -> c_int { + stub("cudnnCreateRNNDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnCreateSpatialTransformerDescriptor() -> c_int { + stub("cudnnCreateSpatialTransformerDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnCTCLoss() -> c_int { + stub("cudnnCTCLoss") +} +#[no_mangle] +pub extern "C" fn cudnnCTCLoss_v8() -> c_int { + stub("cudnnCTCLoss_v8") +} +#[no_mangle] +pub extern "C" fn cudnnDestroyActivationDescriptor() -> c_int { + stub("cudnnDestroyActivationDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnDestroyCTCLossDescriptor() -> c_int { + stub("cudnnDestroyCTCLossDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnDestroyDropoutDescriptor() -> c_int { + stub("cudnnDestroyDropoutDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnDestroyRNNDataDescriptor() -> c_int { + stub("cudnnDestroyRNNDataDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnDestroyRNNDescriptor() -> c_int { + stub("cudnnDestroyRNNDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnDestroySpatialTransformerDescriptor() -> c_int { + stub("cudnnDestroySpatialTransformerDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnDropoutGetStatesSize() -> c_int { + stub("cudnnDropoutGetStatesSize") +} +#[no_mangle] +pub extern "C" fn cudnnFindConvolutionBackwardDataAlgorithmEx() -> c_int { + stub("cudnnFindConvolutionBackwardDataAlgorithmEx") +} +#[no_mangle] +pub extern "C" fn cudnnFindConvolutionBackwardFilterAlgorithmEx() -> c_int { + stub("cudnnFindConvolutionBackwardFilterAlgorithmEx") +} +#[no_mangle] +pub extern "C" fn cudnnFindConvolutionForwardAlgorithmEx() -> c_int { + stub("cudnnFindConvolutionForwardAlgorithmEx") +} +#[no_mangle] +pub extern "C" fn cudnn_get_conv_benchmark_empty_cacheEv() -> c_int { + stub("cudnn_get_conv_benchmark_empty_cacheEv") +} +#[no_mangle] +pub extern "C" fn cudnnGetConvolutionBackwardDataAlgorithm_v7() -> c_int { + stub("cudnnGetConvolutionBackwardDataAlgorithm_v7") +} +#[no_mangle] +pub extern "C" fn cudnnGetConvolutionBackwardDataWorkspaceSize() -> c_int { + stub("cudnnGetConvolutionBackwardDataWorkspaceSize") +} +#[no_mangle] +pub extern "C" fn cudnnGetConvolutionBackwardFilterAlgorithm_v7() -> c_int { + stub("cudnnGetConvolutionBackwardFilterAlgorithm_v7") +} +#[no_mangle] +pub extern "C" fn cudnnGetConvolutionBackwardFilterWorkspaceSize() -> c_int { + stub("cudnnGetConvolutionBackwardFilterWorkspaceSize") +} +#[no_mangle] +pub extern "C" fn cudnnGetConvolutionForwardAlgorithm_v7() -> c_int { + stub("cudnnGetConvolutionForwardAlgorithm_v7") +} +#[no_mangle] +pub extern "C" fn cudnnGetCTCLossWorkspaceSize() -> c_int { + stub("cudnnGetCTCLossWorkspaceSize") +} +#[no_mangle] +pub extern "C" fn cudnnGetCTCLossWorkspaceSize_v8() -> c_int { + stub("cudnnGetCTCLossWorkspaceSize_v8") +} +#[no_mangle] +pub extern "C" fn cudnnGetCudartVersion() -> c_int { + stub("cudnnGetCudartVersion") +} +#[no_mangle] +pub extern "C" fn cudnnGetFilterNdDescriptor() -> c_int { + stub("cudnnGetFilterNdDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnGetLastErrorString() -> c_int { + stub("cudnnGetLastErrorString") +} +#[no_mangle] +pub extern "C" fn cudnnGetRNNTempSpaceSizes() -> c_int { + stub("cudnnGetRNNTempSpaceSizes") +} +#[no_mangle] +pub extern "C" fn cudnnGetRNNWeightParams() -> c_int { + stub("cudnnGetRNNWeightParams") +} +#[no_mangle] +pub extern "C" fn cudnnGetRNNWeightSpaceSize() -> c_int { + stub("cudnnGetRNNWeightSpaceSize") +} +#[no_mangle] +pub extern "C" fn cudnnGetStream() -> c_int { + stub("cudnnGetStream") +} +#[no_mangle] +pub extern "C" fn cudnnGetTensorNdDescriptor() -> c_int { + stub("cudnnGetTensorNdDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnGetVersion() -> c_int { + stub("cudnnGetVersion") +} +#[no_mangle] +pub extern "C" fn cudnn_init_dropout_state4callEdblSt8optionalIN3c1010ScalarTypeEES2_INS3_6LayoutEES2_INS3_6DeviceEES2_IbE( +) -> c_int { + stub("cudnn_init_dropout_state4callEdblSt8optionalIN3c1010ScalarTypeEES2_INS3_6LayoutEES2_INS3_6DeviceEES2_IbE") +} +#[no_mangle] +pub extern "C" fn cudnn_is_acceptableERKNS_10TensorBaseE() -> c_int { + stub("cudnn_is_acceptableERKNS_10TensorBaseE") +} +#[no_mangle] +pub extern "C" fn cudnnPoolingForward() -> c_int { + stub("cudnnPoolingForward") +} +#[no_mangle] +pub extern "C" fn cudnnRestoreDropoutDescriptor() -> c_int { + stub("cudnnRestoreDropoutDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnn_rnn4callERKNS_6TensorEN3c108ArrayRefIS2_EElRKSt8optionalIS2_ES4_SB_lNS5_6SymIntESC_lbdbbNS6_ISC_EESB_( +) -> c_int { + stub("cudnn_rnn4callERKNS_6TensorEN3c108ArrayRefIS2_EElRKSt8optionalIS2_ES4_SB_lNS5_6SymIntESC_lbdbbNS6_ISC_EESB_") +} +#[no_mangle] +pub extern "C" fn cudnnRNNBackwardData_v8() -> c_int { + stub("cudnnRNNBackwardData_v8") +} +#[no_mangle] +pub extern "C" fn cudnnRNNBackwardWeights_v8() -> c_int { + stub("cudnnRNNBackwardWeights_v8") +} +#[no_mangle] +pub extern "C" fn cudnnRNNForward() -> c_int { + stub("cudnnRNNForward") +} +#[no_mangle] +pub extern "C" fn cudnnSetActivationDescriptor() -> c_int { + stub("cudnnSetActivationDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnSetConvolutionGroupCount() -> c_int { + stub("cudnnSetConvolutionGroupCount") +} +#[no_mangle] +pub extern "C" fn cudnnSetConvolutionMathType() -> c_int { + stub("cudnnSetConvolutionMathType") +} +#[no_mangle] +pub extern "C" fn cudnnSetConvolutionNdDescriptor() -> c_int { + stub("cudnnSetConvolutionNdDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnSetCTCLossDescriptorEx() -> c_int { + stub("cudnnSetCTCLossDescriptorEx") +} +#[no_mangle] +pub extern "C" fn cudnnSetCTCLossDescriptor_v9() -> c_int { + stub("cudnnSetCTCLossDescriptor_v9") +} +#[no_mangle] +pub extern "C" fn cudnnSetDropoutDescriptor() -> c_int { + stub("cudnnSetDropoutDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnSetFilterNdDescriptor() -> c_int { + stub("cudnnSetFilterNdDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnSetPooling2dDescriptor() -> c_int { + stub("cudnnSetPooling2dDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnSetRNNDataDescriptor() -> c_int { + stub("cudnnSetRNNDataDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnSetRNNDescriptor_v8() -> c_int { + stub("cudnnSetRNNDescriptor_v8") +} +#[no_mangle] +pub extern "C" fn cudnnSetSpatialTransformerNdDescriptor() -> c_int { + stub("cudnnSetSpatialTransformerNdDescriptor") +} +#[no_mangle] +pub extern "C" fn cudnnSpatialTfGridGeneratorBackward() -> c_int { + stub("cudnnSpatialTfGridGeneratorBackward") +} +#[no_mangle] +pub extern "C" fn cudnnSpatialTfGridGeneratorForward() -> c_int { + stub("cudnnSpatialTfGridGeneratorForward") +} +#[no_mangle] +pub extern "C" fn cudnnSpatialTfSamplerBackward() -> c_int { + stub("cudnnSpatialTfSamplerBackward") +} +#[no_mangle] +pub extern "C" fn cudnnSpatialTfSamplerForward() -> c_int { + stub("cudnnSpatialTfSamplerForward") +} +#[no_mangle] +pub extern "C" fn cudnn_stubE() -> c_int { + stub("cudnn_stubE") +} + +fn rt_stub(name: &str) -> c_int { + if std::env::var_os("SMOLVM_CUDA_SHIM_TRACE").is_some() { + eprintln!("[cudart-stub] CALLED: {name}"); + } + 801 // cudaErrorNotSupported +} +#[no_mangle] +pub extern "C" fn cudaGraphDebugDotPrint() -> c_int { + rt_stub("cudaGraphDebugDotPrint") +} +#[no_mangle] +pub extern "C" fn cudaIpcCloseMemHandle() -> c_int { + rt_stub("cudaIpcCloseMemHandle") +} +#[no_mangle] +pub extern "C" fn cudaIpcGetEventHandle() -> c_int { + rt_stub("cudaIpcGetEventHandle") +} +#[no_mangle] +pub extern "C" fn cudaIpcGetMemHandle() -> c_int { + rt_stub("cudaIpcGetMemHandle") +} +#[no_mangle] +pub extern "C" fn cudaIpcOpenEventHandle() -> c_int { + rt_stub("cudaIpcOpenEventHandle") +} +#[no_mangle] +pub extern "C" fn cudaIpcOpenMemHandle() -> c_int { + rt_stub("cudaIpcOpenMemHandle") +} +#[no_mangle] +pub extern "C" fn cudaMemcpyPeerAsync() -> c_int { + rt_stub("cudaMemcpyPeerAsync") +} +#[no_mangle] +pub extern "C" fn cudaRegisterVar() -> c_int { + rt_stub("cudaRegisterVar") +} +/// `__cudaRegisterVar` — registers a __device__/__constant__ global. No-op: +/// forwarded workloads that don't use cudaMemcpyToSymbol never need the mapping. +#[no_mangle] +pub extern "C" fn __cudaRegisterVar( + _fat: *mut *mut std::os::raw::c_void, + _host_var: *mut std::os::raw::c_char, + _dev_addr: *mut std::os::raw::c_char, + _dev_name: *const std::os::raw::c_char, + _ext: c_int, + _size: usize, + _constant: c_int, + _global: c_int, +) { +} + +// ---- remaining cudart runtime functions PyTorch links ----------------------- +use std::os::raw::c_void; +#[no_mangle] +pub extern "C" fn cudaProfilerStart() -> c_int { + 0 +} +#[no_mangle] +pub extern "C" fn cudaProfilerStop() -> c_int { + 0 +} +#[no_mangle] +pub extern "C" fn cudaInitModule(_x: *mut c_void) -> c_int { + 0 +} +#[no_mangle] +pub extern "C" fn cudaStreamGetPriority(_s: *mut c_void, priority: *mut c_int) -> c_int { + if !priority.is_null() { + unsafe { *priority = 0 } + } + 0 +} +#[no_mangle] +pub extern "C" fn cudaMemcpy2DAsync() -> c_int { + rt_stub("cudaMemcpy2DAsync") +} +#[no_mangle] +pub extern "C" fn cublasCaxpy_v2() -> c_int { + stub("cublasCaxpy_v2") +} +#[no_mangle] +pub extern "C" fn cublasCcopy_v2() -> c_int { + stub("cublasCcopy_v2") +} +#[no_mangle] +pub extern "C" fn cublasCgemmBatched() -> c_int { + stub("cublasCgemmBatched") +} +#[no_mangle] +pub extern "C" fn cublasCgerc_v2() -> c_int { + stub("cublasCgerc_v2") +} +#[no_mangle] +pub extern "C" fn cublasCgeru_v2() -> c_int { + stub("cublasCgeru_v2") +} +#[no_mangle] +pub extern "C" fn cublasChemm_v2() -> c_int { + stub("cublasChemm_v2") +} +#[no_mangle] +pub extern "C" fn cublasChemv_v2() -> c_int { + stub("cublasChemv_v2") +} +#[no_mangle] +pub extern "C" fn cublasCher2k_v2() -> c_int { + stub("cublasCher2k_v2") +} +#[no_mangle] +pub extern "C" fn cublasCher2_v2() -> c_int { + stub("cublasCher2_v2") +} +#[no_mangle] +pub extern "C" fn cublasCherk_v2() -> c_int { + stub("cublasCherk_v2") +} +#[no_mangle] +pub extern "C" fn cublasCher_v2() -> c_int { + stub("cublasCher_v2") +} +#[no_mangle] +pub extern "C" fn cublasCrotg_v2() -> c_int { + stub("cublasCrotg_v2") +} +#[no_mangle] +pub extern "C" fn cublasCrot_v2() -> c_int { + stub("cublasCrot_v2") +} +#[no_mangle] +pub extern "C" fn cublasCscal_v2() -> c_int { + stub("cublasCscal_v2") +} +#[no_mangle] +pub extern "C" fn cublasCsrot_v2() -> c_int { + stub("cublasCsrot_v2") +} +#[no_mangle] +pub extern "C" fn cublasCsscal_v2() -> c_int { + stub("cublasCsscal_v2") +} +#[no_mangle] +pub extern "C" fn cublasCswap_v2() -> c_int { + stub("cublasCswap_v2") +} +#[no_mangle] +pub extern "C" fn cublasCsymm_v2() -> c_int { + stub("cublasCsymm_v2") +} +#[no_mangle] +pub extern "C" fn cublasCsymv_v2() -> c_int { + stub("cublasCsymv_v2") +} +#[no_mangle] +pub extern "C" fn cublasCsyr2k_v2() -> c_int { + stub("cublasCsyr2k_v2") +} +#[no_mangle] +pub extern "C" fn cublasCsyr2_v2() -> c_int { + stub("cublasCsyr2_v2") +} +#[no_mangle] +pub extern "C" fn cublasCsyrk_v2() -> c_int { + stub("cublasCsyrk_v2") +} +#[no_mangle] +pub extern "C" fn cublasCsyr_v2() -> c_int { + stub("cublasCsyr_v2") +} +#[no_mangle] +pub extern "C" fn cublasCtrmm_v2() -> c_int { + stub("cublasCtrmm_v2") +} +#[no_mangle] +pub extern "C" fn cublasCtrmv_v2() -> c_int { + stub("cublasCtrmv_v2") +} +#[no_mangle] +pub extern "C" fn cublasCtrsv_v2() -> c_int { + stub("cublasCtrsv_v2") +} +#[no_mangle] +pub extern "C" fn cublasDasum_v2() -> c_int { + stub("cublasDasum_v2") +} +#[no_mangle] +pub extern "C" fn cublasDgemmBatched() -> c_int { + stub("cublasDgemmBatched") +} +#[no_mangle] +pub extern "C" fn cublasDnrm2_v2() -> c_int { + stub("cublasDnrm2_v2") +} +#[no_mangle] +pub extern "C" fn cublasDrotg_v2() -> c_int { + stub("cublasDrotg_v2") +} +#[no_mangle] +pub extern "C" fn cublasDrotmg_v2() -> c_int { + stub("cublasDrotmg_v2") +} +#[no_mangle] +pub extern "C" fn cublasDrotm_v2() -> c_int { + stub("cublasDrotm_v2") +} +#[no_mangle] +pub extern "C" fn cublasDrot_v2() -> c_int { + stub("cublasDrot_v2") +} +#[no_mangle] +pub extern "C" fn cublasDsymv_v2() -> c_int { + stub("cublasDsymv_v2") +} +#[no_mangle] +pub extern "C" fn cublasDsyr2k_v2() -> c_int { + stub("cublasDsyr2k_v2") +} +#[no_mangle] +pub extern "C" fn cublasDsyr2_v2() -> c_int { + stub("cublasDsyr2_v2") +} +#[no_mangle] +pub extern "C" fn cublasDsyr_v2() -> c_int { + stub("cublasDsyr_v2") +} +#[no_mangle] +pub extern "C" fn cublasDtrmm_v2() -> c_int { + stub("cublasDtrmm_v2") +} +#[no_mangle] +pub extern "C" fn cublasDtrmv_v2() -> c_int { + stub("cublasDtrmv_v2") +} +#[no_mangle] +pub extern "C" fn cublasDtrsv_v2() -> c_int { + stub("cublasDtrsv_v2") +} +#[no_mangle] +pub extern "C" fn cublasDzasum_v2() -> c_int { + stub("cublasDzasum_v2") +} +#[no_mangle] +pub extern "C" fn cublasDznrm2_v2() -> c_int { + stub("cublasDznrm2_v2") +} +#[no_mangle] +pub extern "C" fn cublasGetMatrixAsync() -> c_int { + stub("cublasGetMatrixAsync") +} +#[no_mangle] +pub extern "C" fn cublasGetVectorAsync() -> c_int { + stub("cublasGetVectorAsync") +} +#[no_mangle] +pub extern "C" fn cublasIcamax_v2() -> c_int { + stub("cublasIcamax_v2") +} +#[no_mangle] +pub extern "C" fn cublasIcamin_v2() -> c_int { + stub("cublasIcamin_v2") +} +#[no_mangle] +pub extern "C" fn cublasIdamax_v2() -> c_int { + stub("cublasIdamax_v2") +} +#[no_mangle] +pub extern "C" fn cublasIdamin_v2() -> c_int { + stub("cublasIdamin_v2") +} +#[no_mangle] +pub extern "C" fn cublasIsamax_v2() -> c_int { + stub("cublasIsamax_v2") +} +#[no_mangle] +pub extern "C" fn cublasIsamin_v2() -> c_int { + stub("cublasIsamin_v2") +} +#[no_mangle] +pub extern "C" fn cublasIzamax_v2() -> c_int { + stub("cublasIzamax_v2") +} +#[no_mangle] +pub extern "C" fn cublasIzamin_v2() -> c_int { + stub("cublasIzamin_v2") +} +#[no_mangle] +pub extern "C" fn cublasSasum_v2() -> c_int { + stub("cublasSasum_v2") +} +#[no_mangle] +pub extern "C" fn cublasScasum_v2() -> c_int { + stub("cublasScasum_v2") +} +#[no_mangle] +pub extern "C" fn cublasScnrm2_v2() -> c_int { + stub("cublasScnrm2_v2") +} +#[no_mangle] +pub extern "C" fn cublasSetMatrixAsync() -> c_int { + stub("cublasSetMatrixAsync") +} +#[no_mangle] +pub extern "C" fn cublasSetVectorAsync() -> c_int { + stub("cublasSetVectorAsync") +} +#[no_mangle] +pub extern "C" fn cublasSgemmBatched() -> c_int { + stub("cublasSgemmBatched") +} +#[no_mangle] +pub extern "C" fn cublasSnrm2_v2() -> c_int { + stub("cublasSnrm2_v2") +} +#[no_mangle] +pub extern "C" fn cublasSrotg_v2() -> c_int { + stub("cublasSrotg_v2") +} +#[no_mangle] +pub extern "C" fn cublasSrotmg_v2() -> c_int { + stub("cublasSrotmg_v2") +} +#[no_mangle] +pub extern "C" fn cublasSrotm_v2() -> c_int { + stub("cublasSrotm_v2") +} +#[no_mangle] +pub extern "C" fn cublasSrot_v2() -> c_int { + stub("cublasSrot_v2") +} +#[no_mangle] +pub extern "C" fn cublasSsymv_v2() -> c_int { + stub("cublasSsymv_v2") +} +#[no_mangle] +pub extern "C" fn cublasSsyr2k_v2() -> c_int { + stub("cublasSsyr2k_v2") +} +#[no_mangle] +pub extern "C" fn cublasSsyr2_v2() -> c_int { + stub("cublasSsyr2_v2") +} +#[no_mangle] +pub extern "C" fn cublasSsyr_v2() -> c_int { + stub("cublasSsyr_v2") +} +#[no_mangle] +pub extern "C" fn cublasStrmm_v2() -> c_int { + stub("cublasStrmm_v2") +} +#[no_mangle] +pub extern "C" fn cublasStrmv_v2() -> c_int { + stub("cublasStrmv_v2") +} +#[no_mangle] +pub extern "C" fn cublasStrsv_v2() -> c_int { + stub("cublasStrsv_v2") +} +#[no_mangle] +pub extern "C" fn cublasZaxpy_v2() -> c_int { + stub("cublasZaxpy_v2") +} +#[no_mangle] +pub extern "C" fn cublasZcopy_v2() -> c_int { + stub("cublasZcopy_v2") +} +#[no_mangle] +pub extern "C" fn cublasZdrot_v2() -> c_int { + stub("cublasZdrot_v2") +} +#[no_mangle] +pub extern "C" fn cublasZdscal_v2() -> c_int { + stub("cublasZdscal_v2") +} +#[no_mangle] +pub extern "C" fn cublasZgemmBatched() -> c_int { + stub("cublasZgemmBatched") +} +#[no_mangle] +pub extern "C" fn cublasZgerc_v2() -> c_int { + stub("cublasZgerc_v2") +} +#[no_mangle] +pub extern "C" fn cublasZgeru_v2() -> c_int { + stub("cublasZgeru_v2") +} +#[no_mangle] +pub extern "C" fn cublasZhemm_v2() -> c_int { + stub("cublasZhemm_v2") +} +#[no_mangle] +pub extern "C" fn cublasZhemv_v2() -> c_int { + stub("cublasZhemv_v2") +} +#[no_mangle] +pub extern "C" fn cublasZher2k_v2() -> c_int { + stub("cublasZher2k_v2") +} +#[no_mangle] +pub extern "C" fn cublasZher2_v2() -> c_int { + stub("cublasZher2_v2") +} +#[no_mangle] +pub extern "C" fn cublasZherk_v2() -> c_int { + stub("cublasZherk_v2") +} +#[no_mangle] +pub extern "C" fn cublasZher_v2() -> c_int { + stub("cublasZher_v2") +} +#[no_mangle] +pub extern "C" fn cublasZrotg_v2() -> c_int { + stub("cublasZrotg_v2") +} +#[no_mangle] +pub extern "C" fn cublasZrot_v2() -> c_int { + stub("cublasZrot_v2") +} +#[no_mangle] +pub extern "C" fn cublasZscal_v2() -> c_int { + stub("cublasZscal_v2") +} +#[no_mangle] +pub extern "C" fn cublasZswap_v2() -> c_int { + stub("cublasZswap_v2") +} +#[no_mangle] +pub extern "C" fn cublasZsymm_v2() -> c_int { + stub("cublasZsymm_v2") +} +#[no_mangle] +pub extern "C" fn cublasZsymv_v2() -> c_int { + stub("cublasZsymv_v2") +} +#[no_mangle] +pub extern "C" fn cublasZsyr2k_v2() -> c_int { + stub("cublasZsyr2k_v2") +} +#[no_mangle] +pub extern "C" fn cublasZsyr2_v2() -> c_int { + stub("cublasZsyr2_v2") +} +#[no_mangle] +pub extern "C" fn cublasZsyrk_v2() -> c_int { + stub("cublasZsyrk_v2") +} +#[no_mangle] +pub extern "C" fn cublasZsyr_v2() -> c_int { + stub("cublasZsyr_v2") +} +#[no_mangle] +pub extern "C" fn cublasZtrmm_v2() -> c_int { + stub("cublasZtrmm_v2") +} +#[no_mangle] +pub extern "C" fn cublasZtrmv_v2() -> c_int { + stub("cublasZtrmv_v2") +} +#[no_mangle] +pub extern "C" fn cublasZtrsv_v2() -> c_int { + stub("cublasZtrsv_v2") +} +#[no_mangle] +pub extern "C" fn cudnnGetProperty() -> c_int { + stub("cudnnGetProperty") +} diff --git a/crates/smolvm-cudart-shim/src/generated/cublas_guest.rs b/crates/smolvm-cudart-shim/src/generated/cublas_guest.rs new file mode 100644 index 0000000..da80273 --- /dev/null +++ b/crates/smolvm-cudart-shim/src/generated/cublas_guest.rs @@ -0,0 +1,606 @@ +// @generated by smolvm-cuda-codegen — do not edit. lib='cublas' id=1 +const LIB_ID: u8 = 1; +#[no_mangle] +pub extern "C" fn cublasCreate_v2(handle_out: *mut *mut c_void) -> c_int { + if handle_out.is_null() { return 1; } + // Fire-and-forget create: return a guest-assigned virtual id + // immediately; the host materializes the real descriptor and + // maps the id (see vh_resolve on the host side). + let vh = super::alloc_vhandle(); + match with_client(|c| c.lib_call_deferred(LIB_ID, 0, vh.to_le_bytes().to_vec())) { + Ok(()) => { unsafe { *handle_out = vh as *mut c_void }; 0 } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cublasDestroy_v2(handle: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 1, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSgemm_v2(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f32, A: *const f32, lda: c_int, B: *const f32, ldb: c_int, beta: *const f32, C: *mut f32, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 2, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasHgemm(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const u16, A: *const u16, lda: c_int, B: *const u16, ldb: c_int, beta: *const u16, C: *mut u16, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as u16).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as u16).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 3, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDgemm_v2(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f64, A: *const f64, lda: c_int, B: *const f64, ldb: c_int, beta: *const f64, C: *mut f64, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 4, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSgemmStridedBatched(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f32, A: *const f32, lda: c_int, strideA: i64, B: *const f32, ldb: c_int, strideB: i64, beta: *const f32, C: *mut f32, ldc: c_int, strideC: i64, batchCount: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(strideA as i64).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + a.extend_from_slice(&(strideB as i64).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + a.extend_from_slice(&(strideC as i64).to_le_bytes()); + a.extend_from_slice(&(batchCount as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 5, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDgemmStridedBatched(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f64, A: *const f64, lda: c_int, strideA: i64, B: *const f64, ldb: c_int, strideB: i64, beta: *const f64, C: *mut f64, ldc: c_int, strideC: i64, batchCount: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(strideA as i64).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + a.extend_from_slice(&(strideB as i64).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + a.extend_from_slice(&(strideC as i64).to_le_bytes()); + a.extend_from_slice(&(batchCount as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 6, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSetStream_v2(handle: *mut c_void, stream: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(stream as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 7, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSetWorkspace_v2(handle: *mut c_void, workspace: *mut c_void, size: usize) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(workspace as u64).to_le_bytes()); + a.extend_from_slice(&(size as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 8, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSetMathMode(handle: *mut c_void, mode: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(mode as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 9, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasGetMathMode(handle: *mut c_void, mode: *mut c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + match with_client(|c| c.lib_call(LIB_ID, 10, a)) { + Ok((st, out)) => { let mut o = 0usize; + if !mode.is_null() && out.len() >= o + 4 { unsafe { *mode = i32::from_le_bytes(out[o..o + 4].try_into().unwrap()) as _ }; } o += 4; + let _ = o; st } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cublasGetProperty(prop_type: c_int, value: *mut c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(prop_type as i32).to_le_bytes()); + match with_client(|c| c.lib_call(LIB_ID, 11, a)) { + Ok((st, out)) => { let mut o = 0usize; + if !value.is_null() && out.len() >= o + 4 { unsafe { *value = i32::from_le_bytes(out[o..o + 4].try_into().unwrap()) as _ }; } o += 4; + let _ = o; st } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cublasGemmEx(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f32, A: *const c_void, Atype: c_int, lda: c_int, B: *const c_void, Btype: c_int, ldb: c_int, beta: *const f32, C: *mut c_void, Ctype: c_int, ldc: c_int, computeType: c_int, algo: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(Atype as i32).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(Btype as i32).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(Ctype as i32).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + a.extend_from_slice(&(computeType as i32).to_le_bytes()); + a.extend_from_slice(&(algo as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 12, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasGemmStridedBatchedEx(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f32, A: *const c_void, Atype: c_int, lda: c_int, strideA: i64, B: *const c_void, Btype: c_int, ldb: c_int, strideB: i64, beta: *const f32, C: *mut c_void, Ctype: c_int, ldc: c_int, strideC: i64, batchCount: c_int, computeType: c_int, algo: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(Atype as i32).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(strideA as i64).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(Btype as i32).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + a.extend_from_slice(&(strideB as i64).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(Ctype as i32).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + a.extend_from_slice(&(strideC as i64).to_le_bytes()); + a.extend_from_slice(&(batchCount as i32).to_le_bytes()); + a.extend_from_slice(&(computeType as i32).to_le_bytes()); + a.extend_from_slice(&(algo as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 13, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasGemmBatchedEx(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, k: c_int, alpha: *const f32, Aarray: *const c_void, Atype: c_int, lda: c_int, Barray: *const c_void, Btype: c_int, ldb: c_int, beta: *const f32, Carray: *mut c_void, Ctype: c_int, ldc: c_int, batchCount: c_int, computeType: c_int, algo: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(Aarray as u64).to_le_bytes()); + a.extend_from_slice(&(Atype as i32).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(Barray as u64).to_le_bytes()); + a.extend_from_slice(&(Btype as i32).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(Carray as u64).to_le_bytes()); + a.extend_from_slice(&(Ctype as i32).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + a.extend_from_slice(&(batchCount as i32).to_le_bytes()); + a.extend_from_slice(&(computeType as i32).to_le_bytes()); + a.extend_from_slice(&(algo as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 14, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSaxpy_v2(handle: *mut c_void, n: c_int, alpha: *const f32, x: *const f32, incx: c_int, y: *mut f32, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 15, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDaxpy_v2(handle: *mut c_void, n: c_int, alpha: *const f64, x: *const f64, incx: c_int, y: *mut f64, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 16, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSscal_v2(handle: *mut c_void, n: c_int, alpha: *const f32, x: *mut f32, incx: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 17, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDscal_v2(handle: *mut c_void, n: c_int, alpha: *const f64, x: *mut f64, incx: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 18, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasScopy_v2(handle: *mut c_void, n: c_int, x: *mut f32, incx: c_int, y: *mut f32, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 19, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDcopy_v2(handle: *mut c_void, n: c_int, x: *mut f64, incx: c_int, y: *mut f64, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 20, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSswap_v2(handle: *mut c_void, n: c_int, x: *mut f32, incx: c_int, y: *mut f32, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 21, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDswap_v2(handle: *mut c_void, n: c_int, x: *mut f64, incx: c_int, y: *mut f64, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 22, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSgemv_v2(handle: *mut c_void, trans: c_int, m: c_int, n: c_int, alpha: *const f32, A: *const f32, lda: c_int, x: *const f32, incx: c_int, beta: *const f32, y: *mut f32, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 23, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDgemv_v2(handle: *mut c_void, trans: c_int, m: c_int, n: c_int, alpha: *const f64, A: *const f64, lda: c_int, x: *const f64, incx: c_int, beta: *const f64, y: *mut f64, incy: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 24, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSger_v2(handle: *mut c_void, m: c_int, n: c_int, alpha: *const f32, x: *const f32, incx: c_int, y: *const f32, incy: c_int, A: *mut f32, lda: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 25, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDger_v2(handle: *mut c_void, m: c_int, n: c_int, alpha: *const f64, x: *const f64, incx: c_int, y: *const f64, incy: c_int, A: *mut f64, lda: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(incx as i32).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(incy as i32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 26, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSgeam(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, alpha: *const f32, A: *const f32, lda: c_int, beta: *const f32, B: *const f32, ldb: c_int, C: *mut f32, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 27, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDgeam(handle: *mut c_void, transa: c_int, transb: c_int, m: c_int, n: c_int, alpha: *const f64, A: *const f64, lda: c_int, beta: *const f64, B: *const f64, ldb: c_int, C: *mut f64, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(transa as i32).to_le_bytes()); + a.extend_from_slice(&(transb as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 28, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSsyrk_v2(handle: *mut c_void, uplo: c_int, trans: c_int, n: c_int, k: c_int, alpha: *const f32, A: *const f32, lda: c_int, beta: *const f32, C: *mut f32, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 29, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDsyrk_v2(handle: *mut c_void, uplo: c_int, trans: c_int, n: c_int, k: c_int, alpha: *const f64, A: *const f64, lda: c_int, beta: *const f64, C: *mut f64, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 30, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasSsymm_v2(handle: *mut c_void, side: c_int, uplo: c_int, m: c_int, n: c_int, alpha: *const f32, A: *const f32, lda: c_int, B: *const f32, ldb: c_int, beta: *const f32, C: *mut f32, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(side as i32).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 31, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDsymm_v2(handle: *mut c_void, side: c_int, uplo: c_int, m: c_int, n: c_int, alpha: *const f64, A: *const f64, lda: c_int, B: *const f64, ldb: c_int, beta: *const f64, C: *mut f64, ldc: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(side as i32).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f64).to_le_bytes()); + a.extend_from_slice(&(C as u64).to_le_bytes()); + a.extend_from_slice(&(ldc as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 32, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasStrsm_v2(handle: *mut c_void, side: c_int, uplo: c_int, trans: c_int, diag: c_int, m: c_int, n: c_int, alpha: *const f32, A: *const f32, lda: c_int, B: *mut f32, ldb: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(side as i32).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(diag as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 33, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cublasDtrsm_v2(handle: *mut c_void, side: c_int, uplo: c_int, trans: c_int, diag: c_int, m: c_int, n: c_int, alpha: *const f64, A: *const f64, lda: c_int, B: *mut f64, ldb: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(side as i32).to_le_bytes()); + a.extend_from_slice(&(uplo as i32).to_le_bytes()); + a.extend_from_slice(&(trans as i32).to_le_bytes()); + a.extend_from_slice(&(diag as i32).to_le_bytes()); + a.extend_from_slice(&(m as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f64).to_le_bytes()); + a.extend_from_slice(&(A as u64).to_le_bytes()); + a.extend_from_slice(&(lda as i32).to_le_bytes()); + a.extend_from_slice(&(B as u64).to_le_bytes()); + a.extend_from_slice(&(ldb as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 34, a)) { Ok(()) => 0, Err(_) => 1 } +} diff --git a/crates/smolvm-cudart-shim/src/generated/cudnn_guest.rs b/crates/smolvm-cudart-shim/src/generated/cudnn_guest.rs new file mode 100644 index 0000000..c14e363 --- /dev/null +++ b/crates/smolvm-cudart-shim/src/generated/cudnn_guest.rs @@ -0,0 +1,164 @@ +// @generated by smolvm-cuda-codegen — do not edit. lib='cudnn' id=2 +const LIB_ID: u8 = 2; +#[no_mangle] +pub extern "C" fn cudnnCreate(handle_out: *mut *mut c_void) -> c_int { + if handle_out.is_null() { return 1; } + // Fire-and-forget create: return a guest-assigned virtual id + // immediately; the host materializes the real descriptor and + // maps the id (see vh_resolve on the host side). + let vh = super::alloc_vhandle(); + match with_client(|c| c.lib_call_deferred(LIB_ID, 0, vh.to_le_bytes().to_vec())) { + Ok(()) => { unsafe { *handle_out = vh as *mut c_void }; 0 } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cudnnDestroy(handle: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 1, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnSetStream(handle: *mut c_void, stream: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(stream as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 2, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnCreateTensorDescriptor(handle_out: *mut *mut c_void) -> c_int { + if handle_out.is_null() { return 1; } + // Fire-and-forget create: return a guest-assigned virtual id + // immediately; the host materializes the real descriptor and + // maps the id (see vh_resolve on the host side). + let vh = super::alloc_vhandle(); + match with_client(|c| c.lib_call_deferred(LIB_ID, 3, vh.to_le_bytes().to_vec())) { + Ok(()) => { unsafe { *handle_out = vh as *mut c_void }; 0 } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cudnnSetTensor4dDescriptor(t: *mut c_void, fmt: c_int, dtype: c_int, n: c_int, c: c_int, h: c_int, w: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(t as u64).to_le_bytes()); + a.extend_from_slice(&(fmt as i32).to_le_bytes()); + a.extend_from_slice(&(dtype as i32).to_le_bytes()); + a.extend_from_slice(&(n as i32).to_le_bytes()); + a.extend_from_slice(&(c as i32).to_le_bytes()); + a.extend_from_slice(&(h as i32).to_le_bytes()); + a.extend_from_slice(&(w as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 4, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnDestroyTensorDescriptor(t: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(t as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 5, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnCreateFilterDescriptor(handle_out: *mut *mut c_void) -> c_int { + if handle_out.is_null() { return 1; } + // Fire-and-forget create: return a guest-assigned virtual id + // immediately; the host materializes the real descriptor and + // maps the id (see vh_resolve on the host side). + let vh = super::alloc_vhandle(); + match with_client(|c| c.lib_call_deferred(LIB_ID, 6, vh.to_le_bytes().to_vec())) { + Ok(()) => { unsafe { *handle_out = vh as *mut c_void }; 0 } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cudnnSetFilter4dDescriptor(f: *mut c_void, dtype: c_int, fmt: c_int, k: c_int, c: c_int, h: c_int, w: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(f as u64).to_le_bytes()); + a.extend_from_slice(&(dtype as i32).to_le_bytes()); + a.extend_from_slice(&(fmt as i32).to_le_bytes()); + a.extend_from_slice(&(k as i32).to_le_bytes()); + a.extend_from_slice(&(c as i32).to_le_bytes()); + a.extend_from_slice(&(h as i32).to_le_bytes()); + a.extend_from_slice(&(w as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 7, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnDestroyFilterDescriptor(f: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(f as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 8, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnCreateConvolutionDescriptor(handle_out: *mut *mut c_void) -> c_int { + if handle_out.is_null() { return 1; } + // Fire-and-forget create: return a guest-assigned virtual id + // immediately; the host materializes the real descriptor and + // maps the id (see vh_resolve on the host side). + let vh = super::alloc_vhandle(); + match with_client(|c| c.lib_call_deferred(LIB_ID, 9, vh.to_le_bytes().to_vec())) { + Ok(()) => { unsafe { *handle_out = vh as *mut c_void }; 0 } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cudnnSetConvolution2dDescriptor(cv: *mut c_void, pad_h: c_int, pad_w: c_int, u: c_int, v: c_int, dil_h: c_int, dil_w: c_int, mode: c_int, ctype: c_int) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(cv as u64).to_le_bytes()); + a.extend_from_slice(&(pad_h as i32).to_le_bytes()); + a.extend_from_slice(&(pad_w as i32).to_le_bytes()); + a.extend_from_slice(&(u as i32).to_le_bytes()); + a.extend_from_slice(&(v as i32).to_le_bytes()); + a.extend_from_slice(&(dil_h as i32).to_le_bytes()); + a.extend_from_slice(&(dil_w as i32).to_le_bytes()); + a.extend_from_slice(&(mode as i32).to_le_bytes()); + a.extend_from_slice(&(ctype as i32).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 10, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnDestroyConvolutionDescriptor(cv: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(cv as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 11, a)) { Ok(()) => 0, Err(_) => 1 } +} +#[no_mangle] +pub extern "C" fn cudnnGetConvolutionForwardWorkspaceSize(handle: *mut c_void, x: *mut c_void, w: *mut c_void, cv: *mut c_void, y: *mut c_void, algo: c_int, size: *mut usize) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(w as u64).to_le_bytes()); + a.extend_from_slice(&(cv as u64).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + a.extend_from_slice(&(algo as i32).to_le_bytes()); + match with_client(|c| c.lib_call(LIB_ID, 12, a)) { + Ok((st, out)) => { let mut o = 0usize; + if !size.is_null() && out.len() >= o + 8 { unsafe { *size = u64::from_le_bytes(out[o..o + 8].try_into().unwrap()) as _ }; } o += 8; + let _ = o; st } + Err(_) => 1, + } +} +#[no_mangle] +pub extern "C" fn cudnnConvolutionForward(handle: *mut c_void, alpha: *const f32, xDesc: *mut c_void, x: *mut c_void, wDesc: *mut c_void, w: *mut c_void, convDesc: *mut c_void, algo: c_int, workspace: *mut c_void, wsSize: usize, beta: *const f32, yDesc: *mut c_void, y: *mut c_void) -> c_int { + let mut a: Vec = Vec::new(); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + if alpha.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *alpha } as f32).to_le_bytes()); + a.extend_from_slice(&(xDesc as u64).to_le_bytes()); + a.extend_from_slice(&(x as u64).to_le_bytes()); + a.extend_from_slice(&(wDesc as u64).to_le_bytes()); + a.extend_from_slice(&(w as u64).to_le_bytes()); + a.extend_from_slice(&(convDesc as u64).to_le_bytes()); + a.extend_from_slice(&(algo as i32).to_le_bytes()); + a.extend_from_slice(&(workspace as u64).to_le_bytes()); + a.extend_from_slice(&(wsSize as u64).to_le_bytes()); + if beta.is_null() { return 1; } + a.extend_from_slice(&(unsafe { *beta } as f32).to_le_bytes()); + a.extend_from_slice(&(yDesc as u64).to_le_bytes()); + a.extend_from_slice(&(y as u64).to_le_bytes()); + // No output params: fire-and-forget (failures surface as sticky async errors). + match with_client(|c| c.lib_call_deferred(LIB_ID, 13, a)) { Ok(()) => 0, Err(_) => 1 } +} diff --git a/crates/smolvm-cudart-shim/src/lib.rs b/crates/smolvm-cudart-shim/src/lib.rs new file mode 100644 index 0000000..17ded9f --- /dev/null +++ b/crates/smolvm-cudart-shim/src/lib.rs @@ -0,0 +1,3501 @@ +//! Drop-in `libcudart.so.11.0` for smolvm guests: the CUDA **Runtime API**, +//! implemented by lowering each call to the public CUDA **Driver API** and +//! remoting that over smolvm's CUDA vsock RPC to the host GPU. +//! +//! Why this exists: NVIDIA's own `libcudart` bootstraps through a private, +//! undocumented driver interface (`cuGetExportTable` with an internal UUID), +//! which a black-box `libcuda` shim cannot provide — so Runtime-API programs +//! (anything `nvcc`-compiled, and frameworks on top) cannot run on the Driver- +//! API shim alone. This library replaces `libcudart` instead of the driver: +//! every `cuda*` / `__cuda*` entry point a program links is served here by +//! lowering to public Driver-API calls (`cuModuleLoadData`, `cuLaunchKernel`, +//! `cuMemAlloc`, …) that the existing host CUDA server already executes on the +//! real GPU. No host-side changes; it reuses the Driver-API RPC wholesale. +//! +//! Interpose it with `LD_PRELOAD=/path/libcudart.so.11.0` (or stage it ahead of +//! the program's own copy on `LD_LIBRARY_PATH`). Transport is selected by +//! `SMOLVM_CUDA_RPC` exactly as the libcuda shim: unset/`vsock` (in-guest), +//! `tcp:HOST:PORT`, or `unix:/path` (host-side testing). +//! +//! Semantics: work executes synchronously on the host, so `*Async` calls and +//! streams collapse to ordered-and-complete (permitted — an implementation may +//! be more synchronous than requested). Device pointers are the host's real +//! `CUdeviceptr` values, opaque to the guest; pinned host memory +//! (`cudaHostAlloc`) is plain guest RAM (the "pinned" property is a host-only +//! optimization that does not cross the boundary). Kernel launch reconstructs +//! `kernelParams` from per-argument sizes the host reports via +//! `cuFuncGetParamInfo` (needs a CUDA 12.4+ host driver). + +#![allow(clippy::not_unsafe_ptr_arg_deref)] +#![allow(non_snake_case)] + +use smolvm_cuda::client::{Client, CudaRpcError}; +mod cublas_stubs; +use std::cell::RefCell; +use std::collections::HashMap; +use std::ffi::{c_char, c_int, c_uint, c_void, CStr, CString}; +use std::io::{Read, Write}; +use std::sync::Mutex; + +// ---- cudaError_t codes we produce locally ----------------------------------- + +const CUDA_SUCCESS: c_int = 0; +const CUDA_ERROR_INVALID_VALUE: c_int = 1; +const CUDA_ERROR_MEMORY_ALLOCATION: c_int = 2; +const CUDA_ERROR_INITIALIZATION: c_int = 3; +const CUDA_ERROR_INVALID_DEVICE_POINTER: c_int = 17; +const CUDA_ERROR_INVALID_RESOURCE_HANDLE: c_int = 400; +const CUDA_ERROR_NO_DEVICE: c_int = 100; +const CUDA_ERROR_UNKNOWN: c_int = 999; + +// cudaMemcpyKind +const MEMCPY_HTOH: c_int = 0; +const MEMCPY_HTOD: c_int = 1; +const MEMCPY_DTOH: c_int = 2; +const MEMCPY_DTOD: c_int = 3; +const MEMCPY_DEFAULT: c_int = 4; + +/// A `dim3` passed by value across the C ABI (three unsigned ints). +#[repr(C)] +#[derive(Clone, Copy)] +pub struct Dim3 { + x: c_uint, + y: c_uint, + z: c_uint, +} + +// ---- transport (mirrors smolvm-cuda-shim) ----------------------------------- + +enum Stream { + #[cfg(target_os = "linux")] + Vsock(vsock::VsockStream), + Tcp(std::net::TcpStream), + #[cfg(unix)] + Unix(std::os::unix::net::UnixStream), +} + +impl Read for Stream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + #[cfg(target_os = "linux")] + Stream::Vsock(s) => s.read(buf), + Stream::Tcp(s) => s.read(buf), + #[cfg(unix)] + Stream::Unix(s) => s.read(buf), + } + } +} +impl Write for Stream { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + #[cfg(target_os = "linux")] + Stream::Vsock(s) => s.write(buf), + Stream::Tcp(s) => s.write(buf), + #[cfg(unix)] + Stream::Unix(s) => s.write(buf), + } + } + fn flush(&mut self) -> std::io::Result<()> { + match self { + #[cfg(target_os = "linux")] + Stream::Vsock(s) => s.flush(), + Stream::Tcp(s) => s.flush(), + #[cfg(unix)] + Stream::Unix(s) => s.flush(), + } + } +} + +fn connect() -> Result { + let spec = std::env::var("SMOLVM_CUDA_RPC").unwrap_or_default(); + if let Some(addr) = spec.strip_prefix("tcp:") { + return std::net::TcpStream::connect(addr) + .map(|s| { + let _ = s.set_nodelay(true); // low-latency request/response + Stream::Tcp(s) + }) + .map_err(|_| CUDA_ERROR_NO_DEVICE); + } + #[cfg(unix)] + if let Some(path) = spec.strip_prefix("unix:") { + return std::os::unix::net::UnixStream::connect(path) + .map(Stream::Unix) + .map_err(|_| CUDA_ERROR_NO_DEVICE); + } + #[cfg(target_os = "linux")] + { + const HOST_CID: u32 = 2; + const CUDA_PORT: u32 = 7000; + vsock::VsockStream::connect_with_cid_port(HOST_CID, CUDA_PORT) + .map(Stream::Vsock) + .map_err(|_| CUDA_ERROR_NO_DEVICE) + } + #[cfg(not(target_os = "linux"))] + Err(CUDA_ERROR_NO_DEVICE) +} + +// ---- global state ----------------------------------------------------------- + +/// A registered kernel: its host-side driver function id and the byte size of +/// each `__global__` parameter, in declaration order. +struct FuncRec { + fid: u64, + param_sizes: Vec, +} + +struct ShimState { + client: Client, + initialized: bool, + /// `__cudaRegisterFatBinary` handle (the pointer we minted) → driver module id. + modules: HashMap, + /// `__cudaRegisterFunction` host stub pointer → resolved kernel. + funcs: HashMap, + /// Host pinned-memory allocations (guest RAM) → layout, for cudaFreeHost. + host_allocs: HashMap, + /// Live device allocations, base → size. Range-queried (not exact-match): + /// PyTorch's caching allocator suballocates, so tensor data pointers are + /// interior to a cudaMalloc'd block — `cudaPointerGetAttributes` on one + /// must still report Device or torch's `getDeviceFromPtr` throws. + dev_allocs: std::collections::BTreeMap, + /// Active CUDA graph capture, `(stream_handle, capture_id)`. Kept + /// guest-side so the per-launch hot queries (`cudaStreamIsCapturing`, + /// `cudaStreamGetCaptureInfo` — PyTorch's allocator calls them constantly) + /// answer locally instead of round-tripping. + capture: Option<(u64, u64)>, +} + +static STATE: Mutex> = Mutex::new(None); + +thread_local! { + /// `__cudaPushCallConfiguration` stash, popped by `__cudaPopCallConfiguration`. + static CALL_CONFIG: RefCell> = const { RefCell::new(Vec::new()) }; + /// Last error, for cudaGetLastError / cudaPeekAtLastError. + static LAST_ERROR: std::cell::Cell = const { std::cell::Cell::new(CUDA_SUCCESS) }; +} + +fn set_last(code: c_int) -> c_int { + if code != CUDA_SUCCESS { + if std::env::var_os("SMOLVM_CUDA_SHIM_TRACE").is_some() { + let bt = std::backtrace::Backtrace::force_capture().to_string(); + let frames: Vec<&str> = bt.lines().take(16).collect(); + eprintln!("[shim-err] code={code} frames:\n{}", frames.join("\n")); + } + LAST_ERROR.with(|e| e.set(code)); + } + code +} + +/// Map a Driver-API `CUresult`/transport failure to a `cudaError_t`. +fn map_err(e: CudaRpcError) -> c_int { + match e { + CudaRpcError::Cuda(code) => match code { + 0 => CUDA_SUCCESS, + 1 => CUDA_ERROR_INVALID_VALUE, + 2 => CUDA_ERROR_MEMORY_ALLOCATION, + 200 | 218 => CUDA_ERROR_INVALID_VALUE, // invalid image / PTX + 400 => CUDA_ERROR_INVALID_RESOURCE_HANDLE, + other => { + if std::env::var_os("SMOLVM_CUDA_SHIM_TRACE").is_some() { + eprintln!("[map-err] unmapped driver code {other}"); + } + CUDA_ERROR_UNKNOWN + } + }, + CudaRpcError::Io(_) | CudaRpcError::Protocol(_) => { + if std::env::var_os("SMOLVM_CUDA_SHIM_TRACE").is_some() { + eprintln!("[map-err] transport: {e}"); + } + CUDA_ERROR_UNKNOWN + } + } +} + +/// Lazily connect and bring up a primary context, then run `f` against the +/// client. The first call performs `cuInit` + `cuDevicePrimaryCtxRetain(0)` +/// (which the host binds current on its serving thread), matching how the CUDA +/// runtime brings up its device on first use. +/// Cross-shim ordering hook, dlsym'd by the driver shim (`libcuda.so.1`). +/// The two shims hold separate connections to the host, i.e. two ordering +/// domains for one guest program-order stream; the driver shim fences this +/// connection before each of its own ops so runtime-issued work (deferred in +/// the pipeline) executes first. No-op before the runtime connection exists. +#[no_mangle] +pub extern "C" fn smolvm_cudart_fence() { + if let Ok(mut guard) = STATE.lock() { + if let Some(st) = guard.as_mut() { + let _ = st.client.drain(); + } + } +} + +// ---- driver-shim bridge ------------------------------------------------------- +// The driver shim (`libcuda.so.1`) dlsym-resolves these three and routes ALL +// its traffic through this connection, giving the host one program-ordered +// pipeline for both shims (see smolvm-cuda's `client::Bridge`). Op statuses +// stay in-band in the response payload — nothing is lost to error mapping. + +/// A response too large for the caller's buffer, parked until the caller +/// retries with a big-enough one (null request = fetch). +static BRIDGE_PENDING: Mutex>> = Mutex::new(None); + +/// Fire-and-forget: append one encoded request to the shared pipeline. +/// Nonzero = transport failure. +#[no_mangle] +pub extern "C" fn smolvm_cudart_bridge_quiet(req: *const u8, len: usize) -> i32 { + if req.is_null() { + return 1; + } + let bytes = unsafe { std::slice::from_raw_parts(req, len) }; + match with_client(|c| c.raw_quiet(bytes)) { + Ok(()) => 0, + Err(_) => 999, + } +} + +/// Synchronous round-trip: send one encoded request, write the response +/// payload into `resp`. Returns the response length; -1 = transport failure; +/// other negatives = `cap` too small, retry with `-ret` capacity and a null +/// request to collect the stashed response. +#[no_mangle] +pub extern "C" fn smolvm_cudart_bridge_call( + req: *const u8, + req_len: usize, + resp: *mut u8, + cap: usize, +) -> isize { + let payload = if req.is_null() { + match BRIDGE_PENDING.lock().map(|mut g| g.take()) { + Ok(Some(p)) => p, + _ => return -1, + } + } else { + let bytes = unsafe { std::slice::from_raw_parts(req, req_len) }; + match with_client(|c| c.raw_call(bytes)) { + Ok(p) => p, + Err(_) => return -1, + } + }; + if payload.len() > cap { + let n = payload.len() as isize; + match BRIDGE_PENDING.lock() { + Ok(mut g) => { + *g = Some(payload); + -n + } + Err(_) => -1, + } + } else { + unsafe { std::ptr::copy_nonoverlapping(payload.as_ptr(), resp, payload.len()) }; + payload.len() as isize + } +} + +/// Fence the shared pipeline; returns (and consumes) the first collected +/// quiet-failure status, so the bridged caller can surface it. +#[no_mangle] +pub extern "C" fn smolvm_cudart_bridge_drain() -> i32 { + with_client(|c| { + c.drain()?; + Ok(c.take_sticky()) + }) + .unwrap_or(999) +} + +/// Allocate one ring region: pinned pages + their per-page GPAs. +fn ring_alloc_pages(pages: usize) -> Option<(Vec<*mut u8>, Vec)> { + const PAGE: usize = 4096; + let base = guestmem::alloc(pages * PAGE)? as usize; + // Zero (also faults every page in before pagemap reads). + unsafe { std::ptr::write_bytes(base as *mut u8, 0, pages * PAGE) }; + let segs = guestmem::segments(base, pages * PAGE)?; + let mut gpas = Vec::with_capacity(pages); + for (gpa, len) in segs { + let mut off = 0; + while off < len { + gpas.push(gpa + off); + off += PAGE as u64; + } + } + if gpas.len() != pages { + return None; + } + Some(( + (0..pages).map(|i| (base + i * PAGE) as *mut u8).collect(), + gpas, + )) +} + +/// Try to switch `client` to the shared-memory ring transport. Failure is +/// fine — the connection simply stays on the socket. +fn ring_try_setup(client: &mut Client) { + const PAGE: usize = 4096; + let trace = std::env::var_os("SMOLVM_CUDA_SHIM_TRACE").is_some(); + let Some(req) = ring_alloc_pages(32) else { + if trace { + eprintln!("[ring] no pinned pages (zerocopy off?) — socket mode"); + } + return; + }; + let (Some(resp), Some(bounce)) = (ring_alloc_pages(8), ring_alloc_pages(64)) else { + return; + }; + match client.ring_setup(PAGE, req, resp, bounce) { + Ok(()) => { + if trace { + eprintln!("[ring] shared-memory rings active"); + } + } + Err(e) => { + if trace { + eprintln!("[ring] setup rejected ({e}) — socket mode"); + } + } + } +} + +fn with_client( + f: impl FnOnce(&mut Client) -> Result, +) -> Result { + let mut guard = STATE.lock().map_err(|_| CUDA_ERROR_UNKNOWN)?; + if guard.is_none() { + let stream = connect()?; + #[cfg(target_os = "linux")] + let try_ring = matches!(stream, Stream::Vsock(_)) + && std::env::var("SMOLVM_CUDA_RING").as_deref() != Ok("0"); + #[cfg(not(target_os = "linux"))] + let try_ring = false; + let mut client = Client::new(stream); + client.init().map_err(|_| CUDA_ERROR_INITIALIZATION)?; + let _ = client + .primary_ctx_retain(0) + .map_err(|_| CUDA_ERROR_INITIALIZATION)?; + if try_ring { + ring_try_setup(&mut client); // best-effort; socket mode on failure + } + *guard = Some(ShimState { + client, + initialized: true, + modules: HashMap::new(), + funcs: HashMap::new(), + host_allocs: HashMap::new(), + dev_allocs: std::collections::BTreeMap::new(), + capture: None, + }); + } + let st = guard.as_mut().ok_or(CUDA_ERROR_INITIALIZATION)?; + debug_assert!(st.initialized); + // SAFETY-free: split the borrow so `f` gets the client while we keep the lock. + let client = &mut st.client; + f(client).map_err(map_err) +} + +/// Run `f` with the full state (client + registries) under the lock. +fn with_state(f: impl FnOnce(&mut ShimState) -> Result) -> Result { + // Ensure init first (reuses with_client's bring-up), then re-lock. + with_client(|_| Ok(()))?; + let mut guard = STATE.lock().map_err(|_| CUDA_ERROR_UNKNOWN)?; + let st = guard.as_mut().ok_or(CUDA_ERROR_INITIALIZATION)?; + f(st) +} + +unsafe fn out(p: *mut T, v: T) -> c_int { + if p.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + unsafe { p.write(v) }; + CUDA_SUCCESS +} + +// ---- device / init ---------------------------------------------------------- + +#[no_mangle] +pub extern "C" fn cudaGetDeviceCount(count: *mut c_int) -> c_int { + set_last(match with_client(|c| c.device_get_count()) { + Ok(n) => unsafe { out(count, n) }, + Err(e) => { + // A CUDA program treats "0 devices" as recoverable; surface the count. + unsafe { out(count, 0) }; + e + } + }) +} + +#[no_mangle] +pub extern "C" fn cudaSetDevice(device: c_int) -> c_int { + // Single-device model: only device 0 exists. + set_last(if device == 0 { + CUDA_SUCCESS + } else { + CUDA_ERROR_INVALID_VALUE + }) +} + +#[no_mangle] +pub extern "C" fn cudaGetDevice(device: *mut c_int) -> c_int { + set_last(unsafe { out(device, 0) }) +} + +#[no_mangle] +pub extern "C" fn cudaDeviceSynchronize() -> c_int { + set_last(match with_client(|c| c.ctx_synchronize()) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }) +} + +#[no_mangle] +pub extern "C" fn cudaDriverGetVersion(version: *mut c_int) -> c_int { + static CACHED: std::sync::atomic::AtomicI32 = std::sync::atomic::AtomicI32::new(0); + let c = CACHED.load(std::sync::atomic::Ordering::Relaxed); + if c != 0 { + return set_last(unsafe { out(version, c) }); + } + set_last(match with_client(|c| c.driver_get_version()) { + Ok(v) => { + CACHED.store(v, std::sync::atomic::Ordering::Relaxed); + unsafe { out(version, v) } + } + Err(e) => e, + }) +} + +#[no_mangle] +pub extern "C" fn cudaRuntimeGetVersion(version: *mut c_int) -> c_int { + set_last(unsafe { out(version, 12040) }) +} + +// ---- memory ----------------------------------------------------------------- + +#[no_mangle] +pub extern "C" fn cudaMalloc(dev_ptr: *mut *mut c_void, size: usize) -> c_int { + set_last( + match with_state(|s| { + let d = s.client.mem_alloc(size as u64).map_err(map_err)?; + s.dev_allocs.insert(d, size as u64); + Ok(d) + }) { + Ok(d) => unsafe { out(dev_ptr, d as *mut c_void) }, + Err(e) => e, + }, + ) +} + +/// `cudaMallocManaged` — unified memory the CPU and GPU both access by the same +/// pointer. Through API remoting to a discrete GPU that cannot page-fault into +/// guest RAM, that is unserviceable: a guest-CPU dereference of a real host +/// device address reads garbage. So by default we FAIL (writing a NULL +/// out-pointer), turning silent corruption into an immediate, obvious crash — +/// this is exactly bitsandbytes' *paged* optimizer path (`get_paged` wraps the +/// pointer as a host numpy array). `SMOLVM_CUDA_MANAGED=device` restores the +/// old device-backed behavior for workloads that only ever touch the pointer +/// on the GPU (never on the CPU); use it only when you know that holds. +#[no_mangle] +pub extern "C" fn cudaMallocManaged( + dev_ptr: *mut *mut c_void, + size: usize, + _flags: c_uint, +) -> c_int { + if std::env::var("SMOLVM_CUDA_MANAGED").as_deref() == Ok("device") { + return cudaMalloc(dev_ptr, size); + } + if !dev_ptr.is_null() { + unsafe { *dev_ptr = std::ptr::null_mut() }; + } + // cudaErrorNotSupported: host-coherent managed memory can't cross the + // forwarding boundary. (SMOLVM_CUDA_MANAGED=device to override.) + set_last(801) +} + +/// Managed-memory prefetch hint: nothing to do, our "managed" memory is +/// always device-resident. +#[no_mangle] +pub extern "C" fn cudaMemPrefetchAsync( + _dev_ptr: *const c_void, + _count: usize, + _dst_device: c_int, + _stream: *mut c_void, +) -> c_int { + CUDA_SUCCESS +} + +/// Is `p` inside any live device allocation (base ≤ p < base+size)? +fn dev_contains(allocs: &std::collections::BTreeMap, p: u64) -> bool { + allocs + .range(..=p) + .next_back() + .is_some_and(|(base, size)| p < base + size) +} + +#[no_mangle] +pub extern "C" fn cudaFree(dev_ptr: *mut c_void) -> c_int { + if dev_ptr.is_null() { + return set_last(CUDA_SUCCESS); // cudaFree(NULL) is a no-op + } + set_last( + match with_state(|s| { + s.client.mem_free(dev_ptr as u64).map_err(map_err)?; + s.dev_allocs.remove(&(dev_ptr as u64)); + Ok(()) + }) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }, + ) +} + +#[no_mangle] +pub extern "C" fn cudaHostAlloc(ptr: *mut *mut c_void, size: usize, _flags: c_uint) -> c_int { + cuda_host_malloc(ptr, size.max(1)) +} + +#[no_mangle] +pub extern "C" fn cudaMallocHost(ptr: *mut *mut c_void, size: usize) -> c_int { + cuda_host_malloc(ptr, size.max(1)) +} + +// ---- shared-memory zero-copy staging ---------------------------------------- +// When SMOLVM_CUDA_SHM is set, `cudaMallocHost`/`cudaHostAlloc` bump-allocate +// from a region the host also maps, so a memcpy on that buffer ships only an +// offset (see do_memcpy). Falls back to plain host memory when the region is +// absent or exhausted. + +use std::sync::atomic::{AtomicU64, Ordering}; + +static SHM_NEXT: AtomicU64 = AtomicU64::new(0); + +#[cfg(target_os = "linux")] +fn shm_region() -> Option<&'static smolvm_cuda::shm::ShmRegion> { + smolvm_cuda::shm::get_or_create() +} +#[cfg(not(target_os = "linux"))] +fn shm_region() -> Option<&'static ()> { + None +} + +// ---- guest-RAM zero-copy (microVM) ------------------------------------------ +// When SMOLVM_CUDA_ZEROCOPY is set, `cudaMallocHost`/`cudaHostAlloc` return a +// page-aligned, mlocked buffer whose guest-physical frames we read from +// /proc/self/pagemap. A memcpy on that buffer then ships the guest-physical +// segment list, and the host (which maps guest RAM via krun_get_guest_ram) +// reads it directly. Requires the guest process to have CAP_SYS_ADMIN so +// pagemap exposes real frame numbers (microVM workloads typically run as root). + +#[cfg(target_os = "linux")] +mod guestmem { + use std::os::unix::fs::FileExt; + use std::sync::Mutex; + + const PAGE: usize = 4096; + + struct Pinned { + base: usize, + size: usize, // mmap length (page-rounded) + page_gpas: Vec, // guest-physical base of each page + } + static PINNED: Mutex> = Mutex::new(Vec::new()); + + fn enabled() -> bool { + std::env::var_os("SMOLVM_CUDA_ZEROCOPY").is_some() + } + + /// Allocate a page-aligned, mlocked buffer and record its guest-physical + /// frames. `None` (fall back to byte-shipping) if disabled or unavailable. + pub fn alloc(size: usize) -> Option<*mut u8> { + if !enabled() || size == 0 { + return None; + } + let npages = size.div_ceil(PAGE); + let len = npages * PAGE; + let p = unsafe { + libc::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) + }; + if p == libc::MAP_FAILED { + return None; + } + let base = p as usize; + // Pin (present + non-migratable) and fault every page in. + if unsafe { libc::mlock(p, len) } != 0 { + unsafe { libc::munmap(p, len) }; + return None; + } + for i in 0..npages { + unsafe { (p as *mut u8).add(i * PAGE).write_volatile(0) }; + } + match read_gpas(base, npages) { + Some(page_gpas) => { + PINNED.lock().unwrap().push(Pinned { + base, + size: len, + page_gpas, + }); + Some(p as *mut u8) + } + None => { + unsafe { libc::munmap(p, len) }; + None // pagemap unavailable (no CAP_SYS_ADMIN) → caller falls back + } + } + } + + fn trace() -> bool { + std::env::var_os("SMOLVM_CUDA_ZC_TRACE").is_some() + } + + fn read_gpas(base: usize, npages: usize) -> Option> { + let f = match std::fs::File::open("/proc/self/pagemap") { + Ok(f) => f, + Err(e) => { + if trace() { + eprintln!("[zc] open pagemap failed: {e}"); + } + return None; + } + }; + let mut gpas = Vec::with_capacity(npages); + for i in 0..npages { + let va = base + i * PAGE; + let mut buf = [0u8; 8]; + f.read_exact_at(&mut buf, (va / PAGE) as u64 * 8).ok()?; + let entry = u64::from_le_bytes(buf); + if entry & (1 << 63) == 0 { + if trace() { + eprintln!("[zc] page {i} not present (entry={entry:#x})"); + } + return None; + } + let pfn = entry & ((1u64 << 55) - 1); + if pfn == 0 { + if trace() { + eprintln!("[zc] pagemap PFN hidden (need CAP_SYS_ADMIN); entry={entry:#x}"); + } + return None; + } + gpas.push(pfn * PAGE as u64); + } + if trace() { + eprintln!("[zc] pagemap OK: {npages} pages, gpa[0]={:#x}", gpas[0]); + } + Some(gpas) + } + + /// Coalesced `(gpa, len)` segments for `[ptr, ptr+len)` if it lies wholly in + /// a pinned buffer. + pub fn segments(ptr: usize, len: usize) -> Option> { + if len == 0 { + return None; + } + let reg = PINNED.lock().unwrap(); + let buf = reg + .iter() + .find(|b| ptr >= b.base && ptr + len <= b.base + b.size)?; + let mut segs: Vec<(u64, u64)> = Vec::new(); + let mut cur = ptr - buf.base; + let end = cur + len; + while cur < end { + let page_gpa = buf.page_gpas[cur / PAGE]; + let in_page = cur % PAGE; + let chunk = (PAGE - in_page).min(end - cur); + let gpa = page_gpa + in_page as u64; + match segs.last_mut() { + Some(last) if last.0 + last.1 == gpa => last.1 += chunk as u64, + _ => segs.push((gpa, chunk as u64)), + } + cur += chunk; + } + Some(segs) + } + + pub fn is_pinned(ptr: usize) -> bool { + let reg = PINNED.lock().unwrap(); + reg.iter().any(|b| ptr >= b.base && ptr < b.base + b.size) + } + + pub fn free(ptr: usize) -> bool { + let mut reg = PINNED.lock().unwrap(); + if let Some(i) = reg.iter().position(|b| b.base == ptr) { + let b = reg.remove(i); + unsafe { libc::munmap(b.base as *mut libc::c_void, b.size) }; + true + } else { + false + } + } +} + +#[cfg(not(target_os = "linux"))] +mod guestmem { + pub fn alloc(_: usize) -> Option<*mut u8> { + None + } + pub fn segments(_: usize, _: usize) -> Option> { + None + } + pub fn is_pinned(_: usize) -> bool { + false + } + pub fn free(_: usize) -> bool { + false + } +} + +/// Bump-allocate `size` bytes (256-aligned: ggml asserts host buffers hit +/// TENSOR_ALIGNMENT, and 256 also matches cudaHostAlloc's real alignment). +#[allow(clippy::needless_return)] // `return` is load-bearing across the cfg arms +fn shm_alloc(size: usize) -> Option<*mut u8> { + #[cfg(target_os = "linux")] + { + let r = shm_region()?; + let sz = (size as u64 + 255) & !255; + let off = SHM_NEXT.fetch_add(sz, Ordering::Relaxed); + if off + sz > r.len() as u64 { + return None; // region exhausted → caller falls back + } + return Some(unsafe { r.base().add(off as usize) }); + } + #[cfg(not(target_os = "linux"))] + { + let _ = size; + None + } +} + +/// If `ptr` lies within the shared region, return its offset. +fn shm_offset(ptr: *const c_void) -> Option { + #[cfg(target_os = "linux")] + { + let r = shm_region()?; + let base = r.base() as usize; + let p = ptr as usize; + if p >= base && p < base + r.len() { + return Some((p - base) as u64); + } + None + } + #[cfg(not(target_os = "linux"))] + { + let _ = ptr; + None + } +} + +fn cuda_host_malloc(ptr: *mut *mut c_void, size: usize) -> c_int { + // Zero-copy backings, in order of preference: guest-RAM (microVM) then the + // same-host shared region. Either lets a memcpy skip shipping the bytes. + if let Some(mem) = guestmem::alloc(size) { + return set_last(unsafe { out(ptr, mem as *mut c_void) }); + } + if let Some(mem) = shm_alloc(size) { + return set_last(unsafe { out(ptr, mem as *mut c_void) }); + } + let layout = match std::alloc::Layout::from_size_align(size, 256) { + Ok(l) => l, + Err(_) => return set_last(CUDA_ERROR_INVALID_VALUE), + }; + let mem = unsafe { std::alloc::alloc(layout) }; + if mem.is_null() { + return set_last(CUDA_ERROR_MEMORY_ALLOCATION); + } + set_last( + match with_state(|s| { + s.host_allocs.insert(mem as usize, layout); + Ok(()) + }) { + Ok(()) => unsafe { out(ptr, mem as *mut c_void) }, + Err(e) => { + unsafe { std::alloc::dealloc(mem, layout) }; + e + } + }, + ) +} + +#[no_mangle] +pub extern "C" fn cudaFreeHost(ptr: *mut c_void) -> c_int { + if ptr.is_null() { + return set_last(CUDA_SUCCESS); + } + // Guest-RAM pinned buffers: munmap + unpin. + if guestmem::free(ptr as usize) { + return set_last(CUDA_SUCCESS); + } + // Shared-region allocations are bump-allocated; freeing is a no-op. + if shm_offset(ptr as *const c_void).is_some() { + return set_last(CUDA_SUCCESS); + } + set_last( + match with_state(|s| Ok(s.host_allocs.remove(&(ptr as usize)))) { + Ok(Some(layout)) => { + unsafe { std::alloc::dealloc(ptr as *mut u8, layout) }; + CUDA_SUCCESS + } + Ok(None) => CUDA_ERROR_INVALID_VALUE, + Err(e) => e, + }, + ) +} + +/// Resolve `cudaMemcpyDefault` to a concrete direction from tracked allocations. +fn resolve_kind(s: &ShimState, dst: *const c_void, src: *const c_void, kind: c_int) -> c_int { + if kind != MEMCPY_DEFAULT { + return kind; + } + let dst_dev = dev_contains(&s.dev_allocs, dst as u64); + let src_dev = dev_contains(&s.dev_allocs, src as u64); + match (src_dev, dst_dev) { + (false, true) => MEMCPY_HTOD, + (true, false) => MEMCPY_DTOH, + (true, true) => MEMCPY_DTOD, + (false, false) => MEMCPY_HTOH, + } +} + +fn do_memcpy(dst: *mut c_void, src: *const c_void, n: usize, kind: c_int, stream: u64) -> c_int { + let dbg = std::env::var_os("SMOLVM_CUDA_TRACE_MEMCPY").is_some(); + let r = do_memcpy_inner(dst, src, n, kind, stream); + if dbg && r != CUDA_SUCCESS { + eprintln!("[memcpy-err] kind={kind} n={n} -> {r}"); + } + r +} + +fn do_memcpy_inner( + dst: *mut c_void, + src: *const c_void, + n: usize, + kind: c_int, + stream: u64, +) -> c_int { + with_state(|s| { + let kind = resolve_kind(s, dst, src, kind); + match kind { + MEMCPY_HTOH => { + if n > 0 && (dst.is_null() || src.is_null()) { + return Err(CUDA_ERROR_INVALID_VALUE); + } + unsafe { std::ptr::copy(src as *const u8, dst as *mut u8, n) }; + Ok(()) + } + MEMCPY_HTOD => { + // Zero-copy from a pinned guest buffer: ship guest-physical + // segments; the host reads guest RAM directly. Fall back to + // byte-shipping if the host can't serve it (no mapping). + if let Some(segs) = guestmem::segments(src as usize, n) { + if s.client.memcpy_gpa_htod(dst as u64, segs, stream).is_ok() { + return Ok(()); + } + } + // Zero-copy from the same-host shared region: ship the offset. + if let Some(off) = shm_offset(src) { + return s + .client + .memcpy_shm_htod(dst as u64, off, n as u64, stream) + .map_err(map_err); + } + // Chunk: one frame must stay far below the transport's + // 256 MiB message cap (a 272 MiB embedding tensor here killed + // the connection). Host-synchronous copies chunk safely. + let data = unsafe { std::slice::from_raw_parts(src as *const u8, n) }; + const CHUNK: usize = 64 * 1024 * 1024; + for (i, piece) in data.chunks(CHUNK).enumerate() { + s.client + .memcpy_htod(dst as u64 + (i * CHUNK) as u64, piece, stream) + .map_err(map_err)?; + } + Ok(()) + } + MEMCPY_DTOH => { + if let Some(segs) = guestmem::segments(dst as usize, n) { + if s.client.memcpy_gpa_dtoh(src as u64, segs, stream).is_ok() { + return Ok(()); + } + } + if let Some(off) = shm_offset(dst) { + // Host writes straight into the shared region at `off`. + return s + .client + .memcpy_shm_dtoh(off, src as u64, n as u64, stream) + .map_err(map_err); + } + const CHUNK: usize = 64 * 1024 * 1024; // see H2D: stay under the frame cap + let mut off = 0; + while off < n { + let c = (n - off).min(CHUNK); + let data = s + .client + .memcpy_dtoh(src as u64 + off as u64, c as u64, stream) + .map_err(map_err)?; + if data.len() != c { + return Err(CUDA_ERROR_UNKNOWN); + } + unsafe { + std::ptr::copy_nonoverlapping(data.as_ptr(), (dst as *mut u8).add(off), c) + }; + off += c; + } + Ok(()) + } + MEMCPY_DTOD => s + .client + .memcpy_dtod(dst as u64, src as u64, n as u64) + .map_err(map_err), + _ => Err(CUDA_ERROR_INVALID_VALUE), + } + }) + .err() + .unwrap_or(CUDA_SUCCESS) +} + +#[no_mangle] +pub extern "C" fn cudaMemcpy(dst: *mut c_void, src: *const c_void, n: usize, kind: c_int) -> c_int { + set_last(do_memcpy(dst, src, n, kind, 0)) +} + +#[no_mangle] +pub extern "C" fn cudaMemcpyAsync( + dst: *mut c_void, + src: *const c_void, + n: usize, + kind: c_int, + stream: *mut c_void, +) -> c_int { + // Device-to-device goes through the stream-ordered driver call: it + // pipelines like a launch and — critically — records into an active graph + // capture instead of invalidating it (the sync form is capture-unsafe). + let resolved = with_state(|s| Ok(resolve_kind(s, dst, src, kind))).unwrap_or(kind); + if resolved == MEMCPY_DTOD { + return set_last( + match with_client(|c| { + c.memcpy_dtod_async(dst as u64, src as u64, n as u64, stream as u64) + }) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }, + ); + } + // Other kinds complete before returning (the CUDA API permits a more + // synchronous implementation), but the host orders the copy after prior + // work on `stream` first — torch's non-blocking pool streams don't order + // against the NULL-stream copy the host uses, so dropping the stream let + // a copy overwrite buffers that still-running kernels were reading. + set_last(do_memcpy(dst, src, n, kind, stream as u64)) +} + +#[no_mangle] +pub extern "C" fn cudaMemset(dev_ptr: *mut c_void, value: c_int, count: usize) -> c_int { + set_last( + match with_client(|c| c.memset_d8(dev_ptr as u64, value as u8, count as u64)) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }, + ) +} + +#[no_mangle] +pub extern "C" fn cudaMemsetAsync( + dev_ptr: *mut c_void, + value: c_int, + count: usize, + stream: *mut c_void, +) -> c_int { + // Stream-ordered driver call: pipelines, and records into an active graph + // capture instead of invalidating it. + set_last( + match with_client(|c| { + c.memset_d8_async(dev_ptr as u64, value as u8, count as u64, stream as u64) + }) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }, + ) +} + +// ---- streams ---------------------------------------------------------------- + +#[no_mangle] +pub extern "C" fn cudaStreamCreate(stream: *mut *mut c_void) -> c_int { + set_last(match with_client(|c| c.stream_create(0)) { + Ok(h) => unsafe { out(stream, h as *mut c_void) }, + Err(e) => e, + }) +} + +#[no_mangle] +pub extern "C" fn cudaStreamCreateWithFlags(stream: *mut *mut c_void, flags: c_uint) -> c_int { + set_last(match with_client(|c| c.stream_create(flags)) { + Ok(h) => unsafe { out(stream, h as *mut c_void) }, + Err(e) => e, + }) +} + +#[no_mangle] +pub extern "C" fn cudaStreamDestroy(stream: *mut c_void) -> c_int { + if stream.is_null() { + return set_last(CUDA_SUCCESS); // destroying the default stream is a no-op + } + set_last(match with_client(|c| c.stream_destroy(stream as u64)) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }) +} + +#[no_mangle] +pub extern "C" fn cudaStreamSynchronize(stream: *mut c_void) -> c_int { + set_last(match with_client(|c| c.stream_synchronize(stream as u64)) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }) +} + +/// `cudaLaunchHostFunc` — a host callback that must run AFTER all prior work on +/// `stream`. The deferred pipeline means that prior work may not have executed +/// host-side yet, so we synchronize the stream first; invoking the callback +/// immediately (as the old stub did) let it observe stale GPU results. +#[no_mangle] +pub extern "C" fn cudaLaunchHostFunc( + stream: *mut c_void, + func: Option, + user_data: *mut c_void, +) -> c_int { + let rc = with_client(|c| c.stream_synchronize(stream as u64)); + if let Err(e) = rc { + return set_last(e); + } + if let Some(f) = func { + unsafe { f(user_data) }; + } + CUDA_SUCCESS +} + +// ---- device queries, events, stream/mempool surface (PyTorch runtime API) ---- +// +// Most of this is forward-to-host or no-op: the host serves every connection on +// one thread in call order, so stream/event ordering is implicit and query +// APIs (stream/event "ready?", capture status) can answer synchronously. + +// CUdevice_attribute values used to assemble cudaDeviceProp. +const A_MAX_THREADS_PER_BLOCK: i32 = 1; +const A_MAX_BLOCK_DIM_X: i32 = 2; +const A_MAX_BLOCK_DIM_Y: i32 = 3; +const A_MAX_BLOCK_DIM_Z: i32 = 4; +const A_MAX_GRID_DIM_X: i32 = 5; +const A_MAX_GRID_DIM_Y: i32 = 6; +const A_MAX_GRID_DIM_Z: i32 = 7; +const A_MAX_SHMEM_PER_BLOCK: i32 = 8; +const A_TOTAL_CONST_MEM: i32 = 9; +const A_WARP_SIZE: i32 = 10; +const A_MAX_REGS_PER_BLOCK: i32 = 12; +const A_CLOCK_RATE: i32 = 13; +const A_MP_COUNT: i32 = 16; +const A_KERNEL_EXEC_TIMEOUT: i32 = 17; +const A_CONCURRENT_KERNELS: i32 = 31; +const A_PCI_BUS_ID: i32 = 33; +const A_PCI_DEVICE_ID: i32 = 34; +const A_MEMORY_CLOCK_RATE: i32 = 36; +const A_MEMORY_BUS_WIDTH: i32 = 37; +const A_L2_CACHE_SIZE: i32 = 38; +const A_MAX_THREADS_PER_MP: i32 = 39; +const A_ASYNC_ENGINE_COUNT: i32 = 40; +const A_PCI_DOMAIN_ID: i32 = 50; +const A_COMPUTE_MAJOR: i32 = 75; +const A_COMPUTE_MINOR: i32 = 76; +const A_MAX_SHMEM_PER_MP: i32 = 81; +const A_MAX_REGS_PER_MP: i32 = 82; +const A_MANAGED_MEMORY: i32 = 83; +const A_CONCURRENT_MANAGED_ACCESS: i32 = 89; +const A_COMPUTE_PREEMPTION: i32 = 90; +const A_COOPERATIVE_LAUNCH: i32 = 95; +const A_COOPERATIVE_MULTI_DEVICE: i32 = 96; +const A_SINGLE_TO_DOUBLE_PERF: i32 = 87; +const A_MAX_SHMEM_PER_BLOCK_OPTIN: i32 = 97; +const A_HOST_REGISTER_SUPPORTED: i32 = 99; +const A_SPARSE_CUDA_ARRAY: i32 = 112; +const A_READ_ONLY_HOST_REGISTER: i32 = 113; +const A_MAX_BLOCKS_PER_MP: i32 = 106; +const A_MAX_PERSISTING_L2: i32 = 108; +const A_MAX_ACCESS_POLICY_WINDOW: i32 = 109; +const A_RESERVED_SHMEM_PER_BLOCK: i32 = 111; +const A_TIMELINE_SEMAPHORE: i32 = 114; + +/// Immutable per-(device, attribute) cache (see cudaDeviceGetAttribute). +static DEV_ATTRS: Mutex>> = Mutex::new(None); +fn dev_attr_cached(device: c_int, attr: c_int) -> Option { + DEV_ATTRS + .lock() + .ok()? + .get_or_insert_with(HashMap::new) + .get(&(device, attr)) + .copied() +} +fn dev_attr_store(device: c_int, attr: c_int, v: c_int) { + if let Ok(mut g) = DEV_ATTRS.lock() { + g.get_or_insert_with(HashMap::new).insert((device, attr), v); + } +} + +#[no_mangle] +pub extern "C" fn cudaDeviceGetAttribute(value: *mut c_int, attr: c_int, device: c_int) -> c_int { + // Device attributes are immutable — memoize to spare a host round-trip on + // every repeat (torch queries them thousands of times; a remote server's + // network RTT makes each one expensive). + if let Some(v) = dev_attr_cached(device, attr) { + return set_last(unsafe { out(value, v) }); + } + set_last( + match with_client(|c| c.device_get_attribute(attr, device)) { + Ok(v) => { + dev_attr_store(device, attr, v); + unsafe { out(value, v) } + } + Err(e) => e, + }, + ) +} + +#[no_mangle] +pub extern "C" fn cudaMemGetInfo(free: *mut usize, total: *mut usize) -> c_int { + set_last(match with_client(|c| c.mem_get_info()) { + Ok((f, t)) => unsafe { + let _ = out(free, f as usize); + out(total, t as usize) + }, + Err(e) => e, + }) +} + +/// `cudaDeviceProp`, the exact CUDA 12.x layout (1032 bytes). Offsets verified +/// against the real bundled `libcudart.so.12` filling the struct on this +/// machine, and pinned by the compile-time assertions below — a missing field +/// silently shifts everything after it (that bug has bitten twice: uuid, and a +/// mis-sized texture block that landed the tail up to 76 bytes off). +#[repr(C)] +struct CudaDeviceProp { + name: [u8; 256], + uuid: [u8; 16], + luid: [u8; 8], + luid_device_node_mask: c_uint, + total_global_mem: usize, + shared_mem_per_block: usize, + regs_per_block: c_int, + warp_size: c_int, + mem_pitch: usize, + max_threads_per_block: c_int, + max_threads_dim: [c_int; 3], + max_grid_size: [c_int; 3], + clock_rate: c_int, + total_const_mem: usize, + major: c_int, + minor: c_int, + texture_alignment: usize, + texture_pitch_alignment: usize, + device_overlap: c_int, + multi_processor_count: c_int, + kernel_exec_timeout_enabled: c_int, + integrated: c_int, + can_map_host_memory: c_int, + compute_mode: c_int, + _tex_surf: [c_int; 40], // maxTexture*/maxSurface* block (unused, left zero) + surface_alignment: usize, + concurrent_kernels: c_int, + ecc_enabled: c_int, + pci_bus_id: c_int, + pci_device_id: c_int, + pci_domain_id: c_int, + tcc_driver: c_int, + async_engine_count: c_int, + unified_addressing: c_int, + memory_clock_rate: c_int, + memory_bus_width: c_int, + l2_cache_size: c_int, + persisting_l2_cache_max_size: c_int, + max_threads_per_multiprocessor: c_int, + stream_priorities_supported: c_int, + global_l1_cache_supported: c_int, + local_l1_cache_supported: c_int, + shared_mem_per_multiprocessor: usize, + regs_per_multiprocessor: c_int, + managed_memory: c_int, + is_multi_gpu_board: c_int, + multi_gpu_board_group_id: c_int, + host_native_atomic_supported: c_int, + single_to_double_precision_perf_ratio: c_int, + pageable_memory_access: c_int, + concurrent_managed_access: c_int, + compute_preemption_supported: c_int, + can_use_host_pointer_for_registered_mem: c_int, + cooperative_launch: c_int, + cooperative_multi_device_launch: c_int, + shared_mem_per_block_optin: usize, + pageable_memory_access_uses_host_page_tables: c_int, + direct_managed_mem_access_from_host: c_int, + max_blocks_per_multiprocessor: c_int, + access_policy_max_window_size: c_int, + reserved_shared_mem_per_block: usize, + host_register_supported: c_int, + sparse_cuda_array_supported: c_int, + host_register_read_only_supported: c_int, + timeline_semaphore_interop_supported: c_int, + memory_pools_supported: c_int, + gpu_direct_rdma_supported: c_int, + gpu_direct_rdma_flush_writes_options: c_uint, + gpu_direct_rdma_writes_ordering: c_int, + memory_pool_supported_handle_types: c_uint, + deferred_mapping_cuda_array_supported: c_int, + ipc_event_supported: c_int, + cluster_launch: c_int, + unified_function_pointers: c_int, + _reserved: [c_int; 63], +} + +// Anchor offsets measured from the real 12.4 cudart on this machine; a layout +// drift fails the build instead of shipping a silently shifted struct. +const _: () = { + assert!(std::mem::offset_of!(CudaDeviceProp, clock_rate) == 348); + assert!(std::mem::offset_of!(CudaDeviceProp, multi_processor_count) == 388); + assert!(std::mem::offset_of!(CudaDeviceProp, _tex_surf) == 408); + assert!(std::mem::offset_of!(CudaDeviceProp, memory_clock_rate) == 608); + assert!(std::mem::offset_of!(CudaDeviceProp, max_threads_per_multiprocessor) == 624); + assert!(std::mem::offset_of!(CudaDeviceProp, regs_per_multiprocessor) == 648); + assert!(std::mem::offset_of!(CudaDeviceProp, shared_mem_per_block_optin) == 696); + assert!(std::mem::offset_of!(CudaDeviceProp, reserved_shared_mem_per_block) == 720); + assert!(std::mem::size_of::() == 1032); +}; + +/// CUDA 13 entry point: 13.x renamed the symbol back from `_v2` AND changed +/// the struct (1008 bytes, clock-rate fields removed, everything after +/// `canMapHostMemory` shifted). Callers compiled against 13.x land here; +/// 12.x callers keep `_v2` and its layout. Offsets measured from the 13.3 +/// headers (scratchpad probe), values fetched like the 12.x path. +#[no_mangle] +pub extern "C" fn cudaGetDeviceProperties(prop: *mut c_void, device: c_int) -> c_int { + if prop.is_null() { + return set_last(CUDA_ERROR_INVALID_VALUE); + } + unsafe { std::ptr::write_bytes(prop as *mut u8, 0, 1008) }; + set_last( + with_state(|s| { + let a = |s: &mut ShimState, attr: i32, dflt: i32| { + s.client.device_get_attribute(attr, device).unwrap_or(dflt) + }; + let name = s.client.device_get_name(device).unwrap_or_default(); + let uuid = s.client.device_get_uuid(device).unwrap_or([0; 16]); + let total = s.client.device_total_mem(device).unwrap_or(0); + let base = prop as *mut u8; + let wi = |off: usize, v: i32| unsafe { base.add(off).cast::().write_unaligned(v) }; + let wu = |off: usize, v: u64| unsafe { base.add(off).cast::().write_unaligned(v) }; + let nb = name.as_bytes(); + let n = nb.len().min(255); + unsafe { + std::ptr::copy_nonoverlapping(nb.as_ptr(), base, n); + std::ptr::copy_nonoverlapping(uuid.as_ptr(), base.add(256), 16); + } + wu(288, total); // totalGlobalMem + wu(296, a(s, A_MAX_SHMEM_PER_BLOCK, 49152) as u64); + wi(304, a(s, A_MAX_REGS_PER_BLOCK, 65536)); + wi(308, a(s, A_WARP_SIZE, 32)); + wu(312, 2147483647); // memPitch + wi(320, a(s, A_MAX_THREADS_PER_BLOCK, 1024)); + wi(324, a(s, A_MAX_BLOCK_DIM_X, 1024)); + wi(328, a(s, A_MAX_BLOCK_DIM_Y, 1024)); + wi(332, a(s, A_MAX_BLOCK_DIM_Z, 64)); + wi(336, a(s, A_MAX_GRID_DIM_X, 2147483647)); + wi(340, a(s, A_MAX_GRID_DIM_Y, 65535)); + wi(344, a(s, A_MAX_GRID_DIM_Z, 65535)); + wu(352, a(s, A_TOTAL_CONST_MEM, 65536) as u64); + wi(360, a(s, A_COMPUTE_MAJOR, 8)); + wi(364, a(s, A_COMPUTE_MINOR, 6)); + wu(368, 512); // textureAlignment + wu(376, 32); // texturePitchAlignment + wi(384, a(s, A_MP_COUNT, 1)); + wi(392, 1); // canMapHostMemory + wi(560, a(s, A_CONCURRENT_KERNELS, 1)); + wi(584, a(s, A_ASYNC_ENGINE_COUNT, 2)); + wi(588, 1); // unifiedAddressing + wi(592, a(s, A_MEMORY_BUS_WIDTH, 0)); + wi(596, a(s, A_L2_CACHE_SIZE, 0)); + wi(600, a(s, A_MAX_PERSISTING_L2, 0)); + wi(604, a(s, A_MAX_THREADS_PER_MP, 1536)); + wi(608, 1); // streamPrioritiesSupported + wi(612, 1); // globalL1CacheSupported + wi(616, 1); // localL1CacheSupported + wu(624, a(s, A_MAX_SHMEM_PER_MP, 102400) as u64); + wi(632, a(s, A_MAX_REGS_PER_MP, 65536)); + wi(636, a(s, A_MANAGED_MEMORY, 1)); + wi(656, a(s, A_CONCURRENT_MANAGED_ACCESS, 1)); + wi(660, a(s, A_COMPUTE_PREEMPTION, 1)); + wi(668, a(s, A_COOPERATIVE_LAUNCH, 1)); + wu(672, a(s, A_MAX_SHMEM_PER_BLOCK_OPTIN, 101376) as u64); + wi(688, a(s, A_MAX_BLOCKS_PER_MP, 16)); + wi(692, a(s, A_MAX_ACCESS_POLICY_WINDOW, 0)); + wu(696, a(s, A_RESERVED_SHMEM_PER_BLOCK, 0) as u64); + wi(704, a(s, A_HOST_REGISTER_SUPPORTED, 1)); + Ok(()) + }) + .err() + .unwrap_or(CUDA_SUCCESS), + ) +} + +/// Device-flag scheduling hints have no effect through forwarding. +#[no_mangle] +pub extern "C" fn cudaSetDeviceFlags(_flags: c_uint) -> c_int { + CUDA_SUCCESS +} + +/// Whole-graph exec update: report "update failed" — ggml (and torch) fall +/// back to destroying and re-instantiating the graph, which we forward. +#[no_mangle] +pub extern "C" fn cudaGraphExecUpdate( + _exec: *mut c_void, + _graph: *mut c_void, + result_info: *mut c_void, +) -> c_int { + if !result_info.is_null() { + // cudaGraphExecUpdateResultInfo { result, errorNode, errorFromNode } + unsafe { std::ptr::write_bytes(result_info as *mut u8, 0, 24) }; + } + 910 // cudaErrorGraphExecUpdateFailure +} + +/// Cooperative launches need grid-wide sync the transport can't fake; the +/// caller sees NotSupported and picks a non-cooperative path. +#[no_mangle] +pub extern "C" fn cudaLaunchCooperativeKernel( + _func: *const c_void, + _grid: Dim3, + _block: Dim3, + _args: *mut *mut c_void, + _shared: usize, + _stream: *mut c_void, +) -> c_int { + set_last(801) // cudaErrorNotSupported +} + +#[no_mangle] +pub extern "C" fn cublasGetStatusString(_status: c_int) -> *const c_char { + c"cublas status (forwarded by smolvm)".as_ptr() +} + +#[no_mangle] +pub extern "C" fn cudaGetDeviceProperties_v2(prop: *mut c_void, device: c_int) -> c_int { + if prop.is_null() { + return set_last(CUDA_ERROR_INVALID_VALUE); + } + // Zero the caller's whole 12.x cudaDeviceProp (~1032 bytes) first, then fill + // the prefix fields we know. Trailing fields stay a defined zero. + unsafe { std::ptr::write_bytes(prop as *mut u8, 0, 1032) }; + set_last( + with_state(|s| { + let a = |s: &mut ShimState, attr: i32, dflt: i32| { + s.client.device_get_attribute(attr, device).unwrap_or(dflt) + }; + let name = s.client.device_get_name(device).unwrap_or_default(); + let uuid = s.client.device_get_uuid(device).unwrap_or([0; 16]); + let total = s.client.device_total_mem(device).unwrap_or(0); + let major = a(s, A_COMPUTE_MAJOR, 8); + let minor = a(s, A_COMPUTE_MINOR, 6); + let mp = a(s, A_MP_COUNT, 1); + let max_tpb = a(s, A_MAX_THREADS_PER_BLOCK, 1024); + let warp = a(s, A_WARP_SIZE, 32); + let shmem_blk = a(s, A_MAX_SHMEM_PER_BLOCK, 49152); + let regs_blk = a(s, A_MAX_REGS_PER_BLOCK, 65536); + let max_tpm = a(s, A_MAX_THREADS_PER_MP, 1536); + let shmem_mp = a(s, A_MAX_SHMEM_PER_MP, 102400); + let regs_mp = a(s, A_MAX_REGS_PER_MP, 65536); + let const_mem = a(s, A_TOTAL_CONST_MEM, 65536); + let (bx, by, bz) = ( + a(s, A_MAX_BLOCK_DIM_X, 1024), + a(s, A_MAX_BLOCK_DIM_Y, 1024), + a(s, A_MAX_BLOCK_DIM_Z, 64), + ); + let (gx, gy, gz) = ( + a(s, A_MAX_GRID_DIM_X, 2147483647), + a(s, A_MAX_GRID_DIM_Y, 65535), + a(s, A_MAX_GRID_DIM_Z, 65535), + ); + let clock = a(s, A_CLOCK_RATE, 0); + let mem_clock = a(s, A_MEMORY_CLOCK_RATE, 0); + let bus_width = a(s, A_MEMORY_BUS_WIDTH, 0); + let l2 = a(s, A_L2_CACHE_SIZE, 0); + let persist_l2 = a(s, A_MAX_PERSISTING_L2, 0); + let engines = a(s, A_ASYNC_ENGINE_COUNT, 2); + let timeout = a(s, A_KERNEL_EXEC_TIMEOUT, 0); + let shmem_optin = a(s, A_MAX_SHMEM_PER_BLOCK_OPTIN, shmem_mp); + let reserved_shmem = a(s, A_RESERVED_SHMEM_PER_BLOCK, 0); + let max_blocks_mp = a(s, A_MAX_BLOCKS_PER_MP, 16); + let access_window = a(s, A_MAX_ACCESS_POLICY_WINDOW, 0); + let (pci_bus, pci_dev, pci_dom) = ( + a(s, A_PCI_BUS_ID, 0), + a(s, A_PCI_DEVICE_ID, 0), + a(s, A_PCI_DOMAIN_ID, 0), + ); + // SAFETY: `prop` points at a caller-provided cudaDeviceProp we zeroed. + let p = unsafe { &mut *(prop as *mut CudaDeviceProp) }; + let nb = name.as_bytes(); + let n = nb.len().min(255); + p.name[..n].copy_from_slice(&nb[..n]); + p.uuid = uuid; + p.total_global_mem = total as usize; + p.shared_mem_per_block = shmem_blk as usize; + p.regs_per_block = regs_blk; + p.warp_size = warp; + p.mem_pitch = 2147483647; + p.max_threads_per_block = max_tpb; + p.max_threads_dim = [bx, by, bz]; + p.max_grid_size = [gx, gy, gz]; + p.clock_rate = clock; + p.total_const_mem = const_mem as usize; + p.major = major; + p.minor = minor; + p.texture_alignment = 512; + p.texture_pitch_alignment = 32; + p.device_overlap = (engines > 0) as c_int; + p.multi_processor_count = mp; + p.kernel_exec_timeout_enabled = timeout; + p.can_map_host_memory = 1; + p.surface_alignment = 512; + p.concurrent_kernels = a(s, A_CONCURRENT_KERNELS, 1); + p.pci_bus_id = pci_bus; + p.pci_device_id = pci_dev; + p.pci_domain_id = pci_dom; + p.async_engine_count = engines; + p.unified_addressing = 1; + p.memory_clock_rate = mem_clock; + p.memory_bus_width = bus_width; + p.l2_cache_size = l2; + p.persisting_l2_cache_max_size = persist_l2; + p.max_threads_per_multiprocessor = max_tpm; + p.stream_priorities_supported = 1; + p.global_l1_cache_supported = 1; + p.local_l1_cache_supported = 1; + p.shared_mem_per_multiprocessor = shmem_mp as usize; + p.regs_per_multiprocessor = regs_mp; + p.managed_memory = a(s, A_MANAGED_MEMORY, 1); + p.single_to_double_precision_perf_ratio = a(s, A_SINGLE_TO_DOUBLE_PERF, 32); + p.concurrent_managed_access = a(s, A_CONCURRENT_MANAGED_ACCESS, 1); + p.compute_preemption_supported = a(s, A_COMPUTE_PREEMPTION, 1); + p.cooperative_launch = a(s, A_COOPERATIVE_LAUNCH, 1); + p.cooperative_multi_device_launch = a(s, A_COOPERATIVE_MULTI_DEVICE, 1); + p.shared_mem_per_block_optin = shmem_optin as usize; + p.max_blocks_per_multiprocessor = max_blocks_mp; + p.access_policy_max_window_size = access_window; + p.reserved_shared_mem_per_block = reserved_shmem as usize; + p.host_register_supported = a(s, A_HOST_REGISTER_SUPPORTED, 1); + p.timeline_semaphore_interop_supported = a(s, A_TIMELINE_SEMAPHORE, 1); + p.sparse_cuda_array_supported = a(s, A_SPARSE_CUDA_ARRAY, 0); + p.host_register_read_only_supported = a(s, A_READ_ONLY_HOST_REGISTER, 0); + // Deliberately NOT mirrored from the host GPU: capabilities that + // would steer callers onto paths forwarding can't honor. Pageable / + // registered host memory is guest RAM the host GPU can't reach by + // that pointer, mempools + IPC events are stubbed. + // (pageable_memory_access, can_use_host_pointer_for_registered_mem, + // memory_pools_supported, memory_pool_supported_handle_types, + // ipc_event_supported stay 0.) + Ok(()) + }) + .err() + .unwrap_or(CUDA_SUCCESS), + ) +} + +// ---- events (forward to host) ----------------------------------------------- + +#[no_mangle] +pub extern "C" fn cudaEventCreate(event: *mut *mut c_void) -> c_int { + set_last(match with_client(|c| c.event_create(0)) { + Ok(h) => unsafe { out(event, h as *mut c_void) }, + Err(e) => e, + }) +} +#[no_mangle] +pub extern "C" fn cudaEventCreateWithFlags(event: *mut *mut c_void, flags: c_uint) -> c_int { + set_last(match with_client(|c| c.event_create(flags)) { + Ok(h) => unsafe { out(event, h as *mut c_void) }, + Err(e) => e, + }) +} +#[no_mangle] +pub extern "C" fn cudaEventDestroy(event: *mut c_void) -> c_int { + set_last(match with_client(|c| c.event_destroy(event as u64)) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }) +} +#[no_mangle] +pub extern "C" fn cudaEventRecord(event: *mut c_void, stream: *mut c_void) -> c_int { + set_last( + match with_client(|c| c.event_record(event as u64, stream as u64)) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }, + ) +} +#[no_mangle] +pub extern "C" fn cudaEventRecordWithFlags( + event: *mut c_void, + stream: *mut c_void, + _flags: c_uint, +) -> c_int { + cudaEventRecord(event, stream) +} +#[no_mangle] +pub extern "C" fn cudaEventSynchronize(event: *mut c_void) -> c_int { + set_last(match with_client(|c| c.event_synchronize(event as u64)) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }) +} +#[no_mangle] +pub extern "C" fn cudaEventQuery(event: *mut c_void) -> c_int { + // Must be honest: PyTorch's allocator polls this to decide when freed + // blocks are safe to reuse. Always answering "complete" caused premature + // reuse (ILLEGAL_ADDRESS) once work really ran on side streams. NotReady + // (600) latches into last-error exactly like real cudart; torch clears it. + set_last(match with_client(|c| c.event_query(event as u64)) { + Ok(code) => code, + Err(e) => e, + }) +} +#[no_mangle] +pub extern "C" fn cudaEventElapsedTime( + ms: *mut f32, + start: *mut c_void, + end: *mut c_void, +) -> c_int { + set_last( + match with_client(|c| c.event_elapsed_time(start as u64, end as u64)) { + Ok(t) => unsafe { out(ms, t) }, + Err(e) => e, + }, + ) +} + +// ---- streams: priorities, capture queries, callbacks ------------------------ + +#[no_mangle] +pub extern "C" fn cudaStreamCreateWithPriority( + stream: *mut *mut c_void, + flags: c_uint, + _priority: c_int, +) -> c_int { + set_last(match with_client(|c| c.stream_create(flags)) { + Ok(h) => unsafe { out(stream, h as *mut c_void) }, + Err(e) => e, + }) +} +#[no_mangle] +pub extern "C" fn cudaStreamWaitEvent( + stream: *mut c_void, + event: *mut c_void, + flags: c_uint, +) -> c_int { + // A real cross-stream ordering edge now that work runs on side streams + // (and a graph dependency during capture) — dropping it made replays racy + // (ILLEGAL_ADDRESS). Deferred like a launch. + set_last( + match with_client(|c| c.stream_wait_event(stream as u64, event as u64, flags)) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }, + ) +} +#[no_mangle] +pub extern "C" fn cudaStreamQuery(stream: *mut c_void) -> c_int { + // Honest completion status (0 or 600-NotReady), same as cudaEventQuery. + set_last(match with_client(|c| c.stream_query(stream as u64)) { + Ok(code) => code, + Err(e) => e, + }) +} +// ---- CUDA graphs ------------------------------------------------------------- +// Capture happens on the HOST driver: Begin/End forward, and every launch / +// stream-ordered op issued in between lands on the capturing host stream and is +// recorded (not executed) by the real driver. Replay is a single GraphLaunch +// message for the whole graph — the antidote to per-launch round-trips in +// launch-bound inference. The hot capture-status queries answer from the +// guest-side `capture` field, costing nothing outside capture. + +/// cudaStreamCaptureStatusActive. +const CAPTURE_ACTIVE: c_int = 1; + +#[no_mangle] +pub extern "C" fn cudaStreamBeginCapture(stream: *mut c_void, mode: c_int) -> c_int { + set_last( + match with_state(|s| { + // Fire-and-forget: the host starts capture when this drains, and + // the (also-deferred) launches record in order. The capture id is + // torch-visible only (its allocator correlates via the local + // GetCaptureInfo queries; the host tracks capture by stream), so + // mint it locally. Both save a host round-trip per captured graph, + // which dominates coldstart over a network (~1400 graphs). + s.client + .stream_begin_capture_deferred(stream as u64, mode) + .map_err(map_err)?; + static NEXT_ID: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + let id = NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + s.capture = Some((stream as u64, id)); + Ok(()) + }) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }, + ) +} + +#[no_mangle] +pub extern "C" fn cudaStreamEndCapture(stream: *mut c_void, graph: *mut *mut c_void) -> c_int { + if graph.is_null() { + return set_last(CUDA_ERROR_INVALID_VALUE); + } + set_last( + match with_state(|s| { + // Mint a virtual graph handle; the host maps it to the real + // captured graph when this deferred end drains. + let vh = alloc_vhandle(); + s.client + .stream_end_capture_deferred(stream as u64, vh) + .map_err(map_err)?; + s.capture = None; + Ok(vh) + }) { + Ok(g) => unsafe { out(graph, g as *mut c_void) }, + Err(e) => { + let _ = with_state(|s| { + s.capture = None; + Ok(()) + }); + e + } + }, + ) +} + +#[no_mangle] +pub extern "C" fn cudaStreamIsCapturing(stream: *mut c_void, status: *mut c_int) -> c_int { + let active = with_state(|s| Ok(matches!(s.capture, Some((cs, _)) if cs == stream as u64))) + .unwrap_or(false); + set_last(unsafe { out(status, if active { CAPTURE_ACTIVE } else { 0 }) }) +} + +#[no_mangle] +pub extern "C" fn cudaStreamGetCaptureInfo_v2( + stream: *mut c_void, + status: *mut c_int, + id: *mut u64, + graph: *mut *mut c_void, + deps: *mut *mut *const c_void, + num_deps: *mut usize, +) -> c_int { + let cap = with_state(|s| Ok(s.capture)).unwrap_or(None); + let (st, cid) = match cap { + Some((cs, cid)) if cs == stream as u64 => (CAPTURE_ACTIVE, cid), + _ => (0, 0), + }; + unsafe { + let _ = out(status, st); + if !id.is_null() { + let _ = out(id, cid); + } + if !graph.is_null() { + let _ = out(graph, std::ptr::null_mut()); + } + if !deps.is_null() { + let _ = out(deps, std::ptr::null_mut()); + } + if !num_deps.is_null() { + let _ = out(num_deps, 0usize); + } + } + set_last(CUDA_SUCCESS) +} + +#[no_mangle] +pub extern "C" fn cudaGraphInstantiate( + graph_exec: *mut *mut c_void, + graph: *mut c_void, + _error_node: *mut *mut c_void, + _log_buffer: *mut c_char, + _buffer_size: usize, +) -> c_int { + cudaGraphInstantiateWithFlags(graph_exec, graph, 0) +} + +#[no_mangle] +pub extern "C" fn cudaGraphInstantiateWithFlags( + graph_exec: *mut *mut c_void, + graph: *mut c_void, + _flags: u64, +) -> c_int { + if graph_exec.is_null() { + return set_last(CUDA_ERROR_INVALID_VALUE); + } + set_last( + match with_client(|c| { + // Mint a virtual exec handle; host maps it when this deferred + // instantiate drains (graph is itself a virtual graph handle). + let exec_vh = alloc_vhandle(); + c.graph_instantiate_deferred(graph as u64, exec_vh)?; + Ok(exec_vh) + }) { + Ok(e) => unsafe { out(graph_exec, e as *mut c_void) }, + Err(e) => e, + }, + ) +} + +#[no_mangle] +pub extern "C" fn cudaGraphLaunch(graph_exec: *mut c_void, stream: *mut c_void) -> c_int { + // The whole point: one pipelined message replays every captured kernel. + set_last( + match with_client(|c| c.graph_launch(graph_exec as u64, stream as u64)) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }, + ) +} + +/// Count-only node query (`nodes == NULL`): PyTorch uses it to warn about +/// empty captures. Filling a caller-provided node array is not supported. +#[no_mangle] +pub extern "C" fn cudaGraphGetNodes( + _graph: *mut c_void, + nodes: *mut *mut c_void, + num_nodes: *mut usize, +) -> c_int { + if num_nodes.is_null() { + return set_last(CUDA_ERROR_INVALID_VALUE); + } + // The real node list is never requested (nodes != NULL is rejected below); + // torch calls this only with nodes = NULL to check for an EMPTY graph and + // warn. A captured decode graph is never empty, so answer the count query + // locally with a non-zero value instead of a host round-trip — fetching + // the true count cost one RTT per captured graph (~1400), dominating + // coldstart over a network. (This can only suppress a cosmetic empty-graph + // warning, never cause wrong behavior — unlike the fake-data stubs that + // were replaced with real values.) + if !nodes.is_null() { + return set_last(CUDA_ERROR_INVALID_VALUE); + } + set_last(unsafe { out(num_nodes, 1usize) }) +} + +#[no_mangle] +pub extern "C" fn cudaGraphExecDestroy(graph_exec: *mut c_void) -> c_int { + set_last( + match with_client(|c| c.graph_exec_destroy(graph_exec as u64)) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }, + ) +} + +#[no_mangle] +pub extern "C" fn cudaGraphDestroy(graph: *mut c_void) -> c_int { + set_last(match with_client(|c| c.graph_destroy(graph as u64)) { + Ok(()) => CUDA_SUCCESS, + Err(e) => e, + }) +} +#[no_mangle] +pub extern "C" fn cudaThreadExchangeStreamCaptureMode(mode: *mut c_int) -> c_int { + // PyTorch's allocator wraps capture-time cudaMalloc in a relaxed-mode + // guard via this call. The per-thread mode must take effect on the HOST + // thread that will execute the malloc — each connection is served by one + // host thread, so forwarding maps the semantics exactly. A no-op here + // leaves the host thread in global mode and the malloc fails with 900 + // (cudaErrorStreamCaptureUnsupported), invalidating the capture. + if mode.is_null() { + return set_last(CUDA_ERROR_INVALID_VALUE); + } + let new_mode = unsafe { *mode }; + set_last( + match with_client(|c| c.thread_exchange_capture_mode(new_mode)) { + Ok(old) => { + unsafe { *mode = old }; + CUDA_SUCCESS + } + Err(e) => e, + }, + ) +} +/// `cudaStreamCallback_t` = `void (*)(cudaStream_t, cudaError_t, void*)`. +type StreamCallback = unsafe extern "C" fn(*mut c_void, c_int, *mut c_void); +#[no_mangle] +pub extern "C" fn cudaStreamAddCallback( + stream: *mut c_void, + callback: Option, + user_data: *mut c_void, + _flags: c_uint, +) -> c_int { + // The callback must run after all prior work on `stream`. Under the + // deferred pipeline that work may not have executed host-side yet, so + // synchronize first — invoking it immediately let it observe stale results. + if let Err(e) = with_client(|c| c.stream_synchronize(stream as u64)) { + return set_last(e); + } + if let Some(cb) = callback { + unsafe { cb(stream, CUDA_SUCCESS, user_data) }; + } + set_last(CUDA_SUCCESS) +} +#[no_mangle] +pub extern "C" fn cudaDeviceGetStreamPriorityRange( + least: *mut c_int, + greatest: *mut c_int, +) -> c_int { + unsafe { + if !least.is_null() { + let _ = out(least, 0); + } + if !greatest.is_null() { + let _ = out(greatest, 0); + } + } + set_last(CUDA_SUCCESS) +} + +// ---- async malloc / mempool (map to sync alloc; pool stubs) ------------------ + +#[no_mangle] +pub extern "C" fn cudaMallocAsync( + dev_ptr: *mut *mut c_void, + size: usize, + _stream: *mut c_void, +) -> c_int { + cudaMalloc(dev_ptr, size) +} +#[no_mangle] +pub extern "C" fn cudaFreeAsync(dev_ptr: *mut c_void, _stream: *mut c_void) -> c_int { + cudaFree(dev_ptr) +} +#[no_mangle] +pub extern "C" fn cudaDeviceGetDefaultMemPool(pool: *mut *mut c_void, _device: c_int) -> c_int { + // No pool support; hand back a sentinel so callers that only stash it are ok. + set_last(unsafe { out(pool, std::ptr::without_provenance_mut(1)) }) +} +#[no_mangle] +pub extern "C" fn cudaMemPoolSetAttribute( + _pool: *mut c_void, + _attr: c_int, + _value: *mut c_void, +) -> c_int { + set_last(CUDA_SUCCESS) +} +#[no_mangle] +pub extern "C" fn cudaMemPoolGetAttribute( + _pool: *mut c_void, + _attr: c_int, + value: *mut c_void, +) -> c_int { + // Every mempool attribute is an 8-byte value (thresholds are u64, the + // bool/used/reserved counters are i64). Write a defined 0 rather than + // leaving the caller's buffer as stack garbage. + if !value.is_null() { + unsafe { (value as *mut u64).write_unaligned(0) }; + } + set_last(CUDA_SUCCESS) +} +#[no_mangle] +pub extern "C" fn cudaMemPoolSetAccess( + _pool: *mut c_void, + _desc: *const c_void, + _count: usize, +) -> c_int { + set_last(CUDA_SUCCESS) +} +#[no_mangle] +pub extern "C" fn cudaMemPoolTrimTo(_pool: *mut c_void, _min_bytes_to_keep: usize) -> c_int { + set_last(CUDA_SUCCESS) +} + +// ---- pointer / func attributes, occupancy, host register, peer, misc -------- + +/// `cudaPointerAttributes` (CUDA 12.x): `{ int type; int device; void* devPtr; +/// void* hostPtr; }`. +#[repr(C)] +struct CudaPointerAttributes { + memory_type: c_int, + device: c_int, + device_pointer: *mut c_void, + host_pointer: *mut c_void, +} +#[no_mangle] +pub extern "C" fn cudaPointerGetAttributes(attr: *mut c_void, ptr: *const c_void) -> c_int { + if attr.is_null() { + return set_last(CUDA_ERROR_INVALID_VALUE); + } + // cudaMemoryTypeDevice=2 for our device allocations, Host=1 only for + // buffers from cudaMallocHost/cudaHostAlloc (zero-copy, shm or plain), and + // Unregistered=0 for everything else — real cudart reports plain memory as + // unregistered, and PyTorch's `is_pinned()` relies on that: reporting Host + // for arbitrary pointers made `pin_memory()` a silent no-op, so pinned + // transfers never reached the zero-copy path. + let is_dev = with_state(|s| Ok(dev_contains(&s.dev_allocs, ptr as u64))).unwrap_or(false); + let is_pinned_host = !is_dev + && (guestmem::is_pinned(ptr as usize) + || shm_offset(ptr).is_some() + || with_state(|s| { + Ok(s.host_allocs + .iter() + .any(|(b, l)| ptr as usize >= *b && (ptr as usize) < b + l.size())) + }) + .unwrap_or(false)); + // SAFETY: caller-provided cudaPointerAttributes. + let a = unsafe { &mut *(attr as *mut CudaPointerAttributes) }; + a.memory_type = if is_dev { + 2 + } else if is_pinned_host { + 1 + } else { + 0 + }; + a.device = 0; + a.device_pointer = if is_dev { + ptr as *mut c_void + } else { + std::ptr::null_mut() + }; + a.host_pointer = if is_pinned_host { + ptr as *mut c_void + } else { + std::ptr::null_mut() + }; + set_last(CUDA_SUCCESS) +} +#[no_mangle] +pub extern "C" fn cudaFuncGetAttributes(attr: *mut c_void, func: *const c_void) -> c_int { + if attr.is_null() { + return set_last(CUDA_ERROR_INVALID_VALUE); + } + // cudaFuncAttributes: sharedSizeBytes, constSizeBytes, localSizeBytes + // (size_t @ 0/8/16) then maxThreadsPerBlock, numRegs, ptxVersion, + // binaryVersion (i32 @ 24/28/32/36). Forward the real values — the old + // fixed fakes (numRegs=0) divided by zero in occupancy math. + unsafe { std::ptr::write_bytes(attr as *mut u8, 0, 72) }; + // Kernel attributes are immutable — memoize the packed 72-byte blob per + // function (torch may query per-launch; each miss is 7 host round-trips). + static FUNC_ATTRS: Mutex>> = Mutex::new(None); + if let Ok(mut g) = FUNC_ATTRS.lock() { + if let Some(blob) = g.get_or_insert_with(HashMap::new).get(&(func as usize)) { + unsafe { std::ptr::copy_nonoverlapping(blob.as_ptr(), attr as *mut u8, 72) }; + return set_last(CUDA_SUCCESS); + } + } + // CUfunction_attribute: MAX_THREADS_PER_BLOCK=0, SHARED=1, CONST=2, + // LOCAL=3, NUM_REGS=4, PTX_VERSION=5, BINARY_VERSION=6. + let r = with_state(|s| { + let fid = s + .funcs + .get(&(func as usize)) + .ok_or(CUDA_ERROR_INVALID_DEVICE_POINTER)? + .fid; + let get = |s: &mut ShimState, a: i32| s.client.func_get_attribute(fid, a).unwrap_or(0); + let shared = get(s, 1); + let cst = get(s, 2); + let local = get(s, 3); + let max_tpb = get(s, 0); + let num_regs = get(s, 4); + let ptx = get(s, 5); + let bin = get(s, 6); + unsafe { + let base = attr as *mut u8; + (base as *mut usize).write_unaligned(shared.max(0) as usize); + (base.add(8) as *mut usize).write_unaligned(cst.max(0) as usize); + (base.add(16) as *mut usize).write_unaligned(local.max(0) as usize); + let ints = base.add(24) as *mut c_int; + *ints = if max_tpb > 0 { max_tpb } else { 1024 }; + *ints.add(1) = if num_regs > 0 { num_regs } else { 1 }; // never 0 (occupancy div) + *ints.add(2) = if ptx > 0 { ptx } else { 86 }; + *ints.add(3) = if bin > 0 { bin } else { 86 }; + } + Ok(()) + }); + if r.is_ok() { + if let Ok(mut g) = FUNC_ATTRS.lock() { + let mut blob = [0u8; 72]; + unsafe { std::ptr::copy_nonoverlapping(attr as *const u8, blob.as_mut_ptr(), 72) }; + g.get_or_insert_with(HashMap::new) + .insert(func as usize, blob); + } + } + set_last(r.err().unwrap_or(CUDA_SUCCESS)) +} +/// Forward the shared-memory opt-in (`cudaFuncAttributeMaxDynamicSharedMemorySize` +/// = 8, matching the driver's `CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES`) +/// to the host function. FlashAttention/cutlass kernels needing >48 KiB shared +/// memory raise it here before launching, or the launch fails with INVALID_VALUE. +/// The runtime and driver enum values coincide, so `attr` passes through. +#[no_mangle] +pub extern "C" fn cudaFuncSetAttribute(func: *const c_void, attr: c_int, value: c_int) -> c_int { + // Kernels re-assert the same attribute before every launch (FlashAttention + // raises the shared-memory cap each call) — skip repeats, each was a sync + // round-trip. + static APPLIED: Mutex>> = Mutex::new(None); + if APPLIED + .lock() + .unwrap() + .get_or_insert_with(HashMap::new) + .get(&(func as usize, attr)) + == Some(&value) + { + return CUDA_SUCCESS; + } + let r = with_state(|s| { + let fid = s + .funcs + .get(&(func as usize)) + .ok_or(CUDA_ERROR_INVALID_DEVICE_POINTER)? + .fid; + s.client + .func_set_attribute(fid, attr, value) + .map_err(map_err) + }); + if r.is_ok() { + APPLIED + .lock() + .unwrap() + .get_or_insert_with(HashMap::new) + .insert((func as usize, attr), value); + } + set_last(r.err().unwrap_or(CUDA_SUCCESS)) +} +#[no_mangle] +pub extern "C" fn cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags( + num_blocks: *mut c_int, + _func: *const c_void, + block_size: c_int, + _dynamic_smem: usize, + _flags: c_uint, +) -> c_int { + // Coarse estimate: 2048 threads/SM cap divided by the block size, ≥1. + let bs = block_size.max(1); + set_last(unsafe { out(num_blocks, (2048 / bs).clamp(1, 32)) }) +} +#[no_mangle] +pub extern "C" fn cudaHostRegister(_ptr: *mut c_void, _size: usize, _flags: c_uint) -> c_int { + set_last(CUDA_SUCCESS) +} +#[no_mangle] +pub extern "C" fn cudaHostUnregister(_ptr: *mut c_void) -> c_int { + set_last(CUDA_SUCCESS) +} +#[no_mangle] +pub extern "C" fn cudaHostGetDevicePointer( + p_device: *mut *mut c_void, + host: *mut c_void, + _flags: c_uint, +) -> c_int { + // Unified-addressing convention: device VA == host VA. Correct for the + // memcpy path (the host recognizes guest-RAM addresses and DMAs them). + // KNOWN LIMITATION: a mapped host pointer passed as a KERNEL ARG can't be + // dereferenced by the host GPU (a guest VA isn't device-addressable) — that + // needs true unified memory we don't have. No validated workload does this; + // those that would should use explicit cudaMemcpy instead. + set_last(unsafe { out(p_device, host) }) +} +#[no_mangle] +pub extern "C" fn cudaDeviceCanAccessPeer(can: *mut c_int, _device: c_int, _peer: c_int) -> c_int { + set_last(unsafe { out(can, 0) }) // single device +} +#[no_mangle] +pub extern "C" fn cudaDeviceEnablePeerAccess(_peer: c_int, _flags: c_uint) -> c_int { + set_last(CUDA_SUCCESS) +} +#[no_mangle] +pub extern "C" fn cudaDeviceGetPCIBusId(buf: *mut c_char, len: c_int, _device: c_int) -> c_int { + let id = b"0000:01:00.0\0"; + if !buf.is_null() && len > 0 { + let n = (len as usize - 1).min(id.len() - 1); + unsafe { std::ptr::copy_nonoverlapping(id.as_ptr() as *const c_char, buf, n) }; + unsafe { *buf.add(n) = 0 }; + } + set_last(CUDA_SUCCESS) +} + +// ---- kernel registration + launch ------------------------------------------- + +/// `__fatBinC_Wrapper_t`: what `__cudaRegisterFatBinary` receives. `data` points +/// at the fatbin container (its own header carries the length). +#[repr(C)] +struct FatBinWrapper { + magic: c_int, + version: c_int, + data: *const c_void, + filename_or_fatbins: *const c_void, +} + +/// Length of a fatbin container from its header (magic `0xBA55ED50`, +/// u16 version, u16 headerSize, u64 fatSize). +unsafe fn fatbin_len(data: *const c_void) -> Option { + if data.is_null() { + return None; + } + let p = data as *const u8; + let magic = u32::from_le_bytes(unsafe { *(p as *const [u8; 4]) }); + if magic != 0xBA55_ED50 { + return None; + } + let header_size = u16::from_le_bytes(unsafe { *(p.add(6) as *const [u8; 2]) }) as usize; + let fat_size = u64::from_le_bytes(unsafe { *(p.add(8) as *const [u8; 8]) }) as usize; + Some(header_size + fat_size) +} + +#[no_mangle] +pub extern "C" fn __cudaRegisterFatBinary(fat_cubin: *mut c_void) -> *mut *mut c_void { + // Mint a stable handle the app hands back to Register/Unregister; map it to + // the driver module we load from the embedded fatbin. + let handle = Box::into_raw(Box::new(0u8)) as *mut *mut c_void; + if fat_cubin.is_null() { + return handle; + } + let wrapper = fat_cubin as *const FatBinWrapper; + let data = unsafe { (*wrapper).data }; + let Some(len) = (unsafe { fatbin_len(data) }) else { + return handle; + }; + let blob = unsafe { std::slice::from_raw_parts(data as *const u8, len) }.to_vec(); + let _ = with_state(|s| { + match s.client.module_load_data(&blob) { + Ok(module) => { + s.modules.insert(handle as usize, module); + } + Err(e) => return Err(map_err(e)), + } + Ok(()) + }); + handle +} + +#[no_mangle] +pub extern "C" fn __cudaRegisterFatBinaryEnd(_handle: *mut *mut c_void) {} + +/// Called by some CUDA-compiled modules (e.g. torchvision's `_C.so`) during +/// their static init. The real runtime returns `char` 1 (module usable); we do +/// the same — registration proper happens via `__cudaRegisterFunction`. +#[no_mangle] +pub extern "C" fn __cudaInitModule(_fat_cubin_handle: *mut *mut c_void) -> c_char { + 1 +} + +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn __cudaRegisterFunction( + fat_cubin_handle: *mut *mut c_void, + host_fun: *const c_char, + _device_fun: *mut c_char, + device_name: *const c_char, + _thread_limit: c_int, + _tid: *mut c_void, + _bid: *mut c_void, + _b_dim: *mut c_void, + _g_dim: *mut c_void, + _w_size: *mut c_void, +) { + if device_name.is_null() { + return; + } + let name = match unsafe { CStr::from_ptr(device_name) }.to_str() { + Ok(n) => n.to_string(), + Err(_) => return, + }; + let _ = with_state(|s| { + let module = *s + .modules + .get(&(fat_cubin_handle as usize)) + .ok_or(CUDA_ERROR_INVALID_RESOURCE_HANDLE)?; + let cname = CString::new(name.clone()).map_err(|_| CUDA_ERROR_INVALID_VALUE)?; + let fid = s + .client + .module_get_function(module, cname.to_str().unwrap()) + .map_err(map_err)?; + let param_sizes = s.client.func_get_param_info(fid).map_err(map_err)?; + s.funcs + .insert(host_fun as usize, FuncRec { fid, param_sizes }); + Ok(()) + }); +} + +#[no_mangle] +pub extern "C" fn __cudaUnregisterFatBinary(handle: *mut *mut c_void) { + let _ = with_state(|s| { + if let Some(module) = s.modules.remove(&(handle as usize)) { + let _ = s.client.module_unload(module); + } + Ok(()) + }); + if !handle.is_null() { + unsafe { drop(Box::from_raw(handle as *mut u8)) }; + } +} + +#[no_mangle] +pub extern "C" fn __cudaPushCallConfiguration( + grid: Dim3, + block: Dim3, + shared_mem: usize, + stream: *mut c_void, +) -> c_int { + CALL_CONFIG.with(|c| { + c.borrow_mut() + .push((grid, block, shared_mem, stream as u64)) + }); + CUDA_SUCCESS +} + +#[no_mangle] +pub extern "C" fn __cudaPopCallConfiguration( + grid: *mut Dim3, + block: *mut Dim3, + shared_mem: *mut usize, + stream: *mut *mut c_void, +) -> c_int { + CALL_CONFIG.with(|c| { + let cfg = c.borrow_mut().pop(); + match cfg { + Some((g, b, sh, st)) => unsafe { + if !grid.is_null() { + *grid = g; + } + if !block.is_null() { + *block = b; + } + if !shared_mem.is_null() { + *shared_mem = sh; + } + if !stream.is_null() { + *stream = st as *mut c_void; + } + CUDA_SUCCESS + }, + None => CUDA_ERROR_INVALID_VALUE, + } + }) +} + +#[no_mangle] +pub extern "C" fn cudaLaunchKernel( + func: *const c_void, + grid: Dim3, + block: Dim3, + args: *mut *mut c_void, + shared_mem: usize, + stream: *mut c_void, +) -> c_int { + set_last(do_launch( + func, + [grid.x, grid.y, grid.z], + [block.x, block.y, block.z], + shared_mem, + stream as u64, + args, + )) +} + +/// Shared launch path for both `cudaLaunchKernel` and `cudaLaunchKernelExC`: +/// look up the registered function, gather its argument blobs, forward. +fn do_launch( + func: *const c_void, + grid: [u32; 3], + block: [u32; 3], + shared_mem: usize, + stream: u64, + args: *mut *mut c_void, +) -> c_int { + // Debug bisection: drop launches entirely to isolate marshaling cost. + if std::env::var_os("SMOLVM_CUDA_NOOP_LAUNCH").is_some() { + return CUDA_SUCCESS; + } + with_state(|s| { + let rec = s + .funcs + .get(&(func as usize)) + .ok_or(CUDA_ERROR_INVALID_DEVICE_POINTER)?; + let fid = rec.fid; + let sizes = rec.param_sizes.clone(); + // Reconstruct one byte-blob per kernel argument from `args[i]`. + let params: Vec> = if sizes.is_empty() { + Vec::new() + } else if args.is_null() { + return Err(CUDA_ERROR_INVALID_VALUE); + } else { + let ptrs = unsafe { std::slice::from_raw_parts(args, sizes.len()) }; + sizes + .iter() + .zip(ptrs) + .map(|(&sz, &p)| { + unsafe { std::slice::from_raw_parts(p as *const u8, sz as usize) }.to_vec() + }) + .collect() + }; + s.client + .launch_kernel(fid, grid, block, shared_mem as u32, stream, ¶ms) + .map_err(map_err) + }) + .err() + .unwrap_or(CUDA_SUCCESS) +} + +/// `cudaLaunchConfig_t` (CUDA 12): grid/block dims, dynamic shared bytes, stream, +/// then an attribute array we ignore (cluster dims etc. are not forwarded). +#[repr(C)] +struct CudaLaunchConfig { + grid_dim: Dim3, + block_dim: Dim3, + dynamic_smem_bytes: usize, + stream: *mut c_void, + attrs: *mut c_void, + num_attrs: c_uint, +} + +#[no_mangle] +pub extern "C" fn cudaLaunchKernelExC( + config: *const c_void, + func: *const c_void, + args: *mut *mut c_void, +) -> c_int { + if config.is_null() { + return set_last(CUDA_ERROR_INVALID_VALUE); + } + // SAFETY: caller passes a valid cudaLaunchConfig_t. + let c = unsafe { &*(config as *const CudaLaunchConfig) }; + set_last(do_launch( + func, + [c.grid_dim.x, c.grid_dim.y, c.grid_dim.z], + [c.block_dim.x, c.block_dim.y, c.block_dim.z], + c.dynamic_smem_bytes, + c.stream as u64, + args, + )) +} + +// CUDA 12.0+ launch path. nvcc-generated stubs resolve a "kernel handle" via +// __cudaGetKernel(&handle, hostFun) once, then launch through __cudaLaunchKernel. +// We use the host stub pointer itself as the handle, so both reduce to the same +// funcs-table lookup as the classic cudaLaunchKernel. + +#[no_mangle] +pub extern "C" fn __cudaGetKernel(kernel: *mut *const c_void, host_fun: *const c_void) -> c_int { + if kernel.is_null() { + return CUDA_ERROR_INVALID_VALUE; + } + unsafe { *kernel = host_fun }; + CUDA_SUCCESS +} + +#[no_mangle] +pub extern "C" fn __cudaLaunchKernel( + kernel: *const c_void, + grid: Dim3, + block: Dim3, + args: *mut *mut c_void, + shared_mem: usize, + stream: *mut c_void, +) -> c_int { + // The handle is the host stub pointer (see __cudaGetKernel). + cudaLaunchKernel(kernel, grid, block, args, shared_mem, stream) +} + +/// Per-thread-default-stream variant (compiled with `--default-stream per-thread`). +#[no_mangle] +pub extern "C" fn __cudaLaunchKernel_ptsz( + kernel: *const c_void, + grid: Dim3, + block: Dim3, + args: *mut *mut c_void, + shared_mem: usize, + stream: *mut c_void, +) -> c_int { + cudaLaunchKernel(kernel, grid, block, args, shared_mem, stream) +} + +// ---- errors ----------------------------------------------------------------- + +#[no_mangle] +pub extern "C" fn cudaGetLastError() -> c_int { + merge_sticky_async_error(); + LAST_ERROR.with(|e| { + let v = e.get(); + e.set(CUDA_SUCCESS); + v + }) +} + +#[no_mangle] +pub extern "C" fn cudaPeekAtLastError() -> c_int { + merge_sticky_async_error(); + LAST_ERROR.with(|e| e.get()) +} + +/// Fold any sticky asynchronous-pipeline error (a deferred launch/memcpy that +/// failed on the host) into the thread's last-error slot. Non-blocking: it +/// only reports failures already observed, matching how `cudaGetLastError` +/// reports asynchronous errors "seen so far" without synchronizing. +fn merge_sticky_async_error() { + let _ = with_state(|s| { + let code = s.client.take_sticky(); + if code != 0 { + set_last(map_err(CudaRpcError::Cuda(code))); + } + Ok(()) + }); +} + +#[no_mangle] +pub extern "C" fn cudaGetErrorString(error: c_int) -> *const c_char { + let s: &CStr = match error { + CUDA_SUCCESS => c"no error", + CUDA_ERROR_INVALID_VALUE => c"invalid argument", + CUDA_ERROR_MEMORY_ALLOCATION => c"out of memory", + CUDA_ERROR_INITIALIZATION => c"initialization error", + CUDA_ERROR_INVALID_DEVICE_POINTER => c"invalid device pointer", + CUDA_ERROR_INVALID_RESOURCE_HANDLE => c"invalid resource handle", + CUDA_ERROR_NO_DEVICE => c"no CUDA-capable device is detected", + _ => c"unknown error", + }; + s.as_ptr() +} + +// ---- nvcomp (forward-to-host-lib) ------------------------------------------- +// nvcomp is a dynamic library the workload links (e.g. shadowfax). Interposing +// its API here means its statically-linked cudart never runs in the guest — the +// real nvcomp runs host-side on the shared context. Device-pointer args are +// real host device addresses, forwarded by value; the stream is our handle +// (translated host-side). + +#[no_mangle] +pub extern "C" fn nvcompBatchedDeflateDecompressGetTempSizeEx( + num_chunks: usize, + max_uncompressed_chunk_bytes: usize, + temp_bytes: *mut usize, + max_total_uncompressed_bytes: usize, +) -> c_int { + match with_client(|c| { + c.nvcomp_deflate_temp_size( + num_chunks as u64, + max_uncompressed_chunk_bytes as u64, + max_total_uncompressed_bytes as u64, + ) + }) { + Ok((status, tb)) => { + if !temp_bytes.is_null() { + unsafe { *temp_bytes = tb as usize }; + } + status + } + // Transport/CUDA-layer failure before nvcomp ran → generic nvcomp error. + Err(_) => 1, + } +} + +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn nvcompBatchedDeflateDecompressAsync( + device_compressed_ptrs: *const *const c_void, + device_compressed_bytes: *const usize, + device_uncompressed_bytes: *const usize, + device_actual_uncompressed_bytes: *mut usize, + batch_size: usize, + device_temp: *mut c_void, + temp_bytes: usize, + device_uncompressed_ptrs: *const *mut c_void, + device_statuses: *mut c_int, + stream: *mut c_void, +) -> c_int { + // On success return nvcomp's own status; on a transport/CUDA-layer failure + // before nvcomp ran, report a generic nvcomp error (1). + with_client(|c| { + c.nvcomp_deflate_decompress( + device_compressed_ptrs as u64, + device_compressed_bytes as u64, + device_uncompressed_bytes as u64, + device_actual_uncompressed_bytes as u64, + batch_size as u64, + device_temp as u64, + temp_bytes as u64, + device_uncompressed_ptrs as u64, + device_statuses as u64, + stream as u64, + ) + }) + .unwrap_or(1) +} + +#[no_mangle] +pub extern "C" fn cudaGetErrorName(error: c_int) -> *const c_char { + let s: &CStr = match error { + CUDA_SUCCESS => c"cudaSuccess", + CUDA_ERROR_INVALID_VALUE => c"cudaErrorInvalidValue", + CUDA_ERROR_MEMORY_ALLOCATION => c"cudaErrorMemoryAllocation", + CUDA_ERROR_INITIALIZATION => c"cudaErrorInitializationError", + _ => c"cudaErrorUnknown", + }; + s.as_ptr() +} + +/// Code-generated cuBLAS forwarding stubs over the generic `LibCall` transport. +/// Regenerate with `smolvm-cuda-codegen`; do not edit by hand. +mod gen_cublas { + #![allow(non_snake_case, clippy::unnecessary_cast, unused_mut, dead_code)] + use super::{c_int, c_void, with_client}; + include!("generated/cublas_guest.rs"); +} + +/// Code-generated cuDNN forwarding stubs. Regenerate with `smolvm-cuda-codegen`. +mod gen_cudnn { + #![allow(non_snake_case, clippy::unnecessary_cast, unused_mut, dead_code)] + use super::{c_int, c_void, with_client}; + include!("generated/cudnn_guest.rs"); +} + +// ---- cuDNN v8 backend (graph) API — PyTorch's convolution path -------------- +// Forwarded via the generic LibCall transport under a dedicated lib id. Opaque +// descriptors + device pointers passing through are the server's real host +// pointers, so attribute arrays ship as raw bytes sized by the attribute type. +const LIB_CUDNN_BACKEND: u8 = 3; + +/// Guest-assigned virtual descriptor ids (bit 63 tags them; host userspace +/// pointers and device VAs never set it). Lets create calls fire-and-forget: +/// we invent the id, the host maps it to the real descriptor it creates. +pub(crate) fn alloc_vhandle() -> u64 { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new((1 << 63) | 1); + NEXT.fetch_add(1, Ordering::Relaxed) +} + +/// Byte size of one `cudnnBackendAttributeType_t` element (must match host). +fn cudnn_be_elem_size(t: c_int) -> usize { + match t { + 0 | 3 | 5 | 6 | 15 => 8, // HANDLE, INT64, DOUBLE, VOID_PTR, BACKEND_DESCRIPTOR + 2 | 24 => 1, // BOOLEAN, CHAR + 26 => 16, // FRACTION + _ => 4, // DATA_TYPE / enums / FLOAT / INT32 / ... + } +} + +#[no_mangle] +pub extern "C" fn cudnnBackendCreateDescriptor( + descriptor_type: c_int, + descriptor: *mut *mut c_void, +) -> c_int { + if descriptor.is_null() { + return 2000; // CUDNN_STATUS_BAD_PARAM + } + // Fire-and-forget: hand back a virtual id now; the host materializes the + // descriptor and maps the id. A creation failure surfaces on the next + // synchronous call touching it (Finalize/GetAttribute). + let vh = alloc_vhandle(); + let mut a = descriptor_type.to_le_bytes().to_vec(); + a.extend_from_slice(&vh.to_le_bytes()); + match with_client(|c| c.lib_call_deferred(LIB_CUDNN_BACKEND, 0, a)) { + Ok(()) => { + unsafe { *descriptor = vh as *mut c_void }; + 0 + } + Err(_) => 1, + } +} + +#[no_mangle] +pub extern "C" fn cudnnBackendDestroyDescriptor(descriptor: *mut c_void) -> c_int { + let a = (descriptor as u64).to_le_bytes().to_vec(); + match with_client(|c| c.lib_call_deferred(LIB_CUDNN_BACKEND, 1, a)) { + Ok(()) => 0, + Err(_) => 1, + } +} + +#[no_mangle] +pub extern "C" fn cudnnBackendSetAttribute( + descriptor: *mut c_void, + attribute_name: c_int, + attribute_type: c_int, + element_count: i64, + array_of_elements: *const c_void, +) -> c_int { + let n = (element_count.max(0) as usize) * cudnn_be_elem_size(attribute_type); + let mut a = Vec::with_capacity(24 + n); + a.extend_from_slice(&(descriptor as u64).to_le_bytes()); + a.extend_from_slice(&attribute_name.to_le_bytes()); + a.extend_from_slice(&attribute_type.to_le_bytes()); + a.extend_from_slice(&element_count.to_le_bytes()); + if n > 0 && !array_of_elements.is_null() { + a.extend_from_slice(unsafe { + std::slice::from_raw_parts(array_of_elements as *const u8, n) + }); + } + match with_client(|c| c.lib_call_deferred(LIB_CUDNN_BACKEND, 2, a)) { + Ok(()) => 0, + Err(_) => 1, + } +} + +#[no_mangle] +pub extern "C" fn cudnnBackendGetAttribute( + descriptor: *mut c_void, + attribute_name: c_int, + attribute_type: c_int, + requested_element_count: i64, + element_count: *mut i64, + array_of_elements: *mut c_void, +) -> c_int { + let cap = (requested_element_count.max(0) as usize) * cudnn_be_elem_size(attribute_type); + let mut a = Vec::with_capacity(24 + cap); + a.extend_from_slice(&(descriptor as u64).to_le_bytes()); + a.extend_from_slice(&attribute_name.to_le_bytes()); + a.extend_from_slice(&attribute_type.to_le_bytes()); + a.extend_from_slice(&requested_element_count.to_le_bytes()); + // Seed with current contents: descriptor-array gets pass pre-created handles. + if cap > 0 && !array_of_elements.is_null() { + a.extend_from_slice(unsafe { + std::slice::from_raw_parts(array_of_elements as *const u8, cap) + }); + } + match with_client(|c| c.lib_call(LIB_CUDNN_BACKEND, 3, a)) { + Ok((0, out)) if out.len() >= 8 => { + let cnt = i64::from_le_bytes(out[..8].try_into().unwrap()); + if !element_count.is_null() { + unsafe { *element_count = cnt }; + } + let bytes = &out[8..]; + if !array_of_elements.is_null() && !bytes.is_empty() { + let cap = + (requested_element_count.max(0) as usize) * cudnn_be_elem_size(attribute_type); + let n = bytes.len().min(cap); + // Descriptor arrays are populated *in place*: the returned + // pointers are the caller's own descriptors, so keep the ids + // the caller passed (they may be virtual) instead of the real + // host pointers the server sees. + const TYPE_BACKEND_DESCRIPTOR: c_int = 15; + if attribute_type != TYPE_BACKEND_DESCRIPTOR { + unsafe { + std::ptr::copy_nonoverlapping( + bytes.as_ptr(), + array_of_elements as *mut u8, + n, + ) + }; + } + } + 0 + } + Ok((st, _)) => st, + Err(_) => 1, + } +} + +#[no_mangle] +pub extern "C" fn cudnnBackendFinalize(descriptor: *mut c_void) -> c_int { + let a = (descriptor as u64).to_le_bytes().to_vec(); + match with_client(|c| c.lib_call(LIB_CUDNN_BACKEND, 4, a)) { + Ok((st, _)) => st, + Err(_) => 1, + } +} + +#[no_mangle] +pub extern "C" fn cudnnBackendExecute( + handle: *mut c_void, + execution_plan: *mut c_void, + variant_pack: *mut c_void, +) -> c_int { + let mut a = Vec::with_capacity(24); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&(execution_plan as u64).to_le_bytes()); + a.extend_from_slice(&(variant_pack as u64).to_le_bytes()); + match with_client(|c| c.lib_call_deferred(LIB_CUDNN_BACKEND, 5, a)) { + Ok(()) => 0, + Err(_) => 1, + } +} + +// ---- cuBLASLt matmul API — PyTorch's linear-layer path ----------------------- +// Forwarded via the generic LibCall transport. Descriptors, layouts, preferences +// and device pointers are the server's real host pointers (opaque handles here); +// the opaque 64-byte algo blob and attribute buffers ship as raw bytes. The +// "light handle" is the connection's cuBLAS handle, which torch reuses for Lt. +const LIB_CUBLASLT: u8 = 4; +const CUBLAS_STATUS_SUCCESS: c_int = 0; +const CUBLAS_STATUS_NOT_INITIALIZED: c_int = 1; + +// ---- cuBLASLt descriptor fast path ------------------------------------------- +// torch builds desc + layouts + preference around EVERY Linear matmul; sync +// round-trips here dominated eager decode (125k of 184k). Creates are +// fire-and-forget with guest-minted virtual ids (bit-63-tagged, host maps +// them), Set/Destroy defer, and AlgoGetHeuristic memoizes on the CONTENT of +// the descriptors (ids change every step; the shapes repeat). +/// Live Lt handle → content fingerprint (create args + every attr write). +static LT_FP: Mutex>>> = Mutex::new(None); +/// Heuristic memo entry: (status, out blob). +type LtHeurEntry = (c_int, Vec); +/// Heuristic memo: concatenated fingerprints + request → result. +static LT_HEUR_MEMO: Mutex, LtHeurEntry>>> = Mutex::new(None); + +fn lt_mint(create_args: &[u8]) -> u64 { + // Share the process-wide virtual-handle counter (alloc_vhandle) — a + // second counter minted colliding ids and clobbered the host's map. + let id = alloc_vhandle(); + let mut g = LT_FP.lock().unwrap(); + g.get_or_insert_with(HashMap::new) + .insert(id, create_args.to_vec()); + id +} + +fn lt_fp_append(handle: u64, attr: c_int, buf: &[u8]) { + let mut g = LT_FP.lock().unwrap(); + if let Some(fp) = g.get_or_insert_with(HashMap::new).get_mut(&handle) { + fp.extend_from_slice(&attr.to_le_bytes()); + fp.extend_from_slice(&(buf.len() as u32).to_le_bytes()); + fp.extend_from_slice(buf); + } +} + +fn lt_fp_of(handle: u64) -> Vec { + let mut g = LT_FP.lock().unwrap(); + g.get_or_insert_with(HashMap::new) + .get(&handle) + .cloned() + .unwrap_or_else(|| handle.to_le_bytes().to_vec()) +} + +fn lt_fp_drop(handle: u64) { + let mut g = LT_FP.lock().unwrap(); + g.get_or_insert_with(HashMap::new).remove(&handle); +} +/// sizeof(cublasLtMatmulHeuristicResult_t): algo[64]+workspaceSize(8)+state(4) +/// +wavesCount(4)+reserved[4](16). Must match the host. +const LT_HEUR_RESULT_SZ: usize = 96; + +#[no_mangle] +pub extern "C" fn cublasLtCreate(light_handle: *mut *mut c_void) -> c_int { + if light_handle.is_null() { + return CUBLAS_STATUS_NOT_INITIALIZED; + } + match with_client(|c| c.lib_call(LIB_CUBLASLT, 11, Vec::new())) { + Ok((0, out)) if out.len() >= 8 => { + unsafe { + *light_handle = u64::from_le_bytes(out[..8].try_into().unwrap()) as *mut c_void + }; + CUBLAS_STATUS_SUCCESS + } + Ok((st, _)) => st, + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +#[no_mangle] +pub extern "C" fn cublasLtDestroy(light_handle: *mut c_void) -> c_int { + let a = (light_handle as u64).to_le_bytes().to_vec(); + match with_client(|c| c.lib_call(LIB_CUBLASLT, 12, a)) { + Ok((st, _)) => st, + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +#[no_mangle] +pub extern "C" fn cublasLtMatmulDescCreate( + matmul_desc: *mut *mut c_void, + compute_type: c_int, + scale_type: c_int, +) -> c_int { + if matmul_desc.is_null() { + return CUBLAS_STATUS_NOT_INITIALIZED; + } + let mut a = Vec::with_capacity(16); + a.extend_from_slice(&compute_type.to_le_bytes()); + a.extend_from_slice(&scale_type.to_le_bytes()); + let vh = lt_mint(&a); + a.extend_from_slice(&vh.to_le_bytes()); + match with_client(|c| c.lib_call_deferred(LIB_CUBLASLT, 0, a)) { + Ok(()) => { + unsafe { *matmul_desc = vh as *mut c_void }; + CUBLAS_STATUS_SUCCESS + } + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +#[no_mangle] +pub extern "C" fn cublasLtMatmulDescDestroy(matmul_desc: *mut c_void) -> c_int { + lt_fp_drop(matmul_desc as u64); + let a = (matmul_desc as u64).to_le_bytes().to_vec(); + match with_client(|c| c.lib_call_deferred(LIB_CUBLASLT, 1, a)) { + Ok(()) => CUBLAS_STATUS_SUCCESS, + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +/// Pack an opaque descriptor/layout/preference SetAttribute call: the attribute +/// buffer forwards verbatim (device pointers inside it stay coherent). +fn lt_set_attr( + func: u16, + handle: *mut c_void, + attr: c_int, + buf: *const c_void, + size: usize, +) -> c_int { + let mut a = Vec::with_capacity(12 + size); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&attr.to_le_bytes()); + let bytes: &[u8] = if size > 0 && !buf.is_null() { + unsafe { std::slice::from_raw_parts(buf as *const u8, size) } + } else { + &[] + }; + a.extend_from_slice(bytes); + lt_fp_append(handle as u64, attr, bytes); + match with_client(|c| c.lib_call_deferred(LIB_CUBLASLT, func, a)) { + Ok(()) => CUBLAS_STATUS_SUCCESS, + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +#[no_mangle] +pub extern "C" fn cublasLtMatmulDescSetAttribute( + matmul_desc: *mut c_void, + attr: c_int, + buf: *const c_void, + size_in_bytes: usize, +) -> c_int { + lt_set_attr(2, matmul_desc, attr, buf, size_in_bytes) +} + +#[no_mangle] +pub extern "C" fn cublasLtMatrixLayoutCreate( + mat_layout: *mut *mut c_void, + data_type: c_int, + rows: u64, + cols: u64, + ld: i64, +) -> c_int { + if mat_layout.is_null() { + return CUBLAS_STATUS_NOT_INITIALIZED; + } + let mut a = Vec::with_capacity(36); + a.extend_from_slice(&data_type.to_le_bytes()); + a.extend_from_slice(&rows.to_le_bytes()); + a.extend_from_slice(&cols.to_le_bytes()); + a.extend_from_slice(&ld.to_le_bytes()); + let vh = lt_mint(&a); + a.extend_from_slice(&vh.to_le_bytes()); + match with_client(|c| c.lib_call_deferred(LIB_CUBLASLT, 3, a)) { + Ok(()) => { + unsafe { *mat_layout = vh as *mut c_void }; + CUBLAS_STATUS_SUCCESS + } + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +#[no_mangle] +pub extern "C" fn cublasLtMatrixLayoutDestroy(mat_layout: *mut c_void) -> c_int { + lt_fp_drop(mat_layout as u64); + let a = (mat_layout as u64).to_le_bytes().to_vec(); + match with_client(|c| c.lib_call_deferred(LIB_CUBLASLT, 4, a)) { + Ok(()) => CUBLAS_STATUS_SUCCESS, + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +#[no_mangle] +pub extern "C" fn cublasLtMatrixLayoutSetAttribute( + mat_layout: *mut c_void, + attr: c_int, + buf: *const c_void, + size_in_bytes: usize, +) -> c_int { + lt_set_attr(5, mat_layout, attr, buf, size_in_bytes) +} + +#[no_mangle] +pub extern "C" fn cublasLtMatmulPreferenceCreate(pref: *mut *mut c_void) -> c_int { + if pref.is_null() { + return CUBLAS_STATUS_NOT_INITIALIZED; + } + let vh = lt_mint(&[]); + match with_client(|c| c.lib_call_deferred(LIB_CUBLASLT, 6, vh.to_le_bytes().to_vec())) { + Ok(()) => { + unsafe { *pref = vh as *mut c_void }; + CUBLAS_STATUS_SUCCESS + } + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +#[no_mangle] +pub extern "C" fn cublasLtMatmulPreferenceDestroy(pref: *mut c_void) -> c_int { + lt_fp_drop(pref as u64); + let a = (pref as u64).to_le_bytes().to_vec(); + match with_client(|c| c.lib_call_deferred(LIB_CUBLASLT, 7, a)) { + Ok(()) => CUBLAS_STATUS_SUCCESS, + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +#[no_mangle] +pub extern "C" fn cublasLtMatmulPreferenceSetAttribute( + pref: *mut c_void, + attr: c_int, + buf: *const c_void, + size_in_bytes: usize, +) -> c_int { + lt_set_attr(8, pref, attr, buf, size_in_bytes) +} + +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn cublasLtMatmulAlgoGetHeuristic( + light_handle: *mut c_void, + operation_desc: *mut c_void, + a_desc: *mut c_void, + b_desc: *mut c_void, + c_desc: *mut c_void, + d_desc: *mut c_void, + preference: *mut c_void, + requested_algo_count: c_int, + heuristic_results_array: *mut c_void, + return_algo_count: *mut c_int, +) -> c_int { + // Memo key: the CONTENT of every descriptor (ids change per step; the + // shapes repeat every decode step, so this hits ~always after warmup). + let mut key = Vec::new(); + for h in [operation_desc, a_desc, b_desc, c_desc, d_desc, preference] { + let fp = lt_fp_of(h as u64); + key.extend_from_slice(&(fp.len() as u32).to_le_bytes()); + key.extend_from_slice(&fp); + } + key.extend_from_slice(&requested_algo_count.to_le_bytes()); + if let Some((st, out)) = LT_HEUR_MEMO + .lock() + .unwrap() + .get_or_insert_with(HashMap::new) + .get(&key) + .cloned() + { + if st == 0 && out.len() >= 8 { + let count = i64::from_le_bytes(out[..8].try_into().unwrap()).max(0) as usize; + if !return_algo_count.is_null() { + unsafe { *return_algo_count = count as c_int }; + } + let bytes = &out[8..]; + if !heuristic_results_array.is_null() { + let n = bytes.len().min(count * LT_HEUR_RESULT_SZ); + unsafe { + std::ptr::copy_nonoverlapping( + bytes.as_ptr(), + heuristic_results_array as *mut u8, + n, + ) + }; + } + return CUBLAS_STATUS_SUCCESS; + } + return st; + } + let mut a = Vec::with_capacity(60); + a.extend_from_slice(&(light_handle as u64).to_le_bytes()); + a.extend_from_slice(&(operation_desc as u64).to_le_bytes()); + a.extend_from_slice(&(a_desc as u64).to_le_bytes()); + a.extend_from_slice(&(b_desc as u64).to_le_bytes()); + a.extend_from_slice(&(c_desc as u64).to_le_bytes()); + a.extend_from_slice(&(d_desc as u64).to_le_bytes()); + a.extend_from_slice(&(preference as u64).to_le_bytes()); + a.extend_from_slice(&requested_algo_count.to_le_bytes()); + match with_client(|c| c.lib_call(LIB_CUBLASLT, 9, a)) { + Ok((0, out)) if out.len() >= 8 => { + LT_HEUR_MEMO + .lock() + .unwrap() + .get_or_insert_with(HashMap::new) + .insert(key, (0, out.clone())); + let count = i64::from_le_bytes(out[..8].try_into().unwrap()).max(0) as usize; + if !return_algo_count.is_null() { + unsafe { *return_algo_count = count as c_int }; + } + let bytes = &out[8..]; + if !heuristic_results_array.is_null() { + let n = bytes.len().min(count * LT_HEUR_RESULT_SZ); + unsafe { + std::ptr::copy_nonoverlapping( + bytes.as_ptr(), + heuristic_results_array as *mut u8, + n, + ) + }; + } + CUBLAS_STATUS_SUCCESS + } + Ok((st, _)) => st, + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn cublasLtMatmul( + light_handle: *mut c_void, + compute_desc: *mut c_void, + alpha: *const c_void, + a: *const c_void, + a_desc: *mut c_void, + b: *const c_void, + b_desc: *mut c_void, + beta: *const c_void, + c: *const c_void, + c_desc: *mut c_void, + d: *mut c_void, + d_desc: *mut c_void, + algo: *const c_void, + workspace: *mut c_void, + workspace_size_in_bytes: usize, + stream: *mut c_void, +) -> c_int { + // alpha/beta are host scalars sized by the desc's scale type (≤16 bytes for + // any real/complex type); forward a fixed 16-byte window so either fits. + let read16 = |p: *const c_void| -> [u8; 16] { + let mut v = [0u8; 16]; + if !p.is_null() { + unsafe { std::ptr::copy_nonoverlapping(p as *const u8, v.as_mut_ptr(), 16) }; + } + v + }; + let algo_bytes = { + let mut v = [0u8; 64]; + if !algo.is_null() { + unsafe { std::ptr::copy_nonoverlapping(algo as *const u8, v.as_mut_ptr(), 64) }; + } + v + }; + let mut buf = Vec::with_capacity(208); + buf.extend_from_slice(&(light_handle as u64).to_le_bytes()); + buf.extend_from_slice(&(compute_desc as u64).to_le_bytes()); + buf.extend_from_slice(&read16(alpha)); + buf.extend_from_slice(&(a as u64).to_le_bytes()); + buf.extend_from_slice(&(a_desc as u64).to_le_bytes()); + buf.extend_from_slice(&(b as u64).to_le_bytes()); + buf.extend_from_slice(&(b_desc as u64).to_le_bytes()); + buf.extend_from_slice(&read16(beta)); + buf.extend_from_slice(&(c as u64).to_le_bytes()); + buf.extend_from_slice(&(c_desc as u64).to_le_bytes()); + buf.extend_from_slice(&(d as u64).to_le_bytes()); + buf.extend_from_slice(&(d_desc as u64).to_le_bytes()); + buf.extend_from_slice(&algo_bytes); + buf.extend_from_slice(&(if algo.is_null() { 0u64 } else { 1u64 }).to_le_bytes()); + buf.extend_from_slice(&(workspace as u64).to_le_bytes()); + buf.extend_from_slice(&(workspace_size_in_bytes as u64).to_le_bytes()); + buf.extend_from_slice(&(stream as u64).to_le_bytes()); + match with_client(|c| c.lib_call_deferred(LIB_CUBLASLT, 10, buf)) { + Ok(()) => CUBLAS_STATUS_SUCCESS, + Err(_) => CUBLAS_STATUS_NOT_INITIALIZED, + } +} + +// ---- Legacy cuDNN batch-norm (Ex) + N-D descriptor — BatchNorm2d path -------- +// Forwarded via the generic LibCall transport. Descriptors and device pointers +// are the server's real host pointers; alpha/beta are float scalars, epsilon and +// the averaging factor are doubles, and the N-D descriptor's dim/stride arrays +// forward as raw i32s. +const LIB_CUDNN_BN: u8 = 5; + +/// A valid, static message pointer for `cudnnGetErrorString`. The real function +/// returns `const char*`; torch dereferences it when formatting errors, so a +/// stub that returned an int caused a segfault. The exact text is cosmetic. +#[no_mangle] +pub extern "C" fn cudnnGetErrorString(_status: c_int) -> *const c_char { + c"cudnn status (forwarded by smolvm)".as_ptr() +} + +/// Read a host float behind a `*const c_void` (cuDNN alpha/beta), 0.0 if null. +fn bn_f32(p: *const c_void) -> f32 { + if p.is_null() { + 0.0 + } else { + unsafe { *(p as *const f32) } + } +} + +#[no_mangle] +pub extern "C" fn cudnnSetTensorNdDescriptor( + tensor_desc: *mut c_void, + data_type: c_int, + nb_dims: c_int, + dim_a: *const c_int, + stride_a: *const c_int, +) -> c_int { + let n = nb_dims.max(0) as usize; + let mut a = Vec::with_capacity(16 + n * 8); + a.extend_from_slice(&(tensor_desc as u64).to_le_bytes()); + a.extend_from_slice(&data_type.to_le_bytes()); + a.extend_from_slice(&nb_dims.to_le_bytes()); + // Record this descriptor\'s content fingerprint (everything after the + // handle) so pure size queries over it can be memoized soundly — the + // handle value alone is reusable across different shapes. + record_desc_fingerprint(tensor_desc as u64, &a[8..]); + for arr in [dim_a, stride_a] { + for i in 0..n { + let v = if arr.is_null() { + 0 + } else { + unsafe { *arr.add(i) } + }; + a.extend_from_slice(&v.to_le_bytes()); + } + } + match with_client(|c| c.lib_call_deferred(LIB_CUDNN_BN, 0, a)) { + Ok(()) => 0, + Err(_) => 1, + } +} + +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn cudnnBatchNormalizationForwardInference( + handle: *mut c_void, + mode: c_int, + alpha: *const c_void, + beta: *const c_void, + x_desc: *mut c_void, + x: *const c_void, + y_desc: *mut c_void, + y: *mut c_void, + bn_desc: *mut c_void, + bn_scale: *const c_void, + bn_bias: *const c_void, + est_mean: *const c_void, + est_var: *const c_void, + epsilon: f64, +) -> c_int { + let mut a = Vec::with_capacity(96); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&mode.to_le_bytes()); + a.extend_from_slice(&bn_f32(alpha).to_le_bytes()); + a.extend_from_slice(&bn_f32(beta).to_le_bytes()); + for p in [ + x_desc as u64, + x as u64, + y_desc as u64, + y as u64, + bn_desc as u64, + bn_scale as u64, + bn_bias as u64, + est_mean as u64, + est_var as u64, + ] { + a.extend_from_slice(&p.to_le_bytes()); + } + a.extend_from_slice(&epsilon.to_le_bytes()); + match with_client(|c| c.lib_call_deferred(LIB_CUDNN_BN, 2, a)) { + Ok(()) => 0, + Err(_) => 1, + } +} + +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn cudnnBatchNormalizationForwardTrainingEx( + handle: *mut c_void, + mode: c_int, + bn_ops: c_int, + alpha: *const c_void, + beta: *const c_void, + x_desc: *mut c_void, + x: *const c_void, + z_desc: *mut c_void, + z: *const c_void, + y_desc: *mut c_void, + y: *mut c_void, + bn_desc: *mut c_void, + bn_scale: *const c_void, + bn_bias: *const c_void, + factor: f64, + run_mean: *mut c_void, + run_var: *mut c_void, + epsilon: f64, + save_mean: *mut c_void, + save_ivar: *mut c_void, + act_desc: *mut c_void, + workspace: *mut c_void, + ws_size: usize, + reserve: *mut c_void, + reserve_size: usize, +) -> c_int { + let mut a = Vec::with_capacity(200); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&mode.to_le_bytes()); + a.extend_from_slice(&bn_ops.to_le_bytes()); + a.extend_from_slice(&bn_f32(alpha).to_le_bytes()); + a.extend_from_slice(&bn_f32(beta).to_le_bytes()); + for p in [ + x_desc as u64, + x as u64, + z_desc as u64, + z as u64, + y_desc as u64, + y as u64, + bn_desc as u64, + bn_scale as u64, + bn_bias as u64, + ] { + a.extend_from_slice(&p.to_le_bytes()); + } + a.extend_from_slice(&factor.to_le_bytes()); + a.extend_from_slice(&(run_mean as u64).to_le_bytes()); + a.extend_from_slice(&(run_var as u64).to_le_bytes()); + a.extend_from_slice(&epsilon.to_le_bytes()); + for p in [ + save_mean as u64, + save_ivar as u64, + act_desc as u64, + workspace as u64, + ] { + a.extend_from_slice(&p.to_le_bytes()); + } + a.extend_from_slice(&(ws_size as u64).to_le_bytes()); + a.extend_from_slice(&(reserve as u64).to_le_bytes()); + a.extend_from_slice(&(reserve_size as u64).to_le_bytes()); + match with_client(|c| c.lib_call_deferred(LIB_CUDNN_BN, 3, a)) { + Ok(()) => 0, + Err(_) => 1, + } +} + +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn cudnnBatchNormalizationBackwardEx( + handle: *mut c_void, + mode: c_int, + bn_ops: c_int, + alpha_d: *const c_void, + beta_d: *const c_void, + alpha_p: *const c_void, + beta_p: *const c_void, + x_desc: *mut c_void, + x: *const c_void, + y_desc: *mut c_void, + y: *const c_void, + dy_desc: *mut c_void, + dy: *const c_void, + dz_desc: *mut c_void, + dz: *mut c_void, + dx_desc: *mut c_void, + dx: *mut c_void, + d_bn_desc: *mut c_void, + bn_scale: *const c_void, + bn_bias: *const c_void, + d_bn_scale: *mut c_void, + d_bn_bias: *mut c_void, + epsilon: f64, + saved_mean: *const c_void, + saved_ivar: *const c_void, + act_desc: *mut c_void, + workspace: *mut c_void, + ws_size: usize, + reserve: *mut c_void, + reserve_size: usize, +) -> c_int { + let mut a = Vec::with_capacity(256); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&mode.to_le_bytes()); + a.extend_from_slice(&bn_ops.to_le_bytes()); + a.extend_from_slice(&bn_f32(alpha_d).to_le_bytes()); + a.extend_from_slice(&bn_f32(beta_d).to_le_bytes()); + a.extend_from_slice(&bn_f32(alpha_p).to_le_bytes()); + a.extend_from_slice(&bn_f32(beta_p).to_le_bytes()); + for p in [ + x_desc as u64, + x as u64, + y_desc as u64, + y as u64, + dy_desc as u64, + dy as u64, + dz_desc as u64, + dz as u64, + dx_desc as u64, + dx as u64, + d_bn_desc as u64, + bn_scale as u64, + bn_bias as u64, + d_bn_scale as u64, + d_bn_bias as u64, + ] { + a.extend_from_slice(&p.to_le_bytes()); + } + a.extend_from_slice(&epsilon.to_le_bytes()); + for p in [ + saved_mean as u64, + saved_ivar as u64, + act_desc as u64, + workspace as u64, + ] { + a.extend_from_slice(&p.to_le_bytes()); + } + a.extend_from_slice(&(ws_size as u64).to_le_bytes()); + a.extend_from_slice(&(reserve as u64).to_le_bytes()); + a.extend_from_slice(&(reserve_size as u64).to_le_bytes()); + match with_client(|c| c.lib_call_deferred(LIB_CUDNN_BN, 4, a)) { + Ok(()) => 0, + Err(_) => 1, + } +} + +/// Tensor-descriptor content fingerprints (dtype + dims + strides from the +/// last `cudnnSetTensorNdDescriptor`), keyed by handle value. A handle alone +/// is not a stable identity — destroy/create can reuse the address — so size +/// memoization keys on these contents instead. +static DESC_FP: Mutex>>> = Mutex::new(None); + +fn record_desc_fingerprint(desc: u64, contents: &[u8]) { + if let Ok(mut g) = DESC_FP.lock() { + g.get_or_insert_with(HashMap::new) + .insert(desc, contents.to_vec()); + } +} + +/// Rewrite a BN size-query arg blob into a content-addressed memo key: every +/// descriptor handle is replaced by its recorded fingerprint. `None` (do not +/// cache) if any non-null descriptor has no fingerprint on record. +fn bn_memo_key(func: u16, args: &[u8]) -> Option> { + // Layouts (see the three Get*Size wrappers): handle u64, mode i32, ops i32, + // then only u64 descriptor handles (5/7 for the workspace queries, act+x + // for the reserve query). The leading cudnn handle is identity-stable. + if args.len() < 16 || !(args.len() - 16).is_multiple_of(8) { + return None; + } + let fps = DESC_FP.lock().ok()?; + let fps = fps.as_ref()?; + let mut key = Vec::with_capacity(args.len() * 4); + key.extend_from_slice(&func.to_le_bytes()); + key.extend_from_slice(&args[..16]); + for chunk in args[16..].chunks_exact(8) { + let h = u64::from_le_bytes(chunk.try_into().unwrap()); + if h == 0 { + key.push(0); + continue; + } + let fp = fps.get(&h)?; // unknown descriptor → uncacheable + key.extend_from_slice(&(fp.len() as u32).to_le_bytes()); + key.extend_from_slice(fp); + } + Some(key) +} + +/// Shared tail for the three `Get...Size` queries: forward the packed args and +/// write the returned `size_t` through `size_out`. Memoized on the descriptor +/// *contents* (not handles): the sizes are pure functions of those contents, +/// and PyTorch re-queries them on every batch-norm invocation (~150 sync +/// round-trips per ResNet training step without the cache). +fn bn_size_call(func: u16, args: Vec, size_out: *mut usize) -> c_int { + static MEMO: Mutex, usize>>> = Mutex::new(None); + let key = bn_memo_key(func, &args); + if let Some(k) = &key { + if let Ok(mut g) = MEMO.lock() { + if let Some(sz) = g.get_or_insert_with(HashMap::new).get(k) { + if !size_out.is_null() { + unsafe { *size_out = *sz }; + } + return 0; + } + } + } + match with_client(|c| c.lib_call(LIB_CUDNN_BN, func, args)) { + Ok((0, out)) if out.len() >= 8 => { + let sz = u64::from_le_bytes(out[..8].try_into().unwrap()) as usize; + if !size_out.is_null() { + unsafe { *size_out = sz }; + } + if let Some(k) = key { + if let Ok(mut g) = MEMO.lock() { + g.get_or_insert_with(HashMap::new).insert(k, sz); + } + } + 0 + } + Ok((st, _)) => st, + Err(_) => 1, + } +} + +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn cudnnGetBatchNormalizationForwardTrainingExWorkspaceSize( + handle: *mut c_void, + mode: c_int, + bn_ops: c_int, + x_desc: *mut c_void, + z_desc: *mut c_void, + y_desc: *mut c_void, + bn_desc: *mut c_void, + act_desc: *mut c_void, + size_out: *mut usize, +) -> c_int { + let mut a = Vec::with_capacity(56); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&mode.to_le_bytes()); + a.extend_from_slice(&bn_ops.to_le_bytes()); + for p in [ + x_desc as u64, + z_desc as u64, + y_desc as u64, + bn_desc as u64, + act_desc as u64, + ] { + a.extend_from_slice(&p.to_le_bytes()); + } + bn_size_call(5, a, size_out) +} + +#[no_mangle] +#[allow(clippy::too_many_arguments)] +pub extern "C" fn cudnnGetBatchNormalizationBackwardExWorkspaceSize( + handle: *mut c_void, + mode: c_int, + bn_ops: c_int, + x_desc: *mut c_void, + y_desc: *mut c_void, + dy_desc: *mut c_void, + dz_desc: *mut c_void, + dx_desc: *mut c_void, + d_bn_desc: *mut c_void, + act_desc: *mut c_void, + size_out: *mut usize, +) -> c_int { + let mut a = Vec::with_capacity(64); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&mode.to_le_bytes()); + a.extend_from_slice(&bn_ops.to_le_bytes()); + for p in [ + x_desc as u64, + y_desc as u64, + dy_desc as u64, + dz_desc as u64, + dx_desc as u64, + d_bn_desc as u64, + act_desc as u64, + ] { + a.extend_from_slice(&p.to_le_bytes()); + } + bn_size_call(6, a, size_out) +} + +#[no_mangle] +pub extern "C" fn cudnnGetBatchNormalizationTrainingExReserveSpaceSize( + handle: *mut c_void, + mode: c_int, + bn_ops: c_int, + act_desc: *mut c_void, + x_desc: *mut c_void, + size_out: *mut usize, +) -> c_int { + let mut a = Vec::with_capacity(40); + a.extend_from_slice(&(handle as u64).to_le_bytes()); + a.extend_from_slice(&mode.to_le_bytes()); + a.extend_from_slice(&bn_ops.to_le_bytes()); + a.extend_from_slice(&(act_desc as u64).to_le_bytes()); + a.extend_from_slice(&(x_desc as u64).to_le_bytes()); + bn_size_call(7, a, size_out) +} diff --git a/crates/smolvm-network/Cargo.toml b/crates/smolvm-network/Cargo.toml new file mode 100644 index 0000000..0cf9dfa --- /dev/null +++ b/crates/smolvm-network/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "smolvm-network" +version = "1.5.2" +edition = "2021" +description = "Host-side virtio-net runtime for smolvm" +license = "Apache-2.0" +repository = "https://github.com/smol-machines/smolvm" + +[dependencies] +crossbeam-queue = "0.3" +humantime = "2" +polling = "3" +smoltcp = { version = "0.13", default-features = false, features = [ + "std", + "log", + "medium-ethernet", + "proto-ipv4", + "proto-ipv6", + "socket-tcp", + "socket-udp", + # Raw IP sockets capture the guest's ICMP echo traffic so the gateway can + # relay ping to real host ICMP sockets (see `icmp_relay`). + "socket-raw", + # The gateway interface owns three addresses: IPv4, IPv6 ULA, and the + # IPv6 link-local (default cap is 2). + "iface-max-addr-count-3", +] } +socket2 = "0.6" +tracing = { workspace = true } + +[features] +# Exposes fuzz-only entrypoints (e.g. fuzz_classify_guest_frame). Never enabled in normal builds. +fuzzing = [] diff --git a/crates/smolvm-network/fuzz/.gitignore b/crates/smolvm-network/fuzz/.gitignore new file mode 100644 index 0000000..1a45eee --- /dev/null +++ b/crates/smolvm-network/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/crates/smolvm-network/fuzz/Cargo.lock b/crates/smolvm-network/fuzz/Cargo.lock new file mode 100644 index 0000000..f0d84d4 --- /dev/null +++ b/crates/smolvm-network/fuzz/Cargo.lock @@ -0,0 +1,363 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cc" +version = "1.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "defmt" +version = "0.3.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0963443817029b2024136fc4dd07a5107eb8f977eaf18fcd1fdeb11306b64ad" +dependencies = [ + "defmt 1.1.0", +] + +[[package]] +name = "defmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +dependencies = [ + "bitflags", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +dependencies = [ + "defmt-parser", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "heapless" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" +dependencies = [ + "hash32", + "stable_deref_trait", +] + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smoltcp" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f73d40463bba65efc9adc6370b56df76d563cc46e2482bba58351b4afb7535e" +dependencies = [ + "bitflags", + "byteorder", + "cfg-if", + "defmt 0.3.100", + "heapless", + "log", + "managed", +] + +[[package]] +name = "smolvm-network" +version = "1.0.3" +dependencies = [ + "crossbeam-queue", + "humantime", + "libc", + "smoltcp", + "tracing", +] + +[[package]] +name = "smolvm-network-fuzz" +version = "0.0.0" +dependencies = [ + "libfuzzer-sys", + "smolvm-network", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" diff --git a/crates/smolvm-network/fuzz/Cargo.toml b/crates/smolvm-network/fuzz/Cargo.toml new file mode 100644 index 0000000..cc09132 --- /dev/null +++ b/crates/smolvm-network/fuzz/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "smolvm-network-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" + +[dependencies.smolvm-network] +path = ".." +features = ["fuzzing"] + +[[bin]] +name = "classify_guest_frame" +path = "fuzz_targets/classify_guest_frame.rs" +test = false +doc = false +bench = false + +[workspace] diff --git a/crates/smolvm-network/fuzz/fuzz_targets/classify_guest_frame.rs b/crates/smolvm-network/fuzz/fuzz_targets/classify_guest_frame.rs new file mode 100644 index 0000000..7e436bc --- /dev/null +++ b/crates/smolvm-network/fuzz/fuzz_targets/classify_guest_frame.rs @@ -0,0 +1,10 @@ +#![no_main] +//! Fuzz the host-side parser of guest-driven ethernet frames. +//! +//! A malicious guest sends arbitrary bytes over virtio-net; the host runs each +//! through `classify_guest_frame`. It must never panic / abort on any input. +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + smolvm_network::stack::fuzz_classify_guest_frame(data); +}); diff --git a/crates/smolvm-network/src/device.rs b/crates/smolvm-network/src/device.rs new file mode 100644 index 0000000..382bd64 --- /dev/null +++ b/crates/smolvm-network/src/device.rs @@ -0,0 +1,243 @@ +//! smoltcp `phy::Device` adapter for the virtio-net backend. +//! +//! Context +//! ======= +//! +//! smoltcp talks to an implementation of its `phy::Device` trait: +//! - `receive()` yields one incoming frame and a transmit token +//! - `transmit()` yields a transmit token when space exists for an outgoing +//! frame +//! - `capabilities()` describes medium and MTU +//! +//! This module is the narrow adapter layer between: +//! - smoltcp's abstract device API +//! - the queue-based frame transport used by the rest of the virtio runtime +//! +//! Data flow: +//! +//! ```text +//! guest_to_host queue --stage_next_frame--> smoltcp receive() +//! smoltcp transmit() --host_to_guest queue--> frame_stream writer +//! ``` +//! +//! More concretely: +//! +//! ```text +//! guest frame arrives in guest_to_host +//! -> poll loop calls stage_next_frame() +//! -> poll loop may inspect/classify frame first +//! -> smoltcp calls receive() +//! -> DeviceRxToken hands bytes to smoltcp +//! +//! smoltcp wants to emit a frame +//! -> calls transmit() +//! -> gets DeviceTxToken +//! -> fills provided buffer +//! -> token pushes frame into host_to_guest +//! -> poll loop later wakes frame writer +//! ``` + +use crate::queues::NetworkFrameQueues; +use smoltcp::phy::{self, DeviceCapabilities, Medium}; +use smoltcp::time::Instant; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; + +/// smoltcp `Device` backed by shared frame queues. +/// +/// `staged_guest_frame` exists because the poll loop sometimes needs to inspect +/// a frame before handing it to smoltcp. In particular, the stack wants to +/// classify guest TCP SYN and DNS packets before consumption so it can prepare +/// relay/socket state. +/// +/// The staging pattern looks like: +/// +/// ```text +/// queue -> staged_guest_frame -> RxToken -> smoltcp +/// ``` +pub struct VirtioNetworkDevice { + queues: Arc, + mtu: usize, + staged_guest_frame: Option>, + /// Set when smoltcp emitted at least one frame for the guest. + pub(crate) frames_emitted: AtomicBool, +} + +/// RX token representing one guest ethernet frame. +/// +/// smoltcp consumes RX tokens immediately; the token just owns the frame bytes +/// until the stack asks to inspect them. +pub struct DeviceRxToken { + frame: Vec, +} + +/// TX token representing one outgoing frame from smoltcp. +/// +/// The token borrows the device so it can enqueue the produced frame when +/// smoltcp finishes writing into the provided buffer. +pub struct DeviceTxToken<'a> { + device: &'a mut VirtioNetworkDevice, +} + +impl VirtioNetworkDevice { + /// Create a new device for the given queues and MTU. + /// + /// `mtu` here is the guest IP MTU, not the full Ethernet frame size. + /// `capabilities()` translates it to the Ethernet-frame convention expected + /// by smoltcp. + pub fn new(queues: Arc, mtu: usize) -> Self { + Self { + queues, + mtu, + staged_guest_frame: None, + frames_emitted: AtomicBool::new(false), + } + } + + /// Stage one guest frame so the poll loop can inspect it before smoltcp consumes it. + /// + /// Why staging exists: + /// - the frame arrives first in `guest_to_host` + /// - the poll loop may need to classify it before calling + /// `Interface::poll_ingress_single` + /// - once smoltcp calls `receive()`, ownership moves into an RX token + /// + /// So staging gives the poll loop a temporary peek at the next frame + /// without losing the normal smoltcp `Device` flow. + /// + /// This is the key reason the adapter is not just a direct `queue.pop()` + /// inside `receive()`. + pub fn stage_next_frame(&mut self) -> Option<&[u8]> { + if self.staged_guest_frame.is_none() { + self.staged_guest_frame = self.queues.guest_to_host.pop(); + } + + self.staged_guest_frame.as_deref() + } + + /// Drop the currently staged guest frame. + /// + /// This is used when the poll loop decides not to pass a frame into + /// smoltcp, for example when the MVP intentionally drops unsupported UDP. + pub fn drop_staged_frame(&mut self) { + self.staged_guest_frame = None; + } +} + +impl phy::Device for VirtioNetworkDevice { + type RxToken<'a> = DeviceRxToken; + type TxToken<'a> = DeviceTxToken<'a>; + + fn receive(&mut self, _timestamp: Instant) -> Option<(Self::RxToken<'_>, Self::TxToken<'_>)> { + // smoltcp asks for the next ingress frame only after the poll loop has + // already staged it. If nothing is staged, there is nothing to receive. + let frame = self.staged_guest_frame.take()?; + // This is the single point where a guest-outbound frame is accepted into + // the stack (frames the poll loop dropped never reach here), so it's the + // natural place to meter egress. Count the full ethernet frame. + self.queues.add_egress_bytes(frame.len() as u64); + Some((DeviceRxToken { frame }, DeviceTxToken { device: self })) + } + + fn transmit(&mut self, _timestamp: Instant) -> Option> { + // smoltcp may ask for a transmit token even when the downstream writer + // is temporarily behind. We only hand out a token if the host->guest + // queue still has room. + if self.queues.host_to_guest.len() < self.queues.host_to_guest.capacity() { + Some(DeviceTxToken { device: self }) + } else { + None + } + } + + fn capabilities(&self) -> DeviceCapabilities { + let mut capabilities = DeviceCapabilities::default(); + capabilities.medium = Medium::Ethernet; + // smoltcp wants the maximum Ethernet frame size here, not the Linux IP + // MTU. For Ethernet devices that means "IP MTU + 14-byte Ethernet + // header"; see smoltcp's `DeviceCapabilities::max_transmission_unit` + // documentation. + capabilities.max_transmission_unit = self.mtu + 14; + capabilities + } +} + +impl phy::RxToken for DeviceRxToken { + /// Hand the queued guest frame bytes to smoltcp. + fn consume(self, f: F) -> R + where + F: FnOnce(&[u8]) -> R, + { + f(&self.frame) + } +} + +impl<'a> phy::TxToken for DeviceTxToken<'a> { + /// Let smoltcp build one Ethernet frame and enqueue it for libkrun. + /// + /// Flow: + /// + /// ```text + /// smoltcp fills provided buffer + /// -> adapter enqueues frame into host_to_guest + /// -> sets frames_emitted + /// -> poll loop later wakes the Unix-stream writer + /// ``` + /// + /// The queue push is the handoff point. After that, this adapter no longer + /// owns the frame bytes; the frame writer thread eventually serializes them + /// onto the libkrun Unix stream. + fn consume(self, len: usize, f: F) -> R + where + F: FnOnce(&mut [u8]) -> R, + { + let mut frame = vec![0u8; len]; + let result = f(&mut frame); + if self.device.queues.host_to_guest.push(frame).is_ok() { + self.device.frames_emitted.store(true, Ordering::Relaxed); + } else { + tracing::debug!("dropping outbound ethernet frame because the guest queue is full"); + } + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::queues::DEFAULT_FRAME_QUEUE_CAPACITY; + use smoltcp::phy::{Device, RxToken}; + + #[test] + fn receive_counts_guest_outbound_bytes_as_egress() { + let queues = NetworkFrameQueues::shared(DEFAULT_FRAME_QUEUE_CAPACITY); + let mut dev = VirtioNetworkDevice::new(queues.clone(), 1500); + assert_eq!(queues.egress_bytes(), 0, "starts at zero"); + + // A guest frame must be staged before smoltcp's receive() can take it. + queues.guest_to_host.push(vec![0u8; 100]).unwrap(); + assert!(dev.stage_next_frame().is_some()); + let (rx, _tx) = dev.receive(Instant::ZERO).expect("frame available"); + rx.consume(|f| assert_eq!(f.len(), 100)); + assert_eq!(queues.egress_bytes(), 100, "one frame metered"); + + // A second frame accumulates. + queues.guest_to_host.push(vec![0u8; 40]).unwrap(); + assert!(dev.stage_next_frame().is_some()); + let (rx, _tx) = dev.receive(Instant::ZERO).expect("frame available"); + rx.consume(|_| {}); + assert_eq!(queues.egress_bytes(), 140, "cumulative"); + } + + #[test] + fn dropped_staged_frame_is_not_metered() { + let queues = NetworkFrameQueues::shared(DEFAULT_FRAME_QUEUE_CAPACITY); + let mut dev = VirtioNetworkDevice::new(queues.clone(), 1500); + queues.guest_to_host.push(vec![0u8; 100]).unwrap(); + assert!(dev.stage_next_frame().is_some()); + // The poll loop drops frames it won't forward (e.g. unsupported UDP); + // those never reach the stack, so they must not bill egress. + dev.drop_staged_frame(); + assert_eq!(queues.egress_bytes(), 0, "dropped frame not metered"); + } +} diff --git a/crates/smolvm-network/src/dns.rs b/crates/smolvm-network/src/dns.rs new file mode 100644 index 0000000..ebf53dd --- /dev/null +++ b/crates/smolvm-network/src/dns.rs @@ -0,0 +1,380 @@ +//! Minimal DNS wire parsing + allow-host filtering for the virtio-net gateway. +//! +//! This mirrors libkrun's TSI DNS filter (`vsock/dns_filter.rs`) so `--allow-host` +//! behaves identically on both backends: a query is forwarded upstream only when +//! its name matches the allow-host list (exact match or a subdomain), the answer's +//! A and AAAA records are learned as temporarily-allowed egress IPs, and a +//! disallowed name gets an NXDOMAIN. Hostname matching and TTL clamping match the +//! libkrun rules exactly. + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +const DNS_HEADER_LEN: usize = 12; +const DNS_ID_LEN: usize = 2; +const DNS_U16_LEN: usize = 2; +const DNS_U32_LEN: usize = 4; +const DNS_FLAGS_OFFSET: usize = 2; +const DNS_QDCOUNT_OFFSET: usize = 4; +const DNS_ANCOUNT_OFFSET: usize = 6; +const DNS_QUESTION_FIXED_LEN: usize = 4; +const DNS_RR_FIXED_LEN: usize = 10; +const DNS_RR_TYPE_OFFSET: usize = 0; +const DNS_RR_CLASS_OFFSET: usize = 2; +const DNS_RR_TTL_OFFSET: usize = 4; +const DNS_RR_RDLEN_OFFSET: usize = 8; +const DNS_CLASS_IN: u16 = 1; +const DNS_TYPE_A: u16 = 1; +const DNS_TYPE_AAAA: u16 = 28; +const DNS_A_RDATA_LEN: usize = 4; +const DNS_AAAA_RDATA_LEN: usize = 16; +pub const DNS_RCODE_SERVFAIL: u16 = 2; +pub const DNS_RCODE_NXDOMAIN: u16 = 3; +const DNS_RCODE_MASK: u16 = 0x000f; +const DNS_ONE_QUESTION: u16 = 1; +const DNS_FLAG_RESPONSE: u16 = 0x8000; +const DNS_FLAG_RECURSION_DESIRED: u16 = 0x0100; +const DNS_FLAG_RECURSION_AVAILABLE: u16 = 0x0080; +const DNS_POINTER_TAG: u8 = 0xc0; +const DNS_POINTER_MASK: u8 = 0xc0; +const DNS_POINTER_OFFSET_MASK: u8 = 0x3f; +const DNS_MAX_COMPRESSION_JUMPS: usize = 16; +const DNS_MAX_LABEL_LEN: usize = 63; + +/// Lowercase + strip a trailing dot. `None` for an empty name. +pub fn normalize_hostname(hostname: &str) -> Option { + let hostname = hostname.trim_end_matches('.').to_ascii_lowercase(); + if hostname.is_empty() { + None + } else { + Some(hostname) + } +} + +/// Whether `hostname` matches the allow-list: exact match, or a subdomain of an +/// allowed entry (`foo.example.com` matches `example.com`, `notexample.com` does +/// not). Mirrors libkrun's `is_hostname_allowed`. +pub fn hostname_allowed(hostname: &str, allowed_hosts: &[String]) -> bool { + let Some(hostname) = normalize_hostname(hostname) else { + return false; + }; + allowed_hosts.iter().any(|allowed| { + hostname == *allowed + || hostname + .strip_suffix(allowed) + .is_some_and(|prefix| prefix.ends_with('.')) + }) +} + +/// The question name in a DNS query (first question only). `None` on malformed input. +pub fn question_name(packet: &[u8]) -> Option { + if packet.len() < DNS_HEADER_LEN || read_u16(packet, DNS_QDCOUNT_OFFSET)? != DNS_ONE_QUESTION { + return None; + } + let (name, _) = read_name(packet, DNS_HEADER_LEN)?; + Some(name) +} + +/// Extract `(IpAddr, ttl)` pairs from the A and AAAA records in a DNS response. +pub fn answer_ip_records(packet: &[u8]) -> Vec<(IpAddr, u32)> { + parse_answer_ip_records(packet).unwrap_or_default() +} + +fn parse_answer_ip_records(packet: &[u8]) -> Option> { + if packet.len() < DNS_HEADER_LEN { + return None; + } + let qdcount = read_u16(packet, DNS_QDCOUNT_OFFSET)? as usize; + let ancount = read_u16(packet, DNS_ANCOUNT_OFFSET)? as usize; + let mut offset = DNS_HEADER_LEN; + + for _ in 0..qdcount { + let (_, after_name) = read_name(packet, offset)?; + if after_name + DNS_QUESTION_FIXED_LEN > packet.len() { + return None; + } + offset = after_name + DNS_QUESTION_FIXED_LEN; + } + + let mut ips = Vec::new(); + for _ in 0..ancount { + let (_, after_name) = read_name(packet, offset)?; + if after_name + DNS_RR_FIXED_LEN > packet.len() { + return None; + } + offset = after_name; + + let rr_type = read_u16(packet, offset + DNS_RR_TYPE_OFFSET)?; + let class = read_u16(packet, offset + DNS_RR_CLASS_OFFSET)?; + let ttl = read_u32(packet, offset + DNS_RR_TTL_OFFSET)?; + let rdlen = read_u16(packet, offset + DNS_RR_RDLEN_OFFSET)? as usize; + offset += DNS_RR_FIXED_LEN; + + if offset + rdlen > packet.len() { + return None; + } + if class == DNS_CLASS_IN && rr_type == DNS_TYPE_A && rdlen == DNS_A_RDATA_LEN { + ips.push(( + IpAddr::V4(Ipv4Addr::new( + packet[offset], + packet[offset + 1], + packet[offset + 2], + packet[offset + 3], + )), + ttl, + )); + } else if class == DNS_CLASS_IN && rr_type == DNS_TYPE_AAAA && rdlen == DNS_AAAA_RDATA_LEN { + let octets: [u8; DNS_AAAA_RDATA_LEN] = packet[offset..offset + DNS_AAAA_RDATA_LEN] + .try_into() + .ok()?; + ips.push((IpAddr::V6(Ipv6Addr::from(octets)), ttl)); + } + offset += rdlen; + } + Some(ips) +} + +/// Build a DNS response carrying just an error `rcode` (e.g. NXDOMAIN), echoing +/// the query id and question. Mirrors libkrun's `build_error_response`. +pub fn error_response(query: &[u8], rcode: u16) -> Vec { + let id: &[u8] = if query.len() >= DNS_ID_LEN { + &query[..DNS_ID_LEN] + } else { + &[0, 0] + }; + let req_flags = if query.len() >= DNS_FLAGS_OFFSET + DNS_U16_LEN { + read_u16(query, DNS_FLAGS_OFFSET).unwrap_or(0) + } else { + 0 + }; + let flags = DNS_FLAG_RESPONSE + | (req_flags & DNS_FLAG_RECURSION_DESIRED) + | DNS_FLAG_RECURSION_AVAILABLE + | (rcode & DNS_RCODE_MASK); + + // Echo the question section (qdcount stays as in the query) when parseable. + let question_end = question_section_end(query); + let qdcount = if question_end.is_some() { 1u16 } else { 0 }; + + let mut response = Vec::with_capacity(question_end.unwrap_or(DNS_HEADER_LEN)); + response.extend_from_slice(id); + response.extend_from_slice(&flags.to_be_bytes()); + response.extend_from_slice(&qdcount.to_be_bytes()); + response.extend_from_slice(&0u16.to_be_bytes()); // ancount + response.extend_from_slice(&0u16.to_be_bytes()); // nscount + response.extend_from_slice(&0u16.to_be_bytes()); // arcount + if let Some(end) = question_end { + response.extend_from_slice(&query[DNS_HEADER_LEN..end]); + } + response +} + +fn question_section_end(packet: &[u8]) -> Option { + if packet.len() < DNS_HEADER_LEN || read_u16(packet, DNS_QDCOUNT_OFFSET)? != DNS_ONE_QUESTION { + return None; + } + let (_, after_name) = read_name(packet, DNS_HEADER_LEN)?; + let end = after_name + DNS_QUESTION_FIXED_LEN; + if end > packet.len() { + return None; + } + Some(end) +} + +/// Parse a DNS name (with compression pointers), returning `(name, next_offset)`. +/// `next_offset` is the byte after the name in the *original* (non-jumped) stream. +fn read_name(packet: &[u8], offset: usize) -> Option<(String, usize)> { + let mut labels = Vec::new(); + let mut pos = offset; + let mut next_offset = offset; + let mut jumped = false; + let mut jumps = 0; + + loop { + let len = *packet.get(pos)?; + if len & DNS_POINTER_MASK == DNS_POINTER_TAG { + let lo = *packet.get(pos + 1)?; + let pointer = (((len & DNS_POINTER_OFFSET_MASK) as usize) << 8) | lo as usize; + if pointer >= packet.len() { + return None; + } + if !jumped { + next_offset = pos + DNS_U16_LEN; + } + pos = pointer; + jumped = true; + jumps += 1; + if jumps > DNS_MAX_COMPRESSION_JUMPS { + return None; + } + continue; + } + if len & DNS_POINTER_MASK != 0 { + return None; + } + + pos += 1; + if len == 0 { + if !jumped { + next_offset = pos; + } + break; + } + + let len = len as usize; + if len > DNS_MAX_LABEL_LEN || pos + len > packet.len() { + return None; + } + let label = std::str::from_utf8(&packet[pos..pos + len]).ok()?; + labels.push(label.to_ascii_lowercase()); + pos += len; + if !jumped { + next_offset = pos; + } + } + + Some((labels.join("."), next_offset)) +} + +fn read_u16(buf: &[u8], offset: usize) -> Option { + let bytes = buf.get(offset..offset + DNS_U16_LEN)?; + Some(u16::from_be_bytes([bytes[0], bytes[1]])) +} + +fn read_u32(buf: &[u8], offset: usize) -> Option { + let bytes = buf.get(offset..offset + DNS_U32_LEN)?; + Some(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A query for `name` with one question (A/IN). + fn query_for(name: &str) -> Vec { + let mut q = vec![ + 0x12, 0x34, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + ]; + for label in name.split('.') { + q.push(label.len() as u8); + q.extend_from_slice(label.as_bytes()); + } + q.push(0); + q.extend_from_slice(&DNS_TYPE_A.to_be_bytes()); + q.extend_from_slice(&DNS_CLASS_IN.to_be_bytes()); + q + } + + #[test] + fn parses_question_name() { + assert_eq!( + question_name(&query_for("www.example.com")).as_deref(), + Some("www.example.com") + ); + } + + #[test] + fn hostname_matching_allows_exact_and_subdomain_only() { + let allow = vec!["example.com".to_string()]; + assert!(hostname_allowed("example.com", &allow)); + assert!(hostname_allowed("www.example.com", &allow)); + assert!(hostname_allowed("a.b.example.com", &allow)); + assert!(!hostname_allowed("notexample.com", &allow)); + assert!(!hostname_allowed("example.com.evil.com", &allow)); + } + + #[test] + fn empty_allow_list_blocks_all() { + assert!(!hostname_allowed("example.com", &[])); + } + + #[test] + fn extracts_a_records_with_ttl() { + // Response: echo the question, then one A record 93.184.216.34 ttl 300. + let mut r = query_for("example.com"); + r[2] = 0x81; // QR=1, RD=1 + r[3] = 0x80; // RA=1 + r[6] = 0x00; + r[7] = 0x01; // ancount = 1 + // Answer: name pointer to question (0xc00c), type A, class IN, ttl, rdlen, rdata + r.extend_from_slice(&[0xc0, 0x0c]); + r.extend_from_slice(&DNS_TYPE_A.to_be_bytes()); + r.extend_from_slice(&DNS_CLASS_IN.to_be_bytes()); + r.extend_from_slice(&300u32.to_be_bytes()); + r.extend_from_slice(&(DNS_A_RDATA_LEN as u16).to_be_bytes()); + r.extend_from_slice(&[93, 184, 216, 34]); + + let records = answer_ip_records(&r); + assert_eq!( + records, + vec![(IpAddr::V4(Ipv4Addr::new(93, 184, 216, 34)), 300)] + ); + } + + #[test] + fn extracts_aaaa_records_with_ttl() { + // Response with one AAAA record 2606:2800:21f:cb07:6820:80da:af6b:8b2c ttl 600. + let v6: Ipv6Addr = "2606:2800:21f:cb07:6820:80da:af6b:8b2c".parse().unwrap(); + let mut r = query_for("example.com"); + r[2] = 0x81; + r[3] = 0x80; + r[6] = 0x00; + r[7] = 0x01; // ancount = 1 + r.extend_from_slice(&[0xc0, 0x0c]); + r.extend_from_slice(&DNS_TYPE_AAAA.to_be_bytes()); + r.extend_from_slice(&DNS_CLASS_IN.to_be_bytes()); + r.extend_from_slice(&600u32.to_be_bytes()); + r.extend_from_slice(&(DNS_AAAA_RDATA_LEN as u16).to_be_bytes()); + r.extend_from_slice(&v6.octets()); + + let records = answer_ip_records(&r); + assert_eq!(records, vec![(IpAddr::V6(v6), 600)]); + } + + #[test] + fn extracts_mixed_a_and_aaaa_records() { + let v6: Ipv6Addr = "2606:4700::6810:84e5".parse().unwrap(); + let mut r = query_for("example.com"); + r[2] = 0x81; + r[3] = 0x80; + r[6] = 0x00; + r[7] = 0x02; // ancount = 2 + r.extend_from_slice(&[0xc0, 0x0c]); + r.extend_from_slice(&DNS_TYPE_A.to_be_bytes()); + r.extend_from_slice(&DNS_CLASS_IN.to_be_bytes()); + r.extend_from_slice(&300u32.to_be_bytes()); + r.extend_from_slice(&(DNS_A_RDATA_LEN as u16).to_be_bytes()); + r.extend_from_slice(&[104, 16, 132, 229]); + r.extend_from_slice(&[0xc0, 0x0c]); + r.extend_from_slice(&DNS_TYPE_AAAA.to_be_bytes()); + r.extend_from_slice(&DNS_CLASS_IN.to_be_bytes()); + r.extend_from_slice(&600u32.to_be_bytes()); + r.extend_from_slice(&(DNS_AAAA_RDATA_LEN as u16).to_be_bytes()); + r.extend_from_slice(&v6.octets()); + + let records = answer_ip_records(&r); + assert_eq!( + records, + vec![ + (IpAddr::V4(Ipv4Addr::new(104, 16, 132, 229)), 300), + (IpAddr::V6(v6), 600), + ] + ); + } + + #[test] + fn error_response_is_well_formed() { + let q = query_for("blocked.test"); + let resp = error_response(&q, DNS_RCODE_NXDOMAIN); + // QR bit set + rcode NXDOMAIN in low nibble of flags. + let flags = read_u16(&resp, DNS_FLAGS_OFFSET).unwrap(); + assert_eq!(flags & DNS_FLAG_RESPONSE, DNS_FLAG_RESPONSE); + assert_eq!(flags & DNS_RCODE_MASK, DNS_RCODE_NXDOMAIN); + assert_eq!(&resp[..2], &q[..2]); // echoed id + } + + #[test] + fn malformed_packets_dont_panic() { + assert_eq!(question_name(&[0, 1, 2]), None); + assert!(answer_ip_records(&[0xff; 4]).is_empty()); + } +} diff --git a/crates/smolvm-network/src/dns_relay.rs b/crates/smolvm-network/src/dns_relay.rs new file mode 100644 index 0000000..e070765 --- /dev/null +++ b/crates/smolvm-network/src/dns_relay.rs @@ -0,0 +1,497 @@ +//! Off-poll-thread DNS resolver for the virtio-net backend. +//! +//! Context +//! ======= +//! +//! Guest DNS (UDP/TCP :53) is intercepted by the gateway and forwarded to an +//! upstream resolver (`stack.rs::process_dns_queries` / `process_dns_tcp`). The +//! upstream exchange is a *blocking* host socket round trip with a 2-second +//! timeout. It used to run inline on the single virtio-net poll thread — the +//! thread that owns the smoltcp interface and every one of that VM's TCP/UDP/ +//! ICMP sockets. A slow or dead resolver therefore stalled *all* of the guest's +//! traffic for up to 2 seconds (head-of-line block). +//! +//! This module moves that blocking resolution off the poll thread, mirroring +//! the UDP/ICMP relay offload ([`crate::udp_relay`]). The poll loop only: +//! 1. reads a guest query out of its smoltcp DNS socket, +//! 2. applies the egress allow-host policy itself (cheap, no host I/O), +//! 3. hands allowed queries to this relay over a channel, and +//! 4. later picks the answer back up via the shared relay wake + channel and +//! writes it into the guest socket — never blocking on the resolver. +//! +//! Egress policy (allow/deny, `learn_ip_records`) stays on the poll thread so +//! [`EgressPolicy`](crate::egress::EgressPolicy) is never shared across threads; +//! only the raw upstream forward is offloaded. +//! +//! ```text +//! guest :53 query -> smoltcp gateway socket +//! -> poll loop: classify (allow-host) -> allowed? +//! no -> answer NXDOMAIN/SERVFAIL immediately (no relay) +//! yes -> assign id, remember reply context, channel (id, query) to relay +//! -> relay thread: UDP -> non-blocking connected host socket + poller +//! TCP -> bounded detached worker (rare path) +//! -> answer bytes -> channel back -> reply_wake +//! -> poll loop: learn A/AAAA records, write answer into the guest socket +//! ``` +//! +//! DNS is low-volume and its answers are quick, so the tables here are small and +//! loss under saturation just makes a guest see a normal DNS timeout. + +use crate::queues::WakePipe; +use crate::virtio_net_log; +use polling::{Event, Events}; +use std::io::{Read, Write}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, TcpStream, UdpSocket as HostUdpSocket}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError, TrySendError}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant}; + +/// DNS service port. +const DNS_PORT: u16 = 53; +/// Upstream exchange timeout, matching the previous inline behaviour. +const UPSTREAM_TIMEOUT: Duration = Duration::from_secs(2); +/// Largest DNS message we accept back (EDNS0 / DNS-over-TCP bound). +const DNS_MAX_MSG: usize = 4096; +/// Max in-flight queries buffered in each channel direction. +const CHANNEL_CAPACITY: usize = 256; +/// Max concurrent in-flight UDP queries with a live host socket. +const MAX_INFLIGHT_UDP: usize = 256; +/// Max concurrent in-flight DNS-over-TCP worker threads. +const MAX_INFLIGHT_TCP: usize = 64; +/// Relay thread poll ceiling so shutdown and deadlines are noticed promptly. +const RELAY_POLL_MAX_MS: u64 = 250; + +/// Which upstream transport a query must use. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DnsTransport { + Udp, + Tcp, +} + +/// A query handed from the poll loop to the relay thread. +pub struct DnsQuery { + /// Correlation id chosen by the poll loop; echoed back in the response. + pub id: u64, + /// Transport to reach the upstream resolver on. + pub transport: DnsTransport, + /// Upstream resolver address. + pub upstream: Ipv4Addr, + /// Raw DNS query message (no TCP length prefix). + pub query: Vec, +} + +/// A resolved answer handed back to the poll loop. +pub struct DnsResponse { + /// The id of the originating [`DnsQuery`]. + pub id: u64, + /// Raw DNS answer message, or `None` on error/timeout (guest sees a normal + /// DNS timeout — identical to the old inline `forward` error path). + pub answer: Option>, +} + +/// Channel pair connecting the poll loop and the DNS relay thread. +pub struct DnsRelayChannels { + /// Poll loop -> relay thread. + pub to_relay: SyncSender, + /// Relay thread -> poll loop. + pub from_relay: Receiver, + /// Wakes the relay thread after `to_relay` sends. + pub relay_thread_wake: WakePipe, +} + +/// Start the DNS relay thread. Returns the poll-loop-side channel endpoints. +/// +/// `reply_wake` is the smoltcp poll loop's existing relay wake pipe — pulsed +/// whenever an answer is queued so the loop wakes to deliver it. The thread +/// exits when `shutdown` reports true (checked at least once per +/// [`RELAY_POLL_MAX_MS`]). +pub fn start_dns_relay( + reply_wake: Arc, + shutdown: Arc bool + Send + Sync>, +) -> DnsRelayChannels { + let (to_relay_tx, to_relay_rx) = mpsc::sync_channel(CHANNEL_CAPACITY); + let (from_relay_tx, from_relay_rx) = mpsc::sync_channel(CHANNEL_CAPACITY); + let relay_thread_wake = WakePipe::new(); + let thread_wake = relay_thread_wake.clone(); + + let _ = thread::Builder::new() + .name("smolvm-dns-relay".into()) + .spawn(move || { + run_dns_relay( + to_relay_rx, + from_relay_tx, + thread_wake, + reply_wake, + shutdown, + ); + }); + + DnsRelayChannels { + to_relay: to_relay_tx, + from_relay: from_relay_rx, + relay_thread_wake, + } +} + +/// One in-flight UDP query: a non-blocking connected host socket awaiting its +/// single response, and the deadline after which we give up. +struct InflightUdp { + socket: HostUdpSocket, + deadline: Instant, +} + +/// Send one answer back to the poll loop. Returns `true` if the caller should +/// wake the poll loop, `false` if the channel is closed (relay should exit). +fn deliver_answer(inbound: &SyncSender, id: u64, answer: Option>) -> bool { + match inbound.try_send(DnsResponse { id, answer }) { + Ok(()) => true, + Err(TrySendError::Full(_)) => { + virtio_net_log!( + "virtio-net: dropping DNS answer id={} (inbound queue full)", + id + ); + true + } + Err(TrySendError::Disconnected(_)) => false, + } +} + +fn run_dns_relay( + outbound: Receiver, + inbound: SyncSender, + wake: WakePipe, + reply_wake: Arc, + shutdown: Arc bool + Send + Sync>, +) { + let mut inflight: std::collections::HashMap = + std::collections::HashMap::new(); + let mut recv_buf = vec![0u8; DNS_MAX_MSG]; + let tcp_inflight = Arc::new(AtomicUsize::new(0)); + + loop { + if shutdown() { + return; + } + + let mut woke_reply = false; + + // Outbound: queries handed over by the poll loop. + loop { + match outbound.try_recv() { + Ok(query) => match query.transport { + DnsTransport::Udp => { + if inflight.len() >= MAX_INFLIGHT_UDP { + virtio_net_log!( + "virtio-net: dropping DNS query id={} (in-flight UDP table full)", + query.id + ); + woke_reply |= deliver_answer(&inbound, query.id, None); + continue; + } + let upstream = SocketAddr::new(IpAddr::V4(query.upstream), DNS_PORT); + match start_udp_query(upstream, &query.query) { + Ok(socket) => { + inflight.insert( + query.id, + InflightUdp { + socket, + deadline: Instant::now() + UPSTREAM_TIMEOUT, + }, + ); + } + Err(err) => { + virtio_net_log!( + "virtio-net: DNS/UDP upstream send failed id={} error={}", + query.id, + err + ); + woke_reply |= deliver_answer(&inbound, query.id, None); + } + } + } + DnsTransport::Tcp => { + spawn_tcp_query(&inbound, &reply_wake, &tcp_inflight, query); + } + }, + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => return, + } + } + + // Inbound: block on "wake OR any in-flight UDP socket readable", exactly + // like the UDP relay. Each socket is registered at key `slot + 1`. + let poller = wake.poller(); + let ids: Vec = inflight.keys().copied().collect(); + for (slot, id) in ids.iter().enumerate() { + // SAFETY: the socket is owned by `inflight` and is deleted from the + // poller below before it can be dropped. + let _ = unsafe { poller.add(&inflight[id].socket, Event::readable(slot + 1)) }; + } + + // Wake no later than the nearest deadline so timeouts fire on time. + let now = Instant::now(); + let mut wait = Duration::from_millis(RELAY_POLL_MAX_MS); + for f in inflight.values() { + let remaining = f.deadline.saturating_duration_since(now); + if remaining < wait { + wait = remaining; + } + } + + let mut events = Events::new(); + let _ = poller.wait(&mut events, Some(wait)); + + let mut ready: Vec = vec![false; ids.len()]; + for event in events.iter() { + if event.key >= 1 && event.key - 1 < ready.len() { + ready[event.key - 1] = true; + } + } + + // Deregister every socket before mutating `inflight`. + for id in &ids { + let _ = poller.delete(&inflight[id].socket); + } + + // Deliver ready answers; expire the rest past their deadline. + let now = Instant::now(); + for (slot, id) in ids.iter().enumerate() { + if ready[slot] { + let answer = inflight + .get(id) + .and_then(|f| match f.socket.recv(&mut recv_buf) { + Ok(len) => Some(recv_buf[..len].to_vec()), + Err(_) => None, + }); + match answer { + Some(bytes) => { + inflight.remove(id); + woke_reply |= deliver_answer(&inbound, *id, Some(bytes)); + } + // Spurious readiness with nothing to read: leave it in flight + // to be retried or expired. + None if inflight.get(id).is_some_and(|f| f.deadline <= now) => { + inflight.remove(id); + woke_reply |= deliver_answer(&inbound, *id, None); + } + None => {} + } + } else if inflight.get(id).is_some_and(|f| f.deadline <= now) { + inflight.remove(id); + woke_reply |= deliver_answer(&inbound, *id, None); + } + } + + if woke_reply { + reply_wake.wake(); + } + } +} + +/// Open a non-blocking host UDP socket connected to the upstream resolver and +/// send the query. The single response is collected later by the poll section. +fn start_udp_query(upstream: SocketAddr, query: &[u8]) -> std::io::Result { + let bind: SocketAddr = if upstream.is_ipv4() { + (Ipv4Addr::UNSPECIFIED, 0).into() + } else { + (std::net::Ipv6Addr::UNSPECIFIED, 0).into() + }; + let socket = HostUdpSocket::bind(bind)?; + socket.connect(upstream)?; + socket.set_nonblocking(true)?; + socket.send(query)?; + Ok(socket) +} + +/// Resolve a DNS-over-TCP query on a bounded, detached worker thread. +/// +/// DNS/TCP is the rare fallback path (truncated/large answers). Rather than +/// build a non-blocking length-prefixed TCP state machine, each query gets its +/// own short-lived worker so a slow TCP resolver never blocks the UDP fast path +/// or the poll loop. The worker count is capped; over the cap the query is +/// answered as a timeout. +fn spawn_tcp_query( + inbound: &SyncSender, + reply_wake: &Arc, + tcp_inflight: &Arc, + query: DnsQuery, +) { + if tcp_inflight.load(Ordering::Relaxed) >= MAX_INFLIGHT_TCP { + virtio_net_log!( + "virtio-net: dropping DNS/TCP query id={} (worker cap reached)", + query.id + ); + if deliver_answer(inbound, query.id, None) { + reply_wake.wake(); + } + return; + } + tcp_inflight.fetch_add(1, Ordering::Relaxed); + let worker_inbound = inbound.clone(); + let worker_wake = reply_wake.clone(); + let worker_inflight = tcp_inflight.clone(); + let id = query.id; + let spawned = thread::Builder::new() + .name("smolvm-dns-tcp".into()) + .spawn(move || { + let upstream = SocketAddr::new(IpAddr::V4(query.upstream), DNS_PORT); + let answer = forward_dns_query_tcp(upstream, &query.query).ok(); + if deliver_answer(&worker_inbound, id, answer) { + worker_wake.wake(); + } + worker_inflight.fetch_sub(1, Ordering::Relaxed); + }); + if spawned.is_err() { + // Could not spawn: undo the reservation and answer as a timeout. + tcp_inflight.fetch_sub(1, Ordering::Relaxed); + if deliver_answer(inbound, id, None) { + reply_wake.wake(); + } + } +} + +/// Forward one DNS query to the upstream resolver over TCP (length-prefixed, per +/// RFC 1035 §4.2.2) and return the raw response message. Blocking host TCP +/// exchange with a short timeout — runs only on a detached worker, never the +/// poll thread. +fn forward_dns_query_tcp(upstream: SocketAddr, query: &[u8]) -> std::io::Result> { + use std::io::{Error, ErrorKind}; + let len = u16::try_from(query.len()) + .map_err(|_| Error::new(ErrorKind::InvalidInput, "DNS query too large for TCP"))?; + let mut stream = TcpStream::connect_timeout(&upstream, UPSTREAM_TIMEOUT)?; + stream.set_read_timeout(Some(UPSTREAM_TIMEOUT))?; + stream.set_write_timeout(Some(UPSTREAM_TIMEOUT))?; + stream.write_all(&len.to_be_bytes())?; + stream.write_all(query)?; + stream.flush()?; + + let mut len_buf = [0u8; 2]; + stream.read_exact(&mut len_buf)?; + let resp_len = u16::from_be_bytes(len_buf) as usize; + if resp_len == 0 || resp_len > DNS_MAX_MSG { + return Err(Error::new( + ErrorKind::InvalidData, + "upstream DNS/TCP response length out of range", + )); + } + let mut response = vec![0u8; resp_len]; + stream.read_exact(&mut response)?; + Ok(response) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicBool; + + /// `start_udp_query` sends off a non-blocking connected socket; its single + /// response is collected later without blocking. Exercises the real fn + /// against a loopback echo "resolver" (arbitrary port, so no port-53 dep). + #[test] + fn start_udp_query_sends_and_receives_nonblocking() { + let upstream = HostUdpSocket::bind("127.0.0.1:0").unwrap(); + let upstream_addr = upstream.local_addr().unwrap(); + let resolver = thread::spawn(move || { + let mut buf = [0u8; 512]; + upstream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let (n, peer) = upstream.recv_from(&mut buf).unwrap(); + upstream.send_to(&buf[..n], peer).unwrap(); + }); + + let sock = start_udp_query(upstream_addr, b"\x12\x34hello-dns").unwrap(); + + let deadline = Instant::now() + Duration::from_secs(2); + let mut buf = [0u8; 512]; + loop { + match sock.recv(&mut buf) { + Ok(n) => { + assert_eq!(&buf[..n], b"\x12\x34hello-dns"); + break; + } + Err(_) if Instant::now() < deadline => { + thread::sleep(Duration::from_millis(5)); + } + Err(e) => panic!("no DNS answer: {e}"), + } + } + resolver.join().unwrap(); + } + + /// End-to-end through the relay thread: a live upstream on port 53 needs + /// privileges, so this proves the *offload contract* — the poll thread only + /// sends/receives channel messages and never blocks — using an unreachable + /// upstream that must resolve to a timeout answer, promptly and off-thread. + #[test] + fn poll_thread_never_blocks_on_dead_resolver() { + let reply_wake = Arc::new(WakePipe::new()); + let stop = Arc::new(AtomicBool::new(false)); + let stop_flag = stop.clone(); + let channels = start_dns_relay( + reply_wake.clone(), + Arc::new(move || stop_flag.load(Ordering::Relaxed)), + ); + + // 192.0.2.1 is TEST-NET-1 (RFC 5737): guaranteed unroutable, so the + // upstream never answers and the relay must time it out for us. + let started = Instant::now(); + channels + .to_relay + .send(DnsQuery { + id: 7, + transport: DnsTransport::Udp, + upstream: Ipv4Addr::new(192, 0, 2, 1), + query: b"\x00\x00query".to_vec(), + }) + .unwrap(); + channels.relay_thread_wake.wake(); + + // The send returned immediately (offloaded); this thread is free. + assert!(started.elapsed() < Duration::from_millis(50)); + + // The answer (a timeout -> None) arrives via the channel, driven by the + // relay thread, within a bit over the 2s upstream timeout. + let resp = channels + .from_relay + .recv_timeout(Duration::from_secs(4)) + .expect("relay must always answer, even on timeout"); + assert_eq!(resp.id, 7); + assert!(resp.answer.is_none()); + + stop.store(true, Ordering::Relaxed); + channels.relay_thread_wake.wake(); + } + + /// The real `forward_dns_query_tcp` resolves against a length-prefixed host + /// TCP resolver (arbitrary loopback port) and returns the raw, unprefixed + /// answer message. This is the code the detached DNS/TCP worker runs. + #[test] + fn forward_dns_query_tcp_round_trips() { + use std::net::TcpListener; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + + // Minimal DNS-over-TCP resolver: read the length-prefixed query, reply + // with a length-prefixed canned answer. + let server = thread::spawn(move || { + let (mut sock, _) = listener.accept().unwrap(); + let mut len_buf = [0u8; 2]; + sock.read_exact(&mut len_buf).unwrap(); + let qlen = u16::from_be_bytes(len_buf) as usize; + let mut q = vec![0u8; qlen]; + sock.read_exact(&mut q).unwrap(); + assert_eq!(&q, b"tcp-query"); + let answer = b"answer-bytes"; + sock.write_all(&(answer.len() as u16).to_be_bytes()) + .unwrap(); + sock.write_all(answer).unwrap(); + }); + + let resp = forward_dns_query_tcp(addr, b"tcp-query").unwrap(); + assert_eq!(resp, b"answer-bytes"); + server.join().unwrap(); + } +} diff --git a/crates/smolvm-network/src/egress.rs b/crates/smolvm-network/src/egress.rs new file mode 100644 index 0000000..140b6ba --- /dev/null +++ b/crates/smolvm-network/src/egress.rs @@ -0,0 +1,566 @@ +//! Outbound egress policy for the virtio-net gateway. +//! +//! TSI enforces `allowed_cidrs` + `--allow-host` inside libkrun's socket-intercept +//! layer; the virtio-net gateway terminates every guest flow itself, so it applies +//! the same allow-list at the point it opens a host connection +//! (`TcpRelayTable::create_tcp_socket`). This mirrors libkrun's `vsock/dns_filter.rs` +//! `EgressPolicy` so both backends behave identically: +//! +//! - static `allowed_cidrs` (IPv4 or IPv6) are always permitted; +//! - `--allow-host` names are matched by the gateway's DNS interception, and the +//! A/AAAA records of allowed answers are *learned* as temporarily-allowed IPs +//! (TTL clamped to [60s, 3600s]) so the follow-up connection passes; +//! - with hosts set but no CIDRs, egress is gated entirely by learned IPs. +//! +//! Disallowed destinations are dropped before any host socket is created. DNS +//! forwarding (gateway-internal) is never gated by this filter. + +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use crate::dns; + +/// Learned-IP TTL clamp, matching libkrun's DNS filter. +const MIN_LEARNED_TTL: u64 = 60; +const MAX_LEARNED_TTL: u64 = 3600; + +/// A parsed CIDR (IPv4 or IPv6) with cheap containment testing. +#[derive(Clone, Copy, Debug)] +enum Cidr { + V4 { network: u32, mask: u32 }, + V6 { network: u128, mask: u128 }, +} + +impl Cidr { + /// Parse `"a.b.c.d"` / `"a.b.c.d/n"` / `"x::y"` / `"x::y/n"`. A bare address + /// gets a full-length prefix. Returns `None` for malformed input or a prefix + /// length beyond the address width. + fn parse(spec: &str) -> Option { + let (addr, prefix) = match spec.trim().split_once('/') { + Some((addr, prefix)) => (addr, Some(prefix.parse::().ok()?)), + None => (spec.trim(), None), + }; + match addr.parse::().ok()? { + IpAddr::V4(ip) => { + let prefix = prefix.unwrap_or(32); + if prefix > 32 { + return None; + } + let mask = if prefix == 0 { + 0 + } else { + u32::MAX << (32 - prefix) + }; + Some(Self::V4 { + network: u32::from(ip) & mask, + mask, + }) + } + IpAddr::V6(ip) => { + let prefix = prefix.unwrap_or(128); + if prefix > 128 { + return None; + } + let mask = if prefix == 0 { + 0 + } else { + u128::MAX << (128 - prefix) + }; + Some(Self::V6 { + network: u128::from(ip) & mask, + mask, + }) + } + } + } + + fn contains(&self, ip: IpAddr) -> bool { + match (self, ip) { + (Self::V4 { network, mask }, IpAddr::V4(ip)) => (u32::from(ip) & mask) == *network, + (Self::V6 { network, mask }, IpAddr::V6(ip)) => (u128::from(ip) & mask) == *network, + _ => false, + } + } +} + +struct AllowList { + cidrs: Vec, + /// Normalized allow-host names. `None` = no DNS hostname filtering. + allowed_hosts: Option>, + /// IPs learned from allowed DNS answers → expiry instant. + learned: Mutex>, +} + +/// How much of the platform hard-floor applies, chosen once per policy from the +/// deployment context — NOT a default-on blanket deny. Reaching the host's own +/// LAN from a local VM is legitimate and expected, so the broad internal-subnet +/// floor is reserved for the multi-tenant context where it's actually needed. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum FloorMode { + /// Trusted single-tenant/local override (`SMOLVM_EGRESS_ALLOW_PRIVATE=1`): + /// floor nothing — the guest reaches exactly what the host can. + Off, + /// Local default: deny ONLY the cloud-metadata link-local range + /// (`169.254.0.0/16`, incl. `169.254.169.254`) — never a legitimate + /// destination and the canonical SSRF/credential-theft target, but still + /// protects users running on their own cloud VM. The guest keeps reaching the + /// host's LAN, loopback, etc., so local behavior is reasonable and clear. + MetadataOnly, + /// Multi-tenant/fleet (`SMOLVM_PUBLISH_ADDR` set): the full floor — metadata, + /// host/control internal subnets, loopback, link/unique-local, and the + /// gateway CGNAT range — so a guest can't steal host credentials, pivot to + /// the control plane / worker API, or reach co-resident tenants. + Strict, +} + +/// Parse an explicit `SMOLVM_EGRESS_FLOOR` value into a mode. Returns `None` +/// for an absent/unrecognized value so the caller falls back to the inferred +/// default. Pure (no env) so it is unit-testable. +fn parse_floor_override(v: &str) -> Option { + match v.trim().to_ascii_lowercase().as_str() { + "strict" => Some(FloorMode::Strict), + "metadata" | "metadata-only" | "metadataonly" => Some(FloorMode::MetadataOnly), + "off" | "none" => Some(FloorMode::Off), + _ => None, + } +} + +/// Resolve the floor from the deployment context. Read once at policy creation +/// (never per-packet): explicit `SMOLVM_EGRESS_FLOOR` override wins, else the +/// `ALLOW_PRIVATE` opt-out, else fleet ⇒ strict, else the metadata-only default. +fn floor_mode() -> FloorMode { + // Explicit override wins (highest precedence). A multi-tenant node sets + // `SMOLVM_EGRESS_FLOOR=strict` so the floor is fail-closed and never + // silently degrades to metadata-only if `SMOLVM_PUBLISH_ADDR` is missing + // from the environment (a dropped unit override, a new provisioner, or a + // manual launch must NOT quietly expose the host LAN / control plane / + // co-tenants to a guest). `metadata`/`off` allow a deliberate downgrade. + if let Ok(v) = std::env::var("SMOLVM_EGRESS_FLOOR") { + if let Some(mode) = parse_floor_override(&v) { + return mode; + } + } + let allow_private = std::env::var("SMOLVM_EGRESS_ALLOW_PRIVATE") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + if allow_private { + FloorMode::Off + } else if std::env::var_os("SMOLVM_PUBLISH_ADDR").is_some() { + FloorMode::Strict + } else { + FloorMode::MetadataOnly + } +} + +/// The cloud-metadata link-local range (`169.254.0.0/16` / `fe80::/10`) — the +/// one destination floored in every mode except `Off`, including via an +/// IPv4-mapped IPv6 address so it can't be smuggled past. +fn is_link_local(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => v4.is_link_local(), + IpAddr::V6(v6) => { + (v6.segments()[0] & 0xffc0) == 0xfe80 + || v6.to_ipv4_mapped().is_some_and(|v4| v4.is_link_local()) + } + } +} + +/// The full multi-tenant IPv4 floor (metadata + internal + loopback + CGNAT). +fn is_reserved_v4(v4: Ipv4Addr) -> bool { + v4.is_loopback() // 127.0.0.0/8 + || v4.is_link_local() // 169.254.0.0/16 — incl. 169.254.169.254 (cloud metadata) + || v4.is_private() // 10/8, 172.16/12, 192.168/16 — host/control internal subnet + || v4.is_unspecified() + || v4.is_broadcast() + // 100.64.0.0/10 (CGNAT) — the gateway's own guest/gateway addresses live here. + || matches!(v4.octets(), [100, b, ..] if (64..=127).contains(&b)) +} + +/// Whether `ip` is floored under `mode` — the single hard-floor predicate. Also +/// defeats DNS-rebinding (a learned IP in a floored range is still denied). +fn is_floored(ip: IpAddr, mode: FloorMode) -> bool { + match mode { + FloorMode::Off => false, + FloorMode::MetadataOnly => is_link_local(ip), + FloorMode::Strict => match ip { + IpAddr::V4(v4) => is_reserved_v4(v4), + IpAddr::V6(v6) => { + v6.is_loopback() + || v6.is_unspecified() + || (v6.segments()[0] & 0xffc0) == 0xfe80 // fe80::/10 link-local + || (v6.segments()[0] & 0xfe00) == 0xfc00 // fc00::/7 unique-local + || v6.to_ipv4_mapped().is_some_and(is_reserved_v4) + } + }, + } +} + +/// Outbound egress policy enforced by the gateway before opening a host +/// connection. `unrestricted` allows everything EXCEPT the platform hard-floor +/// (`is_floored`), whose scope is set once by `FloorMode`. +#[derive(Clone)] +pub struct EgressPolicy { + inner: Option>, + /// Hard-floor scope, resolved once from the deployment context at creation. + floor: FloorMode, +} + +impl EgressPolicy { + /// No allow-list — every destination is allowed EXCEPT the platform hard-floor. + pub fn unrestricted() -> Self { + Self { + inner: None, + floor: floor_mode(), + } + } + + /// Build from `VmResources::allowed_cidrs` and the `--allow-host` list. + /// Both `None` → unrestricted. Otherwise a policy is in force: only the + /// listed CIDRs and IPs learned from allowed DNS answers may be reached + /// (an empty CIDR list with no hosts denies everything). + pub fn new(allowed_cidrs: Option<&[String]>, allowed_hosts: Option<&[String]>) -> Self { + if allowed_cidrs.is_none() && allowed_hosts.is_none() { + return Self::unrestricted(); + } + let cidrs = allowed_cidrs + .unwrap_or(&[]) + .iter() + .filter_map(|spec| { + let parsed = Cidr::parse(spec); + if parsed.is_none() { + tracing::warn!(cidr = %spec, "ignoring unparseable egress CIDR"); + } + parsed + }) + .collect(); + let allowed_hosts = allowed_hosts.map(|hosts| { + hosts + .iter() + .filter_map(|h| dns::normalize_hostname(h)) + .collect() + }); + Self { + inner: Some(Arc::new(AllowList { + cidrs, + allowed_hosts, + learned: Mutex::new(HashMap::new()), + })), + floor: floor_mode(), + } + } + + /// Convenience for the CIDR-only case. + pub fn from_allowed_cidrs(allowed: Option<&[String]>) -> Self { + Self::new(allowed, None) + } + + /// Whether any policy is in force (false = allow-all). + pub fn is_restricted(&self) -> bool { + self.inner.is_some() + } + + /// Whether the gateway should DNS-filter queries (an allow-host list is set). + pub fn dns_filter_active(&self) -> bool { + self.inner + .as_ref() + .is_some_and(|list| list.allowed_hosts.is_some()) + } + + /// Whether a DNS query for `hostname` should be forwarded upstream. With no + /// allow-host list, all queries pass (exact + subdomain match otherwise). + pub fn hostname_allowed(&self, hostname: &str) -> bool { + match &self.inner { + None => true, + Some(list) => match &list.allowed_hosts { + None => true, + Some(hosts) => dns::hostname_allowed(hostname, hosts), + }, + } + } + + /// Whether an outbound connection to `ip` (v4 or v6) is permitted. + pub fn allows(&self, ip: IpAddr) -> bool { + // Platform hard-floor: deny per the resolved FloorMode (metadata-only + // locally, full floor under fleet mode) regardless of the allow-list. + if is_floored(ip, self.floor) { + return false; + } + match &self.inner { + None => true, + Some(list) => { + if list.cidrs.iter().any(|cidr| cidr.contains(ip)) { + return true; + } + list.learned + .lock() + .map(|learned| { + learned + .get(&ip) + .is_some_and(|expires_at| *expires_at > Instant::now()) + }) + .unwrap_or(false) + } + } + } + + /// Convenience for IPv4 call sites. + pub fn allows_v4(&self, ip: Ipv4Addr) -> bool { + self.allows(IpAddr::V4(ip)) + } + + /// Convenience for IPv6 call sites. + pub fn allows_v6(&self, ip: Ipv6Addr) -> bool { + self.allows(IpAddr::V6(ip)) + } + + /// Learn the A/AAAA records of an allowed DNS answer as temporarily-allowed + /// IPs. TTLs are clamped to [60s, 3600s]; expired entries are pruned. No-op + /// when unrestricted. + pub fn learn_ip_records(&self, records: &[(IpAddr, u32)]) { + let Some(list) = &self.inner else { + return; + }; + let Ok(mut learned) = list.learned.lock() else { + return; + }; + let now = Instant::now(); + learned.retain(|_, expires_at| *expires_at > now); + for (ip, ttl) in records { + let ttl = u64::from(*ttl).clamp(MIN_LEARNED_TTL, MAX_LEARNED_TTL); + let expires_at = now + Duration::from_secs(ttl); + learned + .entry(*ip) + .and_modify(|existing| *existing = (*existing).max(expires_at)) + .or_insert(expires_at); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn floor_override_parsing() { + assert_eq!(parse_floor_override("strict"), Some(FloorMode::Strict)); + assert_eq!(parse_floor_override(" STRICT "), Some(FloorMode::Strict)); + assert_eq!( + parse_floor_override("metadata"), + Some(FloorMode::MetadataOnly) + ); + assert_eq!( + parse_floor_override("metadata-only"), + Some(FloorMode::MetadataOnly) + ); + assert_eq!(parse_floor_override("off"), Some(FloorMode::Off)); + assert_eq!(parse_floor_override("none"), Some(FloorMode::Off)); + // Unrecognized falls through to the inferred default (None). + assert_eq!(parse_floor_override(""), None); + assert_eq!(parse_floor_override("yes"), None); + } + + #[test] + fn unrestricted_allows_everything() { + let policy = EgressPolicy::unrestricted(); + assert!(!policy.is_restricted()); + assert!(policy.allows_v4(Ipv4Addr::new(8, 8, 8, 8))); + assert!(policy.allows_v6("2001:4860:4860::8888".parse().unwrap())); + assert!(policy.hostname_allowed("anything.test")); + assert!(!policy.dns_filter_active()); + } + + #[test] + fn empty_allowlist_denies_all() { + let policy = EgressPolicy::from_allowed_cidrs(Some(&[])); + assert!(policy.is_restricted()); + assert!(!policy.allows_v4(Ipv4Addr::new(1, 1, 1, 1))); + assert!(!policy.allows_v6("2606:4700::1111".parse().unwrap())); + } + + #[test] + fn cidr_membership_v4() { + // Public CIDRs only — private ranges are denied by the hard-floor below. + let policy = EgressPolicy::new(Some(&["8.8.8.0/24".into(), "1.1.1.1".into()]), None); + assert!(policy.allows_v4(Ipv4Addr::new(8, 8, 8, 7))); + assert!(policy.allows_v4(Ipv4Addr::new(1, 1, 1, 1))); + assert!(!policy.allows_v4(Ipv4Addr::new(1, 1, 1, 2))); + assert!(!policy.allows_v4(Ipv4Addr::new(9, 0, 0, 1))); + } + + #[test] + fn unrestricted_local_floors_only_metadata() { + // Local default (no fleet mode): only the cloud-metadata link-local range + // is denied; the host's LAN, loopback, and CGNAT stay reachable — so a + // local VM behaves predictably. + let p = EgressPolicy::unrestricted(); + assert!(!p.allows_v4(Ipv4Addr::new(169, 254, 169, 254))); // metadata: denied + assert!(p.allows_v4(Ipv4Addr::new(10, 0, 0, 4))); // LAN: reachable + assert!(p.allows_v4(Ipv4Addr::new(127, 0, 0, 1))); // loopback: reachable + assert!(p.allows_v4(Ipv4Addr::new(192, 168, 1, 1))); + assert!(p.allows_v4(Ipv4Addr::new(172, 16, 0, 1))); + assert!(p.allows_v4(Ipv4Addr::new(100, 96, 0, 1))); // CGNAT: reachable + assert!(p.allows_v4(Ipv4Addr::new(1, 1, 1, 1))); // public: reachable + } + + #[test] + fn metadata_floor_overrides_allowlist_and_learned_ips() { + // The metadata range can't be re-opened by allow-listing it... + let p = EgressPolicy::new(Some(&["169.254.0.0/16".into()]), None); + assert!(!p.allows_v4(Ipv4Addr::new(169, 254, 169, 254))); + // ...nor via DNS-rebinding: a learned metadata IP stays denied. + let p2 = EgressPolicy::new(None, Some(&["evil.test".into()])); + let meta = IpAddr::V4(Ipv4Addr::new(169, 254, 169, 254)); + p2.learn_ip_records(&[(meta, 300)]); + assert!(!p2.allows(meta)); + // But a LAN IP in the allow-list IS reachable locally. + let p3 = EgressPolicy::new(Some(&["10.0.0.0/8".into()]), None); + assert!(p3.allows_v4(Ipv4Addr::new(10, 0, 0, 4))); + } + + #[test] + fn metadata_floor_blocks_mapped_and_v6_link_local() { + let p = EgressPolicy::unrestricted(); // MetadataOnly default + // mapped metadata + v6 link-local are denied... + assert!(!p.allows_v6("::ffff:169.254.169.254".parse().unwrap())); + assert!(!p.allows_v6("fe80::1".parse().unwrap())); + // ...but v6 ULA (the LAN equivalent) and global unicast are reachable. + assert!(p.allows_v6("fc00::1".parse().unwrap())); + assert!(p.allows_v6("2606:4700::1111".parse().unwrap())); + } + + #[test] + fn cidr_membership_v6() { + let policy = + EgressPolicy::new(Some(&["2606:4700::/32".into(), "2001:db8::1".into()]), None); + assert!(policy.allows_v6("2606:4700::1111".parse().unwrap())); + assert!(policy.allows_v6("2606:4700:ffff::1".parse().unwrap())); + assert!(policy.allows_v6("2001:db8::1".parse().unwrap())); + assert!(!policy.allows_v6("2001:db8::2".parse().unwrap())); + assert!(!policy.allows_v6("2607::1".parse().unwrap())); + // A v6 CIDR never matches a v4 address and vice versa. + assert!(!policy.allows_v4(Ipv4Addr::new(1, 1, 1, 1))); + } + + #[test] + fn allow_host_gates_dns_and_learns_ips() { + let policy = EgressPolicy::new(None, Some(&["example.com".into()])); + assert!(policy.dns_filter_active()); + assert!(policy.hostname_allowed("example.com")); + assert!(policy.hostname_allowed("www.example.com")); + assert!(!policy.hostname_allowed("evil.test")); + + let v4 = IpAddr::V4(Ipv4Addr::new(93, 184, 216, 34)); + let v6: IpAddr = "2606:2800:21f:cb07:6820:80da:af6b:8b2c" + .parse::() + .unwrap() + .into(); + assert!(!policy.allows(v4)); + assert!(!policy.allows(v6)); + policy.learn_ip_records(&[(v4, 300), (v6, 600)]); + assert!(policy.allows(v4)); + assert!(policy.allows(v6)); + } + + #[test] + fn learned_ip_respects_min_ttl() { + // A tiny TTL is clamped up to MIN_LEARNED_TTL, so the entry is live now. + let policy = EgressPolicy::new(None, Some(&["example.com".into()])); + let ip = IpAddr::V4(Ipv4Addr::new(1, 2, 3, 4)); + policy.learn_ip_records(&[(ip, 1)]); + assert!(policy.allows(ip)); + } + + #[test] + fn unparseable_cidr_is_skipped_not_panicked() { + let policy = EgressPolicy::new(Some(&["nonsense".into(), "1.1.1.1".into()]), None); + assert!(policy.allows_v4(Ipv4Addr::new(1, 1, 1, 1))); + assert!(!policy.allows_v4(Ipv4Addr::new(2, 2, 2, 2))); + } + + #[test] + fn v6_prefix_bounds_checked() { + assert!(Cidr::parse("2001:db8::/129").is_none()); + assert!(Cidr::parse("1.2.3.4/33").is_none()); + assert!(Cidr::parse("::/0").is_some()); + assert!(Cidr::parse("0.0.0.0/0").is_some()); + } + + fn v4(a: u8, b: u8, c: u8, d: u8) -> IpAddr { + IpAddr::V4(Ipv4Addr::new(a, b, c, d)) + } + + #[test] + fn floor_off_blocks_nothing() { + // Trusted override: even metadata/loopback/private pass. + for ip in [ + v4(8, 8, 8, 8), + v4(169, 254, 169, 254), + v4(192, 168, 1, 5), + v4(127, 0, 0, 1), + ] { + assert!( + !is_floored(ip, FloorMode::Off), + "{ip} should not be floored when Off" + ); + } + } + + #[test] + fn floor_metadata_only_blocks_just_link_local() { + // Local default: only the cloud-metadata link-local range is denied; the + // host's LAN, loopback and the public internet stay reachable. + assert!(is_floored(v4(169, 254, 169, 254), FloorMode::MetadataOnly)); + assert!(is_floored(v4(169, 254, 0, 1), FloorMode::MetadataOnly)); + for ip in [ + v4(8, 8, 8, 8), + v4(192, 168, 1, 5), + v4(10, 0, 0, 7), + v4(127, 0, 0, 1), + v4(172, 16, 5, 5), + ] { + assert!( + !is_floored(ip, FloorMode::MetadataOnly), + "{ip} should be reachable locally" + ); + } + // IPv4-mapped metadata must not slip past. + assert!(is_floored( + "::ffff:169.254.169.254".parse().unwrap(), + FloorMode::MetadataOnly + )); + } + + #[test] + fn floor_strict_blocks_internal_and_metadata() { + // Fleet/multi-tenant: the full floor. + for ip in [ + v4(169, 254, 169, 254), // metadata + v4(192, 168, 1, 5), // RFC1918 + v4(10, 0, 0, 7), + v4(172, 16, 5, 5), + v4(127, 0, 0, 1), // loopback + v4(100, 64, 0, 1), // CGNAT gateway range + ] { + assert!( + is_floored(ip, FloorMode::Strict), + "{ip} should be floored under Strict" + ); + } + // Public + just-outside-CGNAT stay reachable. + assert!(!is_floored(v4(8, 8, 8, 8), FloorMode::Strict)); + assert!(!is_floored(v4(100, 128, 0, 1), FloorMode::Strict)); + // IPv6 internal ranges + mapped private. + assert!(is_floored("fe80::1".parse().unwrap(), FloorMode::Strict)); + assert!(is_floored("fc00::1".parse().unwrap(), FloorMode::Strict)); + assert!(is_floored( + "::ffff:10.0.0.1".parse().unwrap(), + FloorMode::Strict + )); + assert!(!is_floored( + "2606:4700::1111".parse().unwrap(), + FloorMode::Strict + )); + } +} diff --git a/crates/smolvm-network/src/frame_stream.rs b/crates/smolvm-network/src/frame_stream.rs new file mode 100644 index 0000000..061e402 --- /dev/null +++ b/crates/smolvm-network/src/frame_stream.rs @@ -0,0 +1,321 @@ +//! libkrun unix-stream framing for the virtio-net backend. +//! +//! Context +//! ======= +//! +//! libkrun's `krun_add_net_unixstream()` interface does not hand us raw virtio +//! rings or a tap device. Instead, it exposes a Unix stream file descriptor +//! carrying Ethernet frames in a tiny framing protocol: +//! +//! ```text +//! [4-byte big-endian frame length][raw ethernet frame bytes] +//! ``` +//! +//! Important details: +//! - the payload is a raw Ethernet frame +//! - there is no virtio-net header on this stream +//! - libkrun adds/removes its internal virtio-net header itself +//! - partial reads and partial writes are normal stream-socket behavior and +//! must be handled explicitly +//! +//! So this module is not the TCP/IP stack. It is just the bridge between: +//! - libkrun's Unix stream transport +//! - the in-process frame queues consumed by the host smoltcp runtime +//! +//! Data flow: +//! +//! ```text +//! guest -> libkrun -> UnixStream -> run_reader() -> guest_to_host queue +//! host <- libkrun <- UnixStream <- run_writer() <- host_to_guest queue +//! ``` +//! +//! In the broader runtime, this module sits here: +//! +//! ```text +//! libkrun unixstream +//! <-> FrameStreamBridge +//! <-> NetworkFrameQueues +//! <-> VirtioNetworkDevice / smoltcp poll loop +//! ``` + +use crate::queues::NetworkFrameQueues; +use socket2::Socket; +use std::io::{self, Read, Write}; +use std::net::Shutdown; +use std::sync::Arc; +use std::thread::{self, JoinHandle}; + +const FRAME_HEADER_LEN: usize = 4; +const SOCKET_SENDBUF_BYTES: usize = 16 * 1024 * 1024; +const MAX_FRAME_LEN: usize = 64 * 1024; + +/// Running libkrun unix-stream bridge for one virtio NIC. +/// +/// The bridge owns: +/// - the shared AF_UNIX stream socket (used to trigger shutdown via `shutdown`) +/// - a reader thread for guest->host frames +/// - a writer thread for host->guest frames +/// +/// The transport is an AF_UNIX stream socket, wrapped in [`socket2::Socket`] so +/// the same code serves Unix hosts and Windows (10 1809+ has native AF_UNIX). +/// On Unix the launcher hands us one end of a `socketpair`; on Windows it hands +/// us the socket it `accept`ed from libkrun on a per-VM path. +/// +/// The socket is shared via `Arc` and the reader/writer threads use it through +/// `&Socket` (socket2 implements `Read`/`Write` for `&Socket`), rather than +/// `try_clone`-ing it three ways: duplicating an AF_UNIX socket handle is +/// unreliable on Windows (a clone may not observe the peer's writes), which +/// silently stalled all guest traffic. Concurrent read + write on one socket is +/// safe — the directions are independent. +pub struct FrameStreamBridge { + socket: Arc, + queues: Arc, + reader_handle: Option>, + writer_handle: Option>, +} + +/// Start the libkrun stream reader and writer threads for one virtio NIC. +/// +/// Takes ownership of an already-connected AF_UNIX stream socket to libkrun. +pub fn start_frame_stream_bridge( + stream: Socket, + queues: Arc, +) -> io::Result { + set_socket_send_buffer(&stream); + let socket = Arc::new(stream); + + let reader_socket = socket.clone(); + let reader_handle = thread::Builder::new() + .name("smolvm-net-reader".into()) + .spawn({ + let queues = queues.clone(); + move || run_reader(reader_socket, queues) + })?; + + let writer_socket = socket.clone(); + let writer_queues = queues.clone(); + let writer_handle = thread::Builder::new() + .name("smolvm-net-writer".into()) + .spawn(move || run_writer(writer_socket, writer_queues))?; + + Ok(FrameStreamBridge { + socket, + queues, + reader_handle: Some(reader_handle), + writer_handle: Some(writer_handle), + }) +} + +impl Drop for FrameStreamBridge { + /// Request shutdown and join the reader/writer workers. + /// + /// `shutdown(Shutdown::Both)` is the important part here: it forces any + /// blocking read/write on the shared socket to wake up and fail, which lets + /// the threads notice shutdown and return. + fn drop(&mut self) { + self.queues.begin_shutdown(); + let _ = self.socket.shutdown(Shutdown::Both); + + if let Some(handle) = self.reader_handle.take() { + let _ = handle.join(); + } + if let Some(handle) = self.writer_handle.take() { + let _ = handle.join(); + } + } +} + +fn run_reader(reader: Arc, queues: Arc) { + // Reader thread: + // libkrun -> Unix stream -> guest_to_host queue -> smoltcp poll loop + let mut sock: &Socket = &reader; + loop { + match read_frame(&mut sock) { + Ok(frame) => { + if queues.guest_to_host.push(frame).is_ok() { + queues.guest_wake.wake(); + } else { + tracing::warn!("dropping guest ethernet frame because the host queue is full"); + } + } + Err(err) => { + queues.begin_shutdown(); + tracing::debug!(error = %err, "virtio-net reader thread stopped"); + return; + } + } + } +} + +fn run_writer(writer: Arc, queues: Arc) { + // Writer thread: + // smoltcp / host runtime -> host_to_guest queue -> Unix stream -> libkrun + let mut sock: &Socket = &writer; + loop { + if queues.is_shutting_down() && queues.host_to_guest.is_empty() { + return; + } + match queues.host_wake.wait(None) { + Ok(true) => queues.host_wake.drain(), + Ok(false) => continue, + Err(err) => { + queues.begin_shutdown(); + tracing::debug!(error = %err, "virtio-net writer wake pipe failed"); + return; + } + } + + while let Some(frame) = queues.host_to_guest.pop() { + if let Err(err) = write_frame(&mut sock, &frame) { + queues.begin_shutdown(); + tracing::debug!(error = %err, "virtio-net writer thread stopped"); + return; + } + } + } +} + +/// Read one raw Ethernet frame using libkrun's 4-byte big-endian length prefix. +/// +/// Wire format: +/// +/// ```text +/// 0 3 4 ... +/// +----------------+----------------------+ +/// | frame_len (BE) | ethernet frame bytes | +/// +----------------+----------------------+ +/// ``` +/// +/// `read_exact` is intentional: +/// - Unix stream sockets are byte streams, not message sockets +/// - one `read` may return only part of the header or part of the frame +/// - the bridge must keep reading until the whole logical frame arrives +/// +/// Outcome: +/// - returns the next complete raw Ethernet frame +/// - rejects zero-length or implausibly large frames as protocol errors +pub(crate) fn read_frame(reader: &mut R) -> io::Result> { + let mut header = [0u8; FRAME_HEADER_LEN]; + reader.read_exact(&mut header)?; + let frame_len = u32::from_be_bytes(header) as usize; + + if frame_len == 0 || frame_len > MAX_FRAME_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid ethernet frame length: {frame_len}"), + )); + } + + let mut frame = vec![0u8; frame_len]; + reader.read_exact(&mut frame)?; + Ok(frame) +} + +/// Write one raw Ethernet frame using libkrun's 4-byte big-endian length prefix. +/// +/// This is the inverse of [`read_frame`]: +/// +/// ```text +/// write 4-byte BE length +/// write raw frame bytes +/// flush stream +/// ``` +/// +/// `write_all` is used instead of a single `write` because stream sockets may +/// accept only part of the buffer. The caller should not need to reason about +/// partial-write state; this helper completes the logical frame write or fails. +pub(crate) fn write_frame(writer: &mut W, frame: &[u8]) -> io::Result<()> { + if frame.is_empty() || frame.len() > MAX_FRAME_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("invalid ethernet frame length: {}", frame.len()), + )); + } + + let header = (frame.len() as u32).to_be_bytes(); + write_all(writer, &header)?; + write_all(writer, frame)?; + writer.flush() +} + +fn write_all(writer: &mut W, mut buf: &[u8]) -> io::Result<()> { + // This is the stream-socket equivalent of "keep sending until the whole + // logical message is written". `Write::write` may legally write fewer bytes + // than requested even on success. + while !buf.is_empty() { + let written = writer.write(buf)?; + if written == 0 { + return Err(io::Error::new( + io::ErrorKind::WriteZero, + "short write while sending ethernet frame", + )); + } + buf = &buf[written..]; + } + Ok(()) +} + +/// Increase the socket's send-buffer size (SO_SNDBUF). This stream carries +/// Ethernet frames toward libkrun; a large send buffer absorbs bursts so we can +/// queue several frames before libkrun catches up. The OS may clamp the request, +/// and the option is non-critical, so a failure is logged and ignored. +/// `socket2` issues the platform `setsockopt` so this works on Unix and Windows. +fn set_socket_send_buffer(stream: &Socket) { + if let Err(err) = stream.set_send_buffer_size(SOCKET_SENDBUF_BYTES) { + tracing::warn!( + error = %err, + "failed to increase SO_SNDBUF for virtio-net unixstream" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + struct PartialWriter { + written: Vec, + chunk_size: usize, + } + + impl Write for PartialWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let take = buf.len().min(self.chunk_size); + self.written.extend_from_slice(&buf[..take]); + Ok(take) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + #[test] + fn write_frame_handles_partial_writes() { + let mut writer = PartialWriter { + written: Vec::new(), + chunk_size: 3, + }; + write_frame(&mut writer, &[1, 2, 3, 4, 5, 6]).unwrap(); + assert_eq!(writer.written[..4], [0, 0, 0, 6]); + assert_eq!(writer.written[4..], [1, 2, 3, 4, 5, 6]); + } + + #[test] + fn read_frame_decodes_length_prefix() { + let mut input = std::io::Cursor::new(vec![0, 0, 0, 3, 7, 8, 9]); + assert_eq!(read_frame(&mut input).unwrap(), vec![7, 8, 9]); + } + + #[cfg(unix)] + #[test] + fn unix_stream_round_trip_multiple_frames() { + use std::os::unix::net::UnixStream; + let (mut left, mut right) = UnixStream::pair().unwrap(); + write_frame(&mut left, &[1, 2, 3]).unwrap(); + write_frame(&mut left, &[4, 5]).unwrap(); + + assert_eq!(read_frame(&mut right).unwrap(), vec![1, 2, 3]); + assert_eq!(read_frame(&mut right).unwrap(), vec![4, 5]); + } +} diff --git a/crates/smolvm-network/src/icmp_relay.rs b/crates/smolvm-network/src/icmp_relay.rs new file mode 100644 index 0000000..c7b54c5 --- /dev/null +++ b/crates/smolvm-network/src/icmp_relay.rs @@ -0,0 +1,595 @@ +//! ICMP echo (ping) relay for the virtio-net backend. +//! +//! Context +//! ======= +//! +//! TCP, UDP, and DNS all reach the outside world through the gateway, but a +//! guest `ping` had nowhere to go: nothing on the host answered ICMP, so echo +//! requests were silently dropped (the same gap TSI has — its syscall hijack +//! only covers stream/datagram sockets, never ICMP). This module closes it by +//! relaying guest echo requests to real host ICMP datagram sockets and feeding +//! the real replies back, so `ping` measures actual reachability. +//! +//! Model — a userspace NAT for echo, built entirely on smoltcp sockets so no +//! Ethernet/IP frames are assembled by hand: +//! +//! ```text +//! guest echo request to D +//! -> normal smoltcp ingress; a raw::Socket(ICMP) captures the full IP packet +//! -> poll loop parses (guest, D, ident, seq, data); egress check +//! -> channels it to the relay thread +//! -> relay thread: one connected host ICMP socket per (guest, D, ident); send +//! -> reply: host socket readable -> channel back -> relay_wake +//! -> poll loop builds an echo *reply* IP packet sourced from D and sends it +//! out the raw::Socket; smoltcp frames it and delivers it to the guest +//! ``` +//! +//! Sourcing the reply from `D` (not the gateway) is the whole point: `ping` +//! rejects replies whose source isn't the address it targeted. smoltcp's +//! `icmp::Socket` can't do that (its TX path picks the interface's own source +//! address), so the relay sends a fully-addressed packet out a `raw::Socket` +//! instead — smoltcp still owns the Ethernet header and neighbor resolution. +//! +//! Echo is connectionless, so lifetime is NAT-style idle expiry: the relay +//! drops host sockets idle for [`FLOW_IDLE_TIMEOUT`]. Loss under pressure (full +//! channels / tables) is acceptable ICMP semantics — logged, never blocking. + +use crate::egress::EgressPolicy; +use crate::queues::WakePipe; +use crate::virtio_net_log; +use polling::{Event, Events}; +use smoltcp::phy::ChecksumCapabilities; +use smoltcp::wire::{ + Icmpv4Packet, Icmpv4Repr, Icmpv6Packet, Icmpv6Repr, IpProtocol, Ipv4Packet, Ipv4Repr, + Ipv6Packet, Ipv6Repr, +}; +use socket2::{Domain, Protocol, SockAddr, Socket as HostSocket, Type}; +use std::collections::HashMap; +use std::mem::MaybeUninit; +use std::net::{IpAddr, SocketAddr}; +use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError, TrySendError}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant}; + +/// Max in-flight echo messages per direction before drops (ICMP may drop). +const CHANNEL_CAPACITY: usize = 256; +/// Max concurrent (guest, destination, ident) flows with live host sockets. +const MAX_FLOWS: usize = 256; +/// Largest ICMP message (type byte through payload) we relay; the guest's MTU +/// bounds requests anyway and replies mirror them. +const MAX_ICMP_BYTES: usize = 1500; +/// Default IP hop limit for relayed echo replies. +const REPLY_HOP_LIMIT: u8 = 64; +/// ICMPv4 echo-request / echo-reply type bytes. +const ICMPV4_ECHO_REQUEST: u8 = 8; +const ICMPV4_ECHO_REPLY: u8 = 0; +/// ICMPv6 echo-request / echo-reply type bytes. +const ICMPV6_ECHO_REQUEST: u8 = 128; +const ICMPV6_ECHO_REPLY: u8 = 129; +/// Drop a flow's host socket after this much inactivity. +const FLOW_IDLE_TIMEOUT: Duration = Duration::from_secs(60); +/// Relay thread poll ceiling, so shutdown and expiry are noticed promptly. +const RELAY_POLL_MAX_MS: i32 = 1000; + +/// One relayed echo message, in either direction. +/// +/// On egress it is the guest's decoded echo request; on ingress it is the host +/// reply with the guest's original `ident` restored (the host kernel rewrites +/// the on-wire identifier to its own port, so the relay stamps it back). +pub struct IcmpEcho { + /// Guest address (echo source on egress, reply target on ingress). + pub guest: IpAddr, + /// External destination the guest pinged (and replies come from). + pub destination: IpAddr, + /// ICMP echo identifier the guest chose. + pub ident: u16, + /// ICMP echo sequence number. + pub seq: u16, + /// Echo payload, mirrored unchanged between request and reply. + pub data: Vec, +} + +/// Channel pair connecting the poll loop and the relay thread. +pub struct IcmpRelayChannels { + /// Poll loop -> relay thread. + pub to_relay: SyncSender, + /// Relay thread -> poll loop. + pub from_relay: Receiver, + /// Wakes the relay thread after `to_relay` sends. + pub relay_thread_wake: WakePipe, +} + +/// Start the ICMP relay thread. Returns the poll-loop-side channel endpoints. +/// +/// `reply_wake` is the smoltcp poll loop's existing relay wake pipe — pulsed +/// whenever a reply is queued so the loop wakes to deliver it to the guest. +/// The thread exits when `shutdown` reports true (checked at least once per +/// [`RELAY_POLL_MAX_MS`]). +pub fn start_icmp_relay( + reply_wake: Arc, + shutdown: Arc bool + Send + Sync>, +) -> IcmpRelayChannels { + let (to_relay_tx, to_relay_rx) = mpsc::sync_channel(CHANNEL_CAPACITY); + let (from_relay_tx, from_relay_rx) = mpsc::sync_channel(CHANNEL_CAPACITY); + let relay_thread_wake = WakePipe::new(); + let thread_wake = relay_thread_wake.clone(); + + let _ = thread::Builder::new() + .name("smolvm-icmp-relay".into()) + .spawn(move || { + run_icmp_relay( + to_relay_rx, + from_relay_tx, + thread_wake, + reply_wake, + shutdown, + ); + }); + + IcmpRelayChannels { + to_relay: to_relay_tx, + from_relay: from_relay_rx, + relay_thread_wake, + } +} + +/// Relay-thread state for one (guest, destination, ident) echo flow. +struct IcmpFlow { + socket: HostSocket, + guest: IpAddr, + destination: IpAddr, + ident: u16, + last_active: Instant, +} + +fn run_icmp_relay( + outbound: Receiver, + inbound: SyncSender, + wake: WakePipe, + reply_wake: Arc, + shutdown: Arc bool + Send + Sync>, +) { + let mut flows: HashMap<(IpAddr, IpAddr, u16), IcmpFlow> = HashMap::new(); + let mut recv_buf = [MaybeUninit::::uninit(); MAX_ICMP_BYTES]; + // The host ICMP socket may be denied (Linux `ping_group_range`, no + // privileges); log that once rather than on every dropped ping. + let mut warned_socket_error = false; + + loop { + if shutdown() { + return; + } + + // Outbound: echo requests handed over by the poll loop. + loop { + match outbound.try_recv() { + Ok(echo) => { + let key = (echo.guest, echo.destination, echo.ident); + if !flows.contains_key(&key) { + if flows.len() >= MAX_FLOWS { + virtio_net_log!( + "virtio-net: dropping ICMP echo {} -> {} (flow table full)", + echo.guest, + echo.destination + ); + continue; + } + match create_flow_socket(echo.destination) { + Ok(socket) => { + flows.insert( + key, + IcmpFlow { + socket, + guest: echo.guest, + destination: echo.destination, + ident: echo.ident, + last_active: Instant::now(), + }, + ); + } + Err(err) => { + if !warned_socket_error { + virtio_net_log!( + "virtio-net: cannot open host ICMP socket for {} (ping disabled): {}", + echo.destination, + err + ); + warned_socket_error = true; + } + continue; + } + } + } + let flow = flows.get_mut(&key).expect("flow inserted above"); + flow.last_active = Instant::now(); + // Best-effort send; ICMP loss is allowed. + let request = echo_request_bytes(echo.destination, echo.seq, &echo.data); + let _ = flow.socket.send(&request); + } + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => return, + } + } + + // Inbound: poll all flow sockets for replies. The wake's poller blocks + // on "wake OR any flow socket readable" in a single call; the wake is a + // notify (no key), and each flow socket is registered at key `slot + 1`. + let poller = wake.poller(); + let keys: Vec<(IpAddr, IpAddr, u16)> = flows.keys().copied().collect(); + for (slot, key) in keys.iter().enumerate() { + // SAFETY: the socket is owned by `flows` and is deleted from the + // poller below before the next iteration may drop it. + let _ = unsafe { poller.add(&flows[key].socket, Event::readable(slot + 1)) }; + } + + let mut events = Events::new(); + let _ = poller.wait( + &mut events, + Some(Duration::from_millis(RELAY_POLL_MAX_MS as u64)), + ); + + // A pending notify is consumed by `wait`; nothing else to drain. + let mut ready: Vec = vec![false; keys.len()]; + for event in events.iter() { + if event.key >= 1 && event.key - 1 < ready.len() { + ready[event.key - 1] = true; + } + } + + // Deregister sockets before the next rebuild so a closed flow's socket + // is never left registered on the poller. + for key in &keys { + let _ = poller.delete(&flows[key].socket); + } + + let mut woke_reply = false; + for (slot, key) in keys.iter().enumerate() { + if !ready[slot] { + continue; + } + let Some(flow) = flows.get_mut(key) else { + continue; + }; + // Drain everything ready on this socket. WouldBlock ends the drain; + // errors (an ICMP error surfaced on the connected socket) just mean + // this flow is quiet — idle expiry reaps it. + while let Ok(len) = flow.socket.recv(&mut recv_buf) { + // SAFETY: `recv` reports `len` initialized leading bytes. + let bytes = + unsafe { &*(&recv_buf[..len] as *const [MaybeUninit] as *const [u8]) }; + let Some((seq, data)) = parse_echo_reply(flow.destination, bytes) else { + continue; + }; + flow.last_active = Instant::now(); + let reply = IcmpEcho { + guest: flow.guest, + destination: flow.destination, + ident: flow.ident, + seq, + data, + }; + match inbound.try_send(reply) { + Ok(()) => woke_reply = true, + Err(TrySendError::Full(_)) => { + virtio_net_log!( + "virtio-net: dropping ICMP reply for {} (inbound queue full)", + flow.guest + ); + } + Err(TrySendError::Disconnected(_)) => return, + } + } + } + if woke_reply { + reply_wake.wake(); + } + + // NAT-style idle expiry. + let now = Instant::now(); + flows.retain(|_, flow| now.duration_since(flow.last_active) < FLOW_IDLE_TIMEOUT); + } +} + +/// Open a non-blocking host ICMP datagram socket connected to `destination`. +/// +/// Uses unprivileged ICMP datagram sockets (`SOCK_DGRAM`/`IPPROTO_ICMP[V6]`), +/// the same mechanism `ping` uses on Linux (gated by `net.ipv4.ping_group_range`) +/// and macOS. `connect` pins the peer so the kernel filters stray ICMP from +/// other hosts — each flow is its own NAT pinhole. +fn create_flow_socket(destination: IpAddr) -> std::io::Result { + let (domain, protocol) = match destination { + IpAddr::V4(_) => (Domain::IPV4, Protocol::ICMPV4), + IpAddr::V6(_) => (Domain::IPV6, Protocol::ICMPV6), + }; + let socket = HostSocket::new(domain, Type::DGRAM, Some(protocol))?; + // ICMP has no ports; the kernel ignores the port on connect. + socket.connect(&SockAddr::from(SocketAddr::new(destination, 0)))?; + socket.set_nonblocking(true)?; + Ok(socket) +} + +/// Build the raw ICMP echo-request bytes (type byte through payload) sent to a +/// host ping socket. The kernel overwrites the identifier with the socket's +/// port and fills the checksum, so both are left zero here. +fn echo_request_bytes(destination: IpAddr, seq: u16, data: &[u8]) -> Vec { + let type_byte = if destination.is_ipv6() { + ICMPV6_ECHO_REQUEST + } else { + ICMPV4_ECHO_REQUEST + }; + let mut buf = Vec::with_capacity(8 + data.len()); + buf.push(type_byte); + buf.push(0); // code + buf.extend_from_slice(&[0, 0]); // checksum (kernel fills) + buf.extend_from_slice(&[0, 0]); // identifier (kernel overwrites) + buf.extend_from_slice(&seq.to_be_bytes()); + buf.extend_from_slice(data); + buf +} + +/// Decode a host ping socket's reply (an ICMP message, no IP header) into the +/// echo `(seq, data)`. The identifier is dropped — the relay restores the +/// guest's own. Returns `None` for anything that isn't an echo reply (e.g. an +/// ICMP error the kernel surfaced on the socket). +fn parse_echo_reply(destination: IpAddr, bytes: &[u8]) -> Option<(u16, Vec)> { + if bytes.len() < 8 { + return None; + } + let expected = if destination.is_ipv6() { + ICMPV6_ECHO_REPLY + } else { + ICMPV4_ECHO_REPLY + }; + if bytes[0] != expected { + return None; + } + let seq = u16::from_be_bytes([bytes[6], bytes[7]]); + Some((seq, bytes[8..].to_vec())) +} + +/// Whether the gateway should relay a guest echo to this destination. Echo +/// obeys the same egress policy as TCP/UDP (static CIDRs + DNS-learned IPs). +pub fn should_relay_icmp(destination: IpAddr, egress: &EgressPolicy) -> bool { + egress.allows(destination) +} + +/// Decode a guest IPv4 ICMP echo *request* captured off the raw socket (a full +/// IP packet) into a relayable [`IcmpEcho`]. Non-echo ICMP returns `None`. +pub fn parse_guest_echo_v4(ip_packet: &[u8]) -> Option { + let ipv4 = Ipv4Packet::new_checked(ip_packet).ok()?; + let icmp = Icmpv4Packet::new_checked(ipv4.payload()).ok()?; + match Icmpv4Repr::parse(&icmp, &ChecksumCapabilities::ignored()).ok()? { + Icmpv4Repr::EchoRequest { + ident, + seq_no, + data, + } => Some(IcmpEcho { + guest: IpAddr::V4(ipv4.src_addr()), + destination: IpAddr::V4(ipv4.dst_addr()), + ident, + seq: seq_no, + data: data.to_vec(), + }), + _ => None, + } +} + +/// IPv6 counterpart of [`parse_guest_echo_v4`]. +pub fn parse_guest_echo_v6(ip_packet: &[u8]) -> Option { + let ipv6 = Ipv6Packet::new_checked(ip_packet).ok()?; + let src = ipv6.src_addr(); + let dst = ipv6.dst_addr(); + let icmp = Icmpv6Packet::new_checked(ipv6.payload()).ok()?; + match Icmpv6Repr::parse(&src, &dst, &icmp, &ChecksumCapabilities::ignored()).ok()? { + Icmpv6Repr::EchoRequest { + ident, + seq_no, + data, + } => Some(IcmpEcho { + guest: IpAddr::V6(src), + destination: IpAddr::V6(dst), + ident, + seq: seq_no, + data: data.to_vec(), + }), + _ => None, + } +} + +/// Build the IPv4 echo-*reply* packet (IP header + ICMP) to hand to the raw +/// socket. Sourced from `reply.destination` so the guest's `ping` accepts it. +pub fn build_echo_reply_v4(reply: &IcmpEcho) -> Option> { + let (IpAddr::V4(src), IpAddr::V4(dst)) = (reply.destination, reply.guest) else { + return None; + }; + let icmp = Icmpv4Repr::EchoReply { + ident: reply.ident, + seq_no: reply.seq, + data: &reply.data, + }; + let ip = Ipv4Repr { + src_addr: src, + dst_addr: dst, + next_header: IpProtocol::Icmp, + payload_len: icmp.buffer_len(), + hop_limit: REPLY_HOP_LIMIT, + }; + let mut buf = vec![0u8; ip.buffer_len() + icmp.buffer_len()]; + let checksum = ChecksumCapabilities::default(); + ip.emit(&mut Ipv4Packet::new_unchecked(&mut buf[..]), &checksum); + icmp.emit( + &mut Icmpv4Packet::new_unchecked(&mut buf[ip.buffer_len()..]), + &checksum, + ); + Some(buf) +} + +/// IPv6 counterpart of [`build_echo_reply_v4`]. +pub fn build_echo_reply_v6(reply: &IcmpEcho) -> Option> { + let (IpAddr::V6(src), IpAddr::V6(dst)) = (reply.destination, reply.guest) else { + return None; + }; + let icmp = Icmpv6Repr::EchoReply { + ident: reply.ident, + seq_no: reply.seq, + data: &reply.data, + }; + let ip = Ipv6Repr { + src_addr: src, + dst_addr: dst, + next_header: IpProtocol::Icmpv6, + payload_len: icmp.buffer_len(), + hop_limit: REPLY_HOP_LIMIT, + }; + let mut buf = vec![0u8; ip.buffer_len() + icmp.buffer_len()]; + ip.emit(&mut Ipv6Packet::new_unchecked(&mut buf[..])); + icmp.emit( + &src, + &dst, + &mut Icmpv6Packet::new_unchecked(&mut buf[ip.buffer_len()..]), + &ChecksumCapabilities::default(), + ); + Some(buf) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{Ipv4Addr, Ipv6Addr}; + use std::sync::atomic::{AtomicBool, Ordering}; + + #[test] + fn relay_thread_round_trips_an_echo_via_loopback() { + // Unprivileged ICMP sockets are gated by `net.ipv4.ping_group_range`; + // skip rather than fail where they aren't permitted (e.g. locked-down CI). + if create_flow_socket(IpAddr::V4(Ipv4Addr::LOCALHOST)).is_err() { + eprintln!("skipping: unprivileged ICMP datagram sockets not permitted here"); + return; + } + + let reply_wake = Arc::new(WakePipe::new()); + let stop = Arc::new(AtomicBool::new(false)); + let stop_flag = stop.clone(); + let channels = start_icmp_relay( + reply_wake.clone(), + Arc::new(move || stop_flag.load(Ordering::Relaxed)), + ); + + let guest = IpAddr::V4(Ipv4Addr::new(100, 96, 0, 2)); + let destination = IpAddr::V4(Ipv4Addr::LOCALHOST); + channels + .to_relay + .send(IcmpEcho { + guest, + destination, + ident: 0x4321, + seq: 9, + data: b"relaytest".to_vec(), + }) + .unwrap(); + channels.relay_thread_wake.wake(); + + // The loopback kernel answers the echo; the relay restores our ident. + let reply = channels + .from_relay + .recv_timeout(Duration::from_secs(3)) + .expect("expected an echo reply from loopback"); + assert_eq!(reply.guest, guest); + assert_eq!(reply.destination, destination); + assert_eq!(reply.ident, 0x4321); + assert_eq!(reply.seq, 9); + assert_eq!(reply.data, b"relaytest"); + + stop.store(true, Ordering::Relaxed); + channels.relay_thread_wake.wake(); + } + + #[test] + fn echo_request_carries_seq_and_payload() { + let bytes = echo_request_bytes(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)), 0x0042, b"ping"); + assert_eq!(bytes[0], ICMPV4_ECHO_REQUEST); + assert_eq!(&bytes[6..8], &[0x00, 0x42]); // sequence + assert_eq!(&bytes[8..], b"ping"); + } + + #[test] + fn echo_request_uses_v6_type_for_v6_destination() { + let bytes = echo_request_bytes(IpAddr::V6(Ipv6Addr::LOCALHOST), 1, b""); + assert_eq!(bytes[0], ICMPV6_ECHO_REQUEST); + } + + #[test] + fn parse_reply_extracts_seq_and_data_ignoring_ident() { + // type=0, code=0, cksum, ident=0xdead (host port), seq=0x0042, data. + let reply = [0u8, 0, 0, 0, 0xde, 0xad, 0x00, 0x42, b'h', b'i']; + let (seq, data) = parse_echo_reply(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)), &reply).unwrap(); + assert_eq!(seq, 0x0042); + assert_eq!(data, b"hi"); + } + + #[test] + fn parse_reply_rejects_non_echo_reply() { + // type=3 (destination unreachable) is not an echo reply. + let err = [3u8, 0, 0, 0, 0, 0, 0, 0]; + assert!(parse_echo_reply(IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)), &err).is_none()); + } + + #[test] + fn round_trips_guest_echo_through_reply_v4() { + // Build a guest echo request IP packet, parse it, build the reply, and + // confirm the reply mirrors ident/seq/data and swaps src/dst. + let request = IcmpEcho { + guest: IpAddr::V4(Ipv4Addr::new(100, 96, 0, 2)), + destination: IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)), + ident: 0x1234, + seq: 7, + data: b"abcdefgh".to_vec(), + }; + // Emit a request packet the way a guest would, then parse it back. + let icmp = Icmpv4Repr::EchoRequest { + ident: request.ident, + seq_no: request.seq, + data: &request.data, + }; + let ip = Ipv4Repr { + src_addr: Ipv4Addr::new(100, 96, 0, 2), + dst_addr: Ipv4Addr::new(1, 1, 1, 1), + next_header: IpProtocol::Icmp, + payload_len: icmp.buffer_len(), + hop_limit: 64, + }; + let mut pkt = vec![0u8; ip.buffer_len() + icmp.buffer_len()]; + let cksum = ChecksumCapabilities::default(); + ip.emit(&mut Ipv4Packet::new_unchecked(&mut pkt[..]), &cksum); + icmp.emit( + &mut Icmpv4Packet::new_unchecked(&mut pkt[ip.buffer_len()..]), + &cksum, + ); + + let parsed = parse_guest_echo_v4(&pkt).unwrap(); + assert_eq!(parsed.destination, request.destination); + assert_eq!(parsed.ident, request.ident); + assert_eq!(parsed.seq, request.seq); + assert_eq!(parsed.data, request.data); + + let reply = build_echo_reply_v4(&parsed).unwrap(); + let reply_ip = Ipv4Packet::new_checked(&reply).unwrap(); + // Reply is sourced from the pinged destination, addressed to the guest. + assert_eq!(reply_ip.src_addr(), Ipv4Addr::new(1, 1, 1, 1)); + assert_eq!(reply_ip.dst_addr(), Ipv4Addr::new(100, 96, 0, 2)); + let reply_icmp = Icmpv4Packet::new_checked(reply_ip.payload()).unwrap(); + match Icmpv4Repr::parse(&reply_icmp, &cksum).unwrap() { + Icmpv4Repr::EchoReply { + ident, + seq_no, + data, + } => { + assert_eq!(ident, request.ident); + assert_eq!(seq_no, request.seq); + assert_eq!(data, &request.data[..]); + } + other => panic!("expected echo reply, got {other:?}"), + } + } +} diff --git a/crates/smolvm-network/src/lib.rs b/crates/smolvm-network/src/lib.rs new file mode 100644 index 0000000..4a199d4 --- /dev/null +++ b/crates/smolvm-network/src/lib.rs @@ -0,0 +1,369 @@ +//! Host-side virtio-net runtime. +//! +//! Context +//! ======= +//! +//! This module is the host-side half of the new networking path: +//! +//! ```text +//! guest app +//! -> guest kernel TCP/IP stack +//! -> virtio-net device +//! -> libkrun unix-stream bridge +//! -> smolvm FrameStreamBridge +//! -> shared frame queues +//! -> smoltcp gateway/runtime +//! -> host sockets / DNS forwarding / TCP relay +//! -> external network +//! ``` +//! +//! Main runtime components: +//! +//! ```text +//! VirtioNetworkRuntime +//! ├─ FrameStreamBridge +//! │ ├─ reader thread +//! │ └─ writer thread +//! ├─ TcpPortListeners +//! │ └─ one non-blocking accept loop per `-p HOST:GUEST` +//! ├─ Arc +//! │ ├─ guest_to_host +//! │ ├─ host_to_guest +//! │ ├─ guest_wake +//! │ ├─ host_wake +//! │ └─ relay_wake +//! └─ smolvm-net-poll thread +//! ├─ VirtioNetworkDevice +//! ├─ smoltcp Interface +//! ├─ SocketSet +//! └─ TcpRelayTable +//! ``` +//! +//! Component roles: +//! - `FrameStreamBridge`: translates libkrun's Unix-stream frame protocol into +//! queue operations +//! - `TcpPortListeners`: accepts host TCP connections for published ports +//! and hands them to the poll loop +//! - `NetworkFrameQueues`: handoff boundary between threads +//! - `VirtioNetworkDevice`: adapts those queues to smoltcp's `phy::Device` +//! - poll thread: acts as the guest-visible gateway and protocol dispatcher +//! - `TcpRelayTable`: maps guest TCP flows onto host-side relay threads +//! +//! This runtime is responsible for: +//! - exchanging raw Ethernet frames with libkrun +//! - presenting a gateway endpoint to the guest +//! - handling DNS through a gateway UDP socket and host UDP forwarding +//! - relaying guest TCP connections to host `TcpStream`s +//! - accepting published host TCP ports and forwarding them into guest TCP +//! connections + +pub mod device; +pub mod dns; +pub mod dns_relay; +pub mod egress; +// The libkrun frame bridge speaks over an AF_UNIX stream socket, available on +// both Unix and Windows (10 1809+), so the whole stack is cross-platform. +pub mod frame_stream; +pub mod icmp_relay; +pub mod queues; +pub mod stack; +pub mod tcp_listeners; +pub mod tcp_relay; +pub mod udp_relay; + +pub use egress::EgressPolicy; + +use socket2::Socket; +use std::fmt; +use std::io; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::thread::JoinHandle; +use std::time::SystemTime; + +use frame_stream::{start_frame_stream_bridge, FrameStreamBridge}; +use queues::NetworkFrameQueues; +use queues::DEFAULT_FRAME_QUEUE_CAPACITY; +use stack::{start_network_stack, VirtioPollConfig}; +use tcp_listeners::create_tcp_channel; +use tcp_listeners::TcpPortListeners; + +/// Default upstream DNS resolver used by the gateway runtime. +pub const DEFAULT_DNS_ADDR: IpAddr = IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)); + +/// Host->guest published TCP port mapping serviced by the virtio gateway. +/// +/// This stays crate-local so the launchers can translate CLI/data-layer port +/// mappings into the gateway runtime without pulling the gateway logic back +/// into the main `smolvm` crate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PortMapping { + /// Port bound on the host loopback interface. + pub host: u16, + /// Port exposed inside the guest. + pub guest: u16, +} + +impl PortMapping { + /// Create a new published port mapping. + pub const fn new(host: u16, guest: u16) -> Self { + Self { host, guest } + } +} + +/// Static guest network configuration for the virtio-net MVP. +/// +/// This struct describes the two endpoints of the single virtual Ethernet link: +/// - the guest NIC (`guest_*`) +/// - the host-side gateway implemented by smolvm (`gateway_*`) +/// +/// The link is dual-stack: a /30 IPv4 point-to-point pair and a /64 ULA IPv6 +/// pair (`fd53:4d00::/64` — `53:4d` = "SM", matching the MAC OUI scheme). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GuestNetworkConfig { + /// Guest IPv4 address. + pub guest_ip: Ipv4Addr, + /// Gateway IPv4 address. + pub gateway_ip: Ipv4Addr, + /// Prefix length. + pub prefix_len: u8, + /// Guest IPv6 (ULA) address. + pub guest_ip6: Ipv6Addr, + /// Gateway IPv6 (ULA) address. + pub gateway_ip6: Ipv6Addr, + /// IPv6 prefix length. + pub prefix_len6: u8, + /// Guest MAC address. + pub guest_mac: [u8; 6], + /// Gateway MAC address. + pub gateway_mac: [u8; 6], + /// DNS server address presented to the guest. + /// + /// For virtio-net this is the gateway's own address (`gateway_ip`): the + /// guest sends DNS to the gateway, which forwards upstream to + /// [`Self::upstream_dns`]. + pub dns_server: Ipv4Addr, + /// Upstream resolver the gateway forwards guest DNS queries to. + /// + /// Defaults to [`DEFAULT_DNS_ADDR`]; the launcher overrides it when the + /// caller passes `--dns ` so a VM on a network that blocks the default + /// resolver can still resolve names. + pub upstream_dns: Ipv4Addr, +} + +impl GuestNetworkConfig { + /// Default Phase 1 guest network configuration. + pub const fn default() -> Self { + Self { + guest_ip: Ipv4Addr::new(100, 96, 0, 2), + gateway_ip: Ipv4Addr::new(100, 96, 0, 1), + prefix_len: 30, + guest_ip6: Ipv6Addr::new(0xfd53, 0x4d00, 0, 0, 0, 0, 0, 2), + gateway_ip6: Ipv6Addr::new(0xfd53, 0x4d00, 0, 0, 0, 0, 0, 1), + prefix_len6: 64, + guest_mac: [0x02, 0x53, 0x4d, 0x00, 0x00, 0x02], + gateway_mac: [0x02, 0x53, 0x4d, 0x00, 0x00, 0x01], + dns_server: Ipv4Addr::new(100, 96, 0, 1), + upstream_dns: match DEFAULT_DNS_ADDR { + IpAddr::V4(ip) => ip, + IpAddr::V6(_) => Ipv4Addr::new(1, 1, 1, 1), + }, + } + } +} + +fn format_network_log_line(timestamp: SystemTime, message: &str) -> String { + format!( + "[{}]: {}", + humantime::format_rfc3339_seconds(timestamp), + message + ) +} + +pub(crate) fn emit_network_log_line(message: fmt::Arguments<'_>) { + eprintln!( + "{}", + format_network_log_line(SystemTime::now(), &message.to_string()) + ); +} + +macro_rules! virtio_net_log { + ($($arg:tt)*) => { + $crate::emit_network_log_line(format_args!($($arg)*)) + }; +} + +pub(crate) use virtio_net_log; + +/// Running host-side virtio-net runtime for one guest NIC. +/// +/// Ownership model: +/// - one runtime instance corresponds to one guest virtio NIC +/// - it owns the queue set shared by the worker threads +/// - it owns the libkrun Unix-stream bridge threads +/// - it owns the published-port listener threads +/// - it owns the smoltcp poll thread +/// +/// Dropping the runtime is the shutdown signal. `Drop` marks the shared queues +/// as shutting down, wakes blocked workers, and joins the poll thread. +pub struct VirtioNetworkRuntime { + queues: std::sync::Arc, + _frame_bridge: FrameStreamBridge, + published_ports: Option, + poll_handle: Option>, +} + +/// Start the host-side virtio-net runtime for one guest NIC. +/// +/// Inputs: +/// - `host_fd`: the host-side Unix stream fd that libkrun will use for this +/// guest NIC. The launcher eventually gets this from the libkrun +/// `krun_add_net_unixstream()` setup path. +/// - `guest_network`: the static guest/gateway addressing and MAC plan for this +/// NIC. +/// - `published_ports`: host->guest TCP port mappings that should be serviced +/// directly by the virtio runtime instead of TSI. +/// +/// High-level flow: +/// +/// ```text +/// start_virtio_network() +/// -> create shared frame queues + wake pipes +/// -> start frame reader/writer threads on the Unix stream +/// -> start host TcpListeners for published ports +/// -> start the smoltcp poll thread +/// -> return a handle that owns the whole runtime +/// ``` +/// +/// Expanded startup picture: +/// +/// ```text +/// host_fd from libkrun +/// -> FrameStreamBridge(host_fd) +/// -> reader thread +/// -> writer thread +/// -> TcpPortListeners +/// -> accept host TcpStreams +/// -> send them to the poll loop over a bounded channel +/// -> NetworkFrameQueues +/// -> start_network_stack(...) +/// -> poll thread owns smoltcp Interface + sockets +/// -> VirtioNetworkRuntime returned to launcher +/// ``` +/// +/// Outcome: +/// - guest->host Ethernet frames start flowing into the queues +/// - host->guest Ethernet frames emitted by smoltcp are written back to libkrun +/// - published host TCP connections can be forwarded toward guest listeners +/// - the poll loop starts acting as the guest-visible gateway +/// +/// Cross-platform: the libkrun frame bridge speaks over an AF_UNIX stream +/// socket, available on Unix and Windows (10 1809+). The caller supplies the +/// already-connected host-side socket (Unix: one end of a `socketpair`; +/// Windows: the socket `accept`ed from libkrun on the per-VM path). +pub fn start_virtio_network( + host_stream: Socket, + guest_network: GuestNetworkConfig, + published_ports: &[PortMapping], + egress: EgressPolicy, +) -> io::Result { + virtio_net_log!( + "virtio-net: starting runtime guest_ip={} gateway_ip={} dns_server={}", + guest_network.guest_ip, + guest_network.gateway_ip, + guest_network.dns_server + ); + let queues = NetworkFrameQueues::shared(DEFAULT_FRAME_QUEUE_CAPACITY); + let frame_bridge = start_frame_stream_bridge(host_stream, queues.clone())?; + // tcp_sender sends the accepted TCP connections to the channel + // tcp_receiver receives the accepted TCP connections via the channel, and let it be consumed in poll thread. + let (tcp_sender, tcp_receiver) = create_tcp_channel(); + let tcp_listeners = if published_ports.is_empty() { + None + } else { + Some(TcpPortListeners::start( + published_ports, + tcp_sender, + queues.relay_wake.clone(), + )?) + }; + let poll_handle = start_network_stack( + queues.clone(), + VirtioPollConfig { + gateway_mac: guest_network.gateway_mac, + guest_mac: guest_network.guest_mac, + gateway_ipv4: guest_network.gateway_ip, + guest_ipv4: guest_network.guest_ip, + gateway_ipv6: guest_network.gateway_ip6, + guest_ipv6: guest_network.guest_ip6, + prefix_len6: guest_network.prefix_len6, + upstream_dns: guest_network.upstream_dns, + mtu: 1500, + }, + tcp_listeners.as_ref().map(|_| tcp_receiver), + egress, + )?; + + Ok(VirtioNetworkRuntime { + queues, + _frame_bridge: frame_bridge, + published_ports: tcp_listeners, + poll_handle: Some(poll_handle), + }) +} + +impl VirtioNetworkRuntime { + /// Cumulative guest-outbound (egress) bytes on this NIC since boot, at the + /// ethernet-frame level. The launcher polls this to surface per-machine + /// egress to the node API for billing. TSI VMs have no virtio runtime, so + /// egress is unavailable for them (a documented metering gap). + pub fn egress_bytes(&self) -> u64 { + self.queues.egress_bytes() + } + + /// A cheap, cloneable read handle to this NIC's egress counter, so the + /// launcher can spawn a thread that periodically flushes the value to the + /// VM's runtime dir for the node API to read (the runtime is not `Clone`). + pub fn egress_counter(&self) -> std::sync::Arc { + self.queues.egress_counter() + } + + /// Take ownership of the runtime and block until the libkrun stream closes + /// (the frame reader marks the queues shutting down on EOF), then drop it for + /// a graceful shutdown. Used on Windows, where the runtime is built on the + /// `accept` thread: that thread parks here so the runtime — and its worker + /// threads — live for the whole VM lifetime instead of being dropped at the + /// end of the `accept` callback. + pub fn block_until_shutdown(self) { + while !self.queues.is_shutting_down() { + std::thread::sleep(std::time::Duration::from_millis(200)); + } + // `self` drops here, joining the worker threads. + } +} + +impl Drop for VirtioNetworkRuntime { + /// Shut down the worker threads in a bounded, cooperative way. + /// + /// The queue shutdown flag wakes the frame bridge and smoltcp poll loop so + /// they can exit on their own. We only explicitly join the poll thread + /// here because the frame bridge joins its own threads in its own `Drop`. + fn drop(&mut self) { + self.queues.begin_shutdown(); + self.published_ports = None; + if let Some(handle) = self.poll_handle.take() { + let _ = handle.join(); + } + } +} + +#[cfg(test)] +mod tests { + use super::format_network_log_line; + use std::time::UNIX_EPOCH; + + #[test] + fn formats_timestamped_network_log_prefix() { + let line = format_network_log_line(UNIX_EPOCH, "virtio-net: smoke test"); + assert_eq!(line, "[1970-01-01T00:00:00Z]: virtio-net: smoke test"); + } +} diff --git a/crates/smolvm-network/src/queues.rs b/crates/smolvm-network/src/queues.rs new file mode 100644 index 0000000..346691f --- /dev/null +++ b/crates/smolvm-network/src/queues.rs @@ -0,0 +1,280 @@ +//! Shared queues and wake notifications for the virtio-net backend. +//! +//! Context +//! ======= +//! +//! The host-side virtio runtime has several independently blocked workers: +//! - the Unix-stream reader thread +//! - the Unix-stream writer thread +//! - the smoltcp poll loop +//! - TCP relay threads +//! +//! They need two kinds of coordination: +//! 1. lock-free frame handoff between threads +//! 2. a way to wake a thread that is blocked waiting on socket readiness +//! +//! This module provides both: +//! - `ArrayQueue>` for frame ownership transfer +//! - `WakePipe` as a tiny readiness primitive built on a cross-platform poller +//! (`polling`, which maps to epoll/kqueue/IOCP). Its `notify()` is the +//! cross-thread wakeup, replacing the old self-pipe. +//! +//! Data flow: +//! +//! ```text +//! guest_to_host queue : reader thread -> smoltcp poll loop +//! host_to_guest queue : smoltcp runtime -> writer thread +//! +//! guest_wake: reader thread / shutdown -> smoltcp poll loop +//! host_wake : smoltcp runtime / shutdown -> Unix-stream writer +//! relay_wake: TCP relay threads / shutdown -> smoltcp poll loop +//! ``` +//! +//! Thread interaction view: +//! +//! ```text +//! FrameStream reader thread +//! -> guest_to_host.push(frame) +//! -> guest_wake.wake() +//! +//! smolvm-net-poll thread +//! -> guest_to_host.pop() +//! -> host_to_guest.push(frame) +//! -> host_wake.wake() +//! -> relay_wake.wait()/drain() +//! +//! FrameStream writer thread +//! -> host_wake.wait() +//! -> host_to_guest.pop() +//! +//! TCP relay thread +//! -> to_smoltcp.send(payload) +//! -> relay_wake.wake() +//! ``` + +use crossbeam_queue::ArrayQueue; +use polling::{Events, Poller}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +/// Default queue capacity for guest/host ethernet frames. +pub const DEFAULT_FRAME_QUEUE_CAPACITY: usize = 1024; + +/// Shared queues and wake handles for the host-side virtio-net runtime. +/// +/// One `NetworkFrameQueues` is shared across all helper threads for a single +/// guest NIC. +/// +/// A useful mental model is: +/// +/// ```text +/// queues = ownership transfer for frame bytes +/// wakes = "go look at the queue now" +/// shutdown= sticky flag + wake all blocked waiters +/// ``` +pub struct NetworkFrameQueues { + /// Raw ethernet frames emitted by the guest and waiting for smoltcp. + pub guest_to_host: ArrayQueue>, + /// Raw ethernet frames emitted by smoltcp and waiting for libkrun. + pub host_to_guest: ArrayQueue>, + /// Wake the smoltcp poll loop when a guest frame arrives. + /// + /// `guest_wake` and `relay_wake` deliberately share one underlying poller: + /// the smoltcp loop blocks on that single poller and either wake unblocks + /// it. The loop re-runs its whole pipeline on every wakeup, so it does not + /// need to know which side fired. + pub guest_wake: WakePipe, + /// Wake the libkrun writer thread when a host frame is ready. + pub host_wake: WakePipe, + /// Wake the smoltcp poll loop when a TCP relay thread has new data. + pub relay_wake: WakePipe, + /// Signals that the helper process should shut down. + shutting_down: AtomicBool, + /// Cumulative guest-outbound (egress) bytes for this NIC since boot, at the + /// ethernet-frame level — every guest frame accepted into the stack is + /// counted. Used for per-machine egress billing/telemetry. Held behind an + /// `Arc` so the runtime owner can hand a cheap read handle to a flush thread + /// without exposing the rest of the queue set. + egress_bytes: Arc, +} + +impl NetworkFrameQueues { + /// Create a new shared queue set wrapped in `Arc`. + pub fn shared(capacity: usize) -> Arc { + // The smoltcp poll loop waits on a single poller; both the guest-frame + // wake and the relay wake notify it, so they share one poller instance. + let poll_loop = WakePipe::new(); + let relay_wake = poll_loop.share(); + Arc::new(Self { + guest_to_host: ArrayQueue::new(capacity), + host_to_guest: ArrayQueue::new(capacity), + guest_wake: poll_loop, + host_wake: WakePipe::new(), + relay_wake, + shutting_down: AtomicBool::new(false), + egress_bytes: Arc::new(AtomicU64::new(0)), + }) + } + + /// Add `n` guest-outbound bytes to the egress counter. Relaxed ordering is + /// fine: the counter is a monotonic statistic, not a synchronization point. + pub fn add_egress_bytes(&self, n: u64) { + self.egress_bytes.fetch_add(n, Ordering::Relaxed); + } + + /// Cumulative guest-outbound bytes for this NIC since boot. + pub fn egress_bytes(&self) -> u64 { + self.egress_bytes.load(Ordering::Relaxed) + } + + /// A cheap, cloneable read handle to the egress counter, for a flush thread + /// owned by the launcher (the runtime itself is not `Clone`). + pub fn egress_counter(&self) -> Arc { + self.egress_bytes.clone() + } + + /// Mark the runtime as shutting down and wake all waiters. + /// + /// The wakes are part of shutdown correctness. Without them, a thread + /// blocked waiting on socket readiness could sleep indefinitely even though + /// the shutdown flag was already set. + pub fn begin_shutdown(&self) { + self.shutting_down.store(true, Ordering::SeqCst); + self.guest_wake.wake(); + self.host_wake.wake(); + self.relay_wake.wake(); + } + + /// Whether shutdown has been requested. + pub fn is_shutting_down(&self) -> bool { + self.shutting_down.load(Ordering::SeqCst) + } +} + +/// Cross-thread wake notification built on a cross-platform poller. +/// +/// The pattern is: +/// - one thread blocks waiting on the poller (optionally alongside registered +/// socket sources) +/// - another thread calls [`WakePipe::wake`] to unblock it +/// - the waiter resumes; the wake is auto-cleared by the next `wait` +/// +/// Why a poller rather than a self-pipe: `polling::Poller::notify()` is a +/// portable cross-thread wakeup that works on Windows (where a `pipe(2)` is not +/// pollable by the IOCP/wepoll backend) as well as Unix. The same poller can +/// have socket sources registered on it, which lets the ICMP/UDP relay loops +/// wait on "wake OR any flow socket readable" in a single blocking call. +#[derive(Clone, Debug)] +pub struct WakePipe { + poller: Arc, +} + +impl WakePipe { + /// Create a wake notification with its own poller. + pub fn new() -> Self { + Self { + poller: Arc::new(Poller::new().expect("create poller for wake notification")), + } + } + + /// Create another handle that shares this wake's underlying poller. + /// + /// Two `WakePipe`s built this way notify the same waiter: useful when one + /// loop must wake on either of two logical events (the smoltcp loop wakes on + /// guest frames or relay data). + pub fn share(&self) -> Self { + Self { + poller: self.poller.clone(), + } + } + + /// The underlying poller, so a caller can register socket sources on it and + /// block on "this wake OR a socket" in a single `wait`. + pub fn poller(&self) -> &Arc { + &self.poller + } + + /// Signal the waiting side. + /// + /// Multiple wakes coalesce: until the waiter next blocks, repeated notifies + /// collapse into a single "there is pending wake state". + pub fn wake(&self) { + // A failed notify only means the waiter will rely on its poll timeout; + // it is never fatal. + let _ = self.poller.notify(); + } + + /// Drain pending wake state. + /// + /// With a poller-backed waker the notification is consumed by `wait` + /// itself, so this is a no-op kept for API symmetry with the old self-pipe. + pub fn drain(&self) {} + + /// Wait until woken or the timeout elapses. + /// + /// Returns `Ok(true)` if the wait ended because of a wake (a `notify` or a + /// registered source becoming ready), `Ok(false)` if the timeout elapsed. + /// + /// A `notify`-driven wakeup reports no events (the poller consumes the + /// notification internally), so a pure wake is detected as either a + /// non-empty event set or an early return: the wait unblocked before the + /// requested deadline. + pub fn wait(&self, timeout: Option) -> std::io::Result { + let mut events = Events::new(); + let start = std::time::Instant::now(); + let count = self.poller.wait(&mut events, timeout)?; + if count > 0 { + return Ok(true); + } + match timeout { + // Without a deadline, the only way `wait` returns is a wake. + None => Ok(true), + // With a deadline, an early return means a `notify` woke us; a + // return at/after the deadline is a genuine timeout. + Some(timeout) => Ok(start.elapsed() < timeout), + } + } +} + +impl Default for WakePipe { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn wake_pipe_round_trip() { + let pipe = WakePipe::new(); + pipe.wake(); + assert!(pipe.wait(Some(Duration::from_millis(10))).unwrap()); + pipe.drain(); + assert!(!pipe.wait(Some(Duration::from_millis(1))).unwrap()); + } + + #[test] + fn shared_wake_notifies_same_waiter() { + let a = WakePipe::new(); + let b = a.share(); + // Waking the shared handle unblocks a waiter on the original. + b.wake(); + assert!(a.wait(Some(Duration::from_millis(10))).unwrap()); + a.drain(); + assert!(!a.wait(Some(Duration::from_millis(1))).unwrap()); + } + + #[test] + fn queues_are_fifo() { + let queues = NetworkFrameQueues::shared(4); + queues.guest_to_host.push(vec![1, 2, 3]).unwrap(); + queues.guest_to_host.push(vec![4, 5, 6]).unwrap(); + + assert_eq!(queues.guest_to_host.pop(), Some(vec![1, 2, 3])); + assert_eq!(queues.guest_to_host.pop(), Some(vec![4, 5, 6])); + assert_eq!(queues.guest_to_host.pop(), None); + } +} diff --git a/crates/smolvm-network/src/stack.rs b/crates/smolvm-network/src/stack.rs new file mode 100644 index 0000000..1452de0 --- /dev/null +++ b/crates/smolvm-network/src/stack.rs @@ -0,0 +1,1253 @@ +//! Host-side smoltcp runtime for the virtio-net backend. +//! +//! Context +//! ======= +//! +//! This file is the in-process "gateway" that sits behind the guest's virtio +//! NIC. It does not configure the guest interface; that already happened inside +//! `smolvm-agent`. Instead, this module: +//! - receives raw Ethernet frames coming out of libkrun +//! - feeds them into smoltcp as if smolvm were the guest's next-hop gateway +//! - forwards guest DNS queries to a host UDP socket +//! - relays guest TCP streams to host `TcpStream`s +//! +//! Conceptually, it plays the role of a tiny virtual router/NAT-side gateway: +//! +//! ```text +//! guest eth0 +//! -> Ethernet frame +//! -> Frame queues +//! -> smoltcp Interface (gateway MAC/IP) +//! -> protocol-specific handling: +//! - TCP -> host relay threads +//! - DNS -> host UDP socket +//! - UDP -> per-flow host socket relay (udp_relay) +//! -> outbound network +//! ``` +//! +//! Poll-loop-centric view: +//! +//! ```text +//! guest_to_host queue +//! -> VirtioNetworkDevice::stage_next_frame() +//! -> classify_guest_frame() +//! -> smoltcp ingress +//! -> protocol-specific side effects +//! - TCP SYN -> create relay/socket state +//! - DNS UDP -> gateway UDP socket +//! - other UDP-> destination-keyed relay socket +//! -> smoltcp egress +//! -> host_to_guest queue +//! -> FrameStream writer +//! ``` +//! +//! Runtime control flow: +//! +//! ```text +//! new guest frame -> guest_wake -> poll loop +//! host relay has data -> relay_wake -> poll loop +//! published host connect -> relay_wake -> poll loop +//! smoltcp emitted frames -> host_wake -> frame writer +//! ``` + +use crate::device::VirtioNetworkDevice; +use crate::dns; +use crate::dns_relay::{self, DnsQuery, DnsResponse, DnsTransport}; +use crate::egress::EgressPolicy; +use crate::icmp_relay; +use crate::queues::NetworkFrameQueues; +use crate::tcp_listeners::AcceptedTcpConnection; +use crate::tcp_relay::{spawn_tcp_relay, TcpRelayTable}; +use crate::udp_relay; +use crate::virtio_net_log; +use smoltcp::iface::{ + Config, Interface, PollIngressSingleResult, PollResult, SocketHandle, SocketSet, +}; +use smoltcp::socket::raw::{ + PacketBuffer as RawPacketBuffer, PacketMetadata as RawPacketMetadata, Socket as RawSocket, +}; +use smoltcp::socket::tcp; +use smoltcp::socket::udp::{PacketBuffer, PacketMetadata, Socket as UdpSocket, UdpMetadata}; +use smoltcp::time::Instant; +use smoltcp::wire::{ + EthernetAddress, EthernetFrame, EthernetProtocol, HardwareAddress, IpAddress, IpCidr, + IpListenEndpoint, IpProtocol, IpVersion, Ipv4Packet, Ipv6Packet, TcpPacket, UdpPacket, +}; +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::sync::atomic::Ordering; +use std::sync::mpsc::{Receiver, SyncSender, TryRecvError, TrySendError}; +use std::sync::Arc; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant as StdInstant}; + +const DNS_SOCKET_PORT: u16 = 53; +const DNS_PACKET_SLOTS: usize = 8; +const DNS_BUFFER_BYTES: usize = 2048; +// DNS-over-TCP to the gateway. Real resolvers (and resolv.conf clients) fall +// back to TCP for truncated answers and EDNS, so the gateway filter must serve +// TCP/53 in addition to UDP/53. A small pool of listening sockets handles +// concurrent queries; DNS/TCP is rare and short-lived (one query per +// connection), so a few suffice. +const DNS_TCP_LISTENERS: usize = 4; +const DNS_TCP_RX_BYTES: usize = 4096; +const DNS_TCP_TX_BYTES: usize = 8192; +// A length-prefixed DNS message is bounded by a 16-bit length, but the gateway +// only needs to handle ordinary queries/responses; cap to keep buffers small +// and reject a guest that sends a bogus oversized prefix. +const DNS_TCP_MAX_MSG: usize = 4096; +const DEFAULT_IDLE_TIMEOUT_MS: i32 = 100; +/// Packet slots per ICMP raw socket buffer (per direction). +const ICMP_PACKET_SLOTS: usize = 16; +/// Payload bytes per ICMP raw socket buffer (per direction). +const ICMP_BUFFER_BYTES: usize = 32 * 1024; + +/// Resolved network parameters for one guest NIC. +/// +/// These are the host-side parameters for the virtual link. Note that the +/// smoltcp interface is configured with the *gateway* MAC/IP, because the host +/// runtime is acting as the guest-visible gateway endpoint. +#[derive(Debug, Clone, Copy)] +pub struct VirtioPollConfig { + /// Host-side gateway MAC visible to the guest. + pub gateway_mac: [u8; 6], + /// Guest MAC address. + pub guest_mac: [u8; 6], + /// Gateway IPv4 address. + pub gateway_ipv4: Ipv4Addr, + /// Guest IPv4 address. + pub guest_ipv4: Ipv4Addr, + /// Gateway IPv6 (ULA) address. + pub gateway_ipv6: Ipv6Addr, + /// Guest IPv6 (ULA) address. + pub guest_ipv6: Ipv6Addr, + /// IPv6 prefix length for the virtual link. + pub prefix_len6: u8, + /// Upstream resolver the gateway forwards guest DNS queries to. + pub upstream_dns: Ipv4Addr, + /// IP-level MTU. + pub mtu: usize, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FrameAction { + TcpSyn { + source: SocketAddr, + destination: SocketAddr, + }, + DnsQuery, + /// Non-DNS guest UDP, relayed to a host socket (see `udp_relay`). + UdpFlow { + destination: SocketAddr, + }, + Passthrough, +} + +/// Start the dedicated smoltcp poll thread for the virtio-net backend. +/// +/// This creates one long-lived poll loop thread per guest NIC. That thread owns +/// the smoltcp `Interface`, its socket set, and the TCP relay table. +/// +/// Ownership boundary: +/// - this thread owns all smoltcp state +/// - relay threads never touch smoltcp sockets directly +/// - frame bridge threads never parse protocols beyond raw Ethernet framing +pub fn start_network_stack( + queues: Arc, + config: VirtioPollConfig, + tcp_receiver: Option>, + egress: EgressPolicy, +) -> std::io::Result> { + virtio_net_log!( + "virtio-net: spawning poll thread guest_ip={} gateway_ip={} mtu={}", + config.guest_ipv4, + config.gateway_ipv4, + config.mtu + ); + thread::Builder::new() + .name("smolvm-net-poll".into()) + .spawn(move || run_network_stack(queues, config, tcp_receiver, egress)) +} + +fn run_network_stack( + queues: Arc, + config: VirtioPollConfig, + mut tcp_receiver: Option>, + egress: EgressPolicy, +) { + // Poll loop overview: + // + // 1. Drain staged guest Ethernet frames from the guest_to_host queue. + // 2. Pre-classify them so we can create relay/socket state before smoltcp + // consumes the frame. + // 3. Poll smoltcp ingress/egress. + // 4. Forward DNS and relay TCP payloads. + // 5. Sleep in poll(2) on wake pipes until guest frames, relay activity, or + // timers require more work. + // + // A useful mental model is: + // + // queue input -> classify -> smoltcp -> protocol handling -> queue output + virtio_net_log!( + "virtio-net: poll loop started guest_ip={} gateway_ip={}", + config.guest_ipv4, + config.gateway_ipv4 + ); + let clock = StdInstant::now(); + let mut device = VirtioNetworkDevice::new(queues.clone(), config.mtu); + let mut interface = create_interface(&mut device, &config); + let mut sockets = SocketSet::new(vec![]); + let dns_socket_handle = add_dns_socket(&mut sockets); + let dns_tcp_handles = add_dns_tcp_sockets(&mut sockets); + let mut dns_tcp_conns: Vec = (0..dns_tcp_handles.len()) + .map(|_| DnsTcpConn::default()) + .collect(); + let (icmp4_handle, icmp6_handle) = add_icmp_raw_sockets(&mut sockets); + // Gateway addresses answer their own pings locally; everything else is + // relayed out to real host ICMP sockets. + let gateway_addrs = [ + IpAddr::V4(config.gateway_ipv4), + IpAddr::V6(config.gateway_ipv6), + IpAddr::V6(link_local_from_mac(config.gateway_mac)), + ]; + let relay_wake = Arc::new(queues.relay_wake.clone()); + let mut relays = TcpRelayTable::new(None, egress.clone()); + let mut udp_sockets = udp_relay::UdpSocketTable::new(); + let udp_channels = { + let shutdown_queues = queues.clone(); + udp_relay::start_udp_relay( + relay_wake.clone(), + Arc::new(move || shutdown_queues.is_shutting_down()), + ) + }; + let icmp_channels = { + let shutdown_queues = queues.clone(); + icmp_relay::start_icmp_relay( + relay_wake.clone(), + Arc::new(move || shutdown_queues.is_shutting_down()), + ) + }; + // DNS resolution is offloaded to its own thread so the blocking upstream + // exchange never stalls this poll loop (which owns every one of the VM's + // sockets). Egress policy stays here; only the raw forward is offloaded. + let dns_channels = { + let shutdown_queues = queues.clone(); + dns_relay::start_dns_relay( + relay_wake.clone(), + Arc::new(move || shutdown_queues.is_shutting_down()), + ) + }; + let mut dns_gateway = DnsGateway::new(); + + // The smoltcp loop is driven by poller wakeups rather than busy spinning. + // guest_wake -> new guest frame or shutdown + // relay_wake -> host TCP relay thread produced data or shutdown + // + // Both wakes share a single underlying poller (see `NetworkFrameQueues`), so + // the loop blocks on that one poller and either side unblocks it. The loop + // re-runs its whole pipeline on every wakeup, so it does not need to know + // which wake fired. + let poller = queues.guest_wake.poller().clone(); + let mut events = polling::Events::new(); + + loop { + if queues.is_shutting_down() { + return; + } + let now = smoltcp_now(clock); + + while let Some(frame) = device.stage_next_frame() { + // We inspect the frame before giving it to smoltcp because certain + // flows need side effects first: + // - TCP SYN: pre-create a matching smoltcp socket + relay entry + // - DNS UDP: allow through for gateway-side forwarding + // - other UDP: pre-create the destination-keyed relay socket + match classify_guest_frame(frame, &gateway_addrs) { + FrameAction::TcpSyn { + source, + destination, + } => { + virtio_net_log!( + "virtio-net: guest TCP SYN source={} destination={}", + source, + destination + ); + if !relays.has_socket_for(&source, &destination) { + relays.create_tcp_socket(source, destination, &mut sockets); + } + if matches!( + interface.poll_ingress_single(now, &mut device, &mut sockets), + PollIngressSingleResult::None + ) { + device.drop_staged_frame(); + } + } + FrameAction::DnsQuery | FrameAction::Passthrough => { + if matches!( + interface.poll_ingress_single(now, &mut device, &mut sockets), + PollIngressSingleResult::None + ) { + device.drop_staged_frame(); + } + } + FrameAction::UdpFlow { destination } => { + // Same egress policy as TCP; a denied destination's datagram + // is silently dropped (a guest sees a normal UDP black hole). + if udp_relay::should_relay_udp(destination, &egress) + && udp_sockets.ensure_socket(destination, &mut sockets) + { + if matches!( + interface.poll_ingress_single(now, &mut device, &mut sockets), + PollIngressSingleResult::None + ) { + device.drop_staged_frame(); + } + } else { + device.drop_staged_frame(); + } + } + } + } + + relay_accepted_tcp_connection( + &mut tcp_receiver, + &mut relays, + &mut interface, + &mut sockets, + config.gateway_ipv4, + config.guest_ipv4, + ); + + // First egress pass: let smoltcp emit any packets caused by the most + // recent ingress work before we service higher-level relays. + flush_interface_egress(&mut interface, &mut device, &mut sockets, now); + interface.poll_maintenance(now); + wake_guest_if_needed(&queues, &device); + + // Move payloads between established smoltcp TCP sockets and host relay + // threads, and service the DNS gateway socket. + relays.relay_data(&mut sockets); + // Enqueue guest DNS queries to the offload thread (or answer blocked + // ones locally), then deliver any answers it has produced back to the + // guest. Neither step blocks on the upstream resolver. + let mut woke_dns = false; + woke_dns |= dispatch_dns_udp( + dns_socket_handle, + &mut sockets, + &egress, + config.upstream_dns, + &mut dns_gateway, + &dns_channels.to_relay, + ); + woke_dns |= process_dns_tcp( + &dns_tcp_handles, + &mut dns_tcp_conns, + &mut sockets, + &egress, + config.upstream_dns, + &mut dns_gateway, + &dns_channels.to_relay, + ); + if woke_dns { + dns_channels.relay_thread_wake.wake(); + } + deliver_dns_responses( + dns_socket_handle, + &dns_tcp_handles, + &mut dns_tcp_conns, + &mut sockets, + &egress, + &mut dns_gateway, + &dns_channels.from_relay, + ); + + // General UDP: forward staged guest datagrams to the relay thread, + // deliver any replies it produced, and expire idle destination sockets. + if udp_sockets.drain_to_relay(&mut sockets, &udp_channels.to_relay) { + udp_channels.relay_thread_wake.wake(); + } + udp_sockets.deliver_replies(&mut sockets, &udp_channels.from_relay); + udp_sockets.expire_idle(&mut sockets); + + // ICMP echo: the raw sockets captured any guest echo requests during + // ingress above. Forward external pings to the relay (answering gateway + // pings locally), then send back any replies it produced. + let mut woke_icmp = false; + woke_icmp |= drain_icmp_echo( + &mut sockets, + icmp4_handle, + false, + &egress, + &gateway_addrs, + &icmp_channels.to_relay, + ); + woke_icmp |= drain_icmp_echo( + &mut sockets, + icmp6_handle, + true, + &egress, + &gateway_addrs, + &icmp_channels.to_relay, + ); + if woke_icmp { + icmp_channels.relay_thread_wake.wake(); + } + deliver_icmp_replies( + &mut sockets, + icmp4_handle, + icmp6_handle, + &icmp_channels.from_relay, + ); + + // Once the guest-side TCP handshake is established inside smoltcp, we + // can spawn the corresponding host relay thread. + for connection in relays.take_new_connections(&mut sockets) { + spawn_tcp_relay( + connection.destination, + connection.relay_target, + connection.from_smoltcp, + connection.to_smoltcp, + relay_wake.clone(), + connection.exit_state, + ); + } + + relays.cleanup_closed(&mut sockets); + + // Second egress pass: DNS responses or relay data may have queued more + // packets for the guest. + flush_interface_egress(&mut interface, &mut device, &mut sockets, now); + wake_guest_if_needed(&queues, &device); + + let timeout = interface + .poll_delay(now, &sockets) + .map(|duration| Duration::from_millis(duration.total_millis().min(u32::MAX as u64))); + let timeout = match timeout { + Some(timeout) => Some(timeout), + None => Some(Duration::from_millis(DEFAULT_IDLE_TIMEOUT_MS as u64)), + }; + + // Block until either wake notifies the shared poller or the timeout + // elapses. The wakes are notify-only, so no events are reported; the + // loop re-runs unconditionally on the next iteration. + events.clear(); + let _ = poller.wait(&mut events, timeout); + } +} + +fn create_interface(device: &mut VirtioNetworkDevice, config: &VirtioPollConfig) -> Interface { + // This interface models the host-side gateway endpoint, not the guest NIC. + // + // Equivalent conceptual state: + // MAC: config.gateway_mac + // IP : config.gateway_ipv4/30 + // config.gateway_ipv6/64 (ULA) + fe80 link-local + // + // The guest IP exists as a peer on the same virtual link; it is not an + // address owned by this interface. + let mut interface = Interface::new( + Config::new(HardwareAddress::Ethernet(EthernetAddress( + config.gateway_mac, + ))), + device, + Instant::ZERO, + ); + interface.update_ip_addrs(|addresses| { + addresses + .push(IpCidr::new(IpAddress::Ipv4(config.gateway_ipv4), 30)) + .expect("failed to add gateway IPv4 address"); + addresses + .push(IpCidr::new( + IpAddress::Ipv6(config.gateway_ipv6), + config.prefix_len6, + )) + .expect("failed to add gateway IPv6 address"); + // RFC-clean NDP wants a link-local peer on the segment; derive the + // standard EUI-64 link-local from the gateway MAC so the guest kernel + // can talk NDP to fe80::… as well as to the ULA. + addresses + .push(IpCidr::new( + IpAddress::Ipv6(link_local_from_mac(config.gateway_mac)), + 64, + )) + .expect("failed to add gateway IPv6 link-local address"); + }); + // The interface acts as the gateway and may need to answer packets for + // destinations other than its directly assigned IP, so the route table and + // "any IP" mode are opened up accordingly. + interface + .routes_mut() + .add_default_ipv4_route(config.gateway_ipv4) + .expect("failed to add default IPv4 route"); + interface + .routes_mut() + .add_default_ipv6_route(config.gateway_ipv6) + .expect("failed to add default IPv6 route"); + interface.set_any_ip(true); + interface +} + +/// Derive the EUI-64 IPv6 link-local address for a MAC (RFC 4291 appendix A): +/// flip the universal/local bit, insert `ff:fe` in the middle. +fn link_local_from_mac(mac: [u8; 6]) -> Ipv6Addr { + Ipv6Addr::new( + 0xfe80, + 0, + 0, + 0, + u16::from_be_bytes([mac[0] ^ 0x02, mac[1]]), + u16::from_be_bytes([mac[2], 0xff]), + u16::from_be_bytes([0xfe, mac[3]]), + u16::from_be_bytes([mac[4], mac[5]]), + ) +} + +/// add_dns_socket is adding an UDP socket inside smoltcp, so that the guest DNS packet will +/// hit this socket first. It is then proxied to the resolver. Note that this will not cause +/// a host side :53 collesion, because the smoltcp Interface, SocketSet is per VM, and the +/// gateway:53 is for that set of Interface and SocketSet, it is not bind to a host-kernel UDP socket. +/// +/// The bind is wildcard (port-only) on purpose: combined with `set_any_ip`, every +/// guest UDP datagram to port 53 — whatever its destination address or family +/// (the v4 gateway, the v6 gateway, or an external resolver IP) — lands on this +/// socket and is answered from that same destination address. That transparently +/// intercepts hardcoded external resolvers too, matching TSI's DNS handling. +fn add_dns_socket(sockets: &mut SocketSet<'_>) -> SocketHandle { + let rx_meta = vec![PacketMetadata::EMPTY; DNS_PACKET_SLOTS]; + let tx_meta = vec![PacketMetadata::EMPTY; DNS_PACKET_SLOTS]; + let rx_buffer = PacketBuffer::new(rx_meta, vec![0u8; DNS_BUFFER_BYTES]); + let tx_buffer = PacketBuffer::new(tx_meta, vec![0u8; DNS_BUFFER_BYTES]); + let mut socket = UdpSocket::new(rx_buffer, tx_buffer); + socket + .bind(smoltcp::wire::IpListenEndpoint { + addr: None, + port: DNS_SOCKET_PORT, + }) + .expect("failed to bind gateway DNS socket"); + sockets.add(socket) +} + +/// Add the two raw IP sockets that capture guest ICMP echo traffic. +/// +/// A `raw::Socket` receives a copy of every matching IP packet *before* the +/// interface's "is this addressed to me?" check, so these capture the guest's +/// echo requests even though their destination is some external host. The same +/// sockets carry the relayed echo *replies* back out, fully addressed (source = +/// the pinged host), letting smoltcp own the Ethernet framing and ARP/NDP. +fn add_icmp_raw_sockets(sockets: &mut SocketSet<'_>) -> (SocketHandle, SocketHandle) { + fn raw_socket(version: IpVersion, protocol: IpProtocol) -> RawSocket<'static> { + let rx = RawPacketBuffer::new( + vec![RawPacketMetadata::EMPTY; ICMP_PACKET_SLOTS], + vec![0u8; ICMP_BUFFER_BYTES], + ); + let tx = RawPacketBuffer::new( + vec![RawPacketMetadata::EMPTY; ICMP_PACKET_SLOTS], + vec![0u8; ICMP_BUFFER_BYTES], + ); + RawSocket::new(Some(version), Some(protocol), rx, tx) + } + + let v4 = sockets.add(raw_socket(IpVersion::Ipv4, IpProtocol::Icmp)); + let v6 = sockets.add(raw_socket(IpVersion::Ipv6, IpProtocol::Icmpv6)); + (v4, v6) +} + +/// Drain guest echo requests captured on one ICMP raw socket. Gateway-destined +/// pings are answered locally (the gateway *is* the source), external ones are +/// forwarded to the relay thread subject to egress policy, and denied +/// destinations are dropped. Returns true if anything was sent to the relay. +fn drain_icmp_echo( + sockets: &mut SocketSet<'_>, + handle: SocketHandle, + is_ipv6: bool, + egress: &EgressPolicy, + gateway_addrs: &[IpAddr], + to_relay: &SyncSender, +) -> bool { + // Phase 1: drain received requests into owned values so the socket can be + // re-borrowed below to emit local gateway replies. + let mut echoes = Vec::new(); + { + let socket = sockets.get_mut::(handle); + while socket.can_recv() { + let Ok(packet) = socket.recv() else { + break; + }; + let parsed = if is_ipv6 { + icmp_relay::parse_guest_echo_v6(packet) + } else { + icmp_relay::parse_guest_echo_v4(packet) + }; + if let Some(echo) = parsed { + echoes.push(echo); + } + } + } + + // Phase 2: route each echo. + let mut woke = false; + let mut local_replies = Vec::new(); + for echo in echoes { + if gateway_addrs.contains(&echo.destination) { + local_replies.push(echo); + } else if icmp_relay::should_relay_icmp(echo.destination, egress) { + match to_relay.try_send(echo) { + Ok(()) => woke = true, + Err(TrySendError::Full(_)) => { + virtio_net_log!("virtio-net: dropping guest ICMP echo (relay queue full)"); + } + Err(TrySendError::Disconnected(_)) => return woke, + } + } + // else: egress policy denies the destination — silent black hole. + } + + // Phase 3: answer gateway pings straight back out the raw socket. + if !local_replies.is_empty() { + let socket = sockets.get_mut::(handle); + for reply in local_replies { + let frame = if is_ipv6 { + icmp_relay::build_echo_reply_v6(&reply) + } else { + icmp_relay::build_echo_reply_v4(&reply) + }; + if let Some(frame) = frame { + let _ = socket.send_slice(&frame); + } + } + } + woke +} + +/// Deliver echo replies produced by the relay thread, sending each as a +/// fully-addressed IP packet (source = the pinged host) out the matching raw +/// socket so smoltcp frames it back to the guest. +fn deliver_icmp_replies( + sockets: &mut SocketSet<'_>, + icmp4_handle: SocketHandle, + icmp6_handle: SocketHandle, + from_relay: &Receiver, +) { + while let Ok(reply) = from_relay.try_recv() { + let (handle, frame) = match reply.guest { + IpAddr::V4(_) => (icmp4_handle, icmp_relay::build_echo_reply_v4(&reply)), + IpAddr::V6(_) => (icmp6_handle, icmp_relay::build_echo_reply_v6(&reply)), + }; + let Some(frame) = frame else { + continue; + }; + let socket = sockets.get_mut::(handle); + if socket.send_slice(&frame).is_err() { + virtio_net_log!( + "virtio-net: dropping ICMP reply to {} (raw socket buffer full)", + reply.guest + ); + } + } +} + +/// Receive the accepted TCP connection from the tcp_channel, and then relay it to +/// the TcpRelayTable where the TCP network packets will be relayed to the guest. +fn relay_accepted_tcp_connection( + tcp_receiver: &mut Option>, + relays: &mut TcpRelayTable, + interface: &mut Interface, + sockets: &mut SocketSet<'_>, + gateway_ipv4: Ipv4Addr, + guest_ipv4: Ipv4Addr, +) { + // Published-port model: + // + // host client -> accepted host TcpStream + // -> create guest-facing smoltcp socket from gateway_ip:ephemeral + // -> guest sees a normal inbound TCP connection to guest_port + // -> once Established, the relay thread bridges payloads + // + // The guest does not see the original host peer address here. This path is + // effectively a small userspace TCP proxy/NAT at the gateway boundary. + let mut disconnected = false; + + if let Some(receiver) = tcp_receiver.as_mut() { + loop { + match receiver.try_recv() { + Ok(connection) => { + let guest_destination = + SocketAddr::new(std::net::IpAddr::V4(guest_ipv4), connection.guest_port); + virtio_net_log!( + "virtio-net: accepted published TCP connection peer={} host_port={} guest_destination={}", + connection.peer_addr, + connection.host_port, + guest_destination + ); + if !relays.create_published_socket( + interface, + gateway_ipv4, + guest_destination, + connection.stream, + sockets, + ) { + tracing::warn!( + host_port = connection.host_port, + guest_port = connection.guest_port, + peer_addr = %connection.peer_addr, + "dropping published TCP connection because the guest relay path could not be created" + ); + } + } + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => { + disconnected = true; + break; + } + } + } + } + + if disconnected { + *tcp_receiver = None; + } +} + +/// Bound on outstanding forwarded DNS queries (UDP context) awaiting an answer +/// from the offload thread. Every enqueued query eventually yields a response +/// (answer or timeout) that clears its entry, so this is only a safety ceiling +/// against a wedged relay; DNS volume is far below it in practice. +const MAX_PENDING_DNS: usize = 512; + +/// Poll-loop-side DNS state: the id generator and the reply context for each +/// in-flight *UDP* query. TCP context lives on the owning [`DnsTcpConn`] +/// (`awaiting`), since a DNS/TCP query is pinned to one listener socket. +struct DnsGateway { + next_id: u64, + pending_udp: HashMap, +} + +/// The guest-reply context we must remember while a UDP query is off-thread. +struct PendingDnsUdp { + endpoint: smoltcp::wire::IpEndpoint, + local_address: Option, + /// Whether to learn A/AAAA answer records into the egress allow-list. + learn: bool, +} + +impl DnsGateway { + fn new() -> Self { + Self { + next_id: 0, + pending_udp: HashMap::new(), + } + } + + fn next_id(&mut self) -> u64 { + let id = self.next_id; + self.next_id = self.next_id.wrapping_add(1); + id + } +} + +/// The decision for a single guest DNS query, made on the poll thread using the +/// egress allow-host policy (no host I/O). Mirrors the previous inline filter. +enum DnsDecision { + /// Answer the guest immediately with these raw DNS message bytes + /// (blocked -> NXDOMAIN, unparseable -> SERVFAIL). + Immediate(Vec), + /// Forward the query upstream via the offload thread; `learn` records the + /// answer's A/AAAA IPs into the egress allow-list when it returns. + Forward { learn: bool }, +} + +/// Classify a query under the allow-host policy. When the DNS filter is +/// inactive everything is forwarded (no learning); otherwise only allow-listed +/// names are forwarded and learned, others get NXDOMAIN and unparseable ones +/// SERVFAIL. Identical policy to the old `filtered_dns_response`. +fn classify_dns_query(query: &[u8], egress: &EgressPolicy) -> DnsDecision { + if !egress.dns_filter_active() { + return DnsDecision::Forward { learn: false }; + } + match dns::question_name(query) { + Some(name) if egress.hostname_allowed(&name) => DnsDecision::Forward { learn: true }, + Some(name) => { + virtio_net_log!( + "virtio-net: blocking DNS query by allow-host policy name={}", + name + ); + DnsDecision::Immediate(dns::error_response(query, dns::DNS_RCODE_NXDOMAIN)) + } + None => DnsDecision::Immediate(dns::error_response(query, dns::DNS_RCODE_SERVFAIL)), + } +} + +/// Drain guest UDP/53 queries out of the gateway socket. Blocked queries are +/// answered inline (no host I/O); allowed queries are handed to the DNS offload +/// thread and their reply context remembered. Returns true if anything was +/// enqueued (the caller then wakes the relay thread). Never blocks upstream. +fn dispatch_dns_udp( + dns_socket_handle: SocketHandle, + sockets: &mut SocketSet<'_>, + egress: &EgressPolicy, + upstream_dns: Ipv4Addr, + gateway: &mut DnsGateway, + to_relay: &SyncSender, +) -> bool { + let mut queued = false; + let socket = sockets.get_mut::(dns_socket_handle); + while socket.can_recv() { + // Copy out the query and the Copy reply-context fields, releasing the + // recv borrow before we may re-borrow the socket to answer inline. + let (query, endpoint, local_address) = match socket.recv() { + Ok((q, m)) => (q.to_vec(), m.endpoint, m.local_address), + Err(_) => break, + }; + match classify_dns_query(&query, egress) { + DnsDecision::Immediate(response) => { + let response_meta = UdpMetadata { + endpoint, + local_address, + meta: Default::default(), + }; + let _ = socket.send_slice(&response, response_meta); + } + DnsDecision::Forward { learn } => { + if gateway.pending_udp.len() >= MAX_PENDING_DNS { + virtio_net_log!("virtio-net: dropping guest DNS query (pending table full)"); + continue; + } + let id = gateway.next_id(); + gateway.pending_udp.insert( + id, + PendingDnsUdp { + endpoint, + local_address, + learn, + }, + ); + match to_relay.try_send(DnsQuery { + id, + transport: DnsTransport::Udp, + upstream: upstream_dns, + query, + }) { + Ok(()) => queued = true, + Err(TrySendError::Full(_)) => { + gateway.pending_udp.remove(&id); + virtio_net_log!("virtio-net: dropping guest DNS query (relay queue full)"); + } + Err(TrySendError::Disconnected(_)) => { + gateway.pending_udp.remove(&id); + return queued; + } + } + } + } + } + queued +} + +/// Deliver answers produced by the DNS offload thread back into the guest-facing +/// smoltcp sockets. UDP answers are matched by id to their remembered reply +/// context; TCP answers are matched to the listener whose connection is awaiting +/// that id. A `None` answer (upstream error/timeout) is dropped for UDP (the +/// guest sees a normal DNS timeout) and closes the TCP connection empty-handed. +fn deliver_dns_responses( + dns_socket_handle: SocketHandle, + dns_tcp_handles: &[SocketHandle], + dns_tcp_conns: &mut [DnsTcpConn], + sockets: &mut SocketSet<'_>, + egress: &EgressPolicy, + gateway: &mut DnsGateway, + from_relay: &Receiver, +) { + while let Ok(response) = from_relay.try_recv() { + if let Some(pending) = gateway.pending_udp.remove(&response.id) { + if let Some(answer) = response.answer { + if pending.learn { + egress.learn_ip_records(&dns::answer_ip_records(&answer)); + } + let socket = sockets.get_mut::(dns_socket_handle); + let response_meta = UdpMetadata { + endpoint: pending.endpoint, + local_address: pending.local_address, + meta: Default::default(), + }; + let _ = socket.send_slice(&answer, response_meta); + } + continue; + } + + // Otherwise it is a DNS/TCP answer: find the awaiting listener. + for (handle, conn) in dns_tcp_handles.iter().zip(dns_tcp_conns.iter_mut()) { + match conn.awaiting { + Some(pending) if pending.id == response.id => { + conn.awaiting = None; + if let Some(answer) = response.answer { + if pending.learn { + egress.learn_ip_records(&dns::answer_ip_records(&answer)); + } + frame_dns_tcp_response(conn, &answer); + } + conn.done = true; + let socket = sockets.get_mut::(*handle); + drain_dns_tcp_tx(socket, conn); + break; + } + _ => {} + } + } + } +} + +/// Append a framed (2-byte length prefix + message) DNS response to a +/// connection's pending TX buffer. +fn frame_dns_tcp_response(conn: &mut DnsTcpConn, response: &[u8]) { + if let Ok(resp_len) = u16::try_from(response.len()) { + conn.tx.extend_from_slice(&resp_len.to_be_bytes()); + conn.tx.extend_from_slice(response); + } +} + +/// Create the pool of smoltcp TCP listening sockets bound to the gateway's +/// port 53. Each accepts one DNS-over-TCP connection at a time and is re-armed +/// by [`process_dns_tcp`] after the connection closes. +fn add_dns_tcp_sockets(sockets: &mut SocketSet<'_>) -> Vec { + (0..DNS_TCP_LISTENERS) + .map(|_| { + let rx_buffer = tcp::SocketBuffer::new(vec![0u8; DNS_TCP_RX_BYTES]); + let tx_buffer = tcp::SocketBuffer::new(vec![0u8; DNS_TCP_TX_BYTES]); + let mut socket = tcp::Socket::new(rx_buffer, tx_buffer); + socket + .listen(IpListenEndpoint { + addr: None, + port: DNS_SOCKET_PORT, + }) + .expect("failed to listen on gateway DNS TCP socket"); + sockets.add(socket) + }) + .collect() +} + +/// Per-listener state for an in-flight DNS-over-TCP connection: the +/// accumulating length-prefixed query, and the framed response we still owe the +/// guest. +#[derive(Default)] +struct DnsTcpConn { + /// Guest -> gateway bytes received so far (2-byte length prefix + query). + rx: Vec, + /// Framed response (2-byte length prefix + message) to send to the guest. + tx: Vec, + /// Bytes of `tx` already written to the socket. + tx_sent: usize, + /// The query has been answered (or rejected); drain `tx` then close. + done: bool, + /// Set once the query has been handed to the DNS offload thread and we are + /// waiting for its answer (carries the correlation id + learn flag). While + /// set, the connection is parked — `deliver_dns_responses` completes it. + awaiting: Option, +} + +/// Correlation state for a DNS/TCP query parked on the offload thread. +#[derive(Clone, Copy)] +struct DnsTcpPending { + id: u64, + learn: bool, +} + +/// Service the DNS-over-TCP listening sockets: accept a connection, read the +/// length-prefixed query, apply the allow-host filter, forward allowed queries +/// upstream over TCP, write the length-prefixed response, and close. Closed +/// sockets are re-armed to listen again. +fn process_dns_tcp( + handles: &[SocketHandle], + conns: &mut [DnsTcpConn], + sockets: &mut SocketSet<'_>, + egress: &EgressPolicy, + upstream_dns: Ipv4Addr, + gateway: &mut DnsGateway, + to_relay: &SyncSender, +) -> bool { + let mut queued = false; + for (handle, conn) in handles.iter().zip(conns.iter_mut()) { + let socket = sockets.get_mut::(*handle); + + // A closed/finished socket: reset state and re-arm to accept the next + // connection. `listen` only succeeds from the CLOSED state; if the + // socket is still draining (e.g. TIME-WAIT) the error is ignored and the + // next poll retries. + if !socket.is_open() { + if !conn.rx.is_empty() || !conn.tx.is_empty() || conn.done || conn.awaiting.is_some() { + *conn = DnsTcpConn::default(); + } + let _ = socket.listen(IpListenEndpoint { + addr: None, + port: DNS_SOCKET_PORT, + }); + continue; + } + + // Parked awaiting the offload thread's answer: nothing to do here until + // `deliver_dns_responses` fills `tx` / sets `done`. + if conn.awaiting.is_some() { + continue; + } + + // Already answered: flush whatever response remains, then close. + if conn.done { + drain_dns_tcp_tx(socket, conn); + continue; + } + + // Accumulate the length-prefixed query. + while socket.can_recv() { + let appended = socket.recv(|data| (data.len(), data.to_vec())); + match appended { + Ok(bytes) if !bytes.is_empty() => conn.rx.extend_from_slice(&bytes), + _ => break, + } + } + + // Reject a guest that floods without ever completing a message. + if conn.rx.len() > DNS_TCP_MAX_MSG + 2 { + conn.done = true; + socket.close(); + continue; + } + + if conn.rx.len() >= 2 { + let msg_len = u16::from_be_bytes([conn.rx[0], conn.rx[1]]) as usize; + if msg_len == 0 || msg_len > DNS_TCP_MAX_MSG { + conn.done = true; + socket.close(); + continue; + } + if conn.rx.len() >= 2 + msg_len { + let query = conn.rx[2..2 + msg_len].to_vec(); + virtio_net_log!( + "virtio-net: DNS/TCP query query_len={} upstream_dns={}", + query.len(), + upstream_dns + ); + match classify_dns_query(&query, egress) { + DnsDecision::Immediate(response) => { + frame_dns_tcp_response(conn, &response); + conn.done = true; + drain_dns_tcp_tx(socket, conn); + } + DnsDecision::Forward { learn } => { + // Hand the query to the offload thread and park the + // connection; the poll loop stays free. + let id = gateway.next_id(); + match to_relay.try_send(DnsQuery { + id, + transport: DnsTransport::Tcp, + upstream: upstream_dns, + query, + }) { + Ok(()) => { + conn.awaiting = Some(DnsTcpPending { id, learn }); + queued = true; + } + Err(_) => { + // Relay unavailable/full: close empty-handed + // (guest sees EOF, then retries — DNS semantics). + conn.done = true; + socket.close(); + } + } + } + } + } + } + } + queued +} + +/// Write as much of the pending framed response as the socket will accept; once +/// fully sent, close the connection (the guest reads the answer then sees EOF). +fn drain_dns_tcp_tx(socket: &mut tcp::Socket<'_>, conn: &mut DnsTcpConn) { + while conn.tx_sent < conn.tx.len() && socket.can_send() { + match socket.send_slice(&conn.tx[conn.tx_sent..]) { + Ok(n) if n > 0 => conn.tx_sent += n, + _ => break, + } + } + if conn.tx_sent >= conn.tx.len() { + socket.close(); + } +} + +fn flush_interface_egress( + interface: &mut Interface, + device: &mut VirtioNetworkDevice, + sockets: &mut SocketSet<'_>, + now: Instant, +) { + // smoltcp may have multiple pending egress packets after a single ingress + // event or timeout. Keep polling until the interface reports there is no + // more immediate work. + loop { + let result = interface.poll_egress(now, device, sockets); + if matches!(result, PollResult::None) { + break; + } + } +} + +fn wake_guest_if_needed(queues: &NetworkFrameQueues, device: &VirtioNetworkDevice) { + // The device records only that "some frame was emitted". We convert that + // sticky bit into one wake for the writer thread and let the writer drain + // the entire host_to_guest queue. + if device.frames_emitted.swap(false, Ordering::Relaxed) { + queues.host_wake.wake(); + } +} + +fn smoltcp_now(clock: StdInstant) -> Instant { + let elapsed = clock.elapsed(); + Instant::from_millis(elapsed.as_millis() as i64) +} + +fn classify_guest_frame(frame: &[u8], gateway_addrs: &[IpAddr]) -> FrameAction { + let ethernet = match EthernetFrame::new_checked(frame) { + Ok(frame) => frame, + Err(_) => return FrameAction::Passthrough, + }; + + // Extract (src, dst, transport protocol, transport payload) from either IP + // family. Anything that isn't plain IPv4/IPv6 — ARP, and IPv6 packets with + // extension headers (which guest TCP/UDP traffic doesn't use) — passes + // through to smoltcp untouched; that also covers ICMPv6/NDP. + let (src_ip, dst_ip, protocol, transport): (IpAddr, IpAddr, _, _) = match ethernet.ethertype() { + EthernetProtocol::Ipv4 => { + let ipv4 = match Ipv4Packet::new_checked(ethernet.payload()) { + Ok(packet) => packet, + Err(_) => return FrameAction::Passthrough, + }; + ( + IpAddr::V4(ipv4.src_addr()), + IpAddr::V4(ipv4.dst_addr()), + ipv4.next_header(), + ipv4.payload(), + ) + } + EthernetProtocol::Ipv6 => { + let ipv6 = match Ipv6Packet::new_checked(ethernet.payload()) { + Ok(packet) => packet, + Err(_) => return FrameAction::Passthrough, + }; + ( + IpAddr::V6(ipv6.src_addr()), + IpAddr::V6(ipv6.dst_addr()), + ipv6.next_header(), + ipv6.payload(), + ) + } + _ => return FrameAction::Passthrough, + }; + + match protocol { + smoltcp::wire::IpProtocol::Tcp => { + let tcp = match TcpPacket::new_checked(transport) { + Ok(packet) => packet, + Err(_) => return FrameAction::Passthrough, + }; + + if tcp.syn() && !tcp.ack() { + // DNS-over-TCP to the gateway itself is intercepted by the local + // listening sockets (process_dns_tcp), not relayed. TCP/53 to an + // external resolver (an allow-listed IP) is left to the egress + // relay so the policy still applies. + if tcp.dst_port() == DNS_SOCKET_PORT && gateway_addrs.contains(&dst_ip) { + FrameAction::Passthrough + } else { + FrameAction::TcpSyn { + source: SocketAddr::new(src_ip, tcp.src_port()), + destination: SocketAddr::new(dst_ip, tcp.dst_port()), + } + } + } else { + FrameAction::Passthrough + } + } + smoltcp::wire::IpProtocol::Udp => { + let udp = match UdpPacket::new_checked(transport) { + Ok(packet) => packet, + Err(_) => return FrameAction::Passthrough, + }; + + if udp.dst_port() == DNS_SOCKET_PORT { + FrameAction::DnsQuery + } else { + FrameAction::UdpFlow { + destination: SocketAddr::new(dst_ip, udp.dst_port()), + } + } + } + _ => FrameAction::Passthrough, + } +} + +/// Fuzz-only entrypoint for `classify_guest_frame`. +/// +/// A malicious guest sends arbitrary ethernet frames over virtio-net, and the +/// host parses every one here — so this MUST NOT panic on any input. Gated +/// behind the `fuzzing` feature so it never ships in a normal build. +#[cfg(feature = "fuzzing")] +pub fn fuzz_classify_guest_frame(frame: &[u8]) { + let _ = classify_guest_frame(frame, &[]); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Minimal Ethernet(IPv4(TCP SYN)) frame with no payload. `new_checked` + /// validates lengths/header fields (not checksums), so dummy checksums are + /// fine for exercising `classify_guest_frame`. + fn tcp_syn_frame(dst_ip: [u8; 4], dst_port: u16) -> Vec { + let mut f = Vec::new(); + // Ethernet: dst MAC, src MAC, ethertype IPv4. + f.extend_from_slice(&[0xff; 6]); + f.extend_from_slice(&[0x02, 0, 0, 0, 0, 1]); + f.extend_from_slice(&[0x08, 0x00]); + // IPv4: v4/IHL5, DSCP, total_len=40, id, flags/frag, ttl, proto=TCP, csum, src, dst. + f.extend_from_slice(&[0x45, 0x00, 0x00, 0x28, 0, 0, 0, 0, 0x40, 0x06, 0, 0]); + f.extend_from_slice(&[10, 0, 0, 2]); // src ip + f.extend_from_slice(&dst_ip); + // TCP: src/dst port, seq, ack, data-offset(5)/flags(SYN), window, csum, urg. + f.extend_from_slice(&54321u16.to_be_bytes()); + f.extend_from_slice(&dst_port.to_be_bytes()); + f.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0]); // seq + ack + f.extend_from_slice(&[0x50, 0x02, 0xff, 0xff, 0, 0, 0, 0]); // offset/SYN/window/csum/urg + f + } + + #[test] + fn dns_tcp_to_gateway_is_intercepted_not_relayed() { + let gw = IpAddr::V4(Ipv4Addr::new(100, 96, 0, 1)); + // TCP/53 to the gateway -> handled by the local DNS listeners (Passthrough). + assert_eq!( + classify_guest_frame(&tcp_syn_frame([100, 96, 0, 1], 53), &[gw]), + FrameAction::Passthrough + ); + } + + #[test] + fn dns_tcp_to_external_resolver_still_relayed() { + let gw = IpAddr::V4(Ipv4Addr::new(100, 96, 0, 1)); + // TCP/53 to an external (allow-listed) resolver must go through the egress + // relay, NOT be swallowed by the gateway DNS listeners. + assert!(matches!( + classify_guest_frame(&tcp_syn_frame([1, 1, 1, 1], 53), &[gw]), + FrameAction::TcpSyn { .. } + )); + } + + #[test] + fn non_dns_tcp_to_gateway_still_relayed() { + let gw = IpAddr::V4(Ipv4Addr::new(100, 96, 0, 1)); + // Only port 53 is intercepted; other gateway ports relay as usual. + assert!(matches!( + classify_guest_frame(&tcp_syn_frame([100, 96, 0, 1], 443), &[gw]), + FrameAction::TcpSyn { .. } + )); + } +} diff --git a/crates/smolvm-network/src/tcp_listeners.rs b/crates/smolvm-network/src/tcp_listeners.rs new file mode 100644 index 0000000..19079fc --- /dev/null +++ b/crates/smolvm-network/src/tcp_listeners.rs @@ -0,0 +1,225 @@ +//! Host-side TCP listeners for published virtio-net ports. +//! +//! Context +//! ======= +//! +//! This module is the host-facing half of `-p HOST:GUEST` for the virtio-net +//! backend. +//! +//! The outbound virtio path already handles guest-initiated TCP: +//! +//! ```text +//! guest TCP connect -> smoltcp socket -> host TcpStream -> remote server +//! ``` +//! +//! Published ports invert the initiator: +//! +//! ```text +//! host client -> host TcpListener -> accepted TcpStream +//! -> smoltcp creates gateway-side TCP connection to guest_ip:GUEST +//! -> relay thread bridges the accepted host socket to the guest flow +//! ``` +//! +//! High-level flow: +//! +//! ```text +//! host client connects to 127.0.0.1:HOST (or [::1]:HOST) +//! -> TcpPortListeners accepts TcpStream +//! -> AcceptedTcpConnection sent over a bounded channel +//! -> relay_wake wakes the smoltcp poll loop +//! -> poll loop creates a guest-facing TCP socket to guest_ip:GUEST +//! -> once Established, tcp_relay uses the accepted host TcpStream directly +//! ``` + +use crate::queues::WakePipe; +use crate::PortMapping; +use std::io; +use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, SyncSender, TrySendError}; +use std::sync::Arc; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +const ACCEPT_POLL_INTERVAL: Duration = Duration::from_millis(25); +/// Maximum number of accepted published sockets queued for the poll loop. +pub const DEFAULT_PUBLISH_QUEUE_CAPACITY: usize = 64; + +/// Accepted host TCP connection waiting for the smoltcp poll loop. +pub struct AcceptedTcpConnection { + /// Connected host-side socket returned by `accept(2)`. + pub stream: TcpStream, + /// Host port that accepted the connection. + pub host_port: u16, + /// Guest port the connection should be forwarded to. + pub guest_port: u16, + /// Remote peer that connected to the published port. + pub peer_addr: SocketAddr, +} + +/// Running published-port listener set for one guest NIC. +pub struct TcpPortListeners { + shutdown: Arc, + handles: Vec>, +} + +impl TcpPortListeners { + /// Start one non-blocking listener thread per published port. + pub fn start( + port_mappings: &[PortMapping], + tcp_sender: SyncSender, + publish_wake: WakePipe, + ) -> io::Result { + let shutdown = Arc::new(AtomicBool::new(false)); + let mut handles = Vec::with_capacity(port_mappings.len()); + + // Published ports bind loopback by default. `SMOLVM_PUBLISH_ADDR` + // widens that for fleet nodes whose ingress proxy connects from + // another host (the control plane): `0.0.0.0` (or any address) makes + // the cross-host hop possible — pair it with a firewall on the port + // range, since whatever can reach the address can reach the port. + let publish_addr: Ipv4Addr = std::env::var("SMOLVM_PUBLISH_ADDR") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(Ipv4Addr::LOCALHOST); + let publish_v6 = if publish_addr == Ipv4Addr::UNSPECIFIED { + Ipv6Addr::UNSPECIFIED + } else { + Ipv6Addr::LOCALHOST + }; + + for mapping in port_mappings { + // The IPv4 listener is required; the IPv6 one is best-effort so + // hosts without IPv6 still publish normally. + let listener = match TcpListener::bind((publish_addr, mapping.host)) { + Ok(listener) => listener, + Err(err) => { + shutdown_all(&shutdown, &mut handles); + return Err(err); + } + }; + let listener_v6 = match TcpListener::bind((publish_v6, mapping.host)) { + Ok(listener) => Some(listener), + Err(err) => { + tracing::debug!( + host_port = mapping.host, + error = %err, + "skipping IPv6 listener for published port" + ); + None + } + }; + + for listener in std::iter::once(listener).chain(listener_v6) { + if let Err(err) = listener.set_nonblocking(true) { + shutdown_all(&shutdown, &mut handles); + return Err(err); + } + + let tcp_sender = tcp_sender.clone(); + let publish_wake = publish_wake.clone(); + let shutdown_flag = shutdown.clone(); + let host_port = mapping.host; + let guest_port = mapping.guest; + + let handle = thread::Builder::new() + .name(format!("smolvm-tcp-{host_port}")) + .spawn(move || { + run_tcp_port_listener( + listener, + host_port, + guest_port, + tcp_sender, + publish_wake, + shutdown_flag, + ) + }) + .map_err(|err| { + shutdown_all(&shutdown, &mut handles); + io::Error::other(format!( + "failed to spawn published-port listener thread for {host_port}: {err}" + )) + })?; + handles.push(handle); + } + } + + Ok(Self { shutdown, handles }) + } +} + +impl Drop for TcpPortListeners { + fn drop(&mut self) { + shutdown_all(&self.shutdown, &mut self.handles); + } +} + +fn shutdown_all(shutdown: &Arc, handles: &mut Vec>) { + shutdown.store(true, Ordering::SeqCst); + for handle in handles.drain(..) { + let _ = handle.join(); + } +} + +fn run_tcp_port_listener( + listener: TcpListener, + host_port: u16, + guest_port: u16, + tcp_sender: SyncSender, + publish_wake: WakePipe, + shutdown: Arc, +) { + loop { + if shutdown.load(Ordering::SeqCst) { + return; + } + + match listener.accept() { + Ok((stream, peer_addr)) => { + let accepted = AcceptedTcpConnection { + stream, + host_port, + guest_port, + peer_addr, + }; + + match tcp_sender.try_send(accepted) { + Ok(()) => publish_wake.wake(), + Err(TrySendError::Full(accepted)) => { + tracing::warn!( + host_port = accepted.host_port, + guest_port = accepted.guest_port, + peer_addr = %accepted.peer_addr, + "dropping published TCP connection because the accept queue is full" + ); + } + Err(TrySendError::Disconnected(_)) => return, + } + } + Err(err) if err.kind() == io::ErrorKind::WouldBlock => { + thread::sleep(ACCEPT_POLL_INTERVAL); + } + Err(err) => { + tracing::warn!( + host_port, + guest_port, + error = %err, + "published port listener accept failed" + ); + thread::sleep(ACCEPT_POLL_INTERVAL); + } + } + } +} + +/// Create the bounded channel used to hand accepted host sockets to the poll loop. +/// Each host port in the provided PortMapping has a listener. When the listener +/// accepts a TCP connection, it "sends" the TcpStream to the poll thread by putting +/// the AcceptedTcpConnection into this channel. The receiver consumes it in the +/// poll thread. +pub fn create_tcp_channel() -> ( + SyncSender, + mpsc::Receiver, +) { + mpsc::sync_channel(DEFAULT_PUBLISH_QUEUE_CAPACITY) +} diff --git a/crates/smolvm-network/src/tcp_relay.rs b/crates/smolvm-network/src/tcp_relay.rs new file mode 100644 index 0000000..4f55790 --- /dev/null +++ b/crates/smolvm-network/src/tcp_relay.rs @@ -0,0 +1,779 @@ +//! TCP relay support for the virtio-net backend. +//! +//! Context +//! ======= +//! +//! In the Phase 1 virtio-net design, guest TCP does not flow directly from the +//! guest to the outside network through the host kernel. Instead, the host-side +//! smoltcp runtime terminates the guest-visible TCP connection in userspace and +//! relays payloads to a normal host `TcpStream`. +//! +//! Conceptually: +//! +//! ```text +//! guest app +//! -> guest kernel TCP +//! -> Ethernet frame +//! -> smoltcp TCP socket (inside smolvm) +//! -> channel +//! -> host TcpStream +//! -> remote server +//! ``` +//! +//! That means: +//! - the host runtime can observe every guest TCP byte stream on this NIC +//! - smoltcp owns the guest-facing TCP state machine +//! - the relay thread owns the host-facing TCP socket +//! - channels bridge payloads between them + +use crate::egress::EgressPolicy; +use crate::queues::WakePipe; +use crate::virtio_net_log; +use smoltcp::iface::{Interface, SocketHandle, SocketSet}; +use smoltcp::socket::tcp; +use smoltcp::wire::IpListenEndpoint; +use std::collections::{HashMap, HashSet}; +use std::io::{self, Read, Write}; +use std::net::{Ipv4Addr, Shutdown, SocketAddr, TcpStream}; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError, TrySendError}; +use std::sync::Arc; +use std::thread; +use std::time::Duration; + +const TCP_RX_BUFFER_BYTES: usize = 64 * 1024; +const TCP_TX_BUFFER_BYTES: usize = 64 * 1024; +const MAX_CONNECTIONS: usize = 256; +const CHANNEL_CAPACITY: usize = 32; +const RELAY_BUFFER_BYTES: usize = 16 * 1024; +const CLOSE_RETRY_LIMIT: u16 = 64; +const PROXY_IDLE_SLEEP: Duration = Duration::from_millis(10); +const PUBLISHED_PORT_START: u16 = 49_152; +const PUBLISHED_PORT_END: u16 = 65_535; + +/// Track all active guest TCP connections bridged through host sockets. +/// +/// One entry corresponds to one `(guest source, destination)` tuple. The table +/// lives in the smoltcp poll thread and owns all guest-facing socket handles. +pub struct TcpRelayTable { + connections: HashMap, + connection_keys: HashSet<(SocketAddr, SocketAddr)>, + used_published_ports: HashSet, + next_published_port: u16, + max_connections: usize, + /// Outbound allow-list applied before opening a host connection for a + /// guest-initiated flow. Inbound published-port connections bypass it. + egress: EgressPolicy, +} + +/// Newly established guest connection ready for a host relay thread. +/// +/// The poll loop emits these once the guest-side smoltcp socket reaches +/// `Established`. At that point we can safely create the host-side relay +/// thread and give it channel endpoints for payload exchange. +pub struct NewTcpConnection { + /// Destination originally requested by the guest. + pub destination: SocketAddr, + /// How the host-side relay should be started. + pub relay_target: RelayTarget, + /// Guest-to-host payloads read from the smoltcp socket. + pub from_smoltcp: Receiver>, + /// Host-to-guest payloads written back into the smoltcp socket. + pub to_smoltcp: SyncSender>, + /// Shared relay exit state. + pub exit_state: RelayExitState, +} + +#[derive(Debug)] +struct TrackedConnection { + // `source` and `destination` identify the guest-side flow. + source: SocketAddr, + destination: SocketAddr, + // guest -> host relay payloads + to_proxy: SyncSender>, + // host -> guest relay payloads + from_proxy: Receiver>, + // endpoints are held here until the guest-side handshake completes + pending_proxy_endpoints: Option, + // once true, a dedicated host relay thread exists + relay_spawned: bool, + // partial guest->host payload already consumed from smoltcp but not yet + // accepted by the relay thread channel + buffered_guest_data: Option>, + // partial host->guest payload not yet fully accepted by smoltcp + buffered_proxy_data: Option<(Vec, usize)>, + // bounded retry count for closing with unsent buffered data + close_attempts: u16, + // relay thread termination mode observed by the poll loop + exit_state: RelayExitState, + // reserved local source port for published inbound connections + reserved_published_port: Option, +} + +#[derive(Debug)] +struct PendingProxyEndpoints { + from_smoltcp: Receiver>, + to_smoltcp: SyncSender>, + relay_target: RelayTarget, +} + +/// How a host-side TCP relay should obtain its remote socket. +#[derive(Debug)] +pub enum RelayTarget { + /// Open a new outbound host `TcpStream` to the destination. + Connect(SocketAddr), + /// Use an already-accepted host `TcpStream` from a published port listener. + Attached(TcpStream), +} + +/// Host relay termination state shared between the poll loop and the relay thread. +/// +/// The relay thread cannot mutate smoltcp sockets directly because those sockets +/// are owned by the poll loop thread. Instead it reports how it finished, and +/// the poll loop interprets that into guest-side socket actions: +/// - `Graceful` -> close guest socket cleanly +/// - `Abort` -> abort/reset guest socket +#[derive(Clone, Debug)] +pub struct RelayExitState { + inner: Arc, +} + +/// How a host TCP relay thread terminated. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum RelayExitMode { + /// Relay thread is still running. + Running = 0, + /// Remote side closed normally; send FIN toward the guest. + Graceful = 1, + /// Remote connect or I/O failed; abort the guest TCP socket. + Abort = 2, +} + +impl RelayExitState { + fn new() -> Self { + Self { + inner: Arc::new(AtomicU8::new(RelayExitMode::Running as u8)), + } + } + + fn load(&self) -> RelayExitMode { + match self.inner.load(Ordering::Relaxed) { + 1 => RelayExitMode::Graceful, + 2 => RelayExitMode::Abort, + _ => RelayExitMode::Running, + } + } + + fn store(&self, mode: RelayExitMode) { + self.inner.store(mode as u8, Ordering::Relaxed); + } +} + +impl TcpRelayTable { + /// Create a new relay table. + pub fn new(max_connections: Option, egress: EgressPolicy) -> Self { + Self { + connections: HashMap::new(), + connection_keys: HashSet::new(), + used_published_ports: HashSet::new(), + next_published_port: PUBLISHED_PORT_START, + max_connections: max_connections.unwrap_or(MAX_CONNECTIONS), + egress, + } + } + + /// Whether a relay socket already exists for the same guest source and destination. + pub fn has_socket_for(&self, source: &SocketAddr, destination: &SocketAddr) -> bool { + self.connection_keys.contains(&(*source, *destination)) + } + + /// Create a smoltcp TCP socket for a guest SYN. + /// + /// Why this happens before full ingress processing: + /// - when the first guest SYN arrives, smoltcp needs a matching socket to + /// receive it + /// - the poll loop therefore pre-creates a listening socket keyed to the + /// destination the guest is trying to reach + /// - only after the guest-facing connection reaches `Established` do we + /// spawn the host relay thread + /// + /// Data path after creation: + /// + /// ```text + /// smoltcp socket --to_proxy channel--> host relay thread + /// host relay thread --from_proxy channel--> smoltcp socket + /// ``` + pub fn create_tcp_socket( + &mut self, + source: SocketAddr, + destination: SocketAddr, + sockets: &mut SocketSet<'_>, + ) -> bool { + if self.connections.len() >= self.max_connections { + tracing::warn!("dropping TCP connection because the relay table is full"); + return false; + } + + // Egress policy: drop the guest SYN before any socket is created when the + // destination isn't allowed, so the guest just sees the connection fail. + // Inbound published-port flows take a separate path and are unaffected. + if !self.egress.allows(destination.ip()) { + tracing::debug!( + %destination, + "virtio-net: blocking outbound connection by egress policy" + ); + return false; + } + + let rx_buffer = tcp::SocketBuffer::new(vec![0u8; TCP_RX_BUFFER_BYTES]); + let tx_buffer = tcp::SocketBuffer::new(vec![0u8; TCP_TX_BUFFER_BYTES]); + let mut socket = tcp::Socket::new(rx_buffer, tx_buffer); + + let listen_endpoint = IpListenEndpoint { + addr: Some(destination.ip().into()), + port: destination.port(), + }; + if socket.listen(listen_endpoint).is_err() { + return false; + } + + let handle = sockets.add(socket); + + let (to_proxy_tx, to_proxy_rx) = mpsc::sync_channel(CHANNEL_CAPACITY); + let (from_proxy_tx, from_proxy_rx) = mpsc::sync_channel(CHANNEL_CAPACITY); + let exit_state = RelayExitState::new(); + + self.connection_keys.insert((source, destination)); + self.connections.insert( + handle, + TrackedConnection { + source, + destination, + to_proxy: to_proxy_tx, + from_proxy: from_proxy_rx, + pending_proxy_endpoints: Some(PendingProxyEndpoints { + from_smoltcp: to_proxy_rx, + to_smoltcp: from_proxy_tx, + relay_target: RelayTarget::Connect(destination), + }), + relay_spawned: false, + buffered_guest_data: None, + buffered_proxy_data: None, + close_attempts: 0, + exit_state, + reserved_published_port: None, + }, + ); + + true + } + + /// Create a guest-facing TCP connection for a published host socket. + /// + /// This is the host->guest mirror of `create_tcp_socket`: + /// + /// ```text + /// host client connects to published port + /// -> host listener accepts TcpStream + /// -> poll loop creates smoltcp TCP socket from gateway_ip:ephemeral + /// to guest_ip:guest_port + /// -> guest kernel sees a normal inbound TCP connection on guest_port + /// ``` + /// + /// The guest-visible source address is the gateway IP, not the original + /// host peer address. That keeps the first version simple and matches the + /// fact that this runtime is acting as a userspace gateway/proxy. + pub fn create_published_socket( + &mut self, + interface: &mut Interface, + gateway_ip: Ipv4Addr, + destination: SocketAddr, + host_stream: TcpStream, + sockets: &mut SocketSet<'_>, + ) -> bool { + if self.connections.len() >= self.max_connections { + tracing::warn!("dropping published TCP connection because the relay table is full"); + return false; + } + + let Some(local_port) = self.allocate_published_port() else { + tracing::warn!( + "dropping published TCP connection because no gateway source port is available" + ); + return false; + }; + + // Inbound published connections always target the guest's IPv4 on the + // internal link (the host listener family is independent of this). + let std::net::IpAddr::V4(destination_ip) = destination.ip() else { + self.used_published_ports.remove(&local_port); + return false; + }; + + let rx_buffer = tcp::SocketBuffer::new(vec![0u8; TCP_RX_BUFFER_BYTES]); + let tx_buffer = tcp::SocketBuffer::new(vec![0u8; TCP_TX_BUFFER_BYTES]); + let mut socket = tcp::Socket::new(rx_buffer, tx_buffer); + let local_endpoint = IpListenEndpoint { + addr: Some(gateway_ip.into()), + port: local_port, + }; + if socket + .connect( + interface.context(), + (destination_ip, destination.port()), + local_endpoint, + ) + .is_err() + { + self.used_published_ports.remove(&local_port); + return false; + } + + let handle = sockets.add(socket); + let source = SocketAddr::new(std::net::IpAddr::V4(gateway_ip), local_port); + + let (to_proxy_tx, to_proxy_rx) = mpsc::sync_channel(CHANNEL_CAPACITY); + let (from_proxy_tx, from_proxy_rx) = mpsc::sync_channel(CHANNEL_CAPACITY); + let exit_state = RelayExitState::new(); + + self.connection_keys.insert((source, destination)); + self.connections.insert( + handle, + TrackedConnection { + source, + destination, + to_proxy: to_proxy_tx, + from_proxy: from_proxy_rx, + pending_proxy_endpoints: Some(PendingProxyEndpoints { + from_smoltcp: to_proxy_rx, + to_smoltcp: from_proxy_tx, + relay_target: RelayTarget::Attached(host_stream), + }), + relay_spawned: false, + buffered_guest_data: None, + buffered_proxy_data: None, + close_attempts: 0, + exit_state, + reserved_published_port: Some(local_port), + }, + ); + + true + } + + /// Relay TCP payloads between smoltcp sockets and host relay threads. + /// + /// This runs in the poll thread. It is responsible for: + /// - draining bytes received from the guest-facing smoltcp socket and + /// pushing them toward the host relay thread + /// - draining bytes received from the host relay thread and writing them + /// back into the smoltcp socket + /// - interpreting relay exit state into guest-side `close()` or `abort()` + pub fn relay_data(&mut self, sockets: &mut SocketSet<'_>) { + let mut read_buffer = [0u8; RELAY_BUFFER_BYTES]; + + for (&handle, connection) in &mut self.connections { + if !connection.relay_spawned { + continue; + } + + let socket = sockets.get_mut::(handle); + + match connection.exit_state.load() { + RelayExitMode::Abort => { + socket.abort(); + continue; + } + RelayExitMode::Graceful => { + flush_proxy_data(socket, connection); + if connection.buffered_proxy_data.is_none() { + socket.close(); + } else { + connection.close_attempts += 1; + if connection.close_attempts >= CLOSE_RETRY_LIMIT { + socket.abort(); + } + } + continue; + } + RelayExitMode::Running => {} + } + + flush_guest_data(connection); + while connection.buffered_guest_data.is_none() && socket.can_recv() { + match socket.recv_slice(&mut read_buffer) { + Ok(bytes_read) if bytes_read > 0 => { + let payload = read_buffer[..bytes_read].to_vec(); + if !send_guest_payload(connection, payload) { + break; + } + } + _ => break, + } + } + + flush_proxy_data(socket, connection); + } + } + + /// Collect connections that reached ESTABLISHED and need a host relay thread. + /// + /// The separation between `create_tcp_socket` and this method is important: + /// the guest TCP handshake is accepted first on the smoltcp side, and only + /// once that succeeds do we commit to opening the host-side `TcpStream`. + pub fn take_new_connections(&mut self, sockets: &mut SocketSet<'_>) -> Vec { + let mut new_connections = Vec::new(); + + for (&handle, connection) in &mut self.connections { + if connection.relay_spawned { + continue; + } + + let socket = sockets.get::(handle); + if socket.state() == tcp::State::Established { + connection.relay_spawned = true; + + if let Some(endpoints) = connection.pending_proxy_endpoints.take() { + new_connections.push(NewTcpConnection { + destination: connection.destination, + relay_target: endpoints.relay_target, + from_smoltcp: endpoints.from_smoltcp, + to_smoltcp: endpoints.to_smoltcp, + exit_state: connection.exit_state.clone(), + }); + } + } + } + + new_connections + } + + /// Remove closed sockets and drop their relay endpoints. + /// + /// This is the final ownership cleanup step for a guest TCP flow. + pub fn cleanup_closed(&mut self, sockets: &mut SocketSet<'_>) { + let keys = &mut self.connection_keys; + let published_ports = &mut self.used_published_ports; + self.connections.retain(|&handle, connection| { + let socket = sockets.get::(handle); + if socket.state() == tcp::State::Closed { + keys.remove(&(connection.source, connection.destination)); + if let Some(port) = connection.reserved_published_port { + published_ports.remove(&port); + } + sockets.remove(handle); + false + } else { + true + } + }); + } + + fn allocate_published_port(&mut self) -> Option { + let start = self.next_published_port; + + loop { + let candidate = self.next_published_port; + self.next_published_port = if candidate == PUBLISHED_PORT_END { + PUBLISHED_PORT_START + } else { + candidate + 1 + }; + + if self.used_published_ports.insert(candidate) { + return Some(candidate); + } + + if self.next_published_port == start { + return None; + } + } + } +} + +/// Spawn one host TCP relay thread for an established guest connection. +/// +/// Thread responsibilities: +/// - connect a host `TcpStream` to the guest-requested destination +/// - copy bytes guest->host from `from_smoltcp` +/// - copy bytes host->guest into `to_smoltcp` +/// - wake the poll loop when host->guest data arrives or guest->host backpressure eases +/// - report termination mode through `exit_state` +pub fn spawn_tcp_relay( + destination: SocketAddr, + relay_target: RelayTarget, + from_smoltcp: Receiver>, + to_smoltcp: SyncSender>, + relay_wake: Arc, + exit_state: RelayExitState, +) { + let thread_name = format!("smolvm-tcp-{}", destination.port()); + virtio_net_log!( + "virtio-net: spawning host TCP relay thread destination={} thread={}", + destination, + thread_name + ); + let _ = thread::Builder::new().name(thread_name).spawn(move || { + run_tcp_relay( + destination, + relay_target, + from_smoltcp, + to_smoltcp, + relay_wake, + exit_state, + ) + }); +} + +fn run_tcp_relay( + destination: SocketAddr, + relay_target: RelayTarget, + from_smoltcp: Receiver>, + to_smoltcp: SyncSender>, + relay_wake: Arc, + exit_state: RelayExitState, +) { + // The relay thread is intentionally isolated from smoltcp internals. Its + // contract is just channels in, channels out, and an exit code back. + virtio_net_log!( + "virtio-net: host TCP relay thread started destination={}", + destination + ); + match tcp_relay_loop( + destination, + relay_target, + from_smoltcp, + to_smoltcp, + relay_wake, + ) { + Ok(mode) => { + virtio_net_log!( + "virtio-net: host TCP relay thread exited destination={} mode={:?}", + destination, + mode + ); + exit_state.store(mode) + } + Err(err) => { + virtio_net_log!( + "virtio-net: host TCP relay failed destination={} error={}", + destination, + err + ); + exit_state.store(RelayExitMode::Abort); + } + } +} + +fn tcp_relay_loop( + destination: SocketAddr, + relay_target: RelayTarget, + from_smoltcp: Receiver>, + to_smoltcp: SyncSender>, + relay_wake: Arc, +) -> io::Result { + // Host-side flow: + // + // 1. Connect a normal host TcpStream to the destination. + // 2. Non-blockingly drain guest payloads from the channel into the socket. + // 3. Non-blockingly read remote payloads from the socket into the channel. + // 4. If neither side made progress, sleep briefly to avoid a hot spin loop. + let mut stream = match relay_target { + RelayTarget::Connect(destination) => { + virtio_net_log!( + "virtio-net: connecting host TCP relay socket destination={}", + destination + ); + let stream = TcpStream::connect(destination)?; + virtio_net_log!( + "virtio-net: host TCP relay socket connected destination={}", + destination + ); + stream + } + RelayTarget::Attached(stream) => { + virtio_net_log!( + "virtio-net: using accepted host TCP socket for published port guest_destination={} peer_addr={:?} local_addr={:?}", + destination, + stream.peer_addr().ok(), + stream.local_addr().ok() + ); + stream + } + }; + stream.set_nonblocking(true)?; + + let mut guest_write_closed = false; + let mut guest_channel_closed = false; + let mut pending_guest_data: Option<(Vec, usize)> = None; + let mut read_buffer = [0u8; RELAY_BUFFER_BYTES]; + + loop { + let mut did_work = false; + + if pending_guest_data.is_none() && !guest_channel_closed { + match from_smoltcp.try_recv() { + Ok(payload) => { + pending_guest_data = Some((payload, 0)); + // Consuming from the bounded guest->host channel may free + // capacity for a payload buffered in the smoltcp poll + // thread. Wake it so backpressure clears promptly even for + // one-way guest->host streams. + relay_wake.wake(); + did_work = true; + } + Err(TryRecvError::Empty) => {} + Err(TryRecvError::Disconnected) => { + guest_channel_closed = true; + } + } + } + + if let Some((payload, offset)) = &mut pending_guest_data { + while *offset < payload.len() { + match stream.write(&payload[*offset..]) { + Ok(0) => { + return Err(io::Error::new( + io::ErrorKind::WriteZero, + "host TCP relay wrote zero bytes", + )); + } + Ok(bytes_written) => { + *offset += bytes_written; + did_work = true; + } + Err(err) if err.kind() == io::ErrorKind::WouldBlock => break, + Err(err) if err.kind() == io::ErrorKind::Interrupted => continue, + Err(err) => return Err(err), + } + } + + if *offset >= payload.len() { + pending_guest_data = None; + } + } + + if guest_channel_closed && pending_guest_data.is_none() && !guest_write_closed { + // The guest side closed its write half. Mirror that toward the + // remote peer only after all buffered guest bytes were written. + let _ = stream.shutdown(Shutdown::Write); + guest_write_closed = true; + } + + match stream.read(&mut read_buffer) { + Ok(0) => return Ok(RelayExitMode::Graceful), + Ok(bytes_read) => { + if to_smoltcp.send(read_buffer[..bytes_read].to_vec()).is_err() { + return Ok(RelayExitMode::Graceful); + } + relay_wake.wake(); + did_work = true; + } + Err(err) if err.kind() == io::ErrorKind::WouldBlock => {} + Err(err) => return Err(err), + } + + if !did_work { + thread::sleep(PROXY_IDLE_SLEEP); + } + } +} + +fn flush_guest_data(connection: &mut TrackedConnection) { + let Some(payload) = connection.buffered_guest_data.take() else { + return; + }; + send_guest_payload(connection, payload); +} + +fn send_guest_payload(connection: &mut TrackedConnection, payload: Vec) -> bool { + match connection.to_proxy.try_send(payload) { + Ok(()) => true, + Err(TrySendError::Full(payload)) => { + connection.buffered_guest_data = Some(payload); + false + } + Err(TrySendError::Disconnected(_)) => false, + } +} + +fn flush_proxy_data(socket: &mut tcp::Socket<'_>, connection: &mut TrackedConnection) { + // smoltcp send windows may accept only part of an inbound host payload. + // `buffered_proxy_data` remembers the unwritten remainder so the next poll + // iteration can continue where it left off instead of dropping bytes. + if let Some((data, offset)) = &mut connection.buffered_proxy_data { + if socket.can_send() { + match socket.send_slice(&data[*offset..]) { + Ok(written) => { + *offset += written; + if *offset >= data.len() { + connection.buffered_proxy_data = None; + } + } + Err(_) => return, + } + } else { + return; + } + } + + while connection.buffered_proxy_data.is_none() { + match connection.from_proxy.try_recv() { + Ok(payload) => { + if socket.can_send() { + match socket.send_slice(&payload) { + Ok(written) if written < payload.len() => { + connection.buffered_proxy_data = Some((payload, written)); + } + Err(_) => { + connection.buffered_proxy_data = Some((payload, 0)); + } + _ => {} + } + } else { + connection.buffered_proxy_data = Some((payload, 0)); + } + } + Err(TryRecvError::Empty | TryRecvError::Disconnected) => break, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_connection(to_proxy: SyncSender>) -> TrackedConnection { + let (_from_proxy_tx, from_proxy) = mpsc::sync_channel(CHANNEL_CAPACITY); + TrackedConnection { + source: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 12_345), + destination: SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 80), + to_proxy, + from_proxy, + pending_proxy_endpoints: None, + relay_spawned: true, + buffered_guest_data: None, + buffered_proxy_data: None, + close_attempts: 0, + exit_state: RelayExitState::new(), + reserved_published_port: None, + } + } + + #[test] + fn guest_payload_is_buffered_when_relay_channel_is_full() { + let (to_proxy, from_smoltcp) = mpsc::sync_channel(1); + to_proxy.send(vec![1]).unwrap(); + let mut connection = test_connection(to_proxy); + + assert!(!send_guest_payload(&mut connection, vec![2])); + assert_eq!(connection.buffered_guest_data.as_deref(), Some(&[2][..])); + + assert_eq!(from_smoltcp.recv().unwrap(), vec![1]); + flush_guest_data(&mut connection); + + assert!(connection.buffered_guest_data.is_none()); + assert_eq!(from_smoltcp.recv().unwrap(), vec![2]); + } +} diff --git a/crates/smolvm-network/src/udp_relay.rs b/crates/smolvm-network/src/udp_relay.rs new file mode 100644 index 0000000..73952ef --- /dev/null +++ b/crates/smolvm-network/src/udp_relay.rs @@ -0,0 +1,537 @@ +//! General UDP relay for the virtio-net backend. +//! +//! Context +//! ======= +//! +//! DNS (UDP :53) is intercepted and answered by the gateway itself +//! (`stack.rs::process_dns_queries`). Every other guest UDP datagram used to be +//! dropped (`FrameAction::UnsupportedUdp`), which broke QUIC/HTTP-3, NTP, and +//! any DNS on a non-standard port. This module relays those flows to real host +//! UDP sockets, giving virtio-net parity with TSI (whose syscall-level hijack +//! gets UDP for free). +//! +//! Model — a tiny userspace NAT, mirroring the TCP relay's shape: +//! +//! ```text +//! guest datagram to D:port (≠ :53) +//! -> classify_guest_frame: FrameAction::UdpFlow +//! -> poll loop: egress check; ensure a smoltcp UDP socket bound to D:port +//! -> smoltcp ingress delivers the datagram into that socket +//! -> poll loop drains it and channels (guest, D, payload) to the relay thread +//! -> relay thread: one connected host UdpSocket per (guest, D) flow; send +//! -> replies: host socket readable -> channel back -> relay_wake +//! -> poll loop writes the reply into the D-bound smoltcp socket with +//! endpoint = guest, local_address = D (guest sees the reply come from D) +//! ``` +//! +//! Flows are connectionless, so lifetime is NAT-style idle expiry: the relay +//! drops host sockets idle for [`FLOW_IDLE_TIMEOUT`]; the poll loop drops +//! destination sockets idle for [`DST_SOCKET_IDLE_TIMEOUT`]. Loss under +//! pressure (full channels / tables) is acceptable UDP semantics — logged, +//! never blocking. + +use crate::egress::EgressPolicy; +use crate::queues::WakePipe; +use crate::virtio_net_log; +use polling::{Event, Events}; +use smoltcp::iface::{SocketHandle, SocketSet}; +use smoltcp::socket::udp::{PacketBuffer, PacketMetadata, Socket as UdpSocket, UdpMetadata}; +use std::collections::HashMap; +use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, UdpSocket as HostUdpSocket}; +use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError, TrySendError}; +use std::sync::Arc; +use std::thread; +use std::time::{Duration, Instant}; + +/// Max in-flight datagrams per direction before drops (UDP may drop). +const CHANNEL_CAPACITY: usize = 256; +/// Max concurrent (guest, destination) flows with live host sockets. +const MAX_FLOWS: usize = 1024; +/// Max destination-keyed smoltcp sockets in the poll loop. +const MAX_DST_SOCKETS: usize = 128; +/// Packet slots per destination socket buffer. +const UDP_PACKET_SLOTS: usize = 16; +/// Payload bytes per destination socket buffer (per direction). +const UDP_BUFFER_BYTES: usize = 64 * 1024; +/// Largest datagram we relay (gateway MTU bounds guest->host anyway). +const MAX_DATAGRAM_BYTES: usize = 65_535; +/// Drop a flow's host socket after this much inactivity. +const FLOW_IDLE_TIMEOUT: Duration = Duration::from_secs(60); +/// Remove a destination's smoltcp socket after this much inactivity. +const DST_SOCKET_IDLE_TIMEOUT: Duration = Duration::from_secs(120); +/// Relay thread poll ceiling, so shutdown and expiry are noticed promptly. +const RELAY_POLL_MAX_MS: i32 = 1000; + +/// One relayed datagram, in either direction. +pub struct UdpDatagram { + /// Guest-side endpoint (source on egress, reply target on ingress). + pub guest: SocketAddr, + /// External destination the guest addressed (and replies come from). + pub destination: SocketAddr, + /// Datagram payload. + pub payload: Vec, +} + +/// Channel pair connecting the poll loop and the relay thread. +pub struct UdpRelayChannels { + /// Poll loop -> relay thread. + pub to_relay: SyncSender, + /// Relay thread -> poll loop. + pub from_relay: Receiver, + /// Wakes the relay thread after `to_relay` sends. + pub relay_thread_wake: WakePipe, +} + +/// Start the UDP relay thread. Returns the poll-loop-side channel endpoints. +/// +/// `reply_wake` is the smoltcp poll loop's existing relay wake pipe — pulsed +/// whenever a reply is queued so the loop wakes to deliver it to the guest. +/// The thread exits when `shutdown` reports true (checked at least once per +/// [`RELAY_POLL_MAX_MS`]). +pub fn start_udp_relay( + reply_wake: Arc, + shutdown: Arc bool + Send + Sync>, +) -> UdpRelayChannels { + let (to_relay_tx, to_relay_rx) = mpsc::sync_channel(CHANNEL_CAPACITY); + let (from_relay_tx, from_relay_rx) = mpsc::sync_channel(CHANNEL_CAPACITY); + let relay_thread_wake = WakePipe::new(); + let thread_wake = relay_thread_wake.clone(); + + let _ = thread::Builder::new() + .name("smolvm-udp-relay".into()) + .spawn(move || { + run_udp_relay( + to_relay_rx, + from_relay_tx, + thread_wake, + reply_wake, + shutdown, + ); + }); + + UdpRelayChannels { + to_relay: to_relay_tx, + from_relay: from_relay_rx, + relay_thread_wake, + } +} + +/// Relay-thread state for one (guest, destination) flow. +struct UdpFlow { + socket: HostUdpSocket, + guest: SocketAddr, + destination: SocketAddr, + last_active: Instant, +} + +fn run_udp_relay( + outbound: Receiver, + inbound: SyncSender, + wake: WakePipe, + reply_wake: Arc, + shutdown: Arc bool + Send + Sync>, +) { + let mut flows: HashMap<(SocketAddr, SocketAddr), UdpFlow> = HashMap::new(); + let mut recv_buf = vec![0u8; MAX_DATAGRAM_BYTES]; + + loop { + if shutdown() { + return; + } + + // Outbound: guest datagrams handed over by the poll loop. + loop { + match outbound.try_recv() { + Ok(datagram) => { + let key = (datagram.guest, datagram.destination); + if !flows.contains_key(&key) { + if flows.len() >= MAX_FLOWS { + virtio_net_log!( + "virtio-net: dropping UDP flow {} -> {} (flow table full)", + datagram.guest, + datagram.destination + ); + continue; + } + match create_flow_socket(datagram.destination) { + Ok(socket) => { + flows.insert( + key, + UdpFlow { + socket, + guest: datagram.guest, + destination: datagram.destination, + last_active: Instant::now(), + }, + ); + } + Err(err) => { + virtio_net_log!( + "virtio-net: failed to open host UDP socket for {} -> {}: {}", + datagram.guest, + datagram.destination, + err + ); + continue; + } + } + } + let flow = flows.get_mut(&key).expect("flow inserted above"); + flow.last_active = Instant::now(); + // Best-effort send; UDP loss is allowed. + let _ = flow.socket.send(&datagram.payload); + } + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => return, + } + } + + // Inbound: poll all flow sockets for replies. The wake's poller blocks + // on "wake OR any flow socket readable" in a single call; the wake is a + // notify (no key), and each flow socket is registered at key `slot + 1`. + let poller = wake.poller(); + let keys: Vec<(SocketAddr, SocketAddr)> = flows.keys().copied().collect(); + for (slot, key) in keys.iter().enumerate() { + // SAFETY: the socket is owned by `flows` and is deleted from the + // poller below before the next iteration may drop it. + let _ = unsafe { poller.add(&flows[key].socket, Event::readable(slot + 1)) }; + } + + let mut events = Events::new(); + let _ = poller.wait( + &mut events, + Some(Duration::from_millis(RELAY_POLL_MAX_MS as u64)), + ); + + // A pending notify is consumed by `wait`; nothing else to drain. + let mut ready: Vec = vec![false; keys.len()]; + for event in events.iter() { + if event.key >= 1 && event.key - 1 < ready.len() { + ready[event.key - 1] = true; + } + } + + // Deregister sockets before the next rebuild so a closed flow's socket + // is never left registered on the poller. + for key in &keys { + let _ = poller.delete(&flows[key].socket); + } + + let mut woke_reply = false; + for (slot, key) in keys.iter().enumerate() { + if !ready[slot] { + continue; + } + let Some(flow) = flows.get_mut(key) else { + continue; + }; + // Drain everything ready on this socket. WouldBlock ends the drain; + // ECONNREFUSED (ICMP unreachable surfaced by the connected socket) + // and friends just mean the flow is dead-ish — idle expiry handles it. + while let Ok(len) = flow.socket.recv(&mut recv_buf) { + flow.last_active = Instant::now(); + let reply = UdpDatagram { + guest: flow.guest, + destination: flow.destination, + payload: recv_buf[..len].to_vec(), + }; + match inbound.try_send(reply) { + Ok(()) => woke_reply = true, + Err(TrySendError::Full(_)) => { + virtio_net_log!( + "virtio-net: dropping UDP reply for {} (inbound queue full)", + flow.guest + ); + } + Err(TrySendError::Disconnected(_)) => return, + } + } + } + if woke_reply { + reply_wake.wake(); + } + + // NAT-style idle expiry. + let now = Instant::now(); + flows.retain(|_, flow| now.duration_since(flow.last_active) < FLOW_IDLE_TIMEOUT); + } +} + +/// Open a non-blocking host UDP socket connected to `destination`. +/// +/// `connect` pins the peer so `send`/`recv` apply and stray traffic from other +/// peers is filtered by the kernel — each flow is its own little NAT pinhole. +fn create_flow_socket(destination: SocketAddr) -> std::io::Result { + let bind_addr: SocketAddr = if destination.is_ipv4() { + (Ipv4Addr::UNSPECIFIED, 0).into() + } else { + (Ipv6Addr::UNSPECIFIED, 0).into() + }; + let socket = HostUdpSocket::bind(bind_addr)?; + socket.connect(destination)?; + socket.set_nonblocking(true)?; + Ok(socket) +} + +/// Poll-loop-side table of destination-keyed smoltcp UDP sockets. +/// +/// One socket per (destination address, port) — the UDP mirror of +/// `TcpRelayTable::create_tcp_socket`'s destination-keyed listen sockets. The +/// guest's own endpoint comes from receive metadata, so all guest flows to the +/// same destination share one smoltcp socket. +pub struct UdpSocketTable { + sockets: HashMap, + last_active: HashMap, +} + +impl UdpSocketTable { + pub fn new() -> Self { + Self { + sockets: HashMap::new(), + last_active: HashMap::new(), + } + } + + /// Ensure a smoltcp UDP socket exists for `destination` so the staged guest + /// datagram has somewhere to land. Returns false when the table is full or + /// the bind fails (the caller drops the frame — UDP semantics). + pub fn ensure_socket(&mut self, destination: SocketAddr, sockets: &mut SocketSet<'_>) -> bool { + if self.sockets.contains_key(&destination) { + self.last_active.insert(destination, Instant::now()); + return true; + } + if self.sockets.len() >= MAX_DST_SOCKETS { + virtio_net_log!( + "virtio-net: dropping UDP datagram to {} (destination socket table full)", + destination + ); + return false; + } + + let rx = PacketBuffer::new( + vec![PacketMetadata::EMPTY; UDP_PACKET_SLOTS], + vec![0u8; UDP_BUFFER_BYTES], + ); + let tx = PacketBuffer::new( + vec![PacketMetadata::EMPTY; UDP_PACKET_SLOTS], + vec![0u8; UDP_BUFFER_BYTES], + ); + let mut socket = UdpSocket::new(rx, tx); + if socket + .bind(smoltcp::wire::IpListenEndpoint { + addr: Some(destination.ip().into()), + port: destination.port(), + }) + .is_err() + { + return false; + } + + let handle = sockets.add(socket); + self.sockets.insert(destination, handle); + self.last_active.insert(destination, Instant::now()); + true + } + + /// Drain guest datagrams from every destination socket toward the relay + /// thread. Returns true if anything was forwarded (the caller then wakes + /// the relay thread). + pub fn drain_to_relay( + &mut self, + sockets: &mut SocketSet<'_>, + to_relay: &SyncSender, + ) -> bool { + let mut forwarded = false; + for (&destination, &handle) in &self.sockets { + let socket = sockets.get_mut::(handle); + while socket.can_recv() { + let Ok((payload, metadata)) = socket.recv() else { + break; + }; + self.last_active.insert(destination, Instant::now()); + let guest = endpoint_to_socket_addr(metadata.endpoint); + let datagram = UdpDatagram { + guest, + destination, + payload: payload.to_vec(), + }; + match to_relay.try_send(datagram) { + Ok(()) => forwarded = true, + Err(TrySendError::Full(_)) => { + virtio_net_log!( + "virtio-net: dropping guest UDP datagram to {} (relay queue full)", + destination + ); + } + Err(TrySendError::Disconnected(_)) => return forwarded, + } + } + } + forwarded + } + + /// Deliver relay replies into the matching destination socket so smoltcp + /// sends them to the guest, sourced from the original destination address. + pub fn deliver_replies( + &mut self, + sockets: &mut SocketSet<'_>, + from_relay: &Receiver, + ) { + while let Ok(reply) = from_relay.try_recv() { + let Some(&handle) = self.sockets.get(&reply.destination) else { + // Destination socket already expired; the reply is late. + continue; + }; + self.last_active.insert(reply.destination, Instant::now()); + let socket = sockets.get_mut::(handle); + let metadata = UdpMetadata { + endpoint: smoltcp::wire::IpEndpoint { + addr: reply.guest.ip().into(), + port: reply.guest.port(), + }, + local_address: Some(reply.destination.ip().into()), + meta: Default::default(), + }; + if socket.send_slice(&reply.payload, metadata).is_err() { + virtio_net_log!( + "virtio-net: dropping UDP reply to {} (guest socket buffer full)", + reply.guest + ); + } + } + } + + /// Remove destination sockets with no traffic for [`DST_SOCKET_IDLE_TIMEOUT`]. + pub fn expire_idle(&mut self, sockets: &mut SocketSet<'_>) { + let now = Instant::now(); + let expired: Vec = self + .last_active + .iter() + .filter(|(_, last)| now.duration_since(**last) >= DST_SOCKET_IDLE_TIMEOUT) + .map(|(dst, _)| *dst) + .collect(); + for destination in expired { + if let Some(handle) = self.sockets.remove(&destination) { + sockets.remove(handle); + } + self.last_active.remove(&destination); + } + } +} + +impl Default for UdpSocketTable { + fn default() -> Self { + Self::new() + } +} + +/// Whether the gateway should relay a guest UDP datagram to this destination. +/// DNS (:53) is excluded — it has its own intercept-and-filter path. Egress +/// policy applies exactly as for TCP (static CIDRs + DNS-learned IPs). +pub fn should_relay_udp(destination: SocketAddr, egress: &EgressPolicy) -> bool { + destination.port() != 53 && egress.allows(destination.ip()) +} + +fn endpoint_to_socket_addr(endpoint: smoltcp::wire::IpEndpoint) -> SocketAddr { + let ip: std::net::IpAddr = endpoint.addr.into(); + SocketAddr::new(ip, endpoint.port) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, Ordering}; + + #[test] + fn flow_socket_round_trip() { + // Echo server on the host loopback. + let server = HostUdpSocket::bind("127.0.0.1:0").unwrap(); + let server_addr = server.local_addr().unwrap(); + + let flow = create_flow_socket(server_addr).unwrap(); + flow.send(b"ping").unwrap(); + + let mut buf = [0u8; 16]; + let (len, peer) = server.recv_from(&mut buf).unwrap(); + assert_eq!(&buf[..len], b"ping"); + server.send_to(b"pong", peer).unwrap(); + + // Non-blocking: poll briefly for the reply. + let deadline = Instant::now() + Duration::from_secs(2); + loop { + match flow.recv(&mut buf) { + Ok(len) => { + assert_eq!(&buf[..len], b"pong"); + break; + } + Err(_) if Instant::now() < deadline => thread::sleep(Duration::from_millis(10)), + Err(e) => panic!("no reply: {e}"), + } + } + } + + #[test] + fn relay_thread_round_trips_a_datagram() { + let server = HostUdpSocket::bind("127.0.0.1:0").unwrap(); + let server_addr = server.local_addr().unwrap(); + + let reply_wake = Arc::new(WakePipe::new()); + let stop = Arc::new(AtomicBool::new(false)); + let stop_flag = stop.clone(); + let channels = start_udp_relay( + reply_wake.clone(), + Arc::new(move || stop_flag.load(Ordering::Relaxed)), + ); + + let guest: SocketAddr = "100.96.0.2:40000".parse().unwrap(); + channels + .to_relay + .send(UdpDatagram { + guest, + destination: server_addr, + payload: b"hello".to_vec(), + }) + .unwrap(); + channels.relay_thread_wake.wake(); + + // Server sees the datagram and answers. + let mut buf = [0u8; 16]; + server + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let (len, peer) = server.recv_from(&mut buf).unwrap(); + assert_eq!(&buf[..len], b"hello"); + server.send_to(b"world", peer).unwrap(); + + // The reply arrives on the inbound channel addressed to the guest flow. + let reply = channels + .from_relay + .recv_timeout(Duration::from_secs(2)) + .unwrap(); + assert_eq!(reply.guest, guest); + assert_eq!(reply.destination, server_addr); + assert_eq!(reply.payload, b"world"); + + stop.store(true, Ordering::Relaxed); + channels.relay_thread_wake.wake(); + } + + #[test] + fn should_relay_respects_dns_carveout_and_policy() { + let open = EgressPolicy::unrestricted(); + assert!(should_relay_udp("1.2.3.4:123".parse().unwrap(), &open)); + assert!(!should_relay_udp("1.2.3.4:53".parse().unwrap(), &open)); + + // Public CIDR — a private allow-list entry would be overridden by the + // egress hard-floor (see egress.rs tests). + let restricted = EgressPolicy::from_allowed_cidrs(Some(&["8.8.8.0/24".into()])); + assert!(should_relay_udp( + "8.8.8.3:123".parse().unwrap(), + &restricted + )); + assert!(!should_relay_udp( + "1.2.3.4:123".parse().unwrap(), + &restricted + )); + } +} diff --git a/crates/smolvm-pack/Cargo.toml b/crates/smolvm-pack/Cargo.toml new file mode 100644 index 0000000..2359046 --- /dev/null +++ b/crates/smolvm-pack/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "smolvm-pack" +version = "1.5.2" +edition = "2021" +description = "Single-binary packaging for smolvm" +license = "Apache-2.0" +repository = "https://github.com/smol-machines/smolvm" + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +smolvm-protocol = { path = "../smolvm-protocol", version = "1.5.2" } +tar = "0.4" +zstd = "0.13" +crc32fast = "1.4" +tempfile = "3" +sha2 = "0.10" +dirs = "5" +time = { version = "0.3", features = ["formatting"] } + +[target.'cfg(unix)'.dependencies] +libc = "0.2" + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.60", features = [ + "Win32_Foundation", + "Win32_Storage_FileSystem", + "Win32_System_IO", +] } diff --git a/crates/smolvm-pack/src/assets.rs b/crates/smolvm-pack/src/assets.rs new file mode 100644 index 0000000..a9cbd33 --- /dev/null +++ b/crates/smolvm-pack/src/assets.rs @@ -0,0 +1,930 @@ +//! Asset collection and compression for packed binaries. +//! +//! This module handles discovering and packaging runtime assets: +//! - Runtime libraries (libkrun, libkrunfw) +//! - Agent rootfs +//! - OCI image layers + +use std::fs::{self, File}; +use std::io::{BufWriter, Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use crate::format::{AssetEntry, AssetInventory, LayerEntry}; +use crate::{PackError, Result}; + +/// Convert a digest string to a filename for layer tars. +/// +/// Strips an optional `sha256:` prefix and uses the full remaining digest hex. +/// Returns an error if the digest portion is shorter than 12 characters. +fn digest_to_filename(digest: &str) -> Result { + let hex = digest.strip_prefix("sha256:").unwrap_or(digest); + if hex.len() < 12 { + return Err(PackError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "digest too short for filename: '{}' ({} chars, need 12)", + digest, + hex.len() + ), + ))); + } + // The filename becomes `layers/{hex}.tar` joined onto the staging dir, so + // the digest MUST be pure hex — otherwise a malicious `.smolmachine` with a + // digest like `sha256:../../evil` would write the layer bytes outside the + // staging root (path traversal). A real content digest is always hex. + if !hex.bytes().all(|b| b.is_ascii_hexdigit()) { + return Err(PackError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("digest is not valid hex: '{}'", digest), + ))); + } + Ok(format!("{}.tar", hex)) +} + +/// Compression level for zstd (3 = zstd default, fast with good ratio). +/// Level 19 was ~100x slower for only ~10% better compression. +pub const ZSTD_LEVEL: i32 = 3; + +/// Find a pre-formatted disk template by filename. +/// +/// Searches in order: +/// 1. `~/.smolvm/{filename}` (installed location) +/// 2. Next to the current executable (development) +fn find_existing_template(filename: &str) -> Option { + if let Some(home) = dirs::home_dir() { + let path = home.join(".smolvm").join(filename); + if path.exists() { + return Some(path); + } + } + if let Ok(exe) = std::env::current_exe() { + if let Some(dir) = exe.parent() { + let path = dir.join(filename); + if path.exists() { + return Some(path); + } + } + } + None +} + +/// Asset collector for gathering runtime components. +pub struct AssetCollector { + staging_dir: PathBuf, + inventory: AssetInventory, +} + +impl AssetCollector { + /// Create a new asset collector with a staging directory. + pub fn new(staging_dir: PathBuf) -> Result { + fs::create_dir_all(&staging_dir)?; + fs::create_dir_all(staging_dir.join("layers"))?; + + Ok(Self { + staging_dir, + inventory: AssetInventory { + libraries: Vec::new(), + agent_rootfs: AssetEntry { + path: "agent-rootfs.tar".to_string(), + size: 0, + }, + layers: Vec::new(), + storage_template: None, + overlay_template: None, + overlay_logical_size: None, + }, + }) + } + + /// Get the staging directory path. + pub fn staging_dir(&self) -> &Path { + &self.staging_dir + } + + /// Discover and copy runtime libraries from the given lib directory. + /// + /// Always copies: + /// - libkrun.dylib / libkrun.so — VM runtime + /// - libkrunfw.5.dylib / libkrunfw.so.5 — kernel firmware + /// + /// Copies when present (GPU passthrough for `gpu = true` guests): + /// - macOS: libvirglrenderer.1.dylib, libMoltenVK.dylib, libepoxy.0.dylib + /// - Linux: libvirglrenderer.so.1, libepoxy.so.0, virgl_render_server binary + /// + /// GPU Vulkan ICDs (ANV, RADV) are hardware-specific and cannot be bundled. + /// When GPU libs are bundled, loading them adds ~3ms overhead even for non-GPU + /// workloads (lib load is unavoidable; virglrenderer init is deferred to GPU use). + pub fn collect_libraries(&mut self, lib_dir: &Path) -> Result<()> { + fs::create_dir_all(self.staging_dir.join("lib"))?; + + let lib_names = if cfg!(target_os = "macos") { + vec!["libkrun.dylib", "libkrunfw.5.dylib"] + } else if cfg!(target_os = "windows") { + // Must match smolvm's loader (util::libkrun_filename): WHP uses the + // Windows DLL names, not the Linux .so names. + vec!["krun.dll", "libkrunfw.dll"] + } else { + vec!["libkrun.so", "libkrunfw.so.5"] + }; + + for name in lib_names { + let src = lib_dir.join(name); + if !src.exists() { + return Err(PackError::AssetNotFound(format!( + "library not found: {}", + src.display() + ))); + } + + let dst = self.staging_dir.join("lib").join(name); + fs::copy(&src, &dst)?; + + let metadata = fs::metadata(&dst)?; + self.inventory.libraries.push(AssetEntry { + path: format!("lib/{}", name), + size: metadata.len(), + }); + } + + // On macOS, bundle GPU rendering libraries when present in the lib dir. + // The virglrenderer chain (Venus/Vulkan) enables hardware-accelerated GPU + // passthrough for guests using virtio-gpu. All paths use @loader_path so + // they resolve relative to where libkrun.dylib is loaded from. + #[cfg(target_os = "macos")] + { + let gpu_libs = [ + "libvirglrenderer.1.dylib", + "libMoltenVK.dylib", + "libepoxy.0.dylib", + ]; + for name in &gpu_libs { + let src = lib_dir.join(name); + if src.exists() { + let dst = self.staging_dir.join("lib").join(name); + fs::copy(&src, &dst)?; + let metadata = fs::metadata(&dst)?; + self.inventory.libraries.push(AssetEntry { + path: format!("lib/{}", name), + size: metadata.len(), + }); + } + } + } + + // On Linux, bundle GPU rendering libraries and render server when present. + // virglrenderer + epoxy enable Venus/Vulkan via virtio-gpu. + // virgl_render_server is the subprocess libkrun spawns during Venus init. + // GPU Vulkan ICDs (ANV, RADV) are hardware-specific and cannot be bundled. + #[cfg(target_os = "linux")] + { + let gpu_libs = ["libvirglrenderer.so.1", "libepoxy.so.0"]; + for name in &gpu_libs { + let src = lib_dir.join(name); + if src.exists() { + let dst = self.staging_dir.join("lib").join(name); + fs::copy(&src, &dst)?; + let metadata = fs::metadata(&dst)?; + self.inventory.libraries.push(AssetEntry { + path: format!("lib/{}", name), + size: metadata.len(), + }); + } + } + let server_src = lib_dir.join("virgl_render_server"); + if server_src.exists() { + let server_dst = self.staging_dir.join("lib").join("virgl_render_server"); + fs::copy(&server_src, &server_dst)?; + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&server_dst, fs::Permissions::from_mode(0o755))?; + let metadata = fs::metadata(&server_dst)?; + self.inventory.libraries.push(AssetEntry { + path: "lib/virgl_render_server".to_string(), + size: metadata.len(), + }); + } + } + + Ok(()) + } + + /// Copy the agent rootfs directory and create a tarball. + pub fn collect_agent_rootfs(&mut self, rootfs_dir: &Path) -> Result<()> { + if !rootfs_dir.exists() { + return Err(PackError::AssetNotFound(format!( + "agent rootfs not found: {}", + rootfs_dir.display() + ))); + } + + let tar_path = self.staging_dir.join("agent-rootfs.tar"); + let tar_file = File::create(&tar_path)?; + let mut tar_builder = tar::Builder::new(BufWriter::new(tar_file)); + + // Don't follow symlinks - preserve them as-is + tar_builder.follow_symlinks(false); + + // The agent-rootfs directory doubles as the virtiofs mount the host + // writes its per-boot readiness markers into (`.smolvm-ready.`, + // see `AGENT_READY_MARKER`). Those are host-side runtime artifacts, not + // part of the guest init system, and they accumulate across boots — and + // a VM that ran under per-VM-uid isolation leaves them owned by a + // foreign uid with mode 0600, unreadable to the packer. `append_dir_all` + // over the whole directory then hard-failed the entire pack with an + // opaque "tar error: Permission denied". Walk the top level instead, + // skip the markers, and name the offending path on any real I/O error. + let mut entries: Vec = + fs::read_dir(rootfs_dir)?.collect::>()?; + // Deterministic ordering so the tar (and its hash) is reproducible. + entries.sort_by_key(|e| e.file_name()); + for entry in entries { + let name = entry.file_name(); + if name + .to_string_lossy() + .starts_with(smolvm_protocol::AGENT_READY_MARKER) + { + continue; + } + let path = entry.path(); + let tar_err = |e: std::io::Error| PackError::Tar(format!("{}: {e}", path.display())); + if entry.file_type()?.is_dir() { + tar_builder.append_dir_all(&name, &path).map_err(tar_err)?; + } else { + // Regular file or symlink (follow_symlinks(false) archives the + // link itself rather than its target). + tar_builder + .append_path_with_name(&path, &name) + .map_err(tar_err)?; + } + } + + tar_builder + .finish() + .map_err(|e| PackError::Tar(e.to_string()))?; + + let metadata = fs::metadata(&tar_path)?; + self.inventory.agent_rootfs = AssetEntry { + path: "agent-rootfs.tar".to_string(), + size: metadata.len(), + }; + + Ok(()) + } + + /// Add an OCI layer tarball. + pub fn add_layer(&mut self, digest: &str, layer_data: &[u8]) -> Result<()> { + let filename = digest_to_filename(digest)?; + let path = format!("layers/{}", filename); + + let dst = self.staging_dir.join(&path); + fs::write(&dst, layer_data)?; + + self.inventory.layers.push(LayerEntry { + digest: digest.to_string(), + path, + size: layer_data.len() as u64, + }); + + Ok(()) + } + + /// Get the staging path where a layer file should be written. + /// + /// Call this before streaming the layer to get the destination path, + /// then call `register_layer()` after writing to register it in the inventory. + pub fn layer_staging_path(&self, digest: &str) -> PathBuf { + let filename = digest_to_filename(digest) + .expect("layer digest must be sha256: with at least 12 hex chars"); + self.staging_dir.join(format!("layers/{}", filename)) + } + + /// Register a layer that was already written to its staging path. + /// + /// Use after streaming a layer directly to `layer_staging_path()`. + pub fn register_layer(&mut self, digest: &str) -> Result<()> { + let filename = digest_to_filename(digest)?; + let path = format!("layers/{}", filename); + let dst = self.staging_dir.join(&path); + + let metadata = fs::metadata(&dst)?; + self.inventory.layers.push(LayerEntry { + digest: digest.to_string(), + path, + size: metadata.len(), + }); + + Ok(()) + } + + /// Add an OCI layer from a file path. + pub fn add_layer_from_file(&mut self, digest: &str, layer_path: &Path) -> Result<()> { + let filename = digest_to_filename(digest)?; + let path = format!("layers/{}", filename); + + let dst = self.staging_dir.join(&path); + fs::copy(layer_path, &dst)?; + + let metadata = fs::metadata(&dst)?; + self.inventory.layers.push(LayerEntry { + digest: digest.to_string(), + path, + size: metadata.len(), + }); + + Ok(()) + } + + /// Create and collect a pre-formatted ext4 storage template. + /// + /// Creates a small sparse ext4 disk image that can be used as a template + /// for the storage disk at runtime. This eliminates the need for mkfs.ext4 + /// on first boot and improves reliability. + /// + /// Tries in order: + /// 1. Copy an existing pre-formatted template from `~/.smolvm/` or next to the exe + /// 2. Format a new one with `mkfs.ext4` (requires e2fsprogs) + /// + /// The template is a 512MB sparse file (actual size ~100KB when empty). + pub fn create_storage_template(&mut self) -> Result<()> { + use std::io::{Seek, SeekFrom, Write}; + use std::process::Command; + + const TEMPLATE_SIZE: u64 = 512 * 1024 * 1024; // 512MB virtual size + const TEMPLATE_NAME: &str = "storage.ext4"; + + let template_path = self.staging_dir.join(TEMPLATE_NAME); + + // Try to copy from an existing pre-formatted template first. + // This avoids requiring e2fsprogs on the build machine. + if let Some(existing) = find_existing_template("storage-template.ext4") { + // Hole-preserving copy: the shipped storage-template.ext4 is a large + // (multi-GiB) sparse file, and a plain fs::copy densifies it into its + // full logical size of zeros on some Linux filesystems/mounts — + // ballooning the staging dir and failing pack builds with ENOSPC. + crate::extract::sparse_copy(&existing, &template_path)?; + let metadata = fs::metadata(&template_path)?; + self.inventory.storage_template = Some(AssetEntry { + path: TEMPLATE_NAME.to_string(), + size: metadata.len(), + }); + return Ok(()); + } + + // No pre-formatted template found — create one with mkfs.ext4. + + // Create sparse file + let mut file = File::create(&template_path)?; + // On Windows/NTFS, File::create makes a dense file; seeking past the end + // and writing a tail byte would allocate every intermediate block. + #[cfg(windows)] + crate::extract::mark_file_sparse(&file)?; + file.seek(SeekFrom::Start(TEMPLATE_SIZE - 1))?; + file.write_all(&[0])?; + file.sync_all()?; + drop(file); + + // Find mkfs.ext4 + let mkfs_paths = [ + "/opt/homebrew/opt/e2fsprogs/sbin/mkfs.ext4", + "/usr/local/opt/e2fsprogs/sbin/mkfs.ext4", + "/opt/homebrew/sbin/mkfs.ext4", + "/usr/local/sbin/mkfs.ext4", + "/sbin/mkfs.ext4", + "/usr/sbin/mkfs.ext4", + "mkfs.ext4", + ]; + + let mkfs_path = mkfs_paths + .iter() + .find(|p| { + if p.contains('/') { + std::path::Path::new(p).exists() + } else { + Command::new(p).arg("--version").output().is_ok() + } + }) + .ok_or_else(|| { + // Windows has no host mkfs.ext4, so point the user at the + // guest-VM recipe for staging a template (the agent rootfs has + // e2fsprogs). On Unix the fix is just installing e2fsprogs. + #[cfg(windows)] + let msg = "storage-template.ext4 not found, and Windows has no host \ + mkfs.ext4 to create one. Format a small template inside a guest VM \ + once and place it next to smolvm.exe (or in %USERPROFILE%\\.smolvm\\):\n \ + smolvm machine create --name mktmpl --volume

:/out\n \ + smolvm machine start --name mktmpl\n \ + smolvm machine exec --name mktmpl -- /bin/busybox sh -c \ + \"truncate -s 512M /out/storage-template.ext4 && \ + mkfs.ext4 -F -q -m 0 /out/storage-template.ext4\"\n \ + smolvm machine delete --name mktmpl --force\n \ + then copy \\storage-template.ext4 next to smolvm.exe"; + #[cfg(not(windows))] + let msg = "mkfs.ext4 not found. Install e2fsprogs or place a pre-formatted \ + storage-template.ext4 in ~/.smolvm/"; + PackError::AssetNotFound(msg.into()) + })?; + + // Format with ext4 + // Reset SIGCHLD to default before spawning to avoid issues after agent stop + #[cfg(unix)] + unsafe { + libc::signal(libc::SIGCHLD, libc::SIG_DFL); + } + + let mut child = Command::new(mkfs_path) + .args([ + "-F", // Force (don't ask) + "-q", // Quiet + "-m", "0", // No reserved blocks + "-L", "smolvm", // Label + ]) + .arg(&template_path) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .map_err(|e| PackError::AssetNotFound(format!("failed to spawn mkfs.ext4: {}", e)))?; + + let status = child.wait().map_err(|e| { + PackError::AssetNotFound(format!("failed to wait for mkfs.ext4: {}", e)) + })?; + + if !status.success() { + return Err(PackError::AssetNotFound( + "mkfs.ext4 failed to format storage template".into(), + )); + } + + // Get actual file size (sparse, so much smaller than 512MB) + let metadata = fs::metadata(&template_path)?; + self.inventory.storage_template = Some(AssetEntry { + path: TEMPLATE_NAME.to_string(), + size: metadata.len(), + }); + + Ok(()) + } + + /// Add an overlay disk template from an existing VM. + /// + /// Copies the VM's overlay disk (overlay.raw) to the staging directory as + /// `overlay.raw`, stripping trailing sparse holes so only actual data bytes + /// enter the tar/zstd pipeline. For a typical 10 GiB overlay with ~50 MB + /// of real ext4 data this reduces pack time by ~100x. + /// + /// The original full size is recorded in `overlay_logical_size` so that + /// the extraction path can restore the sparse skeleton with `ftruncate`. + pub fn add_overlay_template(&mut self, path: &Path) -> Result<()> { + if !path.exists() { + return Err(PackError::AssetNotFound(format!( + "overlay disk not found: {}", + path.display() + ))); + } + + const OVERLAY_NAME: &str = "overlay.raw"; + let dst = self.staging_dir.join(OVERLAY_NAME); + + let (logical_size, truncated_size) = sparse_copy_overlay(path, &dst)?; + + self.inventory.overlay_template = Some(AssetEntry { + path: OVERLAY_NAME.to_string(), + size: truncated_size, + }); + + // Record the original full disk size so extract can restore it. + if logical_size > truncated_size { + self.inventory.overlay_logical_size = Some(logical_size); + } + + Ok(()) + } + + /// Get the current asset inventory. + pub fn inventory(&self) -> &AssetInventory { + &self.inventory + } + + /// Consume the collector and return the final inventory. + pub fn into_inventory(self) -> AssetInventory { + self.inventory + } + + /// Compress staged assets into a single zstd-compressed tarball. + /// + /// When `exclude_libs` is true, the `lib/` directory is excluded + /// (two-file mode: libs are embedded in the stub binary instead). + /// When false, everything is included (single-file mode). + pub fn compress(&self, output: &Path, exclude_libs: bool) -> Result { + let output_file = File::create(output)?; + let encoder = zstd::stream::Encoder::new(output_file, ZSTD_LEVEL) + .map_err(|e| PackError::Compression(e.to_string()))?; + let mut tar_builder = tar::Builder::new(encoder); + + // Sort entries for deterministic tar ordering (consistent checksums) + let mut entries: Vec<_> = fs::read_dir(&self.staging_dir)? + .filter_map(|e| e.ok()) + .collect(); + entries.sort_by_key(|e| e.file_name()); + + for entry in entries { + let name = entry.file_name(); + if exclude_libs && name == "lib" { + continue; // libs go in the stub, not the sidecar + } + let path = entry.path(); + if path.is_dir() { + tar_builder + .append_dir_all(name.to_string_lossy().as_ref(), &path) + .map_err(|e| PackError::Tar(e.to_string()))?; + } else { + tar_builder + .append_path_with_name(&path, name.to_string_lossy().as_ref()) + .map_err(|e| PackError::Tar(e.to_string()))?; + } + } + + let encoder = tar_builder + .into_inner() + .map_err(|e| PackError::Tar(e.to_string()))?; + encoder + .finish() + .map_err(|e| PackError::Compression(e.to_string()))?; + + let metadata = fs::metadata(output)?; + Ok(metadata.len()) + } +} + +// ============================================================================= +// Sparse-aware overlay copy +// ============================================================================= + +/// Copy a sparse overlay disk to `dst`, stripping trailing zeros. +/// +/// Scans backwards from the end of the source to find the last non-zero byte, +/// then copies only bytes `[0, last_nonzero+1]` to `dst`, skipping zero +/// chunks in the forward pass. +/// +/// `SEEK_DATA`/`SEEK_HOLE` is avoided because APFS reports zero-fill extents +/// (efficiently stored but containing zeros) as "data", making lseek-based +/// hole detection return the full file size even when 90%+ is zeros. +/// Content scanning works correctly on both APFS and Linux ext4/xfs. +/// +/// Returns the original full (logical) size so the caller can record it for +/// the extraction side to restore the sparse skeleton via `ftruncate`. +/// Returns `(logical_size, truncated_size)`. +fn sparse_copy_overlay(src: &Path, dst: &Path) -> std::io::Result<(u64, u64)> { + let mut src_file = File::open(src)?; + let logical_size = src_file.metadata()?.len(); + + // Scan backwards to find the last non-zero byte (= safe truncation point). + // On APFS, zero-fill regions are served from page cache without disk I/O, + // so even scanning 8+ GiB of trailing zeros takes only ~100–200 ms. + let truncated_size = find_last_data_byte(&mut src_file, logical_size)?; + + // Create destination as a sparse skeleton; keep the handle for writing. + let mut dst_file = File::create(dst)?; + // On Windows/NTFS, File::create makes a non-sparse file: set_len-ing to the + // (large) truncated size and then writing a chunk at a high offset would + // zero-fill/allocate the entire gap, ballooning a ~50 MB overlay to ~10 GiB + // of real disk. Mark it sparse first so only written extents consume space — + // matching the implicit sparse behavior of Unix filesystems. + #[cfg(windows)] + crate::extract::mark_file_sparse(&dst_file)?; + dst_file.set_len(truncated_size)?; + + if truncated_size == 0 { + return Ok((logical_size, 0)); + } + + // Forward copy: read [0, truncated_size) in 512 KiB chunks, + // writing only non-zero chunks (zero chunks remain as holes). + src_file.seek(SeekFrom::Start(0))?; + let mut buf = vec![0u8; 512 * 1024]; + let mut offset: u64 = 0; + + while offset < truncated_size { + let to_read = (truncated_size - offset).min(buf.len() as u64) as usize; + let n = src_file.read(&mut buf[..to_read])?; + if n == 0 { + break; + } + let chunk = &buf[..n]; + if chunk.iter().any(|&b| b != 0) { + dst_file.seek(SeekFrom::Start(offset))?; + dst_file.write_all(chunk)?; + } + offset += n as u64; + } + + Ok((logical_size, truncated_size)) +} + +/// Find the truncation point: offset of the last non-zero byte + 1. +/// +/// Reads the file backwards in 1 MiB chunks until a non-zero byte is found. +/// Returns 0 if the entire file is zeros. +fn find_last_data_byte(file: &mut File, logical_size: u64) -> std::io::Result { + if logical_size == 0 { + return Ok(0); + } + + const CHUNK: u64 = 1024 * 1024; // 1 MiB scan chunk + let mut buf = vec![0u8; CHUNK as usize]; + let mut pos = logical_size; + + while pos > 0 { + let chunk_start = pos.saturating_sub(CHUNK); + let chunk_size = (pos - chunk_start) as usize; + + file.seek(SeekFrom::Start(chunk_start))?; + let n = file.read(&mut buf[..chunk_size])?; + if n == 0 { + break; + } + + // Scan backwards for the last non-zero byte in this chunk. + for i in (0..n).rev() { + if buf[i] != 0 { + return Ok(chunk_start + i as u64 + 1); + } + } + + pos = chunk_start; + } + + Ok(0) // Entire file is zeros +} + +/// Decompress a zstd-compressed assets blob. +pub fn decompress_assets(compressed: &[u8], output_dir: &Path) -> Result<()> { + fs::create_dir_all(output_dir)?; + + let decoder = zstd::stream::Decoder::new(compressed) + .map_err(|e| PackError::Compression(e.to_string()))?; + let mut archive = tar::Archive::new(decoder); + + archive + .unpack(output_dir) + .map_err(|e| PackError::Tar(e.to_string()))?; + + Ok(()) +} + +/// Decompress assets from a file. +pub fn decompress_assets_from_file(compressed_path: &Path, output_dir: &Path) -> Result<()> { + fs::create_dir_all(output_dir)?; + + let file = File::open(compressed_path)?; + let decoder = + zstd::stream::Decoder::new(file).map_err(|e| PackError::Compression(e.to_string()))?; + let mut archive = tar::Archive::new(decoder); + + archive + .unpack(output_dir) + .map_err(|e| PackError::Tar(e.to_string()))?; + + Ok(()) +} + +/// Calculate CRC32 checksum of data. +pub fn crc32(data: &[u8]) -> u32 { + crc32fast::hash(data) +} + +/// Calculate CRC32 checksum of a file. +pub fn crc32_file(path: &Path) -> Result { + let mut file = File::open(path)?; + let mut hasher = crc32fast::Hasher::new(); + + let mut buf = [0u8; 64 * 1024]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + + Ok(hasher.finalize()) +} + +/// Calculate CRC32 checksum of multiple sections of a file. +pub fn crc32_file_range(path: &Path, offset: u64, size: u64) -> Result { + use std::io::{Seek, SeekFrom}; + + let mut file = File::open(path)?; + file.seek(SeekFrom::Start(offset))?; + + let mut hasher = crc32fast::Hasher::new(); + let mut remaining = size; + let mut buf = [0u8; 64 * 1024]; + + while remaining > 0 { + let to_read = remaining.min(buf.len() as u64) as usize; + let n = file.read(&mut buf[..to_read])?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + remaining -= n as u64; + } + + Ok(hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn digest_to_filename_rejects_path_traversal() { + // A valid hex digest is accepted. + assert_eq!( + digest_to_filename("sha256:abcdef012345").unwrap(), + "abcdef012345.tar" + ); + // Non-hex / traversal digests are rejected before becoming a path. + for bad in [ + "sha256:../../../../etc/evil", + "sha256:..%2f..%2fevil", + "sha256:abc/def/ghij", + "sha256:abcdefabcdeZ", + ] { + assert!( + digest_to_filename(bad).is_err(), + "should reject non-hex digest: {bad}" + ); + } + } + + #[test] + fn test_find_last_data_byte_all_zero() { + let temp = tempfile::NamedTempFile::new().unwrap(); + fs::write(temp.path(), vec![0u8; 4096]).unwrap(); + let mut file = File::open(temp.path()).unwrap(); + assert_eq!(find_last_data_byte(&mut file, 4096).unwrap(), 0); + } + + #[test] + fn test_find_last_data_byte_trailing_zeros() { + let temp = tempfile::NamedTempFile::new().unwrap(); + let mut data = vec![0u8; 4100]; + data[99] = 0xAB; // non-zero at 99, then 4000 trailing zeros + fs::write(temp.path(), &data).unwrap(); + let mut file = File::open(temp.path()).unwrap(); + assert_eq!(find_last_data_byte(&mut file, 4100).unwrap(), 100); + } + + #[test] + fn test_find_last_data_byte_nonzero_at_end() { + // Boundary: no trailing zeros — function must return full length. + let temp = tempfile::NamedTempFile::new().unwrap(); + let mut data = vec![0u8; 1024]; + data[1023] = 1; + fs::write(temp.path(), &data).unwrap(); + let mut file = File::open(temp.path()).unwrap(); + assert_eq!(find_last_data_byte(&mut file, 1024).unwrap(), 1024); + } + + #[test] + fn test_sparse_copy_overlay_all_zero() { + // All-zero source: truncated_size=0, early exit before forward copy. + let temp_dir = tempfile::tempdir().unwrap(); + let src = temp_dir.path().join("src.raw"); + let dst = temp_dir.path().join("dst.raw"); + fs::write(&src, vec![0u8; 8192]).unwrap(); + + let (logical, truncated) = sparse_copy_overlay(&src, &dst).unwrap(); + assert_eq!(logical, 8192); + assert_eq!(truncated, 0); + assert_eq!(fs::metadata(&dst).unwrap().len(), 0); + } + + #[test] + fn test_sparse_copy_overlay_trailing_zeros_and_interior_holes() { + // Verifies: trailing zeros are stripped, sizes are returned, interior + // holes (zero chunks in the forward pass) are preserved in the copy. + let temp_dir = tempfile::tempdir().unwrap(); + let src = temp_dir.path().join("src.raw"); + let dst = temp_dir.path().join("dst.raw"); + + let mut data = vec![0u8; 8192]; + data[0] = 0x01; // non-zero at start + data[511] = 0xFF; // last non-zero byte; trailing 7680 bytes are zeros + fs::write(&src, &data).unwrap(); + + let (logical, truncated) = sparse_copy_overlay(&src, &dst).unwrap(); + assert_eq!(logical, 8192); + assert_eq!(truncated, 512); + + let dst_data = fs::read(&dst).unwrap(); + assert_eq!(dst_data[0], 0x01); + assert_eq!(dst_data[511], 0xFF); + assert_eq!(dst_data[256], 0x00); // interior zero is preserved + } + + #[test] + fn test_crc32_basic() { + let data = b"hello world"; + let checksum = crc32(data); + assert_eq!(checksum, 0x0D4A_1185); // Known CRC32 value + } + + #[test] + fn test_crc32_empty() { + let data = b""; + let checksum = crc32(data); + assert_eq!(checksum, 0); // CRC32 of empty data is 0 + } + + #[test] + fn test_asset_collector_staging() { + let temp_dir = tempfile::tempdir().unwrap(); + let staging = temp_dir.path().join("staging"); + + let _collector = AssetCollector::new(staging.clone()).unwrap(); + + // lib/ is only created when collect_libraries() is called + assert!(!staging.join("lib").exists()); + assert!(staging.join("layers").exists()); + } + + #[test] + fn collect_agent_rootfs_excludes_ready_markers() { + // The agent-rootfs dir doubles as the host's per-boot readiness-marker + // mount (`.smolvm-ready.`). Those markers must never be packed + // into the guest image — and on a host that ran uid-isolated VMs they + // can be foreign-owned/unreadable, which used to hard-fail the whole + // pack ("tar error: Permission denied"). Verify they're skipped while + // real rootfs content is preserved. + let temp = tempfile::tempdir().unwrap(); + let rootfs = temp.path().join("rootfs"); + fs::create_dir_all(rootfs.join("bin")).unwrap(); + fs::create_dir_all(rootfs.join("etc")).unwrap(); + fs::write(rootfs.join("bin/sh"), b"#!/bin/sh\n").unwrap(); + fs::write(rootfs.join("etc/hostname"), b"vm\n").unwrap(); + fs::write(rootfs.join("init"), b"agent").unwrap(); + fs::write( + rootfs.join(format!("{}.deadbeef", smolvm_protocol::AGENT_READY_MARKER)), + b"1", + ) + .unwrap(); + fs::write( + rootfs.join(format!("{}.cafef00d", smolvm_protocol::AGENT_READY_MARKER)), + b"1", + ) + .unwrap(); + + let staging = temp.path().join("staging"); + let mut collector = AssetCollector::new(staging.clone()).unwrap(); + collector.collect_agent_rootfs(&rootfs).unwrap(); + + let tar_path = staging.join("agent-rootfs.tar"); + let names: Vec = tar::Archive::new(File::open(&tar_path).unwrap()) + .entries() + .unwrap() + .map(|e| e.unwrap().path().unwrap().to_string_lossy().into_owned()) + .collect(); + + assert!( + names.iter().any(|n| n.ends_with("bin/sh")), + "real rootfs file missing from pack: {names:?}" + ); + assert!( + names.iter().any(|n| n.ends_with("etc/hostname")), + "real rootfs file missing from pack: {names:?}" + ); + assert!( + names.iter().any(|n| n.ends_with("init")), + "top-level agent binary missing from pack: {names:?}" + ); + assert!( + !names.iter().any(|n| n.contains(".smolvm-ready")), + "readiness marker leaked into the pack: {names:?}" + ); + } + + #[test] + fn test_compression_roundtrip() { + let temp_dir = tempfile::tempdir().unwrap(); + let staging = temp_dir.path().join("staging"); + let output = temp_dir.path().join("output"); + + // Create a file in staging + fs::create_dir_all(&staging).unwrap(); + let test_file = staging.join("test.txt"); + fs::write(&test_file, b"hello world").unwrap(); + + // Create collector and compress + let collector = AssetCollector::new(staging).unwrap(); + let compressed = temp_dir.path().join("assets.tar.zst"); + collector.compress(&compressed, false).unwrap(); + + // Decompress and verify + decompress_assets_from_file(&compressed, &output).unwrap(); + let restored = output.join("test.txt"); + assert!(restored.exists()); + assert_eq!(fs::read_to_string(&restored).unwrap(), "hello world"); + } +} diff --git a/crates/smolvm-pack/src/detect.rs b/crates/smolvm-pack/src/detect.rs new file mode 100644 index 0000000..a8d939e --- /dev/null +++ b/crates/smolvm-pack/src/detect.rs @@ -0,0 +1,257 @@ +//! Packed binary auto-detection. +//! +//! Determines whether the current executable is a packed binary and what mode +//! it is running in: section-embedded (macOS), append-embedded, sidecar, or +//! normal CLI. +//! +//! Called at the very start of `main()` before clap parsing so that a packed +//! binary (e.g. `./my-app echo hello`) shows packed-binary help rather +//! than the full smolvm CLI. + +use crate::format::{PackFooter, FOOTER_SIZE}; +use crate::packer::{read_footer_from_sidecar, sidecar_path_for}; +use std::path::{Path, PathBuf}; + +/// The detected packed binary mode. +pub enum PackedMode { + /// Assets embedded in a Mach-O `__SMOLVM,__smolvm` section (macOS single-file). + #[cfg(target_os = "macos")] + Section { + /// Parsed manifest from the section. + manifest: Box, + /// CRC32 checksum from the section header. + checksum: u32, + /// Pointer to compressed assets in the section (valid for process lifetime). + assets_ptr: *const u8, + /// Size of compressed assets. + assets_size: usize, + }, + /// Assets appended to the binary (Linux single-file, or macOS fallback). + Embedded { + /// Path to the current executable. + exe_path: PathBuf, + /// Footer read from end of binary. + footer: PackFooter, + }, + /// Assets in a `.smolmachine` sidecar file alongside the binary. + Sidecar { + /// Path to the `.smolmachine` file. + sidecar_path: PathBuf, + /// Footer read from end of sidecar. + footer: PackFooter, + }, +} + +/// Detect whether this process is running as a packed binary. +/// +/// Checks in order: +/// 1. macOS Mach-O section (`__SMOLVM,__smolvm`) with `SMOLSECT` magic +/// 2. `.smolmachine` sidecar file with valid footer +/// 3. `SMOLPACK` footer appended to own binary with `assets_offset > 0` +/// +/// Returns `None` for normal `smolvm` invocations (no false positives). +pub fn detect_packed_mode() -> Option { + let exe_path = std::env::current_exe().ok()?; + + // 1. macOS section mode + #[cfg(target_os = "macos")] + { + if let Some(mode) = try_section_mode() { + return Some(mode); + } + } + + // 2. Sidecar mode: .smolmachine next to binary + if let Some(mode) = try_sidecar_mode(&exe_path) { + return Some(mode); + } + + // 3. Embedded-append mode: SMOLPACK footer at end of own binary + if let Some(mode) = try_embedded_mode(&exe_path) { + return Some(mode); + } + + None +} + +fn try_sidecar_mode(exe_path: &Path) -> Option { + let sidecar = sidecar_path_for(exe_path); + if !sidecar.exists() { + return None; + } + let footer = read_footer_from_sidecar(&sidecar).ok()?; + Some(PackedMode::Sidecar { + sidecar_path: sidecar, + footer, + }) +} + +fn try_embedded_mode(exe_path: &Path) -> Option { + let footer = read_footer_direct(exe_path).ok()?; + // assets_offset > 0 means assets are appended after the stub in the binary + if footer.assets_offset > 0 { + Some(PackedMode::Embedded { + exe_path: exe_path.to_path_buf(), + footer, + }) + } else { + None + } +} + +/// Read footer directly from end of file without trying sidecar first. +/// +/// Unlike `packer::read_footer()` which checks for a sidecar, this reads +/// the last 64 bytes of the given file and parses them as a `PackFooter`. +fn read_footer_direct(path: &Path) -> crate::Result { + use std::fs::File; + use std::io::{Read, Seek, SeekFrom}; + + let mut file = File::open(path)?; + let file_size = file.metadata()?.len(); + if file_size < FOOTER_SIZE as u64 { + return Err(crate::PackError::InvalidMagic); + } + file.seek(SeekFrom::End(-(FOOTER_SIZE as i64)))?; + let mut buf = [0u8; FOOTER_SIZE]; + file.read_exact(&mut buf)?; + PackFooter::from_bytes(&buf) +} + +// --------------------------------------------------------------------------- +// macOS Mach-O section reading +// --------------------------------------------------------------------------- + +#[cfg(target_os = "macos")] +fn try_section_mode() -> Option { + let embedded = read_embedded_section()?; + Some(PackedMode::Section { + manifest: Box::new(embedded.manifest), + checksum: embedded.header.checksum, + assets_ptr: embedded.assets_ptr, + assets_size: embedded.assets_size, + }) +} + +/// Data read from the `__SMOLVM,__smolvm` Mach-O section. +#[cfg(target_os = "macos")] +struct EmbeddedData { + header: crate::format::SectionHeader, + manifest: crate::format::PackManifest, + assets_ptr: *const u8, + assets_size: usize, +} + +/// Try to read embedded data from the `__SMOLVM,__smolvm` Mach-O section. +/// +/// Returns `None` if: +/// - Not running on macOS +/// - Section doesn't exist +/// - Section contains only the build-time placeholder (not `SMOLSECT` magic) +#[cfg(target_os = "macos")] +fn read_embedded_section() -> Option { + use crate::format::{PackManifest, SectionHeader, SECTION_HEADER_SIZE, SECTION_MAGIC}; + + extern "C" { + fn getsectiondata( + mhp: *const MachHeader64, + segname: *const i8, + sectname: *const i8, + size: *mut usize, + ) -> *const u8; + } + + #[repr(C)] + struct MachHeader64 { + magic: u32, + cputype: i32, + cpusubtype: i32, + filetype: u32, + ncmds: u32, + sizeofcmds: u32, + flags: u32, + reserved: u32, + } + + extern "C" { + fn _dyld_get_image_header(image_index: u32) -> *const MachHeader64; + } + + unsafe { + let header = _dyld_get_image_header(0); + if header.is_null() { + return None; + } + + let segname = c"__SMOLVM"; + let sectname = c"__smolvm"; + let mut size: usize = 0; + + let data_ptr = getsectiondata(header, segname.as_ptr(), sectname.as_ptr(), &mut size); + + if data_ptr.is_null() || size < SECTION_HEADER_SIZE { + return None; + } + + // Check for SMOLSECT magic (placeholder contains different marker) + let magic_bytes = std::slice::from_raw_parts(data_ptr, 8); + if magic_bytes != SECTION_MAGIC { + return None; + } + + // Parse section header + let header_bytes = std::slice::from_raw_parts(data_ptr, SECTION_HEADER_SIZE); + let section_header = SectionHeader::from_bytes(header_bytes).ok()?; + + // Validate sizes + let expected_size = SECTION_HEADER_SIZE + + section_header.manifest_size as usize + + section_header.assets_size as usize; + if size < expected_size { + return None; + } + + // Parse manifest + let manifest_start = data_ptr.add(SECTION_HEADER_SIZE); + let manifest_bytes = + std::slice::from_raw_parts(manifest_start, section_header.manifest_size as usize); + let manifest = PackManifest::from_json(manifest_bytes).ok()?; + + // Assets follow the manifest + let assets_ptr = manifest_start.add(section_header.manifest_size as usize); + + Some(EmbeddedData { + header: section_header, + manifest, + assets_ptr, + assets_size: section_header.assets_size as usize, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_detect_returns_none_for_normal_binary() { + // The test binary is a normal Rust executable, not a packed binary. + assert!(detect_packed_mode().is_none()); + } + + #[test] + fn test_read_footer_direct_rejects_short_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("short.bin"); + std::fs::write(&path, b"too short").unwrap(); + assert!(read_footer_direct(&path).is_err()); + } + + #[test] + fn test_read_footer_direct_rejects_invalid_magic() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("no_magic.bin"); + std::fs::write(&path, [0u8; 128]).unwrap(); + assert!(read_footer_direct(&path).is_err()); + } +} diff --git a/crates/smolvm-pack/src/extract.rs b/crates/smolvm-pack/src/extract.rs new file mode 100644 index 0000000..1e81c93 --- /dev/null +++ b/crates/smolvm-pack/src/extract.rs @@ -0,0 +1,3177 @@ +//! Asset extraction for packed binaries. +//! +//! Provides shared extraction logic used by both the main `smolvm` binary +//! (sidecar mode via `runpack`) and the standalone stub executable. + +use crate::format::{PackFooter, SIDECAR_EXTENSION}; +use std::fs::{self, File}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; +#[cfg(unix)] +use std::os::unix::io::AsRawFd; + +/// Mark an open file as sparse (Windows/NTFS) so a later `set_len` to a large +/// size — or writing chunks at high offsets after such a `set_len` — doesn't +/// allocate every intermediate block, ballooning a sparse disk image (overlay, +/// storage) to its full logical size on disk. Unix filesystems are sparse by +/// default and never call this. +/// +/// `smolvm-pack` cannot depend on the main crate, so this mirrors the FSCTL +/// approach in the main crate's `src/disk_utils.rs::mark_file_sparse`. +#[cfg(windows)] +pub(crate) fn mark_file_sparse(file: &fs::File) -> std::io::Result<()> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::IO::DeviceIoControl; + // FSCTL_SET_SPARSE control code (winioctl.h). + const FSCTL_SET_SPARSE: u32 = 0x000900C4; + let mut returned: u32 = 0; + // SAFETY: `file` is a valid open handle; FSCTL_SET_SPARSE uses no in/out buffers. + let ok = unsafe { + DeviceIoControl( + file.as_raw_handle(), + FSCTL_SET_SPARSE, + std::ptr::null(), + 0, + std::ptr::null_mut(), + 0, + &mut returned, + std::ptr::null_mut(), + ) + }; + if ok == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +/// Acquire a blocking, exclusive advisory lock on an open lock file. +/// +/// Unix uses `flock(LOCK_EX)`; Windows uses `LockFileEx(LOCKFILE_EXCLUSIVE_LOCK)` +/// on the file handle. Without the Windows path, concurrent first-run +/// extractions of the same checksum race. The lock is released when the OS +/// closes the handle (i.e. when the `File` is dropped). +#[cfg(unix)] +fn lock_file_exclusive(lock_file: &fs::File) -> std::io::Result<()> { + let ret = unsafe { libc::flock(lock_file.as_raw_fd(), libc::LOCK_EX) }; + if ret != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +#[cfg(windows)] +fn lock_file_exclusive(lock_file: &fs::File) -> std::io::Result<()> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Storage::FileSystem::{LockFileEx, LOCKFILE_EXCLUSIVE_LOCK}; + use windows_sys::Win32::System::IO::OVERLAPPED; + + let handle = lock_file.as_raw_handle() as windows_sys::Win32::Foundation::HANDLE; + let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; + // Lock the whole (empty) file: offset 0, maximum byte range. + let ret = unsafe { + LockFileEx( + handle, + LOCKFILE_EXCLUSIVE_LOCK, + 0, + u32::MAX, + u32::MAX, + &mut overlapped, + ) + }; + if ret == 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +/// Set a Unix file mode on `path`, ignoring errors. No-op on non-Unix targets +/// (Windows has no POSIX mode bits). +#[inline] +fn set_mode(path: &Path, mode: u32) { + #[cfg(unix)] + { + let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)); + } + #[cfg(not(unix))] + { + let _ = (path, mode); + } +} + +/// Files larger than this threshold are extracted with a sparse write +/// (ftruncate skeleton + pwrite only non-zero 64 KiB chunks) rather than a +/// dense sequential write. Chosen to match typical overlay disk sizes while +/// staying well above any regular asset file. +const SPARSE_WRITE_THRESHOLD: u64 = 256 * 1024 * 1024; // 256 MiB + +/// Extract a single tar entry as a sparse file. +/// +/// Creates the destination with `ftruncate(entry_size)` so the OS allocates +/// no disk blocks for the zero regions, then streams `entry` in 64 KiB +/// chunks and `pwrite`s only the non-zero ones at their correct offsets. +/// +/// This keeps a 10 GiB overlay disk (with ~50 MB of real data) from +/// materialising as a dense file during sidecar extraction. +fn unpack_sparse( + entry: &mut tar::Entry, + path: &Path, + entry_size: u64, + mode: u32, + real_dest: &Path, +) -> std::io::Result<()> { + // Before creating any directory or opening the file, verify that the real + // (symlink-resolved) parent of `path` stays within `real_dest`. A prior + // tar entry may have placed a symlink parent component (e.g. `foo -> /`) + // that `create_dir_all`/`open` would traverse OUT of the extraction root, + // writing an attacker-controlled host file as root. `O_NOFOLLOW` below only + // guards the FINAL path component, not an escaping parent — this closes + // that gap. (The dense write path gets the same guarantee from the tar + // crate's `validate_inside_dst`, which the sparse path bypasses.) + verify_parent_within_dest(path, real_dest)?; + + // Ensure the parent directory exists (mirrors what entry.unpack_in does). + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + // Reject symlinks and unexpected directories at the destination. + // A prior tar entry may have placed an intra-dest relative symlink at this + // path; File::create would follow it, redirecting writes to the symlink + // target instead of the intended path. + match path.symlink_metadata() { + Ok(meta) if meta.file_type().is_symlink() => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("unpack_sparse: symlink at destination: {}", path.display()), + )); + } + Ok(meta) if meta.file_type().is_dir() => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "unpack_sparse: directory at destination: {}", + path.display() + ), + )); + } + Ok(_) => { + // Regular file: remove it so create_new (O_CREAT|O_EXCL) succeeds. + // This handles idempotent re-extraction without silently overwriting. + fs::remove_file(path)?; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + + // Open with O_CREAT|O_EXCL|O_NOFOLLOW: rejects any symlink placed in the + // TOCTOU window between the check above and the open (defense in depth). + #[cfg(unix)] + let mut file = { + use std::os::unix::fs::OpenOptionsExt; + fs::OpenOptions::new() + .write(true) + .create_new(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path)? + }; + #[cfg(not(unix))] + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path)?; + + // On Windows/NTFS the file is dense by default: set_len to the (large) + // logical size and pwriting non-zero chunks at high offsets would allocate + // every hole, materialising a 10 GiB overlay even though only ~50 MB is + // real data. Mark it sparse first. + #[cfg(windows)] + mark_file_sparse(&file)?; + + // ftruncate: on APFS and ext4 this allocates zero disk blocks for the + // hole regions — only written bytes consume real space. + file.set_len(entry_size)?; + + let mut offset: u64 = 0; + let mut buf = vec![0u8; 64 * 1024]; + + loop { + let n = entry.read(&mut buf)?; + if n == 0 { + break; + } + let chunk = &buf[..n]; + if chunk.iter().any(|&b| b != 0) { + file.seek(SeekFrom::Start(offset))?; + file.write_all(chunk)?; + } + offset += n as u64; + } + + // Only file mode is restored, not timestamps, uid/gid, or xattrs. + // unpack_sparse applies to large cache assets (overlay disks, storage + // images) extracted to a host-local cache directory; the extra metadata + // does not affect functionality for those assets. + // + // Mask to the low permission bits (`& 0o777`) so a hostile header can't + // preserve setuid/setgid/sticky bits on a host-extracted file — mirroring + // the dense path, which never carries those bits through. + #[cfg(unix)] + fs::set_permissions(path, fs::Permissions::from_mode(mode & 0o777))?; + #[cfg(not(unix))] + let _ = mode; + + Ok(()) +} + +/// Verify that the real (symlink-resolved) parent directory of `path` stays +/// within `real_dest`. +/// +/// `canonicalize` resolves every symlink component, so if a prior tar entry +/// planted a symlink parent (e.g. `foo -> /`) that would let a later write +/// escape the extraction root, the deepest existing ancestor resolves to a +/// path outside `real_dest` and we reject before any directory is created or +/// file opened. `real_dest` must itself be a canonicalized path (so the +/// `starts_with` comparison is apples-to-apples, e.g. macOS `/tmp` → +/// `/private/tmp`). +fn verify_parent_within_dest(path: &Path, real_dest: &Path) -> std::io::Result<()> { + let Some(parent) = path.parent() else { + return Ok(()); + }; + // Walk up until we hit a component that already exists on disk; canonicalize + // it (following all symlinks) and require it to be inside real_dest. + for ancestor in parent.ancestors() { + match ancestor.canonicalize() { + Ok(real) => { + if real.starts_with(real_dest) { + return Ok(()); + } + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "resolved parent '{}' of '{}' escapes destination '{}'", + real.display(), + path.display(), + real_dest.display() + ), + )); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => return Err(e), + } + } + Ok(()) +} + +/// Safely unpack a tar archive, rejecting symlinks, hardlinks, and entries +/// that resolve outside `dest`. +/// +/// The standard `tar::Archive::unpack()` strips `..` components but does +/// **not** reject symlinks. A crafted archive could create +/// `lib/libkrun.dylib → /tmp/evil.so`, and subsequent `dlopen()` would +/// load the attacker's library. This function rejects any entry that is +/// not a regular file or directory. +fn safe_unpack(archive: &mut tar::Archive, dest: &Path) -> std::io::Result<()> { + safe_unpack_with_limits(archive, dest, &SafeUnpackLimits::from_env()) +} + +/// Tunable resource ceilings and sparse-write threshold for [`safe_unpack`]. +/// Extracted into a struct so tests can inject small values (exercising the +/// sparse-write path and the cap-rejection paths) without mutating process-wide +/// env vars, which would race with other concurrently-running tests. +#[derive(Clone, Copy)] +struct SafeUnpackLimits { + /// Max number of tar entries before erroring (inode-flood guard). + max_entries: u64, + /// Max total apparent (header-declared) bytes before erroring + /// (disk-exhaustion / decompression-bomb guard). + max_total_bytes: u64, + /// Regular files with a header size >= this use the sparse-write path. + sparse_threshold: u64, +} + +impl SafeUnpackLimits { + fn from_env() -> Self { + Self { + max_entries: max_extract_entries(), + max_total_bytes: max_extract_total_bytes(), + sparse_threshold: SPARSE_WRITE_THRESHOLD, + } + } +} + +fn safe_unpack_with_limits( + archive: &mut tar::Archive, + dest: &Path, + limits: &SafeUnpackLimits, +) -> std::io::Result<()> { + // Use `normalize_path` (not `canonicalize`) for the containment base so it + // matches the per-entry `normalized` paths, which are built from this same + // plain `dest`. On Windows `canonicalize` returns a `\\?\`-verbatim path + // while the entry paths stay plain, so `starts_with` would reject every + // entry; `normalize_path` (which still resolves `..`, preserving the + // traversal defense) keeps both sides in the same form on all platforms. + let canonical_dest = normalize_path(dest); + + // Real (symlink-resolved) destination, used only for the sparse-write + // parent-escape check. `dest` exists by the time we get here (callers + // `create_dir_all` it first), so canonicalize succeeds; fall back to the + // normalized form if not. Kept separate from `canonical_dest` because the + // per-entry containment check compares against plain-joined paths. + let real_dest = dest + .canonicalize() + .unwrap_or_else(|_| canonical_dest.clone()); + + // Bound total work so a hostile layer can't exhaust host disk/inodes across + // co-tenants (decompression bomb / entry flood). Counts apparent (header) + // sizes and entries; both have generous finite ceilings, overridable via + // env for constrained hosts / tests. + let max_entries = limits.max_entries; + let max_total_bytes = limits.max_total_bytes; + let mut entry_count: u64 = 0; + let mut total_bytes: u64 = 0; + + // Track directories with restrictive permissions. We extract all entries + // with directories temporarily set to 0o755, then apply final permissions + // after all children are written. This matches GNU tar / bsdtar behavior + // and prevents extraction failures when a read-only directory appears + // before its children in the tar stream (e.g., Fedora's mode-555 + // /usr/lib64/pm-utils/*.d directories). + let mut deferred_dir_modes: Vec<(PathBuf, u32)> = Vec::new(); + + for entry_result in archive.entries()? { + let mut entry = entry_result?; + let entry_type = entry.header().entry_type(); + let entry_path = entry.path()?.to_path_buf(); + + // Enforce the entry-count and total-bytes ceilings (Fix 3): reject an + // archive that would flood inodes or exhaust disk before we write it. + entry_count += 1; + if entry_count > max_entries { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("tar archive exceeds max entry count ({max_entries})"), + )); + } + total_bytes = total_bytes.saturating_add(entry.header().size().unwrap_or(0)); + if total_bytes > max_total_bytes { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("tar archive exceeds max total size ({max_total_bytes} bytes)"), + )); + } + + match entry_type { + tar::EntryType::Regular + | tar::EntryType::GNUSparse + | tar::EntryType::Directory + | tar::EntryType::Continuous => {} + // GNU/PAX extension headers are metadata for the next entry. + // The tar crate normally consumes them internally, but some + // archives surface them as explicit entries. Skip them. + tar::EntryType::GNULongName + | tar::EntryType::GNULongLink + | tar::EntryType::XGlobalHeader + | tar::EntryType::XHeader => { + continue; + } + tar::EntryType::Symlink => { + // Allow symlinks but validate the target stays within dest. + if let Some(link_target) = entry.link_name()? { + let link_target = link_target.to_path_buf(); + // tar targets use Unix semantics: an absolute target starts + // with '/'. `Path::is_absolute` is false for those on Windows + // (no drive), and `Path::join` would then treat the leading + // slash as a root that wipes `dest`, so detect it by string. + let target_str = link_target.to_string_lossy(); + // Resolve relative symlinks against the entry's parent dir + let resolved = if target_str.starts_with('/') { + // Absolute symlinks: jail to dest (e.g., /lib/foo → dest/lib/foo) + dest.join(target_str.trim_start_matches('/')) + } else { + let parent = entry_path.parent().unwrap_or(Path::new("")); + dest.join(parent).join(&link_target) + }; + // Normalize the path by resolving .. components + let normalized = normalize_path(&resolved); + if !normalized.starts_with(&canonical_dest) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "tar symlink '{}' -> '{}' escapes destination directory", + entry_path.display(), + link_target.display() + ), + )); + } + // Root-cause guard (Fix 1b): reject an ABSOLUTE symlink that + // resolves to the destination ROOT itself (e.g. `foo -> /`, + // `x -> /..`). The jailed check above passes these (they + // "stay within dest" only because the jail rewrite collapses + // them onto dest), but on disk the symlink is created with + // its LITERAL target, turning `foo` into an alias for the + // real filesystem root — the exact primitive a later + // `foo/etc/...` entry uses to escape. A legitimate absolute + // symlink points to a sub-path (`/usr/lib/y`), never at the + // root, so this preserves in-image absolute links. + if target_str.starts_with('/') && normalized == canonical_dest { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "tar symlink '{}' -> '{}' aliases the destination root", + entry_path.display(), + link_target.display() + ), + )); + } + } + } + tar::EntryType::Link => { + // Allow hardlinks but validate the target stays within dest. + if let Some(link_target) = entry.link_name()? { + // Same Unix-absolute handling as symlinks above. + let target_str = link_target.to_string_lossy(); + let full_target = if target_str.starts_with('/') { + dest.join(target_str.trim_start_matches('/')) + } else { + dest.join(link_target.as_ref()) + }; + let normalized = normalize_path(&full_target); + if !normalized.starts_with(&canonical_dest) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "tar hardlink '{}' escapes destination directory", + entry_path.display() + ), + )); + } + // Skip hardlinks whose target was skipped (e.g., overlayfs + // whiteout char devices). The target doesn't exist on disk + // so creating the hardlink would fail. + if !normalized.exists() { + continue; + } + } + } + tar::EntryType::Char | tar::EntryType::Block | tar::EntryType::Fifo => { + // Device nodes and FIFOs appear in overlayfs upper-layer + // exports (e.g., whiteout char devices from package upgrades, + // named pipes from certain RPM scriptlets). These cannot be + // created without root on macOS and aren't needed on the + // host — skip them. + continue; + } + _other => { + // Unknown or unsupported entry types (sockets, vendor + // extensions, future tar formats). Skip rather than fail — + // the packed image runs inside a Linux VM where the agent + // rootfs provides these files; missing non-regular entries + // on the host extraction side don't affect functionality. + continue; + } + } + + // Validate that the unpacked path stays within dest. + let full_path = dest.join(&entry_path); + let normalized = normalize_path(&full_path); + if !normalized.starts_with(&canonical_dest) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "tar entry '{}' escapes destination directory", + entry_path.display() + ), + )); + } + + // Ensure parent directories are writable before extracting any entry. + // OCI layer tars may set restrictive directory modes (e.g., dr-xr-xr-x) + // before child entries, which prevents creating files or subdirectories. + if let Some(parent) = full_path.parent() { + if parent.is_dir() { + set_mode(parent, 0o755); + } + } + + // Save the tar's intended directory mode for deferred application. + if entry_type == tar::EntryType::Directory { + let mode = entry.header().mode().unwrap_or(0o755); + if mode & 0o200 == 0 { + deferred_dir_modes.push((full_path.clone(), mode)); + } + } + + let is_regular = + entry_type == tar::EntryType::Regular || entry_type == tar::EntryType::GNUSparse; + + // For large regular files use a sparse write: ftruncate creates the + // hole skeleton, then we only pwrite non-zero 64 KiB chunks. This + // prevents 10 GiB overlay disks from materialising as dense files on + // disk and causing ENOSPC or slow extraction. + if is_regular && entry.header().size().unwrap_or(0) >= limits.sparse_threshold { + let entry_size = entry.header().size()?; + let mode = entry.header().mode().unwrap_or(0o644); + if let Err(e) = unpack_sparse(&mut entry, &full_path, entry_size, mode, &real_dest) { + return Err(std::io::Error::new( + e.kind(), + format!("failed to unpack '{}': {}", entry_path.display(), e), + )); + } + } else { + if let Err(e) = entry.unpack_in(dest) { + // On macOS, certain entries fail to unpack due to platform + // limitations (xattr encoding, uid/gid mapping, resource forks). + // For non-Regular entries (symlinks, hardlinks, dirs), skip and + // continue rather than aborting the entire extraction. + if !is_regular { + continue; + } + return Err(std::io::Error::new( + e.kind(), + format!("failed to unpack '{}': {}", entry_path.display(), e), + )); + } + + // After extracting a directory, force it writable so subsequent + // entries (children) can be created inside it. Final permissions + // are applied after the loop. + if entry_type == tar::EntryType::Directory && full_path.is_dir() { + set_mode(&full_path, 0o755); + } + } + } + + // Apply deferred directory permissions now that all children are written. + for (path, mode) in deferred_dir_modes { + if path.is_dir() { + set_mode(&path, mode); + } + } + + Ok(()) +} + +/// Normalize a path by resolving `.` and `..` components without requiring +/// the path to exist on disk (unlike `canonicalize()`). +fn normalize_path(path: &Path) -> PathBuf { + let mut components = Vec::new(); + for component in path.components() { + match component { + std::path::Component::ParentDir => { + components.pop(); + } + std::path::Component::CurDir => {} + c => components.push(c), + } + } + components.iter().collect() +} + +/// Resolve a manifest asset path against an extraction cache without allowing +/// absolute paths, parent components, or symlink traversal outside the cache. +pub fn resolve_cache_asset_path( + cache_dir: &Path, + asset_rel_path: &str, + context: &str, +) -> std::io::Result { + if asset_rel_path.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("{} path is empty", context), + )); + } + + let rel = Path::new(asset_rel_path); + if rel.is_absolute() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("{} path must be relative", context), + )); + } + + for component in rel.components() { + match component { + std::path::Component::Normal(_) => {} + std::path::Component::ParentDir + | std::path::Component::CurDir + | std::path::Component::RootDir + | std::path::Component::Prefix(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("{} path contains disallowed components", context), + )); + } + } + } + + let cache_root = cache_dir + .canonicalize() + .unwrap_or_else(|_| normalize_path(cache_dir)); + let candidate = cache_dir.join(rel); + + let resolved = if candidate.exists() { + candidate.canonicalize()? + } else { + // Candidate doesn't exist yet. Canonicalize its parent (which must + // exist — it's the cache dir) and join the filename. This avoids + // the macOS /tmp → /private/tmp symlink mismatch that would cause + // the starts_with check below to fail when cache_root is canonical + // but normalize_path is not. + let parent = candidate.parent().unwrap_or(&candidate); + let canonical_parent = parent + .canonicalize() + .unwrap_or_else(|_| normalize_path(parent)); + canonical_parent.join(candidate.file_name().unwrap_or_default()) + }; + + if !resolved.starts_with(&cache_root) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("{} path escapes cache directory", context), + )); + } + + Ok(resolved) +} + +/// Marker file indicating extraction is complete. +const EXTRACTION_MARKER: &str = ".smolvm-extracted"; + +/// Get the cache directory for a given checksum. +/// +/// Returns `~/.cache/smolvm-pack//` (hex-formatted). +/// +/// FOLLOW-UP (Finding D): `checksum` is a 32-bit CRC content fingerprint, which +/// is not collision-resistant — two distinct packs can share a cache dir. This +/// is a content-addressing weakness (not a write-escape) and warrants migrating +/// the cache key to a SHA-256 digest; tracked separately as it changes the +/// on-disk cache layout and footer format. +pub fn get_cache_dir(checksum: u32) -> std::io::Result { + let base = dirs::cache_dir() + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "no cache directory"))?; + + Ok(base.join("smolvm-pack").join(format!("{:08x}", checksum))) +} + +/// Check if assets have already been extracted. +pub fn is_extracted(cache_dir: &Path) -> bool { + cache_dir.join(EXTRACTION_MARKER).exists() +} + +/// Maximum total size of the pack extraction cache before LRU eviction kicks in. +/// Override with `SMOLVM_PACK_CACHE_MAX_BYTES` (in bytes); default 5 GiB. +pub fn pack_cache_max_bytes() -> u64 { + const DEFAULT: u64 = 5 * 1024 * 1024 * 1024; + std::env::var("SMOLVM_PACK_CACHE_MAX_BYTES") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|&n| n > 0) + .unwrap_or(DEFAULT) +} + +/// Maximum number of entries `safe_unpack` will extract from a single archive +/// before erroring (inode-flood / entry-bomb guard). Override with +/// `SMOLVM_PACK_MAX_ENTRIES`; default 2,000,000 — far above any real image tar. +fn max_extract_entries() -> u64 { + const DEFAULT: u64 = 2_000_000; + std::env::var("SMOLVM_PACK_MAX_ENTRIES") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|&n| n > 0) + .unwrap_or(DEFAULT) +} + +/// Maximum total apparent (header-declared) size `safe_unpack` will extract from +/// a single archive before erroring (disk-exhaustion / decompression-bomb +/// guard). Override with `SMOLVM_PACK_MAX_EXTRACT_BYTES`; default 128 GiB — +/// generous enough for several large (sparse) overlay disks yet finite. +fn max_extract_total_bytes() -> u64 { + const DEFAULT: u64 = 128 * 1024 * 1024 * 1024; + std::env::var("SMOLVM_PACK_MAX_EXTRACT_BYTES") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|&n| n > 0) + .unwrap_or(DEFAULT) +} + +/// Real (sparse-aware) disk usage of a single file. Extraction dirs contain +/// large *sparse* overlay disks (e.g. a 10 GiB disk holding ~50 MB), so we must +/// count allocated blocks, not the apparent length — otherwise the cap would +/// over-count by orders of magnitude and evict far too aggressively. +#[cfg(unix)] +fn file_disk_usage(meta: &fs::Metadata) -> u64 { + use std::os::unix::fs::MetadataExt; + meta.blocks().saturating_mul(512) +} +#[cfg(not(unix))] +fn file_disk_usage(meta: &fs::Metadata) -> u64 { + meta.len() +} + +/// Recursive real disk usage of a directory tree (best-effort; unreadable +/// entries count as zero). Does not follow symlinks. +fn dir_disk_usage(path: &Path) -> u64 { + let mut total = 0u64; + let entries = match fs::read_dir(path) { + Ok(e) => e, + Err(_) => return 0, + }; + for entry in entries.flatten() { + let meta = match entry.metadata() { + Ok(m) => m, + Err(_) => continue, + }; + if meta.is_dir() { + total = total.saturating_add(dir_disk_usage(&entry.path())); + } else if meta.is_file() { + total = total.saturating_add(file_disk_usage(&meta)); + } + } + total +} + +/// Evict least-recently-modified extraction directories under `cache_root` until +/// the cache's total real disk usage is at or below `max_bytes`. Skips +/// directories with active leases (a running pack/VM) — they are never evicted, +/// even if that leaves the cache over the cap. Best-effort: per-entry errors are +/// skipped. Returns the number of bytes freed. +/// +/// This is what bounds the otherwise-unbounded extraction cache; it runs +/// automatically after a new (cache-miss) extraction, and keeps the newest +/// entries (including the one just written) by evicting oldest-first. +pub fn evict_cache_to_size(cache_root: &Path, max_bytes: u64) -> u64 { + let mut entries: Vec<(PathBuf, std::time::SystemTime, u64)> = Vec::new(); + let read_dir = match fs::read_dir(cache_root) { + Ok(rd) => rd, + Err(_) => return 0, + }; + for entry in read_dir.flatten() { + let path = entry.path(); + let meta = match fs::metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if !meta.is_dir() { + continue; // skip *.lock files and other non-extraction entries + } + let modified = meta.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH); + entries.push((path, modified, dir_disk_usage(&entry.path()))); + } + + let total: u64 = entries.iter().map(|(_, _, s)| *s).sum(); + if total <= max_bytes { + return 0; + } + + // Oldest first — evict least-recently-used. + entries.sort_by_key(|(_, modified, _)| *modified); + + let mut over = total - max_bytes; + let mut freed = 0u64; + for (path, _, size) in entries { + if over == 0 { + break; + } + if has_active_leases(&path) { + continue; // never evict a running pack/VM + } + force_detach_layers_volume(&path); + if fs::remove_dir_all(&path).is_ok() { + // Also drop the adjacent .lock file, if any. + let _ = fs::remove_file(path.with_extension("lock")); + freed = freed.saturating_add(size); + over = over.saturating_sub(size); + } + } + freed +} + +/// Check if footer indicates sidecar mode. +fn is_sidecar_mode(footer: &PackFooter) -> bool { + footer.assets_offset == 0 +} + +/// Get sidecar file path for the given executable. +pub fn sidecar_path_for(exe_path: &Path) -> PathBuf { + let filename = exe_path + .file_name() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_default(); + exe_path.with_file_name(format!("{}{}", filename, SIDECAR_EXTENSION)) +} + +/// Extract assets from a sidecar `.smolmachine` file to the cache directory. +/// +/// This is the primary extraction function for `smolvm pack run`. +/// The sidecar file format is: compressed_assets + manifest + footer. +/// +/// Uses file-based locking (`flock`) to prevent races when multiple processes +/// attempt first-run extraction of the same sidecar concurrently. If `force` +/// is false and extraction has already completed (marker file present), this +/// is a no-op (after acquiring the lock to ensure visibility of a concurrent +/// extraction that just finished). +pub fn extract_sidecar( + sidecar_path: &Path, + cache_dir: &Path, + footer: &PackFooter, + force: bool, + debug: bool, +) -> std::io::Result<()> { + if !sidecar_path.exists() { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("sidecar file not found: {}", sidecar_path.display()), + )); + } + + // Ensure parent directory exists for the lockfile + if let Some(parent) = cache_dir.parent() { + fs::create_dir_all(parent)?; + } + + // Acquire an exclusive lock adjacent to the cache directory. + // This serializes concurrent first-run extractions of the same checksum. + let lock_path = cache_dir.with_extension("lock"); + // Held open for the function's duration: backs the advisory lock below + // (flock on Unix, LockFileEx on Windows) and releases on drop. + let lock_file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(&lock_path)?; + + lock_file_exclusive(&lock_file)?; + + // Double-check inside the lock: another process may have completed + // extraction while we were waiting for the lock. + if !force && is_extracted(cache_dir) { + if debug { + eprintln!("debug: assets already extracted (possibly by another process)"); + } + // Lock released on drop of lock_file + return Ok(()); + } + + // If force-extracting over an existing cache, detach any mounted + // case-sensitive volume first, then remove for a clean slate. + if force && cache_dir.exists() { + force_detach_layers_volume(cache_dir); + let _ = fs::remove_dir_all(cache_dir); + } + + let result = extract_sidecar_inner(sidecar_path, cache_dir, footer, debug); + + // If extraction failed mid-stream, partially extracted files remain on + // disk without a completion marker. Subsequent retries hit the same + // error at the same tar entry, never completing. Clean up the partial + // directory so the next attempt starts fresh. + if result.is_err() && cache_dir.exists() && !is_extracted(cache_dir) { + let _ = fs::remove_dir_all(cache_dir); + } + + // After a successful new extraction (cache miss — the early-return above + // handles cache hits), cap the cache so old, unused extractions don't grow + // without bound. LRU + lease-aware: keeps the newest (incl. what we just + // wrote) and never evicts a running pack. + if result.is_ok() { + if let Some(root) = cache_dir.parent() { + let freed = evict_cache_to_size(root, pack_cache_max_bytes()); + if freed > 0 && debug { + eprintln!("debug: pack cache evicted {freed} bytes to stay under cap"); + } + } + } + + result + // Lock released on drop of lock_file +} + +/// Whether the shared content-addressed pack store is usable on this host. +/// +/// The shared store extracts each build-constant pack exactly once per node into +/// `_shared/` (root-owned, read-only) and presents it to each VM via a +/// per-VM idmapped bind mount, instead of re-extracting + re-chowning a private +/// copy per machine. That mechanism is Linux-only (idmapped mounts, kernel ≥5.12) +/// and the per-VM uid isolation it preserves only exists on the Linux fleet. +/// `SMOLVM_DISABLE_SHARED_EXTRACT` is a kill-switch to fall back to the per-machine +/// path without a redeploy. +pub fn shared_extract_enabled() -> bool { + cfg!(target_os = "linux") && std::env::var_os("SMOLVM_DISABLE_SHARED_EXTRACT").is_none() +} + +/// Directory holding the shared copy for one pack checksum, under `shared_root`. +pub fn shared_pack_dir(shared_root: &Path, checksum: u32) -> PathBuf { + shared_root.join(format!("{:08x}", checksum)) +} + +/// Extract a sidecar pack ONCE into the shared content-addressed store and return +/// the path to the shared copy (`shared_root/`). +/// +/// Unlike [`extract_sidecar`] (which writes a private per-machine copy), this is +/// keyed purely by `footer.checksum` (a CRC32 content fingerprint), so every +/// machine on a node whose pack hashes identically reuses the same extracted tree. +/// The build-constant agent-rootfs (~28.6 MB / 362 files) therefore decodes once +/// per node instead of once per machine — the cold-start tax this removes. +/// +/// The shared copy is left **root-owned** (the extractor runs as root and the tar +/// crate does not preserve ownership by default, so all files land `root:root`) +/// and the store directories are locked to `0700 root`. No other uid can read the +/// copy directly; a VM reaches it only through its own idmapped bind mount, which +/// re-presents on-disk uid 0 as that VM's uid — preserving the per-VM isolation +/// (#456) without a per-machine chown. +/// +/// Idempotent + concurrency-safe: delegates to [`extract_sidecar`], whose flock + +/// `.smolvm-extracted` marker serialize concurrent first extractions of the same +/// checksum and make a warm hit a no-op. +pub fn extract_sidecar_shared( + sidecar_path: &Path, + shared_root: &Path, + footer: &PackFooter, + debug: bool, +) -> std::io::Result { + let shared_dir = shared_pack_dir(shared_root, footer.checksum); + extract_sidecar(sidecar_path, &shared_dir, footer, false, debug)?; + // Lock down the store so a dropped per-VM uid can't read the shared copy + // directly (it must go through its idmapped mount). Best-effort: traversal + // by root (the VMM before it drops privileges) is unaffected by 0700. + restrict_to_owner(shared_root); + restrict_to_owner(&shared_dir); + Ok(shared_dir) +} + +/// Set a directory to `0700` (owner-only) if possible. Best-effort; errors are +/// swallowed because the store is already root-owned and root traversal ignores +/// the mode — this only hardens against a *dropped* sibling uid reading the copy. +fn restrict_to_owner(dir: &Path) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Ok(meta) = fs::metadata(dir) { + let mut perms = meta.permissions(); + perms.set_mode(0o700); + let _ = fs::set_permissions(dir, perms); + } + } + #[cfg(not(unix))] + let _ = dir; +} + +/// Inner extraction logic (called under the lock). +fn extract_sidecar_inner( + sidecar_path: &Path, + cache_dir: &Path, + footer: &PackFooter, + debug: bool, +) -> std::io::Result<()> { + fs::create_dir_all(cache_dir)?; + + if debug { + eprintln!( + "debug: reading {} bytes of compressed assets from sidecar {}", + footer.assets_size, + sidecar_path.display() + ); + } + + let sidecar_file = File::open(sidecar_path)?; + let limited_reader = sidecar_file.take(footer.assets_size); + + let decoder = zstd::stream::Decoder::new(limited_reader) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + + let mut archive = tar::Archive::new(decoder); + safe_unpack(&mut archive, cache_dir)?; + + if debug { + eprintln!("debug: extracted assets to {}", cache_dir.display()); + } + + // Layer order from the sidecar manifest (bottom→top). Best-effort: if the + // manifest can't be read the agent falls back to a name sort. + let layer_order = crate::packer::read_manifest_from_sidecar(sidecar_path) + .ok() + .map(|m| { + m.assets + .layers + .iter() + .filter_map(|l| layer_id_from_asset_path(&l.path)) + .collect::>() + }) + .unwrap_or_default(); + + post_process_extraction(cache_dir, &layer_order, debug)?; + Ok(()) +} + +/// Extract assets from a packed binary to the cache directory. +/// +/// Supports both sidecar mode (assets_offset == 0) and embedded mode. +/// This is used by the stub executable. +pub fn extract_from_binary( + exe_path: &Path, + cache_dir: &Path, + footer: &PackFooter, + debug: bool, +) -> std::io::Result<()> { + fs::create_dir_all(cache_dir)?; + + if is_sidecar_mode(footer) { + let sidecar = sidecar_path_for(exe_path); + extract_sidecar(&sidecar, cache_dir, footer, false, debug) + } else { + // Embedded mode: read compressed assets from the executable + let mut exe_file = File::open(exe_path)?; + exe_file.seek(SeekFrom::Start(footer.assets_offset))?; + + if debug { + eprintln!( + "debug: reading {} bytes of compressed assets from offset {}", + footer.assets_size, footer.assets_offset + ); + } + + let limited_reader = (&mut exe_file).take(footer.assets_size); + + let decoder = zstd::stream::Decoder::new(limited_reader) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + + let mut archive = tar::Archive::new(decoder); + safe_unpack(&mut archive, cache_dir)?; + + if debug { + eprintln!("debug: extracted assets to {}", cache_dir.display()); + } + + // Embedded self-exec stub: no separate sidecar manifest to source layer + // order from here, so let the agent fall back to a name sort. + post_process_extraction(cache_dir, &[], debug)?; + Ok(()) + } +} + +/// Extract assets from a memory pointer (for Mach-O section mode on macOS). +/// +/// # Safety +/// +/// `assets_ptr` must point to a valid, readable memory region of at least +/// `assets_size` bytes that remains valid for the duration of the call. +#[cfg(target_os = "macos")] +pub unsafe fn extract_from_section( + cache_dir: &Path, + assets_ptr: *const u8, + assets_size: usize, + debug: bool, +) -> std::io::Result<()> { + fs::create_dir_all(cache_dir)?; + + if debug { + eprintln!( + "debug: extracting {} bytes of compressed assets from section", + assets_size + ); + } + + let assets_slice = unsafe { std::slice::from_raw_parts(assets_ptr, assets_size) }; + let cursor = std::io::Cursor::new(assets_slice); + + let decoder = zstd::stream::Decoder::new(cursor) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + + let mut archive = tar::Archive::new(decoder); + safe_unpack(&mut archive, cache_dir)?; + + if debug { + eprintln!("debug: extracted assets to {}", cache_dir.display()); + } + + // Mach-O section self-exec stub: same as embedded mode — name-sort fallback. + post_process_extraction(cache_dir, &[], debug)?; + Ok(()) +} + +/// Name of the index file written into the extracted-layers dir recording the +/// layers in OCI order (bottom-most first), one short layer id per line. The +/// guest agent honors it when stacking overlayfs lowerdirs; without it, layers +/// (which are named by content digest) sort arbitrarily and a multi-layer pack +/// can be mis-stacked. Must match the agent's `LAYER_ORDER_FILE`. +const LAYER_ORDER_FILE: &str = "layer-order"; + +/// Layer id used as the on-disk layer dir name, derived from the manifest asset +/// path stem so old short paths and new full-digest paths both preserve order. +fn layer_id_from_asset_path(path: &str) -> Option { + let rel = Path::new(path); + (!rel.is_absolute()) + .then_some(rel)? + .file_stem() + .and_then(|s| s.to_str()) + .filter(|s| !s.is_empty()) + .map(ToOwned::to_owned) +} + +/// Post-process extracted assets: unpack agent rootfs, OCI layers, fix +/// permissions, and (when `layer_order` is non-empty) write the layer-order +/// index so the guest stacks the layers in true OCI order rather than by their +/// content-addressed names. `layer_order` is the short layer ids bottom→top. +fn post_process_extraction( + cache_dir: &Path, + layer_order: &[String], + debug: bool, +) -> std::io::Result<()> { + // Extract agent-rootfs.tar to agent-rootfs directory + let rootfs_tar = cache_dir.join("agent-rootfs.tar"); + let rootfs_dir = cache_dir.join("agent-rootfs"); + if rootfs_tar.exists() && !rootfs_dir.exists() { + if debug { + eprintln!("debug: extracting agent-rootfs.tar..."); + } + fs::create_dir_all(&rootfs_dir)?; + let tar_file = File::open(&rootfs_tar)?; + let mut archive = tar::Archive::new(tar_file); + safe_unpack(&mut archive, &rootfs_dir)?; + } + + // Extract OCI layer tars to layers/{digest}/ directories. + // + // On macOS, the default APFS filesystem is case-insensitive. Linux OCI + // layers may contain paths that differ only in case (e.g., "gdebi" script + // and "GDebi/" directory). Extracting these onto case-insensitive APFS + // would silently lose files. Since the extracted directories are mounted + // into the guest via virtiofs as overlayfs lowerdirs, any missing files + // would corrupt the packed image. + // + // To preserve all paths faithfully, we extract layers into a case-sensitive + // APFS sparse disk image on macOS. The image is persisted in the cache and + // re-mounted on subsequent runs. + let layers_dir = cache_dir.join("layers"); + if layers_dir.exists() { + if debug { + eprintln!("debug: extracting OCI layers..."); + } + // On macOS, extract into a case-sensitive volume to preserve Linux + // paths that differ only in case. On Linux (ext4/xfs), the layers + // dir is already case-sensitive. If the volume can't be created on + // macOS, fail rather than silently corrupting case-colliding paths. + let extract_dir = extraction_layers_dir(cache_dir, debug)?; + + for entry in fs::read_dir(&layers_dir)? { + let entry = entry?; + let path = entry.path(); + if path.extension().is_some_and(|ext| ext == "tar") { + let stem = path.file_stem().unwrap_or_default().to_string_lossy(); + let layer_dir = extract_dir.join(&*stem); + if !layer_dir.exists() { + if debug { + eprintln!("debug: extracting layer {}...", stem); + } + fs::create_dir_all(&layer_dir)?; + let tar_file = File::open(&path)?; + let mut archive = tar::Archive::new(tar_file); + safe_unpack(&mut archive, &layer_dir)?; + } + } + } + + // Record the manifest's layer order so the guest stacks overlayfs + // lowerdirs correctly (layer dirs are named by digest and don't sort + // into stack order). Only ids backed by an extracted dir are written. + if !layer_order.is_empty() { + let lines: Vec<&str> = layer_order + .iter() + .filter(|id| extract_dir.join(id).is_dir()) + .map(String::as_str) + .collect(); + if !lines.is_empty() { + fs::write(extract_dir.join(LAYER_ORDER_FILE), lines.join("\n"))?; + } + } + } + + // Write marker file + fs::write(cache_dir.join(EXTRACTION_MARKER), "")?; + + // Make libraries executable (they need to be loadable). + let lib_dir = cache_dir.join("lib"); + if lib_dir.exists() { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for entry in fs::read_dir(&lib_dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_file() { + let mut perms = fs::metadata(&path)?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(&path, perms)?; + } + } + } + } + + Ok(()) +} + +// ============================================================================= +// Case-sensitive layer extraction (macOS) +// +// On macOS, default APFS is case-insensitive. Linux OCI layers may contain +// paths that differ only in case (e.g., `gdebi` vs `GDebi/`). Extracting +// onto case-insensitive APFS silently drops one variant, corrupting the +// packed image. +// +// We extract layers into a case-sensitive APFS sparse disk image. The image +// lives in the cache directory and is mounted on demand. Because the cache +// is shared across concurrent runs of the same packed artifact, we use a +// lease-file protocol to coordinate mount/unmount: +// +// cache_dir/layers-cs.sparseimage — persisted sparse image +// cache_dir/layers-cs/ — mount point +// cache_dir/leases/ — one file per active user +// cache_dir/leases.lock — flock for lease operations +// +// Acquire: lock → gc stale leases → ensure mounted → write lease → unlock +// Release: lock → remove lease → if no leases remain, detach → unlock +// ============================================================================= + +/// Name of the sparse disk image used for case-sensitive layer extraction. +#[cfg(target_os = "macos")] +const CS_IMAGE_NAME: &str = "layers-cs.sparseimage"; + +/// Subdirectory name for the case-sensitive mount point. +#[cfg(target_os = "macos")] +const CS_MOUNT_DIR: &str = "layers-cs"; + +/// Subdirectory for lease files. +#[cfg(target_os = "macos")] +const LEASES_DIR: &str = "leases"; + +/// Lock file for lease coordination. +#[cfg(target_os = "macos")] +const LEASES_LOCK: &str = "leases.lock"; + +/// A lease on the case-sensitive layers volume. On macOS, this ensures the +/// APFS sparse image is mounted while any lease exists, and detaches it +/// when the last lease is released. On Linux, this is a no-op wrapper. +/// +/// Implements `Drop` so all `?` error paths release the lease automatically. +pub struct LayersVolumeLease { + /// Path to the layers directory (case-sensitive mount on macOS, or + /// `cache_dir/layers` on Linux). + pub path: PathBuf, + /// Cache directory this lease belongs to (needed for cleanup on drop). + #[cfg(target_os = "macos")] + cache_dir: PathBuf, +} + +impl Drop for LayersVolumeLease { + fn drop(&mut self) { + #[cfg(target_os = "macos")] + { + release_lease(&self.cache_dir); + } + } +} + +/// Acquire a lease on the case-sensitive layers volume. +/// +/// On macOS: creates the sparse image if needed, mounts it, writes a +/// per-PID lease file. The volume stays mounted until the last lease is +/// released. Returns a `LayersVolumeLease` whose `Drop` releases the lease. +/// +/// On Linux: returns the `cache_dir/layers` path directly (no-op). +/// +/// Called by `post_process_extraction` during first-time extraction and by +/// `pack_run` before launching the VM. +pub fn acquire_layers_lease(cache_dir: &Path, debug: bool) -> std::io::Result { + #[cfg(target_os = "macos")] + { + let image_path = cache_dir.join(CS_IMAGE_NAME); + if image_path.exists() || has_layer_tars(cache_dir) { + // Case-sensitive volume is required on macOS to preserve Linux + // paths faithfully. Fail if it can't be acquired rather than + // silently falling back to case-insensitive extraction. + let path = acquire_lease(cache_dir, debug)?; + return Ok(LayersVolumeLease { + path, + cache_dir: cache_dir.to_path_buf(), + }); + } + } + + let _ = debug; + Ok(LayersVolumeLease { + path: cache_dir.join("layers"), + #[cfg(target_os = "macos")] + cache_dir: cache_dir.to_path_buf(), + }) +} + +/// Acquire a persistent daemon lease that survives process exit. +/// +/// Unlike `acquire_layers_lease` (RAII, released on Drop), this creates a +/// lease file named `daemon` that persists until explicitly released by +/// `release_daemon_lease`. The daemon child PID is recorded in the file +/// so stale daemon leases can be garbage-collected. +/// +/// On Linux this is a no-op that returns the layers path. +pub fn acquire_daemon_lease( + cache_dir: &Path, + daemon_pid: i32, + debug: bool, +) -> std::io::Result { + #[cfg(target_os = "macos")] + { + let image_path = cache_dir.join(CS_IMAGE_NAME); + if image_path.exists() || has_layer_tars(cache_dir) { + let leases_dir = cache_dir.join(LEASES_DIR); + fs::create_dir_all(&leases_dir)?; + let lock = lock_leases(cache_dir)?; + gc_stale_leases(&leases_dir); + ensure_cs_volume_mounted(cache_dir, debug)?; + fs::write(leases_dir.join("daemon"), format!("{}", daemon_pid))?; + drop(lock); + return Ok(cache_dir.join(CS_MOUNT_DIR)); + } + } + + let _ = (daemon_pid, debug); + Ok(cache_dir.join("layers")) +} + +/// Release the persistent daemon lease and detach if no leases remain. +/// +/// Called from `daemon_stop()` after the VM process has been terminated. +pub fn release_daemon_lease(cache_dir: &Path) { + #[cfg(target_os = "macos")] + { + let leases_dir = cache_dir.join(LEASES_DIR); + let daemon_lease = leases_dir.join("daemon"); + if !daemon_lease.exists() { + return; + } + + let Ok(lock) = lock_leases(cache_dir) else { + let _ = fs::remove_file(&daemon_lease); + return; + }; + + let _ = fs::remove_file(&daemon_lease); + gc_stale_leases(&leases_dir); + detach_if_unused(cache_dir); + drop(lock); + } + + #[cfg(not(target_os = "macos"))] + { + let _ = cache_dir; + } +} + +/// Check whether any active leases exist for this cache directory. +/// +/// Used by `pack prune` to skip in-use caches. Garbage-collects stale +/// leases first (dead PIDs, dead daemon processes). +pub fn has_active_leases(cache_dir: &Path) -> bool { + #[cfg(target_os = "macos")] + { + let leases_dir = cache_dir.join(LEASES_DIR); + if !leases_dir.exists() { + return false; + } + + let Ok(lock) = lock_leases(cache_dir) else { + return false; + }; + gc_stale_leases(&leases_dir); + let active = count_leases(&leases_dir); + drop(lock); + active > 0 + } + + #[cfg(not(target_os = "macos"))] + { + let _ = cache_dir; + false + } +} + +/// Force-detach and clean up all leases for a cache directory. +/// +/// Used by `--force-extract` before clearing the cache. NOT used by normal +/// `pack prune` — prune should check `has_active_leases` first and skip +/// active caches. +pub fn force_detach_layers_volume(cache_dir: &Path) { + #[cfg(target_os = "macos")] + { + let mount_point = cache_dir.join(CS_MOUNT_DIR); + if mount_point.exists() && is_mount_point(&mount_point) { + let _ = std::process::Command::new("hdiutil") + .args(["detach", "-quiet", "-force"]) + .arg(&mount_point) + .output(); + } + // Remove all lease files. + let _ = fs::remove_dir_all(cache_dir.join(LEASES_DIR)); + } + + #[cfg(not(target_os = "macos"))] + { + let _ = cache_dir; + } +} + +/// Mount the case-sensitive volume (if needed) and return the extraction +/// directory. Called during initial extraction (already under flock — no +/// lease needed). For runtime use, call `acquire_layers_lease()` instead. +fn extraction_layers_dir(cache_dir: &Path, debug: bool) -> std::io::Result { + #[cfg(target_os = "macos")] + { + ensure_cs_volume_mounted(cache_dir, debug)?; + Ok(cache_dir.join(CS_MOUNT_DIR)) + } + + #[cfg(not(target_os = "macos"))] + { + let _ = debug; + Ok(cache_dir.join("layers")) + } +} + +// --- macOS-only implementation details --- + +#[cfg(target_os = "macos")] +fn has_layer_tars(cache_dir: &Path) -> bool { + let layers_dir = cache_dir.join("layers"); + layers_dir.exists() + && fs::read_dir(&layers_dir) + .ok() + .map(|rd| { + rd.filter_map(|e| e.ok()) + .any(|e| e.path().extension().is_some_and(|ext| ext == "tar")) + }) + .unwrap_or(false) +} + +/// Sum the sizes of all `.tar` files in a directory. +#[cfg(target_os = "macos")] +fn sum_tar_sizes(dir: &Path) -> u64 { + let Ok(entries) = fs::read_dir(dir) else { + return 0; + }; + entries + .filter_map(|e| e.ok()) + .filter(|e| e.path().extension().is_some_and(|ext| ext == "tar")) + .filter_map(|e| e.metadata().ok()) + .map(|m| m.len()) + .sum() +} + +/// Check whether `path` is a mount point by comparing device IDs with parent. +#[cfg(target_os = "macos")] +fn is_mount_point(path: &Path) -> bool { + use std::os::unix::fs::MetadataExt; + let Ok(meta) = fs::metadata(path) else { + return false; + }; + let Ok(parent_meta) = fs::metadata(path.parent().unwrap_or(Path::new("/"))) else { + return false; + }; + meta.dev() != parent_meta.dev() +} + +/// Acquire a lease: lock → gc stale leases → ensure mounted → write lease. +#[cfg(target_os = "macos")] +fn acquire_lease(cache_dir: &Path, debug: bool) -> std::io::Result { + let mount_point = cache_dir.join(CS_MOUNT_DIR); + let leases_dir = cache_dir.join(LEASES_DIR); + fs::create_dir_all(&leases_dir)?; + + let lock = lock_leases(cache_dir)?; + + // Garbage-collect leases from dead processes. + gc_stale_leases(&leases_dir); + + // Ensure the sparse image exists and is mounted. + ensure_cs_volume_mounted(cache_dir, debug)?; + + // Write a lease file for this process. + let lease_path = leases_dir.join(format!("{}", std::process::id())); + fs::write(&lease_path, "")?; + + drop(lock); + Ok(mount_point) +} + +/// Release a lease: lock → remove lease → if no leases remain, detach. +#[cfg(target_os = "macos")] +fn release_lease(cache_dir: &Path) { + let leases_dir = cache_dir.join(LEASES_DIR); + let lease_path = leases_dir.join(format!("{}", std::process::id())); + + let Ok(lock) = lock_leases(cache_dir) else { + let _ = fs::remove_file(&lease_path); + return; + }; + + let _ = fs::remove_file(&lease_path); + gc_stale_leases(&leases_dir); + detach_if_unused(cache_dir); + drop(lock); +} + +/// Remove lease files whose PID is no longer alive. +/// +/// Handles both per-PID leases (named by PID number) and daemon leases +/// (named "daemon", containing the daemon PID as text content). +#[cfg(target_os = "macos")] +fn gc_stale_leases(leases_dir: &Path) { + let Ok(entries) = fs::read_dir(leases_dir) else { + return; + }; + for entry in entries.filter_map(|e| e.ok()) { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + if name_str == "daemon" { + // Daemon lease: PID is stored as file content. + if let Ok(content) = fs::read_to_string(entry.path()) { + if let Ok(pid) = content.trim().parse::() { + if unsafe { libc::kill(pid, 0) } != 0 { + let _ = fs::remove_file(entry.path()); + } + } + } + } else if let Ok(pid) = name_str.parse::() { + // Per-PID lease: file name is the PID. + if unsafe { libc::kill(pid, 0) } != 0 { + let _ = fs::remove_file(entry.path()); + } + } + } +} + +/// Count active lease files in the leases directory. +#[cfg(target_os = "macos")] +fn count_leases(leases_dir: &Path) -> usize { + fs::read_dir(leases_dir) + .ok() + .map(|rd| rd.filter_map(|e| e.ok()).count()) + .unwrap_or(0) +} + +/// Detach the case-sensitive volume if no leases remain. +#[cfg(target_os = "macos")] +fn detach_if_unused(cache_dir: &Path) { + let leases_dir = cache_dir.join(LEASES_DIR); + if count_leases(&leases_dir) == 0 { + let mount_point = cache_dir.join(CS_MOUNT_DIR); + if mount_point.exists() && is_mount_point(&mount_point) { + let _ = std::process::Command::new("hdiutil") + .args(["detach", "-quiet"]) + .arg(&mount_point) + .output(); + } + } +} + +/// Acquire the leases lock (flock-based, like extract_sidecar). +#[cfg(target_os = "macos")] +fn lock_leases(cache_dir: &Path) -> std::io::Result { + let lock_path = cache_dir.join(LEASES_LOCK); + let lock_file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(&lock_path)?; + let ret = unsafe { libc::flock(lock_file.as_raw_fd(), libc::LOCK_EX) }; + if ret != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(lock_file) +} + +/// Create the sparse image if needed and mount it. +#[cfg(target_os = "macos")] +fn ensure_cs_volume_mounted(cache_dir: &Path, debug: bool) -> std::io::Result<()> { + let image_path = cache_dir.join(CS_IMAGE_NAME); + let mount_point = cache_dir.join(CS_MOUNT_DIR); + + // Already mounted — nothing to do. + if mount_point.exists() && is_mount_point(&mount_point) { + return Ok(()); + } + + // Create the sparse image if it doesn't exist. + if !image_path.exists() { + let layers_dir = cache_dir.join("layers"); + let total_tar_bytes = sum_tar_sizes(&layers_dir); + // 2.5x headroom + 512 MiB for fs metadata, minimum 1 GiB. + // Sparse format: only written bytes use real disk. + let size_bytes = std::cmp::max( + (total_tar_bytes as f64 * 2.5) as u64 + 512 * 1024 * 1024, + 1024 * 1024 * 1024, + ); + let size_gib = size_bytes / (1024 * 1024 * 1024) + 1; + let size_arg = format!("{}g", size_gib); + + if debug { + eprintln!( + "debug: creating case-sensitive APFS sparse image ({}g from {} bytes of tars)...", + size_gib, total_tar_bytes + ); + } + let output = std::process::Command::new("hdiutil") + .args([ + "create", + "-size", + &size_arg, + "-fs", + "Case-sensitive APFS", + "-type", + "SPARSE", + "-volname", + "smolvm-layers", + ]) + .arg(&image_path) + .output()?; + if !output.status.success() { + return Err(std::io::Error::other(format!( + "hdiutil create failed: {}", + String::from_utf8_lossy(&output.stderr) + ))); + } + } + + // Mount it. + fs::create_dir_all(&mount_point)?; + if debug { + eprintln!( + "debug: mounting case-sensitive volume at {}", + mount_point.display() + ); + } + let output = std::process::Command::new("hdiutil") + .args(["attach", "-mountpoint"]) + .arg(&mount_point) + .args(["-nobrowse", "-noautoopen"]) + .arg(&image_path) + .output()?; + if !output.status.success() { + return Err(std::io::Error::other(format!( + "hdiutil attach failed: {}", + String::from_utf8_lossy(&output.stderr) + ))); + } + + Ok(()) +} + +/// Marker file indicating libs extraction is complete. +const LIBS_EXTRACTION_MARKER: &str = ".smolvm-libs-extracted"; + +/// Extract runtime libraries from a packed stub binary. +/// +/// Reads the last 32 bytes of the executable looking for a SMOLLIBS footer. +/// If found, extracts the compressed libs bundle to a cache directory and +/// returns the path to the `lib/` directory containing libkrun/libkrunfw. +/// +/// Returns `None` if the binary has no embedded libs (e.g., the base smolvm binary). +pub fn extract_libs_from_binary(exe_path: &Path, debug: bool) -> std::io::Result> { + use crate::format::{LibsFooter, LIBS_FOOTER_SIZE}; + + let mut file = File::open(exe_path)?; + let file_size = file.metadata()?.len(); + if file_size < LIBS_FOOTER_SIZE as u64 { + return Ok(None); + } + + // Read the last 32 bytes + file.seek(SeekFrom::End(-(LIBS_FOOTER_SIZE as i64)))?; + let mut footer_buf = [0u8; LIBS_FOOTER_SIZE]; + file.read_exact(&mut footer_buf)?; + + let footer = match LibsFooter::from_bytes(&footer_buf) { + Ok(f) => f, + Err(_) => return Ok(None), // No SMOLLIBS footer — no embedded libs + }; + + if debug { + eprintln!( + "debug: found SMOLLIBS footer: offset={}, size={}", + footer.libs_offset, footer.libs_size + ); + } + + // Cache key based on libs content hash + file.seek(SeekFrom::Start(footer.libs_offset))?; + let mut hasher = crc32fast::Hasher::new(); + let mut remaining = footer.libs_size; + let mut buf = [0u8; 64 * 1024]; + while remaining > 0 { + let to_read = remaining.min(buf.len() as u64) as usize; + let n = file.read(&mut buf[..to_read])?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + remaining -= n as u64; + } + let libs_checksum = hasher.finalize(); + + let cache_base = dirs::cache_dir() + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "no cache directory"))?; + let libs_cache_dir = cache_base + .join("smolvm-libs") + .join(format!("{:08x}", libs_checksum)); + let lib_dir = libs_cache_dir.join("lib"); + + // Acquire exclusive lock to prevent concurrent extraction races. + if let Some(parent) = libs_cache_dir.parent() { + fs::create_dir_all(parent)?; + } + let lock_path = libs_cache_dir.with_extension("lock"); + let lock_file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(&lock_path)?; + + lock_file_exclusive(&lock_file)?; + + // Re-check after acquiring lock (another process may have finished) + if libs_cache_dir.join(LIBS_EXTRACTION_MARKER).exists() { + if debug { + eprintln!("debug: libs already extracted at {}", lib_dir.display()); + } + // Lock released on drop of lock_file + let _ = lock_file; + return Ok(Some(lib_dir)); + } + + // Extract + fs::create_dir_all(&libs_cache_dir)?; + file.seek(SeekFrom::Start(footer.libs_offset))?; + let limited_reader = (&mut file).take(footer.libs_size); + let decoder = zstd::stream::Decoder::new(limited_reader) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + let mut archive = tar::Archive::new(decoder); + safe_unpack(&mut archive, &libs_cache_dir)?; + + // Make libs executable + if lib_dir.exists() { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + for entry in fs::read_dir(&lib_dir)? { + let entry = entry?; + if entry.path().is_file() { + let mut perms = fs::metadata(entry.path())?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(entry.path(), perms)?; + } + } + } + } + + fs::write(libs_cache_dir.join(LIBS_EXTRACTION_MARKER), "")?; + // Lock released on drop of lock_file + let _ = lock_file; + + if debug { + eprintln!("debug: extracted libs to {}", lib_dir.display()); + } + + Ok(Some(lib_dir)) +} + +/// Copy `src` to `dst` while preserving holes (sparseness), regardless of the +/// platform or filesystem. +/// +/// `std::fs::copy` is *not* reliably hole-preserving. On Linux it prefers +/// `copy_file_range`/`sendfile`, but those fall back to a dense byte-for-byte +/// copy when the source and destination live on different mounts or on a +/// filesystem where the accelerated path isn't available — both common in CI +/// containers and under overlayfs. When that happens, a multi-GiB sparse +/// template (e.g. the 20 GiB `storage-template.ext4`, only ~25 MiB of which is +/// real data) is rehydrated into its full logical size of literal zeros on +/// disk, so two extractions can exhaust the runner and fail with ENOSPC. +/// +/// This copy creates the destination as a sparse skeleton (`set_len` to the +/// source's logical size) and then writes only the chunks that contain non-zero +/// bytes, leaving every zero run as a hole. It mirrors the write-side +/// `assets::sparse_copy_overlay`, so behavior is consistent on both ends and on +/// APFS/ext4/xfs/NTFS alike. Reading over holes costs no disk I/O (the kernel +/// serves zero pages from cache), so scanning even a mostly-empty 20 GiB +/// template is fast. +/// +/// Used on both ends: the extract/run side here, and the pack-create side in +/// `assets::create_storage_template` when it copies the pre-formatted +/// `storage-template.ext4` into the staging directory. +pub(crate) fn sparse_copy(src: &Path, dst: &Path) -> std::io::Result<()> { + let mut src_file = File::open(src)?; + let size = src_file.metadata()?.len(); + + let mut dst_file = File::create(dst)?; + // On Windows/NTFS a fresh file is dense: set_len-ing to a large size and then + // writing chunks at high offsets would allocate the whole gap. Mark it sparse + // first. Unix filesystems are sparse by default. + #[cfg(windows)] + mark_file_sparse(&dst_file)?; + dst_file.set_len(size)?; + + // Forward pass: read in 512 KiB chunks, writing only chunks that contain a + // non-zero byte. Zero chunks are skipped, so they stay as holes in `dst`. + let mut buf = vec![0u8; 512 * 1024]; + let mut offset: u64 = 0; + while offset < size { + let to_read = (size - offset).min(buf.len() as u64) as usize; + let n = src_file.read(&mut buf[..to_read])?; + if n == 0 { + break; + } + let chunk = &buf[..n]; + if chunk.iter().any(|&b| b != 0) { + dst_file.seek(SeekFrom::Start(offset))?; + dst_file.write_all(chunk)?; + } + offset += n as u64; + } + + Ok(()) +} + +/// Create a storage disk file (empty sparse file). +pub fn create_storage_disk(path: &Path, size: u64) -> std::io::Result<()> { + let file = File::create(path)?; + // On Windows/NTFS, File::create makes a dense file; set_len to a multi-GiB + // size would allocate every block. Mark it sparse first. + #[cfg(windows)] + mark_file_sparse(&file)?; + file.set_len(size)?; + Ok(()) +} + +/// Copy overlay disk template from cache to a runtime directory. +/// +/// Copies the overlay template to `dest`, then restores the full sparse +/// skeleton if `overlay_logical_size` is set (new packs store a truncated +/// copy with the trailing hole stripped), and optionally extends further +/// when `size_gb_override` is larger still. +/// +/// Returns an error if the template path is `None` or the template file +/// does not exist in the cache. +pub fn copy_overlay_template( + cache_dir: &Path, + template_path: Option<&str>, + dest: &Path, + size_gb_override: Option, + overlay_logical_size: Option, +) -> std::io::Result<()> { + let template = template_path.ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + "overlay template not specified in manifest", + ) + })?; + + let src = resolve_cache_asset_path(cache_dir, template, "overlay template")?; + if !src.exists() { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("overlay template not found: {}", src.display()), + )); + } + + // Hole-preserving copy: fs::copy can densify a sparse template on some + // Linux filesystems/mounts, ballooning the overlay to its full logical size. + sparse_copy(&src, dest)?; + + // Determine target size: max of the copied size, overlay_logical_size + // (original sparse extent before trailing-hole truncation), and + // size_gb_override (user-requested larger disk). A single ftruncate + // handles all three cases; ftruncate is instant and allocates no disk + // blocks for the extended region. + let copied_size = fs::metadata(dest)?.len(); + let target = [ + Some(copied_size), + overlay_logical_size, + size_gb_override.map(|gb| gb * 1024 * 1024 * 1024), + ] + .into_iter() + .flatten() + .max() + .unwrap_or(copied_size); + + if target > copied_size { + let file = fs::OpenOptions::new().write(true).open(dest)?; + // On Windows/NTFS, extending with set_len would allocate every byte of + // the gap unless the file is sparse. Mark it sparse first (idempotent). + #[cfg(windows)] + mark_file_sparse(&file)?; + file.set_len(target)?; + } + + Ok(()) +} + +/// Create or copy storage disk from template. +/// +/// If a pre-formatted template exists in the cache, copy it. +/// Otherwise, create an empty sparse file (will be formatted by agent on first boot). +/// +/// `size_gb_override` lets callers specify a custom disk size (in GiB). +/// When `None`, falls back to 512 MiB. +pub fn create_or_copy_storage_disk( + cache_dir: &Path, + template_path: Option<&str>, + storage_path: &Path, + size_gb_override: Option, +) -> std::io::Result<()> { + if let Some(template) = template_path { + let template_path = resolve_cache_asset_path(cache_dir, template, "storage template")?; + if template_path.exists() { + // Hole-preserving copy: a plain fs::copy densifies the (mostly-empty) + // multi-GiB storage template on some Linux filesystems/mounts, + // turning ~25 MiB of real data into its full logical size of zeros on + // disk and risking ENOSPC when several extractions run. + sparse_copy(&template_path, storage_path)?; + // If a custom size was requested and it's larger than the template, + // extend the sparse file (resize2fs in the agent will expand the FS). + if let Some(gb) = size_gb_override { + let desired = gb * 1024 * 1024 * 1024; + let current = fs::metadata(storage_path)?.len(); + if desired > current { + let file = fs::OpenOptions::new().write(true).open(storage_path)?; + // On Windows/NTFS, extending with set_len would allocate the + // whole gap unless the file is sparse. Mark sparse first + // (idempotent). + #[cfg(windows)] + mark_file_sparse(&file)?; + file.set_len(desired)?; + } + } + return Ok(()); + } + } + // Fallback: create empty sparse file (agent will format on first boot) + let size = match size_gb_override { + Some(gb) => gb * 1024 * 1024 * 1024, + None => 512 * 1024 * 1024, + }; + create_storage_disk(storage_path, size) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a single-file tar archive in memory with the given name and data. + fn make_tar(name: &str, data: &[u8]) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, name, data).unwrap(); + builder.into_inner().unwrap() + } + + #[cfg(unix)] + #[test] + fn test_unpack_sparse_rejects_symlink_at_destination() { + use std::os::unix::fs::symlink; + + let temp_dir = tempfile::tempdir().unwrap(); + let outside = temp_dir.path().join("outside.bin"); + let dest = temp_dir.path().join("overlay.raw"); + + fs::write(&outside, b"untouched").unwrap(); + symlink(&outside, &dest).unwrap(); // dest is now a symlink → outside + + let data = vec![0xFFu8; 512]; + let tar_bytes = make_tar("overlay.raw", &data); + let mut archive = tar::Archive::new(tar_bytes.as_slice()); + let mut entry = archive.entries().unwrap().next().unwrap().unwrap(); + + let real_dest = temp_dir.path().canonicalize().unwrap(); + let result = unpack_sparse(&mut entry, &dest, data.len() as u64, 0o644, &real_dest); + + assert!(result.is_err(), "should reject symlink at destination"); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData); + // The symlink target must not be modified + assert_eq!(fs::read(&outside).unwrap(), b"untouched"); + } + + /// Build a tar archive carrying a single symlink entry whose link target + /// is `link_target`, plus (optionally) a trailing regular-file entry. + fn make_symlink_tar(name: &str, link_target: &str) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Symlink); + header.set_size(0); + header.set_mode(0o777); + builder.append_link(&mut header, name, link_target).unwrap(); + builder.into_inner().unwrap() + } + + #[cfg(unix)] + #[test] + fn test_safe_unpack_rejects_symlink_entry_escaping_dest_relative() { + // A crafted tar entry `evil -> ../../outside.bin` must be rejected by + // safe_unpack before it is materialized: otherwise a later write + // through `evil` would land outside the extraction directory. + let temp_dir = tempfile::tempdir().unwrap(); + let outside = temp_dir.path().join("outside.bin"); + fs::write(&outside, b"untouched").unwrap(); + + let dest = temp_dir.path().join("dest"); + fs::create_dir(&dest).unwrap(); + + let tar_bytes = make_symlink_tar("evil", "../../outside.bin"); + let mut archive = tar::Archive::new(tar_bytes.as_slice()); + let result = safe_unpack(&mut archive, &dest); + + assert!( + result.is_err(), + "escaping relative symlink must be rejected" + ); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData); + assert!(!dest.join("evil").exists(), "symlink must not be created"); + assert_eq!(fs::read(&outside).unwrap(), b"untouched"); + } + + #[cfg(unix)] + #[test] + fn test_safe_unpack_rejects_symlink_entry_escaping_dest_absolute() { + // An absolute-target symlink `evil -> /etc/passwd` is jailed to + // `dest/etc/passwd`, staying inside dest, so it is allowed. But an + // absolute target with `..` that climbs out (`/../escape`) must be + // rejected — this guards the absolute-symlink jailing branch. + let temp_dir = tempfile::tempdir().unwrap(); + let dest = temp_dir.path().join("dest"); + fs::create_dir(&dest).unwrap(); + + let tar_bytes = make_symlink_tar("evil", "/../../escape"); + let mut archive = tar::Archive::new(tar_bytes.as_slice()); + let result = safe_unpack(&mut archive, &dest); + + assert!( + result.is_err(), + "escaping absolute symlink must be rejected" + ); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData); + assert!(!dest.join("evil").exists(), "symlink must not be created"); + } + + #[cfg(unix)] + #[test] + fn test_safe_unpack_allows_in_dest_symlink() { + // A well-behaved relative symlink that stays within dest is accepted. + let temp_dir = tempfile::tempdir().unwrap(); + let dest = temp_dir.path().join("dest"); + fs::create_dir(&dest).unwrap(); + + let tar_bytes = make_symlink_tar("link", "target"); + let mut archive = tar::Archive::new(tar_bytes.as_slice()); + safe_unpack(&mut archive, &dest).unwrap(); + + let meta = fs::symlink_metadata(dest.join("link")).unwrap(); + assert!( + meta.file_type().is_symlink(), + "in-dest symlink should exist" + ); + } + + // --------------------------------------------------------------------- + // Fix 1 (PRIMARY): the sparse-write path must NOT follow a symlink parent + // component out of `dest`. Reproduces the host-escape: a prior entry plants + // `foo -> `, then a sparse regular file under + // `foo/...` tries to redirect the write through it. `safe_unpack` must + // error AND write nothing at the escape target. + // --------------------------------------------------------------------- + #[cfg(unix)] + #[test] + fn test_safe_unpack_sparse_symlink_parent_escape_blocked() { + use std::os::unix::fs::symlink; + + let temp_dir = tempfile::tempdir().unwrap(); + + // Sentinel directory OUTSIDE dest (sibling under the temp dir). If the + // escape worked, the payload would land here. + let sentinel = temp_dir.path().join("ESCAPE"); + fs::create_dir(&sentinel).unwrap(); + + let dest = temp_dir.path().join("dest"); + fs::create_dir(&dest).unwrap(); + + // Plant the escaping symlink exactly as a prior tar entry would have: + // `dest/foo` -> the sentinel dir outside dest. (Pre-planting instead of + // relying on the tar crate's symlink-creation semantics keeps the test + // deterministic; it reproduces the same on-disk state.) + symlink(&sentinel, dest.join("foo")).unwrap(); + + // A sparse regular file under `foo/...`: create_dir_all + open would + // traverse the symlink parent and write into the sentinel. Small payload + // + low sparse threshold forces the `unpack_sparse` path. + let payload = vec![0xABu8; 4096]; + let tar_bytes = make_tar("foo/pwned.bin", &payload); + + let mut archive = tar::Archive::new(tar_bytes.as_slice()); + let limits = SafeUnpackLimits { + max_entries: 1_000, + max_total_bytes: 1 << 30, + sparse_threshold: 512, // 4096-byte payload takes the sparse path + }; + let result = safe_unpack_with_limits(&mut archive, &dest, &limits); + + assert!( + result.is_err(), + "symlink-parent escape via the sparse path must be rejected" + ); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData); + // Nothing may have been written through the escaping symlink. + assert!( + !sentinel.join("pwned.bin").exists(), + "host file was written OUTSIDE dest — escape not blocked" + ); + } + + // --------------------------------------------------------------------- + // Fix 1b (root cause): an absolute symlink that aliases the dest ROOT + // itself (`foo -> /`) is the parent-escape primitive and must be rejected + // at creation — it must never appear on disk. + // --------------------------------------------------------------------- + #[cfg(unix)] + #[test] + fn test_safe_unpack_rejects_absolute_symlink_aliasing_root() { + let temp_dir = tempfile::tempdir().unwrap(); + let dest = temp_dir.path().join("dest"); + fs::create_dir(&dest).unwrap(); + + let tar_bytes = make_symlink_tar("foo", "/"); + let mut archive = tar::Archive::new(tar_bytes.as_slice()); + let result = safe_unpack(&mut archive, &dest); + + assert!( + result.is_err(), + "absolute symlink aliasing dest root must be rejected" + ); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidData); + assert!( + !dest.join("foo").exists() && fs::symlink_metadata(dest.join("foo")).is_err(), + "root-aliasing symlink must never be created on disk" + ); + } + + // --------------------------------------------------------------------- + // Fix 1 (G): the sparse path must strip setuid/setgid/sticky bits, matching + // the dense path — a hostile header must not yield a setuid host file. + // --------------------------------------------------------------------- + #[cfg(unix)] + #[test] + fn test_unpack_sparse_strips_setuid() { + use std::os::unix::fs::PermissionsExt; + + let temp_dir = tempfile::tempdir().unwrap(); + let dest = temp_dir.path().join("suid.bin"); + let real_dest = temp_dir.path().canonicalize().unwrap(); + + let data = vec![0x11u8; 4096]; + let tar_bytes = make_tar("suid.bin", &data); + let mut archive = tar::Archive::new(tar_bytes.as_slice()); + let mut entry = archive.entries().unwrap().next().unwrap().unwrap(); + + // Header mode carries setuid (0o4000) + setgid (0o2000) + rwxr-xr-x. + unpack_sparse(&mut entry, &dest, data.len() as u64, 0o6755, &real_dest).unwrap(); + + let mode = fs::metadata(&dest).unwrap().permissions().mode() & 0o7777; + assert_eq!( + mode, 0o0755, + "setuid/setgid/sticky bits must be stripped on the sparse path" + ); + } + + // --------------------------------------------------------------------- + // Fix 3: an archive exceeding the entry-count ceiling is rejected. + // --------------------------------------------------------------------- + #[test] + fn test_safe_unpack_rejects_too_many_entries() { + let temp_dir = tempfile::tempdir().unwrap(); + let dest = temp_dir.path().join("dest"); + fs::create_dir(&dest).unwrap(); + + let mut builder = tar::Builder::new(Vec::new()); + for i in 0..10 { + let data = b"x"; + let mut h = tar::Header::new_gnu(); + h.set_size(data.len() as u64); + h.set_mode(0o644); + builder + .append_data(&mut h, format!("file{i}.txt"), &data[..]) + .unwrap(); + } + let tar_bytes = builder.into_inner().unwrap(); + + let mut archive = tar::Archive::new(tar_bytes.as_slice()); + let limits = SafeUnpackLimits { + max_entries: 3, + max_total_bytes: 1 << 30, + sparse_threshold: SPARSE_WRITE_THRESHOLD, + }; + let err = safe_unpack_with_limits(&mut archive, &dest, &limits).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!(err.to_string().contains("max entry count")); + } + + // --------------------------------------------------------------------- + // Fix 3: an archive exceeding the total-bytes ceiling is rejected. + // --------------------------------------------------------------------- + #[test] + fn test_safe_unpack_rejects_total_bytes_over_cap() { + let temp_dir = tempfile::tempdir().unwrap(); + let dest = temp_dir.path().join("dest"); + fs::create_dir(&dest).unwrap(); + + let data = vec![0u8; 4096]; + let tar_bytes = make_tar("big.bin", &data); + + let mut archive = tar::Archive::new(tar_bytes.as_slice()); + let limits = SafeUnpackLimits { + max_entries: 1_000, + max_total_bytes: 1024, // 4096-byte entry exceeds this + sparse_threshold: SPARSE_WRITE_THRESHOLD, + }; + let err = safe_unpack_with_limits(&mut archive, &dest, &limits).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + assert!(err.to_string().contains("max total size")); + } + + #[test] + fn test_unpack_sparse_preserves_data_integrity() { + let temp_dir = tempfile::tempdir().unwrap(); + let dest = temp_dir.path().join("data.raw"); + + // Alternating 64 KiB zero and non-zero blocks: covers the skip-zero + // path, the write-nonzero path, correct seek offsets, and the + // ftruncate skeleton giving the right final size. + let block = 64 * 1024; + let mut data = vec![0u8; 8 * block]; + for i in (0..8).step_by(2) { + data[i * block..(i + 1) * block].fill(0xFF); + } + + let tar_bytes = make_tar("data.raw", &data); + let mut archive = tar::Archive::new(tar_bytes.as_slice()); + let mut entry = archive.entries().unwrap().next().unwrap().unwrap(); + + let real_dest = temp_dir.path().canonicalize().unwrap(); + unpack_sparse(&mut entry, &dest, data.len() as u64, 0o644, &real_dest).unwrap(); + + assert_eq!(fs::read(&dest).unwrap(), data); + } + + #[test] + fn test_cache_dir_format() { + let dir = get_cache_dir(0xDEADBEEF).unwrap(); + assert!(dir.to_string_lossy().contains("deadbeef")); + } + + #[test] + fn test_is_extracted() { + let temp_dir = tempfile::tempdir().unwrap(); + + assert!(!is_extracted(temp_dir.path())); + + fs::write(temp_dir.path().join(EXTRACTION_MARKER), "").unwrap(); + assert!(is_extracted(temp_dir.path())); + } + + #[test] + fn test_is_extracted_partial() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Simulate partial extraction - files exist but no marker + fs::create_dir_all(temp_dir.path().join("lib")).unwrap(); + fs::write(temp_dir.path().join("lib/libkrun.dylib"), "partial").unwrap(); + + assert!(!is_extracted(temp_dir.path())); + } + + #[test] + fn test_sidecar_path_for() { + let exe = Path::new("/path/to/my-app"); + let sidecar = sidecar_path_for(exe); + assert_eq!(sidecar, PathBuf::from("/path/to/my-app.smolmachine")); + } + + #[test] + fn test_sidecar_mode_detection() { + let sidecar_footer = PackFooter { + stub_size: 0, + assets_offset: 0, + assets_size: 1000, + manifest_offset: 1000, + manifest_size: 500, + checksum: 0x12345678, + }; + assert!(is_sidecar_mode(&sidecar_footer)); + + let embedded_footer = PackFooter { + stub_size: 50000, + assets_offset: 50000, + assets_size: 1000, + manifest_offset: 51000, + manifest_size: 500, + checksum: 0x12345678, + }; + assert!(!is_sidecar_mode(&embedded_footer)); + } + + #[test] + fn test_create_storage_disk() { + let temp_dir = tempfile::tempdir().unwrap(); + let disk_path = temp_dir.path().join("test.ext4"); + + create_storage_disk(&disk_path, 1024 * 1024).unwrap(); + + assert!(disk_path.exists()); + assert_eq!(fs::metadata(&disk_path).unwrap().len(), 1024 * 1024); + } + + #[test] + fn test_copy_overlay_template_fails_when_none() { + let temp_dir = tempfile::tempdir().unwrap(); + let dest = temp_dir.path().join("overlay.raw"); + + let result = copy_overlay_template(temp_dir.path(), None, &dest, None, None); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::NotFound); + } + + #[test] + fn test_copy_overlay_template_fails_when_missing() { + let temp_dir = tempfile::tempdir().unwrap(); + let dest = temp_dir.path().join("overlay.raw"); + + let result = + copy_overlay_template(temp_dir.path(), Some("nonexistent.raw"), &dest, None, None); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::NotFound); + } + + #[test] + fn test_copy_overlay_template_copies_and_extends() { + let temp_dir = tempfile::tempdir().unwrap(); + let template = temp_dir.path().join("overlay.raw"); + let dest = temp_dir.path().join("output.raw"); + + // Create a small template file (1 KB) + let template_data = vec![0u8; 1024]; + fs::write(&template, &template_data).unwrap(); + + // Copy without any size override or logical size + copy_overlay_template(temp_dir.path(), Some("overlay.raw"), &dest, None, None).unwrap(); + assert_eq!(fs::metadata(&dest).unwrap().len(), 1024); + + // Copy with overlay_logical_size set — dest should be extended + let dest2 = temp_dir.path().join("output2.raw"); + copy_overlay_template( + temp_dir.path(), + Some("overlay.raw"), + &dest2, + None, + Some(4096), + ) + .unwrap(); + assert_eq!(fs::metadata(&dest2).unwrap().len(), 4096); + } + + #[test] + fn test_copy_overlay_template_size_gb_takes_max() { + let temp_dir = tempfile::tempdir().unwrap(); + let template = temp_dir.path().join("overlay.raw"); + fs::write(&template, vec![0u8; 1024]).unwrap(); + + // size_gb_override wins when larger than overlay_logical_size + let dest = temp_dir.path().join("out_a.raw"); + copy_overlay_template( + temp_dir.path(), + Some("overlay.raw"), + &dest, + Some(1), // 1 GiB + Some(4096), + ) + .unwrap(); + assert_eq!(fs::metadata(&dest).unwrap().len(), 1024 * 1024 * 1024); + + // overlay_logical_size wins when larger than size_gb_override + let dest2 = temp_dir.path().join("out_b.raw"); + copy_overlay_template( + temp_dir.path(), + Some("overlay.raw"), + &dest2, + None, + Some(8192), // overlay_logical_size bigger than template but smaller than size_gb_override test above + ) + .unwrap(); + assert_eq!(fs::metadata(&dest2).unwrap().len(), 8192); + } + + #[test] + fn test_copy_overlay_template_rejects_traversal_path() { + let temp_dir = tempfile::tempdir().unwrap(); + let outside = temp_dir.path().join("outside.raw"); + let dest = temp_dir.path().join("overlay.raw"); + fs::write(&outside, b"x").unwrap(); + + let result = + copy_overlay_template(temp_dir.path(), Some("../outside.raw"), &dest, None, None); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidInput); + } + + #[cfg(unix)] + #[test] + fn test_create_or_copy_storage_disk_rejects_symlink_escape() { + use std::os::unix::fs::symlink; + + let temp_dir = tempfile::tempdir().unwrap(); + let outside_dir = tempfile::tempdir().unwrap(); + let outside_file = outside_dir.path().join("storage-template.ext4"); + fs::write(&outside_file, b"template").unwrap(); + + symlink(outside_dir.path(), temp_dir.path().join("symlink-out")).unwrap(); + + let storage_path = temp_dir.path().join("storage.ext4"); + let result = create_or_copy_storage_disk( + temp_dir.path(), + Some("symlink-out/storage-template.ext4"), + &storage_path, + None, + ); + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), std::io::ErrorKind::InvalidInput); + } + + #[cfg(unix)] + #[test] + fn test_create_or_copy_storage_disk_preserves_sparseness() { + use std::os::unix::fs::MetadataExt; + + // Build a sparse template: a small run of real data at the front, then a + // large trailing hole (the shape of the real storage-template.ext4). + let cache_dir = tempfile::tempdir().unwrap(); + let template = cache_dir.path().join("storage-template.ext4"); + { + let mut f = File::create(&template).unwrap(); + // A few real (non-zero) bytes at the front... + f.write_all(b"real ext4 superblock stand-in").unwrap(); + // ...then 1 GiB logical size, so the rest is a trailing hole. + f.set_len(1024 * 1024 * 1024).unwrap(); + } + + let dest = cache_dir.path().join("storage.ext4"); + create_or_copy_storage_disk(cache_dir.path(), Some("storage-template.ext4"), &dest, None) + .unwrap(); + + let meta = fs::metadata(&dest).unwrap(); + // Logical size is preserved... + assert_eq!(meta.len(), 1024 * 1024 * 1024); + // ...but the destination must NOT be densified: allocated blocks + // (512-byte units) should be a tiny fraction of the logical size. A + // dense copy would report ~2M blocks (1 GiB); a sparse one only a few. + let allocated_bytes = meta.blocks() * 512; + assert!( + allocated_bytes < 16 * 1024 * 1024, + "storage disk was densified: {allocated_bytes} bytes allocated for a sparse template" + ); + } + + #[test] + fn test_extract_sidecar_skips_when_already_extracted() { + // Verifies the double-check pattern inside the lock: + // if the marker exists and force=false, extraction is a no-op. + let temp_dir = tempfile::tempdir().unwrap(); + let cache_dir = temp_dir.path().join("cache"); + fs::create_dir_all(&cache_dir).unwrap(); + + // Write marker to simulate completed extraction + fs::write(cache_dir.join(EXTRACTION_MARKER), "").unwrap(); + + let dummy_footer = PackFooter { + stub_size: 0, + assets_offset: 0, + assets_size: 0, + manifest_offset: 0, + manifest_size: 0, + checksum: 0, + }; + + // Should succeed without trying to open a nonexistent sidecar, + // because the marker check short-circuits. + let result = extract_sidecar( + Path::new("/nonexistent/sidecar.smolmachine"), + &cache_dir, + &dummy_footer, + false, // force=false + false, + ); + // The sidecar doesn't exist, but we never try to open it because + // the marker file is already present. + // Note: the exists() check at the top will fail here, so this test + // verifies the locking path only when the sidecar exists. + // Let's adjust: use a real (empty) sidecar file for the existence check. + drop(result); + + let dummy_sidecar = temp_dir.path().join("dummy.smolmachine"); + fs::write(&dummy_sidecar, b"").unwrap(); + + let result = extract_sidecar( + &dummy_sidecar, + &cache_dir, + &dummy_footer, + false, // force=false + false, + ); + assert!(result.is_ok()); + } + + #[test] + fn test_extract_sidecar_force_clears_marker() { + // Verifies that force=true re-extracts even when the marker exists. + // We can't do a full extraction without a real sidecar, so we verify + // that force=true proceeds past the marker check (and then fails on + // the actual extraction — which is fine for this test). + let temp_dir = tempfile::tempdir().unwrap(); + let cache_dir = temp_dir.path().join("cache-force"); + fs::create_dir_all(&cache_dir).unwrap(); + + // Write marker + fs::write(cache_dir.join(EXTRACTION_MARKER), "").unwrap(); + assert!(is_extracted(&cache_dir)); + + // Create a dummy sidecar (empty — will fail during decompression) + let dummy_sidecar = temp_dir.path().join("force.smolmachine"); + fs::write(&dummy_sidecar, b"not-a-real-zstd-stream").unwrap(); + + let dummy_footer = PackFooter { + stub_size: 0, + assets_offset: 0, + assets_size: 22, // matches "not-a-real-zstd-stream".len() + manifest_offset: 22, + manifest_size: 0, + checksum: 0, + }; + + let result = extract_sidecar( + &dummy_sidecar, + &cache_dir, + &dummy_footer, + true, // force=true should bypass marker + false, + ); + + // Should fail during decompression (not short-circuit on marker), + // proving that force=true re-enters the extraction path. + assert!( + result.is_err(), + "force extraction should attempt (and fail on dummy data)" + ); + } + + /// Builds a tar archive in memory with the given entries. + /// Each entry is (path, is_dir, content). + fn build_tar(entries: &[(&str, bool, &[u8])]) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + for (path, is_dir, content) in entries { + let mut header = tar::Header::new_gnu(); + if *is_dir { + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o755); + } else { + header.set_entry_type(tar::EntryType::Regular); + header.set_size(content.len() as u64); + header.set_mode(0o644); + } + header.set_cksum(); + builder + .append_data(&mut header, *path, &content[..]) + .unwrap(); + } + builder.into_inner().unwrap() + } + + #[test] + fn test_safe_unpack_normal_tar() { + let temp_dir = tempfile::tempdir().unwrap(); + let dest_raw = temp_dir.path().join("out"); + fs::create_dir_all(&dest_raw).unwrap(); + // Canonicalize to resolve macOS /tmp -> /private/tmp symlink + let dest = dest_raw.canonicalize().unwrap(); + + let tar_data = build_tar(&[("dir/", true, b""), ("dir/file.txt", false, b"hello")]); + let mut archive = tar::Archive::new(tar_data.as_slice()); + safe_unpack(&mut archive, &dest).unwrap(); + + assert!(dest.join("dir").is_dir()); + assert_eq!( + fs::read_to_string(dest.join("dir/file.txt")).unwrap(), + "hello" + ); + } + + #[test] + #[cfg(target_os = "macos")] + fn test_safe_unpack_case_collision_fails_on_case_insensitive_fs() { + // On macOS case-insensitive APFS, extracting a tar with paths that + // differ only in case (e.g., "lower" file vs "Lower/" directory) + // should fail — callers must use a case-sensitive volume instead. + let temp_dir = tempfile::tempdir().unwrap(); + let dest_raw = temp_dir.path().join("out"); + fs::create_dir_all(&dest_raw).unwrap(); + let dest = dest_raw.canonicalize().unwrap(); + + let tar_data = build_tar(&[ + ("share/", true, b""), + ("share/pkg/", true, b""), + ("share/pkg/lower", false, b"script content"), + ("share/pkg/Lower/", true, b""), + ("share/pkg/Lower/__init__.py", false, b"python code"), + ]); + let mut archive = tar::Archive::new(tar_data.as_slice()); + + // Should fail on case-insensitive APFS — the caller is responsible + // for providing a case-sensitive destination (via acquire_layers_lease). + let result = safe_unpack(&mut archive, &dest); + assert!( + result.is_err(), + "case collision should fail on case-insensitive FS" + ); + } + + #[test] + #[cfg(target_os = "macos")] + fn test_layers_lease_creates_and_cleans_volume() { + // Verify that acquire_layers_lease creates a case-sensitive sparse + // image, mounts it, and detaches on lease drop. + // Skips gracefully if hdiutil is unavailable (CI, sandboxed envs). + let temp_dir = tempfile::tempdir().unwrap(); + let cache_dir = temp_dir.path().join("cache"); + // Create a dummy tar so has_layer_tars() returns true. + fs::create_dir_all(cache_dir.join("layers")).unwrap(); + fs::write(cache_dir.join("layers/dummy.tar"), b"").unwrap(); + + let lease = match acquire_layers_lease(&cache_dir, false) { + Ok(l) => l, + Err(e) => { + eprintln!("SKIP: hdiutil unavailable: {}", e); + return; + } + }; + assert!(lease.path.exists()); + assert!(is_mount_point(&lease.path)); + + // Both "lower" and "Lower" should coexist on the case-sensitive volume. + fs::write(lease.path.join("lower"), "file").unwrap(); + fs::create_dir_all(lease.path.join("Lower")).unwrap(); + assert!(lease.path.join("lower").exists()); + assert!(lease.path.join("Lower").is_dir()); + + // Lease file should exist while lease is held. + let lease_file = cache_dir + .join(LEASES_DIR) + .join(format!("{}", std::process::id())); + assert!(lease_file.exists()); + + // Drop lease — should detach volume (last lease). + let mount_point = lease.path.clone(); + drop(lease); + assert!( + !is_mount_point(&mount_point), + "volume should be detached after last lease drop" + ); + } + + #[test] + fn test_safe_unpack_skips_char_and_block_devices() { + // Char/Block entries appear in overlayfs exports from Debian images + // (e.g., update-alternatives). They should be skipped, not rejected. + let temp_dir = tempfile::tempdir().unwrap(); + let dest_raw = temp_dir.path().join("out"); + fs::create_dir_all(&dest_raw).unwrap(); + let dest = dest_raw.canonicalize().unwrap(); + + let mut builder = tar::Builder::new(Vec::new()); + + // Regular file before device entries + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(5); + header.set_mode(0o644); + header.set_path("before.txt").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "before.txt", &b"hello"[..]) + .unwrap(); + + // Char device entry (should be skipped) + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Char); + header.set_size(0); + header.set_mode(0o644); + header.set_path("etc/alternatives/pager.1.gz").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "etc/alternatives/pager.1.gz", &b""[..]) + .unwrap(); + + // Block device entry (should be skipped) + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Block); + header.set_size(0); + header.set_mode(0o644); + header.set_path("dev/sda").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "dev/sda", &b""[..]) + .unwrap(); + + // Regular file after device entries (must survive) + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(5); + header.set_mode(0o644); + header.set_path("after.txt").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "after.txt", &b"world"[..]) + .unwrap(); + + let tar_data = builder.into_inner().unwrap(); + + let mut archive = tar::Archive::new(tar_data.as_slice()); + let result = safe_unpack(&mut archive, &dest); + assert!( + result.is_ok(), + "Char/Block entries should be skipped: {:?}", + result.err() + ); + + // Files before AND after device entries are extracted + assert_eq!( + fs::read_to_string(dest.join("before.txt")).unwrap(), + "hello" + ); + assert_eq!(fs::read_to_string(dest.join("after.txt")).unwrap(), "world"); + + // Device entries are not created + assert!(!dest.join("etc/alternatives/pager.1.gz").exists()); + assert!(!dest.join("dev/sda").exists()); + } + + #[test] + fn test_safe_unpack_skips_hardlink_to_whiteout() { + // Overlayfs exports from Fedora produce hardlinks to char-device + // whiteout entries (e.g., .build-id symlinks referencing replaced + // base-layer files). The whiteout is skipped, so the hardlink target + // doesn't exist — the hardlink must be skipped too. + let temp_dir = tempfile::tempdir().unwrap(); + let dest_raw = temp_dir.path().join("out"); + fs::create_dir_all(&dest_raw).unwrap(); + let dest = dest_raw.canonicalize().unwrap(); + + let mut builder = tar::Builder::new(Vec::new()); + + // Char device whiteout (will be skipped) + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Char); + header.set_size(0); + header.set_mode(0o000); + header.set_path("usr/lib/.build-id/84/target").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/lib/.build-id/84/target", &b""[..]) + .unwrap(); + + // Hardlink to the skipped whiteout (should also be skipped) + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Link); + header.set_size(0); + header.set_mode(0o000); + header.set_path("usr/lib/.build-id/d9/link").unwrap(); + header.set_link_name("usr/lib/.build-id/84/target").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/lib/.build-id/d9/link", &b""[..]) + .unwrap(); + + // Regular file after (must survive) + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(2); + header.set_mode(0o644); + header.set_path("ok.txt").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "ok.txt", &b"ok"[..]) + .unwrap(); + + let tar_data = builder.into_inner().unwrap(); + + let mut archive = tar::Archive::new(tar_data.as_slice()); + let result = safe_unpack(&mut archive, &dest); + assert!( + result.is_ok(), + "hardlink to skipped whiteout should be skipped: {:?}", + result.err() + ); + + // Whiteout and hardlink are not created + assert!(!dest.join("usr/lib/.build-id/84/target").exists()); + assert!(!dest.join("usr/lib/.build-id/d9/link").exists()); + // Regular file survives + assert_eq!(fs::read_to_string(dest.join("ok.txt")).unwrap(), "ok"); + } + + #[test] + fn test_safe_unpack_readonly_parent_dir_does_not_block_children() { + // Reproduces the Fedora extraction bug: a mode-555 directory entry + // appears before its children in the tar. Without deferred permissions, + // creating files inside the read-only directory fails. + let temp_dir = tempfile::tempdir().unwrap(); + let dest_raw = temp_dir.path().join("out"); + fs::create_dir_all(&dest_raw).unwrap(); + let dest = dest_raw.canonicalize().unwrap(); + + let mut builder = tar::Builder::new(Vec::new()); + + // Parent directory with restrictive mode (read-only, no write) + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o555); // read + execute only, no write + header.set_path("usr/lib64/pm-utils/").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/lib64/pm-utils/", &b""[..]) + .unwrap(); + + // Child directory inside the read-only parent + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o555); + header.set_path("usr/lib64/pm-utils/module.d/").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/lib64/pm-utils/module.d/", &b""[..]) + .unwrap(); + + // File inside the nested read-only directory + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(4); + header.set_mode(0o644); + header + .set_path("usr/lib64/pm-utils/module.d/test.conf") + .unwrap(); + header.set_cksum(); + builder + .append_data( + &mut header, + "usr/lib64/pm-utils/module.d/test.conf", + &b"data"[..], + ) + .unwrap(); + + let tar_data = builder.into_inner().unwrap(); + + let mut archive = tar::Archive::new(tar_data.as_slice()); + let result = safe_unpack(&mut archive, &dest); + assert!( + result.is_ok(), + "read-only parent should not block children: {:?}", + result.err() + ); + + // Child directory and file must exist + assert!(dest.join("usr/lib64/pm-utils/module.d").is_dir()); + assert_eq!( + fs::read_to_string(dest.join("usr/lib64/pm-utils/module.d/test.conf")).unwrap(), + "data" + ); + + // Final permissions should be restored to the tar's mode (555) + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = fs::metadata(dest.join("usr/lib64/pm-utils")) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o555, "deferred directory mode should be 555"); + } + } + + #[test] + fn test_safe_unpack_mixed_fedora_overlay_layer() { + // Realistic Fedora overlay layer: regular files interspersed with + // whiteout char devices and hardlinks to those whiteouts. + // All good files should extract; bad entries should be skipped. + let temp_dir = tempfile::tempdir().unwrap(); + let dest_raw = temp_dir.path().join("out"); + fs::create_dir_all(&dest_raw).unwrap(); + let dest = dest_raw.canonicalize().unwrap(); + + let mut builder = tar::Builder::new(Vec::new()); + + // Directory + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_mode(0o755); + header.set_path("usr/").unwrap(); + header.set_cksum(); + builder.append_data(&mut header, "usr/", &b""[..]).unwrap(); + + // Good file 1 + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(11); + header.set_mode(0o644); + header.set_path("usr/good1.txt").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/good1.txt", &b"good file 1"[..]) + .unwrap(); + + // Char device whiteout + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Char); + header.set_size(0); + header.set_mode(0o000); + header.set_device_major(0).unwrap(); + header.set_device_minor(0).unwrap(); + header.set_path("usr/.wh.removed-pkg").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/.wh.removed-pkg", &b""[..]) + .unwrap(); + + // Good file 2 + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(11); + header.set_mode(0o644); + header.set_path("usr/good2.txt").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/good2.txt", &b"good file 2"[..]) + .unwrap(); + + // Hardlink to the whiteout (should be skipped) + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Link); + header.set_size(0); + header.set_mode(0o000); + header.set_path("usr/link-to-removed").unwrap(); + header.set_link_name("usr/.wh.removed-pkg").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/link-to-removed", &b""[..]) + .unwrap(); + + // Good file 3 + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(19); // == len("#!/usr/bin/env bash") + header.set_mode(0o755); + header.set_path("usr/good3.sh").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/good3.sh", &b"#!/usr/bin/env bash"[..]) + .unwrap(); + + // Another char device whiteout + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Char); + header.set_size(0); + header.set_mode(0o000); + header.set_device_major(0).unwrap(); + header.set_device_minor(0).unwrap(); + header.set_path("usr/.wh.another-removed").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/.wh.another-removed", &b""[..]) + .unwrap(); + + // Good file 4 (final entry) + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(5); + header.set_mode(0o644); + header.set_path("usr/good4.dat").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "usr/good4.dat", &b"final"[..]) + .unwrap(); + + let tar_data = builder.into_inner().unwrap(); + + let mut archive = tar::Archive::new(tar_data.as_slice()); + let result = safe_unpack(&mut archive, &dest); + assert!( + result.is_ok(), + "mixed Fedora overlay should extract cleanly: {:?}", + result.err() + ); + + // Good files are all extracted + assert_eq!( + fs::read_to_string(dest.join("usr/good1.txt")).unwrap(), + "good file 1" + ); + assert_eq!( + fs::read_to_string(dest.join("usr/good2.txt")).unwrap(), + "good file 2" + ); + assert_eq!( + fs::read_to_string(dest.join("usr/good3.sh")).unwrap(), + "#!/usr/bin/env bash" + ); + assert_eq!( + fs::read_to_string(dest.join("usr/good4.dat")).unwrap(), + "final" + ); + + // Bad entries are not created + assert!(!dest.join("usr/.wh.removed-pkg").exists()); + assert!(!dest.join("usr/link-to-removed").exists()); + assert!(!dest.join("usr/.wh.another-removed").exists()); + } + + #[test] + fn test_safe_unpack_unknown_tar_type_byte() { + // Entry with unknown tar type byte (0x41 = 'A') — a vendor extension + // not recognized by the tar crate (maps to __Nonexhaustive). + // Should be skipped gracefully by safe_unpack's catch-all arm. + // Note: byte '7' maps to EntryType::Continuous which is allowed. + let temp_dir = tempfile::tempdir().unwrap(); + let dest_raw = temp_dir.path().join("out"); + fs::create_dir_all(&dest_raw).unwrap(); + let dest = dest_raw.canonicalize().unwrap(); + + let mut builder = tar::Builder::new(Vec::new()); + + // Regular file before the unknown entry + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(6); + header.set_mode(0o644); + header.set_path("before.txt").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "before.txt", &b"before"[..]) + .unwrap(); + + // Unknown type byte entry ('A' = 0x41, truly unrecognized) + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::new(b'A')); + header.set_size(0); + header.set_mode(0o644); + header.set_path("unknown-type-entry").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "unknown-type-entry", &b""[..]) + .unwrap(); + + // Regular file after + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(5); + header.set_mode(0o644); + header.set_path("after.txt").unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, "after.txt", &b"after"[..]) + .unwrap(); + + let tar_data = builder.into_inner().unwrap(); + + let mut archive = tar::Archive::new(tar_data.as_slice()); + let result = safe_unpack(&mut archive, &dest); + assert!( + result.is_ok(), + "unknown tar type should be skipped: {:?}", + result.err() + ); + + assert_eq!( + fs::read_to_string(dest.join("before.txt")).unwrap(), + "before" + ); + assert_eq!(fs::read_to_string(dest.join("after.txt")).unwrap(), "after"); + assert!(!dest.join("unknown-type-entry").exists()); + } + + #[test] + fn test_evict_cache_to_size_lru() { + use std::ffi::CString; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let mk = |name: &str, mtime_secs: i64| { + let d = root.join(name); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("data"), vec![0u8; 1024 * 1024]).unwrap(); // ~1 MiB real + let c = CString::new(d.to_string_lossy().as_bytes()).unwrap(); + let tv = libc::timeval { + tv_sec: mtime_secs, + tv_usec: 0, + }; + let times = [tv, tv]; + unsafe { + libc::utimes(c.as_ptr(), times.as_ptr()); + } + d + }; + let old = mk("aaaa", 1_000_000); + let mid = mk("bbbb", 2_000_000); + let new = mk("cccc", 3_000_000); + + // Total (~3 MiB) is under the cap → nothing evicted. + assert_eq!(evict_cache_to_size(root, 100 * 1024 * 1024), 0); + assert!(old.exists() && mid.exists() && new.exists()); + + // Cap (~2.5 MiB) forces evicting the single oldest entry, LRU-first. + let freed = evict_cache_to_size(root, 5 * 1024 * 1024 / 2); + assert!(freed > 0, "expected some bytes freed"); + assert!(!old.exists(), "oldest extraction should be evicted"); + assert!(mid.exists() && new.exists(), "newer extractions kept"); + } + + // Lease tracking is a macOS case-sensitive-volume concept; on Linux + // `has_active_leases` is always false (layers live at cache_dir/layers). + #[cfg(target_os = "macos")] + #[test] + fn test_evict_cache_skips_active_lease() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + let d = root.join("aaaa"); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("data"), vec![0u8; 2 * 1024 * 1024]).unwrap(); + // Simulate a running pack: a live daemon lease file (current PID). + let leases = d.join(LEASES_DIR); + fs::create_dir_all(&leases).unwrap(); + fs::write(leases.join("daemon"), format!("{}", std::process::id())).unwrap(); + assert!(has_active_leases(&d), "live lease should be detected"); + // Cap of 0 would evict everything — but the active lease must be spared. + let freed = evict_cache_to_size(root, 0); + assert_eq!(freed, 0, "leased extraction must not be evicted"); + assert!(d.exists()); + } +} diff --git a/crates/smolvm-pack/src/format.rs b/crates/smolvm-pack/src/format.rs new file mode 100644 index 0000000..6eef06c --- /dev/null +++ b/crates/smolvm-pack/src/format.rs @@ -0,0 +1,653 @@ +//! `.smolmachine` binary format specification. +//! +//! # Overview +//! +//! A `.smolmachine` is a portable, self-contained microVM artifact. It bundles +//! everything needed to run a workload: OCI image layers, the agent rootfs, +//! runtime libraries (libkrun), and a manifest describing the configuration. +//! +//! # File Layout +//! +//! A `.smolmachine` file is a zstd-compressed tar archive with a JSON manifest +//! appended as a footer. The manifest is also stored inside the OCI registry +//! as the config blob when pushed. +//! +//! ```text +//! +---------------------------+ +//! | Assets Blob (zstd tar) | 30-150 MB +//! | - agent-rootfs.tar | Guest init system +//! | - layers/*.tar | OCI image layers +//! | - lib/libkrun.* | Runtime libraries (platform-specific) +//! | - storage.ext4 (opt) | Pre-formatted disk template +//! | - overlay.raw (opt) | VM snapshot (VM mode only) +//! +---------------------------+ +//! | Manifest (JSON) | ~2 KB (PackManifest) +//! +---------------------------+ +//! | Footer (64 bytes) | +//! | - magic: "SMOLPACK" | +//! | - version: 1 | +//! | - offsets + sizes | +//! | - CRC32 checksum | +//! +---------------------------+ +//! ``` +//! +//! # OCI Registry Representation +//! +//! When pushed to an OCI registry, a `.smolmachine` is stored as: +//! - **Config blob**: `PackManifest` JSON (`application/vnd.smolmachines.machine.config.v1+json`) +//! - **Layer blob**: The full `.smolmachine` file (`application/vnd.smolmachines.smolmachine.v1`) +//! - **Manifest**: Standard OCI Image Manifest referencing both blobs +//! +//! # Execution Modes +//! +//! - **Container mode** (default): OCI image layers are unpacked and run via crun. +//! - **VM mode**: An overlay disk snapshot is restored directly into the VM. + +use serde::{Deserialize, Serialize}; + +use crate::{PackError, Result}; + +/// Magic bytes identifying a packed smolvm binary. +pub const MAGIC: &[u8; 8] = b"SMOLPACK"; + +/// Magic bytes for embedded section header. +pub const SECTION_MAGIC: &[u8; 8] = b"SMOLSECT"; + +/// Magic bytes for libs footer appended to the stub binary. +pub const LIBS_MAGIC: &[u8; 8] = b"SMOLLIBS"; + +/// Current format version. +pub const FORMAT_VERSION: u32 = 1; + +/// Extension for sidecar assets file. +pub const SIDECAR_EXTENSION: &str = ".smolmachine"; + +/// Footer size in bytes (fixed). +pub const FOOTER_SIZE: usize = 64; + +/// Embedded section header size (fixed). +pub const SECTION_HEADER_SIZE: usize = 32; + +/// Libs footer size in bytes (fixed). +pub const LIBS_FOOTER_SIZE: usize = 32; + +/// Header for data embedded in the __SMOLVM,__smolvm Mach-O section. +/// +/// This format is used for macOS single-file binaries where assets are +/// stored inside the executable's Mach-O structure, allowing proper code signing. +/// +/// Layout (32 bytes total): +/// ```text +/// Offset Size Field +/// 0 8 magic ("SMOLSECT") +/// 8 4 version (u32 LE) +/// 12 4 manifest_size (u32 LE) +/// 16 8 assets_size (u64 LE) +/// 24 4 checksum (u32 LE) +/// 28 4 reserved (zeroes) +/// ``` +/// +/// Following the header: +/// - Manifest JSON (manifest_size bytes) +/// - Compressed assets (assets_size bytes) +#[derive(Debug, Clone, Copy)] +pub struct SectionHeader { + /// Size of manifest JSON in bytes. + pub manifest_size: u32, + /// Size of compressed assets in bytes. + pub assets_size: u64, + /// CRC32 checksum of manifest + assets. + pub checksum: u32, +} + +impl SectionHeader { + /// Serialize header to bytes. + pub fn to_bytes(&self) -> [u8; SECTION_HEADER_SIZE] { + let mut buf = [0u8; SECTION_HEADER_SIZE]; + + // Magic + buf[0..8].copy_from_slice(SECTION_MAGIC); + + // Version + buf[8..12].copy_from_slice(&FORMAT_VERSION.to_le_bytes()); + + // Manifest size + buf[12..16].copy_from_slice(&self.manifest_size.to_le_bytes()); + + // Assets size + buf[16..24].copy_from_slice(&self.assets_size.to_le_bytes()); + + // Checksum + buf[24..28].copy_from_slice(&self.checksum.to_le_bytes()); + + // Reserved (already zeroed) + + buf + } + + /// Deserialize header from bytes. + pub fn from_bytes(buf: &[u8]) -> Result { + if buf.len() < SECTION_HEADER_SIZE { + return Err(PackError::InvalidMagic); + } + + // Validate magic + if &buf[0..8] != SECTION_MAGIC { + return Err(PackError::InvalidMagic); + } + + // Check version + let version = u32::from_le_bytes([buf[8], buf[9], buf[10], buf[11]]); + if version != FORMAT_VERSION { + return Err(PackError::UnsupportedVersion(version)); + } + + Ok(Self { + manifest_size: u32::from_le_bytes([buf[12], buf[13], buf[14], buf[15]]), + assets_size: u64::from_le_bytes([ + buf[16], buf[17], buf[18], buf[19], buf[20], buf[21], buf[22], buf[23], + ]), + checksum: u32::from_le_bytes([buf[24], buf[25], buf[26], buf[27]]), + }) + } +} + +/// Footer appended to the stub binary to locate embedded runtime libraries. +/// +/// The stub reads its own last 32 bytes to find the compressed libs bundle, +/// extracts them to a cache directory, and dlopen's libkrun from there. +/// This keeps the .smolmachine sidecar cross-platform (no platform-specific libs). +/// +/// Layout (32 bytes total): +/// ```text +/// Offset Size Field +/// 0 8 magic ("SMOLLIBS") +/// 8 4 version (u32 LE) +/// 12 8 libs_offset (u64 LE) - offset to compressed libs blob +/// 20 8 libs_size (u64 LE) - size of compressed libs blob +/// 28 4 reserved (zeroes) +/// ``` +#[derive(Debug, Clone, Copy)] +pub struct LibsFooter { + /// Offset from start of file to the compressed libs blob. + pub libs_offset: u64, + /// Size of the compressed libs blob. + pub libs_size: u64, +} + +impl LibsFooter { + /// Serialize footer to bytes. + pub fn to_bytes(&self) -> [u8; LIBS_FOOTER_SIZE] { + let mut buf = [0u8; LIBS_FOOTER_SIZE]; + buf[0..8].copy_from_slice(LIBS_MAGIC); + buf[8..12].copy_from_slice(&1u32.to_le_bytes()); // version 1 + buf[12..20].copy_from_slice(&self.libs_offset.to_le_bytes()); + buf[20..28].copy_from_slice(&self.libs_size.to_le_bytes()); + // 28..32 reserved (zeroed) + buf + } + + /// Deserialize footer from bytes. + pub fn from_bytes(buf: &[u8; LIBS_FOOTER_SIZE]) -> Result { + if &buf[0..8] != LIBS_MAGIC { + return Err(PackError::InvalidMagic); + } + let version = u32::from_le_bytes([buf[8], buf[9], buf[10], buf[11]]); + if version != 1 { + return Err(PackError::UnsupportedVersion(version)); + } + Ok(Self { + libs_offset: u64::from_le_bytes([ + buf[12], buf[13], buf[14], buf[15], buf[16], buf[17], buf[18], buf[19], + ]), + libs_size: u64::from_le_bytes([ + buf[20], buf[21], buf[22], buf[23], buf[24], buf[25], buf[26], buf[27], + ]), + }) + } +} + +/// Fixed-size footer at the end of a packed binary. +/// +/// Layout (64 bytes total): +/// ```text +/// Offset Size Field +/// 0 8 magic ("SMOLPACK") +/// 8 4 version (u32 LE) +/// 12 8 stub_size (u64 LE) - size of stub executable +/// 20 8 assets_offset (u64 LE) - offset to compressed assets +/// 28 8 assets_size (u64 LE) - size of compressed assets +/// 36 8 manifest_offset (u64 LE) - offset to manifest JSON +/// 44 8 manifest_size (u64 LE) - size of manifest JSON +/// 52 4 checksum (u32 LE) - CRC32 of assets + manifest +/// 56 8 reserved (zeroes) +/// ``` +#[derive(Debug, Clone, Copy)] +pub struct PackFooter { + /// Size of the stub executable. + pub stub_size: u64, + /// Offset to compressed assets blob. + pub assets_offset: u64, + /// Size of compressed assets blob. + pub assets_size: u64, + /// Offset to manifest JSON. + pub manifest_offset: u64, + /// Size of manifest JSON. + pub manifest_size: u64, + /// CRC32 checksum of assets + manifest. + pub checksum: u32, +} + +impl PackFooter { + /// Serialize footer to bytes. + pub fn to_bytes(&self) -> [u8; FOOTER_SIZE] { + let mut buf = [0u8; FOOTER_SIZE]; + + // Magic + buf[0..8].copy_from_slice(MAGIC); + + // Version + buf[8..12].copy_from_slice(&FORMAT_VERSION.to_le_bytes()); + + // Stub size + buf[12..20].copy_from_slice(&self.stub_size.to_le_bytes()); + + // Assets offset and size + buf[20..28].copy_from_slice(&self.assets_offset.to_le_bytes()); + buf[28..36].copy_from_slice(&self.assets_size.to_le_bytes()); + + // Manifest offset and size + buf[36..44].copy_from_slice(&self.manifest_offset.to_le_bytes()); + buf[44..52].copy_from_slice(&self.manifest_size.to_le_bytes()); + + // Checksum + buf[52..56].copy_from_slice(&self.checksum.to_le_bytes()); + + // Reserved (already zeroed) + + buf + } + + /// Deserialize footer from bytes. + pub fn from_bytes(buf: &[u8; FOOTER_SIZE]) -> Result { + // Validate magic + if &buf[0..8] != MAGIC { + return Err(PackError::InvalidMagic); + } + + // Check version + let version = u32::from_le_bytes([buf[8], buf[9], buf[10], buf[11]]); + if version != FORMAT_VERSION { + return Err(PackError::UnsupportedVersion(version)); + } + + Ok(Self { + stub_size: u64::from_le_bytes([ + buf[12], buf[13], buf[14], buf[15], buf[16], buf[17], buf[18], buf[19], + ]), + assets_offset: u64::from_le_bytes([ + buf[20], buf[21], buf[22], buf[23], buf[24], buf[25], buf[26], buf[27], + ]), + assets_size: u64::from_le_bytes([ + buf[28], buf[29], buf[30], buf[31], buf[32], buf[33], buf[34], buf[35], + ]), + manifest_offset: u64::from_le_bytes([ + buf[36], buf[37], buf[38], buf[39], buf[40], buf[41], buf[42], buf[43], + ]), + manifest_size: u64::from_le_bytes([ + buf[44], buf[45], buf[46], buf[47], buf[48], buf[49], buf[50], buf[51], + ]), + checksum: u32::from_le_bytes([buf[52], buf[53], buf[54], buf[55]]), + }) + } +} + +/// Execution mode for packed binaries. +/// +/// Determines how commands are executed at runtime: +/// - `Container`: commands run inside a crun container (OCI layers) +/// - `Vm`: commands run directly in the VM rootfs (overlay disk) +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum PackMode { + /// Container mode: OCI image layers + crun container execution. + #[default] + Container, + /// VM mode: overlay disk + direct VM execution. + Vm, +} + +/// Manifest describing the packed image and configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PackManifest { + /// Execution mode (container or VM). + #[serde(default)] + pub mode: PackMode, + + /// Original image reference (e.g., "alpine:latest"). + pub image: String, + + /// Image digest (sha256:...). + pub digest: String, + + /// Target platform (e.g., "linux/arm64"). + pub platform: String, + + /// Entrypoint command (from image config or override). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub entrypoint: Vec, + + /// Default command arguments (from image config or override). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub cmd: Vec, + + /// Default environment variables. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub env: Vec, + + /// Secret references carried with the pack. Each maps an env var name to a + /// reference (host store entry, host env var, or file) — never an inline + /// value. The plaintext is resolved on the run host at exec time, so a + /// `.smolmachine` never contains secret material at rest. + #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")] + pub secret_refs: std::collections::BTreeMap, + + /// Working directory (from image config or override). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workdir: Option, + + /// Default number of vCPUs. + pub cpus: u8, + + /// Default memory in MiB. + pub mem: u32, + + /// Total extracted (on-disk) image size in bytes. + /// Used to auto-size the storage disk at runtime. + #[serde(default)] + pub image_size: u64, + + /// Whether outbound networking is enabled by default. + #[serde(default)] + pub network: bool, + + /// Enable GPU acceleration (Vulkan via virtio-gpu). + #[serde(default)] + pub gpu: bool, + + /// Host platform this .smolmachine runs on (e.g., "darwin/arm64"). + /// Distinct from `platform` which is the guest architecture (always linux). + /// Used for registry Image Index resolution. + pub host_platform: String, + + /// RFC 3339 timestamp when this machine was packed. + pub created: String, + + /// smolvm version that built this machine (e.g., "0.1.15"). + pub smolvm_version: String, + + /// Asset inventory - files included in the assets blob. + pub assets: AssetInventory, +} + +/// Inventory of assets included in the packed binary. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssetInventory { + /// Runtime libraries (relative paths within assets). + pub libraries: Vec, + + /// Agent rootfs tarball. + pub agent_rootfs: AssetEntry, + + /// OCI layer tarballs. + pub layers: Vec, + + /// Pre-formatted storage disk template (optional). + /// When present, copied to cache on first run instead of formatting at runtime. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub storage_template: Option, + + /// Overlay disk template (optional, VM mode only). + /// Contains the VM's persistent rootfs state from a `--from-vm` pack. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub overlay_template: Option, + + /// Original sparse size of the overlay disk, in bytes (optional, VM mode only). + /// + /// When set, the overlay.raw entry in the archive is a truncated copy with + /// the trailing sparse hole removed. On extraction the file is extended to + /// this size via `ftruncate`, restoring the sparse skeleton the VM expects. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub overlay_logical_size: Option, +} + +/// An asset file entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssetEntry { + /// Path within the assets archive. + pub path: String, + + /// Uncompressed size in bytes. + pub size: u64, +} + +/// An OCI layer entry. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LayerEntry { + /// Layer digest (sha256:...). + pub digest: String, + + /// Path within the assets archive. + pub path: String, + + /// Uncompressed size in bytes. + pub size: u64, +} + +/// Generate an RFC 3339 timestamp for the current time in UTC. +fn rfc3339_now() -> String { + let now = time::OffsetDateTime::now_utc(); + now.format(&time::format_description::well_known::Rfc3339) + .expect("RFC 3339 formatting should never fail for a valid OffsetDateTime") +} + +impl PackManifest { + /// Create a new manifest with default values. + pub fn new(image: String, digest: String, platform: String, host_platform: String) -> Self { + Self { + mode: PackMode::default(), + image, + digest, + platform, + entrypoint: Vec::new(), + cmd: Vec::new(), + env: Vec::new(), + secret_refs: std::collections::BTreeMap::new(), + workdir: None, + cpus: 1, + mem: 256, + image_size: 0, + network: false, + gpu: false, + host_platform, + created: rfc3339_now(), + smolvm_version: env!("CARGO_PKG_VERSION").to_string(), + assets: AssetInventory { + libraries: Vec::new(), + agent_rootfs: AssetEntry { + path: "agent-rootfs.tar".to_string(), + size: 0, + }, + layers: Vec::new(), + storage_template: None, + overlay_template: None, + overlay_logical_size: None, + }, + } + } + + /// Serialize manifest to JSON. + pub fn to_json(&self) -> Result> { + Ok(serde_json::to_vec_pretty(self)?) + } + + /// Deserialize manifest from JSON. + pub fn from_json(data: &[u8]) -> Result { + Ok(serde_json::from_slice(data)?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_manifest_secret_refs_roundtrip() { + // Secret refs survive a JSON round-trip through the manifest, and are + // omitted entirely when empty (skip_serializing_if) so existing packs + // stay byte-compatible. + let empty = PackManifest::new( + "alpine".to_string(), + "sha256:abc".to_string(), + "linux/arm64".to_string(), + "linux/arm64".to_string(), + ); + let json = String::from_utf8(empty.to_json().unwrap()).unwrap(); + assert!( + !json.contains("secret_refs"), + "empty secret_refs must not be serialized" + ); + + let mut m = empty; + m.secret_refs.insert( + "DB_PASSWORD".to_string(), + smolvm_protocol::SecretRef { + from_env: Some("PGPASSWORD".to_string()), + from_file: None, + }, + ); + let restored = PackManifest::from_json(&m.to_json().unwrap()).unwrap(); + assert_eq!(restored.secret_refs.len(), 1); + assert_eq!( + restored.secret_refs["DB_PASSWORD"].from_env.as_deref(), + Some("PGPASSWORD") + ); + } + + #[test] + fn test_footer_roundtrip() { + let footer = PackFooter { + stub_size: 512 * 1024, + assets_offset: 512 * 1024, + assets_size: 50 * 1024 * 1024, + manifest_offset: 512 * 1024 + 50 * 1024 * 1024, + manifest_size: 2048, + checksum: 0xDEADBEEF, + }; + + let bytes = footer.to_bytes(); + assert_eq!(bytes.len(), FOOTER_SIZE); + + let restored = PackFooter::from_bytes(&bytes).unwrap(); + assert_eq!(restored.stub_size, footer.stub_size); + assert_eq!(restored.assets_offset, footer.assets_offset); + assert_eq!(restored.assets_size, footer.assets_size); + assert_eq!(restored.manifest_offset, footer.manifest_offset); + assert_eq!(restored.manifest_size, footer.manifest_size); + assert_eq!(restored.checksum, footer.checksum); + } + + #[test] + fn test_footer_invalid_magic() { + let mut bytes = [0u8; FOOTER_SIZE]; + bytes[0..8].copy_from_slice(b"BADMAGIC"); + + let result = PackFooter::from_bytes(&bytes); + assert!(matches!(result, Err(PackError::InvalidMagic))); + } + + #[test] + fn test_footer_unsupported_version() { + let mut bytes = [0u8; FOOTER_SIZE]; + bytes[0..8].copy_from_slice(MAGIC); + bytes[8..12].copy_from_slice(&99u32.to_le_bytes()); // Bad version + + let result = PackFooter::from_bytes(&bytes); + assert!(matches!(result, Err(PackError::UnsupportedVersion(99)))); + } + + #[test] + fn test_manifest_roundtrip() { + let mut manifest = PackManifest::new( + "alpine:latest".to_string(), + "sha256:abc123".to_string(), + "linux/arm64".to_string(), + "darwin/arm64".to_string(), + ); + manifest.cpus = 2; + manifest.mem = 1024; + manifest.entrypoint = vec!["/bin/sh".to_string()]; + manifest.env = vec!["PATH=/usr/local/bin:/usr/bin:/bin".to_string()]; + manifest.assets.libraries.push(AssetEntry { + path: "lib/libkrun.dylib".to_string(), + size: 4 * 1024 * 1024, + }); + + let json = manifest.to_json().unwrap(); + let restored = PackManifest::from_json(&json).unwrap(); + + assert_eq!(restored.image, "alpine:latest"); + assert_eq!(restored.digest, "sha256:abc123"); + assert_eq!(restored.cpus, 2); + assert_eq!(restored.mem, 1024); + assert_eq!(restored.entrypoint, vec!["/bin/sh"]); + assert_eq!(restored.assets.libraries.len(), 1); + } + + #[test] + fn test_manifest_json_format() { + let manifest = PackManifest::new( + "ubuntu:22.04".to_string(), + "sha256:def456".to_string(), + "linux/amd64".to_string(), + "linux/amd64".to_string(), + ); + + let json = String::from_utf8(manifest.to_json().unwrap()).unwrap(); + assert!(json.contains("\"image\": \"ubuntu:22.04\"")); + assert!(json.contains("\"platform\": \"linux/amd64\"")); + // Phase 0 fields: verify key names serialize correctly + assert!(json.contains("\"host_platform\": \"linux/amd64\"")); + assert!(json.contains("\"smolvm_version\"")); + assert!(json.contains("\"created\"")); + } + + #[test] + fn test_pack_mode_default_is_container() { + assert_eq!(PackMode::default(), PackMode::Container); + } + + #[test] + fn test_pack_mode_vm_roundtrip() { + let mut manifest = PackManifest::new( + "vm://myvm".to_string(), + "none".to_string(), + "linux/arm64".to_string(), + "darwin/arm64".to_string(), + ); + manifest.mode = PackMode::Vm; + manifest.assets.overlay_template = Some(AssetEntry { + path: "overlay.raw".to_string(), + size: 2 * 1024 * 1024 * 1024, + }); + + let json = manifest.to_json().unwrap(); + let restored = PackManifest::from_json(&json).unwrap(); + assert_eq!(restored.mode, PackMode::Vm); + assert!(restored.assets.overlay_template.is_some()); + assert_eq!( + restored.assets.overlay_template.unwrap().path, + "overlay.raw" + ); + } +} diff --git a/crates/smolvm-pack/src/lib.rs b/crates/smolvm-pack/src/lib.rs new file mode 100644 index 0000000..25524d0 --- /dev/null +++ b/crates/smolvm-pack/src/lib.rs @@ -0,0 +1,78 @@ +//! `.smolmachine` packaging for smolvm. +//! +//! This crate provides functionality to package an OCI image and all runtime assets +//! into a portable `.smolmachine` artifact that can be pushed to a registry, +//! distributed, and run without smolvm installed. +//! +//! See [`format`] for the full binary format specification. + +#![deny(missing_docs)] + +pub mod assets; +pub mod detect; +pub mod extract; +pub mod format; +#[cfg(target_os = "macos")] +pub mod macho; +pub mod packer; +pub mod signing; + +pub use detect::{detect_packed_mode, PackedMode}; +pub use format::{ + PackFooter, PackManifest, PackMode, SectionHeader, FOOTER_SIZE, MAGIC, SECTION_HEADER_SIZE, + SECTION_MAGIC, SIDECAR_EXTENSION, +}; +pub use packer::{ + read_footer, read_footer_from_sidecar, read_manifest, read_manifest_from_sidecar, + sidecar_path_for, verify_sidecar_checksum, Packer, +}; + +use thiserror::Error; + +/// Errors that can occur during pack operations. +#[derive(Debug, Error)] +pub enum PackError { + /// I/O error. + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + + /// JSON serialization error. + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + + /// Invalid magic bytes in footer. + #[error("invalid magic: expected SMOLPACK")] + InvalidMagic, + + /// Unsupported format version. + #[error("unsupported version: {0}")] + UnsupportedVersion(u32), + + /// Checksum mismatch. + #[error("checksum mismatch: expected {expected:08x}, got {actual:08x}")] + ChecksumMismatch { + /// Expected checksum. + expected: u32, + /// Actual checksum. + actual: u32, + }, + + /// Asset not found. + #[error("asset not found: {0}")] + AssetNotFound(String), + + /// Compression error. + #[error("compression error: {0}")] + Compression(String), + + /// Signing error. + #[error("signing error: {0}")] + Signing(String), + + /// Tar archive error. + #[error("tar error: {0}")] + Tar(String), +} + +/// Result type for pack operations. +pub type Result = std::result::Result; diff --git a/crates/smolvm-pack/src/macho.rs b/crates/smolvm-pack/src/macho.rs new file mode 100644 index 0000000..47c446f --- /dev/null +++ b/crates/smolvm-pack/src/macho.rs @@ -0,0 +1,1305 @@ +//! Mach-O binary manipulation for macOS code signing. +//! +//! This module provides functionality to: +//! - Parse Mach-O binaries +//! - Modify sections (write data into placeholder sections) +//! - Generate adhoc code signatures +//! +//! Based on the approach used by Bun: https://github.com/oven-sh/bun/pull/17207 + +#![allow(missing_docs)] + +use std::io::{self, Read, Write}; + +/// Mach-O magic numbers +pub const MH_MAGIC_64: u32 = 0xfeedfacf; +pub const MH_CIGAM_64: u32 = 0xcffaedfe; + +/// Load command types +pub const LC_SEGMENT_64: u32 = 0x19; +pub const LC_CODE_SIGNATURE: u32 = 0x1d; +pub const LC_DYLD_INFO_ONLY: u32 = 0x80000022; +pub const LC_SYMTAB: u32 = 0x2; +pub const LC_DYSYMTAB: u32 = 0xb; +pub const LC_FUNCTION_STARTS: u32 = 0x26; +pub const LC_DATA_IN_CODE: u32 = 0x29; +pub const LC_DYLD_CHAINED_FIXUPS: u32 = 0x80000034; +pub const LC_DYLD_EXPORTS_TRIE: u32 = 0x80000033; + +/// Code signature magic +pub const CSMAGIC_EMBEDDED_SIGNATURE: u32 = 0xfade0cc0; +pub const CSMAGIC_CODEDIRECTORY: u32 = 0xfade0c02; +pub const CSMAGIC_REQUIREMENTS: u32 = 0xfade0c01; + +/// Code signature slot types (for BlobIndex) +pub const CSSLOT_CODEDIRECTORY: u32 = 0; +pub const CSSLOT_INFOSLOT: u32 = 1; +pub const CSSLOT_REQUIREMENTS: u32 = 2; + +/// Code signature constants +pub const CS_ADHOC: u32 = 0x0002; +pub const CS_LINKER_SIGNED: u32 = 0x20000; +pub const CS_EXECSEG_MAIN_BINARY: u32 = 0x1; + +/// Hash type for SHA256 +pub const CS_HASHTYPE_SHA256: u8 = 2; +pub const CS_SHA256_LEN: usize = 32; + +/// Page size for code signing (16KB on arm64) +pub const CS_PAGE_SIZE: usize = 16384; + +/// Mach-O 64-bit header +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct MachHeader64 { + pub magic: u32, + pub cputype: i32, + pub cpusubtype: i32, + pub filetype: u32, + pub ncmds: u32, + pub sizeofcmds: u32, + pub flags: u32, + pub reserved: u32, +} + +impl MachHeader64 { + pub const SIZE: usize = 32; + + pub fn read(reader: &mut R) -> io::Result { + let mut buf = [0u8; Self::SIZE]; + reader.read_exact(&mut buf)?; + Ok(Self { + magic: u32::from_le_bytes(buf[0..4].try_into().unwrap()), + cputype: i32::from_le_bytes(buf[4..8].try_into().unwrap()), + cpusubtype: i32::from_le_bytes(buf[8..12].try_into().unwrap()), + filetype: u32::from_le_bytes(buf[12..16].try_into().unwrap()), + ncmds: u32::from_le_bytes(buf[16..20].try_into().unwrap()), + sizeofcmds: u32::from_le_bytes(buf[20..24].try_into().unwrap()), + flags: u32::from_le_bytes(buf[24..28].try_into().unwrap()), + reserved: u32::from_le_bytes(buf[28..32].try_into().unwrap()), + }) + } + + pub fn write(&self, writer: &mut W) -> io::Result<()> { + writer.write_all(&self.magic.to_le_bytes())?; + writer.write_all(&self.cputype.to_le_bytes())?; + writer.write_all(&self.cpusubtype.to_le_bytes())?; + writer.write_all(&self.filetype.to_le_bytes())?; + writer.write_all(&self.ncmds.to_le_bytes())?; + writer.write_all(&self.sizeofcmds.to_le_bytes())?; + writer.write_all(&self.flags.to_le_bytes())?; + writer.write_all(&self.reserved.to_le_bytes())?; + Ok(()) + } +} + +/// Load command header +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct LoadCommand { + pub cmd: u32, + pub cmdsize: u32, +} + +impl LoadCommand { + pub const SIZE: usize = 8; + + pub fn read(reader: &mut R) -> io::Result { + let mut buf = [0u8; Self::SIZE]; + reader.read_exact(&mut buf)?; + Ok(Self { + cmd: u32::from_le_bytes(buf[0..4].try_into().unwrap()), + cmdsize: u32::from_le_bytes(buf[4..8].try_into().unwrap()), + }) + } +} + +/// Segment command (64-bit) +#[derive(Debug, Clone)] +pub struct SegmentCommand64 { + pub cmd: u32, + pub cmdsize: u32, + pub segname: [u8; 16], + pub vmaddr: u64, + pub vmsize: u64, + pub fileoff: u64, + pub filesize: u64, + pub maxprot: i32, + pub initprot: i32, + pub nsects: u32, + pub flags: u32, +} + +impl SegmentCommand64 { + pub const SIZE: usize = 72; + + pub fn read(reader: &mut R, cmd: u32, cmdsize: u32) -> io::Result { + let mut buf = [0u8; Self::SIZE - 8]; // Exclude cmd and cmdsize + reader.read_exact(&mut buf)?; + + let mut segname = [0u8; 16]; + segname.copy_from_slice(&buf[0..16]); + + Ok(Self { + cmd, + cmdsize, + segname, + vmaddr: u64::from_le_bytes(buf[16..24].try_into().unwrap()), + vmsize: u64::from_le_bytes(buf[24..32].try_into().unwrap()), + fileoff: u64::from_le_bytes(buf[32..40].try_into().unwrap()), + filesize: u64::from_le_bytes(buf[40..48].try_into().unwrap()), + maxprot: i32::from_le_bytes(buf[48..52].try_into().unwrap()), + initprot: i32::from_le_bytes(buf[52..56].try_into().unwrap()), + nsects: u32::from_le_bytes(buf[56..60].try_into().unwrap()), + flags: u32::from_le_bytes(buf[60..64].try_into().unwrap()), + }) + } + + pub fn name(&self) -> &str { + let len = self.segname.iter().position(|&c| c == 0).unwrap_or(16); + std::str::from_utf8(&self.segname[..len]).unwrap_or("") + } + + pub fn write(&self, writer: &mut W) -> io::Result<()> { + writer.write_all(&self.cmd.to_le_bytes())?; + writer.write_all(&self.cmdsize.to_le_bytes())?; + writer.write_all(&self.segname)?; + writer.write_all(&self.vmaddr.to_le_bytes())?; + writer.write_all(&self.vmsize.to_le_bytes())?; + writer.write_all(&self.fileoff.to_le_bytes())?; + writer.write_all(&self.filesize.to_le_bytes())?; + writer.write_all(&self.maxprot.to_le_bytes())?; + writer.write_all(&self.initprot.to_le_bytes())?; + writer.write_all(&self.nsects.to_le_bytes())?; + writer.write_all(&self.flags.to_le_bytes())?; + Ok(()) + } +} + +/// Section (64-bit) +#[derive(Debug, Clone)] +pub struct Section64 { + pub sectname: [u8; 16], + pub segname: [u8; 16], + pub addr: u64, + pub size: u64, + pub offset: u32, + pub align: u32, + pub reloff: u32, + pub nreloc: u32, + pub flags: u32, + pub reserved1: u32, + pub reserved2: u32, + pub reserved3: u32, +} + +impl Section64 { + pub const SIZE: usize = 80; + + pub fn read(reader: &mut R) -> io::Result { + let mut buf = [0u8; Self::SIZE]; + reader.read_exact(&mut buf)?; + + let mut sectname = [0u8; 16]; + let mut segname = [0u8; 16]; + sectname.copy_from_slice(&buf[0..16]); + segname.copy_from_slice(&buf[16..32]); + + Ok(Self { + sectname, + segname, + addr: u64::from_le_bytes(buf[32..40].try_into().unwrap()), + size: u64::from_le_bytes(buf[40..48].try_into().unwrap()), + offset: u32::from_le_bytes(buf[48..52].try_into().unwrap()), + align: u32::from_le_bytes(buf[52..56].try_into().unwrap()), + reloff: u32::from_le_bytes(buf[56..60].try_into().unwrap()), + nreloc: u32::from_le_bytes(buf[60..64].try_into().unwrap()), + flags: u32::from_le_bytes(buf[64..68].try_into().unwrap()), + reserved1: u32::from_le_bytes(buf[68..72].try_into().unwrap()), + reserved2: u32::from_le_bytes(buf[72..76].try_into().unwrap()), + reserved3: u32::from_le_bytes(buf[76..80].try_into().unwrap()), + }) + } + + pub fn name(&self) -> &str { + let len = self.sectname.iter().position(|&c| c == 0).unwrap_or(16); + std::str::from_utf8(&self.sectname[..len]).unwrap_or("") + } + + pub fn segment_name(&self) -> &str { + let len = self.segname.iter().position(|&c| c == 0).unwrap_or(16); + std::str::from_utf8(&self.segname[..len]).unwrap_or("") + } + + pub fn write(&self, writer: &mut W) -> io::Result<()> { + writer.write_all(&self.sectname)?; + writer.write_all(&self.segname)?; + writer.write_all(&self.addr.to_le_bytes())?; + writer.write_all(&self.size.to_le_bytes())?; + writer.write_all(&self.offset.to_le_bytes())?; + writer.write_all(&self.align.to_le_bytes())?; + writer.write_all(&self.reloff.to_le_bytes())?; + writer.write_all(&self.nreloc.to_le_bytes())?; + writer.write_all(&self.flags.to_le_bytes())?; + writer.write_all(&self.reserved1.to_le_bytes())?; + writer.write_all(&self.reserved2.to_le_bytes())?; + writer.write_all(&self.reserved3.to_le_bytes())?; + Ok(()) + } +} + +/// Code signature linkedit data command +#[derive(Debug, Clone, Copy)] +pub struct LinkeditDataCommand { + pub cmd: u32, + pub cmdsize: u32, + pub dataoff: u32, + pub datasize: u32, +} + +impl LinkeditDataCommand { + pub const SIZE: usize = 16; + + pub fn read(reader: &mut R, cmd: u32, cmdsize: u32) -> io::Result { + let mut buf = [0u8; 8]; + reader.read_exact(&mut buf)?; + Ok(Self { + cmd, + cmdsize, + dataoff: u32::from_le_bytes(buf[0..4].try_into().unwrap()), + datasize: u32::from_le_bytes(buf[4..8].try_into().unwrap()), + }) + } + + pub fn write(&self, writer: &mut W) -> io::Result<()> { + writer.write_all(&self.cmd.to_le_bytes())?; + writer.write_all(&self.cmdsize.to_le_bytes())?; + writer.write_all(&self.dataoff.to_le_bytes())?; + writer.write_all(&self.datasize.to_le_bytes())?; + Ok(()) + } +} + +/// Symtab command +#[derive(Debug, Clone, Copy)] +pub struct SymtabCommand { + pub cmd: u32, + pub cmdsize: u32, + pub symoff: u32, + pub nsyms: u32, + pub stroff: u32, + pub strsize: u32, +} + +impl SymtabCommand { + pub fn read(reader: &mut R, cmd: u32, cmdsize: u32) -> io::Result { + let mut buf = [0u8; 16]; + reader.read_exact(&mut buf)?; + Ok(Self { + cmd, + cmdsize, + symoff: u32::from_le_bytes(buf[0..4].try_into().unwrap()), + nsyms: u32::from_le_bytes(buf[4..8].try_into().unwrap()), + stroff: u32::from_le_bytes(buf[8..12].try_into().unwrap()), + strsize: u32::from_le_bytes(buf[12..16].try_into().unwrap()), + }) + } +} + +/// Dysymtab command +#[derive(Debug, Clone, Copy)] +pub struct DysymtabCommand { + pub cmd: u32, + pub cmdsize: u32, + pub ilocalsym: u32, + pub nlocalsym: u32, + pub iextdefsym: u32, + pub nextdefsym: u32, + pub iundefsym: u32, + pub nundefsym: u32, + pub tocoff: u32, + pub ntoc: u32, + pub modtaboff: u32, + pub nmodtab: u32, + pub extrefsymoff: u32, + pub nextrefsyms: u32, + pub indirectsymoff: u32, + pub nindirectsyms: u32, + pub extreloff: u32, + pub nextrel: u32, + pub locreloff: u32, + pub nlocrel: u32, +} + +impl DysymtabCommand { + pub fn read(reader: &mut R, cmd: u32, cmdsize: u32) -> io::Result { + let mut buf = [0u8; 72]; + reader.read_exact(&mut buf)?; + Ok(Self { + cmd, + cmdsize, + ilocalsym: u32::from_le_bytes(buf[0..4].try_into().unwrap()), + nlocalsym: u32::from_le_bytes(buf[4..8].try_into().unwrap()), + iextdefsym: u32::from_le_bytes(buf[8..12].try_into().unwrap()), + nextdefsym: u32::from_le_bytes(buf[12..16].try_into().unwrap()), + iundefsym: u32::from_le_bytes(buf[16..20].try_into().unwrap()), + nundefsym: u32::from_le_bytes(buf[20..24].try_into().unwrap()), + tocoff: u32::from_le_bytes(buf[24..28].try_into().unwrap()), + ntoc: u32::from_le_bytes(buf[28..32].try_into().unwrap()), + modtaboff: u32::from_le_bytes(buf[32..36].try_into().unwrap()), + nmodtab: u32::from_le_bytes(buf[36..40].try_into().unwrap()), + extrefsymoff: u32::from_le_bytes(buf[40..44].try_into().unwrap()), + nextrefsyms: u32::from_le_bytes(buf[44..48].try_into().unwrap()), + indirectsymoff: u32::from_le_bytes(buf[48..52].try_into().unwrap()), + nindirectsyms: u32::from_le_bytes(buf[52..56].try_into().unwrap()), + extreloff: u32::from_le_bytes(buf[56..60].try_into().unwrap()), + nextrel: u32::from_le_bytes(buf[60..64].try_into().unwrap()), + locreloff: u32::from_le_bytes(buf[64..68].try_into().unwrap()), + nlocrel: u32::from_le_bytes(buf[68..72].try_into().unwrap()), + }) + } +} + +/// DyldInfo command (LC_DYLD_INFO or LC_DYLD_INFO_ONLY) +#[derive(Debug, Clone, Copy)] +pub struct DyldInfoCommand { + pub cmd: u32, + pub cmdsize: u32, + pub rebase_off: u32, + pub rebase_size: u32, + pub bind_off: u32, + pub bind_size: u32, + pub weak_bind_off: u32, + pub weak_bind_size: u32, + pub lazy_bind_off: u32, + pub lazy_bind_size: u32, + pub export_off: u32, + pub export_size: u32, +} + +impl DyldInfoCommand { + pub fn read(reader: &mut R, cmd: u32, cmdsize: u32) -> io::Result { + let mut buf = [0u8; 40]; + reader.read_exact(&mut buf)?; + Ok(Self { + cmd, + cmdsize, + rebase_off: u32::from_le_bytes(buf[0..4].try_into().unwrap()), + rebase_size: u32::from_le_bytes(buf[4..8].try_into().unwrap()), + bind_off: u32::from_le_bytes(buf[8..12].try_into().unwrap()), + bind_size: u32::from_le_bytes(buf[12..16].try_into().unwrap()), + weak_bind_off: u32::from_le_bytes(buf[16..20].try_into().unwrap()), + weak_bind_size: u32::from_le_bytes(buf[20..24].try_into().unwrap()), + lazy_bind_off: u32::from_le_bytes(buf[24..28].try_into().unwrap()), + lazy_bind_size: u32::from_le_bytes(buf[28..32].try_into().unwrap()), + export_off: u32::from_le_bytes(buf[32..36].try_into().unwrap()), + export_size: u32::from_le_bytes(buf[36..40].try_into().unwrap()), + }) + } +} + +/// Parsed load command with data +#[derive(Debug)] +pub enum ParsedLoadCommand { + Segment64 { + segment: SegmentCommand64, + sections: Vec, + }, + CodeSignature(LinkeditDataCommand), + FunctionStarts(LinkeditDataCommand), + DataInCode(LinkeditDataCommand), + DyldChainedFixups(LinkeditDataCommand), + DyldExportsTrie(LinkeditDataCommand), + Symtab(SymtabCommand), + Dysymtab(DysymtabCommand), + DyldInfo(DyldInfoCommand), + Other { + cmd: u32, + data: Vec, + }, +} + +/// Mach-O file for manipulation +pub struct MachoFile { + pub header: MachHeader64, + pub load_commands: Vec, + /// File data after load commands + pub file_data: Vec, + /// Offset where file_data starts + pub data_offset: usize, +} + +impl MachoFile { + /// Parse a Mach-O file from bytes + pub fn parse(data: &[u8]) -> io::Result { + let mut cursor = std::io::Cursor::new(data); + + // Read header + let header = MachHeader64::read(&mut cursor)?; + if header.magic != MH_MAGIC_64 && header.magic != MH_CIGAM_64 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid Mach-O magic: 0x{:x}", header.magic), + )); + } + + // Read load commands + let mut load_commands = Vec::with_capacity(header.ncmds as usize); + for _ in 0..header.ncmds { + let cmd_start = cursor.position() as usize; + let lc = LoadCommand::read(&mut cursor)?; + + let parsed = match lc.cmd { + LC_SEGMENT_64 => { + let segment = SegmentCommand64::read(&mut cursor, lc.cmd, lc.cmdsize)?; + let mut sections = Vec::with_capacity(segment.nsects as usize); + for _ in 0..segment.nsects { + sections.push(Section64::read(&mut cursor)?); + } + ParsedLoadCommand::Segment64 { segment, sections } + } + LC_CODE_SIGNATURE => { + let cmd = LinkeditDataCommand::read(&mut cursor, lc.cmd, lc.cmdsize)?; + ParsedLoadCommand::CodeSignature(cmd) + } + LC_FUNCTION_STARTS => { + let cmd = LinkeditDataCommand::read(&mut cursor, lc.cmd, lc.cmdsize)?; + ParsedLoadCommand::FunctionStarts(cmd) + } + LC_DATA_IN_CODE => { + let cmd = LinkeditDataCommand::read(&mut cursor, lc.cmd, lc.cmdsize)?; + ParsedLoadCommand::DataInCode(cmd) + } + LC_DYLD_CHAINED_FIXUPS => { + let cmd = LinkeditDataCommand::read(&mut cursor, lc.cmd, lc.cmdsize)?; + ParsedLoadCommand::DyldChainedFixups(cmd) + } + LC_DYLD_EXPORTS_TRIE => { + let cmd = LinkeditDataCommand::read(&mut cursor, lc.cmd, lc.cmdsize)?; + ParsedLoadCommand::DyldExportsTrie(cmd) + } + LC_SYMTAB => { + let cmd = SymtabCommand::read(&mut cursor, lc.cmd, lc.cmdsize)?; + ParsedLoadCommand::Symtab(cmd) + } + LC_DYSYMTAB => { + let cmd = DysymtabCommand::read(&mut cursor, lc.cmd, lc.cmdsize)?; + ParsedLoadCommand::Dysymtab(cmd) + } + LC_DYLD_INFO_ONLY => { + let cmd = DyldInfoCommand::read(&mut cursor, lc.cmd, lc.cmdsize)?; + ParsedLoadCommand::DyldInfo(cmd) + } + _ => { + // Read remaining bytes for this command + let remaining = lc.cmdsize as usize - LoadCommand::SIZE; + let mut cmd_data = vec![0u8; remaining]; + cursor.read_exact(&mut cmd_data)?; + ParsedLoadCommand::Other { + cmd: lc.cmd, + data: cmd_data, + } + } + }; + + // Ensure we're at the right position + let expected_end = cmd_start + lc.cmdsize as usize; + cursor.set_position(expected_end as u64); + + load_commands.push(parsed); + } + + let data_offset = cursor.position() as usize; + let file_data = data[data_offset..].to_vec(); + + Ok(Self { + header, + load_commands, + file_data, + data_offset, + }) + } + + /// Find a section by segment and section name + pub fn find_section(&self, seg_name: &str, sect_name: &str) -> Option<(&Section64, usize)> { + for (cmd_idx, cmd) in self.load_commands.iter().enumerate() { + if let ParsedLoadCommand::Segment64 { sections, .. } = cmd { + for section in sections { + if section.segment_name() == seg_name && section.name() == sect_name { + return Some((section, cmd_idx)); + } + } + } + } + None + } + + /// Find the __LINKEDIT segment + pub fn find_linkedit(&self) -> Option<&SegmentCommand64> { + for cmd in &self.load_commands { + if let ParsedLoadCommand::Segment64 { segment, .. } = cmd { + if segment.name() == "__LINKEDIT" { + return Some(segment); + } + } + } + None + } + + /// Get code signature command if present + pub fn code_signature(&self) -> Option<&LinkeditDataCommand> { + for cmd in &self.load_commands { + if let ParsedLoadCommand::CodeSignature(cs) = cmd { + return Some(cs); + } + } + None + } + + /// Align a value up to the page size boundary (16KB for arm64). + fn page_align(size: usize) -> usize { + (size + CS_PAGE_SIZE - 1) & !(CS_PAGE_SIZE - 1) + } + + /// Write data into a section, expanding the binary as needed. + /// + /// This is designed for sections in their own dedicated segment (like __SMOLVM). + /// It updates the section size, segment sizes, and all file offsets for content + /// that follows in the binary. + pub fn write_section( + &mut self, + seg_name: &str, + sect_name: &str, + data: &[u8], + ) -> io::Result<()> { + // Find the section + let (section_offset, old_size, cmd_idx, sect_idx) = self + .find_section_details(seg_name, sect_name) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + format!("Section ({},{}) not found", seg_name, sect_name), + ) + })?; + + let new_size = data.len(); + let aligned_new_size = Self::page_align(new_size); + + // Calculate the page-aligned delta for shifting subsequent content. + // The delta must be page-aligned so LINKEDIT remains at a page boundary. + let delta = if aligned_new_size > old_size { + Self::page_align(aligned_new_size - old_size) as i64 + } else { + -(Self::page_align(old_size - aligned_new_size) as i64) + }; + + // Modify file_data: expand or contract at the section location + let relative_offset = section_offset - self.data_offset; + self.resize_file_data(relative_offset, old_size, aligned_new_size, delta); + + // Write the actual data and zero-pad to alignment + self.file_data[relative_offset..relative_offset + new_size].copy_from_slice(data); + for byte in + &mut self.file_data[relative_offset + new_size..relative_offset + aligned_new_size] + { + *byte = 0; + } + + // Update section and segment metadata + let segment_vmaddr = + self.update_segment_for_section(cmd_idx, sect_idx, aligned_new_size as u64, delta); + + // Shift all content after this section + if delta != 0 { + self.shift_offsets_after(section_offset + old_size, segment_vmaddr, delta); + } + + Ok(()) + } + + /// Find section details: (file_offset, size, cmd_index, section_index). + fn find_section_details( + &self, + seg_name: &str, + sect_name: &str, + ) -> Option<(usize, usize, usize, usize)> { + for (cmd_idx, cmd) in self.load_commands.iter().enumerate() { + if let ParsedLoadCommand::Segment64 { sections, .. } = cmd { + for (sect_idx, section) in sections.iter().enumerate() { + if section.segment_name() == seg_name && section.name() == sect_name { + return Some(( + section.offset as usize, + section.size as usize, + cmd_idx, + sect_idx, + )); + } + } + } + } + None + } + + /// Resize file_data by inserting or removing bytes at the given position. + fn resize_file_data( + &mut self, + relative_offset: usize, + old_size: usize, + new_size: usize, + delta: i64, + ) { + if delta > 0 { + // Expand: insert bytes after old content + let insert_pos = relative_offset + old_size; + self.file_data + .splice(insert_pos..insert_pos, vec![0u8; delta as usize]); + } else if delta < 0 { + // Shrink: remove bytes + let remove_start = relative_offset + new_size; + let remove_end = relative_offset + old_size; + if remove_end > remove_start { + self.file_data.drain(remove_start..remove_end); + } + } + } + + /// Update segment and section metadata after resizing a section. + /// Returns the segment's vmaddr for use in shifting subsequent segments. + fn update_segment_for_section( + &mut self, + cmd_idx: usize, + sect_idx: usize, + new_section_size: u64, + delta: i64, + ) -> u64 { + if let ParsedLoadCommand::Segment64 { segment, sections } = &mut self.load_commands[cmd_idx] + { + let section_addr = sections[sect_idx].addr; + let old_vmsize = segment.vmsize; + + // Update section size + sections[sect_idx].size = new_section_size; + + // Update segment file size + segment.filesize = (segment.filesize as i64 + delta) as u64; + + // Calculate required vmsize to contain the section + let section_offset_in_segment = section_addr - segment.vmaddr; + let required_vmsize = section_offset_in_segment + new_section_size; + let aligned_vmsize = Self::page_align(required_vmsize as usize) as u64; + + segment.vmsize = std::cmp::max((old_vmsize as i64 + delta) as u64, aligned_vmsize); + + segment.vmaddr + } else { + 0 + } + } + + /// Shift file offsets and vmaddrs for all content after the modification point. + fn shift_offsets_after(&mut self, after_offset: usize, modified_vmaddr: u64, delta: i64) { + for cmd in &mut self.load_commands { + match cmd { + ParsedLoadCommand::Segment64 { segment, sections } => { + let is_modified = segment.vmaddr == modified_vmaddr; + let should_move = segment.vmaddr > modified_vmaddr && segment.vmaddr != 0; + + // Shift segment file offset + if segment.fileoff as usize > after_offset { + segment.fileoff = (segment.fileoff as i64 + delta) as u64; + } + + // Shift segment vmaddr for segments after the modified one + if should_move { + segment.vmaddr = (segment.vmaddr as i64 + delta) as u64; + } + + // Shift sections (skip the modified segment - already handled) + if !is_modified { + for section in sections { + if section.offset as usize > after_offset { + section.offset = (section.offset as i64 + delta) as u32; + } + if should_move { + section.addr = (section.addr as i64 + delta) as u64; + } + } + } + } + ParsedLoadCommand::CodeSignature(lc) + | ParsedLoadCommand::FunctionStarts(lc) + | ParsedLoadCommand::DataInCode(lc) + | ParsedLoadCommand::DyldChainedFixups(lc) + | ParsedLoadCommand::DyldExportsTrie(lc) => { + if lc.dataoff as usize > after_offset { + lc.dataoff = (lc.dataoff as i64 + delta) as u32; + } + } + ParsedLoadCommand::Symtab(st) => { + Self::shift_offset_if_after(&mut st.symoff, after_offset, delta); + Self::shift_offset_if_after(&mut st.stroff, after_offset, delta); + } + ParsedLoadCommand::Dysymtab(dst) => { + Self::shift_offset_if_after(&mut dst.tocoff, after_offset, delta); + Self::shift_offset_if_after(&mut dst.modtaboff, after_offset, delta); + Self::shift_offset_if_after(&mut dst.extrefsymoff, after_offset, delta); + Self::shift_offset_if_after(&mut dst.indirectsymoff, after_offset, delta); + Self::shift_offset_if_after(&mut dst.extreloff, after_offset, delta); + Self::shift_offset_if_after(&mut dst.locreloff, after_offset, delta); + } + ParsedLoadCommand::DyldInfo(di) => { + Self::shift_offset_if_after(&mut di.rebase_off, after_offset, delta); + Self::shift_offset_if_after(&mut di.bind_off, after_offset, delta); + Self::shift_offset_if_after(&mut di.weak_bind_off, after_offset, delta); + Self::shift_offset_if_after(&mut di.lazy_bind_off, after_offset, delta); + Self::shift_offset_if_after(&mut di.export_off, after_offset, delta); + } + ParsedLoadCommand::Other { .. } => {} + } + } + } + + /// Helper to shift an offset if it's after the given position and non-zero. + fn shift_offset_if_after(offset: &mut u32, after: usize, delta: i64) { + if *offset != 0 && (*offset as usize) > after { + *offset = (*offset as i64 + delta) as u32; + } + } + + /// Generate and embed an adhoc code signature + pub fn sign_adhoc(&mut self) -> io::Result<()> { + use sha2::{Digest, Sha256}; + + // Remove existing code signature if present + let mut cs_idx = None; + for (i, cmd) in self.load_commands.iter().enumerate() { + if matches!(cmd, ParsedLoadCommand::CodeSignature(_)) { + cs_idx = Some(i); + break; + } + } + + // If there's an existing signature, remove it from file_data + if let Some(idx) = cs_idx { + if let ParsedLoadCommand::CodeSignature(old_cs) = &self.load_commands[idx] { + let sig_offset = old_cs.dataoff as usize; + if sig_offset >= self.data_offset + && sig_offset < self.data_offset + self.file_data.len() + { + let relative = sig_offset - self.data_offset; + self.file_data.truncate(relative); + } + } + // Keep the load command, we'll update it + } + + let code_size = self.data_offset + self.file_data.len(); + let code_size_aligned = (code_size + CS_PAGE_SIZE - 1) & !(CS_PAGE_SIZE - 1); + + // Pad file_data to page alignment + let padding_needed = code_size_aligned - code_size; + self.file_data.extend(vec![0u8; padding_needed]); + + // Calculate signature offset (after padded code) + let sig_offset = self.data_offset + self.file_data.len(); + + // Build the code signature parameters + let identifier = b"smolvm-packed\0"; + let num_pages = code_size_aligned.div_ceil(CS_PAGE_SIZE); + + // Calculate sizes (version 0x20100 format = 48 bytes header) + let cd_hash_offset = 48; // CodeDirectory header size for version 0x20100 + let cd_ident_offset = cd_hash_offset + (num_pages * CS_SHA256_LEN); + let cd_size = cd_ident_offset + identifier.len(); + let cd_size_aligned = (cd_size + 3) & !3; // 4-byte align + + // SuperBlob header (12 bytes) + 1 blob index (8 bytes) + CodeDirectory + let sig_size = 12 + 8 + cd_size_aligned; + let sig_size_aligned = (sig_size + 15) & !15; // 16-byte align + + // IMPORTANT: Update load commands BEFORE hashing so the hash matches what we write + let new_cs = LinkeditDataCommand { + cmd: LC_CODE_SIGNATURE, + cmdsize: LinkeditDataCommand::SIZE as u32, + dataoff: sig_offset as u32, + datasize: sig_size_aligned as u32, + }; + + if let Some(idx) = cs_idx { + self.load_commands[idx] = ParsedLoadCommand::CodeSignature(new_cs); + } else { + self.load_commands + .push(ParsedLoadCommand::CodeSignature(new_cs)); + self.header.ncmds += 1; + self.header.sizeofcmds += LinkeditDataCommand::SIZE as u32; + } + + // Update __LINKEDIT segment to include the signature + for cmd in &mut self.load_commands { + if let ParsedLoadCommand::Segment64 { segment, .. } = cmd { + if segment.name() == "__LINKEDIT" { + let linkedit_end = sig_offset + sig_size_aligned; + segment.filesize = (linkedit_end - segment.fileoff as usize) as u64; + segment.vmsize = + (segment.filesize + CS_PAGE_SIZE as u64 - 1) & !(CS_PAGE_SIZE as u64 - 1); + break; + } + } + } + + // Now build binary for hashing (with updated load commands, but without signature data) + let full_binary = self.build_binary_for_hashing(code_size_aligned); + + // Build CodeDirectory header + let mut code_directory = Vec::with_capacity(cd_size_aligned); + + // CodeDirectory header (version 0x20100 format - 48 bytes) + code_directory.extend(&CSMAGIC_CODEDIRECTORY.to_be_bytes()); // magic + code_directory.extend(&(cd_size as u32).to_be_bytes()); // length + code_directory.extend(&0x20100u32.to_be_bytes()); // version (use 0x20100 for simpler format) + code_directory.extend(&CS_ADHOC.to_be_bytes()); // flags + code_directory.extend(&(cd_hash_offset as u32).to_be_bytes()); // hashOffset + code_directory.extend(&(cd_ident_offset as u32).to_be_bytes()); // identOffset + code_directory.extend(&0u32.to_be_bytes()); // nSpecialSlots + code_directory.extend(&(num_pages as u32).to_be_bytes()); // nCodeSlots + code_directory.extend(&(code_size_aligned as u32).to_be_bytes()); // codeLimit + code_directory.push(CS_SHA256_LEN as u8); // hashSize (32 for SHA256) + code_directory.push(CS_HASHTYPE_SHA256); // hashType + code_directory.push(0); // platform + code_directory.push(14); // pageSize (log2 of 16384 = 14) + code_directory.extend(&0u32.to_be_bytes()); // spare2 + code_directory.extend(&0u32.to_be_bytes()); // scatterOffset (version 0x20100) + + // Hash each page + for page_idx in 0..num_pages { + let page_start = page_idx * CS_PAGE_SIZE; + let page_end = std::cmp::min(page_start + CS_PAGE_SIZE, full_binary.len()); + let page_data = &full_binary[page_start..page_end]; + + let mut hasher = Sha256::new(); + hasher.update(page_data); + let hash = hasher.finalize(); + code_directory.extend(&hash[..]); + } + + // Identifier + code_directory.extend(identifier); + + // Pad to alignment + while code_directory.len() < cd_size_aligned { + code_directory.push(0); + } + + // Build SuperBlob + let mut signature = Vec::with_capacity(sig_size_aligned); + signature.extend(&CSMAGIC_EMBEDDED_SIGNATURE.to_be_bytes()); // magic + signature.extend(&(sig_size as u32).to_be_bytes()); // length + signature.extend(&1u32.to_be_bytes()); // count (1 blob) + + // Blob index for CodeDirectory + signature.extend(&CSSLOT_CODEDIRECTORY.to_be_bytes()); // type (slot type, not magic) + signature.extend(&20u32.to_be_bytes()); // offset (after superblob header + index) + + // CodeDirectory blob + signature.extend(&code_directory); + + // Pad to alignment + while signature.len() < sig_size_aligned { + signature.push(0); + } + + // Append signature to file_data + self.file_data.extend(&signature); + + Ok(()) + } + + /// Build binary data for hashing (header + load commands + file data) + fn build_binary_for_hashing(&self, total_size: usize) -> Vec { + let mut result = Vec::with_capacity(total_size); + + // Write header + let mut header_buf = Vec::new(); + self.header.write(&mut header_buf).unwrap(); + result.extend(&header_buf); + + // Write load commands + for cmd in &self.load_commands { + self.write_load_command(&mut result, cmd); + } + + // Pad to data_offset + while result.len() < self.data_offset { + result.push(0); + } + + // Write file data (up to total_size) + let data_to_write = std::cmp::min(self.file_data.len(), total_size - self.data_offset); + result.extend(&self.file_data[..data_to_write]); + + // Pad to total size + while result.len() < total_size { + result.push(0); + } + + result + } + + fn write_load_command(&self, out: &mut Vec, cmd: &ParsedLoadCommand) { + match cmd { + ParsedLoadCommand::Segment64 { segment, sections } => { + segment.write(out).unwrap(); + for section in sections { + section.write(out).unwrap(); + } + } + ParsedLoadCommand::CodeSignature(lc) + | ParsedLoadCommand::FunctionStarts(lc) + | ParsedLoadCommand::DataInCode(lc) + | ParsedLoadCommand::DyldChainedFixups(lc) + | ParsedLoadCommand::DyldExportsTrie(lc) => { + lc.write(out).unwrap(); + } + ParsedLoadCommand::Symtab(st) => { + out.extend(&st.cmd.to_le_bytes()); + out.extend(&st.cmdsize.to_le_bytes()); + out.extend(&st.symoff.to_le_bytes()); + out.extend(&st.nsyms.to_le_bytes()); + out.extend(&st.stroff.to_le_bytes()); + out.extend(&st.strsize.to_le_bytes()); + } + ParsedLoadCommand::Dysymtab(dst) => { + out.extend(&dst.cmd.to_le_bytes()); + out.extend(&dst.cmdsize.to_le_bytes()); + out.extend(&dst.ilocalsym.to_le_bytes()); + out.extend(&dst.nlocalsym.to_le_bytes()); + out.extend(&dst.iextdefsym.to_le_bytes()); + out.extend(&dst.nextdefsym.to_le_bytes()); + out.extend(&dst.iundefsym.to_le_bytes()); + out.extend(&dst.nundefsym.to_le_bytes()); + out.extend(&dst.tocoff.to_le_bytes()); + out.extend(&dst.ntoc.to_le_bytes()); + out.extend(&dst.modtaboff.to_le_bytes()); + out.extend(&dst.nmodtab.to_le_bytes()); + out.extend(&dst.extrefsymoff.to_le_bytes()); + out.extend(&dst.nextrefsyms.to_le_bytes()); + out.extend(&dst.indirectsymoff.to_le_bytes()); + out.extend(&dst.nindirectsyms.to_le_bytes()); + out.extend(&dst.extreloff.to_le_bytes()); + out.extend(&dst.nextrel.to_le_bytes()); + out.extend(&dst.locreloff.to_le_bytes()); + out.extend(&dst.nlocrel.to_le_bytes()); + } + ParsedLoadCommand::DyldInfo(di) => { + out.extend(&di.cmd.to_le_bytes()); + out.extend(&di.cmdsize.to_le_bytes()); + out.extend(&di.rebase_off.to_le_bytes()); + out.extend(&di.rebase_size.to_le_bytes()); + out.extend(&di.bind_off.to_le_bytes()); + out.extend(&di.bind_size.to_le_bytes()); + out.extend(&di.weak_bind_off.to_le_bytes()); + out.extend(&di.weak_bind_size.to_le_bytes()); + out.extend(&di.lazy_bind_off.to_le_bytes()); + out.extend(&di.lazy_bind_size.to_le_bytes()); + out.extend(&di.export_off.to_le_bytes()); + out.extend(&di.export_size.to_le_bytes()); + } + ParsedLoadCommand::Other { cmd, data } => { + out.extend(&cmd.to_le_bytes()); + out.extend(&((data.len() + 8) as u32).to_le_bytes()); + out.extend(data); + } + } + } + + /// Write the Mach-O file to bytes + pub fn write(&self) -> Vec { + let mut result = Vec::new(); + + // Write header + self.header.write(&mut result).unwrap(); + + // Write load commands + for cmd in &self.load_commands { + self.write_load_command(&mut result, cmd); + } + + // Pad to data_offset if needed + while result.len() < self.data_offset { + result.push(0); + } + + // Write file data + result.extend(&self.file_data); + + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_stub() { + // This test requires the actual stub binary + let stub_path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/release/smolvm"); + if let Ok(data) = std::fs::read(stub_path) { + let macho = MachoFile::parse(&data).expect("Failed to parse Mach-O"); + assert_eq!(macho.header.magic, MH_MAGIC_64); + + // Check we found some segments + let mut found_text = false; + let mut found_data = false; + for cmd in &macho.load_commands { + if let ParsedLoadCommand::Segment64 { segment, .. } = cmd { + if segment.name() == "__TEXT" { + found_text = true; + } + if segment.name() == "__DATA" { + found_data = true; + } + } + } + assert!(found_text, "Should find __TEXT segment"); + assert!(found_data, "Should find __DATA segment"); + } + } + + #[test] + fn test_write_section_and_sign() { + // This test requires the actual stub binary with the __smolvm section + let stub_path = concat!(env!("CARGO_MANIFEST_DIR"), "/../../target/release/smolvm"); + let data = match std::fs::read(stub_path) { + Ok(d) => d, + Err(_) => return, // Skip if stub not built + }; + + let mut macho = match MachoFile::parse(&data) { + Ok(m) => m, + Err(_) => return, // Skip if not a valid Mach-O + }; + + // Check if section exists + if macho.find_section("__SMOLVM", "__smolvm").is_none() { + return; // Skip if section not present + } + + // Write test data to section + let test_data = b"SMOLSECT\x00\x00\x00\x00\x00\x00\x00\x00test data for section"; + macho + .write_section("__SMOLVM", "__smolvm", test_data) + .expect("Failed to write section"); + + // Sign adhoc + macho.sign_adhoc().expect("Failed to sign"); + + // Write to temp file at a known location for debugging + let output_path = std::path::PathBuf::from("/tmp/test-signed-stub"); + let output_data = macho.write(); + std::fs::write(&output_path, &output_data).unwrap(); + + // Make executable + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&output_path).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&output_path, perms).unwrap(); + } + + // Check with otool first to see load commands + let otool_output = std::process::Command::new("otool") + .args(["-l"]) + .arg(&output_path) + .output() + .expect("Failed to run otool"); + let otool_str = String::from_utf8_lossy(&otool_output.stdout); + + // Check if LC_CODE_SIGNATURE is present + assert!( + otool_str.contains("LC_CODE_SIGNATURE"), + "LC_CODE_SIGNATURE not found in output. Output at /tmp/test-signed-stub. otool output:\n{}", + otool_str + ); + + // Verify with codesign + let output = std::process::Command::new("codesign") + .args(["-v", "--verbose"]) + .arg(&output_path) + .output() + .expect("Failed to run codesign"); + + // codesign -v returns 0 for valid signatures + assert!( + output.status.success(), + "Code signature verification failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[test] + fn test_header_roundtrip() { + let header = MachHeader64 { + magic: MH_MAGIC_64, + cputype: 0x0100000c, // ARM64 + cpusubtype: 0, + filetype: 2, // MH_EXECUTE + ncmds: 10, + sizeofcmds: 1000, + flags: 0, + reserved: 0, + }; + + let mut buf = Vec::new(); + header.write(&mut buf).unwrap(); + + let mut cursor = std::io::Cursor::new(&buf); + let parsed = MachHeader64::read(&mut cursor).unwrap(); + + assert_eq!(header.magic, parsed.magic); + assert_eq!(header.cputype, parsed.cputype); + assert_eq!(header.ncmds, parsed.ncmds); + } + + #[test] + fn test_page_align() { + // Test exact page boundaries + assert_eq!(MachoFile::page_align(0), 0); + assert_eq!(MachoFile::page_align(CS_PAGE_SIZE), CS_PAGE_SIZE); + assert_eq!(MachoFile::page_align(CS_PAGE_SIZE * 2), CS_PAGE_SIZE * 2); + + // Test rounding up + assert_eq!(MachoFile::page_align(1), CS_PAGE_SIZE); + assert_eq!(MachoFile::page_align(100), CS_PAGE_SIZE); + assert_eq!(MachoFile::page_align(CS_PAGE_SIZE - 1), CS_PAGE_SIZE); + assert_eq!(MachoFile::page_align(CS_PAGE_SIZE + 1), CS_PAGE_SIZE * 2); + } + + #[test] + fn test_shift_offset_if_after() { + // Test: offset is after threshold, should shift + let mut offset = 1000u32; + MachoFile::shift_offset_if_after(&mut offset, 500, 100); + assert_eq!(offset, 1100); + + // Test: offset is before threshold, should not shift + let mut offset = 400u32; + MachoFile::shift_offset_if_after(&mut offset, 500, 100); + assert_eq!(offset, 400); + + // Test: offset is zero, should not shift (zero means unused) + let mut offset = 0u32; + MachoFile::shift_offset_if_after(&mut offset, 0, 100); + assert_eq!(offset, 0); + + // Test: negative delta (shrinking) + let mut offset = 1000u32; + MachoFile::shift_offset_if_after(&mut offset, 500, -100); + assert_eq!(offset, 900); + } + + #[test] + fn test_segment_name() { + let mut segname = [0u8; 16]; + segname[..8].copy_from_slice(b"__TEXT\0\0"); + + let segment = SegmentCommand64 { + cmd: LC_SEGMENT_64, + cmdsize: 72, + segname, + vmaddr: 0, + vmsize: 0, + fileoff: 0, + filesize: 0, + maxprot: 0, + initprot: 0, + nsects: 0, + flags: 0, + }; + + assert_eq!(segment.name(), "__TEXT"); + } + + #[test] + fn test_section_name() { + let mut sectname = [0u8; 16]; + let mut segname = [0u8; 16]; + sectname[..6].copy_from_slice(b"__text"); + segname[..8].copy_from_slice(b"__TEXT\0\0"); + + let section = Section64 { + sectname, + segname, + addr: 0, + size: 0, + offset: 0, + align: 0, + reloff: 0, + nreloc: 0, + flags: 0, + reserved1: 0, + reserved2: 0, + reserved3: 0, + }; + + assert_eq!(section.name(), "__text"); + assert_eq!(section.segment_name(), "__TEXT"); + } + + #[test] + fn test_load_command_roundtrip() { + let lc = LinkeditDataCommand { + cmd: LC_CODE_SIGNATURE, + cmdsize: 16, + dataoff: 12345, + datasize: 6789, + }; + + let mut buf = Vec::new(); + lc.write(&mut buf).unwrap(); + + assert_eq!(buf.len(), LinkeditDataCommand::SIZE); + + // Parse it back + let cmd = u32::from_le_bytes(buf[0..4].try_into().unwrap()); + let dataoff = u32::from_le_bytes(buf[8..12].try_into().unwrap()); + let datasize = u32::from_le_bytes(buf[12..16].try_into().unwrap()); + + assert_eq!(cmd, LC_CODE_SIGNATURE); + assert_eq!(dataoff, 12345); + assert_eq!(datasize, 6789); + } + + #[test] + fn test_segment_command_roundtrip() { + let mut segname = [0u8; 16]; + segname[..8].copy_from_slice(b"__SMOLVM"); + + let segment = SegmentCommand64 { + cmd: LC_SEGMENT_64, + cmdsize: 72, + segname, + vmaddr: 0x100000000, + vmsize: 0x4000, + fileoff: 0x8000, + filesize: 0x1000, + maxprot: 7, + initprot: 3, + nsects: 1, + flags: 0, + }; + + let mut buf = Vec::new(); + segment.write(&mut buf).unwrap(); + + assert_eq!(buf.len(), SegmentCommand64::SIZE); + + // Verify segment name + assert_eq!(&buf[8..16], b"__SMOLVM"); + + // Verify vmaddr + let vmaddr = u64::from_le_bytes(buf[24..32].try_into().unwrap()); + assert_eq!(vmaddr, 0x100000000); + } +} diff --git a/crates/smolvm-pack/src/packer.rs b/crates/smolvm-pack/src/packer.rs new file mode 100644 index 0000000..1b5aa2a --- /dev/null +++ b/crates/smolvm-pack/src/packer.rs @@ -0,0 +1,1033 @@ +//! Binary packer for assembling packed executables. +//! +//! This module handles combining the stub executable, compressed assets, +//! manifest, and footer into a self-contained `.smolmachine` package. +//! See [`crate::format`] for the binary format specification. + +use std::fs::{self, File}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use crate::assets::{crc32_file_range, AssetCollector}; +use crate::format::{PackFooter, PackManifest, FOOTER_SIZE, SIDECAR_EXTENSION}; +use crate::Result; + +/// Maximum allowed manifest size (16 MiB) to prevent malicious/corrupt sidecars +/// from causing excessive memory allocation. +const MAX_MANIFEST_SIZE: u64 = 16 * 1024 * 1024; + +/// Binary packer for creating self-contained executables. +pub struct Packer { + stub_path: Option, + manifest: PackManifest, + asset_collector: Option, +} + +/// Error type for try_pack_embedded_macho (internal). +#[cfg(target_os = "macos")] +enum PackedEmbeddedError { + /// Not a valid Mach-O with required section, return Packer to try append mode. + NotMachO(Box), + /// Other pack error. + PackError(crate::PackError), +} + +impl Packer { + /// Create a new packer with the given manifest. + pub fn new(manifest: PackManifest) -> Self { + Self { + stub_path: None, + manifest, + asset_collector: None, + } + } + + /// Set the path to the stub executable. + pub fn with_stub(mut self, stub_path: impl AsRef) -> Self { + self.stub_path = Some(stub_path.as_ref().to_path_buf()); + self + } + + /// Set the asset collector and update manifest with its inventory. + pub fn with_assets(mut self, collector: AssetCollector) -> Self { + // Update manifest with the collector's inventory + self.manifest.assets = collector.inventory().clone(); + self.asset_collector = Some(collector); + self + } + + /// Set the asset collector without updating the manifest. + /// + /// Use this when the manifest already has the correct asset inventory + /// (e.g., when it was set from a different collector that was consumed). + pub fn with_asset_collector(mut self, collector: AssetCollector) -> Self { + self.asset_collector = Some(collector); + self + } + + /// Get a mutable reference to the manifest. + pub fn manifest_mut(&mut self) -> &mut PackManifest { + &mut self.manifest + } + + /// Pack everything into the output file using sidecar format. + /// + /// Creates two files: + /// 1. `output` - Stub executable (pure Mach-O, signable) + /// 2. `output.smolmachine` - Compressed assets + manifest + footer + /// + /// This keeps the binary as a pure Mach-O executable that can be + /// properly code-signed on macOS. + pub fn pack(self, output: impl AsRef) -> Result { + let output = output.as_ref(); + let temp_dir = tempfile::tempdir()?; + + // Get stub executable + let stub_path = self + .stub_path + .as_ref() + .ok_or_else(|| crate::PackError::AssetNotFound("stub executable".to_string()))?; + + // 1. Copy stub executable to output (libs are appended later, after signing) + let stub_data = fs::read(stub_path)?; + let stub_size = stub_data.len() as u64; + fs::write(output, &stub_data)?; + + // Make executable + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(output)?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(output, perms)?; + } + + // 3. Build sidecar file with: assets (no libs) + manifest + footer + let sidecar_path = sidecar_path_for(output); + let mut sidecar_file = File::create(&sidecar_path)?; + + // 3a. Write compressed assets (excludes lib/ — those are embedded in the stub) + let assets_temp = temp_dir.path().join("assets.tar.zst"); + let assets_size = if let Some(collector) = &self.asset_collector { + collector.compress(&assets_temp, true)? + } else { + let empty_file = File::create(&assets_temp)?; + let encoder = zstd::stream::Encoder::new(empty_file, 1)?; + let tar_builder = tar::Builder::new(encoder); + let encoder = tar_builder.into_inner()?; + encoder.finish()?; + fs::metadata(&assets_temp)?.len() + }; + + let mut assets_file = File::open(&assets_temp)?; + std::io::copy(&mut assets_file, &mut sidecar_file)?; + + // 3b. Write manifest JSON + let manifest_offset = assets_size; + let manifest_json = self.manifest.to_json()?; + let manifest_size = manifest_json.len() as u64; + sidecar_file.write_all(&manifest_json)?; + + // 3c. Calculate checksum of assets + manifest + sidecar_file.flush()?; + drop(sidecar_file); + let checksum_size = assets_size + manifest_size; + let checksum = crc32_file_range(&sidecar_path, 0, checksum_size)?; + + // 3d. Write footer to sidecar + let footer = PackFooter { + stub_size: 0, // Not used in sidecar mode + assets_offset: 0, // Assets start at beginning of sidecar + assets_size, + manifest_offset, + manifest_size, + checksum, + }; + + let mut sidecar_file = fs::OpenOptions::new().append(true).open(&sidecar_path)?; + sidecar_file.write_all(&footer.to_bytes())?; + + let sidecar_total = assets_size + manifest_size + FOOTER_SIZE as u64; + let total_size = stub_size + sidecar_total; + + Ok(PackedInfo { + stub_size, + assets_size, + manifest_size, + total_size, + checksum, + sidecar_path: Some(sidecar_path), + }) + } + + /// Pack everything into a single executable file (embedded format). + /// + /// On macOS, this uses Mach-O section manipulation to embed assets inside + /// the `__SMOLVM,__smolvm` section, allowing proper code signing. + /// + /// On other platforms, this appends assets to the binary (simpler but + /// not signable on macOS). + pub fn pack_embedded(self, output: impl AsRef) -> Result { + #[cfg(target_os = "macos")] + { + // Try Mach-O section mode first + match self.try_pack_embedded_macho(output.as_ref()) { + Ok(info) => Ok(info), + Err(PackedEmbeddedError::NotMachO(packer)) => { + // Not a valid Mach-O, fall back to append mode + packer.pack_embedded_append(output) + } + Err(PackedEmbeddedError::PackError(e)) => Err(e), + } + } + #[cfg(not(target_os = "macos"))] + { + self.pack_embedded_append(output) + } + } + + /// Try to pack using Mach-O section manipulation. + /// + /// Returns NotMachO with self if the stub isn't a valid Mach-O with + /// the required __smolvm section, allowing fallback to append mode. + #[cfg(target_os = "macos")] + fn try_pack_embedded_macho( + self, + output: &Path, + ) -> std::result::Result { + use crate::macho::MachoFile; + + // Get stub path + let stub_path = match self.stub_path.as_ref() { + Some(p) => p, + None => { + return Err(PackedEmbeddedError::PackError( + crate::PackError::AssetNotFound("stub executable".to_string()), + )) + } + }; + + // Read stub data + let stub_data = match fs::read(stub_path) { + Ok(data) => data, + Err(e) => return Err(PackedEmbeddedError::PackError(e.into())), + }; + + // Try to parse as Mach-O + let macho = match MachoFile::parse(&stub_data) { + Ok(m) => m, + Err(_) => return Err(PackedEmbeddedError::NotMachO(Box::new(self))), // Not a Mach-O, fall back + }; + + // Check for __smolvm section + if macho.find_section("__SMOLVM", "__smolvm").is_none() { + return Err(PackedEmbeddedError::NotMachO(Box::new(self))); // No section, fall back + } + + // Valid Mach-O with section, proceed with Mach-O packing + match self.pack_embedded_macho_inner(output, stub_data, macho) { + Ok(info) => Ok(info), + Err(e) => Err(PackedEmbeddedError::PackError(e)), + } + } + + /// Pack using Mach-O section manipulation (macOS), inner implementation. + /// + /// This writes assets to the `__SMOLVM,__smolvm` section, keeping the + /// binary as a valid Mach-O that can be properly code-signed. + #[cfg(target_os = "macos")] + fn pack_embedded_macho_inner( + self, + output: &Path, + stub_data: Vec, + mut macho: crate::macho::MachoFile, + ) -> Result { + use crate::format::{SectionHeader, SECTION_HEADER_SIZE}; + + let stub_size = stub_data.len() as u64; + let temp_dir = tempfile::tempdir()?; + + // Compress assets + let assets_temp = temp_dir.path().join("assets.tar.zst"); + let assets_size = if let Some(collector) = &self.asset_collector { + collector.compress(&assets_temp, false)? // include libs in single-file mode + } else { + let empty_file = File::create(&assets_temp)?; + let encoder = zstd::stream::Encoder::new(empty_file, 1)?; + let tar_builder = tar::Builder::new(encoder); + let encoder = tar_builder.into_inner()?; + encoder.finish()?; + fs::metadata(&assets_temp)?.len() + }; + + // Serialize manifest + let manifest_json = self.manifest.to_json()?; + let manifest_size = manifest_json.len() as u32; + + // Calculate checksum of manifest + assets + let mut hasher = crc32fast::Hasher::new(); + hasher.update(&manifest_json); + let assets_data = fs::read(&assets_temp)?; + hasher.update(&assets_data); + let checksum = hasher.finalize(); + + // Build section data: header + manifest + assets + let header = SectionHeader { + manifest_size, + assets_size, + checksum, + }; + + let mut section_data = + Vec::with_capacity(SECTION_HEADER_SIZE + manifest_json.len() + assets_data.len()); + section_data.extend_from_slice(&header.to_bytes()); + section_data.extend_from_slice(&manifest_json); + section_data.extend_from_slice(&assets_data); + + // Write section data to Mach-O + macho + .write_section("__SMOLVM", "__smolvm", §ion_data) + .map_err(|e| crate::PackError::Signing(format!("failed to write section: {}", e)))?; + + // Sign the binary with adhoc signature + macho + .sign_adhoc() + .map_err(|e| crate::PackError::Signing(format!("failed to sign: {}", e)))?; + + // Write output + let output_data = macho.write(); + let total_size = output_data.len() as u64; + fs::write(output, &output_data)?; + + // Make executable + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(output)?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(output, perms)?; + } + + Ok(PackedInfo { + stub_size, + assets_size, + manifest_size: manifest_size as u64, + total_size, + checksum, + sidecar_path: None, + }) + } + + /// Pack by appending assets to binary. + /// + /// This is the fallback method on macOS (when stub isn't a valid Mach-O) + /// and the default method on other platforms. + fn pack_embedded_append(self, output: impl AsRef) -> Result { + let output = output.as_ref(); + let temp_dir = tempfile::tempdir()?; + + // Get stub executable + let stub_path = self + .stub_path + .as_ref() + .ok_or_else(|| crate::PackError::AssetNotFound("stub executable".to_string()))?; + + // 1. Copy stub executable to output + let stub_data = fs::read(stub_path)?; + let stub_size = stub_data.len() as u64; + let mut output_file = File::create(output)?; + output_file.write_all(&stub_data)?; + + // 2. Compress and append assets + let assets_temp = temp_dir.path().join("assets.tar.zst"); + let assets_size = if let Some(collector) = &self.asset_collector { + collector.compress(&assets_temp, false)? // include libs in append mode + } else { + let empty_file = File::create(&assets_temp)?; + let encoder = zstd::stream::Encoder::new(empty_file, 1)?; + let tar_builder = tar::Builder::new(encoder); + let encoder = tar_builder.into_inner()?; + encoder.finish()?; + fs::metadata(&assets_temp)?.len() + }; + + let assets_offset = stub_size; // Assets start right after stub + let mut assets_file = File::open(&assets_temp)?; + std::io::copy(&mut assets_file, &mut output_file)?; + + // 3. Append manifest JSON + let manifest_offset = stub_size + assets_size; + let manifest_json = self.manifest.to_json()?; + let manifest_size = manifest_json.len() as u64; + output_file.write_all(&manifest_json)?; + + // 4. Calculate checksum of assets + manifest + output_file.flush()?; + drop(output_file); + let checksum_size = assets_size + manifest_size; + let checksum = crc32_file_range(output, assets_offset, checksum_size)?; + + // 5. Append footer + let footer = PackFooter { + stub_size, + assets_offset, // Non-zero indicates embedded mode + assets_size, + manifest_offset, + manifest_size, + checksum, + }; + + let mut output_file = fs::OpenOptions::new().append(true).open(output)?; + output_file.write_all(&footer.to_bytes())?; + + // Make executable + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(output)?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(output, perms)?; + } + + let total_size = stub_size + assets_size + manifest_size + FOOTER_SIZE as u64; + + Ok(PackedInfo { + stub_size, + assets_size, + manifest_size, + total_size, + checksum, + sidecar_path: None, // No sidecar in embedded mode + }) + } +} + +/// Append compressed runtime libraries to a packed binary. +/// +/// Call this AFTER code signing — the SMOLLIBS footer must be the last +/// bytes of the file so `extract_libs_from_binary()` can find it. +pub fn embed_libs_in_binary(binary_path: &Path, staging_dir: &Path) -> Result<()> { + use crate::format::LibsFooter; + + let lib_dir = staging_dir.join("lib"); + if !lib_dir.exists() { + return Ok(()); // No libs to embed + } + + // Check if there are actual library files + let has_libs = fs::read_dir(&lib_dir)? + .filter_map(|e| e.ok()) + .any(|e| e.path().is_file()); + if !has_libs { + return Ok(()); + } + + let temp_dir = tempfile::tempdir()?; + let libs_temp = temp_dir.path().join("libs.tar.zst"); + + // Compress libs + let output_file = File::create(&libs_temp)?; + let encoder = zstd::stream::Encoder::new(output_file, crate::assets::ZSTD_LEVEL) + .map_err(|e| crate::PackError::Compression(e.to_string()))?; + let mut tar_builder = tar::Builder::new(encoder); + tar_builder + .append_dir_all("lib", &lib_dir) + .map_err(|e| crate::PackError::Tar(e.to_string()))?; + let encoder = tar_builder + .into_inner() + .map_err(|e| crate::PackError::Tar(e.to_string()))?; + encoder + .finish() + .map_err(|e| crate::PackError::Compression(e.to_string()))?; + + let libs_size = fs::metadata(&libs_temp)?.len(); + let binary_size = fs::metadata(binary_path)?.len(); + + // Append libs blob + footer + let mut output_file = fs::OpenOptions::new().append(true).open(binary_path)?; + let mut libs_file = File::open(&libs_temp)?; + std::io::copy(&mut libs_file, &mut output_file)?; + + let libs_footer = LibsFooter { + libs_offset: binary_size, + libs_size, + }; + output_file.write_all(&libs_footer.to_bytes())?; + + Ok(()) +} + +/// Get the sidecar path for a packed binary. +pub fn sidecar_path_for(binary_path: impl AsRef) -> PathBuf { + let mut path = binary_path.as_ref().to_path_buf(); + let filename = path + .file_name() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_default(); + path.set_file_name(format!("{}{}", filename, SIDECAR_EXTENSION)); + path +} + +/// Information about a packed binary. +#[derive(Debug, Clone)] +pub struct PackedInfo { + /// Size of stub executable. + pub stub_size: u64, + /// Size of compressed assets. + pub assets_size: u64, + /// Size of manifest JSON. + pub manifest_size: u64, + /// Total size (binary + sidecar). + pub total_size: u64, + /// CRC32 checksum of assets. + pub checksum: u32, + /// Path to sidecar file (if using sidecar mode). + pub sidecar_path: Option, +} + +/// Read footer from a sidecar file. +/// +/// Validates structural bounds: footer-derived sizes must be consistent with +/// the actual file size and within safe limits. +pub fn read_footer_from_sidecar(sidecar_path: impl AsRef) -> Result { + let mut file = File::open(sidecar_path.as_ref())?; + let file_size = file.metadata()?.len(); + + if file_size < FOOTER_SIZE as u64 { + return Err(crate::PackError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "sidecar file too small to contain footer", + ))); + } + + file.seek(SeekFrom::End(-(FOOTER_SIZE as i64)))?; + let mut footer_bytes = [0u8; FOOTER_SIZE]; + file.read_exact(&mut footer_bytes)?; + + let footer = PackFooter::from_bytes(&footer_bytes)?; + + // Validate footer-derived sizes against actual file size + validate_footer_bounds(&footer, file_size)?; + + Ok(footer) +} + +/// Read manifest from a sidecar file. +pub fn read_manifest_from_sidecar(sidecar_path: impl AsRef) -> Result { + let footer = read_footer_from_sidecar(sidecar_path.as_ref())?; + + let mut file = File::open(sidecar_path.as_ref())?; + file.seek(SeekFrom::Start(footer.manifest_offset))?; + + let mut manifest_bytes = vec![0u8; footer.manifest_size as usize]; + file.read_exact(&mut manifest_bytes)?; + + PackManifest::from_json(&manifest_bytes) +} + +/// Read footer from a packed binary (deprecated - use sidecar instead). +pub fn read_footer(path: impl AsRef) -> Result { + // Try sidecar first + let sidecar = sidecar_path_for(path.as_ref()); + if sidecar.exists() { + return read_footer_from_sidecar(&sidecar); + } + + // Fall back to embedded (v1 format) + let mut file = File::open(path.as_ref())?; + let file_size = file.metadata()?.len(); + + if file_size < FOOTER_SIZE as u64 { + return Err(crate::PackError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "file too small to contain footer", + ))); + } + + file.seek(SeekFrom::End(-(FOOTER_SIZE as i64)))?; + let mut footer_bytes = [0u8; FOOTER_SIZE]; + file.read_exact(&mut footer_bytes)?; + + PackFooter::from_bytes(&footer_bytes) +} + +/// Read manifest from a packed binary (deprecated - use sidecar instead). +pub fn read_manifest(path: impl AsRef) -> Result { + // Try sidecar first + let sidecar = sidecar_path_for(path.as_ref()); + if sidecar.exists() { + return read_manifest_from_sidecar(&sidecar); + } + + // Fall back to embedded (v1 format) + let footer = read_footer(path.as_ref())?; + + let mut file = File::open(path.as_ref())?; + file.seek(SeekFrom::Start(footer.manifest_offset))?; + + let mut manifest_bytes = vec![0u8; footer.manifest_size as usize]; + file.read_exact(&mut manifest_bytes)?; + + PackManifest::from_json(&manifest_bytes) +} + +/// Check if a packed binary uses sidecar mode. +pub fn is_sidecar_mode(footer: &PackFooter) -> bool { + footer.assets_offset == 0 +} + +/// Verify checksum of a packed binary. +pub fn verify_checksum(path: impl AsRef) -> Result { + let footer = read_footer(path.as_ref())?; + + if is_sidecar_mode(&footer) { + // Sidecar mode: checksum is of assets + manifest (not the footer) + let sidecar = sidecar_path_for(path.as_ref()); + if !sidecar.exists() { + return Ok(false); + } + let checksum_size = footer.assets_size + footer.manifest_size; + let actual = crc32_file_range(&sidecar, 0, checksum_size)?; + Ok(actual == footer.checksum) + } else { + // Embedded mode: checksum is of assets + manifest + let checksum_size = footer.assets_size + footer.manifest_size; + let actual = crc32_file_range(path.as_ref(), footer.assets_offset, checksum_size)?; + Ok(actual == footer.checksum) + } +} + +/// Verify checksum of a sidecar file. +/// +/// Computes CRC32 over the assets + manifest region and compares to the +/// checksum stored in the footer. +pub fn verify_sidecar_checksum( + sidecar_path: impl AsRef, + footer: &PackFooter, +) -> Result { + let checksum_size = footer + .assets_size + .checked_add(footer.manifest_size) + .ok_or_else(|| { + crate::PackError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "assets_size + manifest_size overflow", + )) + })?; + let actual = crc32_file_range(sidecar_path.as_ref(), 0, checksum_size)?; + Ok(actual == footer.checksum) +} + +/// Validate that footer-derived sizes are consistent with the actual file size. +fn validate_footer_bounds(footer: &PackFooter, file_size: u64) -> Result<()> { + // Cap manifest size to prevent excessive memory allocation + if footer.manifest_size > MAX_MANIFEST_SIZE { + return Err(crate::PackError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "manifest size ({} bytes) exceeds maximum ({} bytes)", + footer.manifest_size, MAX_MANIFEST_SIZE + ), + ))); + } + + // Verify that assets + manifest + footer fit within the file + let content_end = footer + .assets_size + .checked_add(footer.manifest_size) + .and_then(|s| s.checked_add(FOOTER_SIZE as u64)); + + match content_end { + Some(end) if end <= file_size => {} + _ => { + return Err(crate::PackError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "footer sizes exceed file size (corrupt or truncated sidecar)", + ))); + } + } + + // In sidecar mode (assets_offset == 0), the layout is: + // [assets_blob | manifest_json | footer] + // so manifest_offset MUST equal assets_size. If it doesn't, the + // manifest bytes that read_manifest_from_sidecar parses could differ + // from the region covered by the checksum (0..assets_size+manifest_size), + // creating a trust gap between verified and parsed data. + if footer.assets_offset == 0 && footer.manifest_offset != footer.assets_size { + return Err(crate::PackError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "sidecar manifest_offset ({}) does not match assets_size ({}): corrupt footer", + footer.manifest_offset, footer.assets_size + ), + ))); + } + + // Verify manifest_offset + manifest_size doesn't exceed the pre-footer region + let manifest_end = footer + .manifest_offset + .checked_add(footer.manifest_size) + .and_then(|end| end.checked_add(FOOTER_SIZE as u64)); + + match manifest_end { + Some(end) if end <= file_size => {} + _ => { + return Err(crate::PackError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "manifest offset/size exceeds file bounds", + ))); + } + } + + Ok(()) +} + +/// Extract assets from a packed binary to a directory. +pub fn extract_assets(packed_path: impl AsRef, output_dir: impl AsRef) -> Result<()> { + let footer = read_footer(packed_path.as_ref())?; + + if is_sidecar_mode(&footer) { + // Sidecar mode: read from .smolmachine file + let sidecar = sidecar_path_for(packed_path.as_ref()); + crate::assets::decompress_assets_from_file(&sidecar, output_dir.as_ref())?; + } else { + // Embedded mode: read from the binary itself + let mut file = File::open(packed_path.as_ref())?; + file.seek(SeekFrom::Start(footer.assets_offset))?; + + // Read compressed assets + let mut compressed = vec![0u8; footer.assets_size as usize]; + file.read_exact(&mut compressed)?; + + // Decompress + crate::assets::decompress_assets(&compressed, output_dir.as_ref())?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + fn test_pack_and_read() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create a dummy stub + let stub_path = temp_dir.path().join("stub"); + let mut stub_file = File::create(&stub_path).unwrap(); + stub_file.write_all(b"#!/bin/sh\necho stub").unwrap(); + + // Create manifest + let manifest = PackManifest::new( + "alpine:latest".to_string(), + "sha256:abc123".to_string(), + "linux/arm64".to_string(), + "darwin/arm64".to_string(), + ); + + // Pack + let output_path = temp_dir.path().join("packed"); + let packer = Packer::new(manifest).with_stub(&stub_path); + let info = packer.pack(&output_path).unwrap(); + + assert!(info.stub_size > 0); + assert!(info.total_size > info.stub_size); + + // Verify sidecar file exists + let sidecar = sidecar_path_for(&output_path); + assert!(sidecar.exists()); + + // Read back - sidecar mode has stub_size=0 in footer + let footer = read_footer(&output_path).unwrap(); + assert_eq!(footer.stub_size, 0); // Sidecar mode doesn't track stub size in footer + assert_eq!(footer.assets_offset, 0); // Sidecar mode indicator + + let manifest = read_manifest(&output_path).unwrap(); + assert_eq!(manifest.image, "alpine:latest"); + + // Verify checksum + assert!(verify_checksum(&output_path).unwrap()); + } + + #[test] + fn test_pack_with_assets() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create a dummy stub + let stub_path = temp_dir.path().join("stub"); + fs::write(&stub_path, b"#!/bin/sh\necho stub").unwrap(); + + // Create staging directory with a test file + let staging = temp_dir.path().join("staging"); + let mut collector = AssetCollector::new(staging).unwrap(); + + // Add a fake layer + collector + .add_layer("sha256:abc123def456", b"layer content") + .unwrap(); + + // Create manifest + let manifest = PackManifest::new( + "test:latest".to_string(), + "sha256:test".to_string(), + "linux/arm64".to_string(), + "darwin/arm64".to_string(), + ); + + // Pack with assets + let output_path = temp_dir.path().join("packed"); + let packer = Packer::new(manifest) + .with_stub(&stub_path) + .with_assets(collector); + packer.pack(&output_path).unwrap(); + + // Verify we can read the manifest with layer info + let manifest = read_manifest(&output_path).unwrap(); + assert_eq!(manifest.assets.layers.len(), 1); + assert_eq!(manifest.assets.layers[0].digest, "sha256:abc123def456"); + + // Extract and verify assets + let extract_dir = temp_dir.path().join("extracted"); + extract_assets(&output_path, &extract_dir).unwrap(); + + let layer_file = extract_dir.join("layers/abc123def456.tar"); + assert!(layer_file.exists()); + assert_eq!(fs::read_to_string(&layer_file).unwrap(), "layer content"); + } + + #[test] + fn test_pack_embedded() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create a dummy stub + let stub_path = temp_dir.path().join("stub"); + fs::write(&stub_path, b"#!/bin/sh\necho stub").unwrap(); + + // Create manifest + let manifest = PackManifest::new( + "alpine:latest".to_string(), + "sha256:abc123".to_string(), + "linux/arm64".to_string(), + "darwin/arm64".to_string(), + ); + + // Pack embedded (single file) + let output_path = temp_dir.path().join("packed-single"); + let packer = Packer::new(manifest).with_stub(&stub_path); + let info = packer.pack_embedded(&output_path).unwrap(); + + // Should have no sidecar + assert!(info.sidecar_path.is_none()); + assert!(info.stub_size > 0); + assert!(info.total_size > info.stub_size); + + // Sidecar file should NOT exist + let sidecar = sidecar_path_for(&output_path); + assert!(!sidecar.exists()); + + // Output file should contain everything + let output_size = fs::metadata(&output_path).unwrap().len(); + assert_eq!(output_size, info.total_size); + + // Read back - embedded mode has non-zero assets_offset + let footer = read_footer(&output_path).unwrap(); + assert!(footer.assets_offset > 0); // Embedded mode indicator + assert_eq!(footer.stub_size, info.stub_size); + + let manifest = read_manifest(&output_path).unwrap(); + assert_eq!(manifest.image, "alpine:latest"); + + // Verify checksum + assert!(verify_checksum(&output_path).unwrap()); + } + + #[test] + fn test_pack_embedded_with_assets() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create a dummy stub + let stub_path = temp_dir.path().join("stub"); + fs::write(&stub_path, b"#!/bin/sh\necho stub").unwrap(); + + // Create staging directory with a test file + let staging = temp_dir.path().join("staging"); + let mut collector = AssetCollector::new(staging).unwrap(); + + // Add a fake layer (digest must be at least 12 chars after sha256:) + collector + .add_layer("sha256:embedded123456", b"embedded layer content") + .unwrap(); + + // Create manifest + let manifest = PackManifest::new( + "test:embedded".to_string(), + "sha256:test".to_string(), + "linux/arm64".to_string(), + "darwin/arm64".to_string(), + ); + + // Pack embedded with assets + let output_path = temp_dir.path().join("packed-embedded"); + let packer = Packer::new(manifest) + .with_stub(&stub_path) + .with_assets(collector); + packer.pack_embedded(&output_path).unwrap(); + + // Verify we can read the manifest with layer info + let manifest = read_manifest(&output_path).unwrap(); + assert_eq!(manifest.assets.layers.len(), 1); + assert_eq!(manifest.assets.layers[0].digest, "sha256:embedded123456"); + + // Verify footer indicates embedded mode + let footer = read_footer(&output_path).unwrap(); + assert!(footer.assets_offset > 0); + assert!(!is_sidecar_mode(&footer)); + + // Extract and verify assets + let extract_dir = temp_dir.path().join("extracted"); + extract_assets(&output_path, &extract_dir).unwrap(); + + let layer_file = extract_dir.join("layers/embedded1234.tar"); // First 12 chars + assert!(layer_file.exists()); + assert_eq!( + fs::read_to_string(&layer_file).unwrap(), + "embedded layer content" + ); + } + + #[test] + fn test_sidecar_checksum_verification() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create a dummy stub + let stub_path = temp_dir.path().join("stub"); + fs::write(&stub_path, b"#!/bin/sh\necho stub").unwrap(); + + let manifest = PackManifest::new( + "alpine:latest".to_string(), + "sha256:abc123".to_string(), + "linux/arm64".to_string(), + "darwin/arm64".to_string(), + ); + + // Pack sidecar mode + let output_path = temp_dir.path().join("packed"); + let packer = Packer::new(manifest).with_stub(&stub_path); + packer.pack(&output_path).unwrap(); + + let sidecar = sidecar_path_for(&output_path); + let footer = read_footer_from_sidecar(&sidecar).unwrap(); + + // Valid sidecar should pass + assert!(verify_sidecar_checksum(&sidecar, &footer).unwrap()); + + // Corrupt the sidecar by flipping a byte in the assets region + let mut data = fs::read(&sidecar).unwrap(); + if !data.is_empty() { + data[0] ^= 0xFF; + } + fs::write(&sidecar, &data).unwrap(); + + // Corrupted sidecar should fail checksum + assert!(!verify_sidecar_checksum(&sidecar, &footer).unwrap()); + } + + #[test] + fn test_footer_bounds_reject_oversized_manifest() { + let temp_dir = tempfile::tempdir().unwrap(); + let sidecar = temp_dir.path().join("bad.smolmachine"); + + // Craft a footer with manifest_size exceeding the 16 MiB cap + let footer = PackFooter { + stub_size: 0, + assets_offset: 0, + assets_size: 100, + manifest_offset: 100, + manifest_size: 32 * 1024 * 1024, // 32 MiB — exceeds cap + checksum: 0, + }; + + // Write a minimal sidecar: some bytes + footer + let footer_bytes = footer.to_bytes(); + let mut data = vec![0u8; 200]; // some dummy content + data.extend_from_slice(&footer_bytes); + fs::write(&sidecar, &data).unwrap(); + + let result = read_footer_from_sidecar(&sidecar); + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + assert!(err_msg.contains("exceeds maximum"), "got: {}", err_msg); + } + + #[test] + fn test_footer_bounds_reject_manifest_offset_mismatch() { + let temp_dir = tempfile::tempdir().unwrap(); + let sidecar = temp_dir.path().join("mismatch.smolmachine"); + + // In sidecar mode (assets_offset=0), manifest_offset must == assets_size. + // Craft a footer where manifest_offset points elsewhere. + let footer = PackFooter { + stub_size: 0, + assets_offset: 0, // sidecar mode + assets_size: 100, + manifest_offset: 50, // should be 100 — points into assets region + manifest_size: 50, + checksum: 0, + }; + + let footer_bytes = footer.to_bytes(); + // File needs: 100 (assets) + 50 (manifest) + 64 (footer) = 214 bytes + let mut data = vec![0u8; 150]; + data.extend_from_slice(&footer_bytes); + fs::write(&sidecar, &data).unwrap(); + + let result = read_footer_from_sidecar(&sidecar); + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("manifest_offset") && err_msg.contains("does not match"), + "got: {}", + err_msg + ); + } + + #[test] + fn test_sidecar_vs_embedded_mode_detection() { + let temp_dir = tempfile::tempdir().unwrap(); + + // Create a dummy stub + let stub_path = temp_dir.path().join("stub"); + fs::write(&stub_path, b"#!/bin/sh\necho stub").unwrap(); + + let manifest = PackManifest::new( + "test:latest".to_string(), + "sha256:test".to_string(), + "linux/arm64".to_string(), + "darwin/arm64".to_string(), + ); + + // Pack sidecar mode + let sidecar_output = temp_dir.path().join("sidecar-mode"); + let packer = Packer::new(manifest.clone()).with_stub(&stub_path); + packer.pack(&sidecar_output).unwrap(); + + // Pack embedded mode + let embedded_output = temp_dir.path().join("embedded-mode"); + let packer = Packer::new(manifest).with_stub(&stub_path); + packer.pack_embedded(&embedded_output).unwrap(); + + // Verify mode detection + let sidecar_footer = read_footer(&sidecar_output).unwrap(); + let embedded_footer = read_footer(&embedded_output).unwrap(); + + assert!(is_sidecar_mode(&sidecar_footer)); + assert!(!is_sidecar_mode(&embedded_footer)); + + // Both should have valid checksums + assert!(verify_checksum(&sidecar_output).unwrap()); + assert!(verify_checksum(&embedded_output).unwrap()); + } +} diff --git a/crates/smolvm-pack/src/signing.rs b/crates/smolvm-pack/src/signing.rs new file mode 100644 index 0000000..de497e3 --- /dev/null +++ b/crates/smolvm-pack/src/signing.rs @@ -0,0 +1,199 @@ +//! Code signing support for packed executables. +//! +//! On macOS, executables that use Hypervisor.framework must be signed +//! with the appropriate entitlements. + +use std::path::Path; + +#[cfg(target_os = "macos")] +use std::fs; +#[cfg(target_os = "macos")] +use std::process::Command; + +#[cfg(target_os = "macos")] +use crate::PackError; +use crate::Result; + +/// Default entitlements for packed binaries on macOS. +/// +/// - `hypervisor`: access to Hypervisor.framework for the microVM +/// - `cs.disable-library-validation`: allow dlopen of libkrun from +/// the extracted cache directory (ad-hoc signed libraries) +/// - `cs.allow-jit`: a forkable golden (`machine start --forkable`) maps guest +/// RAM from a file-backed region that HVF executes guest code from; under the +/// hardened runtime that needs a JIT entitlement or `krun_start_enter` returns +/// -22. (`allow-unsigned-executable-memory` also works; `allow-jit` is the +/// narrower capability. Harmless for non-forkable VMs, which use anon RAM.) +#[cfg(target_os = "macos")] +const HYPERVISOR_ENTITLEMENTS: &str = r#" + + + + com.apple.security.hypervisor + + com.apple.security.cs.disable-library-validation + + com.apple.security.cs.allow-jit + + + +"#; + +/// Sign a binary with hypervisor entitlements (macOS only). +/// +/// This uses ad-hoc signing (no certificate required) which is sufficient +/// for local development and distribution. +#[cfg(target_os = "macos")] +pub fn sign_with_hypervisor_entitlements(binary_path: &Path) -> Result<()> { + // Reset SIGCHLD to default before spawning codesign. + // The agent VM startup installs a SIGCHLD handler that reaps all children + // via waitpid(-1), which races with Command::output()'s internal waitpid + // and causes ECHILD (os error 10). + unsafe { + libc::signal(libc::SIGCHLD, libc::SIG_DFL); + } + + // Create temporary entitlements file + let temp_dir = tempfile::tempdir()?; + let entitlements_path = temp_dir.path().join("entitlements.plist"); + fs::write(&entitlements_path, HYPERVISOR_ENTITLEMENTS)?; + + // Run codesign + let output = Command::new("codesign") + .args([ + "--force", + "--sign", + "-", // Ad-hoc signing + "--entitlements", + ]) + .arg(&entitlements_path) + .arg(binary_path) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(PackError::Signing(format!( + "codesign failed: {}", + stderr.trim() + ))); + } + + Ok(()) +} + +/// Sign a binary with hypervisor entitlements (no-op on non-macOS). +#[cfg(not(target_os = "macos"))] +pub fn sign_with_hypervisor_entitlements(_binary_path: &Path) -> Result<()> { + // No signing needed on Linux + Ok(()) +} + +/// Check if a binary is signed (macOS only). +#[cfg(target_os = "macos")] +pub fn is_signed(binary_path: &Path) -> Result { + let output = Command::new("codesign") + .args(["--verify", "--verbose"]) + .arg(binary_path) + .output()?; + + Ok(output.status.success()) +} + +/// Check if a binary is signed (always false on non-macOS). +#[cfg(not(target_os = "macos"))] +pub fn is_signed(_binary_path: &Path) -> Result { + Ok(false) +} + +/// Get signature information for a binary (macOS only). +#[cfg(target_os = "macos")] +pub fn get_signature_info(binary_path: &Path) -> Result> { + let output = Command::new("codesign") + .args(["--display", "--verbose=2"]) + .arg(binary_path) + .output()?; + + if !output.status.success() { + return Ok(None); + } + + let stderr = String::from_utf8_lossy(&output.stderr); + let is_adhoc = stderr.contains("Signature=adhoc"); + + Ok(Some(SignatureInfo { + is_adhoc, + raw_output: stderr.to_string(), + })) +} + +/// Get signature information (always None on non-macOS). +#[cfg(not(target_os = "macos"))] +pub fn get_signature_info(_binary_path: &Path) -> Result> { + Ok(None) +} + +/// Information about a code signature. +#[derive(Debug, Clone)] +pub struct SignatureInfo { + /// Whether this is ad-hoc signed. + pub is_adhoc: bool, + /// Raw codesign output. + pub raw_output: String, +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + #[test] + #[cfg(target_os = "macos")] + fn test_sign_binary() { + let temp_dir = tempfile::tempdir().unwrap(); + let binary_path = temp_dir.path().join("test_binary"); + + // Create a minimal Mach-O binary (just the header) + // This is a valid but non-functional binary that codesign can process + let mut file = fs::File::create(&binary_path).unwrap(); + + // Minimal arm64 Mach-O header + let macho_header: [u8; 32] = [ + 0xCF, 0xFA, 0xED, 0xFE, // Magic (64-bit) + 0x0C, 0x00, 0x00, 0x01, // CPU type (ARM64) + 0x00, 0x00, 0x00, 0x00, // CPU subtype + 0x02, 0x00, 0x00, 0x00, // File type (MH_EXECUTE) + 0x00, 0x00, 0x00, 0x00, // Number of load commands + 0x00, 0x00, 0x00, 0x00, // Size of load commands + 0x00, 0x00, 0x00, 0x00, // Flags + 0x00, 0x00, 0x00, 0x00, // Reserved + ]; + file.write_all(&macho_header).unwrap(); + drop(file); + + // Make executable + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(&binary_path).unwrap().permissions(); + perms.set_mode(0o755); + fs::set_permissions(&binary_path, perms).unwrap(); + } + + // Sign it + let result = sign_with_hypervisor_entitlements(&binary_path); + // This may fail if the binary isn't valid enough for codesign, + // but we're testing that the function runs without panicking + if let Err(e) = &result { + eprintln!("Signing failed (expected for minimal test binary): {}", e); + } + } + + #[test] + #[cfg(target_os = "macos")] + fn test_entitlements_format() { + // Verify the entitlements XML is valid + assert!(HYPERVISOR_ENTITLEMENTS.contains("com.apple.security.hypervisor")); + assert!(HYPERVISOR_ENTITLEMENTS.contains("cs.disable-library-validation")); + assert!(HYPERVISOR_ENTITLEMENTS.contains("")); + } +} diff --git a/crates/smolvm-pack/tests/shared_extract.rs b/crates/smolvm-pack/tests/shared_extract.rs new file mode 100644 index 0000000..6e49b36 --- /dev/null +++ b/crates/smolvm-pack/tests/shared_extract.rs @@ -0,0 +1,176 @@ +//! End-to-end test of the shared, content-addressed pack extraction +//! (`extract_sidecar_shared`) against a *real* `.smolmachine` sidecar — real +//! zstd stream, real tar, real footer/checksum — on a real filesystem. +//! +//! Linux-only: the shared store backs the Linux fleet's per-VM idmapped bind +//! mount (kernel ≥5.12); macOS uses a per-machine case-sensitive sparse image, +//! so the whole module compiles to nothing elsewhere. This validates the +//! *extraction half* (extract-once, content-addressed, locked down); the mount +//! half is validated by the root crate's `tests/idmap_mount.rs`. +#![cfg(target_os = "linux")] + +use std::fs; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; + +use smolvm_pack::assets::AssetCollector; +use smolvm_pack::extract::{extract_sidecar, extract_sidecar_shared, shared_extract_enabled}; +use smolvm_pack::format::PackManifest; +use smolvm_pack::{read_footer_from_sidecar, sidecar_path_for, Packer}; + +/// Build a tiny but *valid* OCI-style layer: a raw tar archive carrying one +/// file. `post_process_extraction` untars every `layers/*.tar`, so the payload +/// must be a real tar (a raw byte blob would fail with "failed to read entire +/// block" — exactly what a real OCI layer never does). +fn layer_tar(filename: &str, content: &[u8]) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Regular); + header.set_size(content.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, filename, content).unwrap(); + builder.into_inner().unwrap() +} + +/// Build a real sidecar `.smolmachine` carrying two OCI layers, returning the +/// sidecar path (under `dir`). +fn build_real_sidecar(dir: &Path) -> std::path::PathBuf { + let stub_path = dir.join("stub"); + fs::write(&stub_path, b"#!/bin/sh\necho stub").unwrap(); + + let mut collector = AssetCollector::new(dir.join("staging")).unwrap(); + collector + .add_layer( + "sha256:aaa111aaa111bbb222", + &layer_tar("etc/base.conf", b"base layer file"), + ) + .unwrap(); + collector + .add_layer( + "sha256:ccc333ccc333ddd444", + &layer_tar("etc/top.conf", b"top layer file"), + ) + .unwrap(); + + let manifest = PackManifest::new( + "test:latest".to_string(), + "sha256:test".to_string(), + "linux/x86_64".to_string(), + "linux/x86_64".to_string(), + ); + + let output = dir.join("packed"); + Packer::new(manifest) + .with_stub(&stub_path) + .with_assets(collector) + .pack(&output) + .unwrap(); + + sidecar_path_for(&output) +} + +/// Recursively collect (relative-path -> file bytes) for every regular file in a +/// tree, so the shared extraction can be compared byte-for-byte to a plain one. +fn file_map(root: &Path) -> std::collections::BTreeMap> { + let mut out = std::collections::BTreeMap::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(d) = stack.pop() { + for entry in fs::read_dir(&d).unwrap() { + let entry = entry.unwrap(); + let ft = entry.file_type().unwrap(); + let p = entry.path(); + if ft.is_dir() { + stack.push(p); + } else if ft.is_file() { + let rel = p.strip_prefix(root).unwrap().to_string_lossy().to_string(); + out.insert(rel, fs::read(&p).unwrap()); + } + } + } + out +} + +#[test] +fn shared_extraction_is_content_addressed_idempotent_and_locked_down() { + let tmp = tempfile::tempdir().unwrap(); + let sidecar = build_real_sidecar(tmp.path()); + let footer = read_footer_from_sidecar(&sidecar).unwrap(); + + let shared_root = tmp.path().join("_shared"); + let pack_plain = tmp.path().join("vm-plain/pack"); + + // First machine extracts into the shared store; a plain per-machine + // extraction is the byte-for-byte reference. + let shared_dir = extract_sidecar_shared(&sidecar, &shared_root, &footer, false).unwrap(); + extract_sidecar(&sidecar, &pack_plain, &footer, false, false).unwrap(); + + // 1) Content-addressed: the shared copy lives at `_shared/`. + assert_eq!( + shared_dir, + shared_root.join(format!("{:08x}", footer.checksum)), + "shared dir must be keyed by the footer checksum" + ); + assert!(shared_dir.is_dir(), "shared extraction dir must exist"); + + // 2) The shared view is byte-identical to a plain per-machine extraction — + // the idmapped mount only remaps ownership, never content. + let reference = file_map(&pack_plain); + assert!(!reference.is_empty(), "extraction produced no files"); + assert_eq!( + file_map(&shared_dir), + reference, + "shared content must match a plain extraction byte-for-byte" + ); + + // 3) Idempotent reuse: a second machine with the same checksum reuses the one + // extracted tree — no second decode, no second store dir. This is the + // cold-start tax the shared store removes. + let shared_dir2 = extract_sidecar_shared(&sidecar, &shared_root, &footer, false).unwrap(); + assert_eq!( + shared_dir2, shared_dir, + "second machine must reuse the copy" + ); + let store_dirs: Vec<_> = fs::read_dir(&shared_root) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.path().is_dir()) + .collect(); + assert_eq!( + store_dirs.len(), + 1, + "only one content-addressed extraction may exist for one checksum" + ); + + // 4) Locked down to 0700: the store root and the checksum dir are owner-only, + // so a sibling VM dropped to a *different* uid cannot read the shared copy + // directly — it must go through its own idmapped mount (#456 isolation). + for dir in [&shared_root, &shared_dir] { + let mode = fs::metadata(dir).unwrap().permissions().mode() & 0o777; + assert_eq!( + mode, + 0o700, + "{} must be 0700, got {:o}", + dir.display(), + mode + ); + } +} + +#[test] +fn kill_switch_disables_the_shared_store() { + // The handler consults `shared_extract_enabled()` to choose the shared vs + // per-machine path; `SMOLVM_DISABLE_SHARED_EXTRACT` forces the fallback + // without a redeploy. (Run as its own test fn so the env mutation cannot race + // the content test on cargo's parallel test threads.) + assert!( + shared_extract_enabled(), + "shared store should be enabled by default on Linux" + ); + std::env::set_var("SMOLVM_DISABLE_SHARED_EXTRACT", "1"); + assert!( + !shared_extract_enabled(), + "kill-switch must disable the shared store" + ); + std::env::remove_var("SMOLVM_DISABLE_SHARED_EXTRACT"); +} diff --git a/crates/smolvm-protocol/Cargo.toml b/crates/smolvm-protocol/Cargo.toml new file mode 100644 index 0000000..4b000e1 --- /dev/null +++ b/crates/smolvm-protocol/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "smolvm-protocol" +version = "1.5.2" +edition = "2021" +description = "Protocol types for smolvm host-guest communication" +license = "Apache-2.0" +repository = "https://github.com/smol-machines/smolvm" + +[dependencies] +base64 = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } diff --git a/crates/smolvm-protocol/src/guest_env.rs b/crates/smolvm-protocol/src/guest_env.rs new file mode 100644 index 0000000..a9ab4e3 --- /dev/null +++ b/crates/smolvm-protocol/src/guest_env.rs @@ -0,0 +1,73 @@ +//! Shared environment-variable contract between the host launcher and guest agent. +//! +//! These names form the protocol boundary between: +//! - the host-side launcher, which decides what features to enable +//! - the guest agent, which reads these vars on startup and acts accordingly +//! +//! They should be treated as stable protocol constants rather than ad hoc +//! launcher strings. + +/// Standard "enabled" value for boolean `SMOLVM_*` sentinel env vars. +/// +/// The host writes this when a feature is enabled; the guest agent +/// compares against it. A single canonical value prevents `true` / `yes` / +/// `1` mismatches between the two sides. +pub const VALUE_ON: &str = "1"; + +/// Env var the host sets on guest init to signal GPU acceleration was requested. +/// +/// Present means "host asked for GPU"; the guest agent reads this and emits a +/// post-boot sanity log confirming whether `/dev/dri/*` nodes actually appeared. +/// Absent means no GPU was requested. +/// +/// This is a boolean sentinel — the value is [`VALUE_ON`] when set. +pub const GPU: &str = "SMOLVM_GPU"; + +/// Env var the host sets on guest init to signal Rosetta 2 x86_64 translation +/// was requested (and is available on the host). +/// +/// Present means the host has attached the RosettaLinux runtime as the virtiofs +/// tag [`crate::ROSETTA_TAG`]; the guest agent reads this on startup and, if set, +/// mounts that runtime at [`crate::ROSETTA_GUEST_PATH`] and registers the ptrace +/// wrapper with `binfmt_misc` as the interpreter for x86_64 ELF binaries. Absent +/// means no Rosetta was requested. +/// +/// This is a boolean sentinel — the value is [`VALUE_ON`] when set. +pub const ROSETTA: &str = "SMOLVM_ROSETTA"; + +/// Filename of this VM's readiness marker, written by the agent into the virtiofs +/// rootfs when boot completes. Per VM (so concurrent boots don't race on one +/// shared file); the host pre-creates and polls the same name. Unset → the agent +/// falls back to the shared [`crate::AGENT_READY_MARKER`] constant. +pub const READY_MARKER: &str = "SMOLVM_READY_MARKER"; + +/// Host wall-clock at VM launch, nanoseconds since the Unix epoch. The agent +/// uses this to set the guest clock when the hypervisor gives the guest no +/// readable paravirt clock (e.g. WHP on Windows, where the guest otherwise +/// boots at ~1999 and every TLS cert validation fails). The agent only applies +/// it when the guest clock already looks obviously wrong, so it never fights an +/// accurate kvmclock (Linux/KVM) or HVF-seeded RTC (macOS). +pub const HOST_TIME_NS: &str = "SMOLVM_HOST_TIME_NS"; + +/// Selects whether the guest should configure a real virtio NIC. +pub const BACKEND: &str = "SMOLVM_NETWORK_BACKEND"; +/// Canonical backend value meaning "configure guest virtio-net". +pub const BACKEND_VIRTIO_NET: &str = "virtio-net"; +/// Guest IPv4 address. +pub const GUEST_IP: &str = "SMOLVM_NETWORK_GUEST_IP"; +/// Guest-visible default gateway IPv4 address. +pub const GATEWAY: &str = "SMOLVM_NETWORK_GATEWAY"; +/// Guest subnet prefix length. +pub const PREFIX_LEN: &str = "SMOLVM_NETWORK_PREFIX_LEN"; +/// Guest MAC address in colon-separated string form. +pub const GUEST_MAC: &str = "SMOLVM_NETWORK_GUEST_MAC"; +/// Guest IPv6 (ULA) address. Optional: absent means IPv4-only guest config. +pub const GUEST_IP6: &str = "SMOLVM_NETWORK_GUEST_IP6"; +/// Guest-visible default gateway IPv6 address. +pub const GATEWAY6: &str = "SMOLVM_NETWORK_GATEWAY6"; +/// Guest IPv6 prefix length. +pub const PREFIX_LEN6: &str = "SMOLVM_NETWORK_PREFIX_LEN6"; +/// Guest-visible DNS server IPv4 address. +pub const DNS: &str = "SMOLVM_NETWORK_DNS"; +/// Enables the guest-side DNS filtering proxy. +pub const DNS_FILTER: &str = "SMOLVM_DNS_FILTER"; diff --git a/crates/smolvm-protocol/src/image_ref.rs b/crates/smolvm-protocol/src/image_ref.rs new file mode 100644 index 0000000..5a8cc9f --- /dev/null +++ b/crates/smolvm-protocol/src/image_ref.rs @@ -0,0 +1,212 @@ +//! Image reference canonicalization. +//! +//! OCI image references have many valid spellings for the same image. +//! This module provides [`normalize_image_ref`], which maps every spelling +//! to a single canonical form so that cache keys, log messages, and +//! protocol messages are consistent regardless of how the caller spelled +//! the reference. + +/// Canonicalize an OCI image reference. +/// +/// All equivalent spellings of the same image produce an identical string, +/// which is safe to use as a cache key or protocol field. +/// +/// # Normalization rules (applied in order) +/// +/// 1. `index.docker.io` is rewritten to `docker.io` (legacy alias). +/// 2. A missing registry defaults to `docker.io`. +/// 3. Single-component names on `docker.io` receive the `library/` prefix +/// (e.g. `alpine` → `docker.io/library/alpine`). +/// 4. A missing tag defaults to `:latest`. When a digest (`@sha256:…`) is +/// present it takes precedence and any tag is dropped. +/// +/// # Examples +/// +/// ``` +/// use smolvm_protocol::normalize_image_ref; +/// +/// assert_eq!(normalize_image_ref("alpine"), +/// "docker.io/library/alpine:latest"); +/// assert_eq!(normalize_image_ref("alpine:3.20"), +/// "docker.io/library/alpine:3.20"); +/// assert_eq!(normalize_image_ref("docker.io/alpine:3.20"), +/// "docker.io/library/alpine:3.20"); +/// assert_eq!(normalize_image_ref("docker.io/library/alpine:3.20"), +/// "docker.io/library/alpine:3.20"); +/// assert_eq!(normalize_image_ref("ghcr.io/owner/repo:v1"), +/// "ghcr.io/owner/repo:v1"); +/// ``` +pub fn normalize_image_ref(image: &str) -> String { + // Local image sources (host-resolved `local:` / `local-dir:`) + // are not registry references — pass them through unchanged. + if image.starts_with("local:") || image.starts_with("local-dir:") { + return image.to_string(); + } + + // 1. Resolve index.docker.io alias. + let owned; + let image = if let Some(rest) = image.strip_prefix("index.docker.io/") { + owned = format!("docker.io/{rest}"); + owned.as_str() + } else { + image + }; + + // 2. and 3. Separate the digest, and the tag from the bare repository. + let (ref_no_tag, tag, digest) = split_suffix(image); + + // 4. Detect registry: the first '/'-delimited component is a registry + // hostname when it contains '.' or ':' (port). Everything else is + // an implicit docker.io reference. + let (registry, path) = split_registry(ref_no_tag); + + // 5. Single-component docker.io paths get the `library/` prefix. + let canonical_path = if registry == "docker.io" && !path.contains('/') { + format!("library/{path}") + } else { + path.to_string() + }; + + // 6. Suffix: digest wins over tag; absent tag defaults to `:latest`. + let suffix = match digest { + Some(d) => format!("@{d}"), + None => format!(":{}", tag.unwrap_or("latest")), + }; + + format!("{registry}/{canonical_path}{suffix}") +} + +/// Split an image reference into `(repo, tag, digest)`. +/// +/// The repo is the `registry/repository` portion with no suffix. +/// +/// The digest is everything after '@'. When a digest is present the tag is +/// informational (the digest is authoritative), but both are still returned. +/// +/// The tag separator is the last ':' with no '/' after it. A colon that is +/// part of a registry hostname (e.g. `localhost:5000/repo`) always has a '/' +/// after it, so this rule does not mistake a port for a tag. +fn split_suffix(image: &str) -> (&str, Option<&str>, Option<&str>) { + let (ref_no_digest, digest) = match image.split_once('@') { + Some((left, right)) => (left, Some(right)), + None => (image, None), + }; + let (repo, tag) = match ref_no_digest.rfind(':') { + Some(pos) if !ref_no_digest[pos..].contains('/') => { + (&ref_no_digest[..pos], Some(&ref_no_digest[pos + 1..])) + } + _ => (ref_no_digest, None), + }; + (repo, tag, digest) +} + +/// Strip the tag and/or digest suffix from an image reference, returning the registry/repo +/// +/// # Examples +/// +/// ``` +/// use smolvm_protocol::image_repo; +/// +/// assert_eq!(image_repo("ghcr.io/owner/repo@sha256:abc"), "ghcr.io/owner/repo"); +/// assert_eq!(image_repo("ghcr.io/owner/repo:v1"), "ghcr.io/owner/repo"); +/// ``` +pub fn image_repo(image: &str) -> &str { + split_suffix(image).0 +} + +/// Split a tag-free, digest-free image string into `(registry, path)`. +/// +/// Returns `("docker.io", whole_string)` when no explicit registry is found. +fn split_registry(image: &str) -> (&str, &str) { + if let Some(slash) = image.find('/') { + let prefix = &image[..slash]; + if prefix.contains('.') || prefix.contains(':') { + return (prefix, &image[slash + 1..]); + } + } + ("docker.io", image) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normalize_image_ref() { + let cases: &[(&str, &str)] = &[ + // Bare name and with tag — docker.io library prefix + :latest default. + ("alpine", "docker.io/library/alpine:latest"), + ("alpine:3.20", "docker.io/library/alpine:3.20"), + // Explicit docker.io without library/ — library/ is inserted. + ("docker.io/alpine:3.20", "docker.io/library/alpine:3.20"), + // Already canonical — idempotent. + ( + "docker.io/library/alpine:3.20", + "docker.io/library/alpine:3.20", + ), + // index.docker.io legacy alias. + ( + "index.docker.io/library/alpine", + "docker.io/library/alpine:latest", + ), + // library/ without a registry prefix. + ("library/alpine", "docker.io/library/alpine:latest"), + // User-namespaced docker.io image — no extra library/ prefix. + ("myuser/myimage:v2", "docker.io/myuser/myimage:v2"), + // Non-docker.io registry. + ("ghcr.io/owner/repo", "ghcr.io/owner/repo:latest"), + ("ghcr.io/owner/repo:v1", "ghcr.io/owner/repo:v1"), + // Port in registry — colon-detection must not confuse port with tag. + ("localhost:5000/myimage:dev", "localhost:5000/myimage:dev"), + ]; + + for (input, expected) in cases { + assert_eq!( + normalize_image_ref(input), + *expected, + "normalize_image_ref({input:?})" + ); + } + } + + #[test] + fn test_normalize_digest_refs() { + let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + // Digest without tag. + assert_eq!( + normalize_image_ref(&format!("alpine@{digest}")), + format!("docker.io/library/alpine@{digest}"), + ); + + // Digest with tag — tag is dropped, digest is authoritative. + assert_eq!( + normalize_image_ref(&format!("alpine:3.20@{digest}")), + format!("docker.io/library/alpine@{digest}"), + ); + } + + #[test] + fn test_image_repo() { + let digest = "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + // Digest suffix is stripped. + assert_eq!( + image_repo(&format!("ghcr.io/owner/repo@{digest}")), + "ghcr.io/owner/repo", + ); + // Tag suffix is stripped. + assert_eq!(image_repo("ghcr.io/owner/repo:v1"), "ghcr.io/owner/repo"); + // No suffix — returned unchanged. + assert_eq!(image_repo("ghcr.io/owner/repo"), "ghcr.io/owner/repo"); + // Port in registry — colon-detection must not confuse port with tag. + assert_eq!( + image_repo("localhost:5000/myimage:dev"), + "localhost:5000/myimage", + ); + assert_eq!( + image_repo("localhost:5000/myimage"), + "localhost:5000/myimage" + ); + } +} diff --git a/crates/smolvm-protocol/src/lib.rs b/crates/smolvm-protocol/src/lib.rs new file mode 100644 index 0000000..7d1aa9f --- /dev/null +++ b/crates/smolvm-protocol/src/lib.rs @@ -0,0 +1,1301 @@ +//! Protocol types for smolvm host-guest communication. +//! +//! This crate defines the wire protocol for vsock communication between +//! the smolvm host and the guest agent (smolvm-agent). +//! +//! # Protocol Overview +//! +//! Communication uses JSON-encoded messages over vsock. Each message is +//! prefixed with a 4-byte big-endian length header. +//! +//! ```text +//! +----------------+-------------------+ +//! | Length (4 BE) | JSON payload | +//! +----------------+-------------------+ +//! ``` + +#![deny(missing_docs)] + +use serde::{Deserialize, Serialize}; + +pub mod guest_env; +pub mod image_ref; +pub mod retry; +pub mod secrets; + +pub use image_ref::{image_repo, normalize_image_ref}; +pub use secrets::{SecretRef, SecretSourceKind}; + +/// Serde helper for encoding `Vec` as a base64 string in JSON. +/// +/// Without this, serde_json serializes `Vec` as a JSON array of numbers +/// (e.g., `[104,101,108,108,111]`), which inflates binary data by ~4x. +/// Base64 encoding reduces this to ~1.33x. +pub mod base64_bytes { + use base64::{engine::general_purpose::STANDARD, Engine}; + use serde::{Deserialize, Deserializer, Serializer}; + + /// Serialize `Vec` as a base64 string. + pub fn serialize(data: &[u8], serializer: S) -> Result { + serializer.serialize_str(&STANDARD.encode(data)) + } + + /// Deserialize a base64 string into `Vec`. + pub fn deserialize<'de, D: Deserializer<'de>>(deserializer: D) -> Result, D::Error> { + let s = String::deserialize(deserializer)?; + STANDARD.decode(&s).map_err(serde::de::Error::custom) + } +} + +/// Protocol version. +pub const PROTOCOL_VERSION: u32 = 1; + +/// virtiofs tag under which the host exposes the Rosetta 2 Linux runtime to the +/// guest. Shared host↔guest so the launcher's `krun_add_virtiofs` tag and the +/// guest agent's `mount -t virtiofs` source can't drift apart. +pub const ROSETTA_TAG: &str = "rosetta"; + +/// Guest mount point for the Rosetta 2 Linux runtime. The ptrace wrapper execs +/// `/rosetta` (the translator), so this path is baked into +/// both the wrapper and the `binfmt_misc` registration. +pub const ROSETTA_GUEST_PATH: &str = "/mnt/rosetta"; + +/// Maximum frame size (32 MB - layer exports use chunked streaming). +pub const MAX_FRAME_SIZE: u32 = 32 * 1024 * 1024; + +/// Chunk size for streaming layer data (~16 MB raw, ~21 MB as base64 JSON). +pub const LAYER_CHUNK_SIZE: usize = 16 * 1024 * 1024; + +/// Files at or below this size are written with a single `FileWrite` +/// message. Larger files must stream via +/// `FileWriteBegin` + `FileWriteChunk` so no single frame approaches +/// [`MAX_FRAME_SIZE`] (base64 + JSON inflation is ~1.4x). +/// +/// Chosen to keep the single-shot frame comfortably under the frame +/// limit while preserving the fast-path latency for small config +/// files / scripts / keys. +pub const FILE_WRITE_SINGLE_SHOT_MAX: usize = 1024 * 1024; + +/// Payload bytes per streaming upload chunk. Deliberately small — +/// equal to [`FILE_WRITE_SINGLE_SHOT_MAX`] — so each chunk's encoded +/// frame (~1.4 MB) fits inside typical kernel Unix-socket send +/// buffers (`SO_SNDBUF` defaults on the order of 200–256 KiB but +/// can grow). Larger chunks would force `write_all` to spin waiting +/// for the agent to drain, and any latency spike trips the 10 s +/// write timeout with `EAGAIN` — exactly the failure David +/// reproduced before this fix landed. +/// +/// Note: [`LAYER_CHUNK_SIZE`] is 16 MiB for agent→host (download) +/// streaming, which works because the host side of the socket has +/// more headroom than the guest side. Upload streaming is the +/// asymmetric case and needs a smaller chunk. +pub const FILE_WRITE_CHUNK_SIZE: usize = FILE_WRITE_SINGLE_SHOT_MAX; + +/// Hard ceiling on a single file transfer in either direction. +/// +/// On the write path: enforced at `FileWriteBegin` by the agent — +/// `total_size > FILE_TRANSFER_MAX_TOTAL` is rejected before any +/// staging file is created. +/// +/// On the read path: enforced by the host's `read_file` loop — +/// after the first chunk that pushes the accumulated total past the +/// cap, the call bails with an error and the partial buffer is +/// dropped. This protects the host process from OOM if the guest +/// (compromised or merely buggy) streams unbounded data. +/// +/// 4 GiB matches the order-of-magnitude of the default overlay disk +/// and the `gpu_vram_mib` cap. Callers that need to move larger +/// blobs should stage via a virtiofs mount instead of `cp`. +pub const FILE_TRANSFER_MAX_TOTAL: u64 = 4 * 1024 * 1024 * 1024; + +/// Filename of the virtiofs-visible marker the agent creates when it is +/// ready to accept vsock connections. +/// +/// The host polls for this file through its virtiofs mount of the guest +/// rootfs. The agent writes it (and optionally a symlink from `/oldroot/`) +/// during deferred init, just before opening the vsock listener. +/// +/// Both sides must agree on this name; keeping it here prevents silent drift. +pub const AGENT_READY_MARKER: &str = ".smolvm-ready"; + +/// Well-known vsock ports. +pub mod ports { + /// Control channel for workload VMs. + pub const WORKLOAD_CONTROL: u32 = 5000; + /// Log streaming from workload VMs. + pub const WORKLOAD_LOGS: u32 = 5001; + /// Agent control port (for OCI operations and management). + pub const AGENT_CONTROL: u32 = 6000; + /// SSH agent forwarding (host SSH_AUTH_SOCK bridged to guest). + pub const SSH_AGENT: u32 = 6001; + /// DNS filtering proxy (guest forwards DNS queries to host for filtering). + pub const DNS_FILTER: u32 = 6002; + /// CUDA-over-vsock (experimental): guest CUDA client forwards Driver-API + /// calls to a host CUDA server that runs them on the host NVIDIA GPU. + pub const CUDA: u32 = 7000; +} + +/// vsock CID constants. +pub mod cid { + /// Host CID (always 2). + pub const HOST: u32 = 2; + /// Guest CID (always 3 for the first/only guest). + pub const GUEST: u32 = 3; + /// Any CID (for listening). + pub const ANY: u32 = u32::MAX; +} + +/// fsnotify event masks, mirroring the kernel's `FS_*` bits in +/// `include/linux/fsnotify_backend.h`. Shared by the host watcher (which maps a +/// host filesystem event to one of these) and the guest agent (which forwards +/// the raw bits to `/proc/smolvm-fsnotify`). Only the subset relevant to +/// file-watching tools is defined. +pub mod fsnotify_mask { + /// File was modified. + pub const FS_MODIFY: u32 = 0x0000_0002; + /// Metadata changed (chmod/chown/utimes). + pub const FS_ATTRIB: u32 = 0x0000_0004; + /// Writable file was closed. + pub const FS_CLOSE_WRITE: u32 = 0x0000_0008; + /// File was moved away from the watched dir. + pub const FS_MOVED_FROM: u32 = 0x0000_0040; + /// File was moved into the watched dir. + pub const FS_MOVED_TO: u32 = 0x0000_0080; + /// Subfile was created. + pub const FS_CREATE: u32 = 0x0000_0100; + /// Subfile was deleted. + pub const FS_DELETE: u32 = 0x0000_0200; + /// Event occurred against a directory. + pub const FS_ISDIR: u32 = 0x4000_0000; +} + +/// A single host-originated filesystem change to replay into the guest. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FsNotifyEvent { + /// Guest-side absolute path the event occurred on (virtiofs staging path). + pub path: String, + /// `fsnotify_mask::FS_*` bitmask for the event. + pub mask: u32, +} + +// ============================================================================ +// Agent Protocol (OCI Operations) +// ============================================================================ + +/// Agent request types (for image management and OCI operations). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "method", rename_all = "snake_case")] +pub enum AgentRequest { + /// Ping to check if agent is alive. + Ping, + + /// Inject host-originated fsnotify events into the guest. + /// + /// virtiofs does not deliver host-side file changes to the guest as + /// fsnotify/inotify events, so inotify-based hot-reload (Vite, webpack, + /// nodemon) never fires when a mounted file is edited on the host. The host + /// watches the mount source and sends the resulting events here; the agent + /// writes them to `/proc/smolvm-fsnotify`, which fires the matching event on + /// the guest inode so watchers on the (bind-mounted) container path wake up. + /// Each `path` is a guest-side absolute path (the virtiofs staging path), + /// `mask` an `fsnotify_mask::FS_*` bitmask. + FsNotify { + /// Host-originated filesystem changes to replay as guest fsnotify events. + #[serde(default)] + events: Vec, + }, + + /// Pull an OCI image and extract layers. + Pull { + /// Image reference (e.g., "alpine:latest", "docker.io/library/ubuntu:22.04"). + image: String, + /// OCI platform to pull (e.g., "linux/arm64", "linux/amd64"). + oci_platform: Option, + /// Optional registry authentication credentials. + #[serde(default, skip_serializing_if = "Option::is_none")] + auth: Option, + /// Proxy URL applied to the registry client (sets HTTP_PROXY and HTTPS_PROXY). + #[serde(default, skip_serializing_if = "Option::is_none")] + proxy: Option, + /// Comma-separated NO_PROXY list of hosts/CIDRs that bypass the proxy. + #[serde(default, skip_serializing_if = "Option::is_none")] + no_proxy: Option, + }, + + /// Query if an image exists locally. + Query { + /// Image reference. + image: String, + }, + + /// List all cached images. + ListImages, + + /// Run garbage collection on unused layers. + GarbageCollect { + /// If true, only report what would be deleted. + dry_run: bool, + /// If true, delete all image manifests and configs first, + /// making all layers unreferenced so they get collected. + #[serde(default)] + purge_all: bool, + }, + + /// Prepare overlay rootfs for a workload. + PrepareOverlay { + /// Image reference. + image: String, + /// Unique workload ID for the overlay. + workload_id: String, + }, + + /// Clean up overlay rootfs for a workload. + CleanupOverlay { + /// Workload ID to clean up. + workload_id: String, + }, + + /// Format the storage disk (first-time setup). + FormatStorage, + + /// Get storage disk status. + StorageStatus, + + /// Test network connectivity directly from the agent (not via chroot). + /// Used to debug TSI networking. + NetworkTest { + /// URL to test (e.g., "http://1.1.1.1") + url: String, + }, + + /// Shutdown the agent. + Shutdown, + + /// Export a layer as a tar archive. + /// + /// Used by `smolvm pack` to extract OCI layers for packaging. + /// The agent streams the layer tar data back via LayerData responses. + ExportLayer { + /// Image digest (sha256:...). + image_digest: String, + /// Layer index (0-based). + layer_index: usize, + }, + + /// Execute a command directly in the VM (not in a container). + /// + /// This runs the command in the agent's Alpine rootfs without any + /// container isolation. Useful for VM-level operations and debugging. + VmExec { + /// Command and arguments. + command: Vec, + /// Environment variables. + #[serde(default)] + env: Vec<(String, String)>, + /// Working directory in the VM. + workdir: Option, + /// Timeout in milliseconds. + #[serde(default)] + timeout_ms: Option, + /// Interactive mode - stream I/O instead of buffering. + #[serde(default)] + interactive: bool, + /// Allocate a pseudo-TTY for the command. + #[serde(default)] + tty: bool, + /// Background mode - spawn and return PID immediately without waiting. + #[serde(default)] + background: bool, + /// Data to pipe to the command's stdin. + #[serde(default)] + stdin_data: Option, + }, + + /// Run a command in an image's rootfs. + /// + /// This prepares an overlay, chroots into it, and executes the command. + /// Returns stdout, stderr, and exit code when the command completes. + Run { + /// Image reference (must be pulled first). + image: String, + /// Command and arguments. + command: Vec, + /// Environment variables. + #[serde(default)] + env: Vec<(String, String)>, + /// Working directory inside the rootfs. + workdir: Option, + /// User inside the rootfs. If omitted, the OCI image default applies. + #[serde(default, skip_serializing_if = "Option::is_none")] + user: Option, + /// Volume mounts to bind into the container. + /// Each tuple is (virtiofs_tag, container_path, read_only). + #[serde(default)] + mounts: Vec<(String, String, bool)>, + /// Timeout in milliseconds. If the command exceeds this duration, + /// it will be killed and return exit code 124. + #[serde(default)] + timeout_ms: Option, + /// Interactive mode - stream I/O instead of buffering. + /// When true, output is streamed via Stdout/Stderr responses, + /// and stdin can be sent via the Stdin request. + #[serde(default)] + interactive: bool, + /// Allocate a pseudo-TTY for the command. + /// Enables terminal features like colors, line editing, and signal handling. + #[serde(default)] + tty: bool, + /// Detached mode — start the container and return immediately with the + /// container ID. Only meaningful when `persistent_overlay_id` is set. + /// Returns a `Completed` response with `stdout` containing the container ID. + #[serde(default)] + detached: bool, + /// Run the workload as an unprivileged container: restricted capabilities, + /// read-only cgroup, and no extra tmpfs. The default (false) is "VM-grade" + /// — since the microVM is the isolation boundary, the workload gets a full + /// capability set and the mounts an init system needs (so any image, incl. + /// systemd, boots). Opt in for defense-in-depth when running untrusted code. + #[serde(default)] + unprivileged: bool, + /// If set, use a persistent overlay that survives across exec sessions. + /// The overlay is identified by this ID (typically the machine name) + /// and reused on subsequent runs. If not set, an ephemeral overlay is + /// created and destroyed after the run. + #[serde(default, skip_serializing_if = "Option::is_none")] + persistent_overlay_id: Option, + /// Data to pipe to the command's stdin (non-interactive runs only). + /// The pipe is closed after writing, so the command sees EOF. + #[serde(default, skip_serializing_if = "Option::is_none")] + stdin_data: Option, + /// Spawn the container and return immediately with the crun PID. + /// The container runs detached; stdout/stderr go to /dev/null. + /// Incompatible with `interactive` and `tty`. + #[serde(default)] + background: bool, + }, + + /// Send stdin data to a running interactive command. + Stdin { + /// Input data to send to the command's stdin. + #[serde(with = "base64_bytes")] + data: Vec, + }, + + /// Resize the PTY window (for TTY mode). + Resize { + /// New width in columns. + cols: u16, + /// New height in rows. + rows: u16, + }, + + // ======================================================================== + // File I/O + // ======================================================================== + /// Write a file inside the VM in a single message. + /// + /// Use only for files up to [`FILE_WRITE_SINGLE_SHOT_MAX`]. Larger + /// files must stream via [`Self::FileWriteBegin`] + + /// [`Self::FileWriteChunk`] to avoid exceeding [`MAX_FRAME_SIZE`] + /// after base64 + JSON inflation. + FileWrite { + /// Absolute path in the VM filesystem. + path: String, + /// File contents. + #[serde(with = "base64_bytes")] + data: Vec, + /// File mode (e.g., 0o644). None = default (0644). + #[serde(default)] + mode: Option, + }, + + /// Open a streaming file upload session on this connection. + /// + /// Must be followed by one or more [`Self::FileWriteChunk`] + /// requests. The final chunk sets `done: true` to finalize. + /// Dropping the connection (or sending any non-chunk request) + /// before `done` aborts the session and leaves no partial file + /// at `path`. + /// + /// Sessions are per-connection — one session at a time. + FileWriteBegin { + /// Absolute path in the VM filesystem. + path: String, + /// File mode (e.g., 0o644). None = default (0644). + #[serde(default)] + mode: Option, + /// Expected total size in bytes. Rejected if it exceeds + /// [`FILE_TRANSFER_MAX_TOTAL`]. The agent uses this for an + /// early-fail check only; the actual size written is the sum + /// of chunk byte lengths. + total_size: u64, + }, + + /// Append a chunk to the currently open streaming upload. + /// If `done` is true, the agent fsyncs and atomically renames the + /// staging file onto the target path. + FileWriteChunk { + /// Chunk bytes. Typically [`FILE_WRITE_CHUNK_SIZE`] except + /// for the last chunk. + #[serde(with = "base64_bytes")] + data: Vec, + /// True on the final chunk; closes and renames the staging + /// file. False on intermediate chunks. + done: bool, + }, + + /// Read a file from the VM. + FileRead { + /// Absolute path in the VM filesystem. + path: String, + }, +} + +impl AgentRequest { + /// A log-safe one-line summary of the request. + /// + /// This string is written to the machine's console log, which is exposed + /// over the logs API — so it must NEVER include credential- or data-bearing + /// fields: registry `auth`, `env` (which can carry host-resolved secrets), + /// `proxy` (may embed credentials), or `data` (file/stdin bytes). Only the + /// variant name plus a non-secret identifier (image) is emitted. + /// + /// The match is exhaustive with no catch-all on purpose: adding a new + /// variant forces a compile error here, so redaction is a deliberate + /// decision rather than an accidental leak in some future request type. + pub fn log_summary(&self) -> String { + match self { + AgentRequest::Ping => "Ping".into(), + AgentRequest::FsNotify { events } => format!("FsNotify {{ count: {} }}", events.len()), + AgentRequest::Pull { image, .. } => format!("Pull {{ image: {image} }}"), + AgentRequest::Query { image, .. } => format!("Query {{ image: {image} }}"), + AgentRequest::ListImages => "ListImages".into(), + AgentRequest::GarbageCollect { .. } => "GarbageCollect".into(), + AgentRequest::PrepareOverlay { .. } => "PrepareOverlay".into(), + AgentRequest::CleanupOverlay { .. } => "CleanupOverlay".into(), + AgentRequest::FormatStorage => "FormatStorage".into(), + AgentRequest::StorageStatus => "StorageStatus".into(), + AgentRequest::NetworkTest { .. } => "NetworkTest".into(), + AgentRequest::Shutdown => "Shutdown".into(), + AgentRequest::ExportLayer { .. } => "ExportLayer".into(), + AgentRequest::VmExec { .. } => "VmExec".into(), + AgentRequest::Run { image, .. } => format!("Run {{ image: {image} }}"), + AgentRequest::Stdin { .. } => "Stdin".into(), + AgentRequest::Resize { .. } => "Resize".into(), + AgentRequest::FileWrite { .. } => "FileWrite".into(), + AgentRequest::FileWriteBegin { .. } => "FileWriteBegin".into(), + AgentRequest::FileWriteChunk { .. } => "FileWriteChunk".into(), + AgentRequest::FileRead { .. } => "FileRead".into(), + } + } +} + +/// Agent response types. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum AgentResponse { + /// Operation completed successfully. + Ok { + /// Response data (varies by request type). + #[serde(default, skip_serializing_if = "Option::is_none")] + data: Option, + }, + + /// Pong response to ping. + Pong { + /// Protocol version. + version: u32, + }, + + /// Progress update (for long operations like pull). + Progress { + /// Human-readable message. + message: String, + /// Completion percentage (0-100). + #[serde(default, skip_serializing_if = "Option::is_none")] + percent: Option, + /// Current layer being processed. + #[serde(default, skip_serializing_if = "Option::is_none")] + layer: Option, + }, + + /// Operation failed. + Error { + /// Error message. + message: String, + /// Error code (for programmatic handling). + #[serde(default, skip_serializing_if = "Option::is_none")] + code: Option, + }, + + /// Command execution completed (non-interactive mode). + Completed { + /// Exit code from the command. + exit_code: i32, + /// Standard output (may be truncated). `Vec` preserves binary + /// output (image bytes, tarballs, etc.) that would be truncated by + /// `String` at the first non-UTF-8 byte. Serialized as base64 JSON + /// string — the same format as the streaming `Stdout` variant. + #[serde(with = "base64_bytes")] + stdout: Vec, + /// Standard error (may be truncated). + #[serde(with = "base64_bytes")] + stderr: Vec, + }, + + /// Command started (interactive mode). + /// Indicates the command is running and ready to receive stdin. + Started, + + /// Stdout data from a running command (interactive mode). + Stdout { + /// Output data. + #[serde(with = "base64_bytes")] + data: Vec, + }, + + /// Stderr data from a running command (interactive mode). + Stderr { + /// Error output data. + #[serde(with = "base64_bytes")] + data: Vec, + }, + + /// Command exited (interactive mode). + Exited { + /// Exit code from the command. + exit_code: i32, + }, + + /// Streaming binary-data chunk. + /// + /// Used by every streaming download direction: the agent sends + /// one or more `DataChunk` responses in sequence, with `done: true` + /// on the final chunk. Current producers: `ExportLayer` and + /// `FileRead`. + /// + /// Payload size per chunk should stay under + /// [`LAYER_CHUNK_SIZE`] so the encoded frame (~1.33× after + /// base64) fits inside [`MAX_FRAME_SIZE`] with JSON overhead to + /// spare. + DataChunk { + /// Chunk bytes. Empty allowed on the final frame (common for + /// EOF-on-clean-boundary cases). + #[serde(with = "base64_bytes")] + data: Vec, + /// True on the final chunk of the stream. + done: bool, + }, +} + +// ============================================================================ +// Error Code Constants +// ============================================================================ +// +// Standard error codes for AgentResponse::Error. Using constants ensures +// consistency across the codebase and makes error handling more reliable. + +/// Error codes for agent responses. +pub mod error_codes { + /// Request payload was invalid or malformed. + pub const INVALID_REQUEST: &str = "INVALID_REQUEST"; + /// Requested resource was not found. + pub const NOT_FOUND: &str = "NOT_FOUND"; + /// Internal error during operation. + pub const INTERNAL_ERROR: &str = "INTERNAL_ERROR"; + /// Image pull operation failed. + pub const PULL_FAILED: &str = "PULL_FAILED"; + /// Image query operation failed. + pub const QUERY_FAILED: &str = "QUERY_FAILED"; + /// Command execution failed. + pub const RUN_FAILED: &str = "RUN_FAILED"; + /// Command execution failed in container. + pub const EXEC_FAILED: &str = "EXEC_FAILED"; + /// Process spawn failed. + pub const SPAWN_FAILED: &str = "SPAWN_FAILED"; + /// Mount operation failed. + pub const MOUNT_FAILED: &str = "MOUNT_FAILED"; + /// File I/O operation failed. + pub const FILE_IO_FAILED: &str = "FILE_IO_FAILED"; + /// Overlay filesystem operation failed. + pub const OVERLAY_FAILED: &str = "OVERLAY_FAILED"; + /// Cleanup operation failed. + pub const CLEANUP_FAILED: &str = "CLEANUP_FAILED"; + /// Storage format operation failed. + pub const FORMAT_FAILED: &str = "FORMAT_FAILED"; + /// Storage status query failed. + pub const STATUS_FAILED: &str = "STATUS_FAILED"; + /// List operation failed. + pub const LIST_FAILED: &str = "LIST_FAILED"; + /// Garbage collection failed. + pub const GC_FAILED: &str = "GC_FAILED"; + /// Container creation failed. + pub const CREATE_FAILED: &str = "CREATE_FAILED"; + /// Container start failed. + pub const START_FAILED: &str = "START_FAILED"; + /// Container stop failed. + pub const STOP_FAILED: &str = "STOP_FAILED"; + /// Container delete failed. + pub const DELETE_FAILED: &str = "DELETE_FAILED"; + /// Export operation failed. + pub const EXPORT_FAILED: &str = "EXPORT_FAILED"; + /// Serialization error. + pub const SERIALIZATION_ERROR: &str = "SERIALIZATION_ERROR"; + /// Message size exceeds maximum. + pub const MESSAGE_TOO_LARGE: &str = "MESSAGE_TOO_LARGE"; + /// Process wait operation failed. + pub const WAIT_FAILED: &str = "WAIT_FAILED"; +} + +impl AgentResponse { + /// Create an error response with the given message and code. + /// + /// # Example + /// + /// ``` + /// use smolvm_protocol::{AgentResponse, error_codes}; + /// + /// let response = AgentResponse::error("image not found", error_codes::NOT_FOUND); + /// ``` + pub fn error(message: impl Into, code: &str) -> Self { + AgentResponse::Error { + message: message.into(), + code: Some(code.to_string()), + } + } + + /// Create an error response from a Result's error, with the given code. + /// + /// # Example + /// + /// ```ignore + /// let response = some_operation() + /// .map(|data| AgentResponse::ok_with_data(data)) + /// .unwrap_or_else(|e| AgentResponse::from_err(e, error_codes::PULL_FAILED)); + /// ``` + pub fn from_err(err: E, code: &str) -> Self { + AgentResponse::Error { + message: err.to_string(), + code: Some(code.to_string()), + } + } + + /// Create an Ok response with optional JSON data. + pub fn ok(data: Option) -> Self { + AgentResponse::Ok { data } + } + + /// Create an Ok response with JSON-serializable data. + /// + /// Returns an error response if serialization fails. + pub fn ok_with_data(data: T) -> Self { + match serde_json::to_value(data) { + Ok(value) => AgentResponse::Ok { data: Some(value) }, + Err(e) => AgentResponse::error( + format!("failed to serialize response: {}", e), + error_codes::SERIALIZATION_ERROR, + ), + } + } + + /// Convert a Result into an AgentResponse. + /// + /// On success, serializes the value to JSON. On error, creates an error response. + /// + /// # Example + /// + /// ```ignore + /// let response = AgentResponse::from_result( + /// storage::pull_image(image), + /// error_codes::PULL_FAILED, + /// ); + /// ``` + pub fn from_result(result: Result, error_code: &str) -> Self + where + T: serde::Serialize, + E: std::fmt::Display, + { + match result { + Ok(data) => Self::ok_with_data(data), + Err(e) => Self::from_err(e, error_code), + } + } +} + +/// Image information returned by Query/ListImages. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ImageInfo { + /// Image reference. + pub reference: String, + /// Image digest (sha256:...). + pub digest: String, + /// Image size in bytes. + pub size: u64, + /// Creation timestamp (ISO 8601). + pub created: Option, + /// Platform architecture. + pub architecture: String, + /// Platform OS. + pub os: String, + /// Number of layers. + pub layer_count: usize, + /// Layer digests in order. + pub layers: Vec, + /// Image entrypoint (from OCI config). + #[serde(default)] + pub entrypoint: Vec, + /// Image default command (from OCI config). + #[serde(default)] + pub cmd: Vec, + /// Image environment variables (from OCI config). + #[serde(default)] + pub env: Vec, + /// Image working directory (from OCI config). + #[serde(default)] + pub workdir: Option, + /// Image default user (from OCI config). + #[serde(default)] + pub user: Option, +} + +/// Overlay preparation result. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OverlayInfo { + /// Path to the merged overlay rootfs. + pub rootfs_path: String, + /// Path to the upper (writable) directory. + pub upper_path: String, + /// Path to the work directory. + pub work_path: String, +} + +/// Storage status information. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StorageStatus { + /// Whether the storage is formatted and ready. + pub ready: bool, + /// Total size in bytes. + pub total_bytes: u64, + /// Used size in bytes. + pub used_bytes: u64, + /// Number of cached layers. + pub layer_count: usize, + /// Number of cached images. + pub image_count: usize, +} + +/// Registry authentication credentials for pulling images. +/// +/// `Debug` is hand-written to redact the password: this value is carried inside +/// `AgentRequest::Pull`, and any `{:?}` of that request (e.g. a tracing span) +/// would otherwise serialize the token verbatim into the machine's console log, +/// which is exposed over the logs API. +#[derive(Clone, Serialize, Deserialize)] +pub struct RegistryAuth { + /// Username for authentication. + pub username: String, + /// Password or token for authentication. + pub password: String, +} + +impl std::fmt::Debug for RegistryAuth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RegistryAuth") + .field("username", &self.username) + .field("password", &"***") + .finish() + } +} + +// ============================================================================ +// Workload VM Protocol (Command Execution) +// ============================================================================ + +/// Messages from host to workload VM. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum HostMessage { + /// Authentication request. + Auth { + /// Authentication token (base64). + token: String, + /// Protocol version. + protocol_version: u32, + }, + + /// Run a command. + Run { + /// Request ID for correlating responses. + request_id: u64, + /// Command and arguments. + command: Vec, + /// Environment variables. + env: Vec<(String, String)>, + /// Working directory. + workdir: Option, + }, + + /// Execute a command in running VM. + Exec { + /// Request ID. + request_id: u64, + /// Command and arguments. + command: Vec, + /// Allocate a TTY. + tty: bool, + }, + + /// Send a signal to a running command. + Signal { + /// Request ID of the command. + request_id: u64, + /// Signal number. + signal: i32, + }, + + /// Request graceful shutdown. + Stop { + /// Timeout in milliseconds. + timeout_ms: u64, + }, +} + +/// Messages from workload VM to host. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum GuestMessage { + /// Authentication successful. + AuthOk, + + /// Authentication failed. + AuthFailed, + + /// VM is ready to receive commands. + Ready, + + /// Command started. + Started { + /// Request ID. + request_id: u64, + }, + + /// Stdout data from command. + Stdout { + /// Request ID. + request_id: u64, + /// Output data. + #[serde(with = "base64_bytes")] + data: Vec, + /// Whether output was truncated. + truncated: bool, + }, + + /// Stderr data from command. + Stderr { + /// Request ID. + request_id: u64, + /// Output data. + #[serde(with = "base64_bytes")] + data: Vec, + /// Whether output was truncated. + truncated: bool, + }, + + /// Command exited. + Exit { + /// Request ID. + request_id: u64, + /// Exit code. + code: i32, + /// Exit reason. + reason: String, + }, + + /// Error occurred. + Error { + /// Request ID (if applicable). + request_id: Option, + /// Error message. + message: String, + }, +} + +// ============================================================================ +// Wire Format Helpers +// ============================================================================ + +/// Envelope that wraps any message with an optional trace ID for correlation. +/// +/// On the wire, the trace_id is flattened into the JSON alongside the message +/// fields: `{"trace_id":"abc123","method":"ping"}`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Envelope { + /// Trace ID for correlating host API requests to agent operations. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub trace_id: Option, + /// The wrapped message. + #[serde(flatten)] + pub body: T, +} + +impl Envelope { + /// Create an envelope with no trace ID. + pub fn new(body: T) -> Self { + Self { + trace_id: None, + body, + } + } + + /// Create an envelope with an optional trace ID. + pub fn with_trace_id(body: T, trace_id: Option) -> Self { + Self { trace_id, body } + } +} + +/// Encode a message to wire format (length-prefixed JSON). +pub fn encode_message(msg: &T) -> Result, serde_json::Error> { + let json = serde_json::to_vec(msg)?; + let len = json.len() as u32; + + let mut buf = Vec::with_capacity(4 + json.len()); + buf.extend_from_slice(&len.to_be_bytes()); + buf.extend_from_slice(&json); + + Ok(buf) +} + +/// Decode a message from wire format. +pub fn decode_message Deserialize<'de>>(data: &[u8]) -> Result { + if data.len() < 4 { + return Err(DecodeError::TooShort); + } + + let len = u32::from_be_bytes([data[0], data[1], data[2], data[3]]) as usize; + + if len > MAX_FRAME_SIZE as usize { + return Err(DecodeError::TooLarge(len)); + } + + if data.len() < 4 + len { + return Err(DecodeError::Incomplete { + expected: len, + got: data.len() - 4, + }); + } + + serde_json::from_slice(&data[4..4 + len]).map_err(DecodeError::Json) +} + +/// Error decoding a wire message. +#[derive(Debug)] +pub enum DecodeError { + /// Data too short to contain length header. + TooShort, + /// Frame size exceeds maximum. + TooLarge(usize), + /// Incomplete frame. + Incomplete { + /// Expected length. + expected: usize, + /// Actual length. + got: usize, + }, + /// JSON parse error. + Json(serde_json::Error), +} + +impl std::fmt::Display for DecodeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DecodeError::TooShort => write!(f, "data too short for length header"), + DecodeError::TooLarge(size) => write!(f, "frame too large: {} bytes", size), + DecodeError::Incomplete { expected, got } => { + write!( + f, + "incomplete frame: expected {} bytes, got {}", + expected, got + ) + } + DecodeError::Json(e) => write!(f, "JSON decode error: {}", e), + } + } +} + +impl std::error::Error for DecodeError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_encode_decode_roundtrip() { + let req = AgentRequest::Pull { + image: "alpine:latest".to_string(), + oci_platform: Some("linux/arm64".to_string()), + auth: None, + proxy: None, + no_proxy: None, + }; + + let encoded = encode_message(&req).unwrap(); + let decoded: AgentRequest = decode_message(&encoded).unwrap(); + + let AgentRequest::Pull { + image, + oci_platform, + auth, + proxy, + no_proxy, + } = decoded + else { + panic!("expected Pull variant, got {:?}", decoded); + }; + assert_eq!(image, "alpine:latest"); + assert_eq!(oci_platform, Some("linux/arm64".to_string())); + assert!(auth.is_none()); + assert!(proxy.is_none()); + assert!(no_proxy.is_none()); + } + + #[test] + fn test_encode_decode_with_auth() { + let req = AgentRequest::Pull { + image: "ghcr.io/owner/repo:latest".to_string(), + oci_platform: None, + auth: Some(RegistryAuth { + username: "testuser".to_string(), + password: "testpass".to_string(), + }), + proxy: None, + no_proxy: None, + }; + + let encoded = encode_message(&req).unwrap(); + let decoded: AgentRequest = decode_message(&encoded).unwrap(); + + let AgentRequest::Pull { + image, + oci_platform, + auth, + proxy: _, + no_proxy: _, + } = decoded + else { + panic!("expected Pull variant, got {:?}", decoded); + }; + assert_eq!(image, "ghcr.io/owner/repo:latest"); + assert!(oci_platform.is_none()); + let auth = auth.expect("auth should be Some"); + assert_eq!(auth.username, "testuser"); + assert_eq!(auth.password, "testpass"); + } + + #[test] + fn test_encode_decode_with_proxy() { + let req = AgentRequest::Pull { + image: "alpine:latest".to_string(), + oci_platform: None, + auth: None, + proxy: Some("http://192.168.127.254:3128".to_string()), + no_proxy: Some("127.0.0.1,localhost,.internal".to_string()), + }; + + let encoded = encode_message(&req).unwrap(); + let decoded: AgentRequest = decode_message(&encoded).unwrap(); + + let AgentRequest::Pull { + proxy, no_proxy, .. + } = decoded + else { + panic!("expected Pull variant, got {:?}", decoded); + }; + assert_eq!(proxy.as_deref(), Some("http://192.168.127.254:3128")); + assert_eq!(no_proxy.as_deref(), Some("127.0.0.1,localhost,.internal")); + } + + #[test] + fn test_decode_too_short() { + let data = [0u8; 2]; + let result: Result = decode_message(&data); + assert!(matches!(result, Err(DecodeError::TooShort))); + } + + #[test] + fn test_decode_incomplete() { + let mut data = vec![0, 0, 0, 100]; // claims 100 bytes + data.extend_from_slice(b"{}"); // only 2 bytes of payload + let result: Result = decode_message(&data); + assert!(matches!(result, Err(DecodeError::Incomplete { .. }))); + } + + #[test] + fn test_agent_request_serialization() { + let req = AgentRequest::Ping; + let json = serde_json::to_string(&req).unwrap(); + assert!(json.contains("ping")); + + let req = AgentRequest::PrepareOverlay { + image: "ubuntu:22.04".to_string(), + workload_id: "wl-123".to_string(), + }; + let json = serde_json::to_string(&req).unwrap(); + assert!(json.contains("prepare_overlay")); + } + + #[test] + fn test_agent_response_serialization() { + let resp = AgentResponse::Pong { + version: PROTOCOL_VERSION, + }; + let json = serde_json::to_string(&resp).unwrap(); + assert!(json.contains("pong")); + + let resp = AgentResponse::Progress { + message: "Pulling layer 1/3".to_string(), + percent: Some(33), + layer: Some("sha256:abc123".to_string()), + }; + let json = serde_json::to_string(&resp).unwrap(); + assert!(json.contains("progress")); + } + + #[test] + fn file_write_begin_roundtrips() { + let req = AgentRequest::FileWriteBegin { + path: "/tmp/target".into(), + mode: Some(0o600), + total_size: 123_456_789, + }; + let bytes = encode_message(&req).unwrap(); + let back: AgentRequest = decode_message(&bytes).unwrap(); + match back { + AgentRequest::FileWriteBegin { + path, + mode, + total_size, + } => { + assert_eq!(path, "/tmp/target"); + assert_eq!(mode, Some(0o600)); + assert_eq!(total_size, 123_456_789); + } + _ => panic!("wrong variant"), + } + } + + #[test] + fn file_write_chunk_roundtrips_binary_data() { + // Binary data (bytes outside UTF-8) must survive the base64 + // trip intact. If the encoding ever silently lossifies, this + // fires. + let payload: Vec = (0u8..=255).collect(); + let req = AgentRequest::FileWriteChunk { + data: payload.clone(), + done: true, + }; + let bytes = encode_message(&req).unwrap(); + let back: AgentRequest = decode_message(&bytes).unwrap(); + match back { + AgentRequest::FileWriteChunk { data, done } => { + assert_eq!(data, payload); + assert!(done); + } + _ => panic!("wrong variant"), + } + } + + #[test] + fn file_write_size_constants_are_frame_safe() { + // Sanity: a single streaming chunk at FILE_WRITE_CHUNK_SIZE + // must fit inside MAX_FRAME_SIZE after base64 (+ ~33%) and + // JSON overhead. If anyone bumps CHUNK_SIZE past the limit, + // this test fires before production does. + let chunk_bytes = FILE_WRITE_CHUNK_SIZE as u64; + let base64_bytes = chunk_bytes.div_ceil(3) * 4; // ceil(n/3)*4 + let json_overhead = 256u64; // method tag, done bool, quotes + let total = base64_bytes + json_overhead; + assert!( + total < MAX_FRAME_SIZE as u64, + "FILE_WRITE_CHUNK_SIZE of {} bytes would produce a frame \ + of ~{} bytes which exceeds MAX_FRAME_SIZE of {}", + chunk_bytes, + total, + MAX_FRAME_SIZE + ); + + // Single-shot threshold must be <= chunk size. They can be + // equal (a 1 MiB file is a single shot; a 1 MiB + 1 byte + // file streams as two chunks); but SINGLE_SHOT > CHUNK would + // be incoherent — a file slightly over the shot threshold + // would need to stream as... a single oversized chunk. + assert!(FILE_WRITE_SINGLE_SHOT_MAX <= FILE_WRITE_CHUNK_SIZE); + } + + #[test] + fn test_ports_constants() { + assert_eq!(ports::WORKLOAD_CONTROL, 5000); + assert_eq!(ports::WORKLOAD_LOGS, 5001); + assert_eq!(ports::AGENT_CONTROL, 6000); + assert_eq!(ports::SSH_AGENT, 6001); + } + + #[test] + fn test_cid_constants() { + assert_eq!(cid::HOST, 2); + assert_eq!(cid::GUEST, 3); + } + + #[test] + fn test_envelope_serialization_with_trace_id() { + let req = AgentRequest::Ping; + let envelope = Envelope::with_trace_id(&req, Some("abc123".to_string())); + let json = serde_json::to_string(&envelope).unwrap(); + + // trace_id should be flattened alongside the method tag + assert!(json.contains("\"trace_id\":\"abc123\"")); + assert!(json.contains("\"method\":\"ping\"")); + + // Deserialize back — Envelope with flatten + let parsed: Envelope = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed.trace_id.as_deref(), Some("abc123")); + assert!(matches!(parsed.body, AgentRequest::Ping)); + } + + #[test] + fn test_envelope_without_trace_id() { + let req = AgentRequest::Ping; + let envelope = Envelope::new(&req); + let json = serde_json::to_string(&envelope).unwrap(); + + // No trace_id field (skip_serializing_if = None) + assert!(!json.contains("trace_id")); + assert!(json.contains("\"method\":\"ping\"")); + } + + #[test] + fn test_envelope_backward_compat_bare_request() { + // A bare AgentRequest (no Envelope) should fail to parse as Envelope + // but succeed as bare AgentRequest — this is the agent's fallback path + let bare_json = r#"{"method":"ping"}"#; + + // Envelope parse should fail (no body field to flatten into) + // Actually with flatten, this may work — let's verify + let envelope_result = serde_json::from_str::>(bare_json); + let bare_result = serde_json::from_str::(bare_json); + + // At least one must succeed for backward compat + assert!( + envelope_result.is_ok() || bare_result.is_ok(), + "Neither Envelope nor bare parse succeeded" + ); + + // Bare parse must always work + assert!(bare_result.is_ok()); + assert!(matches!(bare_result.unwrap(), AgentRequest::Ping)); + + // If Envelope works, trace_id should be None + if let Ok(env) = envelope_result { + assert!(env.trace_id.is_none()); + } + } +} diff --git a/crates/smolvm-protocol/src/retry.rs b/crates/smolvm-protocol/src/retry.rs new file mode 100644 index 0000000..5438879 --- /dev/null +++ b/crates/smolvm-protocol/src/retry.rs @@ -0,0 +1,410 @@ +//! Retry utilities for transient failure recovery. +//! +//! Provides exponential backoff retry logic for operations that may fail +//! transiently (network issues, resource contention, etc.). +//! +//! This module is shared between the host and agent to ensure consistent +//! retry behavior across the system. + +use std::thread; +use std::time::Duration; +use tracing::{debug, warn}; + +// ============================================================================ +// Retry Configuration Constants +// ============================================================================ +// +// These values control retry behavior for different operation types. +// They balance between recovering from transient failures and failing fast +// for permanent errors. + +/// Default maximum retry attempts for general operations. +/// 3 attempts = 1 initial + 2 retries, typically sufficient for transient issues. +const DEFAULT_MAX_ATTEMPTS: u32 = 3; + +/// Default initial delay between retries (100ms). +/// Short enough to not add noticeable latency, long enough to let transient +/// issues resolve (e.g., brief network hiccups). +const DEFAULT_INITIAL_DELAY_MS: u64 = 100; + +/// Default maximum delay cap (5 seconds). +/// Prevents exponential backoff from growing too large while keeping +/// total retry time reasonable. +const DEFAULT_MAX_DELAY_SECS: u64 = 5; + +/// Standard exponential backoff multiplier. +/// Each retry waits 2x longer than the previous (100ms -> 200ms -> 400ms...). +const BACKOFF_MULTIPLIER: f64 = 2.0; + +/// Maximum retry attempts for network operations (image pulls, registry calls). +/// 4 attempts allows recovery from longer network interruptions while still +/// failing within a reasonable time (~45 seconds worst case). +const NETWORK_MAX_ATTEMPTS: u32 = 4; + +/// Initial delay for network operations (500ms). +/// Longer than default because network issues often need more time to resolve +/// (DNS propagation, connection pool recovery, rate limit windows). +const NETWORK_INITIAL_DELAY_MS: u64 = 500; + +/// Maximum delay for network operations (5 seconds). +/// Capped to keep total retry time reasonable. If network issues persist +/// beyond this, they're likely not transient. +const NETWORK_MAX_DELAY_SECS: u64 = 5; + +/// Initial delay for connection operations (100ms). +/// Short because the agent should already be running — we're handling brief +/// unavailability during high load or socket setup under concurrent boot. +const CONNECTION_INITIAL_DELAY_MS: u64 = 100; + +/// Maximum delay for connection operations (2 seconds). +const CONNECTION_MAX_DELAY_SECS: u64 = 2; + +/// Configuration for retry behavior. +#[derive(Debug, Clone)] +pub struct RetryConfig { + /// Maximum number of attempts (including the initial attempt). + pub max_attempts: u32, + /// Initial delay between retries. + pub initial_delay: Duration, + /// Maximum delay between retries (caps exponential growth). + pub max_delay: Duration, + /// Multiplier for exponential backoff (typically 2.0). + pub backoff_multiplier: f64, +} + +impl Default for RetryConfig { + fn default() -> Self { + Self { + max_attempts: DEFAULT_MAX_ATTEMPTS, + initial_delay: Duration::from_millis(DEFAULT_INITIAL_DELAY_MS), + max_delay: Duration::from_secs(DEFAULT_MAX_DELAY_SECS), + backoff_multiplier: BACKOFF_MULTIPLIER, + } + } +} + +impl RetryConfig { + /// Create a config optimized for network operations (image pulls, registry calls). + /// + /// Uses longer delays and more attempts because network issues often need + /// more time to resolve (DNS, rate limits, connection pool recovery). + pub fn for_network() -> Self { + Self { + max_attempts: NETWORK_MAX_ATTEMPTS, + initial_delay: Duration::from_millis(NETWORK_INITIAL_DELAY_MS), + max_delay: Duration::from_secs(NETWORK_MAX_DELAY_SECS), + backoff_multiplier: BACKOFF_MULTIPLIER, + } + } + + /// Create a config for socket/connection operations. + /// + /// Uses short delays because the agent should already be running. + /// These retries handle brief unavailability during high load. + pub fn for_connection() -> Self { + Self { + max_attempts: 6, + initial_delay: Duration::from_millis(CONNECTION_INITIAL_DELAY_MS), + max_delay: Duration::from_secs(CONNECTION_MAX_DELAY_SECS), + backoff_multiplier: BACKOFF_MULTIPLIER, + } + } +} + +/// Execute an operation with retry logic. +/// +/// The `should_retry` function determines whether a given error is transient +/// and worth retrying. +/// +/// # Example +/// +/// ```ignore +/// let result = retry_with_backoff( +/// RetryConfig::for_network(), +/// "fetch manifest", +/// || fetch_manifest(image), +/// |e| is_transient_network_error(&e.to_string()), +/// ); +/// ``` +pub fn retry_with_backoff( + config: RetryConfig, + operation_name: &str, + mut operation: F, + should_retry: R, +) -> Result +where + F: FnMut() -> Result, + R: Fn(&E) -> bool, + E: std::fmt::Display, +{ + let mut attempt = 0; + let mut delay = config.initial_delay; + + loop { + attempt += 1; + + match operation() { + Ok(result) => { + if attempt > 1 { + debug!( + operation = %operation_name, + attempts = attempt, + "operation succeeded after retry" + ); + } + return Ok(result); + } + Err(e) => { + if attempt >= config.max_attempts { + warn!( + operation = %operation_name, + attempts = attempt, + error = %e, + "operation failed after max attempts" + ); + return Err(e); + } + + if !should_retry(&e) { + debug!( + operation = %operation_name, + attempt = attempt, + error = %e, + "operation failed with non-retryable error" + ); + return Err(e); + } + + warn!( + operation = %operation_name, + attempt = attempt, + max_attempts = config.max_attempts, + delay_ms = delay.as_millis(), + error = %e, + "operation failed, will retry" + ); + + thread::sleep(delay); + + // Exponential backoff with cap + delay = Duration::from_secs_f64( + (delay.as_secs_f64() * config.backoff_multiplier) + .min(config.max_delay.as_secs_f64()), + ); + } + } + } +} + +/// Check if an error message indicates a transient network failure. +/// +/// This is a heuristic based on common error messages from network tools +/// and registries. +pub fn is_transient_network_error(error_msg: &str) -> bool { + let error_lower = error_msg.to_lowercase(); + + // Connection errors (transient) + if error_lower.contains("connection refused") + || error_lower.contains("connection reset") + || error_lower.contains("connection timed out") + || error_lower.contains("network is unreachable") + || error_lower.contains("no route to host") + || error_lower.contains("temporary failure") + || error_lower.contains("try again") + || error_lower.contains("resource temporarily unavailable") + { + return true; + } + + // DNS errors (often transient) + if error_lower.contains("name resolution") + || error_lower.contains("dns") + || error_lower.contains("could not resolve") + || error_lower.contains("no such host") + { + return true; + } + + // HTTP errors that may be transient + if error_lower.contains("502 bad gateway") + || error_lower.contains("503 service unavailable") + || error_lower.contains("504 gateway timeout") + || error_lower.contains("429 too many requests") + { + return true; + } + + // Registry-specific transient errors + if error_lower.contains("toomanyrequests") + || error_lower.contains("rate limit") + || error_lower.contains("quota exceeded") + { + return true; + } + + // I/O errors that may be transient + if error_lower.contains("broken pipe") + || error_lower.contains("interrupted") + || error_lower.contains("eagain") + || error_lower.contains("ewouldblock") + { + return true; + } + + false +} + +/// Check if an error message indicates a permanent failure (don't retry). +pub fn is_permanent_error(error_msg: &str) -> bool { + let error_lower = error_msg.to_lowercase(); + + // Authentication/authorization errors + if error_lower.contains("401 unauthorized") + || error_lower.contains("403 forbidden") + || error_lower.contains("authentication required") + || error_lower.contains("access denied") + { + return true; + } + + // Not found errors + if error_lower.contains("404 not found") + || error_lower.contains("manifest unknown") + || error_lower.contains("name unknown") + || error_lower.contains("repository does not exist") + { + return true; + } + + // Invalid input + if error_lower.contains("invalid reference") + || error_lower.contains("invalid image") + || error_lower.contains("malformed") + { + return true; + } + + false +} + +/// Check if an I/O error is likely transient and worth retrying. +pub fn is_transient_io_error(error: &std::io::Error) -> bool { + use std::io::ErrorKind; + + matches!( + error.kind(), + ErrorKind::ConnectionRefused + | ErrorKind::ConnectionReset + | ErrorKind::ConnectionAborted + | ErrorKind::NotConnected + | ErrorKind::BrokenPipe + | ErrorKind::TimedOut + | ErrorKind::Interrupted + | ErrorKind::WouldBlock + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + + #[test] + fn test_retry_success_first_attempt() { + let result: Result = + retry_with_backoff(RetryConfig::default(), "test", || Ok(42), |_| true); + assert_eq!(result.unwrap(), 42); + } + + #[test] + fn test_retry_success_after_failures() { + let attempts = RefCell::new(0); + let result: Result = retry_with_backoff( + RetryConfig { + max_attempts: 3, + initial_delay: Duration::from_millis(1), + max_delay: Duration::from_millis(10), + backoff_multiplier: 2.0, + }, + "test", + || { + *attempts.borrow_mut() += 1; + if *attempts.borrow() < 3 { + Err("transient error") + } else { + Ok(42) + } + }, + |_| true, + ); + assert_eq!(result.unwrap(), 42); + assert_eq!(*attempts.borrow(), 3); + } + + #[test] + fn test_retry_exhausted() { + let attempts = RefCell::new(0); + let result: Result = retry_with_backoff( + RetryConfig { + max_attempts: 3, + initial_delay: Duration::from_millis(1), + max_delay: Duration::from_millis(10), + backoff_multiplier: 2.0, + }, + "test", + || { + *attempts.borrow_mut() += 1; + Err("always fails") + }, + |_| true, + ); + assert!(result.is_err()); + assert_eq!(*attempts.borrow(), 3); + } + + #[test] + fn test_retry_non_retryable_error() { + let attempts = RefCell::new(0); + let result: Result = retry_with_backoff( + RetryConfig::default(), + "test", + || { + *attempts.borrow_mut() += 1; + Err("permanent error") + }, + |_| false, // Never retry + ); + assert!(result.is_err()); + assert_eq!(*attempts.borrow(), 1); + } + + #[test] + fn test_transient_network_errors() { + assert!(is_transient_network_error("connection refused")); + assert!(is_transient_network_error("Connection timed out")); + assert!(is_transient_network_error("503 Service Unavailable")); + assert!(is_transient_network_error("rate limit exceeded")); + assert!(!is_transient_network_error("404 not found")); + assert!(!is_transient_network_error("some random error")); + } + + #[test] + fn test_permanent_errors() { + assert!(is_permanent_error("401 Unauthorized")); + assert!(is_permanent_error("404 Not Found")); + assert!(is_permanent_error("manifest unknown")); + assert!(!is_permanent_error("connection refused")); + assert!(!is_permanent_error("503 Service Unavailable")); + } + + #[test] + fn test_config_presets() { + let network = RetryConfig::for_network(); + assert_eq!(network.max_attempts, 4); + assert_eq!(network.initial_delay, Duration::from_millis(500)); + + let connection = RetryConfig::for_connection(); + assert_eq!(connection.max_attempts, 6); + assert_eq!(connection.initial_delay, Duration::from_millis(100)); + } +} diff --git a/crates/smolvm-protocol/src/secrets.rs b/crates/smolvm-protocol/src/secrets.rs new file mode 100644 index 0000000..70e45fb --- /dev/null +++ b/crates/smolvm-protocol/src/secrets.rs @@ -0,0 +1,140 @@ +//! Secret reference types shared across smolvm surfaces. +//! +//! A [`SecretRef`] is a *pointer* to a secret on the host: either a host +//! environment variable or a host file. Refs travel across trust +//! boundaries (HTTP request bodies, persisted VM records, `.smolmachine` +//! pack manifests); resolved plaintext values do not. smolvm itself does +//! not store secret material — it only references whatever secrets +//! manager already holds it (your shell env, Vault/1Password/etc. rendered +//! to an env var or file). +//! +//! This crate carries only the *shape* of a ref — the on-the-wire and +//! on-disk representation plus trivial introspection. The validation +//! policy (which source kinds are allowed at which trust boundary) +//! lives in the host crate alongside the code that enforces it. See +//! `smolvm::secrets` for `ResolutionScope` and `validate_ref`. + +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +/// Which source a [`SecretRef`] points at, independent of the data +/// inside. Used by audit logging so the logger never sees the path or +/// env-var name (which can themselves be revealing). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SecretSourceKind { + /// The ref points at a host environment variable. + Env, + /// The ref points at a host file path. + File, +} + +impl SecretSourceKind { + /// Human-readable label. + pub fn as_str(self) -> &'static str { + match self { + Self::Env => "env", + Self::File => "file", + } + } +} + +/// A reference to a secret. Exactly one of the two sources must be +/// populated; validation is performed by the host crate's +/// `validate_ref` (policy lives where it's enforced). +/// +/// Round-trips through `serde_json` for persistence in the VM record DB +/// and in `.smolmachine` pack manifests. Refs are not sensitive; the +/// resolved plaintext is produced only at the workload launch site and +/// never touches any of these stores. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SecretRef { + /// Read the secret from a host environment variable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_env: Option, + + /// Read the secret from a host file path (must be absolute). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_file: Option, +} + +impl SecretRef { + /// Return the source kind for this ref, if exactly one source is set. + /// + /// Returns `None` for structurally invalid refs (0 or >1 sources). + /// Callers are expected to have already validated with the host + /// crate's `validate_ref` before calling this; this function is + /// primarily for audit logging of a known-good ref. + pub fn source_kind(&self) -> Option { + match (self.from_env.is_some(), self.from_file.is_some()) { + (true, false) => Some(SecretSourceKind::Env), + (false, true) => Some(SecretSourceKind::File), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn source_kind_reports_variant() { + assert_eq!( + SecretRef { + from_env: Some("Y".into()), + from_file: None, + } + .source_kind(), + Some(SecretSourceKind::Env) + ); + assert_eq!( + SecretRef { + from_env: None, + from_file: Some(PathBuf::from("/z")), + } + .source_kind(), + Some(SecretSourceKind::File) + ); + let empty = SecretRef { + from_env: None, + from_file: None, + }; + assert_eq!(empty.source_kind(), None); + let both = SecretRef { + from_env: Some("Y".into()), + from_file: Some(PathBuf::from("/z")), + }; + assert_eq!(both.source_kind(), None); + } + + #[test] + fn deny_unknown_fields() { + // `from_store` was removed; it must now be rejected like any typo. + let bad = r#"{ "from_store": "gone" }"#; + let res: Result = serde_json::from_str(bad); + assert!(res.is_err()); + } + + #[test] + fn roundtrip_json() { + let r = SecretRef { + from_env: Some("API_KEY".into()), + from_file: None, + }; + let json = serde_json::to_string(&r).unwrap(); + let back: SecretRef = serde_json::from_str(&json).unwrap(); + assert_eq!(r, back); + } + + #[test] + fn serialization_omits_empty_fields() { + let r = SecretRef { + from_env: Some("X".into()), + from_file: None, + }; + let json = serde_json::to_string(&r).unwrap(); + assert!(json.contains("from_env")); + assert!(!json.contains("from_file")); + } +} diff --git a/crates/smolvm-registry/Cargo.toml b/crates/smolvm-registry/Cargo.toml new file mode 100644 index 0000000..bfa0bda --- /dev/null +++ b/crates/smolvm-registry/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "smolvm-registry" +version = "1.5.2" +edition = "2021" +description = "OCI Distribution client for smolmachines registry" +license = "Apache-2.0" +repository = "https://github.com/smol-machines/smolvm" + +[dependencies] +smolvm-pack = { path = "../smolvm-pack", version = "1.5.2" } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream", "json"] } +sha2 = "0.10" +bytes = "1" +hex = "0.4" +tokio = { version = "1", features = ["fs", "io-util"] } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +dirs = "5" +filetime = "0.2" +tokio-util = { version = "0.7", features = ["io"] } +futures-util = "0.3" + +[dev-dependencies] +tokio = { version = "1", features = ["full", "test-util"] } +tempfile = "3" +wiremock = "0.6" diff --git a/crates/smolvm-registry/src/cache.rs b/crates/smolvm-registry/src/cache.rs new file mode 100644 index 0000000..97395b4 --- /dev/null +++ b/crates/smolvm-registry/src/cache.rs @@ -0,0 +1,315 @@ +//! Local blob cache for registry pulls. +//! +//! Content-addressed storage at `~/.cache/smolvm-registry/blobs/sha256/`. +//! Blobs are stored by their digest. LRU eviction keeps total size under a +//! configurable limit — default 5 GB, override with `SMOLVM_BLOB_CACHE_MAX_BYTES`. + +use std::fs; +use std::path::{Path, PathBuf}; + +/// Default maximum blob-cache size when `SMOLVM_BLOB_CACHE_MAX_BYTES` is unset: 5 GB. +const DEFAULT_MAX_SIZE: u64 = 5 * 1024 * 1024 * 1024; + +/// Resolve the blob-cache byte cap from `SMOLVM_BLOB_CACHE_MAX_BYTES`, falling +/// back to [`DEFAULT_MAX_SIZE`]. A fleet pulling multi-GB `.smolmachine` +/// artifacts should set this well above the largest artifact (bounded by disk) +/// so hot worlds are not re-pulled from the registry on every launch. +fn configured_max_size() -> u64 { + parse_cache_limit(std::env::var("SMOLVM_BLOB_CACHE_MAX_BYTES").ok().as_deref()) +} + +/// Parse a byte-count cache limit, ignoring absent / unparseable / zero values. +/// Split out from `configured_max_size` so it is unit-testable without touching +/// the process-global environment. +fn parse_cache_limit(val: Option<&str>) -> u64 { + val.and_then(|v| v.trim().parse::().ok()) + .filter(|&n| n > 0) + .unwrap_or(DEFAULT_MAX_SIZE) +} + +/// True if a cache-dir entry is an in-flight `.partial` download rather than a +/// finalized blob. A partial is being streamed by an active pull, so it must +/// never be counted toward the cache size or evicted — deleting one makes the +/// owning pull's `adopt` rename fail with ENOENT ("No such file or directory"), +/// which surfaces as `registry pull failed` on large artifacts (only large ones +/// push the cache over its cap and trigger eviction in the first place). +fn is_partial(path: &Path) -> bool { + path.extension().and_then(|e| e.to_str()) == Some("partial") +} + +/// Content-addressed blob cache. +pub struct BlobCache { + root: PathBuf, + max_size: u64, +} + +impl BlobCache { + /// Open or create a cache at the default location. + pub fn open_default() -> std::io::Result { + let cache_dir = dirs::cache_dir() + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "no cache dir"))? + .join("smolvm-registry") + .join("blobs"); + Self::open(cache_dir, configured_max_size()) + } + + /// Open or create a cache at a specific path with a size limit. + pub fn open(root: PathBuf, max_size: u64) -> std::io::Result { + fs::create_dir_all(&root)?; + Ok(Self { root, max_size }) + } + + /// Look up a blob by digest. Returns the path if it exists. + pub fn get(&self, digest: &str) -> Option { + let path = self.blob_path(digest); + if path.exists() { + // Touch atime for LRU tracking. + let _ = filetime::set_file_atime(&path, filetime::FileTime::now()); + Some(path) + } else { + None + } + } + + /// Store a blob. Returns the path where it was written. + /// + /// If storing this blob would exceed `max_size`, evicts least-recently-accessed + /// blobs first. + pub fn put(&self, digest: &str, data: &[u8]) -> std::io::Result { + let path = self.blob_path(digest); + if path.exists() { + return Ok(path); + } + + // Evict if needed. + let current = self.total_size()?; + if current + data.len() as u64 > self.max_size { + self.evict_until(self.max_size.saturating_sub(data.len() as u64))?; + } + + // Write atomically via temp file. + let tmp = path.with_extension("partial"); + fs::write(&tmp, data)?; + fs::rename(&tmp, &path)?; + Ok(path) + } + + /// Total size of all cached blobs in bytes. + pub fn total_size(&self) -> std::io::Result { + let mut total = 0u64; + if self.root.exists() { + for entry in fs::read_dir(&self.root)? { + let entry = entry?; + if entry.file_type()?.is_file() && !is_partial(&entry.path()) { + total += entry.metadata()?.len(); + } + } + } + Ok(total) + } + + /// Remove all cached blobs. + pub fn prune_all(&self) -> std::io::Result { + let mut freed = 0u64; + if self.root.exists() { + for entry in fs::read_dir(&self.root)? { + let entry = entry?; + if entry.file_type()?.is_file() { + freed += entry.metadata()?.len(); + fs::remove_file(entry.path())?; + } + } + } + Ok(freed) + } + + /// Evict least-recently-accessed blobs until total size is at or below `target`. + fn evict_until(&self, target: u64) -> std::io::Result<()> { + let mut entries: Vec<(PathBuf, u64, std::time::SystemTime)> = Vec::new(); + + for entry in fs::read_dir(&self.root)? { + let entry = entry?; + // Never evict in-flight `.partial` downloads — a concurrent pull is + // actively writing them; deleting one breaks its adopt with ENOENT. + if !entry.file_type()?.is_file() || is_partial(&entry.path()) { + continue; + } + let meta = entry.metadata()?; + let atime = meta.accessed().unwrap_or(std::time::UNIX_EPOCH); + entries.push((entry.path(), meta.len(), atime)); + } + + // Sort by atime ascending (oldest first). + entries.sort_by_key(|(_, _, atime)| *atime); + + let mut current = entries.iter().map(|(_, size, _)| size).sum::(); + + for (path, size, _) in &entries { + if current <= target { + break; + } + tracing::debug!(path = %path.display(), size, "evicting cached blob"); + fs::remove_file(path)?; + current -= size; + } + + Ok(()) + } + + /// Return the cache path for a blob with this digest. + /// + /// Used by the pull flow to write directly to `path.with_extension("partial")`, + /// then call [`adopt`] to finalize. + pub fn blob_path_for(&self, digest: &str) -> PathBuf { + self.blob_path(digest) + } + + /// Adopt an externally-written partial file into the cache. + /// + /// Expects the file at `blob_path_for(digest).with_extension("partial")` to + /// be fully written and digest-verified by the caller. Handles eviction if + /// needed, then atomically renames the partial file into place. + pub fn adopt(&self, digest: &str, size: u64) -> std::io::Result { + let path = self.blob_path(digest); + let partial = path.with_extension("partial"); + + if path.exists() { + // Already cached (race condition protection). + let _ = fs::remove_file(&partial); + return Ok(path); + } + + let current = self.total_size()?; + if current + size > self.max_size { + self.evict_until(self.max_size.saturating_sub(size))?; + } + + fs::rename(&partial, &path)?; + Ok(path) + } + + /// Path for a blob with the given digest. + fn blob_path(&self, digest: &str) -> PathBuf { + // Store as flat files: "sha256:abc..." → "sha256_abc..." + let filename = digest.replace(':', "_"); + self.root.join(filename) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_put_and_get() { + let tmp = tempfile::tempdir().unwrap(); + let cache = BlobCache::open(tmp.path().to_path_buf(), 1024 * 1024).unwrap(); + + let digest = "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + let data = b"hello world"; + + // Miss before put. + assert!(cache.get(digest).is_none()); + + // Put and get. + let path = cache.put(digest, data).unwrap(); + assert!(path.exists()); + assert_eq!(fs::read(&path).unwrap(), data); + + // Hit after put. + assert!(cache.get(digest).is_some()); + } + + #[test] + fn test_put_idempotent() { + let tmp = tempfile::tempdir().unwrap(); + let cache = BlobCache::open(tmp.path().to_path_buf(), 1024 * 1024).unwrap(); + + let digest = "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + cache.put(digest, b"data1").unwrap(); + // Second put with same digest doesn't overwrite. + cache.put(digest, b"data2").unwrap(); + let path = cache.get(digest).unwrap(); + assert_eq!(fs::read(path).unwrap(), b"data1"); + } + + #[test] + fn test_eviction() { + let tmp = tempfile::tempdir().unwrap(); + // Tiny cache: 20 bytes max. + let cache = BlobCache::open(tmp.path().to_path_buf(), 20).unwrap(); + + let d1 = "sha256:1111111111111111111111111111111111111111111111111111111111111111"; + let d2 = "sha256:2222222222222222222222222222222222222222222222222222222222222222"; + + cache.put(d1, &[0u8; 15]).unwrap(); + assert!(cache.get(d1).is_some()); + + // This should trigger eviction of d1. + cache.put(d2, &[0u8; 15]).unwrap(); + assert!(cache.get(d2).is_some()); + // d1 should have been evicted. + assert!(cache.get(d1).is_none()); + } + + #[test] + fn evict_does_not_delete_in_flight_partial() { + // Regression (large-artifact "registry pull failed: No such file or + // directory"): under cache pressure, a pull's eviction must not delete + // ANOTHER concurrent pull's in-flight `.partial` download, nor its own. + // Before the fix, `total_size`/`evict_until` counted and removed + // `.partial` files, so the victim's `adopt` rename hit ENOENT. Only + // large artifacts trip it because only they push the cache over its cap. + let tmp = tempfile::tempdir().unwrap(); + let cache = BlobCache::open(tmp.path().to_path_buf(), 100).unwrap(); + + let victim = "sha256:1111111111111111111111111111111111111111111111111111111111111111"; + let pulling = "sha256:2222222222222222222222222222222222222222222222222222222222222222"; + + // A concurrent pull is mid-download: its large `.partial` is on disk. + let victim_partial = cache.blob_path_for(victim).with_extension("partial"); + fs::write(&victim_partial, [0u8; 80]).unwrap(); + + // This pull finishes streaming its own large blob and adopts it — which + // triggers eviction because the cache dir now looks over-cap. + let pulling_partial = cache.blob_path_for(pulling).with_extension("partial"); + fs::write(&pulling_partial, [0u8; 80]).unwrap(); + let adopted = cache + .adopt(pulling, 80) + .expect("adopt must not fail — its own .partial must survive eviction"); + + assert!(adopted.exists(), "adopted blob missing after rename"); + assert!( + victim_partial.exists(), + "eviction deleted another concurrent pull's in-flight .partial (the ENOENT bug)" + ); + } + + #[test] + fn cache_limit_parses_env_or_defaults() { + // Valid byte counts are honored. + assert_eq!( + parse_cache_limit(Some("10737418240")), + 10 * 1024 * 1024 * 1024 + ); + assert_eq!(parse_cache_limit(Some(" 5000000000 ")), 5_000_000_000); + // Absent / unparseable / zero fall back to the default. + assert_eq!(parse_cache_limit(None), DEFAULT_MAX_SIZE); + assert_eq!(parse_cache_limit(Some("garbage")), DEFAULT_MAX_SIZE); + assert_eq!(parse_cache_limit(Some("0")), DEFAULT_MAX_SIZE); + } + + #[test] + fn test_prune_all() { + let tmp = tempfile::tempdir().unwrap(); + let cache = BlobCache::open(tmp.path().to_path_buf(), 1024 * 1024).unwrap(); + + let d1 = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + cache.put(d1, b"test data").unwrap(); + assert!(cache.total_size().unwrap() > 0); + + let freed = cache.prune_all().unwrap(); + assert!(freed > 0); + assert_eq!(cache.total_size().unwrap(), 0); + } +} diff --git a/crates/smolvm-registry/src/client.rs b/crates/smolvm-registry/src/client.rs new file mode 100644 index 0000000..b24dc72 --- /dev/null +++ b/crates/smolvm-registry/src/client.rs @@ -0,0 +1,2121 @@ +//! OCI Distribution Spec HTTP client. +//! +//! Implements the subset of the OCI Distribution Spec needed for +//! single-blob artifact push and pull: +//! - Blob existence check (HEAD) +//! - Monolithic blob upload (POST + PUT) +//! - Blob download (GET) +//! - Manifest put/get (PUT/GET) + +use crate::{OciIndex, OciPlatform, RegistryError, Result, INDEX_MEDIA_TYPE, MANIFEST_MEDIA_TYPE}; +use reqwest::header::{ + ACCEPT, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, LINK, LOCATION, WWW_AUTHENTICATE, +}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +/// Maximum bytes buffered in memory for a manifest / image-index document. +/// A hostile or MITM'd registry could otherwise stream a multi-GB (or endless) +/// body that the client buffers entirely BEFORE the digest is checked, OOMing +/// the host. 32 MiB is far above any real OCI manifest (a few KB). +const MAX_MANIFEST_BYTES: usize = 32 * 1024 * 1024; + +/// Maximum bytes buffered in memory for a `pull_blob` response. `pull_blob` is +/// used only for small blobs (the OCI image config, a few KB); large layers use +/// [`RegistryClient::pull_blob_stream`], which streams to disk. 64 MiB is a +/// generous ceiling that still bounds the in-memory buffer. +const MAX_BLOB_BYTES: usize = 64 * 1024 * 1024; + +/// Buffer a response body into memory, refusing to read more than `cap` bytes. +/// +/// Guards against an unbounded body: it rejects on a too-large `Content-Length` +/// up front (cheap), then streams chunk-by-chunk and errors the moment the +/// accumulated size would exceed `cap` — so a lying or absent `Content-Length` +/// can't get past the cap either. This runs BEFORE any digest computation, so +/// the host never buffers an unbounded body first. +async fn read_body_capped(mut resp: reqwest::Response, cap: usize, what: &str) -> Result> { + if let Some(len) = resp.content_length() { + if len > cap as u64 { + return Err(RegistryError::ResponseTooLarge { + what: what.to_string(), + limit: cap, + }); + } + } + let mut buf: Vec = Vec::new(); + while let Some(chunk) = resp.chunk().await? { + if buf.len().saturating_add(chunk.len()) > cap { + return Err(RegistryError::ResponseTooLarge { + what: what.to_string(), + limit: cap, + }); + } + buf.extend_from_slice(&chunk); + } + Ok(buf) +} + +/// Validate that a digest string matches the expected `sha256:<64 hex chars>` format. +/// +/// Public so callers that build a filesystem path (e.g. the serve-side P2P blob +/// handler) can reject a malformed, path-traversing digest at their boundary, +/// exactly as `pull` does before touching the cache. +pub fn validate_digest(digest: &str) -> Result<()> { + if let Some(hex_part) = digest.strip_prefix("sha256:") { + if hex_part.len() == 64 && hex_part.chars().all(|c| c.is_ascii_hexdigit()) { + return Ok(()); + } + } + Err(RegistryError::InvalidManifest(format!( + "invalid digest format: {digest}" + ))) +} + +/// HTTP client for an OCI Distribution registry. +pub struct RegistryClient { + http: reqwest::Client, + /// Base URL including scheme (e.g., "http://localhost:5000" or "https://registry.smolmachines.com"). + base_url: String, + /// Optional Bearer token sent directly to the registry for authenticated requests. + auth_token: Option, + /// Optional identity token exchanged with a registry token service after + /// a `WWW-Authenticate: Bearer ...` challenge. + /// Used for smolmachines registries (Auth0 JWT → token service → OCI bearer). + identity_token: Option, + /// Optional Basic credentials (username, password) exchanged with the registry's + /// own auth service after a `WWW-Authenticate: Bearer ...` challenge. + /// Used for Docker Hub, GHCR, ECR, GCR, and any standard OCI registry. + basic_credentials: Option<(String, String)>, + token_cache: Mutex>, + /// The most recent Bearer challenge received from this registry. + /// + /// Stored after each successful challenge exchange so that subsequent + /// requests can attach a preemptive bearer token (cache hit → no 401 round + /// trip). This is especially important for non-replayable streaming uploads, + /// where a 401 on the PUT body cannot be retried. + last_challenge: Mutex>, +} + +impl RegistryClient { + /// Create a new client for the given registry base URL. + pub fn new(base_url: String) -> Self { + Self { + http: reqwest::Client::new(), + base_url, + auth_token: None, + identity_token: None, + basic_credentials: None, + token_cache: Mutex::new(HashMap::new()), + last_challenge: Mutex::new(None), + } + } + + /// Return the base URL this client is configured for. + pub fn base_url(&self) -> &str { + &self.base_url + } + + /// Return the identity token, if one was configured via [`Self::with_identity_token`]. + /// + /// Useful for inspecting whether the client is in upstream token-exchange mode + /// (identity token path) vs direct bearer mode. + pub fn identity_token(&self) -> Option<&str> { + self.identity_token.as_deref() + } + + /// Set a Bearer token for authenticated requests. + pub fn with_token(mut self, token: String) -> Self { + self.auth_token = Some(token); + self + } + + /// Set an identity token used only to fetch OCI bearer tokens from a + /// registry token service. This token is not sent directly to the registry. + pub fn with_identity_token(mut self, token: String) -> Self { + self.identity_token = Some(token); + self + } + + /// Set Basic credentials for standard Docker/OCI registry Bearer challenge auth. + /// + /// These credentials are sent to the registry's own token endpoint when the + /// registry returns `WWW-Authenticate: Bearer ...`. The resulting short-lived + /// OCI bearer token is then used for the actual registry request. + /// + /// This is the standard path for Docker Hub, GHCR, ECR, GCR/Artifact Registry, + /// ACR, Harbor, Quay, and any OCI-compliant registry. + pub fn with_basic_credentials(mut self, username: String, password: String) -> Self { + self.basic_credentials = Some((username, password)); + self + } + + /// Return the Basic credentials, if configured via [`Self::with_basic_credentials`]. + pub fn basic_credentials(&self) -> Option<(&str, &str)> { + self.basic_credentials + .as_ref() + .map(|(u, p)| (u.as_str(), p.as_str())) + } + + /// Check connectivity: `GET /v2/` must return 200. + pub async fn ping(&self) -> Result<()> { + let url = format!("{}/v2/", self.base_url); + let resp = self + .send_replayable(self.request(reqwest::Method::GET, &url)) + .await?; + if !resp.status().is_success() { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + Ok(()) + } + + /// Check if a blob exists. Returns true if HEAD returns 200. + pub async fn blob_exists(&self, repo: &str, digest: &str) -> Result { + validate_digest(digest)?; + let url = format!("{}/v2/{}/blobs/{}", self.base_url, repo, digest); + let resp = self + .send_replayable(self.request(reqwest::Method::HEAD, &url)) + .await?; + Ok(resp.status() == reqwest::StatusCode::OK) + } + + /// Upload a blob using monolithic upload (POST + PUT). + /// + /// Returns the sha256 digest of the uploaded blob. + pub async fn push_blob(&self, repo: &str, data: &[u8]) -> Result { + let digest = format!("sha256:{}", hex::encode(Sha256::digest(data))); + + // Check if already present (skip upload). + if self.blob_exists(repo, &digest).await? { + tracing::debug!(digest = %digest, "blob already exists, skipping upload"); + return Ok(digest); + } + + // Step 1: POST to initiate upload. + let post_url = format!("{}/v2/{}/blobs/uploads/", self.base_url, repo); + let resp = self + .send_replayable( + self.request(reqwest::Method::POST, &post_url) + .header(CONTENT_LENGTH, 0), + ) + .await?; + + if resp.status() != reqwest::StatusCode::ACCEPTED { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + + // Get the upload URL from Location header. + let location = resp + .headers() + .get(LOCATION) + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| RegistryError::ApiError { + status: 202, + body: "upload accepted but missing Location header".into(), + })? + .to_string(); + + // Resolve Location — validates same-origin for absolute URLs. + let put_url = self.resolve_location(&location)?; + + // Step 2: PUT the blob data with digest. + let separator = if put_url.contains('?') { "&" } else { "?" }; + let put_url = format!("{}{}digest={}", put_url, separator, digest); + + let resp = self + .send_replayable( + self.request(reqwest::Method::PUT, &put_url) + .header(CONTENT_TYPE, "application/octet-stream") + .header(CONTENT_LENGTH, data.len()) + .body(data.to_vec()), + ) + .await?; + + if resp.status() != reqwest::StatusCode::CREATED { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + + Ok(digest) + } + + /// Download a blob by digest. Returns the raw bytes. + /// + /// NOTE: buffers entire blob in memory. For large artifacts, switch to + /// streaming to disk with digest verification via `AsyncRead`. + pub async fn pull_blob(&self, repo: &str, digest: &str) -> Result> { + validate_digest(digest)?; + let url = format!("{}/v2/{}/blobs/{}", self.base_url, repo, digest); + let resp = self + .send_replayable(self.request(reqwest::Method::GET, &url)) + .await?; + + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Err(RegistryError::BlobNotFound(digest.to_string())); + } + + if !resp.status().is_success() { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + + // Cap the in-memory buffer BEFORE hashing so a hostile registry can't + // OOM the host by streaming an unbounded body ahead of the digest check. + let data = read_body_capped(resp, MAX_BLOB_BYTES, "blob").await?; + + // Verify digest. + let actual = format!("sha256:{}", hex::encode(Sha256::digest(&data))); + if actual != digest { + return Err(RegistryError::DigestMismatch { + expected: digest.to_string(), + actual, + }); + } + + Ok(data) + } + + /// Upload a blob from a streamed body with a pre-computed digest. + /// + /// Unlike `push_blob`, this does not buffer the entire blob in memory. + /// The caller pre-computes the digest and provides a **body factory** that + /// can produce a fresh `reqwest::Body` on each call. The factory may be + /// called up to 3 times (initial attempt + 2 auth retries), so it must be + /// able to reopen or rewind the source (e.g., reopen the file). + /// + /// The PUT request is routed through [`send_nonreplayable`], which guarantees + /// that a 401 challenge is always handled correctly: the factory produces a + /// fresh body for each retry, so no data is lost regardless of whether the + /// preemptive token cache held the right scope. + pub async fn push_blob_stream( + &self, + repo: &str, + digest: &str, + size: u64, + make_body: F, + ) -> Result<()> + where + F: Fn() -> Result, + { + validate_digest(digest)?; + + // Skip if already present. + if self.blob_exists(repo, digest).await? { + tracing::debug!(digest = %digest, "blob already exists, skipping upload"); + return Ok(()); + } + + // Step 1: POST to initiate upload (empty body — always replayable). + let post_url = format!("{}/v2/{}/blobs/uploads/", self.base_url, repo); + let resp = self + .send_replayable( + self.request(reqwest::Method::POST, &post_url) + .header(CONTENT_LENGTH, 0), + ) + .await?; + + if resp.status() != reqwest::StatusCode::ACCEPTED { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + + let location = resp + .headers() + .get(LOCATION) + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| RegistryError::ApiError { + status: 202, + body: "upload accepted but missing Location header".into(), + })? + .to_string(); + + // Resolve Location — validates same origin for absolute URLs. + let put_url = self.resolve_location(&location)?; + let separator = if put_url.contains('?') { "&" } else { "?" }; + let put_url = format!("{}{}digest={}", put_url, separator, digest); + + // Step 2: PUT with streaming body via factory. + // Content-Length is set explicitly — reqwest defaults to chunked transfer + // encoding for streamed bodies, which some registries reject. + // The factory is called once per attempt (preemptive + up to 2 retries). + let base_req = self + .request(reqwest::Method::PUT, &put_url) + .header(CONTENT_TYPE, "application/octet-stream") + .header(CONTENT_LENGTH, size); + + let resp = self.send_nonreplayable(base_req, make_body).await?; + + if resp.status() != reqwest::StatusCode::CREATED { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + + Ok(()) + } + + /// Upload a blob in OCI **chunked** mode: POST to open a session, one PATCH + /// per chunk (`Content-Range: -`), then a final PUT carrying the + /// whole-blob digest. + /// + /// Why this exists: a monolithic PUT sends the entire blob as one request + /// body, which a proxy in front of the registry can reject — Cloudflare 413s + /// a multi-hundred-MB upload. Chunking keeps every request under that cap. + /// + /// Each chunk is read into a buffered `Vec` (≤ `chunk_size`) and sent via + /// [`send_replayable`], so every PATCH goes through the full auth path. That + /// matters for large blobs: the OCI access token is short-lived (~5 min), and + /// a multi-chunk upload can outlive it — per-chunk auth lets each request + /// refresh the token instead of failing partway through. + /// + /// Reads from `path` rather than a body factory because chunking requires + /// positioned reads, and reopening the file per attempt would not compose + /// with the per-chunk retry. + pub async fn push_blob_chunked( + &self, + repo: &str, + digest: &str, + path: &std::path::Path, + chunk_size: usize, + ) -> Result<()> { + use tokio::io::AsyncReadExt; + validate_digest(digest)?; + + if self.blob_exists(repo, digest).await? { + tracing::debug!(digest = %digest, "blob already exists, skipping upload"); + return Ok(()); + } + + // Step 1: POST to open an upload session (empty body — replayable). + let post_url = format!("{}/v2/{}/blobs/uploads/", self.base_url, repo); + let resp = self + .send_replayable( + self.request(reqwest::Method::POST, &post_url) + .header(CONTENT_LENGTH, 0), + ) + .await?; + if resp.status() != reqwest::StatusCode::ACCEPTED { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + let mut location = self.upload_location(&resp)?; + + // Step 2: PATCH contiguous chunks. The registry hands back a fresh upload + // URL after each chunk; the next PATCH must target it. + let mut file = tokio::fs::File::open(path).await?; + let mut buf = vec![0u8; chunk_size.max(1)]; + let mut offset: u64 = 0; + loop { + // read() may return short — fill the chunk so all but the last are full. + let mut filled = 0; + while filled < buf.len() { + let n = file.read(&mut buf[filled..]).await?; + if n == 0 { + break; + } + filled += n; + } + if filled == 0 { + break; + } + let start = offset; + let end = offset + filled as u64 - 1; + let resp = self + .send_replayable( + self.request(reqwest::Method::PATCH, &location) + .header(CONTENT_TYPE, "application/octet-stream") + .header(CONTENT_RANGE, format!("{start}-{end}")) + .header(CONTENT_LENGTH, filled) + .body(buf[..filled].to_vec()), + ) + .await?; + if resp.status() != reqwest::StatusCode::ACCEPTED { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + location = self.upload_location(&resp)?; + offset += filled as u64; + } + + // Step 3: PUT (empty body) to close the session, carrying the digest. + let separator = if location.contains('?') { "&" } else { "?" }; + let put_url = format!("{location}{separator}digest={digest}"); + let resp = self + .send_replayable( + self.request(reqwest::Method::PUT, &put_url) + .header(CONTENT_LENGTH, 0), + ) + .await?; + if resp.status() != reqwest::StatusCode::CREATED { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + Ok(()) + } + + /// Extract and resolve the `Location` (next upload URL) from an upload-step + /// response, enforcing same-origin. + fn upload_location(&self, resp: &reqwest::Response) -> Result { + let loc = resp + .headers() + .get(LOCATION) + .and_then(|v| v.to_str().ok()) + .ok_or_else(|| RegistryError::ApiError { + status: 202, + body: "upload step accepted but missing Location header".into(), + })?; + self.resolve_location(loc) + } + + /// Download a blob as a byte stream. + /// + /// Returns the stream after verifying the response status. The caller is + /// responsible for digest verification (hash while writing to disk). + pub async fn pull_blob_stream( + &self, + repo: &str, + digest: &str, + ) -> Result>> { + validate_digest(digest)?; + let url = format!("{}/v2/{}/blobs/{}", self.base_url, repo, digest); + let resp = self + .send_replayable(self.request(reqwest::Method::GET, &url)) + .await?; + + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Err(RegistryError::BlobNotFound(digest.to_string())); + } + + if !resp.status().is_success() { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + + Ok(resp.bytes_stream()) + } + + /// Upload a manifest for the given reference (tag or digest). + pub async fn put_manifest(&self, repo: &str, reference: &str, manifest: &[u8]) -> Result<()> { + // Honor the document's own `mediaType` so this also PUTs an image index + // (the registry validates the Content-Type against the body's mediaType). + let media_type = serde_json::from_slice::(manifest) + .ok() + .and_then(|v| { + v.get("mediaType") + .and_then(|m| m.as_str()) + .map(String::from) + }) + .unwrap_or_else(|| MANIFEST_MEDIA_TYPE.to_string()); + let url = format!("{}/v2/{}/manifests/{}", self.base_url, repo, reference); + let resp = self + .send_replayable( + self.request(reqwest::Method::PUT, &url) + .header(CONTENT_TYPE, media_type) + .body(manifest.to_vec()), + ) + .await?; + + if !resp.status().is_success() { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + Ok(()) + } + + /// Fetch a manifest by reference (tag or digest). + pub async fn get_manifest(&self, repo: &str, reference: &str) -> Result> { + let url = format!("{}/v2/{}/manifests/{}", self.base_url, repo, reference); + let resp = self + .send_replayable( + self.request(reqwest::Method::GET, &url) + .header(ACCEPT, MANIFEST_MEDIA_TYPE), + ) + .await?; + + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Err(RegistryError::BlobNotFound(format!( + "{}:{}", + repo, reference + ))); + } + + if !resp.status().is_success() { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + + // Detect OCI image indexes and Docker manifest lists — smolvm artifacts are + // always single-manifest; an index means the caller referenced a multi-arch + // Docker image rather than a .smolmachine artifact. + let content_type = resp + .headers() + .get(CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + if content_type.contains("application/vnd.oci.image.index.v1+json") + || content_type.contains("application/vnd.docker.distribution.manifest.list.v2+json") + { + return Err(RegistryError::InvalidManifest( + "OCI image indexes (multi-arch manifests) are not supported; \ + this reference points to a Docker image, not a .smolmachine artifact" + .into(), + )); + } + + read_body_capped(resp, MAX_MANIFEST_BYTES, "manifest").await + } + + /// Fetch a manifest OR image index by reference, returning the raw bytes and + /// the document's media type. Unlike [`get_manifest`], this accepts (and does + /// not reject) an image index — the caller decides whether to resolve it to a + /// platform manifest. Used by `pull` for multi-arch fan-out. + pub async fn get_manifest_raw(&self, repo: &str, reference: &str) -> Result<(Vec, String)> { + let url = format!("{}/v2/{}/manifests/{}", self.base_url, repo, reference); + let resp = self + .send_replayable( + self.request(reqwest::Method::GET, &url) + // Advertise BOTH so the registry hands back an index as an + // index (not a server-side-selected manifest). + .header(ACCEPT, format!("{INDEX_MEDIA_TYPE}, {MANIFEST_MEDIA_TYPE}")), + ) + .await?; + + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Err(RegistryError::BlobNotFound(format!( + "{}:{}", + repo, reference + ))); + } + if !resp.status().is_success() { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + + let content_type = resp + .headers() + .get(CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + // Cap the buffered manifest/index BEFORE the digest check below. + let data = read_body_capped(resp, MAX_MANIFEST_BYTES, "manifest").await?; + + // When the reference is a content digest (e.g. an image-index entry being + // resolved to its platform manifest), the bytes MUST hash to it — else a + // malicious or MITM'd registry could swap manifest content while keeping + // the digest the caller trusted. Tags are mutable and carry no such + // guarantee, so only digest references are verifiable here. Mirrors the + // check `pull_blob` already performs for blobs. + if reference.starts_with("sha256:") { + let actual = format!("sha256:{}", hex::encode(Sha256::digest(&data))); + if actual != reference { + return Err(RegistryError::DigestMismatch { + expected: reference.to_string(), + actual, + }); + } + } + + Ok((data, content_type)) + } + + /// Fetch the manifest at `reference`, resolving an OCI image index to THIS + /// machine's host-platform entry (Docker-style fan-out), and return the + /// single-platform manifest bytes. Shared by `pull` and `inspect` so both + /// resolve a multi-arch tag identically (a plain manifest passes through). + pub async fn get_manifest_resolved(&self, repo: &str, reference: &str) -> Result> { + let (doc_bytes, content_type) = self.get_manifest_raw(repo, reference).await?; + + let is_index = content_type.contains(INDEX_MEDIA_TYPE) + || serde_json::from_slice::(&doc_bytes) + .ok() + .map(|v| { + v.get("manifests").is_some() + || v.get("mediaType").and_then(|m| m.as_str()) == Some(INDEX_MEDIA_TYPE) + }) + .unwrap_or(false); + if !is_index { + return Ok(doc_bytes); + } + + let index: OciIndex = serde_json::from_slice(&doc_bytes)?; + // .smolmachine sidecars are cross-platform — libkrun is provided by the + // installed CLI, not bundled — so only the GUEST architecture matters, and + // the guest is always Linux. An index entry must therefore be keyed + // `linux/`; match it strictly. A wrong/missing `os` is bad index data + // and must fail loudly (the error below lists what was published) rather than + // be silently tolerated. The host OS is irrelevant to which sidecar to pull. + let arch = OciPlatform::current().architecture; + let entry = index + .manifests + .iter() + .find(|m| { + m.platform + .as_ref() + .is_some_and(|p| p.os == "linux" && p.architecture == arch) + }) + .ok_or_else(|| { + let available: Vec = index + .manifests + .iter() + .filter_map(|m| m.platform.as_ref().map(|p| p.label())) + .collect(); + RegistryError::InvalidManifest(format!( + "no linux/{arch} build available for this machine; the registry has: {}", + if available.is_empty() { + "(none)".into() + } else { + available.join(", ") + } + )) + })?; + tracing::info!(arch = %arch, digest = %entry.digest, "selected index entry"); + Ok(self.get_manifest_raw(repo, &entry.digest).await?.0) + } + + /// List the repositories in this registry via the OCI catalog endpoint + /// (`GET /v2/_catalog`), following `Link: rel="next"` pagination. + /// + /// Note: `_catalog` is OPTIONAL in the distribution spec and is commonly + /// disabled or access-restricted (Docker Hub does not expose a registry-wide + /// catalog at all). Such registries answer `404`/`401`, surfaced here as an + /// [`RegistryError::ApiError`] the caller can translate into a hint that this + /// registry doesn't support catalog listing. + pub async fn list_repositories(&self) -> Result> { + #[derive(serde::Deserialize)] + struct Catalog { + #[serde(default)] + repositories: Vec, + } + + // First page is the plain endpoint; subsequent pages come from the + // registry-supplied `Link` header (already query-encoded), so we resolve + // each against the base and stop when no `rel="next"` is offered. + let mut next = Some(format!("{}/v2/_catalog", self.base_url)); + let mut repos = Vec::new(); + // Bound the walk so a misbehaving registry can't loop us forever. + for _ in 0..1000 { + let Some(url) = next.take() else { break }; + let resp = self + .send_replayable(self.request(reqwest::Method::GET, &url)) + .await?; + if !resp.status().is_success() { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + next = resp + .headers() + .get(LINK) + .and_then(|v| v.to_str().ok()) + .and_then(Self::parse_next_link) + .map(|rel| self.resolve_location(&rel)) + .transpose()?; + let page: Catalog = serde_json::from_slice(&resp.bytes().await?)?; + repos.extend(page.repositories); + } + Ok(repos) + } + + /// List the tags for a repository (`GET /v2//tags/list`). + /// + /// Unlike the catalog endpoint this is near-universally supported, since it + /// only requires `repository::pull` scope. + pub async fn list_tags(&self, repo: &str) -> Result> { + #[derive(serde::Deserialize)] + struct TagList { + #[serde(default)] + tags: Vec, + } + + let url = format!("{}/v2/{}/tags/list", self.base_url, repo); + let resp = self + .send_replayable(self.request(reqwest::Method::GET, &url)) + .await?; + + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Err(RegistryError::BlobNotFound(repo.to_string())); + } + if !resp.status().is_success() { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + + let list: TagList = serde_json::from_slice(&resp.bytes().await?)?; + Ok(list.tags) + } + + /// Extract the `rel="next"` URL reference from an RFC 5988 `Link` header. + /// + /// The distribution spec formats pagination as `; rel="next"`; we return + /// the bracketed URL (which may be relative and query-encoded) for the caller + /// to resolve against the registry base. + fn parse_next_link(header: &str) -> Option { + // A Link header may hold several comma-separated entries; pick the one + // whose params mark it rel="next". + for entry in header.split(',') { + let mut parts = entry.split(';'); + let Some(url_part) = parts.next() else { + continue; + }; + let is_next = parts.any(|p| { + let p = p.trim().to_ascii_lowercase(); + p == "rel=\"next\"" || p == "rel=next" + }); + if !is_next { + continue; + } + let url = url_part + .trim() + .trim_start_matches('<') + .trim_end_matches('>'); + if !url.is_empty() { + return Some(url.to_string()); + } + } + None + } + + /// Resolve a Location header value against the registry base URL. + /// + /// Relative paths (with or without a leading `/`) are resolved via + /// `Url::join`. Absolute `http(s)://` URLs are parsed and checked against + /// the base origin (scheme + host + port). + /// + /// Cross-origin upload locations are rejected. Registries that return signed + /// storage URLs (S3, GCS) would need an explicit policy allowance here. + fn resolve_location(&self, location: &str) -> Result { + let base = reqwest::Url::parse(&self.base_url).map_err(|e| RegistryError::ApiError { + status: 202, + body: format!("client base URL is not valid: {e}"), + })?; + + let resolved = if location.starts_with("http://") || location.starts_with("https://") { + reqwest::Url::parse(location).map_err(|e| RegistryError::ApiError { + status: 202, + body: format!("Location is not a valid URL '{location}': {e}"), + })? + } else { + // Relative path (with or without leading slash): join against base. + base.join(location).map_err(|e| RegistryError::ApiError { + status: 202, + body: format!("Location is not a valid relative path '{location}': {e}"), + })? + }; + + // Enforce same-origin (scheme + host + port). + if resolved.origin() != base.origin() { + return Err(RegistryError::ApiError { + status: 202, + body: format!( + "Location points to a different origin ('{}'), expected '{}'", + resolved.origin().unicode_serialization(), + base.origin().unicode_serialization(), + ), + }); + } + + Ok(resolved.to_string()) + } + + /// Extract the registry hostname from `base_url` (strips scheme). + /// + /// `"https://registry-1.docker.io"` → `"registry-1.docker.io"` + fn registry_host(&self) -> &str { + self.base_url.split("//").nth(1).unwrap_or(&self.base_url) + } + + /// Build a request with optional auth header. + fn request(&self, method: reqwest::Method, url: &str) -> reqwest::RequestBuilder { + let mut req = self.http.request(method, url); + if let Some(ref token) = self.auth_token { + req = req.bearer_auth(token); + } + req + } + + /// Send a cloneable (replayable) request through the full OCI auth protocol. + /// + /// Use this for requests whose body can be replayed (GET, HEAD, POST with + /// empty body, PUT with a buffered `Vec`). For streaming bodies that + /// cannot be cloned, use [`send_nonreplayable`] with a body factory instead. + /// + /// Protocol steps: + /// 1. Attach a preemptive bearer from the `last_challenge` cache if available + /// (optimization — skips the 401 round trip for warm-cache requests). + /// 2. Send; if not 401, return immediately. + /// 3. Parse `WWW-Authenticate`; exchange credentials for a token + /// (`get_token(false)` — returns cached token if still valid). + /// 4. First retry with that token (uses the pre-cloned builder). + /// 5. If still 401: force-evict the cache entry, fetch a genuinely fresh + /// token (`get_token(true)`), second retry. + /// 6. No third attempt — a freshly-fetched token that is also rejected means + /// the credentials themselves are wrong. + async fn send_replayable(&self, req: reqwest::RequestBuilder) -> Result { + // Clone BEFORE applying the preemptive bearer so that retry clones + // start clean. If we cloned after bearer_auth, every subsequent + // `.bearer_auth(new_token)` call would produce a doubled Authorization + // header ("Bearer stale, Bearer fresh"), which breaks header matching. + let first_retry = req.try_clone(); + let second_retry = req.try_clone(); + + // Step 1: attach a preemptive bearer to the initial send only. + // A valid cached token skips the 401 round trip — especially important + // for non-replayable streaming PUT bodies that cannot be replayed. + let initial_req = if let Some(token) = self.preemptive_token() { + req.bearer_auth(token) + } else { + req + }; + + // Step 2: send. + let resp = initial_req.send().await?; + if resp.status() != reqwest::StatusCode::UNAUTHORIZED { + return Ok(resp); + } + + // If we already have a static bearer, the registry is rejecting the token + // itself — no challenge exchange will help. + if self.auth_token.is_some() { + return Ok(resp); + } + + let Some(www_auth) = resp + .headers() + .get(WWW_AUTHENTICATE) + .and_then(|v| v.to_str().ok()) + else { + return Ok(resp); + }; + + let challenge = BearerChallenge::parse(www_auth)?; + + // Steps 3–4: exchange credentials → first retry. + let token = self.get_token(&challenge, false).await?; + let first_retry = first_retry.ok_or_else(|| RegistryError::Authentication { + message: "registry challenged a non-replayable request; \ + ensure credentials are warm before streaming uploads" + .into(), + })?; + let resp = first_retry.bearer_auth(token).send().await?; + if resp.status() != reqwest::StatusCode::UNAUTHORIZED { + return Ok(resp); + } + + // Step 5: force-evict and fetch a genuinely fresh token → second retry. + let token = self.get_token(&challenge, true).await?; + let second_retry = second_retry.ok_or_else(|| RegistryError::Authentication { + message: "registry challenged a non-replayable request on second attempt".into(), + })?; + Ok(second_retry.bearer_auth(token).send().await?) + } + + /// Send a non-replayable request (e.g., a streaming PUT body) with full auth retry support. + /// + /// Unlike [`send_replayable`], this method does not rely on `try_clone()` of a + /// pre-built body. Instead, the caller supplies a **factory** that produces a + /// fresh `reqwest::Body` on each call. The factory may be invoked up to 3 times. + /// + /// Protocol steps: + /// 1. Attach a preemptive bearer from `last_challenge` cache if available + /// (optimization for the common warm-cache case; may have wrong scope). + /// 2. Call factory for a fresh body; send. + /// 3. On 401: parse the challenge from *this specific request* (correct scope), + /// fetch/cache a token, call factory again, retry. + /// 4. On second 401: force-evict the cache entry, fresh token, factory again, final retry. + /// + /// Correctness is guaranteed by the challenge from step 3, not by the preemptive + /// hint from step 1. A wrong-scope preemptive token just costs one extra round trip. + async fn send_nonreplayable( + &self, + req: reqwest::RequestBuilder, + make_body: F, + ) -> Result + where + F: Fn() -> Result, + { + // Clone the body-less base builder for retries before attaching auth or body. + // try_clone() returns None only for builders that already hold a non-replayable + // streaming body — callers of send_nonreplayable must not pre-attach a body. + let clone_err = || RegistryError::Authentication { + message: "send_nonreplayable called with a non-cloneable request builder; \ + attach the body via the factory, not before calling this method" + .into(), + }; + let retry1 = req.try_clone().ok_or_else(clone_err)?; + let retry2 = req.try_clone().ok_or_else(clone_err)?; + + // Step 1: attach preemptive bearer (optimization — wrong scope → extra 401, not corruption). + let initial = if let Some(token) = self.preemptive_token() { + req.bearer_auth(token) + } else { + req + }; + + // Step 2: send with a fresh body from the factory. + let resp = initial.body(make_body()?).send().await?; + if resp.status() != reqwest::StatusCode::UNAUTHORIZED { + return Ok(resp); + } + + // Static auth_token rejected — challenge exchange won't help. + if self.auth_token.is_some() { + return Ok(resp); + } + + let Some(www_auth) = resp + .headers() + .get(WWW_AUTHENTICATE) + .and_then(|v| v.to_str().ok()) + else { + return Ok(resp); + }; + + let challenge = BearerChallenge::parse(www_auth)?; + + // Step 3: token for the scope this request actually needs, fresh body. + let token = self.get_token(&challenge, false).await?; + let resp = retry1.bearer_auth(token).body(make_body()?).send().await?; + if resp.status() != reqwest::StatusCode::UNAUTHORIZED { + return Ok(resp); + } + + // Step 4: force-evict stale cache entry, genuinely fresh token, final body. + let token = self.get_token(&challenge, true).await?; + Ok(retry2.bearer_auth(token).body(make_body()?).send().await?) + } + + /// Fetch (or return a cached) OCI bearer token for the given challenge. + /// + /// If `force_refresh` is true the cache entry for this challenge is evicted + /// first, guaranteeing a fresh token from the token service. This is used + /// on the second 401 retry, when a cached token has just proven invalid. + async fn get_token(&self, challenge: &BearerChallenge, force_refresh: bool) -> Result { + let key = TokenCacheKey { + realm: challenge.realm.clone(), + service: challenge.service.clone(), + scope: challenge.scope.clone(), + }; + + { + let mut cache = self + .token_cache + .lock() + .map_err(|_| RegistryError::Authentication { + message: "registry token cache lock poisoned".into(), + })?; + if force_refresh { + cache.remove(&key); + } else if let Some(cached) = cache.get(&key) { + if cached.is_valid() { + return Ok(cached.token.clone()); + } + // Stale entry — fall through to fetch a fresh token. + } + } + + let mut url = + reqwest::Url::parse(&challenge.realm).map_err(|e| RegistryError::Authentication { + message: format!("invalid token service realm: {e}"), + })?; + { + let mut pairs = url.query_pairs_mut(); + if let Some(service) = &challenge.service { + pairs.append_pair("service", service); + } + if let Some(scope) = &challenge.scope { + pairs.append_pair("scope", scope); + } + } + + let req = self.http.get(url.clone()); + let resp = if let Some(identity_token) = &self.identity_token { + // smolmachines path: upstream JWT exchanged with the token service. + req.bearer_auth(identity_token).send().await? + } else if let Some((username, password)) = &self.basic_credentials { + // Standard Docker/OCI path: Basic credentials sent to the registry's + // own auth service. + // + // Security checks before sending the PAT: + // 1. HTTPS-only: never send credentials to a plaintext endpoint. + // 2. Realm host allowlist for known registries: a compromised registry + // could serve a valid HTTPS WWW-Authenticate challenge pointing to an + // attacker-controlled token service. For known registries we verify the + // realm host matches the expected auth service. + if url.scheme() != "https" { + return Err(RegistryError::Authentication { + message: format!( + "refusing Basic credentials for non-HTTPS realm: {}", + url.as_str() + ), + }); + } + validate_realm_host(self.registry_host(), &url)?; + req.basic_auth(username, Some(password)).send().await? + } else { + // Anonymous token request — works for public repositories. + req.send().await? + }; + + if !resp.status().is_success() { + return Err(RegistryError::Authentication { + message: format!( + "token service returned {}: {}", + resp.status(), + resp.text().await.unwrap_or_default() + ), + }); + } + + let token_response: TokenResponse = resp.json().await?; + let token = token_response + .token + .or(token_response.access_token) + .ok_or_else(|| RegistryError::Authentication { + message: "token service response did not include token".into(), + })?; + + // Store the raw server-reported expiry. The 30-second safety buffer + // is applied only in `CachedToken::is_valid()`, not here, so that + // short-lived tokens (< 30 s) are immediately stale rather than + // producing expires_at = None (which means "valid forever"). + let expires_at = token_response + .expires_in + .map(|secs| Instant::now() + Duration::from_secs(secs)); + + self.token_cache + .lock() + .map_err(|_| RegistryError::Authentication { + message: "registry token cache lock poisoned".into(), + })? + .insert( + key, + CachedToken { + token: token.clone(), + expires_at, + }, + ); + + // Record the challenge so future requests can attach a preemptive bearer + // token without waiting for a 401 round trip (especially important for + // non-replayable streaming uploads). + if let Ok(mut lc) = self.last_challenge.lock() { + *lc = Some(challenge.clone()); + } + + Ok(token) + } + + /// Return a valid cached bearer token from the most recent challenge exchange, + /// or `None` if the cache is empty, the entry is stale, or `auth_token` is set + /// (in which case `request()` already attaches the static bearer). + /// + /// Called at the top of `send_replayable()` to skip the 401 round trip for + /// authenticated operations — particularly important for streaming PUT bodies + /// that cannot be retried after a 401. + fn preemptive_token(&self) -> Option { + // Static auth_token is always attached by request() — no preemptive needed. + if self.auth_token.is_some() { + return None; + } + let challenge = self.last_challenge.lock().ok()?.as_ref()?.clone(); + let cache = self.token_cache.lock().ok()?; + let key = TokenCacheKey { + realm: challenge.realm, + service: challenge.service, + scope: challenge.scope, + }; + let cached = cache.get(&key)?; + if cached.is_valid() { + Some(cached.token.clone()) + } else { + None + } + } +} + +/// Validate the realm host against an allowlist of known registries. +/// +/// For registries whose auth host is well-known, a compromised registry could +/// send a valid HTTPS `WWW-Authenticate` challenge pointing to an attacker's +/// token service. This check prevents PAT exfiltration in that scenario. +/// +/// Unknown registries are not checked — we cannot enumerate every self-hosted +/// registry's auth topology. The HTTPS-only guard in the caller handles those. +fn validate_realm_host(registry_host: &str, realm_url: &reqwest::Url) -> Result<()> { + let realm_host = realm_url.host_str().unwrap_or(""); + + // Map known registry API hosts to their expected auth service host. + let expected_auth_host: Option<&str> = match registry_host { + "registry-1.docker.io" => Some("auth.docker.io"), + "ghcr.io" => Some("ghcr.io"), + "quay.io" => Some("quay.io"), + // ECR: auth endpoint is on *.amazonaws.com — allow any amazonaws.com subdomain. + h if h.ends_with(".amazonaws.com") => { + if realm_host.ends_with(".amazonaws.com") { + return Ok(()); + } + return Err(RegistryError::Authentication { + message: format!( + "ECR realm host '{realm_host}' is not on amazonaws.com (registry: {registry_host})" + ), + }); + } + // GCR / Artifact Registry: auth is on oauth2.googleapis.com or the same host. + h if h == "gcr.io" || h.ends_with(".gcr.io") || h.ends_with(".pkg.dev") => { + if realm_host == "oauth2.googleapis.com" + || realm_host.ends_with(".gcr.io") + || realm_host.ends_with(".pkg.dev") + { + return Ok(()); + } + return Err(RegistryError::Authentication { + message: format!( + "GCR realm host '{realm_host}' is not on googleapis.com or gcr.io (registry: {registry_host})" + ), + }); + } + _ => None, + }; + + if let Some(expected) = expected_auth_host { + if realm_host != expected { + return Err(RegistryError::Authentication { + message: format!( + "realm host '{realm_host}' does not match expected auth host '{expected}' for registry '{registry_host}'" + ), + }); + } + } + + Ok(()) +} + +/// A cached OCI bearer token with optional expiry. +/// +/// `expires_at` is stored as the raw server-reported expiry (`now + expires_in`). +/// The 30-second safety buffer lives exclusively in `is_valid()`, so short-lived +/// tokens (< 30 s) are immediately considered stale rather than being cached +/// with `expires_at = None` (which would incorrectly mean "valid forever"). +#[derive(Debug, Clone)] +struct CachedToken { + token: String, + expires_at: Option, +} + +impl CachedToken { + /// Returns `true` if the token is still usable (at least 30 s remain). + fn is_valid(&self) -> bool { + match self.expires_at { + None => true, + Some(exp) => Instant::now() + Duration::from_secs(30) < exp, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct TokenCacheKey { + realm: String, + service: Option, + scope: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct BearerChallenge { + realm: String, + service: Option, + scope: Option, +} + +impl BearerChallenge { + fn parse(header: &str) -> Result { + let header = header.trim(); + let params = header + .strip_prefix("Bearer ") + .or_else(|| header.strip_prefix("bearer ")) + .ok_or_else(|| RegistryError::Authentication { + message: format!("unsupported authenticate challenge: {header}"), + })?; + + let mut values = HashMap::new(); + for part in split_auth_params(params) { + let Some((key, value)) = part.split_once('=') else { + continue; + }; + values.insert(key.trim().to_ascii_lowercase(), unquote(value.trim())); + } + + let realm = values + .remove("realm") + .ok_or_else(|| RegistryError::Authentication { + message: "bearer challenge missing realm".into(), + })?; + + Ok(Self { + realm, + service: values.remove("service"), + scope: values.remove("scope"), + }) + } +} + +fn split_auth_params(params: &str) -> Vec<&str> { + let mut parts = Vec::new(); + let mut start = 0; + let mut in_quote = false; + let mut escaped = false; + + for (idx, ch) in params.char_indices() { + if escaped { + escaped = false; + continue; + } + match ch { + '\\' if in_quote => escaped = true, + '"' => in_quote = !in_quote, + ',' if !in_quote => { + parts.push(params[start..idx].trim()); + start = idx + 1; + } + _ => {} + } + } + parts.push(params[start..].trim()); + parts +} + +fn unquote(value: &str) -> String { + let Some(value) = value.strip_prefix('"').and_then(|v| v.strip_suffix('"')) else { + return value.to_string(); + }; + + let mut out = String::with_capacity(value.len()); + let mut escaped = false; + for ch in value.chars() { + if escaped { + out.push(ch); + escaped = false; + } else if ch == '\\' { + escaped = true; + } else { + out.push(ch); + } + } + out +} + +#[derive(Debug, serde::Deserialize)] +struct TokenResponse { + token: Option, + access_token: Option, + /// Token lifetime in seconds, as returned by the token service. + /// Used to populate `CachedToken::expires_at` with a 30-second safety buffer. + expires_in: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_next_link_extracts_rel_next() { + let h = r#"; rel="next""#; + assert_eq!( + RegistryClient::parse_next_link(h).as_deref(), + Some("/v2/_catalog?n=100&last=repo99") + ); + } + + #[test] + fn parse_next_link_ignores_non_next_and_missing() { + assert_eq!( + RegistryClient::parse_next_link(r#"; rel="prev""#), + None + ); + assert_eq!(RegistryClient::parse_next_link(""), None); + // Multiple entries: only the rel="next" one is picked. + let h = r#"; rel="prev", ; rel="next""#; + assert_eq!( + RegistryClient::parse_next_link(h).as_deref(), + Some("/b?n=2") + ); + } + + #[test] + fn validate_digest_accepts_valid_sha256() { + let valid = "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + assert!(validate_digest(valid).is_ok()); + // Uppercase hex is also accepted + let upper = "sha256:ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789"; + assert!(validate_digest(upper).is_ok()); + } + + #[test] + fn validate_digest_rejects_invalid() { + // Wrong/missing prefix + assert!(validate_digest("").is_err()); + assert!(validate_digest("sha256:").is_err()); + assert!(validate_digest( + "sha512:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + ) + .is_err()); + assert!(validate_digest( + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + ) + .is_err()); + // Wrong length + assert!(validate_digest("sha256:abcdef").is_err()); + assert!(validate_digest( + "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234567890" + ) + .is_err()); + // Non-hex chars + assert!(validate_digest( + "sha256:gggggg0123456789abcdef0123456789abcdef0123456789abcdef0123456789" + ) + .is_err()); + } + + fn client(base: &str) -> RegistryClient { + RegistryClient::new(base.to_string()) + } + + #[test] + fn resolve_location_allows_relative_and_same_host() { + let c = client("https://registry.example.com"); + // Relative path + let r = c + .resolve_location("/v2/repo/blobs/uploads/abc?state=xyz") + .unwrap(); + assert_eq!( + r, + "https://registry.example.com/v2/repo/blobs/uploads/abc?state=xyz" + ); + // Absolute same host + let r = c + .resolve_location("https://registry.example.com/v2/uploads/abc") + .unwrap(); + assert_eq!(r, "https://registry.example.com/v2/uploads/abc"); + // Absolute same host with port + let c2 = client("http://localhost:5050"); + let r = c2 + .resolve_location("http://localhost:5050/v2/uploads/xyz") + .unwrap(); + assert_eq!(r, "http://localhost:5050/v2/uploads/xyz"); + } + + #[test] + fn resolve_location_blocks_different_host() { + let c = client("https://registry.example.com"); + // Different host + assert!(c + .resolve_location("https://evil.attacker.com/steal-data") + .is_err()); + // Same host but different port (different origin) + let c2 = client("http://localhost:5050"); + assert!(c2 + .resolve_location("http://localhost:9999/v2/uploads/xyz") + .is_err()); + } + + #[test] + fn client_auth_token() { + let c = RegistryClient::new("https://r.example.com".to_string()); + assert!(c.auth_token.is_none()); + let c = c.with_token("secret".to_string()); + assert_eq!(c.auth_token.as_deref(), Some("secret")); + } + + #[test] + fn client_identity_token_accessor() { + let c = RegistryClient::new("https://registry.smolmachines.com".to_string()); + assert!(c.identity_token().is_none()); + let c = c.with_identity_token("eyJhbG".to_string()); + assert_eq!(c.identity_token(), Some("eyJhbG")); + } + + #[test] + fn client_basic_credentials_accessor() { + let c = RegistryClient::new("https://registry-1.docker.io".to_string()); + assert!(c.basic_credentials().is_none()); + let c = c.with_basic_credentials("alice".to_string(), "ghp_secret".to_string()); + assert_eq!(c.basic_credentials(), Some(("alice", "ghp_secret"))); + } + + #[test] + fn client_credential_modes_are_independent() { + // identity_token and basic_credentials are separate fields; setting one + // does not affect the other. + let c = RegistryClient::new("https://r.example.com".to_string()) + .with_identity_token("jwt".to_string()); + assert_eq!(c.identity_token(), Some("jwt")); + assert!(c.basic_credentials().is_none()); + + let c = RegistryClient::new("https://r.example.com".to_string()) + .with_basic_credentials("user".to_string(), "pass".to_string()); + assert!(c.identity_token().is_none()); + assert_eq!(c.basic_credentials(), Some(("user", "pass"))); + } + + #[test] + fn bearer_challenge_parses_registry_params() { + let challenge = BearerChallenge::parse( + r#"Bearer realm="https://token.smolmachines.com/v2/auth",service="registry.smolmachines.com",scope="repository:binsquare/app:pull,push""#, + ) + .unwrap(); + + assert_eq!(challenge.realm, "https://token.smolmachines.com/v2/auth"); + assert_eq!( + challenge.service.as_deref(), + Some("registry.smolmachines.com") + ); + assert_eq!( + challenge.scope.as_deref(), + Some("repository:binsquare/app:pull,push") + ); + } + + #[test] + fn bearer_challenge_handles_quoted_commas() { + let parts = split_auth_params(r#"realm="https://t.example/auth",scope="a:b:c,d""#); + assert_eq!( + parts, + vec![r#"realm="https://t.example/auth""#, r#"scope="a:b:c,d""#] + ); + } + + // --------------------------------------------------------------------------- + // validate_realm_host + // --------------------------------------------------------------------------- + + fn url(s: &str) -> reqwest::Url { + reqwest::Url::parse(s).unwrap() + } + + #[test] + fn realm_host_docker_hub_allows_auth_endpoint() { + assert!( + validate_realm_host("registry-1.docker.io", &url("https://auth.docker.io/token")) + .is_ok() + ); + } + + #[test] + fn realm_host_docker_hub_rejects_attacker_endpoint() { + let err = validate_realm_host( + "registry-1.docker.io", + &url("https://evil.attacker.com/steal"), + ) + .unwrap_err(); + assert!(err.to_string().contains("auth.docker.io"), "{err}"); + } + + #[test] + fn realm_host_ghcr_allows_self() { + assert!(validate_realm_host("ghcr.io", &url("https://ghcr.io/token")).is_ok()); + } + + #[test] + fn realm_host_ghcr_rejects_other() { + assert!(validate_realm_host("ghcr.io", &url("https://malicious.io/token")).is_err()); + } + + #[test] + fn realm_host_ecr_allows_amazonaws_subdomain() { + assert!(validate_realm_host( + "123456789.dkr.ecr.us-east-1.amazonaws.com", + &url("https://123456789.dkr.ecr.us-east-1.amazonaws.com/token") + ) + .is_ok()); + } + + #[test] + fn realm_host_ecr_rejects_non_amazonaws() { + let err = validate_realm_host( + "123456789.dkr.ecr.us-east-1.amazonaws.com", + &url("https://evil.notamazonaws.com/token"), + ) + .unwrap_err(); + assert!(err.to_string().contains("amazonaws.com"), "{err}"); + } + + #[test] + fn realm_host_gcr_allows_oauth2_googleapis() { + assert!(validate_realm_host("gcr.io", &url("https://oauth2.googleapis.com/token")).is_ok()); + } + + #[test] + fn realm_host_gcr_allows_gcr_subdomain() { + assert!(validate_realm_host("us.gcr.io", &url("https://us.gcr.io/v2/token")).is_ok()); + } + + #[test] + fn realm_host_gcr_rejects_arbitrary() { + assert!(validate_realm_host("gcr.io", &url("https://evil.com/token")).is_err()); + } + + #[test] + fn realm_host_unknown_registry_allows_any_https_realm() { + // Self-hosted registries are not enumerated; HTTPS alone is sufficient. + assert!(validate_realm_host( + "my-registry.internal", + &url("https://auth.my-registry.internal/token") + ) + .is_ok()); + assert!(validate_realm_host( + "harbor.company.io", + &url("https://harbor.company.io/service/token") + ) + .is_ok()); + } + + // --------------------------------------------------------------------------- + // CachedToken expiry + // --------------------------------------------------------------------------- + + #[test] + fn cached_token_valid_without_expiry() { + let t = CachedToken { + token: "tok".into(), + expires_at: None, + }; + assert!(t.is_valid()); + } + + #[test] + fn cached_token_valid_with_future_expiry() { + let t = CachedToken { + token: "tok".into(), + expires_at: Some(Instant::now() + Duration::from_secs(300)), + }; + assert!(t.is_valid()); + } + + #[test] + fn cached_token_invalid_when_within_buffer() { + // expires_at is 10 s from now, but the buffer is 30 s → stale + let t = CachedToken { + token: "tok".into(), + expires_at: Some(Instant::now() + Duration::from_secs(10)), + }; + assert!(!t.is_valid()); + } + + #[test] + fn cached_token_invalid_when_past() { + let t = CachedToken { + token: "tok".into(), + // Already expired — Instant::now() - 1 s; use checked_sub to avoid panic + expires_at: Instant::now().checked_sub(Duration::from_secs(1)), + }; + assert!(!t.is_valid()); + } +} + +// --------------------------------------------------------------------------- +// Mock HTTP tests — prove the protocol flow, not just helpers +// --------------------------------------------------------------------------- +// +// These tests start a real local HTTP server (wiremock) and drive the +// RegistryClient through the OCI auth protocol. They verify: +// +// 1. 401 → WWW-Authenticate → token exchange → retry → success +// 2. Cached token reused on the second request (token endpoint called once) +// 3. Preemptive token attached on second request (no 401 round trip) +// 4. Stale-cached token evicted and re-fetched on repeated 401 +// +// Identity-token mode is used (no HTTPS restriction on the realm URL) +// so the tests work against a plain HTTP wiremock server. + +#[cfg(test)] +mod http_tests { + use super::*; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + const DIGEST: &str = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; + + fn token_body(tok: &str, expires_in: u64) -> serde_json::Value { + serde_json::json!({ "token": tok, "expires_in": expires_in }) + } + + /// Set up HEAD mocks using wiremock priority ordering. + /// + /// Wiremock tries mocks in FIFO order (first mounted = first tried). + /// Mount the `times_401`-shot 401 first (highest priority), then the 404 + /// fallback. The first N HEAD requests get 401; once those shots are + /// exhausted wiremock falls through to the 404 mock. + async fn mount_head_sequence(server: &MockServer, times_401: u64) { + let challenge = format!( + "Bearer realm=\"{}/token\",service=\"test\",scope=\"repository:myrepo:pull\"", + server.uri() + ); + // High-priority N-shot 401 (mounted first = tried first). + Mock::given(method("HEAD")) + .and(path(format!("/v2/myrepo/blobs/{DIGEST}"))) + .respond_with( + ResponseTemplate::new(401).insert_header("WWW-Authenticate", challenge.as_str()), + ) + .up_to_n_times(times_401) + .mount(server) + .await; + // Low-priority fallback: 404 once the 401 shots are exhausted. + Mock::given(method("HEAD")) + .and(path(format!("/v2/myrepo/blobs/{DIGEST}"))) + .respond_with(ResponseTemplate::new(404)) + .mount(server) + .await; + } + + // ----------------------------------------------------------------------- + // Test 1: 401 → token exchange → retry → success. + // Token endpoint called exactly once. + // ----------------------------------------------------------------------- + #[tokio::test] + async fn test_auth_challenge_and_retry() { + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(200).set_body_json(token_body("tok", 300))) + .expect(1) + .mount(&server) + .await; + // first HEAD → 401, second HEAD (retry with bearer) → 404. + mount_head_sequence(&server, 1).await; + + let client = + RegistryClient::new(server.uri().to_string()).with_identity_token("jwt".to_string()); + + let exists = client.blob_exists("myrepo", DIGEST).await.unwrap(); + assert!(!exists); + // MockServer drop asserts expect(1) was satisfied. + } + + // ----------------------------------------------------------------------- + // Test 2: cached token reused across two calls. + // Token endpoint called exactly once total. + // ----------------------------------------------------------------------- + #[tokio::test] + async fn test_cached_token_not_refetched() { + let server = MockServer::start().await; + + Mock::given(method("GET")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(200).set_body_json(token_body("tok", 300))) + .expect(1) + .mount(&server) + .await; + // Only the very first HEAD returns 401; all others return 404. + mount_head_sequence(&server, 1).await; + + let client = + RegistryClient::new(server.uri().to_string()).with_identity_token("jwt".to_string()); + + // Call 1: auth dance → token cached, last_challenge recorded. + client.blob_exists("myrepo", DIGEST).await.unwrap(); + // Call 2: preemptive_token() finds the cached token → initial send returns 404 + // immediately, no auth dance, token endpoint NOT called again. + client.blob_exists("myrepo", DIGEST).await.unwrap(); + // Drop asserts expect(1). + } + + // ----------------------------------------------------------------------- + // Test 3: stale cached token → 401 on retry → force-refresh → success. + // ----------------------------------------------------------------------- + #[tokio::test] + async fn test_stale_token_force_refreshed() { + let server = MockServer::start().await; + + // Priority: first mounted = first tried (wiremock FIFO). + // stale-tok (first) → served on initial fetch; fresh-tok (second) → served on force-refresh. + Mock::given(method("GET")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(200).set_body_json(token_body("stale-tok", 0))) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(200).set_body_json(token_body("fresh-tok", 300))) + .up_to_n_times(1) + .mount(&server) + .await; + + // First 2 HEADs → 401 (initial + first_retry with stale token). + // Third HEAD (second_retry with fresh token) → 404. + mount_head_sequence(&server, 2).await; + + let client = + RegistryClient::new(server.uri().to_string()).with_identity_token("jwt".to_string()); + + // initial → 401 → stale-tok (expires_in=0, immediately stale) + // retry#1 → 401 → get_token(true) force-evict → fresh-tok + // retry#2 → 404 → return Ok + let exists = client.blob_exists("myrepo", DIGEST).await.unwrap(); + assert!(!exists); + } + + // ----------------------------------------------------------------------- + // Test 4: streaming PUT 401 → body factory produces fresh body for retry. + // + // Covers the critical upload path: a non-replayable PUT body that triggers + // a 401 challenge. The factory must be called twice (initial + one retry). + // ----------------------------------------------------------------------- + #[tokio::test] + async fn test_streaming_put_retries_with_fresh_body() { + let server = MockServer::start().await; + + let challenge = format!( + "Bearer realm=\"{}/token\",service=\"test\",scope=\"repository:myrepo:push,pull\"", + server.uri() + ); + + // Token endpoint — must be called exactly once (first retry after 401). + Mock::given(method("GET")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(200).set_body_json(token_body("push-tok", 300))) + .expect(1) + .mount(&server) + .await; + + // blob_exists HEAD → 404 (blob absent; proceed to upload). + Mock::given(method("HEAD")) + .and(path(format!("/v2/myrepo/blobs/{DIGEST}"))) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + + // POST to initiate upload → 202 with upload Location. + Mock::given(method("POST")) + .and(path("/v2/myrepo/blobs/uploads/")) + .respond_with( + ResponseTemplate::new(202) + .insert_header("Location", "/v2/myrepo/blobs/uploads/sess-1"), + ) + .mount(&server) + .await; + + // PUT: first attempt → 401 (unauthenticated); second attempt → 201 Created. + // FIFO order: 401 mock first (higher priority, exhausts after 1 hit), 201 fallback second. + Mock::given(method("PUT")) + .and(path("/v2/myrepo/blobs/uploads/sess-1")) + .respond_with( + ResponseTemplate::new(401).insert_header("WWW-Authenticate", challenge.as_str()), + ) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("PUT")) + .and(path("/v2/myrepo/blobs/uploads/sess-1")) + .respond_with(ResponseTemplate::new(201)) + .mount(&server) + .await; + + let client = + RegistryClient::new(server.uri().to_string()).with_identity_token("jwt".to_string()); + + let body_data: &[u8] = b"fake blob content for upload"; + let call_count = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)); + let call_count_clone = call_count.clone(); + + client + .push_blob_stream("myrepo", DIGEST, body_data.len() as u64, move || { + call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(reqwest::Body::from(body_data)) + }) + .await + .unwrap(); + + // Factory called twice: once for the unauthenticated 401 attempt, + // once for the authenticated retry that succeeds. + assert_eq!(call_count.load(std::sync::atomic::Ordering::SeqCst), 2); + // MockServer drop asserts token endpoint expect(1) was satisfied. + } + + // ----------------------------------------------------------------------- + // Test 5: POST warms the token; PUT uses preemptive bearer — no 401 on PUT. + // + // Verifies that after send_replayable does a challenge dance for the POST, + // last_challenge is set, and send_nonreplayable attaches the token + // preemptively so the PUT body is never sent unauthenticated. + // ----------------------------------------------------------------------- + #[tokio::test] + async fn test_streaming_put_uses_preemptive_token_from_post() { + let server = MockServer::start().await; + + let challenge = format!( + "Bearer realm=\"{}/token\",service=\"test\",scope=\"repository:myrepo:push,pull\"", + server.uri() + ); + + // Token endpoint — called once during the POST auth dance. + Mock::given(method("GET")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(200).set_body_json(token_body("shared-tok", 300))) + .expect(1) + .mount(&server) + .await; + + // blob_exists HEAD → 404. + Mock::given(method("HEAD")) + .and(path(format!("/v2/myrepo/blobs/{DIGEST}"))) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + + // POST → 401 first (triggers auth dance), then 202 with Location. + // FIFO order: 401 first (exhausts after 1 hit), 202 fallback second. + Mock::given(method("POST")) + .and(path("/v2/myrepo/blobs/uploads/")) + .respond_with( + ResponseTemplate::new(401).insert_header("WWW-Authenticate", challenge.as_str()), + ) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v2/myrepo/blobs/uploads/")) + .respond_with( + ResponseTemplate::new(202) + .insert_header("Location", "/v2/myrepo/blobs/uploads/sess-2"), + ) + .mount(&server) + .await; + + // PUT → 201 directly (preemptive token from POST dance). + Mock::given(method("PUT")) + .and(path("/v2/myrepo/blobs/uploads/sess-2")) + .respond_with(ResponseTemplate::new(201)) + .mount(&server) + .await; + + let client = + RegistryClient::new(server.uri().to_string()).with_identity_token("jwt".to_string()); + + let body_data: &[u8] = b"fake blob content"; + let call_count = std::sync::Arc::new(std::sync::atomic::AtomicU32::new(0)); + let call_count_clone = call_count.clone(); + + client + .push_blob_stream("myrepo", DIGEST, body_data.len() as u64, move || { + call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(reqwest::Body::from(body_data)) + }) + .await + .unwrap(); + + // Factory called exactly once — PUT succeeded on first attempt with preemptive token. + assert_eq!(call_count.load(std::sync::atomic::Ordering::SeqCst), 1); + // MockServer drop asserts token endpoint expect(1) was satisfied. + } + + // ----------------------------------------------------------------------- + // Chunked upload: POST → one PATCH per chunk (Content-Range) → PUT finalize. + // 10 bytes with a 3-byte chunk_size = 4 PATCHes ([0-2],[3-5],[6-8],[9-9]), + // then a single PUT carrying the digest. Proves the engine never sends the + // blob as one oversized body (the Cloudflare-413 fix). + // ----------------------------------------------------------------------- + #[tokio::test] + async fn test_chunked_upload_patches_then_finalizes() { + let server = MockServer::start().await; + let content: &[u8] = b"abcdefghij"; // 10 bytes + let digest = format!("sha256:{}", hex::encode(Sha256::digest(content))); + + Mock::given(method("HEAD")) + .and(path(format!("/v2/myrepo/blobs/{digest}"))) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/v2/myrepo/blobs/uploads/")) + .respond_with( + ResponseTemplate::new(202) + .insert_header("Location", "/v2/myrepo/blobs/uploads/sess-1"), + ) + .mount(&server) + .await; + // Each PATCH is accepted and hands back the next upload URL (same one here). + Mock::given(method("PATCH")) + .and(path("/v2/myrepo/blobs/uploads/sess-1")) + .respond_with( + ResponseTemplate::new(202) + .insert_header("Location", "/v2/myrepo/blobs/uploads/sess-1"), + ) + .expect(4) + .mount(&server) + .await; + Mock::given(method("PUT")) + .and(path("/v2/myrepo/blobs/uploads/sess-1")) + .respond_with(ResponseTemplate::new(201)) + .expect(1) + .mount(&server) + .await; + + let dir = std::env::temp_dir().join(format!("smolvm-chunk-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let blob_path = dir.join("blob.bin"); + std::fs::write(&blob_path, content).unwrap(); + + let client = + RegistryClient::new(server.uri().to_string()).with_identity_token("jwt".to_string()); + client + .push_blob_chunked("myrepo", &digest, &blob_path, 3) + .await + .unwrap(); + + std::fs::remove_dir_all(&dir).ok(); + // MockServer drop asserts PATCH expect(4) and PUT expect(1) were satisfied. + } + + // ----------------------------------------------------------------------- + // A digest-addressed manifest whose bytes don't hash to the requested + // digest must be rejected (registry tamper / MITM on index→manifest + // resolution), mirroring the blob-digest check `pull_blob` performs. + // ----------------------------------------------------------------------- + #[tokio::test] + async fn test_manifest_digest_mismatch_rejected() { + let server = MockServer::start().await; + + let honest = br#"{"schemaVersion":2}"#.to_vec(); + let digest = format!("sha256:{}", hex::encode(Sha256::digest(&honest))); + + // Registry serves DIFFERENT bytes under the digest-addressed URL. + let tampered = br#"{"schemaVersion":2,"x":"evil"}"#.to_vec(); + Mock::given(method("GET")) + .and(path(format!("/v2/myrepo/manifests/{digest}"))) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", MANIFEST_MEDIA_TYPE) + .set_body_bytes(tampered), + ) + .mount(&server) + .await; + + let client = RegistryClient::new(server.uri().to_string()); + let err = client + .get_manifest_raw("myrepo", &digest) + .await + .unwrap_err(); + assert!( + matches!(err, RegistryError::DigestMismatch { .. }), + "tampered digest-addressed manifest must be rejected, got {err:?}" + ); + } + + // ----------------------------------------------------------------------- + // The honest case still passes through: bytes that hash to the requested + // digest are returned unchanged. + // ----------------------------------------------------------------------- + #[tokio::test] + async fn test_manifest_digest_match_accepted() { + let server = MockServer::start().await; + + let body = br#"{"schemaVersion":2}"#.to_vec(); + let digest = format!("sha256:{}", hex::encode(Sha256::digest(&body))); + Mock::given(method("GET")) + .and(path(format!("/v2/myrepo/manifests/{digest}"))) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", MANIFEST_MEDIA_TYPE) + .set_body_bytes(body.clone()), + ) + .mount(&server) + .await; + + let client = RegistryClient::new(server.uri().to_string()); + let (bytes, _ct) = client.get_manifest_raw("myrepo", &digest).await.unwrap(); + assert_eq!(bytes, body); + } + + // ----------------------------------------------------------------------- + // Fix 2 (end-to-end): `get_manifest` must reject a body exceeding + // MAX_MANIFEST_BYTES via the up-front Content-Length check, rather than + // buffering the whole (here, > 32 MiB) body into host RAM before hashing. + // The body is honest (matches Content-Length) so the transport accepts it; + // our guard rejects it before reading past the header. + // ----------------------------------------------------------------------- + #[tokio::test] + async fn test_manifest_oversized_rejected() { + let server = MockServer::start().await; + + let oversized = vec![0u8; MAX_MANIFEST_BYTES + 1]; + Mock::given(method("GET")) + .and(path("/v2/myrepo/manifests/latest")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("content-type", MANIFEST_MEDIA_TYPE) + .set_body_bytes(oversized), + ) + .mount(&server) + .await; + + let client = RegistryClient::new(server.uri().to_string()); + let err = client.get_manifest("myrepo", "latest").await.unwrap_err(); + assert!( + matches!(err, RegistryError::ResponseTooLarge { .. }), + "oversized manifest must be rejected, got {err:?}" + ); + } + + // ----------------------------------------------------------------------- + // Fix 2: even with NO / understated Content-Length, the streamed body is + // capped — the accumulating read errors once it crosses the limit rather + // than buffering an unbounded body. Verified against `read_body_capped` + // directly with a tiny cap. + // ----------------------------------------------------------------------- + #[tokio::test] + async fn test_read_body_capped_streams_and_rejects_over_limit() { + let server = MockServer::start().await; + + // 4 KiB body, no explicit Content-Length reliance — cap at 1 KiB. + Mock::given(method("GET")) + .and(path("/big")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(vec![0u8; 4096])) + .mount(&server) + .await; + + let resp = reqwest::get(format!("{}/big", server.uri())).await.unwrap(); + let err = read_body_capped(resp, 1024, "test").await.unwrap_err(); + assert!( + matches!(err, RegistryError::ResponseTooLarge { .. }), + "over-cap streamed body must be rejected, got {err:?}" + ); + + // A body within the cap is returned intact. + let resp2 = reqwest::get(format!("{}/big", server.uri())).await.unwrap(); + let ok = read_body_capped(resp2, 8192, "test").await.unwrap(); + assert_eq!(ok.len(), 4096); + } +} diff --git a/crates/smolvm-registry/src/lib.rs b/crates/smolvm-registry/src/lib.rs new file mode 100644 index 0000000..3486692 --- /dev/null +++ b/crates/smolvm-registry/src/lib.rs @@ -0,0 +1,308 @@ +//! OCI Distribution client for the smolmachines registry. +//! +//! Implements push and pull of `.smolmachine` artifacts as single-blob +//! OCI artifacts, compatible with any OCI Distribution Spec registry. + +pub mod cache; +pub mod client; +pub mod peer; +pub mod pull; +pub mod push; + +pub use cache::BlobCache; +pub use client::{validate_digest, RegistryClient}; +pub use pull::{pull, PullResult}; +pub use push::{push, PushResult}; + +use serde::{Deserialize, Serialize}; + +/// OCI media type for the smolmachine config blob. +pub const CONFIG_MEDIA_TYPE: &str = "application/vnd.smolmachines.machine.config.v1+json"; + +/// OCI media type for the smolmachine sidecar layer blob. +pub const LAYER_MEDIA_TYPE: &str = "application/vnd.smolmachines.smolmachine.v1"; + +/// OCI manifest media type. +pub const MANIFEST_MEDIA_TYPE: &str = "application/vnd.oci.image.manifest.v1+json"; + +/// OCI Image Manifest (minimal, single-layer). +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OciManifest { + pub schema_version: u32, + pub media_type: String, + pub config: OciDescriptor, + pub layers: Vec, +} + +/// OCI content descriptor. +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OciDescriptor { + pub media_type: String, + pub digest: String, + pub size: u64, +} + +/// OCI image index media type — a multi-platform "fan-out" manifest that points +/// at one single-platform manifest per (os, arch). A single tag (e.g. +/// `alpine:latest`) can carry an index so `pull` auto-selects the caller's +/// platform, exactly like a Docker manifest list. +pub const INDEX_MEDIA_TYPE: &str = "application/vnd.oci.image.index.v1+json"; + +/// The platform a manifest targets, as it appears in an index entry. For +/// smolmachines this is the **host** platform the artifact runs on (it bundles +/// host-specific libkrun), e.g. `os=linux, architecture=amd64`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OciPlatform { + pub os: String, + pub architecture: String, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub variant: Option, +} + +impl OciPlatform { + /// The platform of the process running right now, in OCI form (`darwin`/ + /// `linux`, `arm64`/`amd64`) — matches `PackManifest.host_platform`. + pub fn current() -> Self { + let os = match std::env::consts::OS { + "macos" => "darwin", + other => other, + }; + let architecture = match std::env::consts::ARCH { + "aarch64" => "arm64", + "x86_64" => "amd64", + other => other, + }; + Self { + os: os.to_string(), + architecture: architecture.to_string(), + variant: None, + } + } + + /// Parse a `host_platform` string (`"darwin/arm64"`, `"linux/x86_64"`) into a + /// platform descriptor, normalizing the arch (`x86_64`→`amd64`, + /// `aarch64`→`arm64`) so index entries and `current()` always agree. + pub fn parse(host_platform: &str) -> Self { + let (os, arch) = host_platform + .split_once('/') + .unwrap_or(("linux", host_platform)); + let architecture = match arch { + "x86_64" => "amd64", + "aarch64" => "arm64", + other => other, + }; + Self { + os: os.to_string(), + architecture: architecture.to_string(), + variant: None, + } + } + + /// Human label, e.g. `linux/amd64`. + pub fn label(&self) -> String { + format!("{}/{}", self.os, self.architecture) + } +} + +/// One entry in an [`OciIndex`]: a manifest descriptor plus the platform it +/// targets. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OciIndexManifest { + pub media_type: String, + pub digest: String, + pub size: u64, + #[serde(skip_serializing_if = "Option::is_none", default)] + pub platform: Option, +} + +/// OCI image index — references one manifest per platform so a single tag fans +/// out by architecture. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OciIndex { + pub schema_version: u32, + pub media_type: String, + pub manifests: Vec, +} + +/// Error type for registry operations. +#[derive(Debug, thiserror::Error)] +pub enum RegistryError { + #[error("HTTP request failed: {0}")] + Http(#[from] reqwest::Error), + + #[error("registry returned {status}: {body}")] + ApiError { status: u16, body: String }, + + #[error("registry authentication failed: {message}")] + Authentication { message: String }, + + #[error("digest mismatch: expected {expected}, got {actual}")] + DigestMismatch { expected: String, actual: String }, + + #[error("invalid manifest: {0}")] + InvalidManifest(String), + + #[error("registry response too large: {what} exceeds {limit} byte cap")] + ResponseTooLarge { what: String, limit: usize }, + + #[error("blob not found: {0}")] + BlobNotFound(String), + + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + + #[error("pack format error: {0}")] + Pack(#[from] smolvm_pack::PackError), +} + +pub type Result = std::result::Result; + +/// Return `true` if `host` refers to a loopback or any-address that should be +/// reached over plain HTTP rather than HTTPS. +/// +/// Handles all common local registry forms: +/// - `localhost`, `localhost:PORT` +/// - `127.x.x.x`, `127.x.x.x:PORT` (entire 127/8 block is loopback) +/// - `::1`, `[::1]`, `[::1]:PORT` (IPv6 loopback) +/// - `0.0.0.0`, `0.0.0.0:PORT` (bind-all, common in dev) +pub fn is_local_registry(host: &str) -> bool { + // Bare IPv6 (e.g. "::1") must be checked before splitting on ':' because + // "::1".split(':').next() yields "" (the empty token before the first colon). + if host.contains("::") && !host.starts_with('[') { + return host == "::1"; + } + + // Strip port and brackets from bracketed IPv6: [::1]:5000 → ::1 + let bare = if host.starts_with('[') { + host.trim_start_matches('[') + .split(']') + .next() + .unwrap_or(host) + } else { + // hostname or IPv4, split off port + host.split(':').next().unwrap_or(host) + }; + + bare == "localhost" || bare == "::1" || bare == "0.0.0.0" || is_loopback_ipv4(bare) +} + +/// Return true if `s` is a dotted-decimal IPv4 address in the loopback block (127.0.0.0/8). +/// +/// Rejects hostnames that merely start with "127." (e.g. "127.example.com"). +fn is_loopback_ipv4(s: &str) -> bool { + let parts: Vec<&str> = s.split('.').collect(); + if parts.len() != 4 { + return false; + } + parts[0] == "127" && parts[1..].iter().all(|p| p.parse::().is_ok()) +} + +#[cfg(test)] +mod platform_tests { + use super::*; + + #[test] + fn parse_normalizes_arch_and_os() { + // host_platform strings (as written into PackManifest) → OCI platform. + let p = OciPlatform::parse("darwin/arm64"); + assert_eq!( + (p.os.as_str(), p.architecture.as_str()), + ("darwin", "arm64") + ); + // x86_64/aarch64 normalize to amd64/arm64 so index entries and current() + // always compare equal. + assert_eq!(OciPlatform::parse("linux/x86_64").architecture, "amd64"); + assert_eq!(OciPlatform::parse("linux/aarch64").architecture, "arm64"); + assert_eq!(OciPlatform::parse("linux/amd64").label(), "linux/amd64"); + // Missing slash → defaults os to linux. + assert_eq!(OciPlatform::parse("amd64").os, "linux"); + } + + #[test] + fn current_matches_parse_of_its_own_label() { + // The selection in pull compares an index entry to current(); a manifest + // pushed FROM this machine must therefore round-trip equal. + let cur = OciPlatform::current(); + assert_eq!(OciPlatform::parse(&cur.label()), cur); + assert!(matches!(cur.architecture.as_str(), "arm64" | "amd64")); + assert!(matches!(cur.os.as_str(), "darwin" | "linux" | "windows")); + } + + #[test] + fn index_round_trips_and_entry_selection_works() { + let index = OciIndex { + schema_version: 2, + media_type: INDEX_MEDIA_TYPE.to_string(), + manifests: vec![ + OciIndexManifest { + media_type: MANIFEST_MEDIA_TYPE.to_string(), + digest: "sha256:aaa".into(), + size: 10, + platform: Some(OciPlatform::parse("linux/arm64")), + }, + OciIndexManifest { + media_type: MANIFEST_MEDIA_TYPE.to_string(), + digest: "sha256:bbb".into(), + size: 20, + platform: Some(OciPlatform::parse("linux/amd64")), + }, + ], + }; + // serde round-trip (camelCase mediaType/schemaVersion). + let json = serde_json::to_vec(&index).unwrap(); + assert!(String::from_utf8_lossy(&json).contains("\"mediaType\"")); + let back: OciIndex = serde_json::from_slice(&json).unwrap(); + assert_eq!(back.manifests.len(), 2); + // pull's selection: find the entry whose platform == the wanted one. + let want = OciPlatform::parse("linux/amd64"); + let hit = back + .manifests + .iter() + .find(|m| m.platform.as_ref() == Some(&want)) + .unwrap(); + assert_eq!(hit.digest, "sha256:bbb"); + // a platform not present → no match (pull would 404 with the available list). + let miss = OciPlatform::parse("windows/amd64"); + assert!(back + .manifests + .iter() + .all(|m| m.platform.as_ref() != Some(&miss))); + } +} + +#[cfg(test)] +mod is_local_tests { + use super::*; + + #[test] + fn local_variants() { + assert!(is_local_registry("localhost")); + assert!(is_local_registry("localhost:5000")); + assert!(is_local_registry("127.0.0.1")); + assert!(is_local_registry("127.0.0.1:5000")); + assert!(is_local_registry("127.1.2.3:9000")); + assert!(is_local_registry("::1")); + assert!(is_local_registry("[::1]")); + assert!(is_local_registry("[::1]:5000")); + assert!(is_local_registry("0.0.0.0:5000")); + } + + #[test] + fn remote_variants() { + assert!(!is_local_registry("registry.smolmachines.com")); + assert!(!is_local_registry("ghcr.io")); + assert!(!is_local_registry("docker.io")); + assert!(!is_local_registry("registry-1.docker.io")); + assert!(!is_local_registry("192.168.1.10:5000")); + // hostname that starts with "127." but is not an IPv4 address + assert!(!is_local_registry("127.example.com")); + assert!(!is_local_registry("127.example.com:5000")); + } +} diff --git a/crates/smolvm-registry/src/peer.rs b/crates/smolvm-registry/src/peer.rs new file mode 100644 index 0000000..0d0256f --- /dev/null +++ b/crates/smolvm-registry/src/peer.rs @@ -0,0 +1,356 @@ +//! Brokered peer-to-peer layer-blob fetch. +//! +//! On a cache miss, before falling back to the registry, [`pull`](crate::pull) +//! can fetch a layer blob from a sibling fleet node's `GET /p2p/blob/` +//! endpoint over node→node mTLS. The candidate peers are supplied by the control +//! plane (in the machine-create request body); when none are supplied, none of +//! this code runs and the pull path is byte-for-byte the registry path. +//! +//! ## Trust model / security +//! +//! The serve-side `GET /p2p/blob/` is reachable by ANY client whose cert +//! chains to the fleet node-CA, so on a MULTI-TENANT fleet it is a cross-tenant +//! read oracle for private blobs: a node that knows (or guesses) another +//! tenant's private layer digest can read that layer. This is bounded by mTLS +//! (only fleet nodes reach it) and by sha256 pre-image resistance (you must +//! already possess the exact digest), but it is NOT tenant-scoped. +//! +//! A broker-minted, request-scoped token binding the requester to the specific +//! reference's layer digests (verified at the app layer on top of mTLS) is +//! REQUIRED before enabling P2P on a multi-tenant fleet that hosts PRIVATE +//! artifacts. As shipped, P2P is safe on single-tenant or public-artifact fleets +//! and is off by default (peers only arrive when the control plane sends them). + +use crate::cache::BlobCache; +use crate::pull::{stream_verify_adopt, PullResult}; +use crate::{RegistryError, Result}; +use std::path::Path; +use std::sync::OnceLock; +use std::time::Duration; + +/// The node's own mTLS material, REUSED from the serve listener (`serve_tls.rs`): +/// a node presents its server cert/key as the *client* identity to a peer, and +/// trusts peer server certs chained to the same node-CA. +const ENV_CERT: &str = "SMOLVM_SERVE_TLS_CERT"; +const ENV_KEY: &str = "SMOLVM_SERVE_TLS_KEY"; +const ENV_CLIENT_CA: &str = "SMOLVM_SERVE_TLS_CLIENT_CA"; + +/// Bound on how long a single dead/unreachable peer may stall a launch before we +/// give up on it and move to the next peer (and ultimately the registry). +const PEER_CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +/// Idle-read bound: abort a peer that accepts the connection but then stalls +/// mid-body. This is per-read, not a whole-response deadline, so a large but +/// steady blob transfer is never cut off. +const PEER_READ_TIMEOUT: Duration = Duration::from_secs(30); + +/// Process-wide node→node mTLS client, built once from the serve-TLS env vars. +/// +/// Returns `None` (P2P disabled, registry-only) if the mTLS material is absent or +/// unparseable — a node with no serve identity simply never does P2P. +pub(crate) fn peer_client() -> Option<&'static reqwest::Client> { + static CLIENT: OnceLock> = OnceLock::new(); + CLIENT + .get_or_init(|| match build_peer_client() { + Ok(client) => Some(client), + Err(e) => { + tracing::warn!( + error = %e, + "P2P blob fetch disabled: could not build node->node mTLS client" + ); + None + } + }) + .as_ref() +} + +/// Build the node→node mTLS client: our node cert/key as the client identity, +/// trusting only the fleet node-CA (built-in roots disabled — peers are private +/// fleet nodes, never public CAs). +fn build_peer_client() -> std::result::Result { + let cert = read_env_file(ENV_CERT)?; + let key = read_env_file(ENV_KEY)?; + let ca = read_env_file(ENV_CLIENT_CA)?; + + // reqwest's rustls identity wants the cert chain and private key in a single + // PEM buffer. + let mut identity_pem = cert; + identity_pem.push(b'\n'); + identity_pem.extend_from_slice(&key); + let identity = reqwest::Identity::from_pem(&identity_pem) + .map_err(|e| format!("build client identity from {ENV_CERT}/{ENV_KEY}: {e}"))?; + + let ca_cert = reqwest::Certificate::from_pem(&ca) + .map_err(|e| format!("parse node CA from {ENV_CLIENT_CA}: {e}"))?; + + reqwest::Client::builder() + .use_rustls_tls() + // Peers are private fleet nodes signed by our node-CA; do not trust the + // public web PKI for node→node traffic. + .tls_built_in_root_certs(false) + .add_root_certificate(ca_cert) + .identity(identity) + .connect_timeout(PEER_CONNECT_TIMEOUT) + .read_timeout(PEER_READ_TIMEOUT) + .build() + .map_err(|e| format!("build node->node mTLS client: {e}")) +} + +/// Read the file named by env var `name`. Errs (rather than panics) so a missing +/// or unreadable cert just disables P2P instead of taking the process down. +fn read_env_file(name: &str) -> std::result::Result, String> { + let path = std::env::var_os(name) + .filter(|v| !v.is_empty()) + .ok_or_else(|| format!("{name} is unset"))?; + std::fs::read(&path).map_err(|e| format!("read {name} ({}): {e}", Path::new(&path).display())) +} + +/// Try each peer's `GET /p2p/blob/` in order, returning the first blob +/// that streams and verifies against `digest`. Returns `None` when every peer +/// fails — the caller then falls back to the registry. +/// +/// Never returns `Err`: a peer failure (connect / timeout / non-200 / 404 / +/// digest mismatch) is logged and the next peer is tried. Any `.partial` a +/// failed attempt leaves is removed so the next source (peer or registry) starts +/// from a clean slate. +pub(crate) async fn fetch_blob_from_peers( + client: &reqwest::Client, + peers: &[String], + digest: &str, + output: Option<&Path>, + cache: &BlobCache, +) -> Option { + let partial_path = cache.blob_path_for(digest).with_extension("partial"); + + for peer in peers { + match fetch_one(client, peer, digest, output, cache).await { + Ok(result) => { + tracing::info!(peer = %peer, digest = %digest, "fetched layer blob from peer (P2P)"); + return Some(result); + } + Err(e) => { + tracing::warn!( + peer = %peer, + digest = %digest, + error = %e, + "P2P peer fetch failed; trying next source" + ); + // Drop any partial this attempt wrote so the next peer / the + // registry fallback starts clean. + let _ = tokio::fs::remove_file(&partial_path).await; + } + } + } + None +} + +/// Fetch one blob from a single peer: `GET /p2p/blob/`, check the +/// status, then stream→hash→verify→adopt through the shared pull helper. +async fn fetch_one( + client: &reqwest::Client, + peer: &str, + digest: &str, + output: Option<&Path>, + cache: &BlobCache, +) -> Result { + let url = format!("{}/p2p/blob/{}", peer.trim_end_matches('/'), digest); + let resp = client.get(&url).send().await.map_err(RegistryError::Http)?; + + if resp.status() == reqwest::StatusCode::NOT_FOUND { + return Err(RegistryError::BlobNotFound(digest.to_string())); + } + if !resp.status().is_success() { + return Err(RegistryError::ApiError { + status: resp.status().as_u16(), + body: resp.text().await.unwrap_or_default(), + }); + } + + // Identical verification + cache accounting to the registry path: the digest + // is re-checked while streaming, so a lying peer can never poison the cache. + stream_verify_adopt(resp.bytes_stream(), digest, output, cache).await +} + +#[cfg(test)] +mod tests { + use super::*; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + // A valid, well-formed sha256 digest of the test blob below. Computed once + // so the tests assert against a real content address, not a placeholder. + fn blob_and_digest() -> (Vec, String) { + use sha2::{Digest, Sha256}; + let data = b"the-quick-brown-fox-p2p-blob".to_vec(); + let digest = format!("sha256:{}", hex::encode(Sha256::digest(&data))); + (data, digest) + } + + fn cache() -> (tempfile::TempDir, BlobCache) { + let tmp = tempfile::tempdir().unwrap(); + let cache = BlobCache::open(tmp.path().to_path_buf(), 1024 * 1024).unwrap(); + (tmp, cache) + } + + /// Mount `GET /p2p/blob/` on `server` returning `status` with `body`. + async fn mount_blob(server: &MockServer, digest: &str, status: u16, body: Vec) { + Mock::given(method("GET")) + .and(path(format!("/p2p/blob/{digest}"))) + .respond_with(ResponseTemplate::new(status).set_body_bytes(body)) + .mount(server) + .await; + } + + /// Peer hit → the blob is adopted into the cache and returned. The registry + /// is never consulted (this helper takes no registry at all). + #[tokio::test] + async fn peer_hit_adopts_blob() { + let (data, digest) = blob_and_digest(); + let (_tmp, cache) = cache(); + let peer = MockServer::start().await; + mount_blob(&peer, &digest, 200, data.clone()).await; + + let client = reqwest::Client::new(); + let result = fetch_blob_from_peers(&client, &[peer.uri()], &digest, None, &cache) + .await + .expect("peer hit must yield a result"); + + assert_eq!(result.digest, digest); + assert_eq!(result.size, data.len() as u64); + assert!(!result.cached); + // Adopted through the same LRU path — a subsequent cache lookup hits. + let cached = cache + .get(&digest) + .expect("blob must be in cache after peer fetch"); + assert_eq!(std::fs::read(cached).unwrap(), data); + } + + /// First peer 404s; the fetch cleanly falls through to the next peer. + #[tokio::test] + async fn peer_404_falls_through_to_next_peer() { + let (data, digest) = blob_and_digest(); + let (_tmp, cache) = cache(); + + let miss = MockServer::start().await; + mount_blob(&miss, &digest, 404, Vec::new()).await; + let hit = MockServer::start().await; + mount_blob(&hit, &digest, 200, data.clone()).await; + + let client = reqwest::Client::new(); + let result = + fetch_blob_from_peers(&client, &[miss.uri(), hit.uri()], &digest, None, &cache) + .await + .expect("second peer must satisfy the fetch"); + assert_eq!(result.size, data.len() as u64); + } + + /// A peer that returns the WRONG bytes (digest mismatch) is skipped, its + /// partial is cleaned up, and the next peer succeeds. + #[tokio::test] + async fn peer_digest_mismatch_falls_through_and_cleans_partial() { + let (data, digest) = blob_and_digest(); + let (_tmp, cache) = cache(); + + let liar = MockServer::start().await; + mount_blob(&liar, &digest, 200, b"totally-different-bytes".to_vec()).await; + let honest = MockServer::start().await; + mount_blob(&honest, &digest, 200, data.clone()).await; + + let client = reqwest::Client::new(); + let result = + fetch_blob_from_peers(&client, &[liar.uri(), honest.uri()], &digest, None, &cache) + .await + .expect("honest peer must satisfy the fetch"); + assert_eq!(result.size, data.len() as u64); + + // No leftover partial from the mismatching peer. + let partial = cache.blob_path_for(&digest).with_extension("partial"); + assert!(!partial.exists(), "partial must be cleaned after mismatch"); + } + + /// A peer that stalls past the client's timeout is abandoned; the next peer + /// serves the blob. Uses a test-local client with a tiny timeout (production + /// uses connect+read timeouts). + #[tokio::test] + async fn peer_timeout_falls_through() { + let (data, digest) = blob_and_digest(); + let (_tmp, cache) = cache(); + + let slow = MockServer::start().await; + Mock::given(method("GET")) + .and(path(format!("/p2p/blob/{digest}"))) + .respond_with( + ResponseTemplate::new(200) + .set_body_bytes(data.clone()) + .set_delay(Duration::from_secs(30)), + ) + .mount(&slow) + .await; + let fast = MockServer::start().await; + mount_blob(&fast, &digest, 200, data.clone()).await; + + let client = reqwest::Client::builder() + .timeout(Duration::from_millis(150)) + .build() + .unwrap(); + let result = + fetch_blob_from_peers(&client, &[slow.uri(), fast.uri()], &digest, None, &cache) + .await + .expect("fast peer must satisfy the fetch after the slow one times out"); + assert_eq!(result.size, data.len() as u64); + } + + /// Every peer fails → `None`, so the caller falls back to the registry. The + /// cache stays empty and no partial is left behind. + #[tokio::test] + async fn all_peers_fail_returns_none() { + let (_data, digest) = blob_and_digest(); + let (_tmp, cache) = cache(); + + let a = MockServer::start().await; + mount_blob(&a, &digest, 404, Vec::new()).await; + let b = MockServer::start().await; + mount_blob(&b, &digest, 500, b"boom".to_vec()).await; + + let client = reqwest::Client::new(); + let result = + fetch_blob_from_peers(&client, &[a.uri(), b.uri()], &digest, None, &cache).await; + assert!(result.is_none(), "all-peers-fail must return None"); + assert!(cache.get(&digest).is_none(), "cache must stay empty"); + let partial = cache.blob_path_for(&digest).with_extension("partial"); + assert!(!partial.exists(), "no partial should remain"); + } + + /// An unreachable peer (connection refused) is a clean failure, not a panic: + /// the next peer serves the blob. Covers the connect-error branch that a real + /// connect timeout also takes. + #[tokio::test] + async fn unreachable_peer_falls_through() { + let (data, digest) = blob_and_digest(); + let (_tmp, cache) = cache(); + + let hit = MockServer::start().await; + mount_blob(&hit, &digest, 200, data.clone()).await; + + let client = reqwest::Client::new(); + // Port 1 has nothing listening → connection refused. + let dead = "http://127.0.0.1:1".to_string(); + let result = fetch_blob_from_peers(&client, &[dead, hit.uri()], &digest, None, &cache) + .await + .expect("live peer must satisfy the fetch"); + assert_eq!(result.size, data.len() as u64); + } + + /// With the serve-TLS env vars unset, no peer client can be built, so P2P is + /// disabled (registry-only). Guarded by a mutex because it mutates process + /// env; other suites don't touch these vars. + #[test] + fn build_peer_client_errs_when_env_unset() { + static LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _g = LOCK.lock().unwrap_or_else(|e| e.into_inner()); + for v in [ENV_CERT, ENV_KEY, ENV_CLIENT_CA] { + std::env::remove_var(v); + } + let err = build_peer_client().unwrap_err(); + assert!(err.contains(ENV_CERT), "{err}"); + } +} diff --git a/crates/smolvm-registry/src/pull.rs b/crates/smolvm-registry/src/pull.rs new file mode 100644 index 0000000..15c6ae9 --- /dev/null +++ b/crates/smolvm-registry/src/pull.rs @@ -0,0 +1,253 @@ +//! Pull a `.smolmachine` artifact from an OCI registry. +//! +//! The pull flow: +//! 1. Fetch the OCI manifest by tag or digest +//! 2. Parse the manifest to find the layer blob digest +//! 3. Check the local cache for the blob +//! 4. If not cached, try any brokered P2P peers, then stream from the registry, +//! computing the digest while writing +//! 5. Verify the digest and adopt into cache + +use crate::cache::BlobCache; +use crate::client::RegistryClient; +use crate::{OciManifest, RegistryError, Result, LAYER_MEDIA_TYPE}; +use futures_util::StreamExt; +use sha2::{Digest, Sha256}; +use std::path::{Path, PathBuf}; +use tokio::io::AsyncWriteExt; + +/// Result of a successful pull. +#[derive(Debug)] +pub struct PullResult { + /// Path to the downloaded `.smolmachine` file. + pub path: PathBuf, + /// Digest of the layer blob. + pub digest: String, + /// Size of the layer blob in bytes. + pub size: u64, + /// Whether the blob was served from local cache. + pub cached: bool, +} + +/// Pull a `.smolmachine` artifact from the registry. +/// +/// `repo` is the OCI repository path (e.g., "python-dev"). +/// `reference` is the tag or digest (e.g., "latest" or "sha256:abc..."). +/// If `output` is Some, the blob is copied there. Otherwise it's only cached. +/// +/// `blob_peers` are optional brokered P2P peers (node base URLs, e.g. +/// `https://:`) supplied by the control plane. On a cache miss the +/// layer blob is fetched from a peer's `GET /p2p/blob/` (over node→node +/// mTLS) before the registry. When `blob_peers` is empty the peer block is +/// skipped entirely and the registry path is byte-for-byte what it was before. +pub async fn pull( + client: &RegistryClient, + repo: &str, + reference: &str, + output: Option<&Path>, + cache: &BlobCache, + blob_peers: &[String], +) -> Result { + // 1. Fetch the manifest, resolving a multi-platform index to this machine's + // host-platform entry (Docker-style fan-out). Shared with `inspect`. + tracing::info!(repo = %repo, reference = %reference, "fetching manifest..."); + let manifest_bytes = client.get_manifest_resolved(repo, reference).await?; + + let manifest: OciManifest = serde_json::from_slice(&manifest_bytes)?; + + // 2. Find the smolmachine layer. + let layer = manifest + .layers + .iter() + .find(|l| l.media_type == LAYER_MEDIA_TYPE) + .ok_or_else(|| { + RegistryError::InvalidManifest(format!( + "no layer with media type {} in manifest", + LAYER_MEDIA_TYPE + )) + })?; + + let digest = &layer.digest; + let size = layer.size; + + // Validate the digest format BEFORE it is used to build any cache filesystem + // path. `BlobCache::blob_path` only does `digest.replace(':', "_")`, leaving + // `/` and `..` intact, so an attacker-controlled manifest digest could + // otherwise create a `.partial` file or touch atime outside the cache dir. + crate::client::validate_digest(digest)?; + + // 3. Check cache. + if let Some(cached_path) = cache.get(digest) { + tracing::info!(digest = %digest, "blob found in cache"); + + if let Some(out) = output { + tokio::fs::copy(&cached_path, out).await?; + } + + return Ok(PullResult { + path: output.map(PathBuf::from).unwrap_or(cached_path), + digest: digest.clone(), + size, + cached: true, + }); + } + + // 4. Brokered P2P: try sibling nodes before hitting the registry. Inert when + // `blob_peers` is empty — the registry path below is then reached + // byte-for-byte as before. Peers arrive only from the control plane, and + // a node with no serve-TLS identity has no peer client, so this is also a + // no-op there. + if !blob_peers.is_empty() { + if let Some(peer_client) = crate::peer::peer_client() { + if let Some(result) = + crate::peer::fetch_blob_from_peers(peer_client, blob_peers, digest, output, cache) + .await + { + return Ok(result); + } + tracing::info!(digest = %digest, "no peer served the blob; falling back to registry"); + } + } + + // 5. Stream blob from the registry to disk while computing + verifying the + // digest, then adopt into cache. + tracing::info!(digest = %digest, size, "downloading blob..."); + + let stream = client.pull_blob_stream(repo, digest).await?; + let result = stream_verify_adopt(stream, digest, output, cache).await?; + + tracing::info!(digest = %digest, size = result.size, "pull complete"); + + Ok(result) +} + +/// Stream `stream` into the cache's `.partial` file for `digest`, hashing while +/// writing, verify the digest, adopt into the cache, and copy to `output` if +/// requested. +/// +/// Shared by the registry pull path and the P2P peer-fetch path so a blob +/// obtained either way goes through identical digest verification and the same +/// LRU/size accounting ([`BlobCache::adopt`]). On a digest mismatch the +/// `.partial` is removed before returning the error. A mid-stream transport +/// error propagates as-is (leaving the `.partial`, which the next attempt +/// truncates via `File::create`), matching the original registry pull behavior. +pub(crate) async fn stream_verify_adopt( + stream: S, + digest: &str, + output: Option<&Path>, + cache: &BlobCache, +) -> Result +where + S: futures_util::Stream>, +{ + let partial_path = cache.blob_path_for(digest).with_extension("partial"); + let mut file = tokio::fs::File::create(&partial_path).await?; + let mut hasher = Sha256::new(); + let mut total_bytes: u64 = 0; + + let mut stream = std::pin::pin!(stream); + while let Some(chunk_result) = stream.next().await { + let chunk: bytes::Bytes = chunk_result.map_err(RegistryError::Http)?; + hasher.update(&chunk); + file.write_all(&chunk).await?; + total_bytes += chunk.len() as u64; + } + file.flush().await?; + drop(file); + + // Verify digest. + let actual = format!("sha256:{}", hex::encode(hasher.finalize())); + if actual != *digest { + if let Err(e) = tokio::fs::remove_file(&partial_path).await { + tracing::warn!( + error = %e, + path = %partial_path.display(), + "failed to clean up partial blob after digest mismatch" + ); + } + return Err(RegistryError::DigestMismatch { + expected: digest.to_string(), + actual, + }); + } + + // Adopt into cache (handles eviction + atomic rename). + let cached_path = cache.adopt(digest, total_bytes)?; + + // Copy to output if requested. + let result_path = if let Some(out) = output { + tokio::fs::copy(&cached_path, out).await?; + PathBuf::from(out) + } else { + cached_path + }; + + Ok(PullResult { + path: result_path, + digest: digest.to_string(), + size: total_bytes, + cached: false, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{CONFIG_MEDIA_TYPE, MANIFEST_MEDIA_TYPE}; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + /// With empty `blob_peers`, `pull` takes the registry path exactly as before: + /// fetch the manifest, then stream the layer blob from `/v2/.../blobs/...`, + /// verify, and adopt. No peer client is built or consulted. + #[tokio::test] + async fn empty_blob_peers_uses_registry_path() { + use sha2::{Digest, Sha256}; + + let data = b"registry-path-layer-bytes".to_vec(); + let digest = format!("sha256:{}", hex::encode(Sha256::digest(&data))); + let config_digest = + "sha256:1111111111111111111111111111111111111111111111111111111111111111"; + + let manifest = serde_json::json!({ + "schemaVersion": 2, + "mediaType": MANIFEST_MEDIA_TYPE, + "config": { "mediaType": CONFIG_MEDIA_TYPE, "digest": config_digest, "size": 2 }, + "layers": [ { "mediaType": LAYER_MEDIA_TYPE, "digest": digest, "size": data.len() } ], + }); + + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/v2/myrepo/manifests/latest")) + .respond_with( + ResponseTemplate::new(200) + .set_body_raw(serde_json::to_vec(&manifest).unwrap(), MANIFEST_MEDIA_TYPE), + ) + .mount(&server) + .await; + // The layer blob endpoint must be hit exactly once on the registry path. + Mock::given(method("GET")) + .and(path(format!("/v2/myrepo/blobs/{digest}"))) + .respond_with(ResponseTemplate::new(200).set_body_bytes(data.clone())) + .expect(1) + .mount(&server) + .await; + + let tmp = tempfile::tempdir().unwrap(); + let cache = BlobCache::open(tmp.path().to_path_buf(), 1024 * 1024).unwrap(); + let client = RegistryClient::new(server.uri()); + + let result = pull(&client, "myrepo", "latest", None, &cache, &[]) + .await + .expect("registry pull must succeed"); + + assert_eq!(result.digest, digest); + assert_eq!(result.size, data.len() as u64); + assert!(!result.cached); + assert!( + cache.get(&digest).is_some(), + "blob must be adopted into cache" + ); + // MockServer drop asserts the blob endpoint's expect(1) was satisfied. + } +} diff --git a/crates/smolvm-registry/src/push.rs b/crates/smolvm-registry/src/push.rs new file mode 100644 index 0000000..362c702 --- /dev/null +++ b/crates/smolvm-registry/src/push.rs @@ -0,0 +1,184 @@ +//! Push a `.smolmachine` artifact to an OCI registry. +//! +//! The push flow: +//! 1. Compute SHA256 digest of the sidecar (streamed, no full-file buffer) +//! 2. Parse PackManifest from the footer for the OCI config blob +//! 3. Stream-upload the sidecar as a single OCI layer blob +//! 4. Upload the PackManifest JSON as the OCI config blob +//! 5. Build and PUT the OCI Image Manifest referencing both blobs + +use crate::client::RegistryClient; +use crate::{ + OciDescriptor, OciIndex, OciIndexManifest, OciManifest, OciPlatform, Result, CONFIG_MEDIA_TYPE, + INDEX_MEDIA_TYPE, LAYER_MEDIA_TYPE, MANIFEST_MEDIA_TYPE, +}; +use sha2::{Digest, Sha256}; +use std::path::Path; +use tokio::io::AsyncReadExt; + +/// Result of a successful push. +#[derive(Debug)] +pub struct PushResult { + /// Digest of the uploaded sidecar blob. + pub layer_digest: String, + /// Size of the sidecar blob in bytes. + pub layer_size: u64, + /// Digest of the OCI manifest. + pub manifest_digest: String, + /// Host platform this artifact targets (e.g. `linux/amd64`). + pub platform: String, + /// The per-platform tag the manifest was also tagged under (e.g. + /// `latest-linux-amd64`). + pub platform_tag: String, +} + +/// Push a `.smolmachine` sidecar to the registry. +/// +/// Uses a two-pass approach: pass 1 computes the SHA256 digest by streaming +/// through the file, pass 2 streams the file to the registry. The OS page +/// cache makes the second read essentially free. +pub async fn push( + client: &RegistryClient, + repo: &str, + reference: &str, + smolmachine_path: &Path, +) -> Result { + // 1. Compute SHA256 digest (pass 1: stream through hasher, no full-file buffer). + let file = tokio::fs::File::open(smolmachine_path).await?; + let file_meta = file.metadata().await?; + let layer_size = file_meta.len(); + + let mut reader = tokio::io::BufReader::new(file); + let mut hasher = Sha256::new(); + let mut buf = [0u8; 256 * 1024]; + loop { + let n = reader.read(&mut buf).await?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + let layer_digest = format!("sha256:{}", hex::encode(hasher.finalize())); + + tracing::info!( + path = %smolmachine_path.display(), + size = layer_size, + digest = %layer_digest, + "computed sidecar digest" + ); + + // 2. Parse PackManifest from the footer to use as OCI config. + let manifest = smolvm_pack::read_manifest_from_sidecar(smolmachine_path)?; + let config_json = serde_json::to_vec_pretty(&manifest)?; + + // 3. Upload the sidecar as the layer blob in a SINGLE streamed request. + // + // We deliberately do NOT split it into resumable per-chunk PATCH uploads. + // The registry's S3 backend is Cloudflare R2, which (unlike AWS S3) requires + // every non-final part of a multipart upload to be the same size. zot maps + // each OCI PATCH chunk onto an R2 multipart part, so unequal client chunks + // (e.g. 16 MiB + a smaller remainder) fail R2's CompleteMultipartUpload with + // `400 InvalidPart: All non-trailing parts must have the same length`, which + // the registry surfaces as a 500 on the finalizing PUT. Streaming the whole + // blob in one request lets the registry's storage driver choose uniform part + // sizes, which R2 accepts. + // + // Corollary: the registry's push path must NOT sit behind a request-body cap + // (e.g. an orange-cloud Cloudflare proxy that 413s large bodies). The body + // factory reopens the file per attempt so a 401 mid-upload retries with a + // fresh stream; the OS page cache makes reopens cheap. + tracing::info!("uploading sidecar blob..."); + let path = smolmachine_path.to_path_buf(); + client + .push_blob_stream(repo, &layer_digest, layer_size, move || { + // std::fs::File::open is synchronous but fast (just a syscall). + let file = std::fs::File::open(&path).map_err(crate::RegistryError::from)?; + let async_file = tokio::fs::File::from_std(file); + let stream = tokio_util::io::ReaderStream::with_capacity(async_file, 256 * 1024); + Ok(reqwest::Body::wrap_stream(stream)) + }) + .await?; + + // 4. Upload config blob (small, buffered is fine). + tracing::info!("uploading config blob..."); + let config_digest = client.push_blob(repo, &config_json).await?; + let config_size = config_json.len() as u64; + + // 5. Build OCI Image Manifest. + let oci_manifest = OciManifest { + schema_version: 2, + media_type: MANIFEST_MEDIA_TYPE.to_string(), + config: OciDescriptor { + media_type: CONFIG_MEDIA_TYPE.to_string(), + digest: config_digest, + size: config_size, + }, + layers: vec![OciDescriptor { + media_type: LAYER_MEDIA_TYPE.to_string(), + digest: layer_digest.clone(), + size: layer_size, + }], + }; + + let manifest_json = serde_json::to_vec_pretty(&oci_manifest)?; + let manifest_size = manifest_json.len() as u64; + let manifest_digest = format!("sha256:{}", hex::encode(Sha256::digest(&manifest_json))); + + // 6. Store the single-platform manifest content-addressably (by digest), and + // also tag it per platform (e.g. `latest-linux-amd64`) so a specific + // platform is directly pullable. + // + // Key the index entry on the GUEST platform (`linux/`), not the + // builder's `host_platform`. The `.smolmachine` sidecar is selected on + // pull by guest arch (client.rs matches `os == linux && arch == host + // arch`); keying on `host_platform` made artifacts built on macOS index + // as `darwin/arm64`, which pull could never select. On Linux builders + // `platform == host_platform`, so this is a no-op there. + let platform = OciPlatform::parse(&manifest.platform); + let platform_tag = format!("{reference}-{}-{}", platform.os, platform.architecture); + tracing::info!(digest = %manifest_digest, platform = %platform.label(), "uploading manifest..."); + client + .put_manifest(repo, &manifest_digest, &manifest_json) + .await?; + client + .put_manifest(repo, &platform_tag, &manifest_json) + .await?; + + // 7. Maintain an image index at `reference` so the tag fans out by platform. + // Merge with any existing index (replacing this platform's entry); a + // legacy single-manifest tag is replaced by a fresh index. + let entry = OciIndexManifest { + media_type: MANIFEST_MEDIA_TYPE.to_string(), + digest: manifest_digest.clone(), + size: manifest_size, + platform: Some(platform.clone()), + }; + let mut manifests = match client.get_manifest_raw(repo, reference).await { + Ok((bytes, ct)) if ct.contains(INDEX_MEDIA_TYPE) => { + serde_json::from_slice::(&bytes) + .map(|i| i.manifests) + .unwrap_or_default() + } + _ => Vec::new(), + }; + manifests.retain(|m| m.platform.as_ref() != Some(&platform)); + manifests.push(entry); + let index = OciIndex { + schema_version: 2, + media_type: INDEX_MEDIA_TYPE.to_string(), + manifests, + }; + let index_json = serde_json::to_vec_pretty(&index)?; + tracing::info!(reference = %reference, platforms = index.manifests.len(), "updating image index..."); + client.put_manifest(repo, reference, &index_json).await?; + + tracing::info!(digest = %manifest_digest, layer_size, "push complete"); + + Ok(PushResult { + layer_digest, + layer_size, + manifest_digest, + platform: platform.label(), + platform_tag, + }) +} diff --git a/crates/smolvm-smolfile/Cargo.toml b/crates/smolvm-smolfile/Cargo.toml new file mode 100644 index 0000000..ec434e1 --- /dev/null +++ b/crates/smolvm-smolfile/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "smolfile" +version = "1.5.2" +edition = "2021" +description = "Smolfile specification and parser for smolvm microVM workloads" +license = "Apache-2.0" +repository = "https://github.com/smol-machines/smolvm" +keywords = ["smolvm", "microvm", "configuration"] + +[dependencies] +serde = { version = "1", features = ["derive"] } +toml = "0.8" +thiserror = "1" +smolvm-protocol = { path = "../smolvm-protocol", version = "1.5.2" } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/smolvm-smolfile/src/lib.rs b/crates/smolvm-smolfile/src/lib.rs new file mode 100644 index 0000000..eeb1183 --- /dev/null +++ b/crates/smolvm-smolfile/src/lib.rs @@ -0,0 +1,606 @@ +//! Smolfile specification and parser. +//! +//! # Overview +//! +//! A **Smolfile** is a TOML file that declaratively defines a microVM workload. +//! It is the single source of truth for what image to run, how to configure it, +//! and how to package it for distribution. +//! +//! The file is named `Smolfile` (no extension) and lives in the project root. +//! +//! # Specification +//! +//! ## Top-level Fields +//! +//! | Field | Type | Required | Description | +//! |-------|------|----------|-------------| +//! | `image` | string | No | OCI image reference. Omit for bare Alpine VM. | +//! | `entrypoint` | string[] | No | Executable + fixed args. Overrides image ENTRYPOINT. | +//! | `cmd` | string[] | No | Default args appended to entrypoint. Overrides image CMD. | +//! | `env` | string[] | No | Environment variables as `KEY=VALUE`. | +//! | `workdir` | string | No | Working directory inside the VM. | +//! | `cpus` | int | No | Number of vCPUs (default: 1). | +//! | `memory` | int | No | Memory in MiB (default: 256). | +//! | `net` | bool | No | Enable outbound networking via NAT. | +//! | `cuda` | bool | No | Enable CUDA-over-vsock (host NVIDIA GPU). | +//! | `storage` | int | No | Storage disk size in GiB. | +//! | `overlay` | int | No | Overlay disk size in GiB. | +//! | `ports` | string[] | No | Port mappings (`"host:guest"`). Prefer `[dev] ports`. | +//! | `volumes` | string[] | No | Volume mounts (`"host:guest"`). Prefer `[dev] volumes`. | +//! | `init` | string[] | No | Commands run on every VM start. Prefer `[dev] init`. | +//! +//! ## Sections +//! +//! ### `[dev]` — Local development overrides +//! +//! Applied when running `smol up`. Not included in packed artifacts. +//! +//! | Field | Type | Description | +//! |-------|------|-------------| +//! | `volumes` | string[] | Bind mounts (`"./src:/app"`) | +//! | `env` | string[] | Dev-only environment variables | +//! | `init` | string[] | Bootstrap commands run on start | +//! | `workdir` | string | Dev working directory override | +//! | `ports` | string[] | Port mappings for development | +//! +//! ### `[artifact]` — Pack/distribution overrides +//! +//! Applied when running `smol pack create`. Overrides top-level values for the +//! packaged `.smolmachine` artifact. +//! +//! | Field | Type | Description | +//! |-------|------|-------------| +//! | `cpus` | int | vCPU count for packed artifact | +//! | `memory` | int | Memory (MiB) for packed artifact | +//! | `entrypoint` | string[] | Entrypoint override for artifact | +//! | `cmd` | string[] | Cmd override for artifact | +//! | `oci_platform` | string | Target platform (`"linux/amd64"`) | +//! +//! ### `[network]` — Egress filtering +//! +//! Controls outbound network access when `net = true`. +//! +//! | Field | Type | Description | +//! |-------|------|-------------| +//! | `allow_hosts` | string[] | Allowed hostnames (resolved to IPs at start) | +//! | `allow_cidrs` | string[] | Allowed CIDR ranges (`"10.0.0.0/8"`) | +//! +//! ### `[health]` — Health checks +//! +//! | Field | Type | Description | +//! |-------|------|-------------| +//! | `exec` | string[] | Health check command | +//! | `interval` | string | Check interval (`"10s"`, `"1m"`) | +//! | `timeout` | string | Check timeout (`"2s"`) | +//! | `retries` | int | Failures before unhealthy | +//! | `startup_grace` | string | Delay before first check | +//! +//! ### `[restart]` — Restart policy +//! +//! | Field | Type | Description | +//! |-------|------|-------------| +//! | `policy` | string | `"never"`, `"always"`, `"on-failure"`, `"unless-stopped"` | +//! | `max_retries` | int | Max restart attempts | +//! | `max_backoff` | string | Max delay between restarts | +//! +//! ### `[auth]` — Credential forwarding +//! +//! | Field | Type | Description | +//! |-------|------|-------------| +//! | `ssh_agent` | bool | Forward host SSH agent into the VM | +//! +//! ### `[service]` — Deployment metadata +//! +//! | Field | Type | Description | +//! |-------|------|-------------| +//! | `port` | int | Service listen port inside the VM | +//! | `protocol` | string | `"http"` or `"tcp"` | +//! +//! ### `[secrets]` — Secret injection +//! +//! Maps an environment variable name to a *reference*, never an inline value. +//! Each ref names exactly one source. The plaintext is resolved on the host +//! at exec time and is never written to the VM record, the database, or a +//! packed `.smolmachine` — only the reference travels. smolvm stores no +//! secret material itself; bring your own manager (Vault, 1Password, your +//! shell) and render it into an env var or file for the ref to point at. +//! +//! | Source | Type | Description | +//! |--------|------|-------------| +//! | `from_env` | string | Host environment variable to read at exec time | +//! | `from_file` | string | Absolute host file path to read at exec time | +//! +//! # Command Model +//! +//! Follows Docker/OCI semantics: +//! - `entrypoint`: the executable (like Dockerfile ENTRYPOINT) +//! - `cmd`: default arguments (like Dockerfile CMD) +//! - `init`: dev bootstrap commands run at VM start (NOT part of the container command) +//! +//! When set, `entrypoint` and `cmd` override the base image's OCI config. +//! If neither is set, the image's built-in values are used. +//! +//! # Example +//! +//! ```toml +//! image = "ghcr.io/acme/api:1.2.3" +//! entrypoint = ["/app/api"] +//! cmd = ["serve"] +//! workdir = "/app" +//! env = ["PORT=8080"] +//! +//! cpus = 2 +//! memory = 1024 +//! net = true +//! +//! [dev] +//! volumes = ["./src:/app"] +//! init = ["cargo build"] +//! ports = ["8080:8080"] +//! +//! [artifact] +//! cpus = 4 +//! memory = 2048 +//! +//! [network] +//! allow_hosts = ["pypi.org"] +//! allow_cidrs = ["10.0.0.0/8"] +//! +//! [health] +//! exec = ["curl", "-f", "http://localhost:8080/health"] +//! interval = "10s" +//! timeout = "2s" +//! retries = 3 +//! +//! [restart] +//! policy = "on-failure" +//! max_retries = 5 +//! +//! [auth] +//! ssh_agent = true +//! +//! [secrets] +//! DB_PASSWORD = { from_env = "PGPASSWORD" } +//! TLS_KEY = { from_file = "/run/secrets/tls.key" } +//! ``` + +use serde::Deserialize; +use std::path::Path; +use thiserror::Error; + +/// Errors from Smolfile parsing. +#[derive(Debug, Error)] +pub enum SmolfileError { + /// Failed to read the file. + #[error("failed to read {path}: {source}")] + Read { + /// File path. + path: String, + /// Underlying I/O error. + source: std::io::Error, + }, + /// Failed to parse the TOML content. + #[error("failed to parse {path}: {source}")] + Parse { + /// File path. + path: String, + /// Underlying TOML error. + source: toml::de::Error, + }, +} + +// ============================================================================ +// Smolfile types +// ============================================================================ + +/// Parsed Smolfile configuration. +/// +/// The workload command model follows Docker/OCI semantics: +/// +/// - `entrypoint`: the executable and its fixed leading arguments (like Dockerfile ENTRYPOINT) +/// - `cmd`: default arguments appended to entrypoint (like Dockerfile CMD) +/// - `init`: dev bootstrap commands run on every VM start (like RUN at boot, NOT like CMD) +/// +/// When set, `entrypoint` and `cmd` override the base image's OCI config values. +/// If neither is set, the image's built-in entrypoint and cmd are used. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct Smolfile { + /// OCI image (optional — omit for bare Alpine VM). + pub image: Option, + /// Executable and fixed leading arguments (overrides image ENTRYPOINT). + #[serde(default)] + pub entrypoint: Vec, + /// Default arguments appended to entrypoint (overrides image CMD). + #[serde(default)] + pub cmd: Vec, + /// Environment variables as `KEY=VALUE` strings. + #[serde(default)] + pub env: Vec, + /// Secrets injected as environment variables at workload launch. Each + /// entry maps an env var name to a reference (a host store entry, host + /// env var, or file) — never an inline value. The plaintext is resolved + /// on the host at exec time and never written to the VM record or DB. + #[serde(default)] + pub secrets: std::collections::BTreeMap, + /// Working directory inside the VM. + pub workdir: Option, + + // Resources + /// Number of vCPUs. + pub cpus: Option, + /// Memory in MiB. + pub memory: Option, + /// Enable outbound networking. + pub net: Option, + /// Enable GPU acceleration (Vulkan via virtio-gpu). + pub gpu: Option, + /// GPU VRAM (shared memory region) size in MiB. Ignored unless + /// `gpu = true`. Default comes from `DEFAULT_GPU_VRAM_MIB` (4 GiB). + pub gpu_vram: Option, + /// Enable Rosetta 2 x86_64 binary translation. macOS Apple Silicon + /// only; mounts the host Rosetta runtime into the guest via virtiofs. + pub rosetta: Option, + /// Enable CUDA-over-vsock: remote guest CUDA Driver-API calls to the + /// host NVIDIA GPU. Linux-only; requires a host NVIDIA driver. + pub cuda: Option, + /// Storage disk size in GiB. + pub storage: Option, + /// Overlay disk size in GiB. + pub overlay: Option, + + // Legacy top-level fields (prefer [dev] section) + /// Port mappings (e.g., `["8080:8080"]`). + #[serde(default)] + pub ports: Vec, + /// Volume mounts (e.g., `["./src:/app"]`). + #[serde(default)] + pub volumes: Vec, + /// Init commands run on every VM start. + #[serde(default)] + pub init: Vec, + + // Profiles + /// Artifact/pack overrides for `smol pack create`. + pub artifact: Option, + /// Alias for `artifact`. + pub pack: Option, + /// Local development profile. + pub dev: Option, + + // Sections + /// Network egress policy. + pub network: Option, + /// Health check configuration. + pub health: Option, + /// Restart policy. + pub restart: Option, + /// Credential forwarding. + pub auth: Option, + /// Service metadata for deployment. + pub service: Option, +} + +/// Network policy — egress filtering by hostname and/or CIDR. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct NetworkConfig { + /// Allowed egress hostnames (resolved to IPs at VM start). + #[serde(default)] + pub allow_hosts: Vec, + /// Allowed egress CIDR ranges (e.g., `["10.0.0.0/8", "1.1.1.1"]`). + #[serde(default)] + pub allow_cidrs: Vec, +} + +/// Credential forwarding configuration. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct AuthConfig { + /// Forward host SSH agent into the VM. + pub ssh_agent: Option, +} + +/// Distribution-specific overrides for packed artifacts. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct ArtifactConfig { + /// Override vCPU count for artifact. + pub cpus: Option, + /// Override memory (MiB) for artifact. + pub memory: Option, + /// Override entrypoint for artifact. + #[serde(default)] + pub entrypoint: Vec, + /// Override cmd for artifact. + #[serde(default)] + pub cmd: Vec, + /// Target OCI platform (e.g., `linux/amd64`). + pub oci_platform: Option, +} + +/// Local development profile. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct DevConfig { + /// Volume mounts for development (e.g., `["./src:/app"]`). + #[serde(default)] + pub volumes: Vec, + /// Development-only environment variables. + #[serde(default)] + pub env: Vec, + /// Init commands run on every VM start. + #[serde(default)] + pub init: Vec, + /// Development working directory override. + pub workdir: Option, + /// Port mappings for development (e.g., `["8080:8080"]`). + #[serde(default)] + pub ports: Vec, +} + +/// Health check configuration. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct HealthConfig { + /// Health check command (run via `sh -c`). + #[serde(default)] + pub exec: Vec, + /// Check interval (e.g., `"10s"`, `"1m"`). + pub interval: Option, + /// Check timeout (e.g., `"2s"`). + pub timeout: Option, + /// Number of consecutive failures before unhealthy. + pub retries: Option, + /// Grace period before first health check (e.g., `"20s"`). + pub startup_grace: Option, +} + +/// Restart policy configuration. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct RestartConfig { + /// Policy: `"never"`, `"always"`, `"on-failure"`, `"unless-stopped"`. + pub policy: Option, + /// Maximum restart attempts. + pub max_retries: Option, + /// Maximum backoff duration between restarts (e.g., `"60s"`, `"5m"`). + pub max_backoff: Option, +} + +/// Service metadata for deployment. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct ServiceConfig { + /// Port the service listens on inside the VM. + pub port: Option, + /// Protocol (`"http"`, `"tcp"`). + pub protocol: Option, + /// Alternate field name for port. + pub listen: Option, +} + +// ============================================================================ +// Parsing +// ============================================================================ + +/// Parse a Smolfile from a TOML string. +pub fn parse(content: &str) -> Result { + toml::from_str(content) +} + +/// Load and parse a Smolfile from a file path. +pub fn load(path: &Path) -> Result { + let content = std::fs::read_to_string(path).map_err(|e| SmolfileError::Read { + path: path.display().to_string(), + source: e, + })?; + + toml::from_str(&content).map_err(|e| SmolfileError::Parse { + path: path.display().to_string(), + source: e, + }) +} + +// ============================================================================ +// Utilities +// ============================================================================ + +/// Parse a duration string like `"10s"`, `"5m"`, `"2h"` to seconds. +pub fn parse_duration_secs(s: &str) -> Option { + let s = s.trim(); + if let Some(n) = s.strip_suffix('s') { + n.parse().ok() + } else if let Some(n) = s.strip_suffix('m') { + n.parse::().ok().map(|n| n * 60) + } else if let Some(n) = s.strip_suffix('h') { + n.parse::().ok().map(|n| n * 3600) + } else { + s.parse().ok() // bare number = seconds + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_minimal() { + let sf: Smolfile = parse("").unwrap(); + assert_eq!(sf.image, None); + assert_eq!(sf.cpus, None); + } + + #[test] + fn parse_secrets_section() { + // The [secrets] section deserializes into SecretRef references for both + // source kinds. Refs carry no inline value. + let sf = parse( + r#" +image = "alpine:latest" + +[secrets] +FROM_ENV = { from_env = "HOST_VAR" } +FROM_FILE = { from_file = "/etc/secret" } +"#, + ) + .unwrap(); + assert_eq!(sf.secrets.len(), 2); + assert_eq!(sf.secrets["FROM_ENV"].from_env.as_deref(), Some("HOST_VAR")); + assert_eq!( + sf.secrets["FROM_FILE"] + .from_file + .as_ref() + .map(|p| p.to_string_lossy().into_owned()), + Some("/etc/secret".to_string()) + ); + } + + #[test] + fn parse_rejects_unknown_secret_field() { + // deny_unknown_fields on SecretRef makes a typo (`from_stor`) a hard + // parse error instead of a silently-ignored empty ref. + let res = parse( + r#" +[secrets] +X = { from_stor = "typo" } +"#, + ); + assert!( + res.is_err(), + "expected parse error for unknown secret field" + ); + } + + #[test] + fn parse_full_example() { + let sf = parse( + r#" +image = "alpine" +cpus = 2 +memory = 1024 +net = true +entrypoint = ["/bin/sh"] +cmd = ["-c", "echo hello"] +env = ["FOO=bar"] +workdir = "/app" + +[dev] +volumes = ["./src:/app"] +init = ["echo hello"] +ports = ["8080:8080"] + +[artifact] +cpus = 4 +memory = 2048 + +[network] +allow_hosts = ["pypi.org"] +allow_cidrs = ["10.0.0.0/8"] + +[health] +exec = ["curl", "-f", "http://localhost/health"] +interval = "10s" +timeout = "2s" +retries = 3 + +[restart] +policy = "on-failure" +max_retries = 5 + +[auth] +ssh_agent = true + +[service] +port = 8080 +protocol = "http" +"#, + ) + .unwrap(); + + assert_eq!(sf.image.as_deref(), Some("alpine")); + assert_eq!(sf.cpus, Some(2)); + assert_eq!(sf.memory, Some(1024)); + assert_eq!(sf.net, Some(true)); + assert_eq!(sf.entrypoint, vec!["/bin/sh"]); + assert_eq!(sf.cmd, vec!["-c", "echo hello"]); + assert_eq!(sf.env, vec!["FOO=bar"]); + assert_eq!(sf.workdir.as_deref(), Some("/app")); + + let dev = sf.dev.unwrap(); + assert_eq!(dev.volumes, vec!["./src:/app"]); + assert_eq!(dev.init, vec!["echo hello"]); + + let artifact = sf.artifact.unwrap(); + assert_eq!(artifact.cpus, Some(4)); + assert_eq!(artifact.memory, Some(2048)); + + let network = sf.network.unwrap(); + assert_eq!(network.allow_hosts, vec!["pypi.org"]); + + let health = sf.health.unwrap(); + assert_eq!(health.retries, Some(3)); + + let restart = sf.restart.unwrap(); + assert_eq!(restart.policy.as_deref(), Some("on-failure")); + + assert_eq!(sf.auth.unwrap().ssh_agent, Some(true)); + assert_eq!(sf.service.unwrap().port, Some(8080)); + } + + #[test] + fn parse_cuda_field() { + let sf = parse("cuda = true").unwrap(); + assert_eq!(sf.cuda, Some(true)); + + let sf = parse("cuda = false").unwrap(); + assert_eq!(sf.cuda, Some(false)); + + let sf = parse("").unwrap(); + assert_eq!(sf.cuda, None); + } + + #[test] + fn parse_rejects_unknown_fields() { + let err = parse("bogus_field = true"); + assert!(err.is_err()); + } + + #[test] + fn load_from_file() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("Smolfile"); + std::fs::write( + &path, + r#" +image = "alpine" +cpus = 2 +memory = 1024 +net = true + +[dev] +volumes = ["./src:/app"] +init = ["echo hello"] +"#, + ) + .unwrap(); + let sf = load(&path).unwrap(); + assert_eq!(sf.image.as_deref(), Some("alpine")); + assert_eq!(sf.cpus, Some(2)); + assert_eq!(sf.dev.unwrap().volumes, vec!["./src:/app"]); + } + + #[test] + fn load_nonexistent_file() { + let err = load(Path::new("/nonexistent/Smolfile")).unwrap_err(); + assert!(matches!(err, SmolfileError::Read { .. })); + } + + #[test] + fn parse_duration_secs_formats() { + assert_eq!(parse_duration_secs("10s"), Some(10)); + assert_eq!(parse_duration_secs("5m"), Some(300)); + assert_eq!(parse_duration_secs("2h"), Some(7200)); + assert_eq!(parse_duration_secs("42"), Some(42)); + } +} diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md new file mode 100644 index 0000000..f1b0972 --- /dev/null +++ b/docs/DEVELOPMENT.md @@ -0,0 +1,127 @@ +# Development + +## Prerequisites + +- Rust toolchain +- [git-lfs](https://git-lfs.com) (required for library binaries) +- smolvm itself (for cross-compiling the agent — builds inside a `rust:alpine` VM) +- e2fsprogs (for storage template creation; `mkfs.ext4`; on macOS: `brew install e2fsprogs`) +- LLVM (macOS only, for building libkrun: `brew install llvm`) +- [cargo-make](https://github.com/sagiegurari/cargo-make): `cargo install cargo-make` + +## Quick Start + +We use [`cargo-make`](https://github.com/sagiegurari/cargo-make) to orchestrate build tasks: + +```bash +# Install cargo-make (one-time) +cargo install cargo-make + +# View all available tasks +cargo make --list-all-steps + +# Build and codesign (macOS) - binary ready at ./target/release/smolvm +cargo make dev + +# Run smolvm with environment variables set up automatically +cargo make smolvm --version +cargo make smolvm machine run --net --image alpine:latest -- echo hello +cargo make smolvm machine ls + +# Or run the binary directly with environment variables: +DYLD_LIBRARY_PATH="./lib" SMOLVM_AGENT_ROOTFS="./target/agent-rootfs" ./target/release/smolvm +``` + +**How it works:** +- `cargo make dev` builds + codesigns (macOS only), binary ready at `./target/release/smolvm` +- `cargo make smolvm ` runs smolvm with `DYLD_LIBRARY_PATH` and `SMOLVM_AGENT_ROOTFS` set up +- On macOS, binary is automatically signed with hypervisor entitlements + +## Building Distribution Packages + +```bash +# Build distribution package +cargo make dist + +# Build using local libkrun changes from ../libkrun +./scripts/build-dist.sh --with-local-libkrun +``` + +## Running Tests + +```bash +# Run all tests +cargo make test + +# Run specific test suites +cargo make test-cli # CLI tests only +cargo make test-sandbox # Sandbox tests only +cargo make test-machine # MicroVM tests only +cargo make test-pack # Pack tests only +cargo make test-lib # Unit tests (no VM required) +``` + +## Agent Rootfs + +The agent rootfs resolution order is: +1. `SMOLVM_AGENT_ROOTFS` env var (explicit override) +2. `./target/agent-rootfs` (local development) +3. Platform data directory (`~/.local/share/smolvm/` on Linux, `~/Library/Application Support/smolvm/` on macOS) + +```bash +# Build agent for Linux (size-optimized) +cargo make build-agent + +# Build agent rootfs +cargo make agent-rootfs + +# Rebuild agent and update rootfs +cargo make agent-rebuild +``` + +## Code Quality + +```bash +# Run clippy and fmt checks +cargo make lint + +# Auto-fix linting issues +cargo make fix-lints +``` + +## Other Tasks + +```bash +# Install locally from dist package +cargo make install +``` + +The `cargo make dist` task wraps `scripts/build-dist.sh`. Other scripts: + +```bash +./scripts/build-dist.sh +./scripts/build-agent-rootfs.sh +./scripts/install-local.sh +``` + +## Rebuilding Libraries + +The pre-built library binaries in `lib/` cover most development workflows. If you +need to rebuild them (after submodule updates, kernel config changes, or enabling +new features), see: + +- [Building libkrun](building-libkrun.md) — rebuild `lib/libkrun.dylib` (GPU support, blk, net) +- [Building libkrunfw](building-libkrunfw-macos.md) — rebuild `lib/libkrunfw.5.dylib` (kernel blob) + +## Troubleshooting + +**Database lock errors** ("Database already open"): +```bash +pkill -f "smolvm serve" +pkill -f "smolvm-bin machine start" +``` + +**Hung tests**: Check for stuck VM processes: +```bash +ps aux | grep smolvm +``` diff --git a/docs/cross-os-smolmachine-validation.md b/docs/cross-os-smolmachine-validation.md new file mode 100644 index 0000000..61cd147 --- /dev/null +++ b/docs/cross-os-smolmachine-validation.md @@ -0,0 +1,89 @@ +# Cross-OS `.smolmachine` validation (macOS → Linux arm64) + +**Status:** verified 2026-06-02 on GCP `c4a-highmem-96-metal` (Google Axion, arm64 bare metal). + +## What this proves (and what it doesn't) + +A `.smolmachine` packed on **macOS arm64** (Apple Silicon, libkrun-on-Darwin) was +transferred to a **remote Linux arm64 bare-metal host**, rehydrated, **booted +under Linux KVM with a from-source libkrun/libkrunfw**, and executed the packed +image. + +- ✅ **Cross-OS, same-arch** (`darwin/arm64` pack → `linux/arm64` run): proven. + The guest is `linux/arm64` either way; only the *host* runtime changes + (libkrun-on-Darwin/HVF vs libkrun-on-Linux/KVM). The seam that matters — an + artifact assembled against a macOS filesystem, reconstructed against ext4 on + bare-metal KVM — holds with no macOS-ism leaking into the packed layers. +- ❌ **Cross-architecture** (one artifact on both arm64 and x86_64) is **false by + design.** A `.smolmachine` packs native OCI layers + agent rootfs for ONE CPU + arch. The arch-aware scheduler ([smolfleet]) enforces this: an arm64 artifact + is *refused* on an x86 node, not run on it. + +So "cross-platform compatibility" = **OS-portable, arch-specific**. + +## Result + +``` +$ # exec inside the macOS-packed image, now on Linux arm64 bare metal +Python 3.12.13 +aarch64 +PRETTY_NAME="Alpine Linux v3.23" +exit: 0 +``` + +The artifact was `python:3.12-alpine` packed on a Mac — a plain rootfs has no +`python3`, so finding 3.12.13 proves the packed image state was rehydrated. + +## Why bare metal + +Linux libkrun needs `/dev/kvm`. Regular cloud VMs (GCP T2A/Tau, AWS Nitro) do +**not** expose nested virtualization, so smolvm workers must be **bare metal**: +GCP `c4a-*-metal` (Axion arm64) or AWS Graviton `*.metal`. On bare metal `/dev/kvm` +is native — there's no nesting to be unsupported. Confirmed on the host: +`uname -m` = `aarch64`, 96 cores, `/dev/kvm` present. + +## Reproduction + +Provision an arm64 Linux bare-metal host (see `smolfleet/deploy/terraform/gcp` +`arm_worker_*` or `.../aws`). Then, on the host: + +1. **Build the arm64 libkrun/libkrunfw libs** from the patched submodules: + `scripts/build-libkrun-linux.sh` → `lib/linux-aarch64/`. (Already committed; + rebuild only on a libkrun/libkrunfw bump.) +2. **Build the smolvm host binary:** `cargo build --release --bin smolvm`. + (Needs `AGENTS.md` present — it's `include_str!`'d by `main.rs`.) +3. **Build + install the agent-rootfs:** + `scripts/build-agent-rootfs.sh --arch aarch64` → copy `target/agent-rootfs` + to the smolvm data dir (`/.local/share/smolvm/agent-rootfs`). The node + needs its OWN agent-rootfs; a `.smolmachine`'s image layers run *on top of* it. +4. **Run the runtime:** + `SMOLVM_LIB_DIR=/lib/linux-aarch64 smolvm serve start --listen 0.0.0.0:8080` + (KVM access needs root or `kvm` group membership). +5. **Rehydrate a Mac-built artifact:** copy the `.smolmachine` over, then + `POST /api/v1/machines {"name":..., "from":"/path/to.smolmachine", ...}` → + `POST .../start` → `POST .../exec`. + +## Gotchas hit (and fixed in the tooling) + +Discovered building on the real host; folded back into the scripts where applicable: + +- **`make -C libkrunfw` fails `No rule to make target 'w'`** — `-C` auto-enables + `-w` (print-directory), which lands in `MAKEFLAGS` as a bare `w` and the + kernel's `$(MAKE) $(MAKEFLAGS)` recipe treats it as a target. Fix: build from + *inside* the dir, not with `-C`. (Fixed in `build-libkrun-linux.sh`.) +- **libkrunfw needs `python3-pyelftools`** (for `bin2cbundle.py`). +- **libkrun GPU feature needs** `libepoxy-dev` + `libvirglrenderer-dev` + + `libdrm-dev` + `libgbm-dev` (build) and `libvirglrenderer1` + `libepoxy0` + (runtime). +- **`error: linker 'cc' not found`** — install `build-essential` (cloud-init may + still hold the apt lock on first boot; wait for `lock-frontend`). +- **smolvm needs the agent-rootfs installed** even for `from`/`.smolmachine` + machines (step 3) — the artifact supplies image layers, not the guest init. + +## Versions built + +- `libkrunfw.so.5.4.0` (linux/aarch64, kernel 6.12.87) +- `libkrun.so.1.17.3` (linux/aarch64, BLK+NET+GPU) +- `smolvm` 0.8.1 (linux/aarch64 host binary) + +[smolfleet]: ../smolfleet diff --git a/docs/install-arch.md b/docs/install-arch.md new file mode 100644 index 0000000..d9331ca --- /dev/null +++ b/docs/install-arch.md @@ -0,0 +1,41 @@ +# Installing smolvm on Arch Linux + +smolvm ships an **official pacman repository** maintained by the smol-machines +team. It serves prebuilt packages for `x86_64` and `aarch64`, updated +automatically with every release. + +> The `smolvm`, `smolvm-bin` and `smolvm-git` packages on the AUR are +> third-party and not maintained by us. As of 1.4.7 they link against the +> stock Arch `libkrun`, which lacks six symbols smolvm needs — disk overlays, +> snapshots and egress policy do not work with those packages. The official +> package bundles the smol-machines libkrun fork and is fully functional. + +## Setup + +Add the repository to `/etc/pacman.conf`: + +```ini +[smol-machines] +SigLevel = Optional TrustAll +Server = https://smol-machines.github.io/smolvm/pacman/$arch +``` + +Then install: + +```sh +sudo pacman -Sy smolvm +``` + +Upgrades arrive with your normal `pacman -Syu`. + +## Notes + +- The repository is served from GitHub Pages (the `pacman-repo` branch); + packages are built by CI from the release artifacts and published on every + release (`.github/workflows/pacman-repo.yml`). It is a rolling repo — the + latest release only (older versions remain on the GitHub Releases page). +- Package signing (GPG) is planned; until then integrity is provided by + sha256-pinned sources built in CI and served over HTTPS. +- The package bundles the smol-machines `libkrun`/`libkrunfw` fork under + `/usr/lib/smolvm/lib` — it does not conflict with the system `libkrun` + package, which other software may continue to use. diff --git a/docs/lossless-serve-restart.md b/docs/lossless-serve-restart.md new file mode 100644 index 0000000..8bab14a --- /dev/null +++ b/docs/lossless-serve-restart.md @@ -0,0 +1,174 @@ +# Lossless `smolvm serve` restart + +## Goal + +Restart the `smolvm` serve process (the node runtime / local API) — for a binary +upgrade **or** an unexpected crash-auto-restart — **without killing or orphaning +running VMs**, and reconnect to them afterward so exec / interactive / management +keep working. + +This turns a fleet serve upgrade from a drain-everything operation into a rolling +`systemctl restart` that running workloads sail through. + +## Scope — what "lossless" does and does NOT cover + +The key distinction is **guest compute vs. host control-plane**. Processes +*inside* the VM (a job, a server, an agent task) live in the guest kernel, not in +serve, so they keep executing across a serve restart. What blips is the host-side +control channel for the ~1–3 s restart window: new exec/API calls fail and +**attached interactive sessions drop** (the WebSocket dies; the guest-side shell +process survives but the terminal must reconnect). Published-port traffic can blip +for that window too. + +| Update type | Lossless? | +|---|---| +| **serve binary upgrade** (node runtime) | ✅ yes — the target win | +| serve **crash** / systemd auto-restart | ✅ yes (scopes cover it) | +| **libkrun upgrade** (hypervisor dylib) | ❌ no — only new VMs get it; running VMs keep the loaded copy | +| **host kernel / OS / instance reboot** | ❌ no — kills every host process, VMs included | +| guest rootfs / agent | ❌ no for running VMs (baked at boot) | + +Infrastructure-level updates (libkrun, kernel, host reboot, instance replacement) +still bounce VMs — those need drain+reschedule today, and live-migration +(libkrun checkpoint/restore + the fork-clone work) long term. "Lossless restart" +≠ "survives anything." + +## Why it's hard: three independent kill vectors + +A running VM is, today, a child process of `smolvm-node.service` in that unit's +**delegated cgroup**. Three separate mechanisms tear it down on restart: + +1. **serve signals the PID** — `impl Drop for AgentManager` (`src/agent/manager.rs`) + calls `stop()` (SIGTERM/SIGKILL) when the manager owns the child and isn't + `detached`. serve never detached on shutdown (the CLI does). +2. **systemd kills the service cgroup** — default `KillMode=control-group` SIGKILLs + the whole subtree on unit stop. (`KillMode=process` mitigates; scopes make it moot.) +3. **systemd refuses to recreate the cgroup** — on `systemctl restart`, a surviving + VM left in the service's delegated cgroup yields **`status=219/CGROUP`**: the new + serve never starts and crash-loops. This fires on *any* unit (re)start, including + systemd's automatic crash-restart. + +Losslessness requires neutralizing all three plus reconnect. Missing any one still +kills the VM or crash-loops serve. + +## The model: VMs become systemd scopes; serve becomes a reconnecting manager + +Invert ownership: each VM becomes its own first-class systemd unit +`smolvm-vm-.scope`, owned by PID1 in a **sibling cgroup** (not under the service). +serve *attaches to* and *detaches from* VMs instead of *containing* them — the +libvirt/`machined` pattern, and the supported way to put externally-forked +processes under systemd cgroup management. + +``` +serve.start_vm(): + fork _boot-vm ──► PID + StartTransientUnit("smolvm-vm-.scope", PIDs=[PID], + properties=[MemoryMax, CPUQuota, TasksMax, CPUWeight]) + └─ systemd adopts PID into a sibling cgroup + applies caps + +serve dies / restarts: + detach_all() → Drop won't signal the PID (vector 1) + VM reparents to init; scope persists (systemd owns the cgroup) + new serve starts in a CLEAN service cgroup (no 219 — VM isn't there) + +new serve startup: + list_vms() (DB) ✕ enumerate smolvm-vm-*.scope (cross-check / reconcile) + for each live VM: reconnect agent.sock + ping (vector 3 → reconnect) + +machine stop/delete: + kill PID → scope auto-GCs when its last process exits (no extra teardown) +``` + +Scopes auto-remove when empty, so stop/delete needs no special scope teardown — +killing the VM PID is enough and systemd reaps the scope. + +## Plan (phased) + +**Phase 1 — Scoped VM launch (`manager.rs`, core change).** After forking +`_boot-vm`, `busctl StartTransientUnit` creates `smolvm-vm-.scope` adopting the +PID, with per-VM caps ported from the current cgroup code to scope properties. +**Availability gate + fallback** (`systemd_scope::is_available()`): requires +`/run/systemd/system` + `busctl` + **root** (system-bus `StartTransientUnit` needs +root/polkit; serve is root on the worker). Anything else — macOS dev, containers, +OpenRC, **or an unprivileged local Linux `serve`** — returns false and routes to the +existing delegated-cgroup placement, which keeps resource caps (just not lossless, +fine for dev). The root gate specifically prevents a non-root local serve from +entering scope-mode and then failing every adopt *uncapped* (scope-mode skips the +cgroup fallback). One detection point, two launch paths sharing everything +downstream. Accept the microsecond fork→adopt race (only risks a brand-new VM). +A runtime adopt failure *after* the gate passed (broken D-Bus) logs loudly; the VM +runs uncapped + not-restart-safe. Future: support the per-user bus (`busctl --user`) +for unprivileged local serve. + +**Phase 2 — Detach on shutdown (DONE).** `ApiState::detach_all()` on serve's +non-drain shutdown path. Still required: scopes stop *systemd* from killing the VM; +detach is the only thing that stops serve's *application-level* signal. + +**Phase 3 — Reliable reconnect.** `try_connect_existing` (connect `agent.sock` + +`ping`) works once serve actually starts; add a supervisor **retry loop** (bounded +backoff) for "alive but not yet reachable" machines so reconnect absorbs agent +settle time. Cross-check DB vs. enumerated scopes and reconcile drift. + +**Phase 4 — Drain decoupling.** Remove `SMOLVM_DRAIN_ON_SHUTDOWN` from the worker +unit + Terraform. Preserve clean decommission via an **explicit control-driven +drain**: the autoscaler calls a serve drain trigger before `provider.terminate()`. +Drain becomes a deliberate decommission step, never a restart side-effect. + +**Phase 5 — Crash-restart.** Falls out for free: with VMs in scopes, systemd's +automatic restart of a crashed serve also starts clean and reconnects. (This is why +a self-re-exec trick is insufficient — it covers only planned upgrades.) + +**Phase 6 — Validation (gate before fleet rollout).** Idle VM survives a +`systemctl restart` (same PID) + reconnect + exec, ×3. Busy VM: restart while an +interactive exec streams → VM survives, session drops, fresh exec reconnects. Crash +sim: `kill -9` serve → auto-restart → survive + reconnect. Caps enforced on the +scope. Fallback: macOS/non-systemd still boots via direct fork. + +**Phase 7 — Rollout.** Deploy to workers (one-time bounce — legacy VMs in the old +service cgroup still drop on that first restart); new VMs launch in scopes; every +subsequent restart is lossless. Codify unit/TF changes. + +## Decisions & trade-offs + +| Decision | Choice | Why | +|---|---|---| +| Scope vs hand-rolled cgroup move | **Scope** | Supported API; hand-moving fights `Delegate=` and is systemd-version-fragile | +| Scope vs self-re-exec | **Scope** | re-exec misses crash-restart; scopes cover both | +| Scope vs separate `vmd` daemon | **Scope** | daemon-split is the same losslessness at ~3× the architecture; revisit only if API/VM need independent scaling | +| fork-then-adopt race | **Accept** | window is microseconds, only risks a brand-new VM | +| non-systemd hosts | **Fallback to direct fork** | contains systemd coupling to the cloud worker; keeps dev/CI working | + +## Risks + +- **Blast radius** — touches the most critical path (VM spawn). Mitigate with the + dual-path fallback (dev unaffected) and the Phase 6 validation gate. +- **systemd version differences** in scope/property semantics — pin behavior in an + integration test on the actual worker image. +- **Reconnect to a busy VM** — the guest agent must tolerate a client vanishing + mid-stream (the accept-loop design suggests it does; Phase 6's busy-VM test proves it). + +## Status + +- **Phases 1 + 2 implemented + validated live on worker-1 (2026-06-14).** A VM was + placed (landed in `smolvm-vm-.scope`, a sibling cgroup), serve restarted, and: + VM PID survived; the scope stayed `active`; serve started clean (**no + `219/CGROUP`**); `reconnected to machine ... pid=Some()` (the live VM, not a + stale record); and exec returned `LOSSLESS_OK_x86_64` (exit 0, 38 ms) through the + reconnected serve. Scope binary sha `921eb504`. +- New `src/systemd_scope.rs` (busctl `StartTransientUnit`, no D-Bus crate); + `serve.rs` chooses scope-mode when systemd is present (else delegated-cgroup + fallback); `manager.rs` adopts the forked PID after fork. `internal_boot.rs` + needed no change (it already skips self-placement when `SMOLVM_CGROUP_ROOT` is + unset). +- **Phase 3 (reconnect retry) is now optional** — validation showed the existing + `try_connect_existing` reconnects immediately once serve can start; the retry + loop is belt-and-suspenders, not load-bearing. +- **Phase 4 code DONE (compiles):** explicit drain trigger — smolvm `POST /drain` + (`handlers::machines::drain_node` → `drain_machines`, control-only via the mTLS + listener); smolfleet `RuntimeDriver::drain()` (default no-op) + `SmolvmDriver` + POST impl; autoscaler drains before `provider.terminate()` (best-effort, never + blocks decommission). Ops half (remove `SMOLVM_DRAIN_ON_SHUTDOWN` from units + TF) + pairs with the Phase 7 binary rollout. +- Pending: Phase 3 (optional retry hardening), Phase 4 ops (env removal), Phase 6 + busy-VM + crash-sim live cases, Phase 7 PR + fleet rollout. Code uncommitted + across both repos; worker-1 reverted to fleet-standard `d8f57614`. See task #210. diff --git a/docs/network-virtio-policy-macos-plan.md b/docs/network-virtio-policy-macos-plan.md new file mode 100644 index 0000000..18af2bb --- /dev/null +++ b/docs/network-virtio-policy-macos-plan.md @@ -0,0 +1,123 @@ +# Policy-complete, cross-platform virtio-net + +## Problem +The HTTP API accepts `ports` but always provisions TSI (outbound-only), so published/inbound +ports are silently non-functional through the API. virtio-net — the only backend with an +inbound path — is reachable only via the hidden `--net-backend virtio-net` CLI flag, is +Linux-only, and can't enforce egress policy (the `PolicyRequiresTsi` fallback). This closes +those gaps so virtio-net is a first-class, policy-complete, cross-platform backend. + +## Decisions +1. DNS filtering on virtio-net **reuses** TSI's host-side filter (guest `dns_proxy` → vsock → + `dns_filter_socket`) so semantics match by construction. Keep both backends. +2. Keep both backends — no convergence on one. +3. **TSI stays the default.** virtio-net is opt-in. Requesting `ports` without virtio-net is a + clear error, not a silent break or an auto-switch. + +## Architecture notes (verified) +- `plan_launch_network()` (`src/network/launch.rs`) is the single shared backend-decision + chokepoint; both `launcher.rs` (CLI/fork) and `launcher_dynamic.rs` (the `_boot-vm` the API + server spawns) call it with `port_count`. +- The published-port inbound relay lives only in the virtio-net path (`tcp_relay.rs`, + `RelayTarget::Attached`); TSI has no inbound handling. +- `smolvm-network` is POSIX-only (`poll`/`pipe`/`fcntl`/`UnixStream`/`setsockopt` + userspace + smoltcp) — Linux gating is incidental, not fundamental. +- libkrun `krun_add_net_unixstream` is `#[cfg(feature = "net")]` (not OS-gated); the macOS + dylib is already built `NET=1`. +- TSI egress policy: the **DNS filter is host-side in smolvm** (reusable via vsock); the + **CIDR filter is inside libkrun's TSI** (`set_egress`, `launcher.rs:569`) — TSI-only, so the + gateway needs its own CIDR enforcement. + +## Workstream A — egress policy on virtio-net +- A1. CIDR filter in the smoltcp gateway: enforce `allowed_cidrs` at host-connection setup; + drop + RST disallowed destinations. Thread `allowed_cidrs` into `start_virtio_network`. +- A2. DNS filter: wire the existing `dns_filter_socket` + guest `dns_proxy` on the virtio-net + launch path (same host-side filter code as TSI → exact parity, no reimplementation). +- A3. Integration: feed DNS-filter-resolved IPs (for allowed hostnames) into the gateway's + dynamic allow-set so follow-up connections to resolved addresses pass A1. +- A4. Remove the `VirtioNet if has_policy → PolicyRequiresTsi` fallback in `launch.rs`. +- A5. Tests: allowed/blocked CIDR, allowed/blocked host, ports + `allowedCidrs` together. + +## Workstream B — virtio-net on macOS +Correction after audit: virtio-net is **already cross-platform in source** — there is +nothing to un-gate. +- B1/B2 — N/A. `smolvm-network` is a main (cross-platform) dependency with no internal + `cfg(target_os)`; the launcher's virtio-net arm and `create_unix_stream_pair` + (`socketpair`) are not gated. The Linux-only dependency block holds only + `seccompiler`/`landlock`. +- B3 — already handled: `SO_SNDBUF` over-request is non-fatal (logged, clamped); writes + surface `EPIPE` rather than `SIGPIPE` (Rust ignores it process-wide on macOS too). No code + change needed unless validation finds otherwise. +- B4. Confirm `lib/libkrun.dylib` exports `krun_add_net_unixstream` (NET=1 build) — same + class of risk as the `krun_create_disk_overlay` vendoring gap. +- B5. Build this branch on the isolated `~/smolvm-f5` Mac clone (never touch + `~/Documents/smolvm`) and live-validate virtio-net on HVF: outbound, CIDR, allow-host, + published ports. Fix any runtime quirks that surface. + +Remaining B work is pure Mac validation (B4 + B5); both require Mac access. + +**VALIDATED on M4 Max / macOS 26.1 / HVF (2026-06-09, `~/smolvm-f5` fast-fork build):** +- B4: `lib/libkrun.dylib` exports `_krun_add_net_unixstream` (NET=1 build). +- virtio-net VM boots with `eth0 100.96.0.2/30` (smoltcp gateway active, not TSI). +- Outbound TCP works (`wget http://1.1.1.1/` succeeds) — the portable POSIX gateway + + `krun_add_net_unixstream` + HVF move packets end-to-end. +- Inbound published-port host listener binds (`smolvm` holds `TCP localhost:18080 LISTEN`). +- Validated with the *base* virtio-net (fast-fork). The A+C egress-policy additions + (`egress.rs`/`dns.rs`, CIDR + allow-host) are portable Rust with no `cfg(target_os)` and are + Linux-live-validated; running them specifically on macOS needs this branch built on the Mac. + +## Workstream C — API + UX (no default change) +- C1. Add `networkBackend: Option` to `CreateMachineRequest` + (`api/types.rs`); thread through `resource_spec_to_vm_resources` (`api/state.rs`, replacing + hardcoded `None`). +- C2. Error when `ports` are requested without virtio-net (default or explicit TSI): + "published ports require networkBackend: virtio-net". +- C3. Unhide `--net-backend`; `plan_launch_network` default stays TSI. +- C4. API tests: virtio-net + port reachable; TSI + port → clean 4xx. + +## Workstream D — IPv6 dual-stack (DONE, live-validated 2026-06-10) +The gateway was IPv4-only by Phase-1 scoping, not by architecture. Now dual-stack: +- smoltcp `proto-ipv6` + `iface-max-addr-count-3`; gateway owns IPv4 + IPv6 ULA + + EUI-64 link-local; default v6 route; `any_ip` covers v6 (verified in smoltcp 0.13). +- Link addressing: guest `fd53:4d00::2/64`, gateway `fd53:4d00::1` (`53:4d` = "SM", + matching the MAC OUI scheme). +- `classify_guest_frame` handles `Ipv6Packet` (extension-headered frames pass through); + outbound TCP relay is family-agnostic (V4-only guards removed). +- DNS socket binds wildcard `:53` — works over both families and transparently + intercepts hardcoded external resolvers (TSI parity); AAAA records are learned into + the egress allow-set alongside A (`dns::answer_ip_records`). +- `EgressPolicy` accepts v4 + v6 CIDRs; learned IPs are `IpAddr`. +- Published ports bind `[::1]` as well as `127.0.0.1` (v6 best-effort). +- Guest agent: optional `SMOLVM_NETWORK_{GUEST_IP6,GATEWAY6,PREFIX_LEN6}` env trio → + rtnetlink `AF_INET6` addr (IFA_F_NODAD) + default route. Launchers export the trio. + +**Validated live (Linux/KVM):** guest dual-stack (`fd53:4d00::2/64` NODAD + default via +gateway); NDP answered (gateway REACHABLE); guest TCP over v6 to a host ULA fetched +content end-to-end; v6 egress CIDR A/B (allow `fd7a::/48` → fetch OK; allow +`2001:db8::/32` → same target refused; v4 cross-family blocked); published port serves +both `127.0.0.1` and `[::1]` (explicit AF_INET6 client). AAAA learning unit-tested +(host lacks external IPv6 for a live allow-host→AAAA run). + +Known remaining gaps (both families): ICMP (ping). General UDP is now relayed — see +Workstream E. + +## Workstream E — general UDP relay (DONE, live-validated 2026-06-12) +Non-DNS guest UDP used to be dropped (`FrameAction::UnsupportedUdp`), breaking +QUIC/HTTP-3, NTP, and DNS on non-standard ports. `udp_relay.rs` is a tiny userspace +NAT mirroring the TCP relay's shape: destination-keyed smoltcp UDP sockets +(`UdpSocketTable`, the UDP twin of `create_tcp_socket`), one relay thread owning a +connected host `UdpSocket` per (guest, destination) flow, channels + wake pipes in +both directions, and NAT-style idle expiry (flows 60s, destination sockets 120s). +Replies are written back with `local_address` = the original destination, so the +guest sees them come from the right peer. Egress policy applies exactly as for TCP +(static CIDRs + DNS-learned IPs; denied destinations are a silent UDP black hole); +DNS :53 keeps its own intercept-and-filter path. Both IP families. + +**Validated live (Linux/KVM):** guest `nc -u` to a host echo server on :9099 +round-trips over IPv4 and IPv6; with `--allow-cidr 1.1.1.1/32` the same UDP flow is +black-holed while TCP to the allowed CIDR still works. Unit tests cover the relay +thread end-to-end, flow sockets, and the policy/DNS carve-out. + +## Sequencing +A and B are independent (parallelizable). C depends on A removing the conflict. A is the +highest-value and fully testable on Linux; start there. D builds on A. diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..d85669f --- /dev/null +++ b/examples/README.md @@ -0,0 +1,121 @@ +# Smolfile Examples + +A Smolfile is the declarative source of truth for a microVM workload. It describes what runs, what resources it needs, and how it behaves — then smolvm uses the same spec for local execution, artifact creation, and future deployment. + +## Quick Start + +```bash +# Run the OpenClaw gateway from a Smolfile +smolvm machine run -d -s examples/openclaw-app/openclaw.smolfile +curl http://localhost:18789/health + +# Run a Python dev environment +smolvm machine run -s examples/python-app/python.smolfile + +# Run Doom in a browser +smolvm machine run -d -s examples/doom-web/doom.smolfile +open http://localhost:8080 + +# Headless Chromium with GPU acceleration +smolvm machine create --name browser -s examples/headless-browser/browser.smolfile +smolvm machine start --name browser +smolvm machine exec --name browser -- \ + chromium --headless=new --no-sandbox --disable-dev-shm-usage \ + --use-gl=angle --use-angle=vulkan \ + --screenshot=/tmp/out.png --window-size=1280,800 \ + https://example.com +smolvm machine exec --name browser -- base64 /tmp/out.png | base64 -d > out.png +``` + +### Persistent microVMs + +```bash +smolvm machine create --name dev -s examples/python-app/python.smolfile +smolvm machine start --name dev +smolvm machine exec --name dev -- python3 --version +smolvm machine stop --name dev +``` + +### Pack a distributable binary + +```bash +smolvm pack create -s examples/openclaw-app/openclaw.smolfile -o openclaw-packed +./openclaw-packed +``` + +## Smolfile Reference + +```toml +# Top-level workload fields +image = "ghcr.io/acme/api:1.2.3" # OCI image (optional — omit for bare Alpine VM) +entrypoint = ["/app/api"] # executable and fixed leading arguments +cmd = ["serve"] # default arguments appended to entrypoint +env = ["PORT=8080"] # runtime environment variables +workdir = "/app" # working directory + +# Resources +cpus = 2 # vCPUs (default: 4) +memory = 1024 # MiB (default: 8192) +net = true # outbound networking (default: false) +gpu = true # GPU acceleration via virtio-gpu/Venus (Vulkan) +gpu_vram = 2048 # GPU shared-memory region in MiB (default: 4096) +storage = 40 # storage disk GiB (default: 20) +overlay = 4 # overlay disk GiB (default: 2) + +# Network policy — egress filtering by hostname and/or CIDR +[network] +allow_hosts = ["api.stripe.com", "db.example.com"] # resolved at VM start +allow_cidrs = ["10.0.0.0/8"] # IP/CIDR ranges + +# Health checks +[health] +exec = ["curl", "-f", "http://127.0.0.1:8080/health"] +interval = "10s" +timeout = "2s" +retries = 3 + +# Restart policy (parsed, not yet wired) +[restart] +policy = "always" + +# Local development profile +[dev] +volumes = ["./src:/app"] # host bind mounts +env = ["APP_MODE=development"] # dev-only env (extends top-level) +init = ["npm install"] # dev bootstrap commands +workdir = "/app" # dev-only workdir override +ports = ["8080:8080"] # host:guest port forwarding + +# Artifact profile (for smolvm pack create -s) +[artifact] +cpus = 4 # override resources for distribution +memory = 2048 +entrypoint = ["/app/api"] # override entrypoint for packed binary +oci_platform = "linux/amd64" # target OCI platform + +``` + +### Merge precedence + +CLI flags override Smolfile values. For `machine run`: + +``` +image: --image flag > Smolfile image > None (bare Alpine VM) +entrypoint: --entrypoint flag > Smolfile entrypoint > image metadata +cmd: trailing args (after --) > Smolfile cmd > image metadata +env: top-level env + [dev].env + CLI -e +init: [dev].init + CLI --init +volumes: [dev].volumes + CLI -v +ports: [dev].ports + CLI -p +``` + +For `pack create`: + +``` +image: --image flag > Smolfile image +entrypoint: --entrypoint flag > [artifact].entrypoint > Smolfile entrypoint > image metadata +cmd: [artifact].cmd > Smolfile cmd > image metadata +cpus: --cpus flag > [artifact].cpus > top-level cpus +env: image env + Smolfile env (dedup by key) +workdir: Smolfile workdir > image workdir +``` diff --git a/examples/docker-in-vm/docker.smolfile b/examples/docker-in-vm/docker.smolfile new file mode 100644 index 0000000..33963a3 --- /dev/null +++ b/examples/docker-in-vm/docker.smolfile @@ -0,0 +1,118 @@ +# Docker inside a smolvm microVM +# +# Runs Docker Engine in a bare Alpine VM. Init commands run directly in the VM +# as root via VmExec, so Docker gets full kernel privileges without any +# capability restrictions. +# +# This also works with `--image`: a privileged (default) image-based machine +# runs in an OCI container, but /storage is bind-mounted into it, so the same +# `mount --bind /storage/docker /var/lib/docker` resolves to the ext4 disk just +# as in a bare VM. +# +# ── Kernel requirements ─────────────────────────────────────────────────────── +# +# Requires libkrunfw built from config-libkrunfw_aarch64_lean (or later) with: +# CONFIG_OVERLAY_FS=y — overlayfs (rootfs overlay) +# CONFIG_NETFILTER=y — iptables/nftables for Docker networking +# CONFIG_NF_TABLES=y — nftables netlink (Alpine's iptables-nft) +# CONFIG_NFT_COMPAT=y — iptables compatibility via nftables +# CONFIG_BRIDGE=y — docker0 bridge interface +# +# If the current libkrunfw.5.dylib lacks these, see docs/building-libkrunfw-macos.md. +# +# ── Storage driver note ─────────────────────────────────────────────────────── +# +# Docker's /var/lib/docker MUST be on ext4 (/dev/vda), not on the rootfs +# overlayfs. This is a hard requirement, not a workaround. +# +# Why: the smolvm rootfs overlay uses the initramfs (ramfs) as its lower +# layer. Ramfs has no file-handle support (no exportfs). This propagates up +# through the overlayfs — any directory ON the rootfs overlay (like +# /var/lib/docker) also lacks file-handle support. The kernel then falls +# back to index=off and rejects it as an upper dir for Docker's nested +# overlay, regardless of CONFIG_OVERLAY_FS_REDIRECT_DIR or index=on mount +# options. +# +# Fix: bind-mount /storage/docker → /var/lib/docker before starting dockerd. +# /storage/ is mounted from /dev/vda (ext4), which supports file handles. +# Docker's overlay2 layers then live on ext4 and nest correctly. +# +# This bind-mount must be re-applied after every VM start (mounts don't +# persist across stop/start). The init commands create the directory on +# first start; the "Start dockerd" exec command does the bind-mount each +# time. +# +# ── Create and start ───────────────────────────────────────────────────────── +# +# smolvm machine create --name docker -s examples/docker-in-vm/docker.smolfile --net-backend virtio-net +# smolvm machine start --name docker +# +# ── Start dockerd ───────────────────────────────────────────────────────────── +# +# smolvm machine exec --name docker -- sh -c ' +# mkdir -p /storage/docker /var/lib/docker +# mount --bind /storage/docker /var/lib/docker +# rm -f /var/run/docker.pid +# dockerd --storage-driver=overlay2 >/tmp/dockerd.log 2>&1 & +# for i in $(seq 1 30); do docker info >/dev/null 2>&1 && break; sleep 1; done +# docker info | grep "Storage Driver"' +# +# ── Run containers ──────────────────────────────────────────────────────────── +# +# # Pull an image +# smolvm machine exec --name docker -- docker pull alpine +# +# # Run with bridge networking (default — requires netfilter kernel config) +# smolvm machine exec --name docker -- docker run --rm alpine echo "hello from docker" +# +# # Run with host networking (works even without netfilter) +# smolvm machine exec --name docker -- docker run --rm --network=host alpine wget -qO- https://example.com +# +# # Build an image +# smolvm machine exec --name docker -- sh -c ' +# printf "FROM alpine\nRUN echo built" > /tmp/Dockerfile +# docker build -t myapp /tmp' +# +# ── Interactive shell inside a Docker container ─────────────────────────────── +# +# smolvm machine exec -i --name docker -- docker run --rm -i alpine sh +# +# ── Kubernetes (K3d / kind) ─────────────────────────────────────────────────── +# +# The guest kernel + this setup run k3s. The engine now creates /dev/kmsg in +# the container /dev (the kubelet hard-requires it), so K3d works with no extra +# prep beyond a running dockerd: +# +# smolvm machine exec --name docker -- sh -c ' +# curl -sL https://github.com/k3d-io/k3d/releases/latest/download/k3d-linux-$(uname -m | sed s/aarch64/arm64/;s/x86_64/amd64/) -o /usr/local/bin/k3d +# chmod +x /usr/local/bin/k3d +# k3d cluster create dev --no-lb +# kubectl get nodes' # -> node Ready +# +# ── Cloud (smolmachines) note ───────────────────────────────────────────────── +# +# On the cloud platform each `exec` runs in its own mount namespace, so the +# `mount --bind` above is invisible to dockerd. Point docker's data-root +# straight at the ext4 disk instead: +# +# dockerd --data-root=/storage/docker --storage-driver=overlay2 +# +# ── Stop ───────────────────────────────────────────────────────────────────── +# +# smolvm machine stop --name docker +# +# ───────────────────────────────────────────────────────────────────────────── + +cpus = 2 +memory = 2048 +net = true +storage = 20 + +init = [ + "apk update -q", + # docker: daemon + CLI + "apk add docker -q", + # Bind /var/lib/docker to ext4 storage so overlay2 works (avoids overlay-on-overlay) + "mkdir -p /storage/docker /var/lib/docker", + "mount --bind /storage/docker /var/lib/docker", +] diff --git a/examples/doom-web/doom.smolfile b/examples/doom-web/doom.smolfile new file mode 100644 index 0000000..63f3c0f --- /dev/null +++ b/examples/doom-web/doom.smolfile @@ -0,0 +1,17 @@ +# Doom in a browser, served from a hardware-isolated microVM +# +# Run: +# smolvm machine run -d -s examples/doom-web/doom.smolfile +# open http://localhost:8080 + +image = "python:3.12-alpine" +entrypoint = ["python3"] +cmd = ["-m", "http.server", "8080", "--directory", "/var/www"] + +cpus = 2 +memory = 512 +net = true + +[dev] +volumes = ["./examples/doom-web:/var/www"] +ports = ["8080:8080"] diff --git a/examples/doom-web/index.html b/examples/doom-web/index.html new file mode 100644 index 0000000..6503eab --- /dev/null +++ b/examples/doom-web/index.html @@ -0,0 +1,63 @@ + + + + + + DOOM - smolvm + + + +
+

DOOM — running in smolvm

+

Hardware-isolated microVM • served via port forwarding

+
+ +
+ +
+ Powered by js-dos • + DOOM shareware © id Software +
+ + + + + diff --git a/examples/doubledelete_test.rs b/examples/doubledelete_test.rs new file mode 100644 index 0000000..c7db365 --- /dev/null +++ b/examples/doubledelete_test.rs @@ -0,0 +1,27 @@ +//! Verifies delete_machine is idempotent (double-delete no longer errors). +use smolvm::embedded::{EmbeddedRuntime, MachineSpec}; +use smolvm::VmResources; + +fn main() { + let rt = EmbeddedRuntime::new().expect("runtime"); + let name = "doubledelete-test"; + let _ = rt.delete_machine(name); // clean slate + rt.create_machine(MachineSpec { + name: name.into(), + mounts: vec![], + ports: vec![], + resources: VmResources { + cpus: 1, + memory_mib: 512, + network: false, + ..Default::default() + }, + persistent: false, + }) + .expect("create"); + rt.delete_machine(name).expect("first delete"); + match rt.delete_machine(name) { + Ok(()) => println!("PASS: second delete is idempotent (no error)"), + Err(e) => println!("FAIL: second delete errored: {e}"), + } +} diff --git a/examples/gpu-chrome/gpu-chrome.smolfile b/examples/gpu-chrome/gpu-chrome.smolfile new file mode 100644 index 0000000..4f1abec --- /dev/null +++ b/examples/gpu-chrome/gpu-chrome.smolfile @@ -0,0 +1,38 @@ +# Headless Chrome with GPU acceleration in a hardware-isolated microVM +# +# Runs Chromium with Vulkan rendering via virtio-gpu. Useful for +# Playwright, Puppeteer, or any headless browser workload that needs +# a real GPU (WebGL, Canvas, CSS transforms, screenshots). +# +# ── Quick start ────────────────────────────────────────────────────── +# +# smolvm machine run -it --gpu -s examples/gpu-chrome/gpu-chrome.smolfile +# +# ── Verify GPU works ───────────────────────────────────────────────── +# +# smolvm machine run --gpu -s examples/gpu-chrome/gpu-chrome.smolfile \ +# -- vulkaninfo --summary +# +# ── Host requirements (macOS) ──────────────────────────────────────── +# +# brew install slp/krun/virglrenderer slp/krun/molten-vk +# +# Then rebuild libkrun with GPU support: +# cd libkrun && make BLK=1 NET=1 GPU=1 TIMESYNC=1 +# +# ───────────────────────────────────────────────────────────────────── + +image = "fedora:42" +entrypoint = ["sh", "-c"] +cmd = ["echo 'GPU VM ready — use exec to run commands'"] + +cpus = 4 +memory = 4096 +net = true +gpu = true + +[dev] +init = [ + "dnf copr enable -y slp/mesa-libkrun-vulkan", + "dnf install -y --allowerasing mesa-vulkan-drivers vulkan-tools chromium", +] diff --git a/examples/headless-browser/README.md b/examples/headless-browser/README.md new file mode 100644 index 0000000..4defd49 --- /dev/null +++ b/examples/headless-browser/README.md @@ -0,0 +1,118 @@ +# Headless browsers in smolvm — and pre-warmed browser pools via fork + +`browser.smolfile` covers the basics: a GPU-accelerated headless Chromium you +`create` + `start` + `exec` against. This doc covers the harder-won part — +running Chromium as a **persistent, forkable workload** so you can keep a pool +of pre-warmed browsers and skip cold startup on every task. + +## Why fork a browser at all + +A browser's cold start (process launch + shared-library load + JS-engine init + +first navigation) costs **1–3+ seconds**. A smolvm fork is a copy-on-write clone +of a *running* VM, so a clone inherits the golden's **already-running, +already-initialized browser** — same process, same warmed heap, same listening +CDP port. You pay the cold start **once** (in the golden) and materialize warm +clones in **~50–130 ms** each. + +``` +warm one golden ──fork──▶ clone 1 (browser already up) ~90 ms to agent-ready + ──fork──▶ clone 2 (browser already up) + ──fork──▶ clone N ... +``` + +## The #1 gotcha: fork-friendly Chromium flags + +**Chromium MUST be launched with `--no-zygote` (plus `--no-sandbox` and +`--disable-dev-shm-usage`) for the browser to survive a fork.** + +Chromium's default process model uses a *zygote* — a pre-forked template process +the browser clones renderers from. That model does **not** survive a +cross-process VM fork cleanly: after the fork, joining the restored container +(`crun exec`, which is what `machine exec` does for an image VM) **hangs**. With +`--no-zygote`, Chromium runs a flat process tree that forks and execs correctly +on the clone. + +``` +# fork-friendly: +chromium --headless=new --no-zygote --no-sandbox --disable-dev-shm-usage \ + --remote-debugging-port=9222 --user-data-dir=/tmp/cdata about:blank + +# NOT fork-friendly (the image's bare default CMD, zygote on): +# → golden runs fine, but `machine exec` into a forked clone hangs. +``` + +`--no-sandbox` is required because the microVM guest typically lacks the +user-namespace / setuid-sandbox plumbing Chromium expects; `--disable-dev-shm-usage` +avoids the tiny default `/dev/shm`. + +## Setup: a persistent, forkable browser golden + +The browser has to be **running at fork time**, so launch it as the machine's +persistent workload — not via a one-shot `machine exec` (which is torn down when +the exec returns). Pass the command at `create` (an image machine with no +command instead adopts the image's OCI `CMD`/`ENTRYPOINT`): + +```sh +smolvm machine create --name browser-golden \ + --image chromedp/headless-shell:latest --net \ + --workdir /headless-shell \ + -- ./headless-shell --headless=new --no-zygote --no-sandbox \ + --disable-dev-shm-usage --remote-debugging-port=9222 --user-data-dir=/tmp/cd + +# --forkable enables the CoW-fork machinery (memfd-backed RAM + control socket). +smolvm machine start --name browser-golden --forkable + +# Fork a warm clone on demand (one golden → N clones): +smolvm machine fork --golden browser-golden --name worker-1 +smolvm machine fork --golden browser-golden --name worker-2 + +# Each clone's browser is already up; exec / drive it: +smolvm machine exec --name worker-1 -- /bin/sh -c 'echo ready' +``` + +The golden stays **frozen** as the CoW base while clones exist — don't `start` +it again until the clones are gone. + +## What survives the fork (and what doesn't) + +| Preserved (it's in the restored RAM/CoW disk) | NOT preserved | +|---|---| +| The running browser process + warmed heap / JIT | **Live network connections** (reset on restore — freeze the golden at an *idle* point, not mid-request) | +| Loaded shared libraries, parsed/blank page | **GPU renderer context** (host-side virgl/Venus state isn't transferred; headless / software rendering is unaffected) | +| The listening CDP port (`127.0.0.1:9222`) | Wall-clock-sensitive timers can jump forward by the freeze duration | +| Open file handles into the rootfs | | + +Per-clone identity is rejuvenated automatically (distinct hostname, fresh +entropy), and each clone gets its own CoW disk overlay, so clones are isolated. + +## Driving the warm browser (CDP) + +Chromium binds the DevTools endpoint to **`127.0.0.1`** regardless of +`--remote-debugging-address` (security hardening), so reach it from *inside* the +VM (the agent shares the guest network namespace) or proxy it out. + +Use a real CDP/WebSocket client — **puppeteer** or **chrome-remote-interface** — +not raw `curl`/`socat`. A raw TCP client connects but won't get the DevTools +JSON body back reliably, and the WebSocket CDP path needs +`--remote-allow-origins=*`. The HTTP `/json/*` endpoints also require a +`Host: localhost` header (DNS-rebinding protection). + +## Performance + +Measured on Apple M4 Max (APFS, instant `clonefile` CoW disks): + +- End-to-end `machine fork` to a usable, agent-reachable warm clone: **~50–130 ms**. +- Clone boot-from-snapshot to agent-ready: **~90 ms**. +- Density: golden ~300 MB RSS, each clone only ~30–40 MB (RAM CoW-shared). + +On Linux the per-fork time is the same class on a reflink-capable filesystem +(btrfs/xfs); on ext4 the disk copy dominates (~0.4–0.9 s). + +## Status + +Forking (`--forkable`, `machine fork`) is part of the fast-fork work and is +validated on both Linux/KVM (x86_64) and macOS/HVF (aarch64). The persistent +warm browser → fork → drive path is validated end-to-end with the flags above. +The cross-arch caveat is fundamental: a clone runs on the **same CPU arch + +hypervisor + host** as its golden (a fork is a live CoW clone, not a portable +snapshot). For portable, cold artifacts use `smolvm pack` instead. diff --git a/examples/headless-browser/browser.smolfile b/examples/headless-browser/browser.smolfile new file mode 100644 index 0000000..bbaecfe --- /dev/null +++ b/examples/headless-browser/browser.smolfile @@ -0,0 +1,68 @@ +# Headless Chromium with GPU acceleration +# +# For pre-warmed browser pools via fork (warm-state preservation, the required +# --no-zygote/--no-sandbox flags, persistent-workload setup, and CDP access), +# see README.md in this directory. +# +# Guest Vulkan (Venus/virtio-gpu) forwards calls over the virtio-gpu transport +# to the host GPU. ANGLE inside Chromium uses Venus as its Vulkan backend, so +# WebGL and compositing run on real hardware — no SwiftShader fallback. +# +# Confirmed GL renderer: +# ANGLE (Intel, Vulkan 1.4.307 (Virtio-GPU Venus (Intel(R) UHD Graphics 620 +# (KBL GT2)) (0x00005917)), venus) +# +# ── Setup (one time) ───────────────────────────────────────────────────────── +# +# smolvm machine create --name browser -s examples/headless-browser/browser.smolfile +# smolvm machine start --name browser +# +# ── Take a screenshot ──────────────────────────────────────────────────────── +# +# smolvm machine exec --name browser -- \ +# chromium --headless=new --no-sandbox --disable-dev-shm-usage \ +# --use-gl=angle --use-angle=vulkan \ +# --screenshot=/tmp/out.png --window-size=1280,800 \ +# https://example.com +# +# # Copy screenshot to host +# smolvm machine exec --name browser -- base64 /tmp/out.png | base64 -d > out.png +# +# ── Verify GPU renderer ─────────────────────────────────────────────────────── +# +# smolvm machine exec --name browser -- sh -c ' +# vulkaninfo --summary 2>/dev/null | grep deviceName +# ' +# # → deviceName = Virtio-GPU Venus (Intel(R) UHD Graphics 620 ...) +# +# ── Stop ───────────────────────────────────────────────────────────────────── +# +# smolvm machine stop --name browser +# +# ───────────────────────────────────────────────────────────────────────────── + +# mesa-vulkan-virtio (the guest virtio Vulkan ICD) is not in Alpine 3.21 stable. +# Use edge until Alpine 3.22 is released. +image = "alpine:edge" + +cpus = 4 +memory = 4096 +gpu = true +gpu_vram = 2048 +net = true + +# Venus Vulkan ICD — tells the Vulkan loader where to find the virtio GPU driver. +# The path is architecture-specific. Change x86_64 → aarch64 on ARM hosts. +# +# x86_64: VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/virtio_icd.x86_64.json +# aarch64: VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/virtio_icd.aarch64.json +env = [ + "VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/virtio_icd.x86_64.json", +] + +[dev] +# Install Chromium and the Mesa Venus Vulkan driver. +# apk is idempotent — subsequent starts skip packages already on disk. +init = [ + "apk add --no-cache chromium chromium-angle mesa-vulkan-virtio vulkan-loader font-opensans", +] diff --git a/examples/largeoutput_test.rs b/examples/largeoutput_test.rs new file mode 100644 index 0000000..14d35af --- /dev/null +++ b/examples/largeoutput_test.rs @@ -0,0 +1,67 @@ +//! Verifies the large-output guard: a huge non-streaming exec returns a CLEAR +//! error (not a SIGPIPE-truncated result) and leaves the machine HEALTHY. +use smolvm::embedded::{EmbeddedRuntime, MachineSpec}; +use smolvm::VmResources; + +fn main() { + let rt = EmbeddedRuntime::new().expect("runtime"); + let name = "largeoutput-test"; + let _ = rt.delete_machine(name); + let _ = rt.create_machine(MachineSpec { + name: name.into(), + mounts: vec![], + ports: vec![], + resources: VmResources { + cpus: 1, + memory_mib: 1024, + network: false, + ..Default::default() + }, + persistent: false, + }); + rt.start_machine(name).expect("start"); + // small output: works + let (c, out, _) = rt + .exec( + name, + vec!["echo".into(), "small".into()], + vec![], + None, + None, + ) + .unwrap(); + println!( + "small exec: exit={c} out={:?}", + String::from_utf8_lossy(&out).trim() + ); + // ~40 MiB output: expect a clean error, not a truncated/SIGPIPE result + match rt.exec( + name, + vec![ + "sh".into(), + "-c".into(), + "head -c 41943040 /dev/zero | tr '\\0' a".into(), + ], + vec![], + None, + None, + ) { + Ok((c, out, _)) => println!("BIG exec: UNEXPECTED ok exit={c} len={}", out.len()), + Err(e) => println!("BIG exec: clean error -> {e}"), + } + // machine still healthy? + match rt.exec( + name, + vec!["echo".into(), "after".into()], + vec![], + None, + None, + ) { + Ok((_, out, _)) => println!( + "after BIG: HEALTHY out={:?}", + String::from_utf8_lossy(&out).trim() + ), + Err(e) => println!("after BIG: POISONED -> {e}"), + } + rt.delete_machine(name).expect("delete"); +} diff --git a/examples/local-llm/local-llm.smolfile b/examples/local-llm/local-llm.smolfile new file mode 100644 index 0000000..7015724 --- /dev/null +++ b/examples/local-llm/local-llm.smolfile @@ -0,0 +1,98 @@ +# Local LLM inference with GPU acceleration +# +# Runs llama.cpp with the Vulkan backend inside a microVM, forwarding compute +# to the host GPU via virtio-gpu → virglrenderer → MoltenVK → Metal. +# +# Confirmed on Apple M4 Max: +# Vulkan0: Virtio-GPU Venus (Apple M4 Max) (36864 MiB, 36864 MiB free) +# Prompt: 634.3 t/s | Generation: 170.2 t/s (Qwen2-0.5B Q4_K_M, -ngl 99) +# +# ── Setup (runs once, takes ~5 min to build llama.cpp) ─────────────────────── +# +# smolvm machine create --name llm -s examples/local-llm/local-llm.smolfile +# smolvm machine start --name llm +# +# ── Download a model ──────────────────────────────────────────────────────── +# +# # Download directly into the VM (stored on its persistent overlay disk): +# +# smolvm machine exec --name llm -- sh -c \ +# 'mkdir -p /models && curl -L -o /models/qwen2-0.5b-q4_k_m.gguf \ +# https://huggingface.co/Qwen/Qwen2-0.5B-Instruct-GGUF/resolve/main/qwen2-0_5b-instruct-q4_k_m.gguf' +# +# # Or for Llama 3.2 3B (better quality, ~2 GB): +# smolvm machine exec --name llm -- sh -c \ +# 'mkdir -p /models && curl -L -o /models/llama-3.2-3b-q4_k_m.gguf \ +# https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF/resolve/main/Llama-3.2-3B-Instruct-Q4_K_M.gguf' +# +# ── Run inference ──────────────────────────────────────────────────────────── +# +# smolvm machine exec --name llm -- sh -c \ +# '/opt/llama.cpp/build/bin/llama-completion \ +# -m /models/qwen2-0.5b-q4_k_m.gguf \ +# -ngl 99 --temp 0.8 -n 256 \ +# -p "Explain the Venus Vulkan driver in one paragraph." \ +# 2>&1 1>/tmp/resp.txt | grep -E "prompt eval|eval time"; cat /tmp/resp.txt' +# +# # Prints prompt and generation speed (tokens/sec), then the response. +# # Drop the grep pipe and use 2>/dev/null instead to suppress all stats. +# +# ── Verify GPU is active ───────────────────────────────────────────────────── +# +# smolvm machine exec --name llm -- vulkaninfo --summary +# # → deviceName = Virtio-GPU Venus (Apple M4 Max) +# +# ── Stop ───────────────────────────────────────────────────────────────────── +# +# smolvm machine stop --name llm +# +# ───────────────────────────────────────────────────────────────────────────── + +image = "fedora:42" + +cpus = 8 +memory = 8192 +gpu = true +gpu_vram = 4096 +net = true + +# Venus Vulkan ICD — tells the Vulkan loader to use the virtio GPU driver. +# The path is architecture-specific: +# +# aarch64 (Apple Silicon, ARM): virtio_icd.aarch64.json ← default below +# x86_64 (Intel/AMD): virtio_icd.x86_64.json +# +env = [ + "VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/virtio_icd.aarch64.json", +] + +[dev] +# Mount your local model directory into /models inside the VM. +# Set the host path to wherever you store .gguf files, for example: +# +# volumes = ["~/.cache/smolvm/models:/models"] +# +# Without this, copy models in with: +# smolvm machine exec --name llm -- sh -c 'cat > /tmp/model.gguf' < ./model.gguf + +init = [ + # Enable the patched Mesa Venus driver from COPR. + # Required for Apple Silicon (fixes 16 KB page-size alignment in virglrenderer). + "dnf copr enable -y slp/mesa-libkrun-vulkan", + + # Install Mesa Vulkan driver + llama.cpp build dependencies. + # vulkan-loader-devel provides libvulkan.so and for CMake's FindVulkan. + # glslc and spirv-*-devel are needed to compile Vulkan compute shaders. + "dnf install -y --allowerasing --quiet mesa-vulkan-drivers vulkan-loader-devel vulkan-tools git cmake gcc gcc-c++ glslc spirv-headers-devel spirv-tools-devel", + + # Swap the stock mesa-vulkan-drivers for the slp COPR build so the Apple + # Silicon 16 KB page-size patch is active. + "dnf swap -y --quiet mesa-vulkan-drivers mesa-vulkan-drivers --repo 'copr:copr.fedorainfracloud.org:slp:mesa-libkrun-vulkan'", + + # Clone llama.cpp (skip if already present). + "test -d /opt/llama.cpp || git clone --depth 1 https://github.com/ggml-org/llama.cpp /opt/llama.cpp", + + # Build llama.cpp with the Vulkan backend (skip if binary already exists). + # First run takes ~5 minutes; subsequent starts are instant. + "test -x /opt/llama.cpp/build/bin/llama-cli || (cmake -B /opt/llama.cpp/build -S /opt/llama.cpp -DGGML_VULKAN=ON -DCMAKE_BUILD_TYPE=Release -DLLAMA_BUILD_TESTS=OFF && cmake --build /opt/llama.cpp/build --target llama-cli llama-completion llama-server -j$(nproc))", +] diff --git a/examples/node-app/node.smolfile b/examples/node-app/node.smolfile new file mode 100644 index 0000000..cf156f5 --- /dev/null +++ b/examples/node-app/node.smolfile @@ -0,0 +1,17 @@ +# Node.js dev environment in a hardware-isolated microVM +# +# smolvm machine run -s examples/node-app/node.smolfile +# smolvm machine create --name dev -s node.smolfile +# smolvm machine start --name dev +# smolvm machine exec --name dev -- node -e "console.log('hello')" + +image = "node:22-alpine" +entrypoint = ["node"] + +cpus = 2 +memory = 512 +net = true + +[dev] +ports = ["3000:3000"] +init = ["npm install -g nodemon"] diff --git a/examples/openclaw-app/openclaw.smolfile b/examples/openclaw-app/openclaw.smolfile new file mode 100644 index 0000000..2eb1a6b --- /dev/null +++ b/examples/openclaw-app/openclaw.smolfile @@ -0,0 +1,33 @@ +# OpenClaw — LLM gateway in a hardware-isolated microVM +# +# OpenClaw proxies requests to OpenAI, Anthropic, Google, Mistral, etc. +# Running it inside a microVM means all outbound traffic is restricted +# to only the LLM provider APIs listed below — no data exfiltration possible. +# +# ── Quick start ────────────────────────────────────────────────────── +# +# smolvm machine run -d -s openclaw.smolfile +# curl http://localhost:18789/health +# curl http://localhost:18789/v1/chat/completions \ +# -H "Authorization: Bearer $OPENAI_API_KEY" \ +# -H "Content-Type: application/json" \ +# -d '{"model":"gpt-4o","messages":[{"role":"user","content":"hello"}]}' +# +# ── Interact with the running machine ──────────────────────────────── +# +# smolvm machine ls # see running machines +# smolvm machine exec -it -- sh # open a shell inside the VM +# smolvm machine stop # shut it down +# +# ───────────────────────────────────────────────────────────────────── + +image = "alpine/openclaw:main" +entrypoint = ["openclaw"] +cmd = ["gateway", "--port", "18789", "--allow-unconfigured"] + +cpus = 2 +memory = 1024 +net = true + +[dev] +ports = ["18789:18789"] diff --git a/examples/orphan_test.rs b/examples/orphan_test.rs new file mode 100644 index 0000000..039078e --- /dev/null +++ b/examples/orphan_test.rs @@ -0,0 +1,42 @@ +//! Manual verification harness for the parent-death watchdog (orphaned-VMM leak). +//! +//! Mimics an SDK embedder: it owns a VM in-process (sets SMOLVM_BOOT_BINARY, +//! never detaches), prints its pid, then blocks forever. The driver script +//! `kill -9`s this process to simulate a host crash and asserts the `_boot-vm` +//! VMM exits instead of orphaning. Run via `cargo run --example orphan_test`. +use smolvm::embedded::{EmbeddedRuntime, MachineSpec}; +use smolvm::VmResources; + +fn main() { + let name = std::env::var("ORPHAN_TEST_VM").unwrap_or_else(|_| "orphan-test".into()); + let rt = EmbeddedRuntime::new().expect("create embedded runtime"); + + let spec = MachineSpec { + name: name.clone(), + mounts: vec![], + ports: vec![], + resources: VmResources { + cpus: 1, + memory_mib: 512, + network: true, + ..Default::default() + }, + persistent: false, + }; + let _ = rt.create_machine(spec); // ignore "already exists" on reruns + rt.start_machine(&name).expect("start machine"); + + let pid = rt.pid(&name).unwrap_or(-1); + // Sentinel the driver greps for: embedder pid + the VMM child pid. + println!( + "ORPHAN_TEST_READY embedder={} vmm={}", + std::process::id(), + pid + ); + use std::io::Write; + let _ = std::io::stdout().flush(); + + loop { + std::thread::sleep(std::time::Duration::from_secs(3600)); + } +} diff --git a/examples/python-app/python.smolfile b/examples/python-app/python.smolfile new file mode 100644 index 0000000..c48118e --- /dev/null +++ b/examples/python-app/python.smolfile @@ -0,0 +1,17 @@ +# Python dev environment in a hardware-isolated microVM +# +# smolvm machine run -s examples/python-app/python.smolfile +# smolvm machine create --name dev -s python.smolfile +# smolvm machine start --name dev +# smolvm machine exec --name dev -- python3 --version + +image = "python:3.12-alpine" +entrypoint = ["python3"] + +cpus = 2 +memory = 512 +net = true + +[dev] +ports = ["8080:8080"] +init = ["pip install --no-cache-dir ipython"] diff --git a/examples/readfile_fix_test.rs b/examples/readfile_fix_test.rs new file mode 100644 index 0000000..0b0eed4 --- /dev/null +++ b/examples/readfile_fix_test.rs @@ -0,0 +1,71 @@ +//! Verifies the read_file non-regular-file fix: reading a directory or an +//! unbounded special file must return a CLEAN error and NOT poison the agent +//! connection (previously a dir read bricked the machine; /dev/zero hung). +use smolvm::embedded::{EmbeddedRuntime, MachineSpec}; +use smolvm::VmResources; +use std::time::Duration; + +fn main() { + let rt = EmbeddedRuntime::new().expect("runtime"); + let name = "readfile-fix-test"; + let _ = rt.create_machine(MachineSpec { + name: name.into(), + mounts: vec![], + ports: vec![], + resources: VmResources { + cpus: 1, + memory_mib: 512, + network: false, + ..Default::default() + }, + persistent: false, + }); + rt.start_machine(name).expect("start"); + + let echo = |label: &str| match rt.exec( + name, + vec!["echo".into(), label.into()], + vec![], + None, + Some(Duration::from_secs(10)), + ) { + Ok((code, out, _)) => println!( + " {label}: HEALTHY exit={code} out={:?}", + String::from_utf8_lossy(&out).trim() + ), + Err(e) => println!(" {label}: POISONED → {e}"), + }; + + echo("before"); + match rt.read_file(name, "/tmp") { + Ok(b) => println!("readFile(/tmp dir): UNEXPECTED ok ({} bytes)", b.len()), + Err(e) => println!("readFile(/tmp dir): clean error → {e}"), + } + echo("after-dir"); + match rt.read_file(name, "/dev/zero") { + Ok(b) => println!("readFile(/dev/zero): UNEXPECTED ok ({} bytes)", b.len()), + Err(e) => println!("readFile(/dev/zero): clean error → {e}"), + } + echo("after-devzero"); + // sanity: a real file still reads + let _ = rt.exec( + name, + vec![ + "sh".into(), + "-c".into(), + "echo content > /tmp/real.txt".into(), + ], + vec![], + None, + None, + ); + match rt.read_file(name, "/tmp/real.txt") { + Ok(b) => println!( + "readFile(/tmp/real.txt): ok → {:?}", + String::from_utf8_lossy(&b).trim() + ), + Err(e) => println!("readFile(/tmp/real.txt): UNEXPECTED error → {e}"), + } + + rt.delete_machine(name).expect("delete"); +} diff --git a/examples/reattach_test.rs b/examples/reattach_test.rs new file mode 100644 index 0000000..1213078 --- /dev/null +++ b/examples/reattach_test.rs @@ -0,0 +1,64 @@ +//! Verifies start-or-reconnect for a stopped persistent machine — the path the +//! SDK's local `Machine.connect()` now uses (native connect → start_machine). +use smolvm::embedded::{EmbeddedRuntime, MachineSpec}; +use smolvm::VmResources; + +fn main() { + let rt = EmbeddedRuntime::new().expect("runtime"); + let name = "reattach-test"; + let _ = rt.delete_machine(name); // clean slate + + rt.create_machine(MachineSpec { + name: name.into(), + mounts: vec![], + ports: vec![], + resources: VmResources { + cpus: 1, + memory_mib: 512, + network: false, + ..Default::default() + }, + persistent: true, + }) + .expect("create"); + rt.start_machine(name).expect("start"); + rt.exec( + name, + vec![ + "sh".into(), + "-c".into(), + "echo survived-restart > /root/persist.txt".into(), + ], + vec![], + None, + None, + ) + .expect("write"); + println!("wrote /root/persist.txt; stopping machine..."); + rt.stop_machine(name).expect("stop"); + + println!("reattaching to stopped persistent machine via start_machine() ..."); + rt.start_machine(name) + .expect("reattach (start-or-reconnect)"); + let (code, out, _) = rt + .exec( + name, + vec!["cat".into(), "/root/persist.txt".into()], + vec![], + None, + None, + ) + .expect("read after reattach"); + let content = String::from_utf8_lossy(&out); + println!("readback: exit={code} content={:?}", content.trim()); + println!( + "{}", + if content.trim() == "survived-restart" { + "RESULT: PASS — reattached + data persisted" + } else { + "RESULT: FAIL" + } + ); + + rt.delete_machine(name).expect("delete"); +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..5a4b24e --- /dev/null +++ b/flake.lock @@ -0,0 +1,63 @@ +{ + "nodes": { + "libkrun-src": { + "flake": false, + "locked": { + "lastModified": 1779235645, + "narHash": "sha256-rmRMZ5j3HlueCAXQLydjmMq45+hwhxSLWM02VvHokcY=", + "owner": "smol-machines", + "repo": "libkrun", + "rev": "98163265197caa24a789699f16a68b98e917b65b", + "type": "github" + }, + "original": { + "owner": "smol-machines", + "repo": "libkrun", + "rev": "98163265197caa24a789699f16a68b98e917b65b", + "type": "github" + } + }, + "libkrunfw-src": { + "flake": false, + "locked": { + "lastModified": 1778821066, + "narHash": "sha256-g2DxDnnh52JCPg4ktPWFBi0J9203FCncGOzB3eiCRxQ=", + "owner": "smol-machines", + "repo": "libkrunfw", + "rev": "516ceece6aed60ccc84ac8faa459885062e39400", + "type": "github" + }, + "original": { + "owner": "smol-machines", + "repo": "libkrunfw", + "rev": "516ceece6aed60ccc84ac8faa459885062e39400", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1779357205, + "narHash": "sha256-cCO8aTqss5x9Ky8GWkpY0Hy5fyTZEbtifSUV8QjSzic=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f83fc3c307e74bc5fd5adb7eb6b8b13ffd2a36e1", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "libkrun-src": "libkrun-src", + "libkrunfw-src": "libkrunfw-src", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..36e2f78 --- /dev/null +++ b/flake.nix @@ -0,0 +1,92 @@ +{ + description = "Ship and run software with isolation by default."; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + libkrun-src = { + url = "github:smol-machines/libkrun/98163265197caa24a789699f16a68b98e917b65b"; + flake = false; + }; + + libkrunfw-src = { + url = "github:smol-machines/libkrunfw/516ceece6aed60ccc84ac8faa459885062e39400"; + flake = false; + }; + }; + + outputs = + { + self, + nixpkgs, + libkrun-src, + libkrunfw-src, + ... + }: + let + forAllSystems = + function: + nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed ( + system: function (nixpkgs.legacyPackages.${system}.extend overlay) + ); + + overlay = final: prev: { + smolvm-libkrunfw = final.callPackage ./nix/libkrunfw.nix { + src = libkrunfw-src; + }; + + smolvm-libkrun = final.callPackage ./nix/libkrun.nix { + src = libkrun-src; + libkrunfw = final.smolvm-libkrunfw; + withBlk = true; + withNet = true; + withGpu = final.stdenv.hostPlatform.isLinux; + }; + + smolvm = final.callPackage ./nix/smolvm.nix { }; + }; + in + { + overlays.default = overlay; + + formatter = forAllSystems (pkgs: pkgs.alejandra); + + packages = forAllSystems (pkgs: { + libkrunfw = pkgs.smolvm-libkrunfw; + libkrun = pkgs.smolvm-libkrun; + smolvm = pkgs.smolvm; + default = pkgs.smolvm; + }); + + apps = forAllSystems (pkgs: { + default = { + type = "app"; + program = "${self.packages.${pkgs.stdenv.hostPlatform.system}.default}/bin/smolvm"; + }; + }); + + devShells = forAllSystems (pkgs: { + default = pkgs.mkShell { + inputsFrom = [ + self.packages.${pkgs.stdenv.hostPlatform.system}.libkrun + ]; + + packages = with pkgs; [ + cargo + rustc + rustfmt + clippy + rust-analyzer + cargo-make + pkg-config + ]; + + LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib"; + LIBKRUN_DIR = "${pkgs.lib.getLib self.packages.${pkgs.stdenv.hostPlatform.system}.libkrun}/${ + if pkgs.stdenv.hostPlatform.isDarwin then "lib" else "lib64" + }"; + RUST_SRC_PATH = "${pkgs.rustPlatform.rustLibSrc}"; + }; + }); + }; +} diff --git a/lib/libMoltenVK.dylib b/lib/libMoltenVK.dylib new file mode 100644 index 0000000..574b9c8 --- /dev/null +++ b/lib/libMoltenVK.dylib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:945c9d86e886edc19a9cc3975336f4c67279d3ca884bf55638b041d8921fd16f +size 5109344 diff --git a/lib/libepoxy.0.dylib b/lib/libepoxy.0.dylib new file mode 100644 index 0000000..6944e72 --- /dev/null +++ b/lib/libepoxy.0.dylib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4907b7a0cf4ef3515bde843dc187f22482d142898f6c44f2859b59c6298e3324 +size 1188496 diff --git a/lib/libkrun.1.dylib b/lib/libkrun.1.dylib new file mode 120000 index 0000000..2199424 --- /dev/null +++ b/lib/libkrun.1.dylib @@ -0,0 +1 @@ +libkrun.dylib \ No newline at end of file diff --git a/lib/libkrun.dylib b/lib/libkrun.dylib new file mode 100755 index 0000000..f8c8b2a --- /dev/null +++ b/lib/libkrun.dylib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26f61a8ba3795c008dbc0fdfb20bef47d2e0f13d0d6572d56fb1560d38632b3a +size 6158304 diff --git a/lib/libkrun.provenance b/lib/libkrun.provenance new file mode 100644 index 0000000..7a14721 --- /dev/null +++ b/lib/libkrun.provenance @@ -0,0 +1,6 @@ +# Auto-generated by scripts/stamp-libkrun-provenance.sh — do not edit by hand. +# The libkrun/libkrunfw submodule commits the bundled libs in this dir were +# built from. Verified against the pinned submodules by +# scripts/check-libkrun-provenance.sh (run in CI). +libkrun=b982e75995def5751a13dccee49a5f4731d648f5 +libkrunfw=37f73dadb7e64610642d7041c163b8dbf0e9a1ef diff --git a/lib/libkrunfw.5.dylib b/lib/libkrunfw.5.dylib new file mode 100755 index 0000000..847501c --- /dev/null +++ b/lib/libkrunfw.5.dylib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ddcca79131bdbbd3f1d4dc2832459d990c303252260ff1329c0321d0e955935 +size 13118480 diff --git a/lib/libkrunfw.dylib b/lib/libkrunfw.dylib new file mode 120000 index 0000000..42737c6 --- /dev/null +++ b/lib/libkrunfw.dylib @@ -0,0 +1 @@ +libkrunfw.5.dylib \ No newline at end of file diff --git a/lib/libvirglrenderer.1.dylib b/lib/libvirglrenderer.1.dylib new file mode 100644 index 0000000..3eb485b --- /dev/null +++ b/lib/libvirglrenderer.1.dylib @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8a4b62f9b9c37de3bbe093eeccfd39c575378e3cb752c916d1a27f1bcbb9352 +size 1656832 diff --git a/lib/linux-aarch64/libkrun.provenance b/lib/linux-aarch64/libkrun.provenance new file mode 100644 index 0000000..7a14721 --- /dev/null +++ b/lib/linux-aarch64/libkrun.provenance @@ -0,0 +1,6 @@ +# Auto-generated by scripts/stamp-libkrun-provenance.sh — do not edit by hand. +# The libkrun/libkrunfw submodule commits the bundled libs in this dir were +# built from. Verified against the pinned submodules by +# scripts/check-libkrun-provenance.sh (run in CI). +libkrun=b982e75995def5751a13dccee49a5f4731d648f5 +libkrunfw=37f73dadb7e64610642d7041c163b8dbf0e9a1ef diff --git a/lib/linux-aarch64/libkrun.so b/lib/linux-aarch64/libkrun.so new file mode 100755 index 0000000..f195814 --- /dev/null +++ b/lib/linux-aarch64/libkrun.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93723b1580b8d01893e8f7517182fd7331da90cd83f52576e868f5b29da59b3a +size 6731808 diff --git a/lib/linux-aarch64/libkrun.so.2 b/lib/linux-aarch64/libkrun.so.2 new file mode 120000 index 0000000..879bdab --- /dev/null +++ b/lib/linux-aarch64/libkrun.so.2 @@ -0,0 +1 @@ +libkrun.so \ No newline at end of file diff --git a/lib/linux-aarch64/libkrun.so.2.0.0 b/lib/linux-aarch64/libkrun.so.2.0.0 new file mode 100755 index 0000000..f195814 --- /dev/null +++ b/lib/linux-aarch64/libkrun.so.2.0.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93723b1580b8d01893e8f7517182fd7331da90cd83f52576e868f5b29da59b3a +size 6731808 diff --git a/lib/linux-aarch64/libkrunfw.so b/lib/linux-aarch64/libkrunfw.so new file mode 120000 index 0000000..0d3f125 --- /dev/null +++ b/lib/linux-aarch64/libkrunfw.so @@ -0,0 +1 @@ +libkrunfw.so.5 \ No newline at end of file diff --git a/lib/linux-aarch64/libkrunfw.so.5 b/lib/linux-aarch64/libkrunfw.so.5 new file mode 120000 index 0000000..982c50e --- /dev/null +++ b/lib/linux-aarch64/libkrunfw.so.5 @@ -0,0 +1 @@ +libkrunfw.so.5.5.0 \ No newline at end of file diff --git a/lib/linux-aarch64/libkrunfw.so.5.5.0 b/lib/linux-aarch64/libkrunfw.so.5.5.0 new file mode 100755 index 0000000..29d3d2c --- /dev/null +++ b/lib/linux-aarch64/libkrunfw.so.5.5.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7da48d831e85c9908d713dd24f860a1324f4554a63804c3cc7e8a90dcf3dcfe3 +size 13239912 diff --git a/lib/linux-x86_64/libkrun.provenance b/lib/linux-x86_64/libkrun.provenance new file mode 100644 index 0000000..7a14721 --- /dev/null +++ b/lib/linux-x86_64/libkrun.provenance @@ -0,0 +1,6 @@ +# Auto-generated by scripts/stamp-libkrun-provenance.sh — do not edit by hand. +# The libkrun/libkrunfw submodule commits the bundled libs in this dir were +# built from. Verified against the pinned submodules by +# scripts/check-libkrun-provenance.sh (run in CI). +libkrun=b982e75995def5751a13dccee49a5f4731d648f5 +libkrunfw=37f73dadb7e64610642d7041c163b8dbf0e9a1ef diff --git a/lib/linux-x86_64/libkrun.so b/lib/linux-x86_64/libkrun.so new file mode 100755 index 0000000..716b743 --- /dev/null +++ b/lib/linux-x86_64/libkrun.so @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56bfc7b654b82acc4737563dda5fa786fb7212bb59109404cc67e79fce21da72 +size 7837488 diff --git a/lib/linux-x86_64/libkrun.so.2 b/lib/linux-x86_64/libkrun.so.2 new file mode 120000 index 0000000..879bdab --- /dev/null +++ b/lib/linux-x86_64/libkrun.so.2 @@ -0,0 +1 @@ +libkrun.so \ No newline at end of file diff --git a/lib/linux-x86_64/libkrun.so.2.0.0 b/lib/linux-x86_64/libkrun.so.2.0.0 new file mode 100755 index 0000000..716b743 --- /dev/null +++ b/lib/linux-x86_64/libkrun.so.2.0.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56bfc7b654b82acc4737563dda5fa786fb7212bb59109404cc67e79fce21da72 +size 7837488 diff --git a/lib/linux-x86_64/libkrunfw.so b/lib/linux-x86_64/libkrunfw.so new file mode 120000 index 0000000..0d3f125 --- /dev/null +++ b/lib/linux-x86_64/libkrunfw.so @@ -0,0 +1 @@ +libkrunfw.so.5 \ No newline at end of file diff --git a/lib/linux-x86_64/libkrunfw.so.5 b/lib/linux-x86_64/libkrunfw.so.5 new file mode 120000 index 0000000..982c50e --- /dev/null +++ b/lib/linux-x86_64/libkrunfw.so.5 @@ -0,0 +1 @@ +libkrunfw.so.5.5.0 \ No newline at end of file diff --git a/lib/linux-x86_64/libkrunfw.so.5.5.0 b/lib/linux-x86_64/libkrunfw.so.5.5.0 new file mode 100755 index 0000000..a33cccf --- /dev/null +++ b/lib/linux-x86_64/libkrunfw.so.5.5.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a50055fa114459642ae45e4492f81a5ced2e4cbc5226eb715cb34c637c6b8e5 +size 14878392 diff --git a/lib/windows-x86_64/krun.dll b/lib/windows-x86_64/krun.dll new file mode 100755 index 0000000..811b69c --- /dev/null +++ b/lib/windows-x86_64/krun.dll @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3178b9c27270801420173aa3bc811de2ac6b8ab1a74120b77e41e1dbb0922b63 +size 10104091 diff --git a/lib/windows-x86_64/libkrun.provenance b/lib/windows-x86_64/libkrun.provenance new file mode 100644 index 0000000..7a14721 --- /dev/null +++ b/lib/windows-x86_64/libkrun.provenance @@ -0,0 +1,6 @@ +# Auto-generated by scripts/stamp-libkrun-provenance.sh — do not edit by hand. +# The libkrun/libkrunfw submodule commits the bundled libs in this dir were +# built from. Verified against the pinned submodules by +# scripts/check-libkrun-provenance.sh (run in CI). +libkrun=b982e75995def5751a13dccee49a5f4731d648f5 +libkrunfw=37f73dadb7e64610642d7041c163b8dbf0e9a1ef diff --git a/lib/windows-x86_64/libkrunfw.dll b/lib/windows-x86_64/libkrunfw.dll new file mode 100644 index 0000000..86b3c89 --- /dev/null +++ b/lib/windows-x86_64/libkrunfw.dll @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d07efdc977572b50797c5cc9656d2d3ba9db06f54568a59bea402d5c9d58bdb6 +size 19338676 diff --git a/nix/libkrun.nix b/nix/libkrun.nix new file mode 100644 index 0000000..5d502fb --- /dev/null +++ b/nix/libkrun.nix @@ -0,0 +1,155 @@ +{ + lib, + stdenv, + rustPlatform, + src, + cargo, + curl, + cacert, + pkg-config, + glibc, + openssl, + libcap_ng, + libepoxy, + libdrm, + pipewire, + virglrenderer, + libkrunfw, + rustc, + pkgsCross, + darwin, + apple-sdk_15 ? null, + libiconv, + libarchive ? null, + libnsm ? null, + withBlk ? false, + withNet ? false, + withGpu ? false, + withSound ? false, + withInput ? false, + withTimesync ? false, + withAwsNitro ? false, + variant ? null, +}: + +assert lib.elem variant [ + null + "sev" + "tdx" +]; +assert withAwsNitro -> variant == null; +assert withAwsNitro -> stdenv.hostPlatform.isLinux; +assert withAwsNitro -> libarchive != null && libnsm != null; + +let + linuxCrossPkgs = if stdenv.hostPlatform.isAarch64 + then pkgsCross.aarch64-multiplatform + else pkgsCross.gnu64; + libkrunfw' = (libkrunfw.override { inherit variant; }); +in +stdenv.mkDerivation (finalAttrs: { + pname = "libkrun" + + lib.optionalString (variant != null) "-${variant}" + + lib.optionalString withAwsNitro "-awsnitro"; + version = "1.17.3"; + + inherit src; + + outputs = [ + "out" + "dev" + ]; + + cargoDeps = rustPlatform.fetchCargoVendor { + inherit (finalAttrs) src; + hash = "sha256-b6Rk6ljpHNKwVD7r/NhppGE1TLKRXo8MxZNAszcOH5I="; + }; + + postPatch = '' + substituteInPlace Makefile \ + --replace 'cargo build --release $(FEATURE_FLAGS)' 'cargo build -p libkrun --release $(FEATURE_FLAGS)' \ + --replace 'cargo build $(FEATURE_FLAGS)' 'cargo build -p libkrun $(FEATURE_FLAGS)' + '' + lib.optionalString stdenv.hostPlatform.isDarwin '' + substituteInPlace Makefile \ + --replace 'CC_LINUX=$(CLANG) -target $(GCC_TRIPLET) -fuse-ld=lld -Wl,-strip-debug --sysroot $(abspath $(SYSROOT_LINUX)) -B$(GCC_LIB_DIR) -L$(GCC_LIB_DIR) -Wno-c23-extensions' 'CC_LINUX=${linuxCrossPkgs.stdenv.cc}/bin/${linuxCrossPkgs.stdenv.cc.targetPrefix}gcc -L${linuxCrossPkgs.glibc.static}/lib' \ + --replace 'mv target/release/libkrun.dylib target/release/$(KRUN_BASE_$(OS))' 'true' + ''; + + nativeBuildInputs = [ + rustPlatform.cargoSetupHook + rustPlatform.bindgenHook + cargo + pkg-config + rustc + curl + cacert + ] ++ lib.optional stdenv.hostPlatform.isDarwin linuxCrossPkgs.stdenv.cc; + + buildInputs = [ + libkrunfw' + ] + ++ lib.optionals stdenv.hostPlatform.isLinux [ + libcap_ng + glibc + glibc.static + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + apple-sdk_15 + libiconv + ] + ++ lib.optionals withGpu [ + libepoxy + libdrm + virglrenderer + ] + ++ lib.optional withSound pipewire + ++ lib.optional (variant == "sev" || variant == "tdx") openssl + ++ lib.optionals withAwsNitro [ + libarchive + libnsm + ]; + + makeFlags = [ + "PREFIX=${placeholder "out"}" + ] + ++ lib.optional withBlk "BLK=1" + ++ lib.optional withNet "NET=1" + ++ lib.optional withGpu "GPU=1" + ++ lib.optional withSound "SND=1" + ++ lib.optional withInput "INPUT=1" + ++ lib.optional withTimesync "TIMESYNC=1" + ++ lib.optional withAwsNitro "AWS_NITRO=1" + ++ lib.optional (variant == "sev") "SEV=1" + ++ lib.optional (variant == "tdx") "TDX=1"; + + postInstall = '' + mkdir -p $dev/lib/pkgconfig + mv $out/${if stdenv.hostPlatform.isDarwin then "lib" else "lib64"}/pkgconfig $dev/lib/ + mv $out/include $dev/ + ''; + + env = { + OPENSSL_NO_VENDOR = true; + } + // lib.optionalAttrs stdenv.hostPlatform.isLinux { + # Make sure libkrunfw can be found by dlopen(). + RUSTFLAGS = toString ( + map (flag: "-C link-arg=" + flag) [ + "-Wl,--push-state,--no-as-needed" + ("-lkrunfw" + lib.optionalString (variant != null) "-${variant}") + "-Wl,--pop-state" + ] + ); + } + // lib.optionalAttrs stdenv.hostPlatform.isDarwin { + CC_LINUX = "${linuxCrossPkgs.stdenv.cc}/bin/${linuxCrossPkgs.stdenv.cc.targetPrefix}gcc -L${linuxCrossPkgs.glibc.static}/lib"; + SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + }; + + meta = { + description = "Dynamic library providing Virtualization-based process isolation capabilities"; + homepage = "https://github.com/smol-machines/libkrun"; + license = lib.licenses.asl20; + platforms = libkrunfw'.meta.platforms; + }; +}) diff --git a/nix/libkrunfw.nix b/nix/libkrunfw.nix new file mode 100644 index 0000000..0a5ff4f --- /dev/null +++ b/nix/libkrunfw.nix @@ -0,0 +1,119 @@ +{ + lib, + stdenv, + fetchurl, + src, + pkgsCross, + flex, + bison, + bc, + cpio, + perl, + elfutils, + libelf, + python3, + variant ? null, +}: + +assert lib.elem variant [ + null + "sev" + "tdx" +]; + +let + linuxCrossCc = if stdenv.hostPlatform.isAarch64 + then pkgsCross.aarch64-multiplatform.stdenv.cc + else pkgsCross.gnu64.stdenv.cc; +in +stdenv.mkDerivation (finalAttrs: { + pname = "libkrunfw" + lib.optionalString (variant != null) "-${variant}"; + version = "5.4.0"; + + inherit src; + + kernelSrc = fetchurl { + url = "mirror://kernel/linux/kernel/v6.x/linux-6.12.87.tar.xz"; + hash = "sha256-zBKnZEtM754GYnsp3odT4is9B2cDqbUr6EJj4FyLmDA="; + }; + + postPatch = '' + substituteInPlace Makefile \ + --replace 'curl $(KERNEL_REMOTE) -o $(KERNEL_TARBALL)' 'ln -s $(kernelSrc) $(KERNEL_TARBALL)' + '' + lib.optionalString stdenv.hostPlatform.isDarwin '' + substituteInPlace Makefile \ + --replace 'for patch in $(KERNEL_PATCHES); do patch -p1 -d $(KERNEL_SOURCES) < "$$patch"; done' 'for patch in $(KERNEL_PATCHES); do patch -p1 -d $(KERNEL_SOURCES) < "$$patch"; done; perl -0pi -e "s/typedef struct \\{\\n\\t__u8 b\\[16\\];\\n\\} uuid_t;/#ifndef __APPLE__\\ntypedef struct {\\n\\t__u8 b[16];\\n} uuid_t;\\n#endif/; s/uuid->b\\[/(*uuid)[/g" $(KERNEL_SOURCES)/scripts/mod/file2alias.c' \ + --replace '$(MAKE) olddefconfig' '$(MAKE) HOSTCC=cc HOSTCFLAGS=-I../host-include olddefconfig' \ + --replace '$(MAKE) $(MAKEFLAGS) $(KERNEL_FLAGS)' '$(MAKE) HOSTCC=cc HOSTCFLAGS=-I../host-include $(MAKEFLAGS) $(KERNEL_FLAGS)' + ''; + + preBuild = lib.optionalString stdenv.hostPlatform.isDarwin '' + mkdir -p host-include + cp ${linuxCrossCc.libc.dev}/include/elf.h host-include/ + cat > host-include/byteswap.h <<'EOF' + #pragma once + #define bswap_16(x) __builtin_bswap16(x) + #define bswap_32(x) __builtin_bswap32(x) + #define bswap_64(x) __builtin_bswap64(x) + EOF + export HOSTCC=cc + ''; + + nativeBuildInputs = [ + stdenv.cc + flex + bison + bc + cpio + perl + python3 + python3.pkgs.pyelftools + ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ + linuxCrossCc + libelf + ]; + + buildInputs = lib.optionals stdenv.hostPlatform.isLinux [ + elfutils + ]; + + makeFlags = [ + "PREFIX=${placeholder "out"}" + ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + "OS=Linux" + "ARCH=${stdenv.hostPlatform.linuxArch}" + "CROSS_COMPILE=${linuxCrossCc.targetPrefix}" + ] + ++ lib.optionals (variant == "sev") [ + "SEV=1" + ] + ++ lib.optionals (variant == "tdx") [ + "TDX=1" + ]; + + # Fixes https://github.com/containers/libkrunfw/issues/55 + env = lib.optionalAttrs stdenv.targetPlatform.isAarch64 { + NIX_CFLAGS_COMPILE = "-march=armv8-a+crypto"; + }; + + enableParallelBuilding = true; + + meta = { + description = "Dynamic library bundling the guest payload consumed by libkrun"; + homepage = "https://github.com/smol-machines/libkrunfw"; + license = with lib.licenses; [ + lgpl2Only + lgpl21Only + ]; + platforms = [ + "x86_64-linux" + ] + ++ lib.optionals (variant == null) [ + "aarch64-linux" + "riscv64-linux" + "aarch64-darwin" + "x86_64-darwin" + ]; + }; +}) diff --git a/nix/smolvm.nix b/nix/smolvm.nix new file mode 100644 index 0000000..eb7bf15 --- /dev/null +++ b/nix/smolvm.nix @@ -0,0 +1,96 @@ +{ + lib, + stdenv, + fetchurl, + makeWrapper, + patchelf, + gcc-unwrapped, + bzip2, +}: let + version = "1.4.7"; + + releases = { + x86_64-linux = { + asset = "smolvm-${version}-linux-x86_64.tar.gz"; + root = "smolvm-${version}-linux-x86_64"; + hash = "sha256-KmdN7GDx+u4PB3RqWGDnSXgtC+4JoGnTn2HFUADLTfc="; + }; + aarch64-linux = { + asset = "smolvm-${version}-linux-arm64.tar.gz"; + root = "smolvm-${version}-linux-arm64"; + hash = "sha256-SFwSbfO9K7stwcq60IwMQ7ceKvDpBYkHcFJnnUtxL80="; + }; + aarch64-darwin = { + asset = "smolvm-${version}-darwin-arm64.tar.gz"; + root = "smolvm-${version}-darwin-arm64"; + hash = "sha256-pJkgLg9uCnNZUz/MEtnbpof3YkcLLfWGLt5UL02PQjo="; + + }; + }; + + release = releases.${stdenv.hostPlatform.system} or (throw "smolvm release tarball is not available for ${stdenv.hostPlatform.system}"); + + linuxRpath = lib.makeLibraryPath [ + gcc-unwrapped.lib + bzip2 + ]; +in + stdenv.mkDerivation { + pname = "smolvm"; + inherit version; + + src = fetchurl { + url = "https://github.com/smol-machines/smolvm/releases/download/v${version}/${release.asset}"; + inherit (release) hash; + }; + + sourceRoot = release.root; + + nativeBuildInputs = + [ + makeWrapper + ] + ++ lib.optionals stdenv.hostPlatform.isLinux [ + patchelf + ]; + + dontPatchELF = true; + dontPatchShebangs = true; + dontStrip = true; + + installPhase = + '' + runHook preInstall + + mkdir -p $out/libexec/smolvm $out/bin + cp -R . $out/libexec/smolvm/ + chmod +x $out/libexec/smolvm/smolvm $out/libexec/smolvm/smolvm-bin + patchShebangs $out/libexec/smolvm/smolvm + '' + + lib.optionalString stdenv.hostPlatform.isLinux '' + patchelf --set-interpreter ${stdenv.cc.bintools.dynamicLinker} \ + --set-rpath '$ORIGIN/lib:${linuxRpath}' \ + $out/libexec/smolvm/smolvm-bin + + for library in $out/libexec/smolvm/lib/*.so*; do + if patchelf --print-needed "$library" >/dev/null 2>&1; then + patchelf --set-rpath '$ORIGIN:${linuxRpath}' "$library" + fi + done + '' + + '' + makeWrapper $out/libexec/smolvm/smolvm $out/bin/smolvm \ + --set-default SMOLVM_AGENT_ROOTFS $out/libexec/smolvm/agent-rootfs + + runHook postInstall + ''; + + meta = { + description = "Ship and run software with isolation by default"; + homepage = "https://github.com/smol-machines/smolvm"; + license = lib.licenses.asl20; + platforms = builtins.attrNames releases; + mainProgram = "smolvm"; + sourceProvenance = with lib.sourceTypes; [binaryNativeCode]; + }; + } diff --git a/packaging/arch/.gitignore b/packaging/arch/.gitignore new file mode 100644 index 0000000..6c4b6f5 --- /dev/null +++ b/packaging/arch/.gitignore @@ -0,0 +1,5 @@ +*.pkg.tar.zst +*.tar.gz +src/ +pkg/ +LICENSE-* diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD new file mode 100644 index 0000000..0dba602 --- /dev/null +++ b/packaging/arch/PKGBUILD @@ -0,0 +1,68 @@ +# Maintainer: BinSquare +# Official smol-machines package — served from the smol-machines pacman repo. +# Repackages the upstream release binaries (includes the smol-machines +# libkrun fork, soname libkrun.so.2, which stock Arch libkrun cannot replace: +# disk overlays, snapshots, egress policy and control-socket support need it). +pkgname=smolvm +pkgver=1.5.0 +pkgrel=1 +pkgdesc='Build & run portable, lightweight, self-contained virtual machines (official smol-machines package)' +arch=('x86_64' 'aarch64') +url='https://github.com/smol-machines/smolvm' +license=('Apache-2.0') +depends=( + 'seatd' + 'crun' + 'jq' + 'e2fsprogs' + 'util-linux' + 'libcap' + 'gzip' + 'tar' +) +options=(!debug !strip) +conflicts=('smolvm-bin' 'smolvm-git') +source=( + "LICENSE-$pkgver::https://raw.githubusercontent.com/smol-machines/$pkgname/refs/tags/v$pkgver/LICENSE" +) +source_x86_64=( + "$pkgname-$pkgver-linux-x86_64.tar.gz::$url/releases/download/v$pkgver/$pkgname-$pkgver-linux-x86_64.tar.gz" +) +source_aarch64=( + "$pkgname-$pkgver-linux-arm64.tar.gz::$url/releases/download/v$pkgver/$pkgname-$pkgver-linux-arm64.tar.gz" +) +sha256sums=( + 'ac6a4050f2f415a02f3c223ddee932a07de627bc143059e9a1ea9df088e46909' +) +sha256sums_x86_64=( + '986bc230befea542af18d85a849fca056a18d728c9c7b75cfca382adb0fe1a64' +) +sha256sums_aarch64=( + 'f3e339635359b29cb60407e5d7deac3d4199870d8af2034c37c74f0f1c5530e8' +) + +package() { + case "$CARCH" in + x86_64) _dir="$pkgname-$pkgver-linux-x86_64" ;; + aarch64) _dir="$pkgname-$pkgver-linux-arm64" ;; + esac + cd "$_dir" + + # Point the wrapper at the packaged locations instead of script-relative ones + sed -i \ + -e 's|^SMOLVM_BIN=.*|SMOLVM_BIN=/usr/lib/smolvm/smolvm-bin|' \ + -e 's|^SMOLVM_LIB=.*|SMOLVM_LIB=/usr/lib/smolvm/lib|' \ + -e 's|^SMOLVM_BUNDLED_ROOTFS=.*|SMOLVM_BUNDLED_ROOTFS=/usr/lib/smolvm/agent-rootfs|' \ + smolvm + + install -Dm0755 smolvm "$pkgdir/usr/bin/smolvm" + install -Dm0755 smolvm-bin "$pkgdir/usr/lib/smolvm/smolvm-bin" + cp -a lib "$pkgdir/usr/lib/smolvm/lib" + cp -r agent-rootfs "$pkgdir/usr/lib/smolvm/agent-rootfs" + # pre-formatted storage template (optional at runtime; saves a mkfs on first pack) + install -Dm644 storage-template.ext4 "$pkgdir/usr/lib/smolvm/storage-template.ext4" + install -Dm644 "../LICENSE-$pkgver" "$pkgdir/usr/share/licenses/$pkgname/LICENSE" + install -Dm644 README.txt "$pkgdir/usr/share/doc/$pkgname/README.txt" +} + +# vim: ts=4 sw=4 et: diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..73cb934 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "stable" +components = ["rustfmt", "clippy"] diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..60ba245 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,3 @@ +edition = "2021" +max_width = 100 +use_small_heuristics = "Default" diff --git a/scripts/build-agent-rootfs.sh b/scripts/build-agent-rootfs.sh new file mode 100755 index 0000000..a7030e9 --- /dev/null +++ b/scripts/build-agent-rootfs.sh @@ -0,0 +1,351 @@ +#!/usr/bin/env bash +# Build the agent VM rootfs +# +# This script creates an Alpine-based rootfs with: +# - crane (for OCI image operations) +# - crun (OCI container runtime) +# - smolvm-agent daemon +# - Required utilities (jq, e2fsprogs, util-linux) +# +# Usage: ./scripts/build-agent-rootfs.sh [--arch aarch64|x86_64] [--no-build-agent] [--install] [output-dir] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Parse flags +INSTALL_ROOTFS=0 +OVERRIDE_ARCH="" +NO_BUILD_AGENT=0 +POSITIONAL_ARGS=() +while [[ $# -gt 0 ]]; do + case "$1" in + --install) INSTALL_ROOTFS=1; shift ;; + --arch) + if [[ -z "${2:-}" ]]; then + echo "Error: --arch requires a value (aarch64 or x86_64)" + exit 1 + fi + OVERRIDE_ARCH="$2"; shift 2 ;; + --no-build-agent) NO_BUILD_AGENT=1; shift ;; + *) POSITIONAL_ARGS+=("$1"); shift ;; + esac +done +export INSTALL_ROOTFS + +OUTPUT_DIR="${POSITIONAL_ARGS[0]:-$PROJECT_ROOT/target/agent-rootfs}" + +# Alpine version +ALPINE_VERSION="3.19" + +# Detect or override architecture +DETECTED_ARCH="${OVERRIDE_ARCH:-$(uname -m)}" +case "$DETECTED_ARCH" in + arm64|aarch64) + ALPINE_ARCH="aarch64" + CRANE_ARCH="arm64" + RUST_TARGET="aarch64-unknown-linux-musl" + ;; + x86_64|amd64) + ALPINE_ARCH="x86_64" + CRANE_ARCH="x86_64" + RUST_TARGET="x86_64-unknown-linux-musl" + ;; + *) + echo "Unsupported architecture: $DETECTED_ARCH" + exit 1 + ;; +esac + +ALPINE_MIRROR="https://dl-cdn.alpinelinux.org/alpine" +ALPINE_MINIROOTFS="alpine-minirootfs-${ALPINE_VERSION}.0-${ALPINE_ARCH}.tar.gz" +ALPINE_URL="${ALPINE_MIRROR}/v${ALPINE_VERSION}/releases/${ALPINE_ARCH}/${ALPINE_MINIROOTFS}" + +# Crane version +CRANE_VERSION="0.19.0" +CRANE_URL="https://github.com/google/go-containerregistry/releases/download/v${CRANE_VERSION}/go-containerregistry_Linux_${CRANE_ARCH}.tar.gz" + +echo "Building agent rootfs..." +echo " Alpine: ${ALPINE_VERSION} (${ALPINE_ARCH})" +echo " Crane: ${CRANE_VERSION}" +echo " Output: ${OUTPUT_DIR}" + +# Create output directory +rm -rf "$OUTPUT_DIR" +mkdir -p "$OUTPUT_DIR" + +# Download Alpine minirootfs +echo "Downloading Alpine minirootfs..." +ALPINE_TAR="/tmp/${ALPINE_MINIROOTFS}" +if [ ! -f "$ALPINE_TAR" ]; then + curl -fsSL -o "$ALPINE_TAR" "$ALPINE_URL" +fi + +# Extract Alpine +echo "Extracting Alpine..." +tar -xzf "$ALPINE_TAR" -C "$OUTPUT_DIR" + +# Download crane +echo "Downloading crane..." +CRANE_TAR="/tmp/crane-${CRANE_VERSION}-${CRANE_ARCH}.tar.gz" +if [ ! -f "$CRANE_TAR" ]; then + curl -fsSL -o "$CRANE_TAR" "$CRANE_URL" +fi + +# Extract crane to rootfs +echo "Installing crane..." +mkdir -p "$OUTPUT_DIR/usr/local/bin" +tar -xzf "$CRANE_TAR" -C "$OUTPUT_DIR/usr/local/bin" crane + +# Install additional Alpine packages into the rootfs. +# Strategies: +# 1. apk.static (Linux only) — runs natively, supports cross-arch via --arch +# 2. smolvm (any host) — only for native-arch builds (pulls host-arch image) +echo "Installing additional packages..." +APK_PACKAGES="jq e2fsprogs e2fsprogs-extra crun util-linux libcap seatd" + +# Determine if this is a cross-arch build +HOST_ARCH="$(uname -m)" +case "$HOST_ARCH" in + arm64) HOST_ALPINE_ARCH="aarch64" ;; + amd64) HOST_ALPINE_ARCH="x86_64" ;; + *) HOST_ALPINE_ARCH="$HOST_ARCH" ;; +esac +CROSS_ARCH=0 +if [[ "$ALPINE_ARCH" != "$HOST_ALPINE_ARCH" ]]; then + CROSS_ARCH=1 +fi + +install_packages_apk_static() { + echo " Using apk.static..." + # Download Alpine's static apk binary — runs natively on Linux, + # can install packages for any target architecture via --arch. + APK_STATIC_MIRROR="${ALPINE_MIRROR}/v${ALPINE_VERSION}/main/${HOST_ARCH}" + APK_STATIC_PKG=$(curl -fsSL "$APK_STATIC_MIRROR/" | grep -o 'apk-tools-static-[^"]*\.apk' | head -1) + if [[ -z "$APK_STATIC_PKG" ]]; then + echo "Error: could not find apk-tools-static package at $APK_STATIC_MIRROR" + exit 1 + fi + curl -fsSL -o /tmp/apk-static.apk "${APK_STATIC_MIRROR}/${APK_STATIC_PKG}" + mkdir -p /tmp/apk-static + tar -xzf /tmp/apk-static.apk -C /tmp/apk-static 2>/dev/null || true + + # Set up apk repositories in the rootfs + mkdir -p "$OUTPUT_DIR/etc/apk" + echo "${ALPINE_MIRROR}/v${ALPINE_VERSION}/main" > "$OUTPUT_DIR/etc/apk/repositories" + echo "${ALPINE_MIRROR}/v${ALPINE_VERSION}/community" >> "$OUTPUT_DIR/etc/apk/repositories" + + # --no-scripts: skip pre/post-install scripts and triggers. + # When cross-building (e.g. aarch64 rootfs on x86_64 host), those scripts + # are aarch64 ELF binaries that the host kernel can't exec, causing exit + # code 127. The minirootfs already ships busybox symlinks, and seatd runs + # as root in the VM so the 'seat' group creation is not required. + /tmp/apk-static/sbin/apk.static \ + --root "$OUTPUT_DIR" \ + --initdb \ + --no-cache \ + --allow-untrusted \ + --no-scripts \ + --arch "$ALPINE_ARCH" \ + add $APK_PACKAGES + echo "Packages installed successfully" +} + +repair_executable_modes() { + local rootfs_dir="$1" + local dirs=( + "$rootfs_dir/bin" + "$rootfs_dir/sbin" + "$rootfs_dir/usr/bin" + "$rootfs_dir/usr/sbin" + "$rootfs_dir/usr/local/bin" + "$rootfs_dir/usr/local/sbin" + ) + + echo "Normalizing executable permissions..." + for dir in "${dirs[@]}"; do + if [[ ! -d "$dir" ]]; then + continue + fi + + # On the macOS build path, apk install into the host-mounted rootfs can + # strip execute bits from package-installed guest tools. We observed + # this on crun, resize2fs, and e2fsck, and the failures only surfaced + # later during packed/container execution. These directories are the + # standard executable locations in the guest rootfs, so normalize their + # contents before install/pack preserves the bad modes. + find "$dir" -type d -exec chmod 755 {} + + find "$dir" -type f -exec chmod 755 {} + + done +} + +if [[ "$(uname -s)" == "Linux" ]]; then + # On Linux, apk.static is preferred — it handles cross-arch correctly + install_packages_apk_static +elif [[ "$CROSS_ARCH" == "1" ]]; then + echo "Error: cross-arch rootfs builds (--arch $ALPINE_ARCH on $HOST_ALPINE_ARCH host)" + echo " are only supported on Linux (uses apk.static)." + echo " On macOS, omit --arch or use the same architecture as your host." + exit 1 +elif command -v smolvm &> /dev/null; then + echo " Using smolvm..." + smolvm machine run --net -v "$OUTPUT_DIR:/rootfs" --image "alpine:${ALPINE_VERSION}" \ + -- sh -c "apk add --root /rootfs --initdb --no-cache $APK_PACKAGES" + echo "Packages installed successfully" +else + echo "Error: smolvm is required to build the agent rootfs on macOS" + echo "Install smolvm first: https://github.com/smolvm/smolvm" + exit 1 +fi + +repair_executable_modes "$OUTPUT_DIR" + +# Create necessary directories +mkdir -p "$OUTPUT_DIR/storage" +mkdir -p "$OUTPUT_DIR/etc/init.d" +mkdir -p "$OUTPUT_DIR/run" + +# Bake in the agent's /mnt mount points so the rootfs is self-sufficient. +# +# At boot, setup_persistent_rootfs() (crates/smolvm-agent/src/main.rs) mounts +# the overlay/storage disks and stages pivot_root under these paths BEFORE any +# writable overlay exists — its create_dir_all() calls run against the agent +# rootfs itself. On a read-only rootfs, or one built from scratch without the +# Alpine base's empty /mnt, those mkdirs fail, the mounts fail, and the VM +# boots without its persistent overlay. Pre-creating the dirs here makes those +# runtime create_dir_all() calls a no-op (the agent keeps them as a backstop +# and WARNs if a mount point is ever missing on a RO rootfs). +# +# Keep this list in sync with the agent's mount-point constants: +# /mnt/overlay OVERLAY_MOUNT } setup_persistent_rootfs(), required at +# /mnt/storage STORAGE_TEMP_MOUNT } boot before the overlay is writable +# /mnt/newroot NEWROOT } +# /mnt/virtiofs paths::VIRTIOFS_MOUNT_ROOT parent for per-tag virtiofs shares +# /mnt/rosetta vm::rosetta::ROSETTA_GUEST_PATH macOS Rosetta binfmt share +for mnt_dir in overlay storage newroot virtiofs rosetta; do + mkdir -p "$OUTPUT_DIR/mnt/$mnt_dir" +done + +# Install the pre-built Rosetta ptrace wrapper. This static binary intercepts +# Rosetta's Virtualization.framework ioctl validation, allowing x86_64 +# translation to work under libkrun's Hypervisor.framework backend. +# Source: scripts/rosetta/rosetta-wrapper.c +ROSETTA_WRAPPER="$(dirname "$0")/rosetta/rosetta-wrapper" +if [ -f "$ROSETTA_WRAPPER" ]; then + cp "$ROSETTA_WRAPPER" "$OUTPUT_DIR/usr/bin/rosetta-wrapper" + chmod 755 "$OUTPUT_DIR/usr/bin/rosetta-wrapper" +fi + +# Remove existing init (it's a symlink to busybox) and replace with +# symlink to the agent binary. The agent handles overlayfs setup + +# pivot_root internally before starting the vsock listener. +rm -f "$OUTPUT_DIR/sbin/init" +ln -sf /usr/local/bin/smolvm-agent "$OUTPUT_DIR/sbin/init" + +# Create resolv.conf +echo "nameserver 1.1.1.1" > "$OUTPUT_DIR/etc/resolv.conf" + +# Remove seatd socket if baked in during build (build artifact, not runtime state) +rm -f "$OUTPUT_DIR/run/seatd.sock" + +PROFILE="release-small" + +if [[ -n "${AGENT_BINARY:-}" ]] && [[ -f "${AGENT_BINARY}" ]]; then + echo "Using pre-built agent binary: $AGENT_BINARY" +elif [[ "$NO_BUILD_AGENT" == "1" ]]; then + echo "Skipping agent build (--no-build-agent)" +else + AGENT_BINARY="" + + # Strategy 1: Native build on Linux with musl target installed + if [[ "$(uname -s)" == "Linux" ]] && command -v cargo &> /dev/null; then + if rustup target list --installed 2>/dev/null | grep -q "$RUST_TARGET"; then + echo "Building natively with musl target..." + # Build the agent and the CUDA-over-vsock guest runner together. + cargo build --profile "$PROFILE" -p smolvm-agent -p smolvm-cuda-guest \ + --target "$RUST_TARGET" --manifest-path "$PROJECT_ROOT/Cargo.toml" + AGENT_BINARY="$PROJECT_ROOT/target/$RUST_TARGET/$PROFILE/smolvm-agent" + CUDA_GUEST_BINARY="$PROJECT_ROOT/target/$RUST_TARGET/$PROFILE/smolvm-cuda-run" + fi + fi + + # Strategy 2: smolvm with rust:alpine (dogfooding) + if [[ -z "$AGENT_BINARY" ]] || [[ ! -f "$AGENT_BINARY" ]]; then + if command -v smolvm &> /dev/null; then + echo "Building via smolvm (rust:alpine)..." + smolvm machine run --net --mem 2048 -v "$PROJECT_ROOT:/work" --image rust:alpine \ + -- sh -c ". /usr/local/cargo/env && apk add musl-dev && cd /work && cargo build --profile $PROFILE -p smolvm-agent -p smolvm-cuda-guest" + AGENT_BINARY="$PROJECT_ROOT/target/$PROFILE/smolvm-agent" + CUDA_GUEST_BINARY="$PROJECT_ROOT/target/$PROFILE/smolvm-cuda-run" + else + echo "Error: Cannot build smolvm-agent" + echo " Either install the musl target: rustup target add $RUST_TARGET" + echo " Or install smolvm for cross-compilation" + exit 1 + fi + fi +fi + +# Install the agent binary into the rootfs (if we have one) +if [[ -n "${AGENT_BINARY:-}" ]] && [[ -f "${AGENT_BINARY}" ]]; then + echo "Installing smolvm-agent binary..." + cp "$AGENT_BINARY" "$OUTPUT_DIR/usr/local/bin/smolvm-agent" + chmod +x "$OUTPUT_DIR/usr/local/bin/smolvm-agent" +elif [[ "$NO_BUILD_AGENT" != "1" ]]; then + echo "Error: smolvm-agent binary not found at ${AGENT_BINARY:-}" + exit 1 +fi + +# Install the CUDA-over-vsock guest runner (best-effort: a missing binary just +# means the image ships without it; CUDA is opt-in via `machine create --cuda`). +if [[ -n "${CUDA_GUEST_BINARY:-}" ]] && [[ -f "${CUDA_GUEST_BINARY}" ]]; then + echo "Installing smolvm-cuda-run binary..." + cp "$CUDA_GUEST_BINARY" "$OUTPUT_DIR/usr/local/bin/smolvm-cuda-run" + chmod +x "$OUTPUT_DIR/usr/local/bin/smolvm-cuda-run" +fi + +# CUDA guest shims: glibc cdylibs the agent bind-mounts into workload +# containers on `--cuda` (see crates/smolvm-agent/src/cuda.rs). They are loaded +# by the container's glibc — not by the (musl) agent — so they build for the +# native gnu target. Best-effort: without them `--cuda` still works, the user +# just stages shims manually; cross-arch rootfs builds skip them. +if [[ "$NO_BUILD_AGENT" != "1" && "$(uname -s)" == "Linux" && "$(uname -m)" == "$ALPINE_ARCH" ]] \ + && command -v cargo &> /dev/null; then + echo "Building CUDA guest shims (native gnu target)..." + if cargo build --release -p smolvm-cudart-shim -p smolvm-cuda-shim \ + --manifest-path "$PROJECT_ROOT/Cargo.toml"; then + mkdir -p "$OUTPUT_DIR/usr/local/lib/smolvm-cuda" + cp "$PROJECT_ROOT/target/release/libcudart.so" \ + "$OUTPUT_DIR/usr/local/lib/smolvm-cuda/libcudart-shim.so" + cp "$PROJECT_ROOT/target/release/libcuda.so" \ + "$OUTPUT_DIR/usr/local/lib/smolvm-cuda/libcuda.so.1" + echo "Installed CUDA guest shims" + else + echo "CUDA guest shim build failed — rootfs ships without auto-staging" + fi +else + echo "Skipping CUDA guest shims (cross-arch or no cargo) — auto-staging disabled in this rootfs" +fi + +echo "" +echo "Agent rootfs created at: $OUTPUT_DIR" +if [[ -n "${AGENT_BINARY:-}" ]]; then + echo "Agent binary: $AGENT_BINARY" +fi +echo "Rootfs size: $(du -sh "$OUTPUT_DIR" | cut -f1)" + +# Install to runtime data directory if --install flag is passed +if [[ "${INSTALL_ROOTFS:-}" == "1" ]]; then + if [[ "$(uname -s)" == "Darwin" ]]; then + DATA_DIR="$HOME/Library/Application Support/smolvm" + else + DATA_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/smolvm" + fi + + echo "Installing agent-rootfs to $DATA_DIR..." + mkdir -p "$DATA_DIR" + rm -rf "$DATA_DIR/agent-rootfs" + cp -a "$OUTPUT_DIR" "$DATA_DIR/agent-rootfs" + echo "Installed successfully." +fi diff --git a/scripts/build-dist-windows.sh b/scripts/build-dist-windows.sh new file mode 100755 index 0000000..8dc1b99 --- /dev/null +++ b/scripts/build-dist-windows.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# Build a distributable smolvm package for Windows (x86_64). +# +# Unlike build-dist.sh (which builds a native Unix dist), this is a CROSS build: +# it runs on a Linux host, cross-compiles smolvm.exe with the mingw-w64 toolchain, +# and assembles the layout that boots on Windows/WHP — smolvm.exe with krun.dll + +# libkrunfw.dll beside it (Windows resolves DLLs from the exe's directory), the +# Linux x86_64 agent-rootfs (as a tarball), and the pre-formatted ext4 disk templates. The guest +# is Linux, so the rootfs/templates are the same artifacts the linux-x86_64 dist +# ships; only the host binary + libraries are Windows-specific. +# +# Output: dist/smolvm--windows-x86_64.zip +# +# Inputs (mirrors build-dist.sh --skip-agent-build): +# target/agent-rootfs/ Linux x86_64 rootfs with smolvm-agent baked in +# $LIB_DIR (default lib/windows-x86_64) krun.dll + libkrunfw.dll +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +cd "$PROJECT_ROOT" + +VERSION="${VERSION:-$(grep '^version' Cargo.toml | head -1 | cut -d'"' -f2)}" +PLATFORM="windows-x86_64" +DIST_NAME="smolvm-${VERSION}-${PLATFORM}" +DIST_DIR="dist/${DIST_NAME}" +LIB_DIR="${LIB_DIR:-lib/windows-x86_64}" +TARGET="x86_64-pc-windows-gnu" +ROOTFS_SRC="$PROJECT_ROOT/target/agent-rootfs" + +# The Windows host dlopens libkrun at runtime (libloading), so the cross build +# has no link-time libkrun dependency — only the mingw linker is required. +export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER="${CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER:-x86_64-w64-mingw32-gcc}" + +echo "Building smolvm Windows distribution: ${DIST_NAME}" + +# --- Preconditions --------------------------------------------------------- +for dll in krun.dll libkrunfw.dll; do + if [[ ! -f "$LIB_DIR/$dll" ]]; then + echo "Error: $dll not found in $LIB_DIR (set LIB_DIR to the Windows lib dir)" >&2 + exit 1 + fi +done +if [[ ! -d "$ROOTFS_SRC" ]]; then + echo "Error: target/agent-rootfs not found." >&2 + echo " Run ./scripts/build-agent-rootfs.sh --arch x86_64 first (CI downloads it)." >&2 + exit 1 +fi +if ! command -v "$CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER" >/dev/null 2>&1; then + echo "Error: mingw linker $CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER not found." >&2 + echo " Install gcc-mingw-w64-x86-64 (apt) or x86_64-w64-mingw32-gcc (brew)." >&2 + exit 1 +fi + +# --- Cross-build smolvm.exe ------------------------------------------------- +echo "Cross-compiling smolvm.exe for $TARGET..." +rustup target list --installed 2>/dev/null | grep -q "$TARGET" || rustup target add "$TARGET" +cargo build --release --target "$TARGET" --bin smolvm +EXE="target/$TARGET/release/smolvm.exe" +[[ -f "$EXE" ]] || { echo "Error: cross build did not produce $EXE" >&2; exit 1; } + +# --- Assemble the dist directory ------------------------------------------- +echo "Assembling distribution..." +rm -rf "$DIST_DIR" +mkdir -p "$DIST_DIR" + +# Host binary + DLLs sit together; Windows resolves krun.dll (and its +# libkrunfw.dll dependency) from the directory containing smolvm.exe. +cp "$EXE" "$DIST_DIR/smolvm.exe" +cp "$LIB_DIR/krun.dll" "$DIST_DIR/krun.dll" +cp "$LIB_DIR/libkrunfw.dll" "$DIST_DIR/libkrunfw.dll" + +# Ship the Linux x86_64 guest rootfs as a TARBALL, not an extracted dir: a `.zip` +# can't carry the dir tree (busybox symlinks, modes, special files) — that is what +# broke the first attempt. smolvm.exe extracts `agent-rootfs.tar.gz` on first run +# via ensure_extracted_rootfs (Windows `tar.exe`; symlinks it can't make are +# skipped, tolerated by the runtime). The caller must have injected the agent +# binary into the rootfs already (the rootfs artifact ships agent-less). +if [[ ! -f "$ROOTFS_SRC/usr/local/bin/smolvm-agent" ]]; then + echo "Error: target/agent-rootfs is missing usr/local/bin/smolvm-agent." >&2 + echo " Inject the Linux x86_64 agent into the rootfs before packaging." >&2 + exit 1 +fi +# The rootfs is extracted as root in CI; use sudo to read any restricted files, +# then hand the tarball back to the runner user so the later `zip` can read it. +TAR_SUDO="" +if command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then TAR_SUDO="sudo"; fi +echo "Packing agent-rootfs.tar.gz..." +$TAR_SUDO tar -czf "$DIST_DIR/agent-rootfs.tar.gz" -C "$ROOTFS_SRC" . +[[ -n "$TAR_SUDO" ]] && $TAR_SUDO chown "$(id -u):$(id -g)" "$DIST_DIR/agent-rootfs.tar.gz" +echo "Agent rootfs tarball: $(du -h "$DIST_DIR/agent-rootfs.tar.gz" | cut -f1)" + +# --- Pre-formatted ext4 disk templates ------------------------------------- +# Windows has no host mkfs.ext4, so the release ships pre-formatted templates +# next to smolvm.exe (pack/create copies them instead of formatting at runtime). +# +# Ship them at their REAL 512 MiB size — do NOT extend to the 20/10 GiB virtual +# sizes here. Zip has no sparse-file representation, so an extended template +# materializes as tens of GiB of zeros on extraction (and, no longer holey, gets +# dense-copied per machine start until the disk fills). The engine extends the +# per-machine copy to the requested virtual size itself (copy_disk_from_template +# marks the copy sparse and set_lens it; the guest grows the ext4 with resize2fs +# at boot), so nothing on Windows needs a pre-extended template. The Linux/macOS +# tarballs keep full-size templates via GNU tar --sparse (see build-dist.sh). +if command -v mkfs.ext4 >/dev/null 2>&1; then + echo "Creating disk templates..." + make_template() { # $1=path $2=label + dd if=/dev/zero of="$1" bs=1 count=0 seek=$((512 * 1024 * 1024)) 2>/dev/null + mkfs.ext4 -F -q -m 0 -L "$2" "$1" + } + make_template "$DIST_DIR/storage-template.ext4" smolvm + make_template "$DIST_DIR/overlay-template.ext4" smolvm-overlay + echo "Templates: storage + overlay (512 MiB each; engine extends copies to their virtual size)" +else + echo "Error: mkfs.ext4 not found — install e2fsprogs to build the Windows dist." >&2 + exit 1 +fi + +# --- README ---------------------------------------------------------------- +cat > "$DIST_DIR/README.txt" < Optional features, or: + dism /online /enable-feature /featurename:HypervisorPlatform /all) + +INSTALL + Unzip this archive anywhere and keep all files together. krun.dll and + libkrunfw.dll must stay beside smolvm.exe. Optionally add the folder to PATH. + +USAGE (run smolvm.exe directly) + smolvm.exe machine run --net --image alpine -- echo "Hello from Windows" + smolvm.exe machine create --net --name myvm + smolvm.exe machine start --name myvm + smolvm.exe machine exec --name myvm -- /bin/sh + smolvm.exe machine stats --name myvm + smolvm.exe machine stop --name myvm + +NOT YET SUPPORTED ON WINDOWS + GPU acceleration; machine fork / snapshot. + +More: https://github.com/smol-machines/smolvm +EOF + +# --- Checksums + zip -------------------------------------------------------- +echo "Generating checksums..." +( cd "$DIST_DIR" && sha256sum smolvm.exe krun.dll libkrunfw.dll agent-rootfs.tar.gz > checksums.txt ) + +echo "Creating zip..." +( cd dist && rm -f "${DIST_NAME}.zip" && zip -qr "${DIST_NAME}.zip" "${DIST_NAME}" ) + +echo "" +echo "Distribution package created: dist/${DIST_NAME}.zip" +echo "Contents:" +ls -la "$DIST_DIR" diff --git a/scripts/build-dist.sh b/scripts/build-dist.sh new file mode 100755 index 0000000..f022335 --- /dev/null +++ b/scripts/build-dist.sh @@ -0,0 +1,665 @@ +#!/usr/bin/env bash +# Build a distributable smolvm package +# +# Usage: +# ./scripts/build-dist.sh +# ./scripts/build-dist.sh --with-local-libkrun +# +# Output: dist/smolvm--.tar.gz + +set -e + +# Options +WITH_LOCAL_LIBKRUN=0 +SKIP_AGENT_BUILD=0 +LOCAL_LIBKRUN_DIR="" +LIBKRUN_MAKE_FLAGS="${LIBKRUN_MAKE_FLAGS:-BLK=1 NET=1 GPU=1}" + +print_help() { + cat <<'EOF' +Build a distributable smolvm package. + +Usage: + ./scripts/build-dist.sh [options] + +Options: + --with-local-libkrun Build libkrun from local checkout and refresh bundled lib/ + --local-libkrun-dir PATH Local libkrun checkout (default: ../libkrun) + --skip-agent-build Skip agent cross-compilation (use pre-built binary) + -h, --help Show this help text + +Environment: + LIBKRUN_MAKE_FLAGS make flags for local libkrun build (default: BLK=1 NET=1 GPU=1) + LIBCLANG_PATH path to libclang.dylib (auto-detected from brew llvm on macOS) + LIB_DIR Override bundled library directory used by smolvm build + CODESIGN_IDENTITY macOS code signing identity (default: - for ad-hoc) +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --with-local-libkrun) + WITH_LOCAL_LIBKRUN=1 + shift + ;; + --skip-agent-build) + SKIP_AGENT_BUILD=1 + shift + ;; + --local-libkrun-dir) + if [[ -z "${2:-}" ]]; then + echo "Error: --local-libkrun-dir requires a path" + exit 1 + fi + LOCAL_LIBKRUN_DIR="$2" + shift 2 + ;; + -h|--help) + print_help + exit 0 + ;; + *) + echo "Error: unknown option: $1" + print_help + exit 1 + ;; + esac +done + +# Configuration +VERSION="${VERSION:-$(grep '^version' Cargo.toml | head -1 | cut -d'"' -f2)}" +# Normalize architecture: aarch64 -> arm64 for consistent naming across platforms +_ARCH="$(uname -m)" +if [[ "$_ARCH" == "aarch64" ]]; then + _ARCH="arm64" +fi +PLATFORM="$(uname -s | tr '[:upper:]' '[:lower:]')-${_ARCH}" +DIST_NAME="smolvm-${VERSION}-${PLATFORM}" +DIST_DIR="dist/${DIST_NAME}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +WORKSPACE_SRC_ROOT="$(cd "$PROJECT_ROOT/.." && pwd)" +LOCAL_STAGE_DIR="$PROJECT_ROOT/target/local-lib-stage" +LOCAL_INIT_KRUN="" + +if [[ -z "$LOCAL_LIBKRUN_DIR" ]]; then + LOCAL_LIBKRUN_DIR="$WORKSPACE_SRC_ROOT/libkrun" +fi + +echo "Building smolvm distribution: ${DIST_NAME}" + +# Check for git-lfs (required for library binaries) +if ! command -v git-lfs &> /dev/null && ! git lfs version &> /dev/null 2>&1; then + echo "Error: git-lfs is required to build smolvm distributions" + exit 1 +fi + +# Resolve bundled library directory +if [[ "$(uname -s)" == "Linux" ]]; then + ARCH="$(uname -m)" + DEFAULT_LIB_DIR="./lib/linux-${ARCH}" + STAGED_LIB_DIR="$LOCAL_STAGE_DIR/usr/local/lib64" +else + DEFAULT_LIB_DIR="./lib" + STAGED_LIB_DIR="$LOCAL_STAGE_DIR/usr/local/lib" +fi + +BASE_LIB_DIR="${LIB_DIR:-$DEFAULT_LIB_DIR}" +WORK_LIB_DIR="$BASE_LIB_DIR" +LOCAL_BUNDLE_DIR="$PROJECT_ROOT/target/local-lib-bundle" + +run_make() { + local repo="$1" + local flags="$2" + shift 2 + local -a args=() + if [[ -n "$flags" ]]; then + read -r -a args <<< "$flags" + fi + make -C "$repo" "${args[@]}" "$@" +} + +copy_matching_libraries() { + local src_dir="$1" + local pattern="$2" + local dst_dir="$3" + + if compgen -G "$src_dir/$pattern" > /dev/null; then + cp -a "$src_dir"/$pattern "$dst_dir"/ + fi +} + +setup_macos_libkrun_env() { + if [[ "$(uname -s)" != "Darwin" ]]; then + return + fi + if [[ ! -f "$LOCAL_LIBKRUN_DIR/Makefile" ]]; then + return + fi + + # libkrun build scripts use bindgen and require libclang.dylib at runtime. + if [[ -z "${LIBCLANG_PATH:-}" ]] && command -v brew &> /dev/null; then + local llvm_prefix + llvm_prefix="$(brew --prefix llvm 2>/dev/null || true)" + if [[ -n "$llvm_prefix" ]] && [[ -f "$llvm_prefix/lib/libclang.dylib" ]]; then + export LIBCLANG_PATH="$llvm_prefix/lib" + echo "Using libclang from $LIBCLANG_PATH" + fi + fi + + if [[ -n "${LIBCLANG_PATH:-}" ]]; then + export DYLD_FALLBACK_LIBRARY_PATH="$LIBCLANG_PATH:${DYLD_FALLBACK_LIBRARY_PATH:-}" + else + echo "Warning: LIBCLANG_PATH is not set." + echo " If libkrun build fails with 'Library not loaded: @rpath/libclang.dylib'," + echo " install llvm via brew and set LIBCLANG_PATH to its lib directory." + fi + + if [[ "$LIBKRUN_MAKE_FLAGS" == *"BUILD_INIT=0"* ]] && [[ ! -f "$LOCAL_LIBKRUN_DIR/init/init" ]]; then + echo "Error: LIBKRUN_MAKE_FLAGS includes BUILD_INIT=0 but init binary is missing:" + echo " $LOCAL_LIBKRUN_DIR/init/init" + echo "Build init first (for example: make -C \"$LOCAL_LIBKRUN_DIR\" BLK=1)," + echo "or remove BUILD_INIT=0 from LIBKRUN_MAKE_FLAGS." + exit 1 + fi +} + +refresh_bundled_libs_from_local() { + local repo="$1" + local flags="$2" + local prefix="$3" + + if [[ ! -f "$repo/Makefile" ]]; then + echo "Error: local repo not found: $repo" + exit 1 + fi + + mkdir -p "$WORK_LIB_DIR" + rm -rf "$LOCAL_STAGE_DIR" + mkdir -p "$LOCAL_STAGE_DIR" + + # On Linux, link with partial RELRO so libkrun's symbols bind lazily. Full + # RELRO forces BIND_NOW, which would defeat the lazy virglrenderer loading + # (RTLD_LAZY in src/agent/krun.rs + the patchelf --remove-needed below) that + # lets one GPU-enabled libkrun load on non-GPU hosts. Harmless for the + # install step and non-cargo builds; not applicable to macOS dylibs. + if [[ "$(uname -s)" == "Linux" ]]; then + RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C relro-level=partial" run_make "$repo" "$flags" + else + run_make "$repo" "$flags" + fi + run_make "$repo" "$flags" install "DESTDIR=$LOCAL_STAGE_DIR" "PREFIX=/usr/local" + + if [[ ! -d "$STAGED_LIB_DIR" ]]; then + echo "Error: no staged libraries found in $STAGED_LIB_DIR" + exit 1 + fi + + if ! compgen -G "$STAGED_LIB_DIR/${prefix}*" > /dev/null; then + echo "Error: no staged ${prefix} artifacts found in $STAGED_LIB_DIR" + exit 1 + fi + + cp -a "$STAGED_LIB_DIR"/${prefix}* "$WORK_LIB_DIR"/ +} + +if [[ "$WITH_LOCAL_LIBKRUN" == "1" ]]; then + if [[ ! -d "$BASE_LIB_DIR" ]]; then + echo "Error: base library directory does not exist: $BASE_LIB_DIR" + echo "Set LIB_DIR to a directory containing libkrun/libkrunfw artifacts." + exit 1 + fi + + rm -rf "$LOCAL_BUNDLE_DIR" + mkdir -p "$LOCAL_BUNDLE_DIR" + copy_matching_libraries "$BASE_LIB_DIR" "libkrun*" "$LOCAL_BUNDLE_DIR" + copy_matching_libraries "$BASE_LIB_DIR" "libkrunfw*" "$LOCAL_BUNDLE_DIR" + # GPU rendering libraries are not rebuilt by --with-local-libkrun, but must + # be carried over from the base lib dir so the dist stays GPU-capable. + copy_matching_libraries "$BASE_LIB_DIR" "libvirglrenderer*" "$LOCAL_BUNDLE_DIR" + copy_matching_libraries "$BASE_LIB_DIR" "libMoltenVK*" "$LOCAL_BUNDLE_DIR" + copy_matching_libraries "$BASE_LIB_DIR" "libepoxy*" "$LOCAL_BUNDLE_DIR" + # Linux render server binary (required for Venus Vulkan). + [[ -f "$BASE_LIB_DIR/virgl_render_server" ]] && cp "$BASE_LIB_DIR/virgl_render_server" "$LOCAL_BUNDLE_DIR/" + WORK_LIB_DIR="$LOCAL_BUNDLE_DIR" + echo "Staging local build bundle in $WORK_LIB_DIR" +fi + +if [[ "$WITH_LOCAL_LIBKRUN" == "1" ]]; then + echo "Building local libkrun from $LOCAL_LIBKRUN_DIR..." + setup_macos_libkrun_env + refresh_bundled_libs_from_local "$LOCAL_LIBKRUN_DIR" "$LIBKRUN_MAKE_FLAGS" "libkrun" + if [[ -f "$LOCAL_LIBKRUN_DIR/init/init" ]]; then + LOCAL_INIT_KRUN="$LOCAL_LIBKRUN_DIR/init/init" + fi +fi + +# Check for required libraries +if [[ ! -f "$WORK_LIB_DIR/libkrun.dylib" ]] && [[ ! -f "$WORK_LIB_DIR/libkrun.so" ]]; then + echo "Error: libkrun not found in $WORK_LIB_DIR" + echo "Set LIB_DIR to point to your libkrun library directory." + exit 1 +fi +if [[ ! -f "$WORK_LIB_DIR/libkrunfw.5.dylib" ]] && [[ ! -f "$WORK_LIB_DIR/libkrunfw.so" ]]; then + echo "Error: libkrunfw not found in $WORK_LIB_DIR" + echo "Set LIB_DIR to point to your libkrunfw library directory." + exit 1 +fi + +# Build release binaries +echo "Building release binaries..." +LIBKRUN_BUNDLE="$WORK_LIB_DIR" cargo build --release --bin smolvm + +# Build the unified `smol` CLI if its source is present (it lives in a sibling +# repo checked out at ./smol). It is a separate cargo workspace that depends on +# the engine by path, so build it from inside ./smol with an ABSOLUTE +# LIBKRUN_BUNDLE so its build.rs finds the same bundled libraries we ship. +BUILD_SMOL=0 +if [[ -f "./smol/Cargo.toml" ]]; then + echo "Building unified smol CLI..." + _ABS_LIB_BUNDLE="$(cd "$WORK_LIB_DIR" && pwd)" + ( cd ./smol && LIBKRUN_BUNDLE="$_ABS_LIB_BUNDLE" cargo build --release --bin smol ) + BUILD_SMOL=1 +else + echo "smol CLI source (./smol) not found — building engine-only distribution" +fi + +# Build smolvm-agent for Linux (size-optimized) +if [[ "$SKIP_AGENT_BUILD" == "1" ]]; then + echo "Skipping agent build (--skip-agent-build)" + if [[ ! -f "./target/release-small/smolvm-agent" ]]; then + echo "Error: --skip-agent-build requires a pre-built agent at target/release-small/smolvm-agent" + exit 1 + fi +else + echo "Building smolvm-agent for Linux (optimized for size)..." + if [[ "$(uname -s)" == "Linux" ]]; then + # On Linux, build natively with musl for static linking + MUSL_TARGET="$(uname -m)-unknown-linux-musl" + if command -v cargo &> /dev/null; then + if rustup target list --installed 2>/dev/null | grep -q "$MUSL_TARGET"; then + cargo build --profile release-small -p smolvm-agent --target "$MUSL_TARGET" + # Copy to the non-target-triple path that the rest of the script expects + mkdir -p ./target/release-small + cp "./target/${MUSL_TARGET}/release-small/smolvm-agent" \ + "./target/release-small/smolvm-agent" + fi + fi + fi + + # If native build didn't produce the binary, use smolvm + if [[ ! -f "./target/release-small/smolvm-agent" ]]; then + if command -v smolvm &> /dev/null; then + echo "Building via smolvm (rust:alpine)..." + smolvm machine run --net --mem 2048 -v "$PROJECT_ROOT:/work" --image rust:alpine \ + -- sh -c ". /usr/local/cargo/env && apk add musl-dev && cd /work && cargo build --profile release-small -p smolvm-agent" + else + echo "Error: Cannot build smolvm-agent." + echo " Install smolvm or the musl target (rustup target add x86_64-unknown-linux-musl)" + exit 1 + fi + fi +fi + +# Sign binary (macOS only) +# Set CODESIGN_IDENTITY to a Developer ID for distribution signing. +# Defaults to ad-hoc signing (-) for local development. +if [[ "$(uname -s)" == "Darwin" ]]; then + IDENTITY="${CODESIGN_IDENTITY:--}" + CODESIGN_ARGS=(--force --sign "$IDENTITY" --entitlements smolvm.entitlements) + if [[ "$IDENTITY" != "-" ]]; then + # Developer ID signing requires hardened runtime for notarization + CODESIGN_ARGS+=(--options runtime) + fi + echo "Signing binary (identity: $IDENTITY)..." + codesign "${CODESIGN_ARGS[@]}" ./target/release/smolvm + # The smol binary also calls the macOS hypervisor, so it needs the same + # entitlements + signature (otherwise it cannot start a VM). + if [[ "$BUILD_SMOL" == "1" ]]; then + codesign "${CODESIGN_ARGS[@]}" ./smol/target/release/smol + fi +fi + +# Create distribution directory +echo "Creating distribution package..." +rm -rf "$DIST_DIR" +mkdir -p "$DIST_DIR/lib" + +# Copy binary (renamed to smolvm-bin) +cp ./target/release/smolvm "$DIST_DIR/smolvm-bin" + +# Copy wrapper script +cp ./scripts/smolvm-wrapper.sh "$DIST_DIR/smolvm" +chmod +x "$DIST_DIR/smolvm" + +# Copy the unified smol CLI (binary renamed to smol-bin + its own wrapper). +# The wrapper points at the same lib/ and agent-rootfs, so one tarball serves +# both `smol` (the user-facing CLI) and `smolvm` (the lower-level engine). +if [[ "$BUILD_SMOL" == "1" ]]; then + cp ./smol/target/release/smol "$DIST_DIR/smol-bin" + cp ./scripts/smol-wrapper.sh "$DIST_DIR/smol" + chmod +x "$DIST_DIR/smol" +fi + +# Copy libraries +if [[ "$(uname -s)" == "Darwin" ]]; then + cp "$WORK_LIB_DIR/libkrun.dylib" "$DIST_DIR/lib/" + cp "$WORK_LIB_DIR/libkrunfw.5.dylib" "$DIST_DIR/lib/" + # Create symlink for compatibility + ln -sf libkrunfw.5.dylib "$DIST_DIR/lib/libkrunfw.dylib" + # Bundle GPU rendering libraries if present (virglrenderer → MoltenVK + epoxy). + # All three dylibs use @loader_path refs so they resolve correctly regardless + # of where the lib/ directory is placed after installation. + for gpu_lib in libvirglrenderer.1.dylib libMoltenVK.dylib libepoxy.0.dylib; do + if [[ -f "$WORK_LIB_DIR/$gpu_lib" ]]; then + cp "$WORK_LIB_DIR/$gpu_lib" "$DIST_DIR/lib/" + echo "Bundled GPU library: $gpu_lib ($(du -h "$DIST_DIR/lib/$gpu_lib" | cut -f1))" + fi + done +else + copy_so_with_symlinks() { + local lib_prefix="$1" + local required="$2" + local local_so="$WORK_LIB_DIR/${lib_prefix}.so" + if [[ ! -e "$local_so" ]]; then + if [[ "$required" == "required" ]]; then + echo "Error: ${lib_prefix}.so not found in $WORK_LIB_DIR" + exit 1 + fi + return 0 + fi + + # Copy each library's real file plus all symlinks that reference it. + # Only copies files in the active symlink chain — stale old versions + # (e.g. libkrunfw.so.5.2.0 when current is 5.3.0) are excluded. + real_file="$(readlink -f "$local_so")" + real_name="$(basename "$real_file")" + cp "$real_file" "$DIST_DIR/lib/$real_name" + + # Copy every symlink in the directory that ultimately points to + # the same real file. This catches both directions: + # libkrun.so.1 → libkrun.so (SONAME → real) + # libkrunfw.so → libkrunfw.so.5 → libkrunfw.so.5.3.0 + for candidate in "$WORK_LIB_DIR"/${lib_prefix}.so*; do + [[ -L "$candidate" ]] || continue + candidate_real="$(readlink -f "$candidate")" + if [[ "$candidate_real" == "$real_file" ]]; then + cp -a "$candidate" "$DIST_DIR/lib/" + fi + done + echo "Bundled library: ${lib_prefix} ($(du -h "$DIST_DIR/lib/$real_name" | cut -f1))" + } + + copy_so_with_symlinks libkrun required + copy_so_with_symlinks libkrunfw required + + # Strip the hard NEEDED on virglrenderer from the GPU-enabled libkrun so a + # host without it can still dlopen libkrun (paired with the RTLD_LAZY load in + # src/agent/krun.rs). virglrenderer is loaded by soname at runtime only when + # the GPU path actually runs — so one build serves both GPU and non-GPU hosts. + if command -v patchelf >/dev/null 2>&1; then + for lk in "$DIST_DIR"/lib/libkrun.so*; do + [[ -f "$lk" && ! -L "$lk" ]] || continue + if patchelf --print-needed "$lk" 2>/dev/null | grep -q libvirglrenderer; then + patchelf --remove-needed libvirglrenderer.so.1 "$lk" + echo "Stripped libvirglrenderer NEEDED from $(basename "$lk") — GPU stays optional at runtime" + fi + done + else + echo "Warning: patchelf not found — libkrun keeps its hard virglrenderer NEEDED;" + echo " non-GPU Linux hosts will fail to load it. Install patchelf in the build env." + fi + + # Bundle GPU rendering libraries if present (virglrenderer chain for Venus/Vulkan). + # libMoltenVK is macOS-only — not included here. + for gpu_lib_prefix in libvirglrenderer libepoxy; do + copy_so_with_symlinks "$gpu_lib_prefix" optional + done + # Bundle render server binary (required for Venus Vulkan on Linux). + if [[ -f "$WORK_LIB_DIR/virgl_render_server" ]]; then + cp "$WORK_LIB_DIR/virgl_render_server" "$DIST_DIR/lib/" + chmod +x "$DIST_DIR/lib/virgl_render_server" + echo "Bundled: virgl_render_server ($(du -h "$DIST_DIR/lib/virgl_render_server" | cut -f1))" + fi +fi + +# Copy init.krun for Linux only if a real (non-empty) init binary is present. +# libkrun 2.0.0 embeds the guest init (krun-init) in libkrun.so (INIT_BLOB on by +# default), so no external init.krun is needed — a VM boots from the embedded +# init (verified). The committed `libkrun/init/init` is now an empty placeholder, +# so the `-s` (non-empty) checks skip it; a real prebuilt init still gets bundled +# for older libkrun builds that expect an external init.krun. +if [[ "$(uname -s)" == "Linux" ]]; then + # Look for a non-empty init.krun in the submodule or system locations. + INIT_KRUN="" + if [[ -n "$LOCAL_INIT_KRUN" ]] && [[ -s "$LOCAL_INIT_KRUN" ]]; then + INIT_KRUN="$LOCAL_INIT_KRUN" + elif [[ -s "$PROJECT_ROOT/libkrun/init/init" ]]; then + INIT_KRUN="$PROJECT_ROOT/libkrun/init/init" + elif [[ -s "/usr/local/share/smolvm/init.krun" ]]; then + INIT_KRUN="/usr/local/share/smolvm/init.krun" + fi + + if [[ -n "$INIT_KRUN" ]]; then + echo "Copying init.krun from $INIT_KRUN..." + cp "$INIT_KRUN" "$DIST_DIR/init.krun" + chmod +x "$DIST_DIR/init.krun" + + # init.krun runs as the guest PID 1, so it must match the target arch. + # libkrun/init/init is a committed binary that does NOT track the build + # host's arch, so a native dist build can silently ship a wrong-arch init + # (the x86_64 release once shipped the aarch64 init). Linux dist builds + # run natively, so the correct arch is `uname -m`; fail loudly otherwise. + host_arch="$(uname -m)" + case "$host_arch" in + x86_64) want="x86-64" ;; + aarch64|arm64) want="aarch64" ;; + *) want="" ;; + esac + init_desc="$(file -b "$DIST_DIR/init.krun")" + if [[ -n "$want" ]] && [[ "$init_desc" != *"$want"* ]]; then + echo "ERROR: init.krun is the wrong architecture for a $host_arch build." >&2 + echo " expected '$want', got: $init_desc" >&2 + echo " source: $INIT_KRUN — rebuild it for $host_arch (see scripts/build-libkrun-linux.sh)." >&2 + exit 1 + fi + echo "init.krun arch OK ($init_desc)" + else + echo "No external init.krun bundled (libkrun 2.0.0 embeds the guest init)." + fi +fi + +# Build agent-rootfs +echo "Building agent-rootfs..." +ROOTFS_SRC="$PROJECT_ROOT/target/agent-rootfs" +if [[ ! -d "$ROOTFS_SRC" ]]; then + echo "Error: target/agent-rootfs not found" + echo "Run ./scripts/build-agent-rootfs.sh first to create the base rootfs." + exit 1 +fi + +# Copy rootfs and update agent binary +# Use cp -a to preserve symlinks (busybox creates many symlinks in /bin) +mkdir -p "$DIST_DIR/agent-rootfs" +cp -a "$ROOTFS_SRC"/* "$DIST_DIR/agent-rootfs/" + +# Copy freshly built agent binary (from release-small profile) +# Remove existing symlinks first (busybox creates init as symlink) +rm -f "$DIST_DIR/agent-rootfs/usr/local/bin/smolvm-agent" +rm -f "$DIST_DIR/agent-rootfs/sbin/init" +cp ./target/release-small/smolvm-agent "$DIST_DIR/agent-rootfs/usr/local/bin/smolvm-agent" +chmod +x "$DIST_DIR/agent-rootfs/usr/local/bin/smolvm-agent" +# Symlink /sbin/init → agent (saves ~1.8MB in initramfs vs a copy). +# The agent handles overlayfs setup + pivot_root internally. +ln -sf /usr/local/bin/smolvm-agent "$DIST_DIR/agent-rootfs/sbin/init" + +echo "Agent rootfs size: $(du -sh "$DIST_DIR/agent-rootfs" | cut -f1)" + +# Create pre-formatted storage template +# This eliminates the e2fsprogs dependency for end users +echo "Creating storage template..." +TEMPLATE_SIZE=$((512 * 1024 * 1024)) # 512MB +TEMPLATE_PATH="$DIST_DIR/storage-template.ext4" + +# Find mkfs.ext4 +MKFS_PATHS=( + "/opt/homebrew/opt/e2fsprogs/sbin/mkfs.ext4" + "/usr/local/opt/e2fsprogs/sbin/mkfs.ext4" + "/opt/homebrew/sbin/mkfs.ext4" + "/usr/local/sbin/mkfs.ext4" + "/sbin/mkfs.ext4" + "/usr/sbin/mkfs.ext4" +) + +MKFS_BIN="" +for path in "${MKFS_PATHS[@]}"; do + if [[ -x "$path" ]]; then + MKFS_BIN="$path" + break + fi +done + +if [[ -z "$MKFS_BIN" ]] && command -v mkfs.ext4 &> /dev/null; then + MKFS_BIN="mkfs.ext4" +fi + +if [[ -z "$MKFS_BIN" ]]; then + echo "Warning: mkfs.ext4 not found, skipping storage template creation" + echo " Users will need e2fsprogs installed" +else + # Set a file's virtual size (sparse), portably (macOS lacks GNU truncate). + extend_sparse() { # $1=path $2=bytes + if command -v truncate >/dev/null 2>&1; then + truncate -s "$2" "$1" + else + perl -e 'truncate($ARGV[0], $ARGV[1]) or die "truncate: $!"' "$1" "$2" + fi + } + + # Create sparse file + dd if=/dev/zero of="$TEMPLATE_PATH" bs=1 count=0 seek=$TEMPLATE_SIZE 2>/dev/null + + # Format with ext4 + "$MKFS_BIN" -F -q -m 0 -L smolvm "$TEMPLATE_PATH" + + # Size to the default storage virtual size (DEFAULT_STORAGE_SIZE_GIB=20) so a + # fresh VM boots from an instant qcow2 overlay (the guest grows the 512 MiB + # ext4 with resize2fs); the runtime treats the template as immutable. Sparse. + extend_sparse "$TEMPLATE_PATH" $((20 * 1024 * 1024 * 1024)) + echo "Storage template created: $(du -h "$TEMPLATE_PATH" | cut -f1) physical (20 GiB virtual)" + + # Create overlay template (same format, different label) + OVERLAY_TEMPLATE_PATH="$DIST_DIR/overlay-template.ext4" + dd if=/dev/zero of="$OVERLAY_TEMPLATE_PATH" bs=1 count=0 seek=$TEMPLATE_SIZE 2>/dev/null + "$MKFS_BIN" -F -q -m 0 -L smolvm-overlay "$OVERLAY_TEMPLATE_PATH" + # Size to the default overlay virtual size (DEFAULT_OVERLAY_SIZE_GIB=10). + extend_sparse "$OVERLAY_TEMPLATE_PATH" $((10 * 1024 * 1024 * 1024)) + echo "Overlay template created: $(du -h "$OVERLAY_TEMPLATE_PATH" | cut -f1) physical (10 GiB virtual)" +fi + +# Copy README +cat > "$DIST_DIR/README.txt" << 'EOF' +smolvm - OCI-native microVM runtime + +INSTALLATION +============ + +1. Extract this archive to a location of your choice: + tar -xzf smolvm-*.tar.gz + cd smolvm-* + +2. Run the smolvm wrapper script. It automatically uses the bundled + agent-rootfs/ directory when present. + +3. (Optional) Add to PATH: + # Add to ~/.bashrc or ~/.zshrc: + export PATH="/path/to/smolvm-directory:$PATH" + +4. (Optional) Create a symlink: + sudo ln -s /path/to/smolvm-directory/smolvm /usr/local/bin/smolvm + +PREREQUISITES +============= + +macOS: + - macOS 11.0 (Big Sur) or later + - Apple Silicon or Intel Mac + +Linux: + - KVM support (/dev/kvm must exist) + - User must have access to /dev/kvm (typically via 'kvm' group) + +USAGE +===== + +Run the 'smolvm' script (not smolvm-bin directly): + + ./smolvm machine run --net --image alpine -- echo "Hello World" + ./smolvm machine create --net --name myvm + ./smolvm machine start --name myvm + ./smolvm machine exec --name myvm -- /bin/sh + ./smolvm machine ls + ./smolvm machine stop --name myvm + ./smolvm machine delete --name myvm + +TROUBLESHOOTING +=============== + +"library not found" errors: + Make sure you're running the 'smolvm' wrapper script, not 'smolvm-bin' + directly. The wrapper sets up the library path automatically. + +"agent did not become ready within 30 seconds": + This usually means the storage disk couldn't be formatted. + Check that the storage-template.ext4 file exists in ~/.smolvm/ + If not, you may need to reinstall smolvm or install e2fsprogs: + macOS: brew install e2fsprogs + Linux: apt install e2fsprogs + +For more information: https://github.com/smolvm/smolvm +EOF + +# Generate checksums. Use sha256sum where available (Linux/coreutils, incl. +# Arch) and fall back to `shasum -a 256` (the macOS default, which has no +# sha256sum). Both emit the same " " format. +echo "Generating checksums..." +if command -v sha256sum >/dev/null 2>&1; then + SHA256_CMD=(sha256sum) +elif command -v shasum >/dev/null 2>&1; then + SHA256_CMD=(shasum -a 256) +else + echo "Error: neither sha256sum nor shasum found; cannot generate checksums" >&2 + exit 1 +fi +(cd "$DIST_DIR" && "${SHA256_CMD[@]}" smolvm smolvm-bin lib/* > checksums.txt) + +# Delete existing tarball. This is because when a new release is created, there could be +# tarball of the old release left in dist/, and ./install-local.sh may pick up the wrong tarball +echo "Cleaning up existing tarball..." +rm -f "smolvm-*.tar.gz" + +# Create tarball. Preserve sparseness so the 20/10 GiB-virtual disk templates +# (a few hundred KiB physical) don't balloon to full size on extraction. GNU tar +# needs --sparse; bsdtar (macOS) detects holes automatically. +echo "Creating tarball..." +cd dist +if tar --version 2>/dev/null | grep -qi gnu; then + tar --sparse -czf "${DIST_NAME}.tar.gz" "${DIST_NAME}" +else + tar -czf "${DIST_NAME}.tar.gz" "${DIST_NAME}" +fi +cd .. + +# Summary +echo "" +echo "Distribution package created:" +echo " dist/${DIST_NAME}.tar.gz" +echo "" +echo "Contents:" +ls -la "$DIST_DIR" +echo "" +echo "To test locally:" +echo " cd $DIST_DIR && ./smolvm --help" +echo "" +echo "To install locally:" +echo " ./scripts/install-local.sh" diff --git a/scripts/build-embedded-node.sh b/scripts/build-embedded-node.sh new file mode 100755 index 0000000..485ed32 --- /dev/null +++ b/scripts/build-embedded-node.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +detect_lib_bundle() { + if [[ "$(uname -s)" == "Linux" ]]; then + echo "$REPO_ROOT/lib/linux-$(uname -m)" + else + echo "$REPO_ROOT/lib" + fi +} + +require_match() { + local dir="$1" + local pattern="$2" + + if ! compgen -G "$dir/$pattern" > /dev/null; then + echo "Error: expected '$pattern' in $dir" >&2 + exit 1 + fi +} + +LIBKRUN_BUNDLE="$(detect_lib_bundle)" + +if [[ ! -d "$LIBKRUN_BUNDLE" ]]; then + echo "Error: embedded library bundle not found: $LIBKRUN_BUNDLE" >&2 + exit 1 +fi + +if [[ "$(uname -s)" == "Linux" ]]; then + require_match "$LIBKRUN_BUNDLE" "libkrun.so*" + require_match "$LIBKRUN_BUNDLE" "libkrunfw.so*" +else + require_match "$LIBKRUN_BUNDLE" "libkrun*.dylib" + require_match "$LIBKRUN_BUNDLE" "libkrunfw*.dylib" +fi + +echo "Using embedded lib bundle: $LIBKRUN_BUNDLE" +echo "Building smolvm-napi..." + +# TODO(release): add CI/release automation for publishing the public package +# and internal platform packages. +# TODO(multi-language): add the python/go/c embedded SDK workspaces that reuse +# the same bundled lib staging model. + +( + cd "$REPO_ROOT" + LIBKRUN_BUNDLE="$LIBKRUN_BUNDLE" cargo build --release -p smolvm-napi +) + +if [[ -f "$REPO_ROOT/sdks/node/package.json" ]]; then + echo "Building embedded Node workspace..." + ( + cd "$REPO_ROOT/sdks/node" + npm run build + ) +fi + +cat <<'EOF' + +Built smolvm-napi and the current-host smolvm-embedded Node packages. + +Useful follow-ups: + - cd sdks/node && npm test + - cd sdks/node && npm run smoke + - cd sdks/node && npm exec --workspace smolvm-embedded tsx examples/basic.ts +EOF diff --git a/scripts/build-libkrun-linux.sh b/scripts/build-libkrun-linux.sh new file mode 100755 index 0000000..a82e498 --- /dev/null +++ b/scripts/build-libkrun-linux.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# +# Build libkrunfw + libkrun for the HOST Linux architecture from smolvm's +# patched submodules, and assemble them into lib/linux-/ in the layout +# the release pipeline + smolvm runtime expect. +# +# Run this ON a native Linux host of the target arch — there is no cross-compile +# here. The intended use is layer 3 of Arm cloud support: SSH into a GCP +# C4A.metal (Axion arm64) worker and run this to produce lib/linux-aarch64/. +# It also works on x86_64 Linux to refresh lib/linux-x86_64/. +# +# Why a native build: libkrunfw compiles a full Linux kernel (linux-6.12.87 + +# patches) with the per-arch config (config-libkrunfw_), and libkrun +# embeds a static init ELF of the guest arch. Both are far simpler and more +# reliable built natively than cross-compiled. +# +# NOTE: smolvm's libkrun/libkrunfw are a PATCHED fork (init.c, virtiofs ioctl, +# DNS egress, etc.) — upstream prebuilt .so files will not work. Always build +# from the in-tree submodules. +# +# Usage: ./scripts/build-libkrun-linux.sh +# GPU=0 skip the GPU feature (drops the virglrenderer dep) +# SKIP_DEPS=1 assume build deps are already installed +# SKIP_LIBKRUNFW=1 reuse the committed lib/linux-/libkrunfw.so and +# rebuild ONLY libkrun. libkrunfw is host-glibc-independent +# (it wraps a guest-kernel blob; its floor is GLIBC_2.2), +# so when the goal is lowering libkrun's glibc floor there's +# no need to recompile the kernel — skips the slow step. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_ROOT" + +if [[ "$(uname -s)" != "Linux" ]]; then + echo "ERROR: this script must run on Linux (native build). For macOS use 'make smolvm' in libkrun/." >&2 + exit 1 +fi + +ARCH="$(uname -m)" # aarch64 | x86_64 +LIBDIR="lib/linux-${ARCH}" +GPU="${GPU:-1}" +GPU_FLAG="GPU=${GPU}" + +echo "=== building libkrun stack for linux-${ARCH} -> ${LIBDIR} ===" + +# --- 1. Build dependencies (Debian/Ubuntu) ----------------------------------- +if [[ "${SKIP_DEPS:-0}" != "1" ]] && command -v apt-get >/dev/null 2>&1; then + echo "--- installing build dependencies ---" + sudo apt-get update -q + # kernel build (libkrunfw) + libkrun (rust/bindgen) + git-lfs/runtime + sudo apt-get install -y -q \ + build-essential flex bison libelf-dev libssl-dev bc cpio rsync kmod \ + python3 python3-pyelftools pkg-config clang llvm libclang-dev curl git git-lfs \ + $([[ "$GPU" == "1" ]] && echo "libvirglrenderer-dev libepoxy-dev libdrm-dev libgbm-dev" || true) + if ! command -v cargo >/dev/null 2>&1; then + echo "--- installing rust toolchain ---" + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + # shellcheck disable=SC1091 + source "$HOME/.cargo/env" + fi +fi +command -v cargo >/dev/null 2>&1 || { source "$HOME/.cargo/env" 2>/dev/null || true; } + +# Ensure the submodules + LFS are present (the kernel config lives in-tree). +# Tolerate running on rsync'd working trees (no parent .git) — the libkrun / +# libkrunfw dirs just need to already contain the patched source. +if [ -d .git ]; then + git submodule update --init libkrun libkrunfw || true + git lfs pull 2>/dev/null || true +fi +[ -d libkrun/src ] \ + || { echo "ERROR: libkrun/ source must be present (clone or rsync it here)." >&2; exit 1; } +# Only the libkrunfw kernel source is needed when we actually rebuild it — under +# SKIP_LIBKRUNFW the committed lib/linux-/libkrunfw.so is reused, so a +# light rsync of just libkrun/ + lib/ is enough (no multi-GB kernel tree). +if [[ "${SKIP_LIBKRUNFW:-0}" != "1" ]]; then + [ -f libkrunfw/Makefile ] \ + || { echo "ERROR: libkrunfw/ source must be present (or set SKIP_LIBKRUNFW=1 to reuse the committed lib)." >&2; exit 1; } +fi + +# bindgen needs to find libclang at runtime on some distros. +if [[ -z "${LIBCLANG_PATH:-}" ]]; then + LIBCLANG_PATH="$(dirname "$(find /usr/lib -name 'libclang.so*' 2>/dev/null | head -1)")" + export LIBCLANG_PATH +fi + +# --- 2. libkrunfw (guest kernel) --------------------------------------------- +if [[ "${SKIP_LIBKRUNFW:-0}" == "1" ]]; then + echo "--- SKIP_LIBKRUNFW=1: reusing committed ${LIBDIR}/libkrunfw.so.* (host-glibc-independent) ---" + KRUNFW_SO="" # signals "do not re-assemble libkrunfw" below +else + echo "--- building libkrunfw (kernel ${ARCH}; this is the slow step) ---" + # Build from INSIDE the dir, NOT `make -C`: `-C` auto-enables -w (print-dir), + # which lands in MAKEFLAGS as a bare `w`; the kernel's `$(MAKE) $(MAKEFLAGS)` + # recipe then dies with "No rule to make target 'w'". -j scales the kernel build. + ( cd libkrunfw && make clean >/dev/null 2>&1 || true; make -j"$(nproc)" GUESTARCH="${ARCH}" ) + KRUNFW_SO="$(ls -1 libkrunfw/libkrunfw.so.*.*.* 2>/dev/null | head -1)" + [[ -n "$KRUNFW_SO" ]] || { echo "ERROR: libkrunfw build produced no .so" >&2; exit 1; } +fi + +# --- 3. init (guest PID-1) --------------------------------------------------- +# Two libkrun layouts, auto-detected by the presence of init/init.c: +# - old (<= libkrun 1.x): PID-1 is a static C init compiled here and handed to +# the build via KRUN_INIT_BINARY_PATH. +# - libkrun 2.0+: PID-1 is a Rust crate (init/) that src/init_blob/build.rs +# cross-compiles to musl ITSELF. `init/init` is only an empty placeholder, and +# KRUN_INIT_BINARY_PATH must be LEFT UNSET — init_blob/build.rs uses that var +# verbatim when set, so pointing it at the empty placeholder embeds a 0-byte +# init and bricks the guest (PID-1 dies, VM never reaches agent-ready). The +# musl std target must be present or the init links dynamically and can't run +# as the guest's PID 1. +INIT_ENV=() +if [[ -f libkrun/init/init.c ]]; then + echo "--- building guest init (legacy C; native ${ARCH} ELF) ---" + cc -O2 -static -Wall -o libkrun/init/init libkrun/init/init.c libkrun/init/dhcp.c + file libkrun/init/init | grep -q 'statically linked' \ + || { echo "ERROR: init/init is not a static ELF" >&2; exit 1; } + INIT_ENV=("KRUN_INIT_BINARY_PATH=$(realpath libkrun/init/init)") +else + echo "--- guest init: Rust crate (init_blob cross-compiles to musl) ---" + rustup target add "${ARCH}-unknown-linux-musl" >/dev/null 2>&1 \ + || echo "WARN: could not add ${ARCH}-unknown-linux-musl; init may link dynamically" +fi + +# --- 4. libkrun (VMM) -------------------------------------------------------- +echo "--- building libkrun (BLK=1 NET=1 ${GPU_FLAG}) ---" +# Link with partial RELRO (lazy binding), NOT full RELRO. The GPU-enabled libkrun +# carries a hard virglrenderer NEEDED that build-dist.sh strips via patchelf so +# non-GPU hosts can dlopen it (paired with RTLD_LAZY in src/agent/krun.rs). That +# strip only holds if symbols bind lazily — full RELRO's BIND_NOW would eagerly +# resolve the now-unprovided virgl symbols and fail the load. Must match the +# relro-level=partial that build-dist.sh applies on its own libkrun compiles. +( cd libkrun && make clean >/dev/null 2>&1 || true; \ + env "${INIT_ENV[@]}" \ + RUSTFLAGS="${RUSTFLAGS:+$RUSTFLAGS }-C relro-level=partial" \ + make --no-print-directory BLK=1 NET=1 "${GPU_FLAG}" ) +KRUN_SO="$(ls -1 libkrun/target/release/libkrun.so.*.*.* 2>/dev/null | head -1)" +[[ -n "$KRUN_SO" ]] || { echo "ERROR: libkrun build produced no .so" >&2; exit 1; } + +# --- 5. Assemble lib/linux-/ (mirror the x86_64 layout + sonames) ------ +echo "--- assembling ${LIBDIR} ---" +mkdir -p "$LIBDIR" + +assemble() { + # $1 = built .so.X.Y.Z $2 = base name (libkrun.so / libkrunfw.so) + local built="$1" base="$2" ver fname major + fname="$(basename "$built")" # e.g. libkrunfw.so.5.3.0 + ver="${fname#"${base}."}" # 5.3.0 + major="${ver%%.*}" # 5 + cp -f "$built" "$LIBDIR/$fname" + ln -sf "$fname" "$LIBDIR/${base}.${major}" # libkrunfw.so.5 -> .so.5.3.0 + ln -sf "${base}.${major}" "$LIBDIR/${base}" # libkrunfw.so -> .so.5 +} + +if [[ -n "$KRUNFW_SO" ]]; then + assemble "$KRUNFW_SO" "libkrunfw.so" +else + echo "--- keeping committed ${LIBDIR}/libkrunfw.so (SKIP_LIBKRUNFW) ---" +fi +# libkrun: x86_64 keeps libkrun.so as a real file too — mirror that. +KRUN_FNAME="$(basename "$KRUN_SO")"; KRUN_VER="${KRUN_FNAME#libkrun.so.}"; KRUN_MAJOR="${KRUN_VER%%.*}" +cp -f "$KRUN_SO" "$LIBDIR/$KRUN_FNAME" +cp -f "$KRUN_SO" "$LIBDIR/libkrun.so" +ln -sf "libkrun.so" "$LIBDIR/libkrun.so.${KRUN_MAJOR}" + +# Record which submodule commits these libs came from, so CI can detect a stale +# bundle (scripts/check-libkrun-provenance.sh). When libkrunfw was reused rather +# than rebuilt, keep its recorded commit via --skip-libkrunfw. +STAMP_ARGS=("$LIBDIR") +[[ -z "$KRUNFW_SO" ]] && STAMP_ARGS+=(--skip-libkrunfw) +"$(dirname "$0")/stamp-libkrun-provenance.sh" "${STAMP_ARGS[@]}" + +echo "" +echo "=== done: ${LIBDIR} ===" +ls -la "$LIBDIR" +echo "" +echo "Commit (these are git-LFS tracked via .gitattributes 'lib/linux-aarch64/*.so'):" +echo " git add ${LIBDIR} && git commit -m 'feat: linux-${ARCH} libkrun/libkrunfw libs'" diff --git a/scripts/check-krun-exports.sh b/scripts/check-krun-exports.sh new file mode 100755 index 0000000..d2ef2d3 --- /dev/null +++ b/scripts/check-krun-exports.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Verify every bundled libkrun exports all the symbols smolvm *requires* at +# runtime — the `load_sym!` set in src/agent/krun.rs. +# +# Why this exists: smolvm dlopen()s the platform libkrun and resolves each +# required symbol. A lib missing one fails at startup with +# "symbol not found: " on that OS only. CI cross-compiles smolvm and +# packages the libs but never dlopens them on the target, so a lib built with +# the wrong feature set (e.g. the Windows krun.dll shipped once without the +# `blk`/disk API) sails through green and breaks only on a user's machine. +# This check is that missing dlopen, done statically at build time. +# +# Optional symbols (load_optional_sym!) are intentionally NOT required. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +KRUN_RS="$ROOT/src/agent/krun.rs" + +# Source of truth: the identifiers passed to load_sym!(...) (required), not the +# string literals passed to load_optional_sym!(...) (optional). +mapfile -t REQUIRED < <(grep -oE 'load_sym!\(krun_[a-z0-9_]+\)' "$KRUN_RS" \ + | sed -E 's/.*\((krun_[a-z0-9_]+)\)/\1/' | sort -u) +if [ "${#REQUIRED[@]}" -eq 0 ]; then + echo "ERROR: parsed 0 required symbols from $KRUN_RS — check the load_sym! pattern" + exit 1 +fi +echo "smolvm requires ${#REQUIRED[@]} krun symbols (load_sym!):" +printf ' %s\n' "${REQUIRED[@]}" +echo + +# Print the exported krun_* names of a shared library, format-detected. +exports_of() { + local f="$1" + case "$f" in + *.dll) + local od + od="$(command -v x86_64-w64-mingw32-objdump || command -v llvm-objdump || command -v objdump)" + # PE export directory lists the exported names; grep the krun_* ones. + "$od" -p "$f" 2>/dev/null | grep -oE 'krun_[a-z0-9_]+' + ;; + *) + # ELF (.so) or Mach-O (.dylib). llvm-nm reads both; fall back to nm. + local nmtool + nmtool="$(command -v llvm-nm || command -v nm)" + # Defined external text symbols; leading '_' on Mach-O is ignored by the + # krun_* grep. Try dynamic (-D, ELF) then general defined (-gU, Mach-O). + { "$nmtool" -D --defined-only "$f" 2>/dev/null; "$nmtool" -gU "$f" 2>/dev/null; } \ + | grep -oE 'krun_[a-z0-9_]+' + ;; + esac | sort -u +} + +# Resolve to the real files (skip symlinks; pick the versioned .so). +libs=() +[ -f "$ROOT/lib/libkrun.dylib" ] && libs+=("$ROOT/lib/libkrun.dylib") +for d in linux-x86_64 linux-aarch64; do + so="$(ls "$ROOT/lib/$d"/libkrun.so.*.* 2>/dev/null | head -1 || true)" + [ -n "$so" ] && libs+=("$so") +done +[ -f "$ROOT/lib/windows-x86_64/krun.dll" ] && libs+=("$ROOT/lib/windows-x86_64/krun.dll") + +rc=0 +for lib in "${libs[@]}"; do + rel="${lib#$ROOT/}" + present="$(exports_of "$lib")" + if [ -z "$present" ]; then + echo "⚠ $rel: could not read exports (missing tool for this format?) — skipping" + continue + fi + missing=() + for sym in "${REQUIRED[@]}"; do + grep -qxF "$sym" <<<"$present" || missing+=("$sym") + done + if [ "${#missing[@]}" -eq 0 ]; then + echo "✓ $rel — all ${#REQUIRED[@]} required symbols exported" + else + echo "✗ $rel — MISSING ${#missing[@]} required symbol(s): ${missing[*]}" + rc=1 + fi +done + +if [ "$rc" -ne 0 ]; then + echo + echo "FAIL: a bundled libkrun is missing symbols smolvm requires — it would" + echo "fail at dlopen on that platform. Rebuild that lib with the correct" + echo "features (the disk API needs the 'blk' feature) and re-stamp lib/." +fi +exit "$rc" diff --git a/scripts/check-libkrun-provenance.sh b/scripts/check-libkrun-provenance.sh new file mode 100755 index 0000000..de1c270 --- /dev/null +++ b/scripts/check-libkrun-provenance.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Verify the committed libkrun/libkrunfw binaries in lib/ match the pinned +# submodule commits. Fails loudly if a lib dir is stale or unstamped — this is +# the guard that stops a stale prebuilt libkrun (e.g. one missing the latest TSI +# egress enforcement) from shipping in a release. +# +# Uses the superproject's recorded submodule commit (the gitlink), so it works +# in CI without initialising the submodules. Run in CI (see ci.yml). +# +# PASS → every lib dir's libkrun.provenance matches the pinned submodules +# FAIL → a lib dir is missing provenance or was built from an older submodule +# commit; rebuild + restamp before merging. + +set -euo pipefail +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# Pinned submodule commits (gitlinks recorded in the superproject). +want_krun="$(git rev-parse "HEAD:libkrun")" +want_fw="$(git rev-parse "HEAD:libkrunfw")" + +# Lib dirs that bundle a libkrun (macOS dylib + each Linux arch + Windows DLL). +LIB_DIRS=(lib lib/linux-x86_64 lib/linux-aarch64 lib/windows-x86_64) + +fail=0 +for dir in "${LIB_DIRS[@]}"; do + # Skip dirs that don't actually ship a libkrun. The lib is `libkrun.*` on + # macOS/Linux and `krun.dll` on Windows. + { compgen -G "$dir/libkrun.*" || compgen -G "$dir/krun.dll"; } >/dev/null 2>&1 || continue + prov="$dir/libkrun.provenance" + + if [[ ! -f "$prov" ]]; then + echo "FAIL $dir: no libkrun.provenance — binary's source commit is unknown." + fail=1 + continue + fi + got_krun="$(grep '^libkrun=' "$prov" | cut -d= -f2)" + got_fw="$(grep '^libkrunfw=' "$prov" | cut -d= -f2)" + + dir_ok=1 + if [[ "$got_krun" != "$want_krun" ]]; then + echo "FAIL $dir: libkrun built from $got_krun but submodule is pinned at $want_krun" + dir_ok=0; fail=1 + fi + if [[ "$got_fw" != "$want_fw" ]]; then + echo "FAIL $dir: libkrunfw built from $got_fw but submodule is pinned at $want_fw" + dir_ok=0; fail=1 + fi + [[ "$dir_ok" == "1" ]] && echo "OK $dir (libkrun=$got_krun)" +done + +if [[ "$fail" != "0" ]]; then + cat >&2 </ +then commit the updated lib/ (binaries + libkrun.provenance). This is the guard +that prevents shipping a stale libkrun (e.g. missing TSI egress enforcement). +EOF + exit 1 +fi +echo "All bundled libkrun binaries match the pinned submodules." diff --git a/scripts/check-secrets-guards.sh b/scripts/check-secrets-guards.sh new file mode 100755 index 0000000..8b1453a --- /dev/null +++ b/scripts/check-secrets-guards.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# +# Static guardrails for host-side secret references. +# +# The whole point of secret refs is that secret *plaintext* is resolved on the +# host at exec time and never persists anywhere a guest, a record, or the +# database can see it. smolvm stores no secret material itself; only opaque +# *references* (which env var / file a value comes from) are persisted. These +# greps fail CI the moment a change breaks one of those invariants, so a +# reviewer doesn't have to spot it by eye. +# +# The checks are deliberately conservative: they match on patterns that are +# never legitimate, so a green run means "no known-bad pattern present", not +# "provably correct". + +set -euo pipefail +cd "$(dirname "$0")/.." + +fail=0 +note() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; fail=1; } +ok() { printf ' \033[32mok\033[0m %s\n' "$1"; } + +echo "secrets guardrails:" + +# 1. SecretRef must reject unknown fields, so a Smolfile/API typo +# (`from_stor`) is a hard error instead of a silently-ignored empty ref. +if grep -q 'deny_unknown_fields' crates/smolvm-protocol/src/secrets.rs; then + ok "SecretRef denies unknown fields" +else + note "SecretRef is missing #[serde(deny_unknown_fields)] in crates/smolvm-protocol/src/secrets.rs" +fi + +# 2. resolve_secret_ref must hand back a Zeroizing buffer so resolved plaintext +# is scrubbed from memory on drop rather than lingering on the heap. +if grep -A6 'fn resolve_secret_ref(' src/secrets.rs | grep -q 'Zeroizing'; then + ok "resolve_secret_ref returns a Zeroizing buffer" +else + note "resolve_secret_ref no longer returns Zeroizing — plaintext would not be scrubbed" +fi + +# 3. The protocol SecretRef must carry only references, never an inline value. +# A `value`/`plaintext`/`secret` field would let plaintext ride along into +# every record and DB row that persists a ref. +if grep -nE '^\s*pub (value|plaintext|secret)\b' crates/smolvm-protocol/src/secrets.rs; then + note "SecretRef gained an inline plaintext field — refs must stay opaque" +else + ok "SecretRef carries references only (no inline plaintext field)" +fi + +# 4. Resolved plaintext must never be assigned back onto a persisted record's +# env. Persisted `*.env =` assignments may only come from plain sources +# (parse_env_list, the request, an override) — never from a resolver. +# NOTE: this catches the DIRECT case only; multi-hop laundering (resolver -> +# local -> defaults.env -> persist) is NOT visible to grep and still needs +# review — the durable fix is a non-Serialize `Secret` newtype. +# (Earlier this used a literal `**` pathspec + `\s`, which git grep silently +# matches against nothing, so the check never ran. Use real dir pathspecs and +# POSIX-ERE ` *`, and scan the pack crate too.) +if git grep -nE '\.env *=' -- src crates \ + | grep -E 'resolve_refs_to_env|record_env_with_secrets|resolve_secret_ref|resolve_secret_refs_for_env'; then + note "a persisted .env is assigned from a secret resolver — plaintext would reach the DB" +else + ok "no persisted .env directly assigned from a secret resolver" +fi + +# 5. A .smolmachine is a portable, untrusted artifact: its packed secret refs +# must be validated/resolved under the Untrusted scope (which rejects every +# source kind) so a downloaded pack cannot read the running host's env/files +# via from_env/from_file. +if git grep -nE 'manifest\.secret_refs' -- src/cli/pack_run.rs \ + | grep -qE 'RecordReplay|TrustedLocal'; then + note "pack_run resolves manifest secret refs under a trusting scope — host env/file exfil risk" +else + ok "packed secret refs resolve under Untrusted scope" +fi + +# 6. The resolved-plaintext `Secret` newtype must never gain a Serialize/Display +# impl, and must not be `derive`d with Serialize. Either would re-open the +# accidental-leak surface (JSON/logs) the type exists to close — the no-leak +# invariant is now enforced by the type system, and this guards that. +if git grep -nE 'impl[^=]*(Serialize|Display)[^=]*for Secret\b' -- src/secrets.rs \ + || git grep -nE -B1 'struct Secret\(' -- src/secrets.rs | grep -q 'Serialize'; then + note "the Secret newtype gained a Serialize/Display impl — resolved plaintext could leak" +else + ok "Secret has no Serialize/Display impl (plaintext can't serialize/log by accident)" +fi + +if [ "$fail" -ne 0 ]; then + echo "secrets guardrails FAILED" >&2 + exit 1 +fi +echo "secrets guardrails passed" diff --git a/scripts/install-local.sh b/scripts/install-local.sh new file mode 100755 index 0000000..de3284b --- /dev/null +++ b/scripts/install-local.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +# Install smolvm from a local tarball or build directory +# +# Usage: +# ./scripts/install-local.sh # Install from dist/ +# ./scripts/install-local.sh path/to/tarball # Install from tarball +# ./scripts/install-local.sh --uninstall # Uninstall + +set -e + +INSTALL_PREFIX="${HOME}/.smolvm" +BIN_DIR="${HOME}/.local/bin" + +# Colors +if [ -t 1 ]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[0;33m' + BLUE='\033[0;34m' + BOLD='\033[1m' + NC='\033[0m' +else + RED='' GREEN='' YELLOW='' BLUE='' BOLD='' NC='' +fi + +info() { echo -e "${BLUE}info:${NC} $1"; } +success() { echo -e "${GREEN}success:${NC} $1"; } +warn() { echo -e "${YELLOW}warning:${NC} $1"; } +error() { echo -e "${RED}error:${NC} $1" >&2; } + +# Check platform requirements +check_requirements() { + # macOS-specific checks + if [[ "$(uname -s)" == "Darwin" ]]; then + local macos_version + macos_version=$(sw_vers -productVersion 2>/dev/null || echo "0.0") + local major_version + major_version=$(echo "$macos_version" | cut -d. -f1) + + if [[ "$major_version" -lt 11 ]]; then + error "smolvm requires macOS 11.0 or later (you have $macos_version)" + exit 1 + fi + fi + + # Linux-specific checks + if [[ "$(uname -s)" == "Linux" ]]; then + if [[ ! -e /dev/kvm ]]; then + warn "/dev/kvm not found. smolvm requires KVM support." + warn "Make sure your system supports virtualization and KVM is enabled." + fi + fi +} + +# Find tarball in dist directory +find_tarball() { + local platform + platform="$(uname -s | tr '[:upper:]' '[:lower:]')-$(uname -m)" + + # Try exact match first + local tarball + tarball=$(find dist -maxdepth 1 -name "smolvm-*-${platform}.tar.gz" 2>/dev/null | head -1) + + # Try with arm64 variant + if [[ -z "$tarball" && "$platform" == *-aarch64 ]]; then + platform="${platform/aarch64/arm64}" + tarball=$(find dist -maxdepth 1 -name "smolvm-*-${platform}.tar.gz" 2>/dev/null | head -1) + fi + + echo "$tarball" +} + +# Install from directory +install_from_dir() { + local src_dir="$1" + + # Verify required files exist + if [[ ! -f "$src_dir/smolvm" ]] || [[ ! -f "$src_dir/smolvm-bin" ]]; then + error "Invalid smolvm distribution: missing smolvm or smolvm-bin" + exit 1 + fi + + if [[ ! -d "$src_dir/lib" ]]; then + error "Invalid smolvm distribution: missing lib directory" + exit 1 + fi + + # Get version from distribution or default + local version="dev" + if [[ -f "$src_dir/README.txt" ]]; then + version=$(echo "$src_dir" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || echo "dev") + fi + + info "Installing smolvm $version to $INSTALL_PREFIX" + + # Create installation directory + mkdir -p "$INSTALL_PREFIX" + + # Remove old installation + rm -rf "$INSTALL_PREFIX/lib" + rm -f "$INSTALL_PREFIX/smolvm" "$INSTALL_PREFIX/smolvm-bin" + + # Copy files + cp -r "$src_dir/lib" "$INSTALL_PREFIX/" + cp "$src_dir/smolvm" "$INSTALL_PREFIX/" + cp "$src_dir/smolvm-bin" "$INSTALL_PREFIX/" + chmod +x "$INSTALL_PREFIX/smolvm" "$INSTALL_PREFIX/smolvm-bin" + + # Install agent-rootfs to data directory + local data_dir + if [[ "$(uname -s)" == "Darwin" ]]; then + data_dir="$HOME/Library/Application Support/smolvm" + else + data_dir="${XDG_DATA_HOME:-$HOME/.local/share}/smolvm" + fi + + if [[ -d "$src_dir/agent-rootfs" ]]; then + info "Installing agent-rootfs to $data_dir..." + mkdir -p "$data_dir" + rm -rf "$data_dir/agent-rootfs" + # Use cp -a to preserve symlinks (busybox creates many symlinks) + cp -a "$src_dir/agent-rootfs" "$data_dir/" + success "Agent rootfs installed" + else + warn "agent-rootfs not found in distribution" + fi + + # Copy init.krun if present (Linux only, required by libkrunfw kernel) + if [[ -f "$src_dir/init.krun" ]]; then + info "Installing init.krun to $data_dir..." + cp "$src_dir/init.krun" "$data_dir/init.krun" + chmod +x "$data_dir/init.krun" + success "init.krun installed" + fi + + # Store version + echo "$version" > "$INSTALL_PREFIX/.version" + + # Create symlink + mkdir -p "$BIN_DIR" + ln -sf "$INSTALL_PREFIX/smolvm" "$BIN_DIR/smolvm" + + success "Installed smolvm to $INSTALL_PREFIX" + success "Symlink created at $BIN_DIR/smolvm" +} + +# Uninstall +uninstall() { + info "Uninstalling smolvm..." + + if [[ -d "$INSTALL_PREFIX" ]]; then + rm -rf "$INSTALL_PREFIX" + success "Removed $INSTALL_PREFIX" + fi + + if [[ -L "$BIN_DIR/smolvm" ]]; then + rm -f "$BIN_DIR/smolvm" + success "Removed $BIN_DIR/smolvm" + fi + + success "smolvm uninstalled" +} + +# Main +main() { + echo "" + echo -e "${BOLD}smolvm local installer${NC}" + echo "" + + # Handle uninstall + if [[ "$1" == "--uninstall" ]]; then + uninstall + exit 0 + fi + + # Check platform requirements + check_requirements + + local tarball="$1" + local tmp_dir="" + local install_dir="" + + # If no argument, find tarball in dist/ + if [[ -z "$tarball" ]]; then + tarball=$(find_tarball) + if [[ -z "$tarball" ]]; then + error "No tarball found in dist/" + error "Run './scripts/build-dist.sh' first or specify a tarball path." + exit 1 + fi + info "Found tarball: $tarball" + fi + + # Check if it's a tarball or directory + if [[ -f "$tarball" && "$tarball" == *.tar.gz ]]; then + # Extract tarball + tmp_dir=$(mktemp -d) + info "Extracting $tarball..." + tar -xzf "$tarball" -C "$tmp_dir" + + install_dir=$(find "$tmp_dir" -maxdepth 1 -type d -name "smolvm-*" | head -1) + if [[ -z "$install_dir" ]]; then + error "Could not find smolvm directory in tarball" + rm -rf "$tmp_dir" + exit 1 + fi + elif [[ -d "$tarball" ]]; then + # Use directory directly + install_dir="$tarball" + else + error "Not a valid tarball or directory: $tarball" + exit 1 + fi + + # Install + install_from_dir "$install_dir" + + # Cleanup + if [[ -n "$tmp_dir" ]]; then + rm -rf "$tmp_dir" + fi + + # Check PATH + echo "" + if ! echo "$PATH" | grep -q "$BIN_DIR"; then + warn "$BIN_DIR is not in your PATH" + echo "" + echo "Add to your shell profile:" + echo " export PATH=\"$BIN_DIR:\$PATH\"" + echo "" + fi + + echo "Test your installation:" + echo " smolvm --help" + echo "" +} + +main "$@" diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..3721ef7 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,722 @@ +#!/usr/bin/env bash +# smolvm installer +# +# CANONICAL SOURCE: scripts/install.sh in the smolvm repo +# The website copy at smolmachines/docs/public/install.sh must be kept in sync. +# After editing this file, copy it to smolmachines/docs/public/install.sh +# +# Usage: +# curl -sSL https://smolmachines.com/install.sh | bash +# curl -sSL https://smolmachines.com/install.sh | bash -s -- --version 0.1.1 +# curl -sSL https://smolmachines.com/install.sh | bash -s -- --prefix /opt/smolvm +# +# Options: +# --version VERSION Install specific version (default: latest) +# --prefix DIR Install to DIR (default: ~/.smolvm) +# --no-modify-path Don't modify shell profile +# --uninstall Remove smolvm installation +# --help Show this help message + +set -e + +# Configuration +GITHUB_REPO="smol-machines/smolvm" +INSTALL_PREFIX="${HOME}/.smolvm" +BIN_DIR="${HOME}/.local/bin" +MODIFY_PATH=true +VERSION="" +UNINSTALL=false + +# Colors (disabled if not a terminal) +if [ -t 1 ]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[0;33m' + BLUE='\033[0;34m' + BOLD='\033[1m' + NC='\033[0m' # No Color +else + RED='' + GREEN='' + YELLOW='' + BLUE='' + BOLD='' + NC='' +fi + +# Print functions +info() { + echo -e "${BLUE}info:${NC} $1" +} + +success() { + echo -e "${GREEN}success:${NC} $1" +} + +warn() { + echo -e "${YELLOW}warning:${NC} $1" +} + +error() { + echo -e "${RED}error:${NC} $1" >&2 +} + +# Detect platform +detect_platform() { + local os arch + + # Detect OS + case "$(uname -s)" in + Darwin) + os="darwin" + ;; + Linux) + os="linux" + ;; + *) + error "Unsupported operating system: $(uname -s)" + error "smolvm supports macOS and Linux only." + exit 1 + ;; + esac + + # Detect architecture + case "$(uname -m)" in + x86_64|amd64) + arch="x86_64" + ;; + aarch64|arm64) + arch="aarch64" + ;; + *) + error "Unsupported architecture: $(uname -m)" + error "smolvm supports x86_64 and arm64 only." + exit 1 + ;; + esac + + echo "${os}-${arch}" +} + +# Check system requirements +check_requirements() { + local platform="$1" + + # Check for curl or wget + if ! command -v curl &> /dev/null && ! command -v wget &> /dev/null; then + error "curl or wget is required to download smolvm." + exit 1 + fi + + # Check for tar + if ! command -v tar &> /dev/null; then + error "tar is required to extract smolvm." + exit 1 + fi + + # macOS-specific checks + if [[ "$platform" == darwin-* ]]; then + # Check macOS version (need 11.0+) + local macos_version + macos_version=$(sw_vers -productVersion 2>/dev/null || echo "0.0") + local major_version + major_version=$(echo "$macos_version" | cut -d. -f1) + + if [[ "$major_version" -lt 11 ]]; then + error "smolvm requires macOS 11.0 or later (you have $macos_version)" + exit 1 + fi + fi + + # Linux-specific checks + if [[ "$platform" == linux-* ]]; then + # Check for KVM support + if [[ ! -e /dev/kvm ]]; then + warn "/dev/kvm not found. smolvm requires KVM support." + warn "" + warn "To enable KVM:" + warn " 1. Ensure virtualization is enabled in your BIOS/UEFI" + warn " 2. Load the KVM kernel module:" + warn " sudo modprobe kvm" + warn " sudo modprobe kvm_intel # For Intel CPUs" + warn " sudo modprobe kvm_amd # For AMD CPUs" + warn "" + warn "For persistent loading, add to /etc/modules-load.d/kvm.conf:" + warn " kvm" + warn " kvm_intel # or kvm_amd" + elif [[ ! -r /dev/kvm ]] || [[ ! -w /dev/kvm ]]; then + warn "Cannot access /dev/kvm (permission denied)." + warn "" + warn "Add your user to the 'kvm' group:" + warn " sudo usermod -aG kvm $USER" + warn "" + warn "Then log out and log back in for the change to take effect." + else + info "KVM access verified" + fi + fi +} + +# Get latest version from GitHub +get_latest_version() { + local url="https://api.github.com/repos/${GITHUB_REPO}/releases/latest" + local version + + if command -v curl &> /dev/null; then + version=$(curl -sSL "$url" 2>/dev/null | grep '"tag_name"' | sed -E 's/.*"tag_name": *"v?([^"]+)".*/\1/') + else + version=$(wget -qO- "$url" 2>/dev/null | grep '"tag_name"' | sed -E 's/.*"tag_name": *"v?([^"]+)".*/\1/') + fi + + if [[ -z "$version" ]]; then + # Fallback to a default version if GitHub API fails + echo "0.1.1" + else + echo "$version" + fi +} + +# Download file +download() { + local url="$1" + local output="$2" + + info "Downloading $url" + + if command -v curl &> /dev/null; then + curl -fSL --progress-bar "$url" -o "$output" + else + wget --show-progress -q "$url" -O "$output" + fi +} + +# Get download URL for a version and platform +get_download_url() { + local version="$1" + local platform="$2" + + # Convert platform format (darwin-aarch64 -> darwin-arm64 for compatibility) + local download_platform="$platform" + if [[ "$platform" == *-aarch64 ]]; then + download_platform="${platform/aarch64/arm64}" + fi + + echo "https://github.com/${GITHUB_REPO}/releases/download/v${version}/smolvm-${version}-${download_platform}.tar.gz" +} + +# Get checksum URL for a version and platform +get_checksum_url() { + local version="$1" + echo "https://github.com/${GITHUB_REPO}/releases/download/v${version}/checksums.sha256" +} + +# Verify file checksum +verify_checksum() { + local file="$1" + local checksums_file="$2" + local filename + filename=$(basename "$file") + + # Extract expected checksum for this file + local expected + expected=$(grep "$filename" "$checksums_file" 2>/dev/null | awk '{print $1}') + + if [[ -z "$expected" ]]; then + warn "Checksum not found for $filename, skipping verification" + return 0 + fi + + # Calculate actual checksum + local actual + if command -v sha256sum &> /dev/null; then + actual=$(sha256sum "$file" | awk '{print $1}') + elif command -v shasum &> /dev/null; then + actual=$(shasum -a 256 "$file" | awk '{print $1}') + else + warn "sha256sum/shasum not found, skipping checksum verification" + return 0 + fi + + if [[ "$expected" != "$actual" ]]; then + error "Checksum verification failed!" + error " Expected: $expected" + error " Actual: $actual" + return 1 + fi + + info "Checksum verified" + return 0 +} + +# Install smolvm +install_smolvm() { + local version="$1" + local platform="$2" + local prefix="$3" + + local url + url=$(get_download_url "$version" "$platform") + local checksums_url + checksums_url=$(get_checksum_url "$version") + local tmp_dir + tmp_dir=$(mktemp -d) + local archive_name + archive_name=$(basename "$url") + local archive="${tmp_dir}/${archive_name}" + local checksums="${tmp_dir}/checksums.sha256" + + # Download archive + download "$url" "$archive" || { + error "Failed to download smolvm from $url" + error "Please check if version $version exists for platform $platform" + rm -rf "$tmp_dir" + exit 1 + } + + # Download and verify checksums (optional - don't fail if checksums unavailable) + if download "$checksums_url" "$checksums" 2>/dev/null; then + verify_checksum "$archive" "$checksums" || { + error "Archive failed checksum verification - aborting for security" + rm -rf "$tmp_dir" + exit 1 + } + else + warn "Checksums not available for this release, skipping verification" + fi + + # Extract + info "Extracting archive..." + tar -xzf "$archive" -C "$tmp_dir" || { + error "Failed to extract archive" + rm -rf "$tmp_dir" + exit 1 + } + + # Find extracted directory + local extracted_dir + extracted_dir=$(find "$tmp_dir" -maxdepth 1 -type d -name "smolvm-*" | head -1) + + if [[ -z "$extracted_dir" ]]; then + error "Could not find extracted smolvm directory" + rm -rf "$tmp_dir" + exit 1 + fi + + # Safety: refuse to install to system directories + case "$prefix" in + /|/usr|/usr/*|/bin|/sbin|/lib|/lib64|/etc|/var|/opt|/tmp|/System|/System/*|/Library|/Library/*) + error "Refusing to install to system directory: $prefix" + error "Use a user-writable directory like ~/.smolvm (the default)" + rm -rf "$tmp_dir" + exit 1 + ;; + esac + + # Safety: warn if installing outside home directory + if [[ "$prefix" != "$HOME"* ]] && [[ "$prefix" != /tmp/* ]]; then + warn "Installing outside of home directory: $prefix" + warn "This will remove $prefix/lib/ and $prefix/smolvm if they exist." + if [ -t 0 ]; then + printf "Continue? [y/N] " + read -r REPLY + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + error "Aborted." + rm -rf "$tmp_dir" + exit 1 + fi + else + error "Non-interactive install to non-home path. Aborting for safety." + error "Use --prefix with a path under \$HOME, or run interactively." + rm -rf "$tmp_dir" + exit 1 + fi + fi + + # Create installation directory + info "Installing to $prefix..." + mkdir -p "$prefix" + + # Remove old smolvm installation files only (not arbitrary lib/ directories) + if [[ -d "$prefix/lib" ]] && [[ -f "$prefix/.version" ]]; then + # Only remove lib/ if this looks like an existing smolvm installation + rm -rf "$prefix/lib" + elif [[ -d "$prefix/lib" ]]; then + warn "$prefix/lib exists but no .version file found — skipping lib/ removal" + warn "If this is a previous smolvm install, remove it manually first" + fi + if [[ -f "$prefix/smolvm" ]]; then + rm -f "$prefix/smolvm" + fi + if [[ -f "$prefix/smolvm-bin" ]]; then + rm -f "$prefix/smolvm-bin" + fi + if [[ -f "$prefix/smol" ]]; then + rm -f "$prefix/smol" + fi + if [[ -f "$prefix/smol-bin" ]]; then + rm -f "$prefix/smol-bin" + fi + if [[ -f "$prefix/smolvm-stub" ]]; then + rm -f "$prefix/smolvm-stub" + fi + if [[ -f "$prefix/storage-template.ext4" ]]; then + rm -f "$prefix/storage-template.ext4" + fi + if [[ -f "$prefix/overlay-template.ext4" ]]; then + rm -f "$prefix/overlay-template.ext4" + fi + + # Copy files + cp -r "$extracted_dir/lib" "$prefix/" + cp "$extracted_dir/smolvm" "$prefix/" + cp "$extracted_dir/smolvm-bin" "$prefix/" + chmod +x "$prefix/smolvm" + chmod +x "$prefix/smolvm-bin" + + # Copy the unified `smol` CLI if the distribution includes it. Older + # engine-only tarballs won't have it; newer ones ship both. + local has_smol=false + if [[ -f "$extracted_dir/smol" ]] && [[ -f "$extracted_dir/smol-bin" ]]; then + cp "$extracted_dir/smol" "$prefix/" + cp "$extracted_dir/smol-bin" "$prefix/" + chmod +x "$prefix/smol" + chmod +x "$prefix/smol-bin" + has_smol=true + fi + + # Copy disk templates if present + if [[ -f "$extracted_dir/storage-template.ext4" ]]; then + cp "$extracted_dir/storage-template.ext4" "$prefix/" + fi + if [[ -f "$extracted_dir/overlay-template.ext4" ]]; then + cp "$extracted_dir/overlay-template.ext4" "$prefix/" + fi + + # Size the disk templates to their default virtual size at install time, so + # the runtime boots fresh VMs from an instant qcow2 copy-on-write overlay + # (no per-boot template copy) instead of mutating a shared template lazily. + # The ext4 inside stays 512 MiB; the guest grows it with resize2fs at boot. + # Sparse, so it costs no real disk. Linux-only (the overlay path is gated to + # Linux) and needs `truncate` (GNU coreutils); skipped otherwise, in which + # case the runtime safely falls back to the copy path. The sizes mirror + # DEFAULT_STORAGE_SIZE_GIB (20) and DEFAULT_OVERLAY_SIZE_GIB (10). + if command -v truncate >/dev/null 2>&1; then + [[ -f "$prefix/storage-template.ext4" ]] && truncate -s 20G "$prefix/storage-template.ext4" + [[ -f "$prefix/overlay-template.ext4" ]] && truncate -s 10G "$prefix/overlay-template.ext4" + fi + + # Install agent-rootfs to data directory + local data_dir + if [[ "$(uname -s)" == "Darwin" ]]; then + data_dir="$HOME/Library/Application Support/smolvm" + else + data_dir="${XDG_DATA_HOME:-$HOME/.local/share}/smolvm" + fi + + if [[ -d "$extracted_dir/agent-rootfs" ]]; then + info "Installing agent-rootfs to $data_dir..." + mkdir -p "$data_dir" + rm -rf "$data_dir/agent-rootfs" + # Use cp -a to preserve symlinks (busybox creates many symlinks) + cp -a "$extracted_dir/agent-rootfs" "$data_dir/" + else + warn "agent-rootfs not found in distribution - some features may not work" + fi + + # Copy init.krun if present (Linux only, required by libkrunfw kernel) + if [[ -f "$extracted_dir/init.krun" ]]; then + info "Installing init.krun to $data_dir..." + cp "$extracted_dir/init.krun" "$data_dir/init.krun" + chmod +x "$data_dir/init.krun" + fi + + # Store version info + echo "$version" > "$prefix/.version" + + # Cleanup + rm -rf "$tmp_dir" + + # macOS: files downloaded via curl carry a com.apple.quarantine attribute, + # and Gatekeeper will refuse to run the binaries (which call the hypervisor) + # until it's cleared. Strip it from the whole install tree, then check the + # code signature so we can warn clearly instead of failing cryptically. + if [[ "$(uname -s)" == "Darwin" ]]; then + if command -v xattr &> /dev/null; then + xattr -dr com.apple.quarantine "$prefix" 2>/dev/null || true + fi + if command -v codesign &> /dev/null; then + local _sig_bin="$prefix/smolvm-bin" + [[ "$has_smol" == true ]] && _sig_bin="$prefix/smol-bin" + if ! codesign --verify --deep "$_sig_bin" 2>/dev/null; then + warn "The installed binary is not validly code-signed." + warn "It needs the com.apple.security.hypervisor entitlement to start VMs." + elif command -v spctl &> /dev/null && ! spctl --assess --type execute "$_sig_bin" &> /dev/null; then + warn "Binary is signed but not notarized by Apple." + warn "It will run (quarantine was cleared), but Gatekeeper may warn on first launch." + fi + fi + fi + + # Create symlinks in bin directory. `smol` is the primary, user-facing CLI; + # `smolvm` remains available as the lower-level engine command. + mkdir -p "$BIN_DIR" + ln -sf "$prefix/smolvm" "$BIN_DIR/smolvm" + if [[ "$has_smol" == true ]]; then + ln -sf "$prefix/smol" "$BIN_DIR/smol" + success "smol $version installed to $prefix (also installed: smolvm)" + else + success "smolvm $version installed to $prefix" + fi +} + +# Modify shell profile to add to PATH +modify_path() { + local bin_dir="$1" + local profile="" + local export_line="export PATH=\"$bin_dir:\$PATH\"" + + # Determine shell profile + case "$SHELL" in + */zsh) + profile="$HOME/.zshrc" + ;; + */bash) + if [[ -f "$HOME/.bash_profile" ]]; then + profile="$HOME/.bash_profile" + else + profile="$HOME/.bashrc" + fi + ;; + */fish) + profile="$HOME/.config/fish/config.fish" + export_line="set -gx PATH $bin_dir \$PATH" + ;; + *) + profile="$HOME/.profile" + ;; + esac + + # Check if already in PATH + if echo "$PATH" | grep -q "$bin_dir"; then + info "$bin_dir is already in PATH" + return + fi + + # Check if already in profile + if [[ -f "$profile" ]] && grep -q "$bin_dir" "$profile" 2>/dev/null; then + info "PATH already configured in $profile" + return + fi + + # Add to profile + info "Adding $bin_dir to PATH in $profile" + echo "" >> "$profile" + echo "# smolvm" >> "$profile" + echo "$export_line" >> "$profile" + + warn "PATH updated. Run 'source $profile' or open a new terminal." +} + +# Uninstall smolvm +uninstall_smolvm() { + local prefix="$1" + + info "Uninstalling smolvm..." + + # Remove installation directory + if [[ -d "$prefix" ]]; then + rm -rf "$prefix" + success "Removed $prefix" + else + warn "Installation directory not found: $prefix" + fi + + # Remove symlink + if [[ -L "$BIN_DIR/smolvm" ]]; then + rm -f "$BIN_DIR/smolvm" + success "Removed symlink $BIN_DIR/smolvm" + fi + + # Remove data directory (agent-rootfs, storage) + local data_dir + if [[ "$(uname -s)" == "Darwin" ]]; then + data_dir="$HOME/Library/Application Support/smolvm" + else + data_dir="${XDG_DATA_HOME:-$HOME/.local/share}/smolvm" + fi + if [[ -d "$data_dir" ]]; then + rm -rf "$data_dir" + success "Removed data directory $data_dir" + fi + + # Remove cache directories + local cache_dir cache_pack_dir + if [[ "$(uname -s)" == "Darwin" ]]; then + cache_dir="$HOME/Library/Caches/smolvm" + cache_pack_dir="$HOME/Library/Caches/smolvm-pack" + else + cache_dir="${XDG_CACHE_HOME:-$HOME/.cache}/smolvm" + cache_pack_dir="${XDG_CACHE_HOME:-$HOME/.cache}/smolvm-pack" + fi + if [[ -d "$cache_dir" ]]; then + rm -rf "$cache_dir" + success "Removed cache directory $cache_dir" + fi + if [[ -d "$cache_pack_dir" ]]; then + # On macOS, detach any hdiutil-mounted case-sensitive volumes + # (layers-cs) before removing the directory. Without this, rm + # fails with "Resource busy" on active mount points. + if [[ "$(uname -s)" == "Darwin" ]]; then + find "$cache_pack_dir" -name layers-cs -type d -exec hdiutil detach {} -force \; 2>/dev/null || true + fi + rm -rf "$cache_pack_dir" + success "Removed pack cache directory $cache_pack_dir" + fi + + # Remove libs extraction cache (from packed binary SMOLLIBS) + local cache_libs_dir + if [[ "$(uname -s)" == "Darwin" ]]; then + cache_libs_dir="$HOME/Library/Caches/smolvm-libs" + else + cache_libs_dir="${XDG_CACHE_HOME:-$HOME/.cache}/smolvm-libs" + fi + if [[ -d "$cache_libs_dir" ]]; then + rm -rf "$cache_libs_dir" + success "Removed libs cache directory $cache_libs_dir" + fi + + # Note about remaining files + warn "You may want to remove the PATH entry from your shell profile." + local config_dir="$HOME/.config/smolvm" + if [[ -d "$config_dir" ]]; then + warn "Registry credentials preserved at $config_dir" + warn "Remove manually if no longer needed: rm -rf $config_dir" + fi + + success "smolvm has been uninstalled" +} + +# Print usage +usage() { + cat << EOF +smolvm installer + +Usage: + install.sh [OPTIONS] + +Options: + --version VERSION Install specific version (default: latest) + --prefix DIR Install to DIR (default: ~/.smolvm) + --no-modify-path Don't modify shell profile + --uninstall Remove smolvm installation + --help Show this help message + +Examples: + # Install latest version + curl -sSL https://smolmachines.com/install.sh | bash + + # Install specific version + curl -sSL https://smolmachines.com/install.sh | bash -s -- --version 0.1.1 + + # Install to custom directory + curl -sSL https://smolmachines.com/install.sh | bash -s -- --prefix /opt/smolvm + + # Uninstall + curl -sSL https://smolmachines.com/install.sh | bash -s -- --uninstall +EOF +} + +# Parse arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --version) + VERSION="$2" + shift 2 + ;; + --prefix) + INSTALL_PREFIX="$2" + shift 2 + ;; + --no-modify-path) + MODIFY_PATH=false + shift + ;; + --uninstall) + UNINSTALL=true + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + error "Unknown option: $1" + usage + exit 1 + ;; + esac + done +} + +# Main +main() { + parse_args "$@" + + echo "" + echo -e "${BOLD}smolvm installer${NC}" + echo "" + + # Handle uninstall + if [[ "$UNINSTALL" == true ]]; then + uninstall_smolvm "$INSTALL_PREFIX" + exit 0 + fi + + # Detect platform + local platform + platform=$(detect_platform) + info "Detected platform: $platform" + + # Check requirements + check_requirements "$platform" + + # Get version + if [[ -z "$VERSION" ]]; then + info "Fetching latest version..." + VERSION=$(get_latest_version) + fi + info "Installing version: $VERSION" + + # Check for existing installation + if [[ -f "$INSTALL_PREFIX/.version" ]]; then + local current_version + current_version=$(cat "$INSTALL_PREFIX/.version") + if [[ "$current_version" == "$VERSION" ]]; then + info "Reinstalling smolvm $VERSION..." + else + info "Upgrading from $current_version to $VERSION" + fi + fi + + # Install + install_smolvm "$VERSION" "$platform" "$INSTALL_PREFIX" + + # Modify PATH + if [[ "$MODIFY_PATH" == true ]]; then + modify_path "$BIN_DIR" + fi + + echo "" + echo -e "${GREEN}Installation complete!${NC}" + echo "" + echo "To get started, run:" + echo "" + if ! echo "$PATH" | grep -q "$BIN_DIR"; then + echo " export PATH=\"$BIN_DIR:\$PATH\"" + fi + echo " smolvm --help" + echo "" +} + +main "$@" diff --git a/scripts/rebuild-agent.sh b/scripts/rebuild-agent.sh new file mode 100755 index 0000000..06f1ef5 --- /dev/null +++ b/scripts/rebuild-agent.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Rebuild the smolvm-agent binary for Linux and install it +# +# Usage: ./scripts/rebuild-agent.sh [--clean] +# +# Options: +# --clean Force clean rebuild (required after protocol changes) + +set -ex + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" +ROOTFS_DIR="$HOME/Library/Application Support/smolvm/agent-rootfs" + +cd "$PROJECT_DIR" + +# Clean build artifacts if requested +CLEAN_CMD="" +if [[ "$1" == "--clean" ]]; then + echo "Cleaning build artifacts..." + CLEAN_CMD="rm -rf target/release-small/deps/smolvm_protocol* \ + target/release-small/deps/smolvm_agent* \ + target/release-small/.fingerprint/smolvm-protocol* \ + target/release-small/.fingerprint/smolvm-agent* \ + target/release-small/smolvm-agent && " +fi + +echo "Building smolvm-agent for Linux..." +# Prefer the locally built binary over the installed one — it has the latest +# fixes (e.g., registry config path) that the installed version may lack. +SMOLVM_BIN="${PROJECT_DIR}/target/release/smolvm" +if [ ! -f "$SMOLVM_BIN" ] && command -v smolvm &> /dev/null; then + SMOLVM_BIN="smolvm" +elif [ ! -f "$SMOLVM_BIN" ]; then + echo "Error: smolvm is required to cross-compile the agent" + echo "Build with: cargo build --release" + exit 1 +fi + +"$SMOLVM_BIN" machine run --net --mem 2048 -v "$PROJECT_DIR:/work" --image rust:alpine \ + -- sh -c ". /usr/local/cargo/env && apk add musl-dev && cd /work && ${CLEAN_CMD}cargo build --profile release-small -p smolvm-agent" + +# Check if rootfs directory exists +if [[ ! -d "$ROOTFS_DIR/usr/local/bin" ]]; then + echo "Error: Agent rootfs not found at $ROOTFS_DIR" + echo "Run ./scripts/build-agent-rootfs.sh first" + exit 1 +fi + +echo "Installing agent binary..." +cp target/release-small/smolvm-agent "$ROOTFS_DIR/usr/local/bin/" + +# /sbin/init is the kernel's entry point — symlink to the agent binary. +# The agent handles overlayfs setup + pivot_root internally before +# starting the vsock listener. +ln -sf /usr/local/bin/smolvm-agent "$ROOTFS_DIR/sbin/init" + +# Also update target/agent-rootfs if it exists — pack create reads from +# there first, so it must stay in sync with the installed rootfs. +if [[ -d "$PROJECT_DIR/target/agent-rootfs/usr/local/bin" ]]; then + cp target/release-small/smolvm-agent "$PROJECT_DIR/target/agent-rootfs/usr/local/bin/" + echo "Updated: target/agent-rootfs (keeps pack create in sync)" +fi + +echo "Stopping running agent (if any)..." +export DYLD_LIBRARY_PATH="$PROJECT_DIR/lib" +"$PROJECT_DIR/target/release/smolvm" agent stop 2>/dev/null || true + +echo "" +echo "Agent rebuilt and installed successfully!" +echo "Binary: $ROOTFS_DIR/usr/local/bin/smolvm-agent" +echo "Init: $ROOTFS_DIR/sbin/init (symlink to agent)" +ls -la "$ROOTFS_DIR/usr/local/bin/smolvm-agent" "$ROOTFS_DIR/sbin/init" diff --git a/scripts/rosetta/rosetta-wrapper b/scripts/rosetta/rosetta-wrapper new file mode 100755 index 0000000..8f732ff Binary files /dev/null and b/scripts/rosetta/rosetta-wrapper differ diff --git a/scripts/rosetta/rosetta-wrapper.c b/scripts/rosetta/rosetta-wrapper.c new file mode 100644 index 0000000..3272790 --- /dev/null +++ b/scripts/rosetta/rosetta-wrapper.c @@ -0,0 +1,200 @@ +/* + * rosetta-wrapper — ptrace shim for Apple Rosetta under Hypervisor.framework + * + * Rosetta's Linux binary validates it's running under Virtualization.framework + * via an undocumented ioctl (type byte 0x61). Under libkrun's + * Hypervisor.framework backend that ioctl fails, causing Rosetta to abort. + * + * This wrapper intercepts the validation ioctl via ptrace, returns the + * expected magic string, and then detaches — running at full speed for the + * rest of the process's lifetime. + * + * Installed at /usr/bin/rosetta-wrapper in the agent rootfs and registered + * as the binfmt_misc interpreter for x86_64 ELF binaries. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct user_regs { + uint64_t regs[31]; + uint64_t sp; + uint64_t pc; + uint64_t pstate; +}; + +static const char ROSETTA_MAGIC[] = + "Our hard work\nby these words guarded\nplease don't steal\n\xC2\xA9 Apple Inc\n"; + +#define SYS_IOCTL 29 + +static int get_regs(pid_t pid, struct user_regs *regs) { + struct iovec iov = { regs, sizeof(*regs) }; + return ptrace(PTRACE_GETREGSET, pid, (void *)NT_PRSTATUS, &iov); +} + +static int set_regs(pid_t pid, struct user_regs *regs) { + struct iovec iov = { regs, sizeof(*regs) }; + return ptrace(PTRACE_SETREGSET, pid, (void *)NT_PRSTATUS, &iov); +} + +static int write_mem(pid_t pid, uint64_t addr, const void *buf, size_t len) { + const unsigned char *src = buf; + size_t i; + + for (i = 0; i + 8 <= len; i += 8) { + uint64_t word; + memcpy(&word, src + i, 8); + if (ptrace(PTRACE_POKEDATA, pid, (void *)(addr + i), (void *)word) < 0) + return -1; + } + if (i < len) { + uint64_t word = 0; + errno = 0; + word = (uint64_t)ptrace(PTRACE_PEEKDATA, pid, (void *)(addr + i), NULL); + if (errno) + return -1; + memcpy(&word, src + i, len - i); + if (ptrace(PTRACE_POKEDATA, pid, (void *)(addr + i), (void *)word) < 0) + return -1; + } + return 0; +} + +int main(int argc, char *argv[], char *envp[]) { + if (argc < 2) { + fprintf(stderr, "Usage: %s [args...]\n", argv[0]); + return 1; + } + + const char *rosetta = "/mnt/rosetta/rosetta"; + pid_t child = fork(); + if (child < 0) { + perror("fork"); + return 1; + } + + if (child == 0) { + ptrace(PTRACE_TRACEME, 0, NULL, NULL); + char **new_argv = malloc((argc + 1) * sizeof(char *)); + new_argv[0] = (char *)rosetta; + for (int i = 1; i < argc; i++) + new_argv[i] = argv[i]; + new_argv[argc] = NULL; + execve(rosetta, new_argv, envp); + perror("execve"); + _exit(127); + } + + /* Wait for exec-stop (SIGTRAP from execve after TRACEME). */ + int status; + waitpid(child, &status, 0); + if (!WIFSTOPPED(status) || WSTOPSIG(status) != SIGTRAP) { + fprintf(stderr, "rosetta-wrapper: unexpected initial stop: %d\n", status); + kill(child, SIGKILL); + return 1; + } + + ptrace(PTRACE_SETOPTIONS, child, NULL, (void *)(long)PTRACE_O_TRACESYSGOOD); + ptrace(PTRACE_SYSCALL, child, NULL, NULL); + + int in_syscall = 0; + int intercept_exit = 0; + uint64_t ioctl_buf_addr = 0; + + while (1) { + waitpid(child, &status, 0); + + if (WIFEXITED(status)) + return WEXITSTATUS(status); + if (WIFSIGNALED(status)) + return 128 + WTERMSIG(status); + + if (!WIFSTOPPED(status)) + continue; + + int sig = WSTOPSIG(status); + + if (sig != (SIGTRAP | 0x80)) { + /* Non-syscall signal: deliver it. */ + ptrace(PTRACE_SYSCALL, child, NULL, (void *)(long)sig); + continue; + } + + /* Syscall stop. */ + struct user_regs regs; + get_regs(child, ®s); + + if (!in_syscall) { + /* Syscall entry. */ + if (regs.regs[8] == SYS_IOCTL) { + uint32_t cmd = (uint32_t)regs.regs[1]; + uint8_t ioc_type = (cmd >> 8) & 0xff; + uint16_t ioc_size = (cmd >> 16) & 0x3fff; + /* + * Match on type byte 0x61 (Rosetta's Virtualization.framework + * validation). The command number changed between macOS versions + * so we match on the type byte for version independence. + */ + if (ioc_type == 0x61) { + if (ioc_size == 0x45) { + intercept_exit = 1; + ioctl_buf_addr = regs.regs[2]; + } else if (ioc_size == 0x80) { + intercept_exit = 2; + ioctl_buf_addr = regs.regs[2]; + } + } + } + in_syscall = 1; + } else { + /* Syscall exit. */ + if (intercept_exit == 1) { + /* 0x45-byte validation: write the magic string. */ + char buf[0x45]; + memset(buf, 0, sizeof(buf)); + size_t mlen = strlen(ROSETTA_MAGIC); + if (mlen >= sizeof(buf)) + mlen = sizeof(buf) - 1; + memcpy(buf, ROSETTA_MAGIC, mlen); + write_mem(child, ioctl_buf_addr, buf, sizeof(buf)); + regs.regs[0] = 1; + set_regs(child, ®s); + } else if (intercept_exit == 2) { + /* 0x80-byte validation: zero-fill. */ + char buf[0x80]; + memset(buf, 0, sizeof(buf)); + write_mem(child, ioctl_buf_addr, buf, sizeof(buf)); + regs.regs[0] = 1; + set_regs(child, ®s); + } + + if (intercept_exit) { + /* + * Validation intercepted — Rosetta will proceed. Detach so the + * translated process runs at full speed with no ptrace overhead. + */ + ptrace(PTRACE_DETACH, child, NULL, NULL); + waitpid(child, &status, 0); + if (WIFEXITED(status)) + return WEXITSTATUS(status); + if (WIFSIGNALED(status)) + return 128 + WTERMSIG(status); + return 0; + } + + in_syscall = 0; + } + + ptrace(PTRACE_SYSCALL, child, NULL, NULL); + } +} diff --git a/scripts/smol-wrapper.sh b/scripts/smol-wrapper.sh new file mode 100755 index 0000000..124685f --- /dev/null +++ b/scripts/smol-wrapper.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# smol - the unified Smol Machines CLI (local microVMs + cloud) +# This wrapper sets up the library path and runs the smol binary. +# +# It mirrors smolvm-wrapper.sh: the `smol` binary embeds the same engine, so it +# discovers libkrun/libkrunfw and the agent rootfs the same way (relative to its +# own install directory), making the distribution relocatable. + +set -e + +# Resolve symlinks to get the actual script location +resolve_symlink() { + local target="$1" + while [[ -L "$target" ]]; do + local link_dir + link_dir="$(cd "$(dirname "$target")" && pwd)" + target="$(readlink "$target")" + # Handle relative symlinks + if [[ "$target" != /* ]]; then + target="$link_dir/$target" + fi + done + echo "$target" +} + +# Get the directory where the actual script lives (resolving symlinks) +SCRIPT_PATH="$(resolve_symlink "${BASH_SOURCE[0]}")" +SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_PATH")" && pwd)" + +# The actual binary and libraries are in the same directory +SMOL_BIN="$SCRIPT_DIR/smol-bin" +SMOL_LIB="$SCRIPT_DIR/lib" +SMOL_BUNDLED_ROOTFS="$SCRIPT_DIR/agent-rootfs" + +if [[ -d "$SMOL_BUNDLED_ROOTFS" ]]; then + export SMOLVM_AGENT_ROOTFS="${SMOLVM_AGENT_ROOTFS:-$SMOL_BUNDLED_ROOTFS}" +fi + +# Check if binary exists +if [[ ! -x "$SMOL_BIN" ]]; then + echo "Error: smol binary not found at $SMOL_BIN" >&2 + echo "Make sure you extracted the full distribution." >&2 + exit 1 +fi + +# Check if libraries exist +if [[ ! -d "$SMOL_LIB" ]]; then + echo "Error: library directory not found at $SMOL_LIB" >&2 + echo "Make sure you extracted the full distribution." >&2 + exit 1 +fi + +# Set library path based on OS and run +if [[ "$(uname -s)" == "Darwin" ]]; then + export DYLD_LIBRARY_PATH="$SMOL_LIB${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" +else + export LD_LIBRARY_PATH="$SMOL_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +fi +exec "$SMOL_BIN" "$@" diff --git a/scripts/smolvm-wrapper.sh b/scripts/smolvm-wrapper.sh new file mode 100755 index 0000000..0f7ff1e --- /dev/null +++ b/scripts/smolvm-wrapper.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# smolvm - OCI-native microVM runtime +# This wrapper sets up the library path and runs the smolvm binary. + +set -e + +# Resolve symlinks to get the actual script location +resolve_symlink() { + local target="$1" + while [[ -L "$target" ]]; do + local link_dir + link_dir="$(cd "$(dirname "$target")" && pwd)" + target="$(readlink "$target")" + # Handle relative symlinks + if [[ "$target" != /* ]]; then + target="$link_dir/$target" + fi + done + echo "$target" +} + +# Get the directory where the actual script lives (resolving symlinks) +SCRIPT_PATH="$(resolve_symlink "${BASH_SOURCE[0]}")" +SCRIPT_DIR="$(cd "$(dirname "$SCRIPT_PATH")" && pwd)" + +# The actual binary and libraries are in the same directory +SMOLVM_BIN="$SCRIPT_DIR/smolvm-bin" +SMOLVM_LIB="$SCRIPT_DIR/lib" +SMOLVM_BUNDLED_ROOTFS="$SCRIPT_DIR/agent-rootfs" + +if [[ -d "$SMOLVM_BUNDLED_ROOTFS" ]]; then + export SMOLVM_AGENT_ROOTFS="${SMOLVM_AGENT_ROOTFS:-$SMOLVM_BUNDLED_ROOTFS}" +fi + +# Check if binary exists +if [[ ! -x "$SMOLVM_BIN" ]]; then + echo "Error: smolvm binary not found at $SMOLVM_BIN" >&2 + echo "Make sure you extracted the full distribution." >&2 + exit 1 +fi + +# Check if libraries exist +if [[ ! -d "$SMOLVM_LIB" ]]; then + echo "Error: library directory not found at $SMOLVM_LIB" >&2 + echo "Make sure you extracted the full distribution." >&2 + exit 1 +fi + +# Set library path based on OS and run +if [[ "$(uname -s)" == "Darwin" ]]; then + export DYLD_LIBRARY_PATH="$SMOLVM_LIB${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" +else + export LD_LIBRARY_PATH="$SMOLVM_LIB${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +fi +exec "$SMOLVM_BIN" "$@" diff --git a/scripts/stamp-libkrun-provenance.sh b/scripts/stamp-libkrun-provenance.sh new file mode 100755 index 0000000..d65e9f5 --- /dev/null +++ b/scripts/stamp-libkrun-provenance.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Record which libkrun/libkrunfw submodule commits the bundled libs in a given +# lib dir were built from. Called by the libkrun build scripts after they copy +# fresh binaries into place, so the committed lib/ always carries proof of which +# source it came from. check-libkrun-provenance.sh later verifies this matches +# the pinned submodules — that's what stops a stale binary from shipping. +# +# Usage: stamp-libkrun-provenance.sh [--skip-libkrunfw] +# e.g. lib, lib/linux-x86_64, lib/linux-aarch64 +# --skip-libkrunfw keep the existing libkrunfw= line (the build reused the +# committed libkrunfw.* instead of rebuilding it) + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LIB_DIR="${1:?usage: stamp-libkrun-provenance.sh [--skip-libkrunfw]}" +SKIP_FW=0 +[[ "${2:-}" == "--skip-libkrunfw" ]] && SKIP_FW=1 + +cd "$REPO_ROOT" +PROV="$LIB_DIR/libkrun.provenance" +mkdir -p "$LIB_DIR" + +# The commit each submodule is actually checked out at right now (what we built). +krun_sha="$(git -C libkrun rev-parse HEAD)" + +if [[ "$SKIP_FW" == "1" && -f "$PROV" ]]; then + fw_sha="$(grep '^libkrunfw=' "$PROV" | cut -d= -f2)" +else + fw_sha="$(git -C libkrunfw rev-parse HEAD)" +fi + +cat > "$PROV" < /dev/null 2>&1 || true + exit 1 +} +info() { echo -e "${YELLOW}INFO${NC}: $1"; } + +# --------------------------------------------------------------------------- +# Setup +# --------------------------------------------------------------------------- + +# Build if needed. +if [ ! -f "$SMOLVM" ]; then + info "Building smolvm..." + cargo build --manifest-path "$REPO_DIR/Cargo.toml" +fi + +# Find or create a test .smolmachine file. +TEST_SMOLMACHINE="${1:-}" +if [ -z "$TEST_SMOLMACHINE" ]; then + # Look for any existing .smolmachine in the repo. + TEST_SMOLMACHINE=$(find "$REPO_DIR" -maxdepth 1 -name "*.smolmachine" -type f | head -1) +fi + +if [ -z "$TEST_SMOLMACHINE" ] || [ ! -f "$TEST_SMOLMACHINE" ]; then + info "No .smolmachine file found. Skipping integration test." + info "To run: smolvm pack create --image alpine:latest -o /tmp/test-alpine" + info "Then: $0 /tmp/test-alpine.smolmachine" + exit 0 +fi + +info "Using test artifact: $TEST_SMOLMACHINE" + +# --------------------------------------------------------------------------- +# Start local registry +# --------------------------------------------------------------------------- + +info "Starting local zot registry..." +if ! docker compose -f "$INFRA_DIR/docker-compose.yml" up -d 2>/dev/null; then + info "docker compose failed — is Docker running?" + info "Skipping integration test." + exit 0 +fi + +# Wait for registry to be ready. +for i in $(seq 1 10); do + if curl -sf "http://$REGISTRY/v2/" > /dev/null 2>&1; then + break + fi + if [ "$i" -eq 10 ]; then + fail "Registry did not start within 10 seconds" + fi + sleep 1 +done +pass "Registry is running at $REGISTRY" + +# --------------------------------------------------------------------------- +# Test push +# --------------------------------------------------------------------------- + +info "Pushing $TEST_SMOLMACHINE to $TEST_REF..." +$SMOLVM registry push "$TEST_REF" -f "$TEST_SMOLMACHINE" +pass "Push succeeded" + +# --------------------------------------------------------------------------- +# Test pull +# --------------------------------------------------------------------------- + +PULL_OUTPUT=$(mktemp -d)/pulled.smolmachine + +info "Pulling $TEST_REF to $PULL_OUTPUT..." +$SMOLVM registry pull "$TEST_REF" -o "$PULL_OUTPUT" +pass "Pull succeeded" + +# Verify file sizes match. +ORIG_SIZE=$(wc -c < "$TEST_SMOLMACHINE" | tr -d ' ') +PULL_SIZE=$(wc -c < "$PULL_OUTPUT" | tr -d ' ') + +if [ "$ORIG_SIZE" -eq "$PULL_SIZE" ]; then + pass "File sizes match ($ORIG_SIZE bytes)" +else + fail "Size mismatch: original=$ORIG_SIZE pulled=$PULL_SIZE" +fi + +# Verify file contents match. +ORIG_SHA=$(shasum -a 256 "$TEST_SMOLMACHINE" | cut -d' ' -f1) +PULL_SHA=$(shasum -a 256 "$PULL_OUTPUT" | cut -d' ' -f1) + +if [ "$ORIG_SHA" = "$PULL_SHA" ]; then + pass "SHA256 digests match ($ORIG_SHA)" +else + fail "Digest mismatch: original=$ORIG_SHA pulled=$PULL_SHA" +fi + +# --------------------------------------------------------------------------- +# Test cached pull +# --------------------------------------------------------------------------- + +PULL_OUTPUT2=$(mktemp -d)/pulled2.smolmachine + +info "Pulling again (should hit cache)..." +OUTPUT=$($SMOLVM registry pull "$TEST_REF" -o "$PULL_OUTPUT2" 2>&1) +if echo "$OUTPUT" | grep -q "cached"; then + pass "Second pull used cache" +else + info "Second pull did not report cache hit (may be expected)" +fi + +# --------------------------------------------------------------------------- +# Cleanup +# --------------------------------------------------------------------------- + +rm -f "$PULL_OUTPUT" "$PULL_OUTPUT2" +info "Stopping local registry..." +docker compose -f "$INFRA_DIR/docker-compose.yml" down > /dev/null 2>&1 + +echo "" +echo -e "${GREEN}All registry integration tests passed.${NC}" diff --git a/scripts/update-formula.sh b/scripts/update-formula.sh new file mode 100755 index 0000000..b1f425a --- /dev/null +++ b/scripts/update-formula.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Update the Homebrew formula with SHA256 hashes from dist/ tarballs +# +# Usage: ./scripts/update-formula.sh +# +# This script reads tarballs from dist/ and updates the formula with correct SHA256 hashes. + +set -e + +FORMULA="homebrew-tap/Formula/smolvm.rb" +VERSION="${VERSION:-$(grep '^version' Cargo.toml | head -1 | cut -d'"' -f2)}" + +# Colors +if [ -t 1 ]; then + GREEN='\033[0;32m' + YELLOW='\033[0;33m' + NC='\033[0m' +else + GREEN='' YELLOW='' NC='' +fi + +info() { echo -e "${GREEN}info:${NC} $1"; } +warn() { echo -e "${YELLOW}warning:${NC} $1"; } + +echo "Updating formula for version $VERSION" + +# Check formula exists +if [[ ! -f "$FORMULA" ]]; then + echo "Error: Formula not found at $FORMULA" + exit 1 +fi + +# Update version in formula +sed -i '' "s/version \"[^\"]*\"/version \"$VERSION\"/" "$FORMULA" +info "Updated version to $VERSION" + +# Platform mapping: tarball name -> placeholder name in formula +declare -A PLATFORMS=( + ["darwin-arm64"]="PLACEHOLDER_DARWIN_ARM64_SHA256" + ["darwin-x86_64"]="PLACEHOLDER_DARWIN_X86_64_SHA256" + ["linux-arm64"]="PLACEHOLDER_LINUX_ARM64_SHA256" + ["linux-x86_64"]="PLACEHOLDER_LINUX_X86_64_SHA256" +) + +# Calculate and update SHA256 for each platform +echo "" +echo "Updating SHA256 hashes:" + +for platform in "${!PLATFORMS[@]}"; do + tarball="dist/smolvm-${VERSION}-${platform}.tar.gz" + placeholder="${PLATFORMS[$platform]}" + + if [[ -f "$tarball" ]]; then + sha=$(shasum -a 256 "$tarball" | cut -d' ' -f1) + info "$platform: $sha" + + # Replace placeholder or existing SHA with new one + # First try to replace the placeholder + if grep -q "$placeholder" "$FORMULA"; then + sed -i '' "s/$placeholder/$sha/" "$FORMULA" + fi + + # Also try to replace any existing 64-char hex string on lines mentioning this platform + # This handles re-running the script + else + warn "$platform: tarball not found at $tarball" + fi +done + +echo "" +info "Formula updated at $FORMULA" + +# Generate checksums.txt for releases +CHECKSUMS_FILE="dist/checksums.txt" +echo "" +echo "Generating $CHECKSUMS_FILE for release..." +rm -f "$CHECKSUMS_FILE" + +for tarball in dist/smolvm-${VERSION}-*.tar.gz; do + if [[ -f "$tarball" ]]; then + shasum -a 256 "$tarball" >> "$CHECKSUMS_FILE" + fi +done + +if [[ -f "$CHECKSUMS_FILE" ]]; then + info "Created $CHECKSUMS_FILE:" + cat "$CHECKSUMS_FILE" +else + warn "No tarballs found to generate checksums" +fi + +echo "" +echo "Next steps:" +echo " 1. Review changes: git diff $FORMULA" +echo " 2. Test formula locally: brew install --build-from-source $FORMULA" +echo " 3. Commit and push to homebrew-tap repo" diff --git a/sdks/README.md b/sdks/README.md new file mode 100644 index 0000000..0595a98 --- /dev/null +++ b/sdks/README.md @@ -0,0 +1,174 @@ +# SmolVM Embedded SDKs + +## Overview + +This directory holds language bindings that embed `smolvm` directly into the +host process instead of talking to the API server. + +Layout convention: + +- `sdks/scripts/` contains shared helpers used by all embedded SDKs. +- `sdks/node/` contains the Node.js embedded SDK and its internal platform + packages. +- Future embedded SDKs should live in sibling directories such as `sdks/go/`, and `sdks/c/`. + +Bundled native library rule: + +- Embedded SDKs ship package-local copies of `libkrun` and `libkrunfw`. +- Those libraries are always staged from the `smolvm` repo's bundled `./lib` + directory, not from Homebrew or other system locations. +- Shared helpers in `sdks/scripts/` should be used to copy the current host's + libraries into each SDK package's `lib/` directory. + +Current status: + +- **The embedded SDK currently creates a machine without involving the DB storage. +This means machines created via the embedded SDK are not visible via the smolvm CLI. +This is a bug, and we are actively working on a fix.** + +## Hint + +This directory contains the **embedded SDKs** explicitly: +That means those that run in the same process as smolvm itself. +There is no external daemon to run, making it easier and safer to script. + +For the currently only other SDK - that follows the traditional approach with a separate process - see [here.](https://github.com/smol-machines/smolvm-sdk) + +## Development + +### Build + +Install the Node workspace dependencies, then run the repo-level embedded SDK +build. That flow compiles `smolvm-napi`, stages the bundled `libkrun` and +`libkrunfw` libraries into the current-host platform package, and builds the +public `smolvm-embedded` package. + +```bash +cd sdks/node +npm install + +cd ../.. +./scripts/build-embedded-node.sh +``` + +If you are already inside `sdks/node`, `npm run build` rebuilds the current-host +platform package plus the public package without leaving the workspace. + +### Test + +Use the Node workspace for the main verification flow: + +```bash +cd sdks/node +npm test +npm run --workspace smolvm-embedded test:integration +npm run smoke +npm exec --workspace smolvm-embedded tsx examples/basic.ts +npm exec --workspace smolvm-embedded tsx examples/create-and-start.ts +``` + +- `npm test` rebuilds the current platform package and runs the + `smolvm-embedded` Vitest suite. +- `npm run --workspace smolvm-embedded test:integration` runs only the embedded + SDK integration tests under `sdks/node/smolvm-embedded/integration-tests/`. +- `npm run smoke` performs the fresh-install validation from the PR by packing + the public package plus the host platform package, installing them into a + temporary project, and checking that the native binding loads correctly. +- `npm exec --workspace smolvm-embedded tsx examples/basic.ts` runs the local + integration example that exercises `quickExec`, container execution, managed + machine lifecycle, and explicit machine cleanup. +- `npm exec --workspace smolvm-embedded tsx examples/create-and-start.ts` + creates `created-by-node`, explicitly starts it, and intentionally leaves it + running so it can be inspected with `smolvm machine status --name + created-by-node` or `smolvm machine ls`. + +### Manual Fresh-Install Check + +If you want to test from a freshly installed +`smolvm-embedded` Node SDK" flow by hand, pack both the public package and the +current-host platform package, then install them into a throwaway project. + +Supported platform package directories: + +- `smolvm-embedded-darwin-arm64` +- `smolvm-embedded-darwin-x64` +- `smolvm-embedded-linux-arm64-gnu` +- `smolvm-embedded-linux-x64-gnu` + +Example using the current-host package name in `PLATFORM_PKG`: + +```bash +TMP_PACK_DIR="$(mktemp -d /tmp/smolvm-embedded-pack.XXXXXX)" +TMP_PROJECT_DIR="$(mktemp -d /tmp/smolvm-embedded-project.XXXXXX)" +PLATFORM_PKG="smolvm-embedded-darwin-arm64" + +cd sdks/node/"$PLATFORM_PKG" +npm pack --pack-destination "$TMP_PACK_DIR" + +cd ../smolvm-embedded +npm pack --pack-destination "$TMP_PACK_DIR" + +cd "$TMP_PROJECT_DIR" +npm init -y +npm install typescript tsx @types/node +npm install \ + "$TMP_PACK_DIR"/smolvm-embedded-0.1.0.tgz \ + "$TMP_PACK_DIR"/"$PLATFORM_PKG"-0.1.0.tgz +``` + +Create `index.ts` in the temporary project: + +```ts +import { quickExec, withMachine } from "smolvm-embedded"; + +async function main() { + const hello = await quickExec(["echo", "hello from smolvm-embedded"]); + console.log("quickExec stdout:", hello.stdout.trim()); + console.log("quickExec exitCode:", hello.exitCode); + + await withMachine({ name: "demo-machine" }, async (sb) => { + const result = await sb.exec(["uname", "-a"]); + console.log("machine uname:", result.stdout.trim()); + }); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); +``` + +Run the smoke program: + +```bash +cd "$TMP_PROJECT_DIR" +npx tsx index.ts +``` + +Alternative `index.ts` that does not use `quickExec` and does not delete the VM: + +```ts +import { Machine } from "smolvm-embedded"; + +async function main() { + const machine = await Machine.create({ + name: "created-by-node", + persistent: true, + }); + + console.log("created machine:", machine.name); + console.log("state before start:", machine.state); + + await machine.start(); + + console.log("state after start:", machine.state); + console.log("is running:", machine.isRunning); + console.log("pid:", machine.pid ?? "unknown"); + console.log("VM was not deleted."); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); +``` diff --git a/sdks/node/.gitignore b/sdks/node/.gitignore new file mode 100644 index 0000000..6724587 --- /dev/null +++ b/sdks/node/.gitignore @@ -0,0 +1,7 @@ +node_modules/ +package-lock.json +*/dist/ +*/lib/ +*/native/index.js +*/native/index.d.ts +*/native/*.node diff --git a/sdks/node/README.md b/sdks/node/README.md new file mode 100644 index 0000000..f8836ee --- /dev/null +++ b/sdks/node/README.md @@ -0,0 +1,31 @@ +# Embedded Node SDK Workspace + +This workspace contains: + +- `smolvm-embedded`: the public package users install +- `smolvm-embedded-*`: internal platform packages that carry the `.node` + binary plus bundled `libkrun` and `libkrunfw` + +Users should only install `smolvm-embedded`. The platform packages are an +implementation detail used by npm's optional dependency resolution. + +Local maintainer workflow: + +```bash +cd sdks/node +npm install +npm run build +npm test +npm run smoke +``` + +During local development, `npm install` may warn about unpublished optional +platform packages. That is expected. Local builds resolve the sibling +`smolvm-embedded-*` package directories directly instead of relying on npm to +install those internal packages. + +Repo-root build entrypoint: + +```bash +./scripts/build-embedded-node.sh +``` diff --git a/sdks/node/package.json b/sdks/node/package.json new file mode 100644 index 0000000..e80198e --- /dev/null +++ b/sdks/node/package.json @@ -0,0 +1,22 @@ +{ + "name": "smolvm-embedded-node-workspace", + "private": true, + "scripts": { + "build:platform": "./scripts/build-platform-package.sh", + "build:public": "npm run --workspace smolvm-embedded build:ts", + "build": "./scripts/build-current-platform.sh", + "test": "npm run build:platform && npm run --workspace smolvm-embedded test", + "smoke": "./scripts/smoke-install.sh" + }, + "workspaces": [ + "smolvm-embedded" + ], + "devDependencies": { + "@napi-rs/cli": "^2.18.0", + "@types/node": "^20.0.0", + "tsup": "^8.0.0", + "tsx": "^4.19.3", + "typescript": "^5.3.0", + "vitest": "^4.1.8" + } +} diff --git a/sdks/node/scripts/build-current-platform.sh b/sdks/node/scripts/build-current-platform.sh new file mode 100755 index 0000000..bd17a56 --- /dev/null +++ b/sdks/node/scripts/build-current-platform.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +WORKSPACE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +if [[ ! -d "$WORKSPACE_DIR/node_modules" ]]; then + echo "Error: Node workspace dependencies are not installed." >&2 + echo "Run 'cd $WORKSPACE_DIR && npm install' first." >&2 + exit 1 +fi + +( + cd "$WORKSPACE_DIR" + npm run build:platform + npm run build:public +) diff --git a/sdks/node/scripts/build-platform-package.sh b/sdks/node/scripts/build-platform-package.sh new file mode 100755 index 0000000..a53d2e2 --- /dev/null +++ b/sdks/node/scripts/build-platform-package.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +WORKSPACE_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +REPO_ROOT="$(cd "$WORKSPACE_DIR/../.." && pwd)" + +detect_lib_bundle() { + if [[ -n "${SMOLVM_EMBEDDED_LIB_BUNDLE:-}" ]]; then + echo "$SMOLVM_EMBEDDED_LIB_BUNDLE" + return 0 + fi + + if [[ "$(uname -s)" == "Linux" ]]; then + echo "$REPO_ROOT/lib/linux-$(uname -m)" + else + echo "$REPO_ROOT/lib" + fi +} + +detect_platform_package() { + case "$(uname -s):$(uname -m)" in + Darwin:arm64) + echo "smolvm-embedded-darwin-arm64" + ;; + Darwin:x86_64) + echo "smolvm-embedded-darwin-x64" + ;; + Linux:aarch64) + echo "smolvm-embedded-linux-arm64-gnu" + ;; + Linux:x86_64) + echo "smolvm-embedded-linux-x64-gnu" + ;; + *) + echo "Unsupported platform: $(uname -s) $(uname -m)" >&2 + exit 1 + ;; + esac +} + +PACKAGE_NAME="$(detect_platform_package)" +PACKAGE_DIR="$WORKSPACE_DIR/$PACKAGE_NAME" +LIBKRUN_BUNDLE="$(detect_lib_bundle)" + +echo "Building embedded Node platform package: $PACKAGE_NAME" + +( + cd "$PACKAGE_DIR" + LIBKRUN_BUNDLE="$LIBKRUN_BUNDLE" napi build --platform --release --cargo-cwd ../../../crates/smolvm-napi native +) + +"$REPO_ROOT/sdks/scripts/stage-embedded-libs.sh" "$PACKAGE_DIR/lib" + +cat <&2 + exit 1 + ;; + esac +} + +if [[ ! -d "$WORKSPACE_DIR/node_modules" ]]; then + echo "Error: Node workspace dependencies are not installed." >&2 + echo "Run 'cd $WORKSPACE_DIR && npm install' first." >&2 + exit 1 +fi + +PLATFORM_PACKAGE="$(detect_platform_package)" + +( + cd "$WORKSPACE_DIR" + npm run build +) + +PUBLIC_TARBALL="$( + cd "$WORKSPACE_DIR/smolvm-embedded" && + npm pack --pack-destination "$PACK_DIR" | tail -n 1 +)" +PLATFORM_TARBALL="$( + cd "$WORKSPACE_DIR/$PLATFORM_PACKAGE" && + npm pack --pack-destination "$PACK_DIR" | tail -n 1 +)" + +cat > "$PROJECT_DIR/package.json" <<'EOF' +{ + "name": "smolvm-embedded-smoke", + "private": true +} +EOF + +( + cd "$PROJECT_DIR" + npm install --offline --no-audit --no-fund \ + "$PACK_DIR/$PLATFORM_TARBALL" "$PACK_DIR/$PUBLIC_TARBALL" >/dev/null + node - <<'EOF' +const embedded = require("smolvm-embedded"); +if (typeof embedded.quickExec !== "function") { + throw new Error("smolvm-embedded smoke install failed: quickExec export missing"); +} +console.log("smolvm-embedded smoke install OK"); +EOF +) diff --git a/sdks/node/smolvm-embedded-darwin-arm64/package.json b/sdks/node/smolvm-embedded-darwin-arm64/package.json new file mode 100644 index 0000000..fe8d3e0 --- /dev/null +++ b/sdks/node/smolvm-embedded-darwin-arm64/package.json @@ -0,0 +1,18 @@ +{ + "name": "smolvm-embedded-darwin-arm64", + "version": "0.1.0", + "description": "Internal platform package for smolvm-embedded on macOS arm64", + "main": "native/index.js", + "types": "native/index.d.ts", + "files": [ + "native", + "lib" + ], + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "license": "Apache-2.0" +} diff --git a/sdks/node/smolvm-embedded-darwin-x64/package.json b/sdks/node/smolvm-embedded-darwin-x64/package.json new file mode 100644 index 0000000..333eb99 --- /dev/null +++ b/sdks/node/smolvm-embedded-darwin-x64/package.json @@ -0,0 +1,18 @@ +{ + "name": "smolvm-embedded-darwin-x64", + "version": "0.1.0", + "description": "Internal platform package for smolvm-embedded on macOS x64", + "main": "native/index.js", + "types": "native/index.d.ts", + "files": [ + "native", + "lib" + ], + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "license": "Apache-2.0" +} diff --git a/sdks/node/smolvm-embedded-linux-arm64-gnu/package.json b/sdks/node/smolvm-embedded-linux-arm64-gnu/package.json new file mode 100644 index 0000000..882e460 --- /dev/null +++ b/sdks/node/smolvm-embedded-linux-arm64-gnu/package.json @@ -0,0 +1,18 @@ +{ + "name": "smolvm-embedded-linux-arm64-gnu", + "version": "0.1.0", + "description": "Internal platform package for smolvm-embedded on Linux arm64 GNU", + "main": "native/index.js", + "types": "native/index.d.ts", + "files": [ + "native", + "lib" + ], + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "license": "Apache-2.0" +} diff --git a/sdks/node/smolvm-embedded-linux-x64-gnu/package.json b/sdks/node/smolvm-embedded-linux-x64-gnu/package.json new file mode 100644 index 0000000..60b9396 --- /dev/null +++ b/sdks/node/smolvm-embedded-linux-x64-gnu/package.json @@ -0,0 +1,18 @@ +{ + "name": "smolvm-embedded-linux-x64-gnu", + "version": "0.1.0", + "description": "Internal platform package for smolvm-embedded on Linux x64 GNU", + "main": "native/index.js", + "types": "native/index.d.ts", + "files": [ + "native", + "lib" + ], + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "license": "Apache-2.0" +} diff --git a/sdks/node/smolvm-embedded/README.md b/sdks/node/smolvm-embedded/README.md new file mode 100644 index 0000000..7fde52e --- /dev/null +++ b/sdks/node/smolvm-embedded/README.md @@ -0,0 +1,41 @@ +# smolvm-embedded + +Public embedded Node.js SDK package. + +This package will expose the TypeScript API surface and resolve the correct +internal platform package at install/runtime. + +Current local workflow from the `smolvm` repo: + +```bash +cd sdks/node +npm install +npm run build +npm test +npm run --workspace smolvm-embedded test:integration +npm run smoke +``` + +To run only the embedded SDK integration suite: + +```bash +cd sdks/node +npm run --workspace smolvm-embedded test:integration +``` + +Example: + +```bash +cd sdks/node/smolvm-embedded +npx tsx examples/basic.ts +``` + +The package resolves one of the internal platform packages at runtime: + +- `smolvm-embedded-darwin-arm64` +- `smolvm-embedded-darwin-x64` +- `smolvm-embedded-linux-arm64-gnu` +- `smolvm-embedded-linux-x64-gnu` + +Those packages carry the `.node` binary plus the bundled `libkrun` and +`libkrunfw` libraries staged from the `smolvm` repo's `./lib` directory. diff --git a/sdks/node/smolvm-embedded/__tests__/machine.test.ts b/sdks/node/smolvm-embedded/__tests__/machine.test.ts new file mode 100644 index 0000000..8f0f597 --- /dev/null +++ b/sdks/node/smolvm-embedded/__tests__/machine.test.ts @@ -0,0 +1,168 @@ +/** + * Integration tests for smolvm-embedded. + * + * Requirements: + * - macOS with Hypervisor.framework or Linux with KVM + * - repo-local or packaged libkrun/libkrunfw build outputs available + * - smolvm agent rootfs at the default path: + * - macOS: ~/Library/Application Support/smolvm/agent-rootfs + * - Linux: ~/.local/share/smolvm/agent-rootfs + * - The current platform package must be built first: `cd sdks/node && npm run build` + */ + +import { describe, it, expect, afterAll } from "vitest"; +import { + Machine, + withMachine, + quickExec, + quickRun, + ExecResult, +} from "../src/index"; + +describe("Machine lifecycle", () => { + it("should create, exec, and delete a machine", async () => { + const sb = await Machine.create({ name: "test-lifecycle" }); + try { + expect(sb.state).toBe("running"); + + const result = await sb.exec(["echo", "hello from smolvm"]); + expect(result).toBeInstanceOf(ExecResult); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe("hello from smolvm"); + expect(result.success).toBe(true); + } finally { + await sb.delete(); + } + }); + + it("should handle non-zero exit codes", async () => { + await withMachine({ name: "test-exit-code" }, async (sb) => { + const result = await sb.exec(["sh", "-c", "exit 42"]); + expect(result.exitCode).toBe(42); + expect(result.success).toBe(false); + }); + }); + + it("should capture stderr", async () => { + await withMachine({ name: "test-stderr" }, async (sb) => { + const result = await sb.exec([ + "sh", + "-c", + 'echo "out" && echo "err" >&2', + ]); + expect(result.stdout.trim()).toBe("out"); + expect(result.stderr.trim()).toBe("err"); + }); + }); + + it("should pass environment variables", async () => { + await withMachine({ name: "test-env" }, async (sb) => { + const result = await sb.exec(["sh", "-c", "echo $MY_VAR"], { + env: { MY_VAR: "hello-env" }, + }); + expect(result.stdout.trim()).toBe("hello-env"); + }); + }); +}); + +describe("quickExec", () => { + it("should execute a command in a temporary machine", async () => { + const result = await quickExec(["echo", "quick"]); + expect(result.exitCode).toBe(0); + expect(result.stdout.trim()).toBe("quick"); + }); + + it("should execute multiple commands", async () => { + const result = await quickExec([ + "sh", + "-c", + "uname -s && echo done", + ]); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("done"); + }); +}); + +describe("Container image execution", () => { + it("should run a command in an Alpine container", async () => { + const result = await quickRun("alpine:latest", [ + "cat", + "/etc/os-release", + ], { + resources: { network: true }, + }); + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("Alpine"); + }); + + it("should pull and list images", async () => { + await withMachine({ + name: "test-images", + resources: { network: true }, + }, async (sb) => { + const info = await sb.pullImage("alpine:latest"); + expect(info.reference).toContain("alpine"); + expect(info.digest).toMatch(/^sha256:/); + expect(info.size).toBeGreaterThan(0); + + const images = await sb.listImages(); + expect(images.length).toBeGreaterThanOrEqual(1); + expect(images.some((img) => img.reference.includes("alpine"))).toBe( + true + ); + }); + }); +}); + +describe("withMachine", () => { + it("should clean up on success", async () => { + const result = await withMachine( + { name: "test-cleanup-success" }, + async (sb) => { + return sb.exec(["echo", "ok"]); + } + ); + expect(result.exitCode).toBe(0); + }); + + it("should clean up on error", async () => { + await expect( + withMachine({ name: "test-cleanup-error" }, async () => { + throw new Error("test error"); + }) + ).rejects.toThrow("test error"); + }); +}); + +describe("ExecResult", () => { + it("assertSuccess should pass for exit code 0", async () => { + const result = await quickExec(["true"]); + expect(() => result.assertSuccess()).not.toThrow(); + }); + + it("assertSuccess should throw for non-zero exit code", async () => { + const result = await quickExec(["false"]); + expect(() => result.assertSuccess()).toThrow("Command failed"); + }); +}); + +describe("Machine with resources", () => { + it("should create a machine with custom resources", async () => { + await withMachine( + { + name: "test-resources", + resources: { + cpus: 2, + memoryMb: 1024, + network: true, + }, + }, + async (sb) => { + // Verify CPU count visible in guest + const result = await sb.exec(["nproc"]); + expect(result.exitCode).toBe(0); + expect(parseInt(result.stdout.trim())).toBe(2); + } + ); + }); +}); diff --git a/sdks/node/smolvm-embedded/examples/basic.ts b/sdks/node/smolvm-embedded/examples/basic.ts new file mode 100644 index 0000000..d206bf2 --- /dev/null +++ b/sdks/node/smolvm-embedded/examples/basic.ts @@ -0,0 +1,82 @@ +/** + * Basic usage example for smolvm-embedded. + * + * Run with: npx tsx examples/basic.ts + */ + +import { + Machine, + withMachine, + quickExec, + quickRun, +} from "../src/index"; + +async function main() { + console.log("=== smolvm-embedded examples ===\n"); + + // 1. Quick one-liner execution + console.log("1. quickExec — run a command in a temporary VM:"); + const hello = await quickExec(["echo", "Hello from a microVM!"]); + console.log(` stdout: ${hello.stdout.trim()}`); + console.log(` exit code: ${hello.exitCode}\n`); + + // 2. Quick container execution + console.log("2. quickRun — run a command in an Alpine container:"); + const alpine = await quickRun("alpine:latest", [ + "cat", + "/etc/os-release", + ], { + resources: { network: true }, + }); + console.log(` stdout (first line): ${alpine.stdout.split("\n")[0]}`); + console.log(` exit code: ${alpine.exitCode}\n`); + + // 3. Managed machine lifecycle + console.log("3. withMachine — managed lifecycle:"); + await withMachine({ name: "example-machine" }, async (sb) => { + console.log(` machine "${sb.name}" state: ${sb.state}`); + + const date = await sb.exec(["date", "+%Y-%m-%d"]); + console.log(` date: ${date.stdout.trim()}`); + + const uname = await sb.exec(["uname", "-a"]); + console.log(` uname: ${uname.stdout.trim()}`); + }); + console.log(" machine cleaned up.\n"); + + // 4. Full control + console.log("4. Full control — create, use, delete:"); + const sb = await Machine.create({ + name: "example-full-control", + resources: { + cpus: 2, + memoryMb: 1024, + network: true, + }, + }); + + try { + const nproc = await sb.exec(["nproc"]); + console.log(` vCPUs: ${nproc.stdout.trim()}`); + + // Pull an image and run in it + const info = await sb.pullImage("alpine:latest"); + console.log( + ` pulled: ${info.reference} (${(info.size / 1024 / 1024).toFixed(1)} MB)` + ); + + const result = await sb.run("alpine:latest", [ + "sh", + "-c", + "echo Running in Alpine on $(uname -m)", + ]); + console.log(` container output: ${result.stdout.trim()}`); + } finally { + await sb.delete(); + console.log(" machine deleted.\n"); + } + + console.log("=== All examples completed ==="); +} + +main().catch(console.error); diff --git a/sdks/node/smolvm-embedded/examples/create-and-start.ts b/sdks/node/smolvm-embedded/examples/create-and-start.ts new file mode 100644 index 0000000..8fa3b26 --- /dev/null +++ b/sdks/node/smolvm-embedded/examples/create-and-start.ts @@ -0,0 +1,36 @@ +/** + * Create and start a named microVM from the embedded Node SDK. + * + * Run with: npx tsx examples/create-and-start.ts + * + * This intentionally does not delete the VM. It leaves `created-by-node` + * running so it can be inspected from the smolvm CLI. + */ + +import { Machine } from "../src/index"; + +async function main() { + const machine = await Machine.create({ + name: "created-by-node", + persistent: true, + }); + + console.log(`created machine: ${machine.name}`); + console.log(`state before start: ${machine.state}`); + + await machine.start(); + + console.log(`state after start: ${machine.state}`); + console.log(`is running: ${machine.isRunning}`); + console.log(`pid: ${machine.pid ?? "unknown"}`); + console.log(""); + console.log("VM was not deleted."); + console.log("Inspect it with:"); + console.log(" smolvm machine status --name created-by-node"); + console.log(" smolvm machine ls"); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/sdks/node/smolvm-embedded/integration-tests/exec-and-files.test.ts b/sdks/node/smolvm-embedded/integration-tests/exec-and-files.test.ts new file mode 100644 index 0000000..bb8f3e2 --- /dev/null +++ b/sdks/node/smolvm-embedded/integration-tests/exec-and-files.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; + +import { withMachine } from "../src/index.js"; + +import { collectExecStream, uniqueName } from "./helpers.js"; + +describe("embedded sdk exec and file io", () => { + it("supports exec env, workdir, stderr, and non-zero exit codes", async () => { + await withMachine({ name: uniqueName("exec") }, async (machine) => { + const result = await machine.exec( + ["sh", "-lc", 'echo "$GREETING"; pwd; echo "warn" >&2; exit 7'], + { + env: { GREETING: "hello-sdk" }, + workdir: "/tmp", + } + ); + + expect(result.exitCode).toBe(7); + expect(result.stdout).toContain("hello-sdk"); + expect(result.stdout).toContain("/tmp"); + expect(result.stderr).toContain("warn"); + }); + }); + + it("collects stdout, stderr, and exit events from streaming exec", async () => { + await withMachine({ name: uniqueName("stream") }, async (machine) => { + const events = await machine.execStreaming([ + "sh", + "-lc", + "echo line-1 && echo line-2 >&2 && echo line-3", + ]); + + const result = collectExecStream(events); + expect(result.stdout).toContain("line-1"); + expect(result.stdout).toContain("line-3"); + expect(result.stderr).toContain("line-2"); + expect(result.exitCode).toBe(0); + expect(result.errors).toEqual([]); + }); + }); + + it("supports file upload and download", async () => { + await withMachine({ name: uniqueName("files") }, async (machine) => { + const upload = `hello-from-host-${Date.now().toString(36)}`; + + await machine.writeFile("/tmp/uploaded.txt", upload, { mode: 0o640 }); + + const uploadCheck = await machine.exec(["cat", "/tmp/uploaded.txt"]); + expect(uploadCheck.exitCode).toBe(0); + expect(uploadCheck.stdout.trim()).toBe(upload); + + await machine.exec([ + "sh", + "-lc", + "echo 'hello from vm' > /tmp/from-vm.txt", + ]); + + const downloaded = await machine.readFile("/tmp/from-vm.txt"); + expect(downloaded.toString("utf8").trim()).toBe("hello from vm"); + }); + }); +}); diff --git a/sdks/node/smolvm-embedded/integration-tests/helpers.ts b/sdks/node/smolvm-embedded/integration-tests/helpers.ts new file mode 100644 index 0000000..a8b363a --- /dev/null +++ b/sdks/node/smolvm-embedded/integration-tests/helpers.ts @@ -0,0 +1,134 @@ +import { createServer } from "node:net"; +import * as http from "node:http"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +export function uniqueName(prefix: string): string { + const safePrefix = prefix.replace(/[^a-z0-9-]/gi, "").slice(0, 10) || "sdk"; + return [ + safePrefix, + process.pid.toString(36), + Date.now().toString(36), + Math.random().toString(36).slice(2, 6), + ].join("-"); +} + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function makeTempDir(prefix: string): Promise { + return mkdtemp(join(tmpdir(), `${prefix}-`)); +} + +export async function removeTempDir(path: string): Promise { + await rm(path, { recursive: true, force: true }); +} + +export async function getFreePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (!address || typeof address === "string") { + server.close(() => reject(new Error("Failed to allocate a TCP port"))); + return; + } + + const { port } = address; + server.close((err) => { + if (err) { + reject(err); + return; + } + resolve(port); + }); + }); + }); +} + +export async function httpGetText(url: string): Promise { + return new Promise((resolve, reject) => { + const req = http.get(url, (res) => { + let body = ""; + res.setEncoding("utf8"); + res.on("data", (chunk) => { + body += chunk; + }); + res.on("end", () => { + if ((res.statusCode ?? 0) >= 400) { + reject(new Error(`HTTP ${res.statusCode}: ${body}`)); + return; + } + resolve(body); + }); + }); + + req.once("error", reject); + }); +} + +export async function waitForHttpText( + url: string, + attempts = 20, + intervalMs = 500 +): Promise { + let lastError: unknown; + + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { + return await httpGetText(url); + } catch (error) { + lastError = error; + await sleep(intervalMs); + } + } + + throw lastError instanceof Error + ? lastError + : new Error(`Timed out waiting for ${url}`); +} + +export function collectExecStream(events: Array<{ + kind: string; + data?: string; + exitCode?: number; + message?: string; +}>): { + stdout: string; + stderr: string; + exitCode: number | undefined; + errors: string[]; +} { + let stdout = ""; + let stderr = ""; + let exitCode: number | undefined; + const errors: string[] = []; + + for (const event of events) { + switch (event.kind) { + case "stdout": + stdout += event.data ?? ""; + break; + case "stderr": + stderr += event.data ?? ""; + break; + case "exit": + exitCode = event.exitCode; + break; + case "error": + if (event.message) { + errors.push(event.message); + } + break; + default: + errors.push(`unknown event kind: ${event.kind}`); + break; + } + } + + return { stdout, stderr, exitCode, errors }; +} diff --git a/sdks/node/smolvm-embedded/integration-tests/images.test.ts b/sdks/node/smolvm-embedded/integration-tests/images.test.ts new file mode 100644 index 0000000..5a3b231 --- /dev/null +++ b/sdks/node/smolvm-embedded/integration-tests/images.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; + +import { + quickExec, + quickRun, + withMachine, +} from "../src/index.js"; + +import { uniqueName } from "./helpers.js"; + +describe("embedded sdk one-shot and image workflows", () => { + it("quickExec runs one-shot commands in a temporary VM", async () => { + const result = await quickExec(["sh", "-lc", "echo quick && uname -s"]); + + expect(result.assertSuccess().stdout).toContain("quick"); + }); + + it("pulls, lists, and runs OCI images with env and workdir", async () => { + await withMachine( + { + name: uniqueName("images"), + resources: { network: true }, + }, + async (machine) => { + const image = await machine.pullImage("alpine:latest"); + expect(image.reference).toContain("alpine"); + expect(image.digest).toMatch(/^sha256:/); + expect(image.size).toBeGreaterThan(0); + + const images = await machine.listImages(); + expect(images.some((entry) => entry.reference.includes("alpine"))).toBe( + true + ); + + const result = await machine.run( + "alpine:latest", + [ + "sh", + "-lc", + 'echo "$RUN_VALUE" && pwd && grep "^NAME=" /etc/os-release', + ], + { + env: { RUN_VALUE: "from-run" }, + workdir: "/tmp", + } + ); + + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("from-run"); + expect(result.stdout).toContain("/tmp"); + expect(result.stdout).toContain("Alpine"); + } + ); + }); + + it("quickRun executes container commands without explicit lifecycle management", async () => { + const result = await quickRun( + "alpine:latest", + [ + "sh", + "-lc", + 'echo quick-run && grep "^NAME=" /etc/os-release', + ], + { + resources: { network: true }, + } + ); + + expect(result.exitCode).toBe(0); + expect(result.stdout).toContain("quick-run"); + expect(result.stdout).toContain("Alpine"); + }); +}); diff --git a/sdks/node/smolvm-embedded/integration-tests/lifecycle.test.ts b/sdks/node/smolvm-embedded/integration-tests/lifecycle.test.ts new file mode 100644 index 0000000..2d3dec5 --- /dev/null +++ b/sdks/node/smolvm-embedded/integration-tests/lifecycle.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from "vitest"; + +import { + Machine, + NotFoundError, + withMachine, +} from "../src/index.js"; + +import { uniqueName } from "./helpers.js"; + +describe("embedded sdk lifecycle", () => { + it("supports persistent create, start, reconnect, stop, and restart", async () => { + const name = uniqueName("persist"); + const machine = await Machine.create({ name, persistent: true }); + + try { + expect(machine.isStarted).toBe(false); + expect(machine.isRunning).toBe(false); + expect(machine.state).toBe("stopped"); + expect(machine.pid).toBeNull(); + + await machine.start(); + + expect(machine.isStarted).toBe(true); + expect(machine.isRunning).toBe(true); + expect(machine.state).toBe("running"); + expect(machine.pid).toEqual(expect.any(Number)); + + const connected = await Machine.connect(name); + expect(connected.name).toBe(name); + expect(connected.isRunning).toBe(true); + + const result = await connected.exec(["echo", "connected"]); + expect(result.stdout.trim()).toBe("connected"); + + await machine.stop(); + + expect(machine.isStarted).toBe(false); + expect(machine.isRunning).toBe(false); + expect(machine.state).toBe("stopped"); + expect(machine.pid).toBeNull(); + + await machine.start(); + expect(machine.state).toBe("running"); + } finally { + await machine.delete().catch(() => { + // Best-effort cleanup for persistent machines. + }); + } + }); + + it("withMachine cleans up ephemeral machines after the callback exits", async () => { + const name = uniqueName("cleanup"); + + const result = await withMachine({ name }, async (machine) => { + expect(machine.state).toBe("running"); + return machine.exec(["echo", "ephemeral"]); + }); + + expect(result.stdout.trim()).toBe("ephemeral"); + await expect(Machine.connect(name)).rejects.toBeInstanceOf(NotFoundError); + }); +}); diff --git a/sdks/node/smolvm-embedded/integration-tests/network-and-mounts.test.ts b/sdks/node/smolvm-embedded/integration-tests/network-and-mounts.test.ts new file mode 100644 index 0000000..f640903 --- /dev/null +++ b/sdks/node/smolvm-embedded/integration-tests/network-and-mounts.test.ts @@ -0,0 +1,120 @@ +import { writeFile, readFile } from "node:fs/promises"; + +import { describe, expect, it } from "vitest"; + +import { withMachine } from "../src/index.js"; + +import { + getFreePort, + makeTempDir, + removeTempDir, + sleep, + uniqueName, + waitForHttpText, +} from "./helpers.js"; + +describe("embedded sdk networking, mounts, and ports", () => { + it("keeps outbound network disabled by default", async () => { + await withMachine({ name: uniqueName("no-net") }, async (machine) => { + const result = await machine.exec(["nslookup", "cloudflare.com"]); + expect(result.exitCode).not.toBe(0); + }); + }); + + it("supports DNS lookups when network is enabled", async () => { + await withMachine( + { + name: uniqueName("dns"), + resources: { network: true }, + }, + async (machine) => { + const result = await machine.exec([ + "sh", + "-lc", + "nslookup cloudflare.com && nslookup github.com", + ]); + + expect(result.exitCode).toBe(0); + expect(result.output).toContain("Address"); + } + ); + }); + + it("supports read-only and read-write host mounts", async () => { + const readWriteDir = await makeTempDir("smolvm-rw"); + const readOnlyDir = await makeTempDir("smolvm-ro"); + + try { + await writeFile(`${readWriteDir}/input.txt`, "rw-data\n", "utf8"); + await writeFile(`${readOnlyDir}/readonly.txt`, "ro-data\n", "utf8"); + + await withMachine( + { + name: uniqueName("mounts"), + mounts: [ + { source: readWriteDir, target: "/mnt/rw", readOnly: false }, + { source: readOnlyDir, target: "/mnt/ro", readOnly: true }, + ], + }, + async (machine) => { + const readResult = await machine.exec([ + "sh", + "-lc", + "cat /mnt/rw/input.txt && cat /mnt/ro/readonly.txt", + ]); + + expect(readResult.exitCode).toBe(0); + expect(readResult.stdout).toContain("rw-data"); + expect(readResult.stdout).toContain("ro-data"); + + const writeResult = await machine.exec([ + "sh", + "-lc", + "echo written-from-vm > /mnt/rw/output.txt", + ]); + expect(writeResult.exitCode).toBe(0); + + const blockedWrite = await machine.exec([ + "sh", + "-lc", + "echo blocked > /mnt/ro/blocked.txt", + ]); + expect(blockedWrite.exitCode).not.toBe(0); + } + ); + + const written = await readFile(`${readWriteDir}/output.txt`, "utf8"); + expect(written.trim()).toBe("written-from-vm"); + } finally { + await removeTempDir(readWriteDir); + await removeTempDir(readOnlyDir); + } + }); + + it("supports host-to-guest port mappings", async () => { + const hostPort = await getFreePort(); + + await withMachine( + { + name: uniqueName("ports"), + ports: [{ host: hostPort, guest: 8080 }], + resources: { network: true }, + }, + async (machine) => { + const serverPromise = machine.exec([ + "sh", + "-lc", + "printf 'HTTP/1.1 200 OK\\r\\nContent-Length: 2\\r\\n\\r\\nok' | nc -l -p 8080 -w 15", + ]); + + await sleep(1000); + + const body = await waitForHttpText(`http://127.0.0.1:${hostPort}/`); + expect(body).toBe("ok"); + + const serverResult = await serverPromise; + expect(serverResult.exitCode).toBe(0); + } + ); + }); +}); diff --git a/sdks/node/smolvm-embedded/integration-tests/resources.test.ts b/sdks/node/smolvm-embedded/integration-tests/resources.test.ts new file mode 100644 index 0000000..e247d05 --- /dev/null +++ b/sdks/node/smolvm-embedded/integration-tests/resources.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest"; + +import { withMachine } from "../src/index.js"; + +import { uniqueName } from "./helpers.js"; + +describe("embedded sdk resource configuration", () => { + it("applies cpu, memory, and overlay sizing", async () => { + await withMachine( + { + name: uniqueName("res"), + resources: { + cpus: 2, + memoryMb: 1024, + overlayGib: 4, + }, + }, + async (machine) => { + const cpuResult = await machine.exec(["nproc"]); + expect(cpuResult.exitCode).toBe(0); + expect(Number.parseInt(cpuResult.stdout.trim(), 10)).toBe(2); + + const memResult = await machine.exec([ + "sh", + "-lc", + "awk '/MemTotal/ { print $2 }' /proc/meminfo", + ]); + expect(memResult.exitCode).toBe(0); + expect(Number.parseInt(memResult.stdout.trim(), 10)).toBeGreaterThan( + 900_000 + ); + + const overlayResult = await machine.exec([ + "sh", + "-lc", + "df -m / | tail -1 | awk '{print $2}'", + ]); + expect(overlayResult.exitCode).toBe(0); + expect( + Number.parseInt(overlayResult.stdout.trim(), 10) + ).toBeGreaterThan(3_000); + } + ); + }); +}); diff --git a/sdks/node/smolvm-embedded/package.json b/sdks/node/smolvm-embedded/package.json new file mode 100644 index 0000000..6ca8d63 --- /dev/null +++ b/sdks/node/smolvm-embedded/package.json @@ -0,0 +1,51 @@ +{ + "name": "smolvm-embedded", + "version": "0.2.0", + "description": "Embedded Node.js SDK for smolvm", + "main": "dist/index.js", + "module": "dist/index.mjs", + "types": "dist/index.d.ts", + "exports": { + ".": { + "import": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + }, + "require": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + } + }, + "files": [ + "dist" + ], + "scripts": { + "build:ts": "../node_modules/.bin/tsup", + "build": "npm run build:ts", + "dev": "../node_modules/.bin/tsup src/index.ts --format cjs,esm --dts --watch", + "test": "../node_modules/.bin/vitest run", + "test:integration": "../node_modules/.bin/vitest run integration-tests", + "test:watch": "../node_modules/.bin/vitest", + "example:basic": "../node_modules/.bin/tsx examples/basic.ts", + "example:create-and-start": "../node_modules/.bin/tsx examples/create-and-start.ts" + }, + "optionalDependencies": { + "smolvm-embedded-darwin-arm64": "0.1.0", + "smolvm-embedded-darwin-x64": "0.1.0", + "smolvm-embedded-linux-arm64-gnu": "0.1.0", + "smolvm-embedded-linux-x64-gnu": "0.1.0" + }, + "engines": { + "node": ">= 18.0.0" + }, + "keywords": [ + "smolvm", + "embedded", + "microvm", + "machine", + "virtualization", + "napi" + ], + "license": "Apache-2.0" +} diff --git a/sdks/node/smolvm-embedded/src/errors.ts b/sdks/node/smolvm-embedded/src/errors.ts new file mode 100644 index 0000000..e9e7e40 --- /dev/null +++ b/sdks/node/smolvm-embedded/src/errors.ts @@ -0,0 +1,75 @@ +/** + * Error classes for smolvm-embedded. + */ + +/** + * Base error class for all smolvm embedded SDK errors. + */ +export class SmolvmError extends Error { + readonly code: string; + + constructor(message: string, code: string) { + super(message); + this.name = "SmolvmError"; + this.code = code; + } +} + +/** VM or resource not found. */ +export class NotFoundError extends SmolvmError { + constructor(message: string) { + super(message, "NOT_FOUND"); + this.name = "NotFoundError"; + } +} + +/** Invalid VM state for the requested operation. */ +export class InvalidStateError extends SmolvmError { + constructor(message: string) { + super(message, "INVALID_STATE"); + this.name = "InvalidStateError"; + } +} + +/** Hypervisor not available on this system. */ +export class HypervisorUnavailableError extends SmolvmError { + constructor(message: string) { + super(message, "HYPERVISOR_UNAVAILABLE"); + this.name = "HypervisorUnavailableError"; + } +} + +/** Resource conflict (e.g., machine already exists). */ +export class ConflictError extends SmolvmError { + constructor(message: string) { + super(message, "CONFLICT"); + this.name = "ConflictError"; + } +} + +/** + * Parse a NAPI error into a typed SmolvmError. + * + * NAPI errors from the Rust side are formatted as `[CODE] message`. + */ +export function parseNativeError(err: Error): SmolvmError { + const match = err.message.match(/^\[(\w+)\]\s*(.*)/s); + if (!match) { + return new SmolvmError(err.message, "SMOLVM_ERROR"); + } + + const [, code, message] = match; + + switch (code) { + case "NOT_FOUND": + return new NotFoundError(message); + case "INVALID_STATE": + return new InvalidStateError(message); + case "HYPERVISOR_UNAVAILABLE": + return new HypervisorUnavailableError(message); + case "CONFLICT": + return new ConflictError(message); + default: + return new SmolvmError(message, code); + } +} diff --git a/sdks/node/smolvm-embedded/src/execution.ts b/sdks/node/smolvm-embedded/src/execution.ts new file mode 100644 index 0000000..5e1a285 --- /dev/null +++ b/sdks/node/smolvm-embedded/src/execution.ts @@ -0,0 +1,60 @@ +/** + * Execution result types for smolvm-embedded. + */ + +/** + * Error thrown when assertSuccess() is called on a failed execution. + */ +export class ExecutionError extends Error { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; + + constructor(exitCode: number, stdout: string, stderr: string) { + super(`Command failed with exit code ${exitCode}`); + this.name = "ExecutionError"; + this.exitCode = exitCode; + this.stdout = stdout; + this.stderr = stderr; + } +} + +/** + * Rich result object from command execution. + */ +export class ExecResult { + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; + + constructor(exitCode: number, stdout: string, stderr: string) { + this.exitCode = exitCode; + this.stdout = stdout; + this.stderr = stderr; + } + + /** Whether the command exited successfully (exit code 0). */ + get success(): boolean { + return this.exitCode === 0; + } + + /** Combined stdout and stderr output. */ + get output(): string { + if (this.stdout && this.stderr) { + return `${this.stdout}\n${this.stderr}`; + } + return this.stdout || this.stderr; + } + + /** + * Assert that the command succeeded (exit code 0). + * Throws ExecutionError if the command failed. + * Returns this for method chaining. + */ + assertSuccess(): this { + if (!this.success) { + throw new ExecutionError(this.exitCode, this.stdout, this.stderr); + } + return this; + } +} diff --git a/sdks/node/smolvm-embedded/src/index.ts b/sdks/node/smolvm-embedded/src/index.ts new file mode 100644 index 0000000..61753b9 --- /dev/null +++ b/sdks/node/smolvm-embedded/src/index.ts @@ -0,0 +1,57 @@ +/** + * smolvm-embedded — Embedded Node.js SDK for smolvm. + * + * Embed microVMs directly in your Node.js process via NAPI-RS. + * No daemon required. + * + * @example + * ```ts + * import { quickExec, withMachine, Machine } from "smolvm-embedded"; + * + * // One-liner + * const result = await quickExec(["echo", "hello"]); + * + * // Managed lifecycle + * await withMachine({ name: "my-machine" }, async (sb) => { + * const r = await sb.exec(["date"]); + * console.log(r.stdout); + * }); + * + * // Full control + * const sb = await Machine.create({ name: "my-vm" }); + * const r = await sb.run("alpine:latest", ["cat", "/etc/os-release"]); + * console.log(r.stdout); + * await sb.delete(); + * ``` + */ + +// Core classes +export { Machine, withMachine, quickExec, quickRun } from "./machine.js"; +export { ExecResult, ExecutionError } from "./execution.js"; + +// Presets +export { PythonMachine } from "./presets/python.js"; +export { NodeMachine } from "./presets/node.js"; + +// Error classes +export { + SmolvmError, + NotFoundError, + InvalidStateError, + HypervisorUnavailableError, + ConflictError, + parseNativeError, +} from "./errors.js"; + +// Types +export type { + MachineConfig, + MountSpec, + PortSpec, + ResourceSpec, + ExecOptions, + ExecStreamEvent, + FileWriteOptions, + CodeOptions, + ImageInfo, +} from "./types.js"; diff --git a/sdks/node/smolvm-embedded/src/machine.ts b/sdks/node/smolvm-embedded/src/machine.ts new file mode 100644 index 0000000..2678aa2 --- /dev/null +++ b/sdks/node/smolvm-embedded/src/machine.ts @@ -0,0 +1,346 @@ +/** + * Machine — high-level wrapper around NapiMachine. + * + * Provides the same ergonomic API as @smolvm/node but runs entirely + * in-process via native bindings (no daemon required). + */ + +import { ExecResult } from "./execution.js"; +import { parseNativeError } from "./errors.js"; +import { loadNativeBinding } from "./native-binding.js"; +import type { + MachineConfig, + ExecOptions, + ExecStreamEvent, + FileWriteOptions, + ImageInfo, + MountSpec, + PortSpec, + ResourceSpec, +} from "./types.js"; + +const { NapiMachine } = loadNativeBinding(); + +/** + * Convert SDK ExecOptions to the NAPI format. + */ +function toNapiExecOptions( + options?: ExecOptions +): { env?: Array<{ key: string; value: string }>; workdir?: string; timeoutSecs?: number } | undefined { + if (!options) return undefined; + return { + env: options.env + ? Object.entries(options.env).map(([key, value]) => ({ key, value })) + : undefined, + workdir: options.workdir, + timeoutSecs: options.timeout, + }; +} + +function toSdkExecStreamEvent(event: { + kind: string; + data?: string; + exitCode?: number; + message?: string; +}): ExecStreamEvent { + switch (event.kind) { + case "stdout": + return { kind: "stdout", data: event.data }; + case "stderr": + return { kind: "stderr", data: event.data }; + case "exit": + return { kind: "exit", exitCode: event.exitCode }; + case "error": + return { kind: "error", message: event.message }; + default: + throw new Error(`Unknown exec stream event kind: ${event.kind}`); + } +} + +/** + * Convert SDK config to NAPI format. + */ +function toNapiConfig(config: MachineConfig) { + return { + name: config.name, + mounts: config.mounts?.map((m: MountSpec) => ({ + source: m.source, + target: m.target, + readOnly: m.readOnly, + })), + ports: config.ports?.map((p: PortSpec) => ({ + host: p.host, + guest: p.guest, + })), + resources: config.resources + ? { + cpus: config.resources.cpus, + memoryMib: config.resources.memoryMb, + network: config.resources.network, + storageGib: config.resources.storageGib, + overlayGib: config.resources.overlayGib, + } + : undefined, + persistent: config.persistent, + }; +} + +/** + * Wrap a native call with error translation. + */ +async function wrapNative(fn: () => Promise): Promise { + try { + return await fn(); + } catch (err) { + throw parseNativeError(err as Error); + } +} + +/** + * A machine wrapping a microVM with native bindings. + * + * No daemon required — the VM runs directly in the Node.js process + * via libkrun (Hypervisor.framework on macOS, KVM on Linux). + */ +export class Machine { + readonly name: string; + private native: InstanceType; + private started = false; + + protected constructor(config: MachineConfig) { + this.name = config.name; + this.native = new NapiMachine(toNapiConfig(config)); + } + + private static fromNative( + name: string, + native: InstanceType + ): Machine { + const machine = Object.create(Machine.prototype) as Machine; + (machine as any).name = name; + (machine as any).native = native; + (machine as any).started = true; + return machine; + } + + /** + * Create a new machine. Auto-starts unless `persistent: true` is set. + */ + static async create(config: MachineConfig): Promise { + const machine = new Machine(config); + if (!config.persistent) { + await machine.start(); + } + return machine; + } + + /** + * Connect to an already-running machine by name. + * + * Throws NotFoundError if no running VM exists with the given name. + */ + static async connect(name: string): Promise { + try { + return Machine.fromNative(name, NapiMachine.connect(name)); + } catch (err) { + throw parseNativeError(err as Error); + } + } + + /** + * Start the machine VM. + * + * Boots a microVM via fork + libkrun, waits for the agent to be ready, + * then establishes a vsock connection. If the VM is already running + * with matching config, this is a no-op. + */ + async start(): Promise { + await wrapNative(() => this.native.start()); + this.started = true; + } + + /** Whether the machine has been started. */ + get isStarted(): boolean { + return this.started; + } + + /** Get the current VM state: "stopped", "starting", "running", or "stopping". */ + get state(): string { + return this.native.state(); + } + + /** Whether the VM process is currently running. */ + get isRunning(): boolean { + return this.native.isRunning; + } + + /** The child PID of the VM process, or null if not running. */ + get pid(): number | null { + return this.native.pid ?? null; + } + + /** + * Execute a command directly in the VM. + * + * @param command - Command and arguments (e.g., ["echo", "hello"]) + * @param options - Execution options (env, workdir, timeout) + */ + async exec(command: string[], options?: ExecOptions): Promise { + const result = await wrapNative<{ exitCode: number; stdout: string; stderr: string }>(() => + this.native.exec(command, toNapiExecOptions(options)) + ); + return new ExecResult(result.exitCode, result.stdout, result.stderr); + } + + /** + * Pull an OCI image and run a command inside it. + * + * @param image - OCI image reference (e.g., "alpine:latest") + * @param command - Command and arguments + * @param options - Execution options + */ + async run( + image: string, + command: string[], + options?: ExecOptions + ): Promise { + const result = await wrapNative<{ exitCode: number; stdout: string; stderr: string }>(() => + this.native.run(image, command, toNapiExecOptions(options)) + ); + return new ExecResult(result.exitCode, result.stdout, result.stderr); + } + + /** + * Pull an OCI image into the machine's storage. + */ + async pullImage(image: string): Promise { + return wrapNative(() => this.native.pullImage(image)); + } + + /** + * List all cached OCI images. + */ + async listImages(): Promise { + return wrapNative(() => this.native.listImages()); + } + + /** + * Write a file into the running VM. + */ + async writeFile( + path: string, + data: string | Uint8Array, + options?: FileWriteOptions + ): Promise { + const bytes = typeof data === "string" ? Buffer.from(data) : Buffer.from(data); + await wrapNative(() => this.native.writeFile(path, bytes, options)); + } + + /** + * Read a file from the running VM. + */ + async readFile(path: string): Promise { + const data = await wrapNative(() => this.native.readFile(path)); + return Buffer.from(data); + } + + /** + * Execute a command and collect streaming stdout/stderr/exit events. + */ + async execStreaming( + command: string[], + options?: ExecOptions + ): Promise { + const events = await wrapNative< + Array<{ kind: string; data?: string; exitCode?: number; message?: string }> + >(() => this.native.execStreaming(command, toNapiExecOptions(options))); + return events.map(toSdkExecStreamEvent); + } + + /** + * Stop the machine VM gracefully. + */ + async stop(): Promise { + await wrapNative(() => this.native.stop()); + this.started = false; + } + + /** + * Stop the machine and delete all associated storage. + */ + async delete(): Promise { + await wrapNative(() => this.native.delete()); + this.started = false; + } +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +/** + * Create a machine, run a function with it, then clean up. + * + * @example + * ```ts + * const result = await withMachine({ name: "my-task" }, async (sb) => { + * return await sb.exec(["echo", "hello"]); + * }); + * ``` + */ +export async function withMachine( + config: MachineConfig, + fn: (machine: Machine) => Promise +): Promise { + const machine = await Machine.create(config); + try { + return await fn(machine); + } finally { + await machine.delete().catch(() => { + // Best-effort cleanup + }); + } +} + +/** + * Quick one-shot command execution in a temporary machine. + * + * Creates a machine, runs the command, cleans up, and returns the result. + * + * @example + * ```ts + * const result = await quickExec(["echo", "hello"]); + * console.log(result.stdout); // "hello\n" + * ``` + */ +export async function quickExec( + command: string[], + options?: MachineConfig & ExecOptions +): Promise { + const name = options?.name ?? `quick-${Date.now().toString(36)}`; + return withMachine({ ...options, name }, (sb) => + sb.exec(command, options) + ); +} + +/** + * Quick one-shot command execution in a container image. + * + * Creates a machine, pulls the image, runs the command, cleans up. + * + * @example + * ```ts + * const result = await quickRun("alpine:latest", ["cat", "/etc/os-release"]); + * console.log(result.stdout); + * ``` + */ +export async function quickRun( + image: string, + command: string[], + options?: MachineConfig & ExecOptions +): Promise { + const name = options?.name ?? `quick-${Date.now().toString(36)}`; + return withMachine({ ...options, name }, (sb) => + sb.run(image, command, options) + ); +} diff --git a/sdks/node/smolvm-embedded/src/native-binding.ts b/sdks/node/smolvm-embedded/src/native-binding.ts new file mode 100644 index 0000000..6ac39c1 --- /dev/null +++ b/sdks/node/smolvm-embedded/src/native-binding.ts @@ -0,0 +1,28 @@ +import { getPlatformPackageName } from "./platform-package.js"; +import { getPlatformPackageRoot } from "./platform-package.js"; +import { prepareNativeRuntime } from "./runtime-env.js"; +import { resolve } from "node:path"; + +let nativeBinding: any; + +export function loadNativeBinding(): any { + if (nativeBinding) { + return nativeBinding; + } + + const packageName = getPlatformPackageName(); + const packageRoot = getPlatformPackageRoot(); + prepareNativeRuntime(); + + try { + nativeBinding = require(resolve(packageRoot, "native/index.js")); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `Failed to load smolvm-embedded native binding from '${packageName}'. ` + + `Original error: ${message}` + ); + } + + return nativeBinding; +} diff --git a/sdks/node/smolvm-embedded/src/platform-package.ts b/sdks/node/smolvm-embedded/src/platform-package.ts new file mode 100644 index 0000000..1814a6c --- /dev/null +++ b/sdks/node/smolvm-embedded/src/platform-package.ts @@ -0,0 +1,52 @@ +import { existsSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +function currentPlatformKey(): string { + return `${process.platform}:${process.arch}`; +} + +function currentPackageRoot(): string { + if (typeof __dirname === "string") { + return resolve(__dirname, ".."); + } + + return resolve(dirname(fileURLToPath(import.meta.url)), ".."); +} + +export function getPlatformPackageName(): string { + switch (currentPlatformKey()) { + case "darwin:arm64": + return "smolvm-embedded-darwin-arm64"; + case "darwin:x64": + return "smolvm-embedded-darwin-x64"; + case "linux:arm64": + return "smolvm-embedded-linux-arm64-gnu"; + case "linux:x64": + return "smolvm-embedded-linux-x64-gnu"; + default: + throw new Error( + `Unsupported smolvm-embedded platform: ${process.platform}/${process.arch}` + ); + } +} + +export function getPlatformPackageRoot(): string { + const packageName = getPlatformPackageName(); + const packageRoot = currentPackageRoot(); + const siblingPackageRoot = resolve(packageRoot, "..", packageName); + const siblingPackageJson = resolve(siblingPackageRoot, "package.json"); + + if (existsSync(siblingPackageJson)) { + return siblingPackageRoot; + } + + try { + return dirname(require.resolve(`${packageName}/package.json`)); + } catch { + throw new Error( + `Missing internal platform package '${packageName}' for smolvm-embedded. ` + + "Build the current platform package from sdks/node first." + ); + } +} diff --git a/sdks/node/smolvm-embedded/src/presets/node.ts b/sdks/node/smolvm-embedded/src/presets/node.ts new file mode 100644 index 0000000..c41b7c4 --- /dev/null +++ b/sdks/node/smolvm-embedded/src/presets/node.ts @@ -0,0 +1,79 @@ +/** + * NodeMachine — pre-configured machine for running Node.js code. + */ + +import { Machine } from "../machine.js"; +import { ExecResult } from "../execution.js"; +import type { MachineConfig, ExecOptions, CodeOptions } from "../types.js"; + +export class NodeMachine extends Machine { + static readonly DEFAULT_IMAGE = "node:22-alpine"; + + static async create(config: MachineConfig): Promise { + const machine = new NodeMachine(config); + await machine.start(); + // Pre-pull the Node image + await machine.pullImage(NodeMachine.DEFAULT_IMAGE); + return machine; + } + + private constructor(config: MachineConfig) { + super(config); + } + + /** Run JavaScript code. */ + async runCode(code: string, options?: CodeOptions): Promise { + const image = options?.image ?? NodeMachine.DEFAULT_IMAGE; + return this.run(image, ["node", "-e", code], options); + } + + /** Run a JavaScript file (must be in a mounted directory). */ + async runFile(path: string, options?: CodeOptions): Promise { + const image = options?.image ?? NodeMachine.DEFAULT_IMAGE; + return this.run(image, ["node", path], options); + } + + /** Run ESM code. */ + async runESM(code: string, options?: CodeOptions): Promise { + const image = options?.image ?? NodeMachine.DEFAULT_IMAGE; + return this.run( + image, + ["node", "--input-type=module", "-e", code], + options + ); + } + + /** Evaluate a JavaScript expression and return the result. */ + async evaluate( + expression: string, + options?: CodeOptions + ): Promise { + const code = `console.log(JSON.stringify(${expression}))`; + return this.runCode(code, options); + } + + /** Run npm commands. */ + async npm(args: string[], options?: ExecOptions): Promise { + return this.run(NodeMachine.DEFAULT_IMAGE, ["npm", ...args], options); + } + + /** Install npm packages. */ + async npmInstall( + packages: string[], + options?: ExecOptions + ): Promise { + return this.npm(["install", ...packages], options); + } + + /** Run npx commands. */ + async npx(args: string[], options?: ExecOptions): Promise { + return this.run(NodeMachine.DEFAULT_IMAGE, ["npx", ...args], options); + } + + /** Get Node.js version. */ + async version(options?: CodeOptions): Promise { + const image = options?.image ?? NodeMachine.DEFAULT_IMAGE; + const result = await this.run(image, ["node", "--version"], options); + return result.stdout.trim(); + } +} diff --git a/sdks/node/smolvm-embedded/src/presets/python.ts b/sdks/node/smolvm-embedded/src/presets/python.ts new file mode 100644 index 0000000..0004dd7 --- /dev/null +++ b/sdks/node/smolvm-embedded/src/presets/python.ts @@ -0,0 +1,74 @@ +/** + * PythonMachine — pre-configured machine for running Python code. + */ + +import { Machine } from "../machine.js"; +import { ExecResult } from "../execution.js"; +import type { MachineConfig, ExecOptions, CodeOptions } from "../types.js"; + +export class PythonMachine extends Machine { + static readonly DEFAULT_IMAGE = "python:3.12-alpine"; + + static async create(config: MachineConfig): Promise { + const machine = new PythonMachine(config); + await machine.start(); + // Pre-pull the Python image + await machine.pullImage(PythonMachine.DEFAULT_IMAGE); + return machine; + } + + private constructor(config: MachineConfig) { + super(config); + } + + /** Run Python code. */ + async runCode(code: string, options?: CodeOptions): Promise { + const image = options?.image ?? PythonMachine.DEFAULT_IMAGE; + return this.run(image, ["python3", "-c", code], options); + } + + /** Run a Python file (must be in a mounted directory). */ + async runFile(path: string, options?: CodeOptions): Promise { + const image = options?.image ?? PythonMachine.DEFAULT_IMAGE; + return this.run(image, ["python3", path], options); + } + + /** Run setup code, then main code. */ + async runWithSetup( + setupCode: string, + mainCode: string, + options?: CodeOptions + ): Promise { + const combined = `${setupCode}\n${mainCode}`; + return this.runCode(combined, options); + } + + /** Install pip packages. */ + async pip( + packages: string[], + options?: ExecOptions + ): Promise { + return this.run( + PythonMachine.DEFAULT_IMAGE, + ["pip", "install", ...packages], + options + ); + } + + /** List installed packages. */ + async listPackages(options?: ExecOptions): Promise { + const result = await this.run( + PythonMachine.DEFAULT_IMAGE, + ["pip", "list", "--format=freeze"], + options + ); + return result.stdout.trim().split("\n").filter(Boolean); + } + + /** Get Python version. */ + async version(options?: CodeOptions): Promise { + const image = options?.image ?? PythonMachine.DEFAULT_IMAGE; + const result = await this.run(image, ["python3", "--version"], options); + return result.stdout.trim(); + } +} diff --git a/sdks/node/smolvm-embedded/src/runtime-env.ts b/sdks/node/smolvm-embedded/src/runtime-env.ts new file mode 100644 index 0000000..44d790f --- /dev/null +++ b/sdks/node/smolvm-embedded/src/runtime-env.ts @@ -0,0 +1,42 @@ +/** + * Runtime environment setup for bundled native libraries. + * + * The embedded SDK runs inside `node`, so Rust-side `current_exe()` points + * at the Node executable rather than this package. Expose an explicit library + * directory so smolvm can find bundled libkrun/libkrunfw assets reliably. + */ + +import { existsSync } from "node:fs"; +import { delimiter, resolve } from "node:path"; + +import { getPlatformPackageRoot } from "./platform-package.js"; + +function prependEnvPath(name: string, entry: string): void { + const current = process.env[name]; + if (!current) { + process.env[name] = entry; + return; + } + + const parts = current.split(delimiter); + if (!parts.includes(entry)) { + process.env[name] = `${entry}${delimiter}${current}`; + } +} + +export function prepareNativeRuntime(): void { + const bundledLibDir = resolve(getPlatformPackageRoot(), "lib"); + if (!existsSync(bundledLibDir)) { + return; + } + + if (!process.env.SMOLVM_LIB_DIR) { + process.env.SMOLVM_LIB_DIR = bundledLibDir; + } + + if (process.platform === "darwin") { + prependEnvPath("DYLD_LIBRARY_PATH", bundledLibDir); + } else if (process.platform === "linux") { + prependEnvPath("LD_LIBRARY_PATH", bundledLibDir); + } +} diff --git a/sdks/node/smolvm-embedded/src/types.ts b/sdks/node/smolvm-embedded/src/types.ts new file mode 100644 index 0000000..b0a54d5 --- /dev/null +++ b/sdks/node/smolvm-embedded/src/types.ts @@ -0,0 +1,130 @@ +/** + * Type definitions for smolvm-embedded. + * + * These types mirror the existing smolvm-node SDK API shape + * but operate in-process via NAPI-RS (no daemon required). + */ + +// ============================================================================ +// Configuration Types +// ============================================================================ + +/** + * Configuration for creating a machine. + */ +export interface MachineConfig { + /** Unique name for the machine. Used as the VM identifier. */ + name: string; + /** Host directories to mount into the VM. */ + mounts?: MountSpec[]; + /** Port mappings from host to guest. */ + ports?: PortSpec[]; + /** VM resource configuration. */ + resources?: ResourceSpec; + /** If true, create() does NOT auto-start — call start() explicitly. Storage persists across stop/start. */ + persistent?: boolean; +} + +/** + * A host directory mount specification. + */ +export interface MountSpec { + /** Absolute path on the host. */ + source: string; + /** Absolute path inside the guest. */ + target: string; + /** Mount as read-only (default: true). */ + readOnly?: boolean; +} + +/** + * A port mapping from host to guest. + */ +export interface PortSpec { + /** Port on the host. */ + host: number; + /** Port inside the guest. */ + guest: number; +} + +/** + * VM resource allocation. + */ +export interface ResourceSpec { + /** Number of vCPUs (default: 4). */ + cpus?: number; + /** Memory in MiB (default: 8192). */ + memoryMb?: number; + /** Enable outbound network access (default: false). */ + network?: boolean; + /** Storage disk size in GiB (default: 20). */ + storageGib?: number; + /** Overlay disk size in GiB (default: 10). */ + overlayGib?: number; +} + +// ============================================================================ +// Execution Types +// ============================================================================ + +/** + * Options for command execution. + */ +export interface ExecOptions { + /** Environment variables as key-value pairs. */ + env?: Record; + /** Working directory for the command. */ + workdir?: string; + /** Timeout in seconds. */ + timeout?: number; +} + +/** + * Options for writing a file into the VM. + */ +export interface FileWriteOptions { + /** Optional octal file mode, for example `0o644`. */ + mode?: number; +} + +/** + * Options for code execution (extends ExecOptions). + */ +export interface CodeOptions extends ExecOptions { + /** Override default image. */ + image?: string; +} + +// ============================================================================ +// Response Types +// ============================================================================ + +/** + * Information about an OCI image. + */ +export interface ImageInfo { + /** Image reference (e.g., "alpine:latest"). */ + reference: string; + /** Image digest (sha256:...). */ + digest: string; + /** Image size in bytes. */ + size: number; + /** Platform architecture (e.g., "arm64"). */ + architecture: string; + /** Platform OS (e.g., "linux"). */ + os: string; +} + +/** + * Event emitted by a streaming exec session. + */ +export interface ExecStreamEvent { + /** Event kind. */ + kind: "stdout" | "stderr" | "exit" | "error"; + /** Text payload for stdout/stderr events. */ + data?: string; + /** Exit code for exit events. */ + exitCode?: number; + /** Error message for error events. */ + message?: string; +} diff --git a/sdks/node/smolvm-embedded/tsconfig.json b/sdks/node/smolvm-embedded/tsconfig.json new file mode 100644 index 0000000..ee07781 --- /dev/null +++ b/sdks/node/smolvm-embedded/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "declaration": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": "src", + "sourceMap": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"], + "exclude": ["node_modules", "dist", "__tests__", "examples"] +} diff --git a/sdks/node/smolvm-embedded/tsup.config.ts b/sdks/node/smolvm-embedded/tsup.config.ts new file mode 100644 index 0000000..1471aad --- /dev/null +++ b/sdks/node/smolvm-embedded/tsup.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["cjs", "esm"], + dts: true, +}); diff --git a/sdks/node/smolvm-embedded/vitest.config.ts b/sdks/node/smolvm-embedded/vitest.config.ts new file mode 100644 index 0000000..15fdb34 --- /dev/null +++ b/sdks/node/smolvm-embedded/vitest.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + // Daemonless VM startup is reliable under process workers but not thread workers. + pool: "forks", + fileParallelism: false, + testTimeout: 120_000, // 2 minutes — VM boot + image pull can be slow + hookTimeout: 60_000, + include: [ + "__tests__/**/*.test.ts", + "integration-tests/**/*.test.ts", + ], + }, +}); diff --git a/sdks/scripts/stage-embedded-libs.sh b/sdks/scripts/stage-embedded-libs.sh new file mode 100755 index 0000000..4fb6643 --- /dev/null +++ b/sdks/scripts/stage-embedded-libs.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +copy_matching_libraries() { + local src_dir="$1" + local pattern="$2" + local dst_dir="$3" + + if compgen -G "$src_dir/$pattern" > /dev/null; then + cp -a "$src_dir"/$pattern "$dst_dir"/ + fi +} + +detect_lib_bundle() { + if [[ -n "${SMOLVM_EMBEDDED_LIB_BUNDLE:-}" ]]; then + echo "$SMOLVM_EMBEDDED_LIB_BUNDLE" + return 0 + fi + + if [[ "$(uname -s)" == "Linux" ]]; then + echo "$REPO_ROOT/lib/linux-$(uname -m)" + else + echo "$REPO_ROOT/lib" + fi +} + +usage() { + cat <<'EOF' +Usage: + ./sdks/scripts/stage-embedded-libs.sh + +Environment: + SMOLVM_EMBEDDED_LIB_BUNDLE Override the bundled library source directory. +EOF +} + +if [[ $# -ne 1 ]]; then + usage >&2 + exit 1 +fi + +TARGET_LIB_DIR="$1" +SOURCE_LIB_DIR="$(detect_lib_bundle)" + +if [[ ! -d "$SOURCE_LIB_DIR" ]]; then + echo "Error: bundled library directory not found: $SOURCE_LIB_DIR" >&2 + exit 1 +fi + +rm -rf "$TARGET_LIB_DIR" +mkdir -p "$TARGET_LIB_DIR" + +if [[ "$(uname -s)" == "Linux" ]]; then + copy_matching_libraries "$SOURCE_LIB_DIR" "libkrun.so*" "$TARGET_LIB_DIR" + copy_matching_libraries "$SOURCE_LIB_DIR" "libkrunfw.so*" "$TARGET_LIB_DIR" +else + copy_matching_libraries "$SOURCE_LIB_DIR" "libkrun*.dylib" "$TARGET_LIB_DIR" + copy_matching_libraries "$SOURCE_LIB_DIR" "libkrunfw*.dylib" "$TARGET_LIB_DIR" +fi + +echo "Staged embedded SDK libraries from $SOURCE_LIB_DIR into $TARGET_LIB_DIR" diff --git a/smolvm.entitlements b/smolvm.entitlements new file mode 100644 index 0000000..cd7c9da --- /dev/null +++ b/smolvm.entitlements @@ -0,0 +1,12 @@ + + + + + com.apple.security.hypervisor + + com.apple.security.cs.disable-library-validation + + com.apple.security.cs.allow-jit + + + diff --git a/src/agent/boot_config.rs b/src/agent/boot_config.rs new file mode 100644 index 0000000..7e86e30 --- /dev/null +++ b/src/agent/boot_config.rs @@ -0,0 +1,75 @@ +//! Boot configuration for subprocess-based VM launch. +//! +//! On macOS, `fork()` in a multi-threaded process (e.g., the tokio-based API +//! server) creates unstable children because Apple frameworks like +//! Hypervisor.framework detect the forked state and abort. To avoid this, +//! the server spawns a fresh single-threaded `smolvm _boot-vm` subprocess +//! that safely runs `krun_start_enter`. +//! +//! This module defines the serializable config passed to that subprocess. + +use crate::data::disk::DiskFormat; +use crate::data::network::PortMapping; +use crate::data::resources::VmResources; +use crate::data::storage::HostMount; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +/// Configuration for the `_boot-vm` subprocess. +/// +/// Written to a temp file by the parent and read by the child. +#[derive(Debug, Serialize, Deserialize)] +pub struct BootConfig { + /// Path to the agent rootfs directory. + pub rootfs_path: PathBuf, + /// Path to the storage disk file. + pub storage_disk_path: PathBuf, + /// Path to the overlay disk file. + pub overlay_disk_path: PathBuf, + /// Path to the vsock Unix socket. + pub vsock_socket: PathBuf, + /// Optional path to console log file. + pub console_log: Option, + /// Path to write startup errors. + pub startup_error_log: PathBuf, + /// Storage disk size in GiB. + pub storage_size_gb: u64, + /// Overlay disk size in GiB. + pub overlay_size_gb: u64, + /// Host directory mounts. + pub mounts: Vec, + /// Port mappings. + pub ports: Vec, + /// VM resources (CPU, memory, network, disk sizes). + pub resources: VmResources, + /// Path to the host-side Unix socket for SSH agent forwarding. + /// When set, a vsock port is registered so the guest can reach the host's SSH agent. + #[serde(default)] + pub ssh_agent_socket: Option, + /// Enable CUDA-over-vsock: smolvm starts a host CUDA server and remotes the + /// guest's CUDA Driver-API calls to the host GPU. + #[serde(default)] + pub cuda: bool, + /// Hostnames for DNS filtering. When set, the host starts a DNS filter + /// listener and the guest agent proxies DNS queries through it. + #[serde(default)] + pub dns_filter_hosts: Option>, + /// Pre-extracted OCI layers directory for .smolmachine-sourced machines. + /// When `pack_idmap_source` is set, this is an empty per-VM mountpoint the + /// boot subprocess idmap-binds the shared pack onto; otherwise it is the + /// directory holding the extracted pack directly. + #[serde(default)] + pub packed_layers_dir: Option, + /// Root-owned shared pack directory (`_shared/`) to present at + /// `packed_layers_dir` via a per-VM idmapped bind mount that maps on-disk + /// uid 0 -> the VM's dropped uid. Set only when per-VM uid isolation is + /// active (Linux fleet); the mount lives in the boot subprocess's private + /// mount namespace and is torn down automatically on exit. When `None`, + /// `packed_layers_dir` is consumed as-is (no idmap mount). + #[serde(default)] + pub pack_idmap_source: Option, + /// Additional disk images to attach (path, read_only, format). The format + /// lets the `pack --from-vm` exporter attach a source qcow2 disk read-only. + #[serde(default)] + pub extra_disks: Vec<(PathBuf, bool, DiskFormat)>, +} diff --git a/src/agent/client.rs b/src/agent/client.rs new file mode 100644 index 0000000..0471e60 --- /dev/null +++ b/src/agent/client.rs @@ -0,0 +1,2761 @@ +//! vsock client for communicating with the smolvm-agent. +//! +//! This module provides a client for sending requests to the agent +//! and receiving responses. + +use crate::error::{Error, Result}; +use crate::platform::uds::UdsStream; +use crate::registry::{extract_registry, rewrite_image_registry, RegistryAuth}; +use crate::settings::SmolSettings; +use smolvm_protocol::normalize_image_ref; +use smolvm_protocol::{ + encode_message, AgentRequest, AgentResponse, Envelope, FsNotifyEvent, ImageInfo, OverlayInfo, + StorageStatus, FILE_TRANSFER_MAX_TOTAL, FILE_WRITE_CHUNK_SIZE, FILE_WRITE_SINGLE_SHOT_MAX, + MAX_FRAME_SIZE, PROTOCOL_VERSION, +}; +use std::io::{Read, Write}; +use std::path::Path; +use std::time::Duration; + +/// Events from a streaming exec session. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ExecEvent { + /// Standard output data. + Stdout(Vec), + /// Standard error data. + Stderr(Vec), + /// Command exited with this code. + Exit(i32), + /// An error occurred. + Error(String), +} + +/// One input event fed into a channel-driven interactive session +/// ([`AgentClient::interactive_session_io`]). This decouples the interactive +/// poll loop from the process's real stdin so the session can be driven by a +/// remote transport (e.g. a WebSocket terminal) instead of a local TTY. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InteractiveInput { + /// Bytes to forward to the command's stdin. + Stdin(Vec), + /// Terminal resize (PTY window change). + Resize { + /// New terminal width in columns. + cols: u16, + /// New terminal height in rows. + rows: u16, + }, + /// End of input — sends an empty stdin frame (EOF) to the command. + Eof, +} + +/// One output chunk produced by a channel-driven interactive session. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InteractiveOutput { + /// A chunk of bytes from the command's stdout. + Stdout(Vec), + /// A chunk of bytes from the command's stderr. + Stderr(Vec), +} + +// ============================================================================ +// Socket Timeout Constants +// ============================================================================ +// +// These timeouts control how long the client waits for various operations. +// They balance between allowing slow operations to complete and failing fast +// when the agent is unresponsive. + +/// Default socket read timeout (30 seconds). +/// Used for most request/response operations. Long enough for the agent to +/// process requests, short enough to detect hung connections. +const DEFAULT_READ_TIMEOUT_SECS: u64 = 30; + +/// Default socket write timeout (10 seconds). +/// Writes should complete quickly - if they don't, the connection is likely broken. +const DEFAULT_WRITE_TIMEOUT_SECS: u64 = 10; + +/// Read timeout for image pull operations (10 minutes). +/// Image pulls can take a long time for large images over slow connections. +const IMAGE_PULL_TIMEOUT_SECS: u64 = 600; + +// (Removed INTERACTIVE_TIMEOUT_SECS — no-user-timeout execs now disable +// the socket read timeout entirely, matching interactive_session behavior.) + +/// Buffer time added to user-specified timeouts (5 seconds). +/// When users specify a command timeout, we add this buffer to the socket +/// timeout to allow for protocol overhead and response transmission. +const TIMEOUT_BUFFER_SECS: u64 = 5; + +/// Timeout for shutdown acknowledgment (5 seconds). +/// sync() + ack transmission is typically <100ms, but heavy writes or +/// large journals may take longer. If no ack within 5s, the VM has +/// likely already torn down — safe to proceed with SIGTERM. +const SHUTDOWN_ACK_TIMEOUT_SECS: u64 = 5; + +// ============================================================================ +// I/O Constants +// ============================================================================ + +/// Buffer size for reading stdin during interactive sessions. +const STDIN_BUF_SIZE: usize = 4096; + +/// Poll timeout in milliseconds for interactive I/O loops. +/// Short enough for responsive SIGWINCH handling, long enough to avoid busy-waiting. +const POLL_TIMEOUT_MS: i32 = 100; + +/// Exit code reported when a channel-driven interactive session ends because the +/// remote peer (e.g. a WebSocket terminal) disconnected rather than the command +/// exiting. 128 + SIGINT(2), matching the shell convention for an interrupted job. +const DISCONNECT_EXIT_CODE: i32 = 130; + +/// RAII guard that resets the socket read timeout on drop. +/// +/// Ensures the timeout is always restored, even if the operation +/// returns early due to an error. Uses a cloned UnixStream handle +/// (shares the underlying fd) to avoid borrow conflicts. +pub struct ReadTimeoutGuard { + stream: UdsStream, +} + +impl ReadTimeoutGuard { + /// Create a guard from a reference to the stream. + /// Clones the underlying fd so the guard doesn't borrow the original. + fn new(stream: &UdsStream) -> Option { + stream.try_clone().ok().map(|s| Self { stream: s }) + } +} + +impl Drop for ReadTimeoutGuard { + fn drop(&mut self) { + if let Err(e) = self + .stream + .set_read_timeout(Some(Duration::from_secs(DEFAULT_READ_TIMEOUT_SECS))) + { + tracing::warn!(error = %e, "failed to reset socket read timeout to default"); + } + } +} + +/// Configuration for running a command interactively. +#[derive(Debug, Clone)] +pub struct RunConfig { + /// OCI image to run. + pub image: String, + /// Command and arguments to execute. + pub command: Vec, + /// Environment variables as (key, value) pairs. + pub env: Vec<(String, String)>, + /// Working directory inside the container. + pub workdir: Option, + /// User to execute as inside the container. + pub user: Option, + /// Volume mounts as (tag, guest_path, read_only) tuples. + pub mounts: Vec<(String, String, bool)>, + /// Timeout for command execution. + pub timeout: Option, + /// Whether to allocate a TTY. + pub tty: bool, + /// Persistent overlay ID. If set, the overlay persists across exec sessions + /// so filesystem changes (e.g. package installs) survive. + pub persistent_overlay_id: Option, + /// Data to pipe to the command's stdin (non-interactive runs only). The + /// pipe is closed after writing so the command sees EOF. + pub stdin: Option, + /// Run as an unprivileged container (restricted caps, ro cgroup, no extra + /// tmpfs). Default false = "VM-grade" (the microVM is the boundary). + pub unprivileged: bool, +} + +impl RunConfig { + /// Create a new run configuration with the given image and command. + /// + /// The image reference is canonicalized immediately so all downstream + /// code (cache keys, logs, protocol messages) sees the same form + /// regardless of how the caller spelled it. + pub fn new(image: impl Into, command: Vec) -> Self { + Self { + image: normalize_image_ref(&image.into()), + command, + env: Vec::new(), + workdir: None, + user: None, + mounts: Vec::new(), + timeout: None, + tty: false, + persistent_overlay_id: None, + stdin: None, + unprivileged: false, + } + } + + /// Set environment variables. + pub fn with_env(mut self, env: Vec<(String, String)>) -> Self { + self.env = env; + self + } + + /// Set working directory. + pub fn with_workdir(mut self, workdir: Option) -> Self { + self.workdir = workdir; + self + } + + /// Set container user. + pub fn with_user(mut self, user: Option) -> Self { + self.user = user; + self + } + + /// Set volume mounts. + pub fn with_mounts(mut self, mounts: Vec<(String, String, bool)>) -> Self { + self.mounts = mounts; + self + } + + /// Set timeout. + pub fn with_timeout(mut self, timeout: Option) -> Self { + self.timeout = timeout; + self + } + + /// Enable TTY mode. + pub fn with_tty(mut self, tty: bool) -> Self { + self.tty = tty; + self + } + + /// Set stdin data piped to the command (non-interactive runs). + pub fn with_stdin(mut self, stdin: Option) -> Self { + self.stdin = stdin; + self + } + + /// Set persistent overlay ID for cross-session filesystem persistence. + pub fn with_persistent_overlay(mut self, id: Option) -> Self { + self.persistent_overlay_id = id; + self + } + + /// Run as an unprivileged container (defense-in-depth for untrusted code). + pub fn with_unprivileged(mut self, unprivileged: bool) -> Self { + self.unprivileged = unprivileged; + self + } +} + +/// Options for pulling an OCI image. +/// +/// Use `PullOptions::new()` to create with defaults, then chain methods +/// to customize behavior. +/// +/// # Example +/// +/// ```ignore +/// let options = PullOptions::new() +/// .oci_platform("linux/arm64") +/// .use_registry_config(true) +/// .progress(|cur, total, layer| println!("{}/{}: {}", cur, total, layer)); +/// +/// client.pull("alpine:latest", options)?; +/// ``` +#[derive(Default)] +pub struct PullOptions +where + F: FnMut(usize, usize, &str), +{ + /// OCI platform to pull (e.g., "linux/arm64"). + pub oci_platform: Option, + /// Explicit authentication credentials. + pub auth: Option, + /// Whether to load credentials from registry config file. + pub use_registry_config: bool, + /// Proxy URL applied to the in-VM registry client (HTTP_PROXY/HTTPS_PROXY). + pub proxy: Option, + /// Comma-separated NO_PROXY list of hosts/CIDRs that bypass the proxy. + pub no_proxy: Option, + /// Progress callback: (current, total, layer_id). + pub progress: Option, +} + +impl PullOptions { + /// Create new pull options with defaults. + pub fn new() -> Self { + Self { + oci_platform: None, + auth: None, + use_registry_config: false, + proxy: None, + no_proxy: None, + progress: None, + } + } +} + +impl PullOptions { + /// Set the target OCI platform (e.g., "linux/arm64"). + pub fn oci_platform(mut self, oci_platform: impl Into) -> Self { + self.oci_platform = Some(oci_platform.into()); + self + } + + /// Set explicit authentication credentials. + pub fn auth(mut self, auth: RegistryAuth) -> Self { + self.auth = Some(auth); + self + } + + /// Enable loading credentials from registry config file. + /// + /// When enabled, loads `~/.config/smolvm/registries.toml` and + /// automatically provides credentials for matching registries. + /// Also applies registry mirrors if configured. + pub fn use_registry_config(mut self, enabled: bool) -> Self { + self.use_registry_config = enabled; + self + } + + /// Set the proxy URL applied to the in-VM registry client. + pub fn proxy(mut self, proxy: impl Into) -> Self { + self.proxy = Some(proxy.into()); + self + } + + /// Set the NO_PROXY list for the in-VM registry client. + pub fn no_proxy(mut self, no_proxy: impl Into) -> Self { + self.no_proxy = Some(no_proxy.into()); + self + } + + /// Set a progress callback. + /// + /// The callback receives (current_percent, total=100, layer_id) for each layer. + pub fn progress(self, callback: G) -> PullOptions { + PullOptions { + oci_platform: self.oci_platform, + auth: self.auth, + use_registry_config: self.use_registry_config, + proxy: self.proxy, + no_proxy: self.no_proxy, + progress: Some(callback), + } + } +} + +/// Raw descriptor of the process's stdin, in the portable form the terminal +/// poll loop expects. Unix: the real fd; Windows: the `STD_INPUT_HANDLE` +/// console handle cast into the portable `Fd`. +fn stdin_raw_fd() -> crate::agent::terminal::Fd { + #[cfg(unix)] + { + use std::os::unix::io::AsRawFd; + std::io::stdin().as_raw_fd() + } + #[cfg(not(unix))] + { + // SAFETY: GetStdHandle has no preconditions; it returns the process's + // standard-input HANDLE (or INVALID_HANDLE_VALUE), which the Windows + // poll loop interprets. + let handle = unsafe { + windows_sys::Win32::System::Console::GetStdHandle( + windows_sys::Win32::System::Console::STD_INPUT_HANDLE, + ) + }; + handle as crate::agent::terminal::Fd + } +} + +/// Check if a shutdown receive error is a benign race condition. +/// +/// During shutdown the VM may tear down before the ack response is flushed, +/// causing EAGAIN, connection reset, or similar errors. These are expected +/// and don't indicate a problem — sync() has likely already completed. +fn is_benign_shutdown_error(error_str: &str) -> bool { + error_str.contains("os error 35") // EAGAIN on macOS + || error_str.contains("os error 11") // EAGAIN on Linux + || error_str.contains("temporarily unavailable") + || error_str.contains("Connection reset") + || error_str.contains("connection reset") +} + +/// Client for communicating with the smolvm-agent. +pub struct AgentClient { + stream: UdsStream, + /// Trace ID for correlating this client session's requests with host API calls. + trace_id: Option, +} + +// ============================================================================ +// Response match helpers +// ============================================================================ + +/// Extract typed data from an `Ok` response. +fn expect_data(resp: AgentResponse, op: &str) -> Result { + match resp { + AgentResponse::Ok { + data: Some(data), .. + } => { + serde_json::from_value(data).map_err(|e| Error::agent("parse response", e.to_string())) + } + AgentResponse::Error { message, .. } => Err(Error::agent(op, message)), + _ => Err(Error::agent(op, "unexpected response type")), + } +} + +/// Expect an `Ok` response, ignoring any data. +fn expect_ok(resp: AgentResponse, op: &str) -> Result<()> { + match resp { + AgentResponse::Ok { .. } => Ok(()), + AgentResponse::Error { message, .. } => Err(Error::agent(op, message)), + _ => Err(Error::agent(op, "unexpected response type")), + } +} + +/// Extract exit code, stdout, stderr from a `Completed` response. +fn expect_completed(resp: AgentResponse, op: &str) -> Result<(i32, Vec, Vec)> { + match resp { + AgentResponse::Completed { + exit_code, + stdout, + stderr, + } => Ok((exit_code, stdout, stderr)), + AgentResponse::Error { message, .. } => Err(Error::agent(op, message)), + _ => Err(Error::agent(op, "unexpected response type")), + } +} + +#[cfg(test)] +impl AgentClient { + /// Build an `AgentClient` from a pre-connected `UnixStream`. + /// + /// Test-only: production code must go through [`AgentClient::connect`] + /// so socket timeouts are configured correctly. Used by the regression + /// tests that drive the client against a `UdsStream::pair()`. + pub(crate) fn from_stream(stream: UdsStream) -> Self { + Self { + stream, + trace_id: None, + } + } +} + +impl AgentClient { + /// Set socket read timeout, returning an error if it fails. + /// + /// This is a helper to ensure timeout failures are always handled properly, + /// preventing indefinite hangs on read operations. + fn set_read_timeout(&self, timeout: Duration) -> Result<()> { + self.stream.set_read_timeout(Some(timeout)).map_err(|e| { + Error::agent( + "set read timeout", + format!("failed to set socket read timeout to {:?}: {}", timeout, e), + ) + }) + } + + /// Connect to the agent via Unix socket. + /// + /// # Arguments + /// + /// * `socket_path` - Path to the vsock Unix socket + /// + /// # Errors + /// + /// Returns an error if: + /// - Connection to the socket fails + /// - Socket timeouts cannot be configured (prevents indefinite hangs) + pub fn connect(socket_path: impl AsRef) -> Result { + Self::connect_once(socket_path.as_ref()) + } + + /// Connect to the agent with retry logic for transient failures. + /// + /// This is useful when the agent might be temporarily unavailable + /// (e.g., during high load or brief network issues). + pub fn connect_with_retry(socket_path: impl AsRef) -> Result { + use crate::util::{retry_with_backoff, RetryConfig}; + + let path = socket_path.as_ref(); + + retry_with_backoff( + RetryConfig::for_connection(), + "agent connect", + || Self::connect_once(path), + |e| { + // Check if this is a transient error worth retrying + let error_msg = e.to_string(); + // Connection refused/reset are transient during VM startup. + // "No such file or directory" occurs when the vsock socket + // file hasn't been created yet by libkrun's muxer thread — + // transient under concurrent boot contention. + error_msg.contains("Connection refused") + || error_msg.contains("connection refused") + || error_msg.contains("Connection reset") + || error_msg.contains("connection reset") + || error_msg.contains("Broken pipe") + || error_msg.contains("Resource temporarily unavailable") + || error_msg.contains("No such file or directory") + }, + ) + } + + /// Connect with a short timeout, for use during startup ping probes. + /// Uses 100ms read timeout instead of 30s to fail fast during boot. + /// The agent completes init in ~130ms of guest uptime, so 100ms is enough + /// to detect a ready agent without wasting time on a full 1s timeout. + pub fn connect_with_short_timeout(socket_path: impl AsRef) -> Result { + Self::connect_with_timeouts_ms(socket_path.as_ref(), 100, 100) + } + + /// Connect with a moderate timeout, for state-probe "is this agent alive" + /// checks from `machine ls` / `machine status`. 3 seconds is long enough + /// to avoid false "unreachable" readings when the agent is momentarily + /// busy (e.g., processing a Run request's overlayfs setup), but short + /// enough to not make `ls` feel sluggish when the agent is truly dead. + pub fn connect_for_state_probe(socket_path: impl AsRef) -> Result { + Self::connect_with_timeouts_ms(socket_path.as_ref(), 3000, 3000) + } + + /// Connect with a very short timeout for boot-time probe cycles. + /// Uses 5ms timeout to minimize blocking between ready-marker checks. + /// Only used in the fallback path (old agents without ready markers). + pub fn connect_with_boot_probe_timeout(socket_path: impl AsRef) -> Result { + Self::connect_with_timeouts_ms(socket_path.as_ref(), 5, 5) + } + + /// Internal connect implementation (single attempt). + fn connect_once(socket_path: &Path) -> Result { + Self::connect_with_timeouts( + socket_path, + DEFAULT_READ_TIMEOUT_SECS, + DEFAULT_WRITE_TIMEOUT_SECS, + ) + } + + /// Connect to the agent socket and configure read/write timeouts (in seconds). + fn connect_with_timeouts(socket_path: &Path, read_secs: u64, write_secs: u64) -> Result { + Self::connect_with_timeouts_ms(socket_path, read_secs * 1000, write_secs * 1000) + } + + /// Connect to the agent socket and configure read/write timeouts (in milliseconds). + fn connect_with_timeouts_ms(socket_path: &Path, read_ms: u64, write_ms: u64) -> Result { + let stream = UdsStream::connect(socket_path) + .map_err(|e| Error::agent("connect to agent", e.to_string()))?; + + stream + .set_read_timeout(Some(Duration::from_millis(read_ms))) + .map_err(|e| Error::agent("set read timeout", e.to_string()))?; + stream + .set_write_timeout(Some(Duration::from_millis(write_ms))) + .map_err(|e| Error::agent("set write timeout", e.to_string()))?; + + Ok(Self { + stream, + trace_id: None, + }) + } + + /// Set a trace ID for correlating this client session's requests with host API calls. + /// All subsequent requests will include this trace_id in the Envelope. + pub fn set_trace_id(&mut self, trace_id: String) { + self.trace_id = Some(trace_id); + } + + /// Encode a request wrapped in an Envelope with the current trace_id. + fn encode_traced(&self, req: &AgentRequest) -> Result> { + let envelope = Envelope::with_trace_id(req, self.trace_id.clone()); + encode_message(&envelope).map_err(|e| Error::agent("encode message", e.to_string())) + } + + /// Send a request and receive a response. + fn request(&mut self, req: &AgentRequest) -> Result { + // Encode and send request + let data = self.encode_traced(req)?; + self.stream + .write_all(&data) + .map_err(|e| Error::agent("send message", e.to_string()))?; + + // Read response + self.receive() + } + + /// Ping the helper daemon and validate the protocol version. + /// + /// Returns the agent's protocol version. Logs a warning if the version + /// doesn't match the host's expected version. + pub fn ping(&mut self) -> Result { + let resp = self.request(&AgentRequest::Ping)?; + + match resp { + AgentResponse::Pong { version } => { + if version != PROTOCOL_VERSION { + tracing::warn!( + host_version = PROTOCOL_VERSION, + agent_version = version, + "protocol version mismatch — agent may be outdated or newer than host" + ); + } + Ok(version) + } + AgentResponse::Error { message, .. } => Err(Error::agent("ping", message)), + _ => Err(Error::agent("ping", "unexpected response type")), + } + } + + /// Replay host-originated filesystem changes into the guest as fsnotify + /// events so inotify-based watchers on `-v` mounts fire. Used by the host + /// [`FsNotifyWatcher`](super::FsNotifyWatcher); a stale connection surfaces as + /// an error the watcher treats as "VM gone, stop". + pub fn fsnotify(&mut self, events: Vec) -> Result<()> { + if events.is_empty() { + return Ok(()); + } + match self.request(&AgentRequest::FsNotify { events })? { + AgentResponse::Ok { .. } => Ok(()), + AgentResponse::Error { message, .. } => Err(Error::agent("fsnotify", message)), + _ => Err(Error::agent("fsnotify", "unexpected response type")), + } + } + + /// Pull an OCI image with the given options. + /// + /// This is the primary pull method. Use `PullOptions` to configure + /// authentication, platform, and progress tracking. + /// + /// # Example + /// + /// ```ignore + /// // Simple pull + /// client.pull("alpine:latest", PullOptions::new())?; + /// + /// // Pull with registry config (loads credentials from config file) + /// client.pull("ghcr.io/owner/repo", PullOptions::new().use_registry_config(true))?; + /// + /// // Pull with explicit auth and progress + /// client.pull("private.registry/image", PullOptions::new() + /// .auth(RegistryAuth { username: "user".into(), password: "pass".into() }) + /// .progress(|cur, total, layer| eprintln!("{}%", cur)))?; + /// ``` + /// + /// # Note + /// + /// This operation uses a 10-minute timeout to accommodate large images. + pub fn pull( + &mut self, + image: &str, + options: PullOptions, + ) -> Result { + // Resolve effective image and auth based on options + let (effective_image, effective_auth) = if options.use_registry_config { + let registry_config = SmolSettings::load().unwrap_or_default().images; + let registry = extract_registry(image); + + // Get credentials from config if not explicitly provided + let auth = options.auth.or_else(|| { + registry_config.get_credentials(®istry).inspect(|creds| { + tracing::debug!( + registry = %registry, + username = %creds.username, + "using configured registry credentials" + ); + }) + }); + + // Apply mirror if configured + let img = if let Some(mirror) = registry_config.get_mirror(®istry) { + let mirrored = rewrite_image_registry(image, mirror); + tracing::debug!( + original = %image, + mirrored = %mirrored, + mirror = %mirror, + "using registry mirror" + ); + mirrored + } else { + image.to_string() + }; + + (img, auth) + } else { + (image.to_string(), options.auth) + }; + + self.pull_image_internal( + &effective_image, + options.oci_platform.as_deref(), + effective_auth.as_ref(), + options.proxy.as_deref(), + options.no_proxy.as_deref(), + options.progress, + ) + } + + /// Internal implementation of image pull. + fn pull_image_internal( + &mut self, + image: &str, + oci_platform: Option<&str>, + auth: Option<&RegistryAuth>, + proxy: Option<&str>, + no_proxy: Option<&str>, + mut progress: Option, + ) -> Result { + let image = normalize_image_ref(image); + let image = image.as_str(); + + // Use a long timeout for pull - large images can take minutes to download/extract. + // The guard resets the timeout on drop (including error paths). + self.set_read_timeout(Duration::from_secs(IMAGE_PULL_TIMEOUT_SECS))?; + let _timeout_guard = ReadTimeoutGuard::new(&self.stream); + + // Send the pull request + let data = self.encode_traced(&AgentRequest::Pull { + image: image.to_string(), + oci_platform: oci_platform.map(String::from), + auth: auth.cloned(), + proxy: proxy.map(String::from), + no_proxy: no_proxy.map(String::from), + })?; + + self.stream + .write_all(&data) + .map_err(|e| Error::agent("send request", e.to_string()))?; + + // Read responses - loop until we get Ok or Error (skip Progress) + loop { + match self.receive()? { + AgentResponse::Progress { + percent, + layer, + message: _, + } => { + if let Some(ref mut cb) = progress { + let current = percent.unwrap_or(0) as usize; + let layer_id = layer.as_deref().unwrap_or(""); + cb(current, 100, layer_id); + } + } + AgentResponse::Ok { data: Some(data) } => { + return serde_json::from_value(data) + .map_err(|e| Error::agent("parse response", e.to_string())); + } + AgentResponse::Error { message, .. } => { + return Err(Error::agent("pull image", message)); + } + _ => { + return Err(Error::agent("pull image", "unexpected response type")); + } + } + } + } + + // ========================================================================= + // Convenience methods for common pull patterns + // ========================================================================= + + /// Pull an OCI image with default options. + /// + /// Shorthand for `pull(image, PullOptions::new())`. + pub fn pull_simple(&mut self, image: &str) -> Result { + self.pull(image, PullOptions::new()) + } + + /// Pull an OCI image with automatic registry credential lookup. + /// + /// Loads credentials from `~/.config/smolvm/registries.toml` and applies + /// registry mirrors if configured. + /// + /// Shorthand for `pull(image, PullOptions::new().use_registry_config(true))`. + pub fn pull_with_registry_config(&mut self, image: &str) -> Result { + self.pull(image, PullOptions::new().use_registry_config(true)) + } + + /// Pull an OCI image with registry config and progress callback. + pub fn pull_with_registry_config_and_progress( + &mut self, + image: &str, + oci_platform: Option<&str>, + proxy: Option<&str>, + no_proxy: Option<&str>, + progress: F, + ) -> Result { + let mut opts = PullOptions::new() + .use_registry_config(true) + .progress(progress); + if let Some(p) = oci_platform { + opts = opts.oci_platform(p); + } + if let Some(p) = proxy { + opts = opts.proxy(p); + } + if let Some(np) = no_proxy { + opts = opts.no_proxy(np); + } + self.pull(image, opts) + } + + /// Query if an image exists locally. + pub fn query(&mut self, image: &str) -> Result> { + let resp = self.request(&AgentRequest::Query { + image: image.to_string(), + })?; + + match resp { + AgentResponse::Ok { data: Some(data) } => { + let info: ImageInfo = serde_json::from_value(data) + .map_err(|e| Error::agent("parse response", e.to_string()))?; + Ok(Some(info)) + } + AgentResponse::Error { code, .. } if code.as_deref() == Some("NOT_FOUND") => Ok(None), + AgentResponse::Error { message, .. } => Err(Error::agent("query image", message)), + _ => Err(Error::agent("query image", "unexpected response type")), + } + } + + /// List all cached images. + pub fn list_images(&mut self) -> Result> { + let resp = self.request(&AgentRequest::ListImages)?; + expect_data(resp, "list images") + } + + /// Run garbage collection. + /// + /// # Arguments + /// + /// * `dry_run` - If true, only report what would be deleted + /// * `purge_all` - If true, delete all manifests/configs first so all layers are collected + pub fn garbage_collect(&mut self, dry_run: bool, purge_all: bool) -> Result { + let resp = self.request(&AgentRequest::GarbageCollect { dry_run, purge_all })?; + + match resp { + AgentResponse::Ok { data: Some(data) } => { + let freed = data["freed_bytes"].as_u64().unwrap_or(0); + Ok(freed) + } + AgentResponse::Error { message, .. } => Err(Error::agent("garbage collect", message)), + _ => Err(Error::agent("garbage collect", "unexpected response type")), + } + } + + /// Prepare an overlay filesystem for a workload. + /// + /// # Arguments + /// + /// * `image` - Image reference + /// * `workload_id` - Unique workload identifier + pub fn prepare_overlay(&mut self, image: &str, workload_id: &str) -> Result { + let resp = self.request(&AgentRequest::PrepareOverlay { + image: image.to_string(), + workload_id: workload_id.to_string(), + })?; + expect_data(resp, "prepare overlay") + } + + /// Clean up an overlay filesystem. + pub fn cleanup_overlay(&mut self, workload_id: &str) -> Result<()> { + let resp = self.request(&AgentRequest::CleanupOverlay { + workload_id: workload_id.to_string(), + })?; + expect_ok(resp, "cleanup overlay") + } + + /// Format the storage disk. + pub fn format_storage(&mut self) -> Result<()> { + let resp = self.request(&AgentRequest::FormatStorage)?; + expect_ok(resp, "format storage") + } + + /// Get storage status. + pub fn storage_status(&mut self) -> Result { + let resp = self.request(&AgentRequest::StorageStatus)?; + expect_data(resp, "storage status") + } + + /// Test network connectivity directly from the agent (not via chroot). + /// Used to debug TSI networking. + pub fn network_test(&mut self, url: &str) -> Result { + let resp = self.request(&AgentRequest::NetworkTest { + url: url.to_string(), + })?; + + match resp { + AgentResponse::Ok { data: Some(data) } => Ok(data), + AgentResponse::Error { message, .. } => Err(Error::agent("network test", message)), + _ => Err(Error::agent("network test", "unexpected response type")), + } + } + + /// Request agent shutdown. + /// + /// Waits for the agent to acknowledge the shutdown request before returning. + /// This ensures the agent has called sync() to flush filesystem caches + /// before we send SIGTERM to terminate the VM. + /// + /// The acknowledgment is critical for data integrity - without it, the VM + /// may be killed before ext4 journal commits are flushed, causing layer + /// corruption on next boot. + pub fn shutdown(&mut self) -> Result<()> { + // Set a timeout for shutdown acknowledgment. + // The agent calls sync() then sends the ack — typically <100ms, + // but heavy writes or large journals may take longer. + // If no ack within 5s, the VM has likely already torn down. + let _ = self + .stream + .set_read_timeout(Some(Duration::from_secs(SHUTDOWN_ACK_TIMEOUT_SECS))); + + let data = self.encode_traced(&AgentRequest::Shutdown)?; + self.stream + .write_all(&data) + .map_err(|e| Error::agent("send shutdown", e.to_string()))?; + + // Wait for acknowledgment - this confirms sync() completed. + // Returns Ok only when the ack is actually received, so callers + // can distinguish "sync confirmed" from "sync unknown". + match self.receive() { + Ok(_) => { + tracing::debug!("agent acknowledged shutdown (sync complete)"); + Ok(()) + } + Err(e) => { + let error_str = e.to_string(); + if is_benign_shutdown_error(&error_str) { + tracing::debug!( + "shutdown ack not received (connection closed) - sync may have completed" + ); + } else { + tracing::warn!(error = %e, "shutdown acknowledgment failed"); + } + Err(Error::agent("shutdown ack", error_str)) + } + } + } + + // ======================================================================== + // VM-Level Exec (Direct Execution in VM) + // ======================================================================== + + /// Execute a command directly in the VM (not in a container). + /// + /// Runs the command in the agent's Alpine rootfs without container + /// isolation. Returns `(exit_code, stdout_bytes, stderr_bytes)`. Output + /// is raw bytes — binary data (image bytes, tarballs) is preserved. + /// Callers that need a string can use `String::from_utf8_lossy(&bytes)`. + pub fn vm_exec( + &mut self, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + stdin_data: Option, + ) -> Result<(i32, Vec, Vec)> { + let _timeout_guard = self.set_exec_timeout(timeout)?; + let timeout_ms = timeout.map(|t| t.as_millis() as u64); + + let resp = self.request(&AgentRequest::VmExec { + command, + env, + workdir, + timeout_ms, + interactive: false, + tty: false, + background: false, + stdin_data, + })?; + + expect_completed(resp, "vm exec") + } + + /// Execute a command in the background inside the VM. + /// + /// Spawns the process and returns immediately with the PID. + /// The process runs detached — stdout/stderr go to /dev/null. + pub fn vm_exec_background( + &mut self, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + ) -> Result { + let resp = self.request(&AgentRequest::VmExec { + command, + env, + workdir, + timeout_ms: None, + interactive: false, + tty: false, + background: true, + stdin_data: None, + })?; + + let (exit_code, stdout, _stderr) = expect_completed(resp, "vm exec background")?; + if exit_code != 0 { + return Err(Error::agent("vm exec background", "spawn failed")); + } + // PID output is always ASCII digits — lossy conversion is safe. + let pid: u32 = String::from_utf8_lossy(&stdout) + .trim() + .parse() + .map_err(|_| Error::agent("vm exec background", "invalid PID in response"))?; + Ok(pid) + } + + /// Raw descriptor of the underlying agent socket, in the portable form the + /// terminal poll loop expects. Unix: the socket fd; Windows: the underlying + /// WinSock `SOCKET` cast into the portable `Fd`. + fn stream_raw_fd(&self) -> crate::agent::terminal::Fd { + #[cfg(unix)] + { + use std::os::unix::io::AsRawFd; + self.stream.as_raw_fd() + } + #[cfg(not(unix))] + { + self.stream.raw_socket() as crate::agent::terminal::Fd + } + } + + /// Run an interactive I/O session. + /// + /// Sends `request`, waits for `Started`, then runs the poll loop + /// streaming stdout/stderr and forwarding stdin until `Exited`. + fn interactive_session(&mut self, request: AgentRequest, tty: bool, op: &str) -> Result { + use crate::agent::terminal::{ + check_sigwinch, flush_retry, get_terminal_size, install_sigwinch_handler, poll_io, + stdin_is_tty, write_all_retry, NonBlockingStdin, RawModeGuard, + }; + use std::io::{stderr, stdin, stdout, Read}; + + // Disable socket read timeout for interactive sessions — the poll loop + // handles readiness checking, and the session runs until the user exits. + self.stream + .set_read_timeout(None) + .map_err(|e| Error::agent("set read timeout", e.to_string()))?; + + self.send(&request)?; + + // Wait for Started response + let started = self.receive()?; + match started { + AgentResponse::Started => {} + AgentResponse::Error { message, .. } => { + return Err(Error::agent(op, message)); + } + _ => { + return Err(Error::agent(op, "expected Started response")); + } + } + + // Enable raw mode if TTY requested and stdin is a TTY + // The guard will restore terminal settings on drop (even on panic) + let _raw_mode = if tty && stdin_is_tty() { + RawModeGuard::new(stdin_raw_fd()) + } else { + None + }; + + // Send initial terminal size so PTY starts at the right dimensions + if tty { + if let Some((cols, rows)) = get_terminal_size() { + self.send(&AgentRequest::Resize { cols, rows })?; + } + install_sigwinch_handler(); + } + + // Set stdin to non-blocking (guard restores on drop) + let _nonblock_stdin = NonBlockingStdin::new() + .map_err(|e| Error::agent("set stdin nonblocking", e.to_string()))?; + + // Socket stays blocking — poll() determines readiness, then blocking + // read/write completes immediately. This avoids partial-read/write bugs + // that occur with non-blocking read_exact/write_all. + let mut stdin_handle = stdin(); + let stdin_fd = stdin_raw_fd(); + let socket_fd = self.stream_raw_fd(); + let mut stdin_buf = [0u8; STDIN_BUF_SIZE]; + let mut stdin_eof = false; + + let exit_code = loop { + let effective_stdin_fd = if stdin_eof { -1 } else { stdin_fd }; + let poll_result = poll_io(effective_stdin_fd, socket_fd, POLL_TIMEOUT_MS) + .map_err(|e| Error::agent("poll", e.to_string()))?; + + // Check for terminal resize (SIGWINCH) + if tty && check_sigwinch() { + if let Some((cols, rows)) = get_terminal_size() { + self.send(&AgentRequest::Resize { cols, rows })?; + } + } + + // Handle socket data FIRST — drain agent output before writing stdin + // to prevent deadlock when send buffer is full + if poll_result.socket_ready { + match self.receive() { + Ok(AgentResponse::Stdout { data }) => { + write_all_retry(&mut stdout(), &data)?; + flush_retry(&mut stdout())?; + } + Ok(AgentResponse::Stderr { data }) => { + write_all_retry(&mut stderr(), &data)?; + flush_retry(&mut stderr())?; + } + Ok(AgentResponse::Exited { exit_code }) => { + break exit_code; + } + Ok(AgentResponse::Error { message, .. }) => { + return Err(Error::agent(op, message)); + } + Ok(_) => {} + Err(e) => { + // EAGAIN/WouldBlock can occur when poll() reports readiness + // but the data isn't available yet (common with vsock on macOS). + // Retry on next poll iteration instead of crashing. + if e.is_io() + && matches!( + e.source_io_error_kind(), + Some(std::io::ErrorKind::WouldBlock) + ) + { + tracing::debug!("socket read returned EAGAIN, retrying"); + continue; + } + return Err(e); + } + } + } + + // Socket peer closed without sending Exited — VM crashed or was killed + if poll_result.socket_hangup && !poll_result.socket_ready { + return Err(Error::agent(op, "connection to VM lost".to_string())); + } + + // Handle stdin input — send to agent + if poll_result.stdin_ready && !stdin_eof { + match stdin_handle.read(&mut stdin_buf) { + Ok(0) => { + stdin_eof = true; + self.send(&AgentRequest::Stdin { data: Vec::new() })?; + } + Ok(n) => { + self.send(&AgentRequest::Stdin { + data: stdin_buf[..n].to_vec(), + })?; + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {} + Err(e) => { + tracing::debug!(error = %e, "stdin read error, treating as EOF"); + stdin_eof = true; + self.send(&AgentRequest::Stdin { data: Vec::new() })?; + } + } + } + }; + + Ok(exit_code) + } + + /// Execute a command directly in the VM with interactive I/O. + pub fn vm_exec_interactive( + &mut self, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + tty: bool, + ) -> Result { + let timeout_ms = timeout.map(|t| t.as_millis() as u64); + self.interactive_session( + AgentRequest::VmExec { + command, + env, + workdir, + timeout_ms, + interactive: true, + tty, + background: false, + stdin_data: None, + }, + tty, + "vm exec interactive", + ) + } + + /// Run a command in an image's rootfs (non-interactive). + /// + /// This is the non-interactive counterpart to `run_interactive()`. + /// Both accept a `RunConfig` for consistency. + /// + /// # Returns + /// + /// A tuple of (exit_code, stdout, stderr) + pub fn run_non_interactive(&mut self, config: RunConfig) -> Result<(i32, Vec, Vec)> { + let _timeout_guard = self.set_exec_timeout(config.timeout)?; + let timeout_ms = config.timeout.map(|t| t.as_millis() as u64); + + let resp = self.request(&AgentRequest::Run { + image: config.image, + command: config.command, + env: config.env, + workdir: config.workdir, + user: config.user, + mounts: config.mounts, + timeout_ms, + interactive: false, + tty: false, + detached: false, + unprivileged: config.unprivileged, + persistent_overlay_id: config.persistent_overlay_id, + stdin_data: config.stdin, + background: false, + })?; + + expect_completed(resp, "run command") + } + + /// Run a command in an image's rootfs in the background. + /// + /// Spawns the container and returns immediately with the crun PID. + /// stdout/stderr go to /dev/null inside the guest. Use a persistent + /// overlay ID so subsequent `exec` sessions see the same filesystem. + pub fn run_background(&mut self, config: RunConfig) -> Result { + let resp = self.request(&AgentRequest::Run { + image: config.image, + command: config.command, + env: config.env, + workdir: config.workdir, + user: config.user, + mounts: config.mounts, + timeout_ms: None, + interactive: false, + tty: false, + detached: false, + unprivileged: config.unprivileged, + persistent_overlay_id: config.persistent_overlay_id, + stdin_data: None, + background: true, + })?; + + let (exit_code, stdout, _stderr) = expect_completed(resp, "run background")?; + if exit_code != 0 { + return Err(Error::agent("run background", "spawn failed")); + } + let pid: u32 = String::from_utf8_lossy(&stdout) + .trim() + .parse() + .map_err(|_| Error::agent("run background", "invalid PID in response"))?; + Ok(pid) + } + + /// Run a command in an image's rootfs and handle streamed events as they arrive. + /// + /// Unlike `run_interactive`, this does not forward stdin. It is the + /// image-backed counterpart to `vm_exec_streaming_with`. + pub fn run_streaming_with(&mut self, config: RunConfig, on_event: F) -> Result<()> + where + F: FnMut(ExecEvent), + { + let timeout_ms = config.timeout.map(|t| t.as_millis() as u64); + + self.stream + .set_read_timeout(None) + .map_err(|e| Error::agent("set read timeout", e.to_string()))?; + + self.send(&AgentRequest::Run { + image: config.image, + command: config.command, + env: config.env, + workdir: config.workdir, + user: config.user, + mounts: config.mounts, + timeout_ms, + interactive: true, + tty: false, + detached: false, + unprivileged: config.unprivileged, + persistent_overlay_id: config.persistent_overlay_id, + stdin_data: None, + background: false, + })?; + + collect_exec_events(self, "run streaming", on_event) + } + + /// Run a command interactively with streaming I/O. + /// + /// This method streams output directly to stdout/stderr and forwards stdin. + /// It blocks until the command exits. + /// + /// # Arguments + /// + /// * `config` - Run configuration including image, command, environment, etc. + /// + /// # Returns + /// + /// The exit code of the command + pub fn run_interactive(&mut self, config: RunConfig) -> Result { + let timeout_ms = config.timeout.map(|t| t.as_millis() as u64); + let tty = config.tty; + self.interactive_session( + AgentRequest::Run { + image: config.image, + command: config.command, + env: config.env, + workdir: config.workdir, + user: config.user, + mounts: config.mounts, + timeout_ms, + interactive: true, + tty, + detached: false, + unprivileged: config.unprivileged, + persistent_overlay_id: config.persistent_overlay_id, + stdin_data: None, + background: false, + }, + tty, + "run interactive", + ) + } + + /// Start a container in detached mode and return its container ID. + /// + /// Sends a `Run { detached: true }` request to the agent, which starts the + /// container in the background via `crun run --detach` and immediately + /// returns the container ID. Subsequent `machine exec` calls against the + /// same `persistent_overlay_id` will join this container's namespaces via + /// `crun exec` instead of creating a new isolated container. + /// + /// Requires `config.persistent_overlay_id` to be set — detached containers + /// only make sense when there is a persistent overlay to associate with. + pub fn run_container_detached(&mut self, config: RunConfig) -> Result { + // Container startup involves overlay setup + crun init which can exceed + // the default 30s read timeout on first run (cold overlay, cold image). + let _timeout_guard = self.set_extended_read_timeout(Duration::from_secs(120))?; + + let resp = self.request(&AgentRequest::Run { + image: config.image, + command: config.command, + env: config.env, + workdir: config.workdir, + user: config.user, + mounts: config.mounts, + timeout_ms: None, + interactive: false, + tty: false, + detached: true, + unprivileged: config.unprivileged, + persistent_overlay_id: config.persistent_overlay_id, + stdin_data: None, + background: false, + })?; + let (exit_code, stdout, _) = expect_completed(resp, "run container detached")?; + if exit_code != 0 { + return Err(Error::agent( + "run container detached", + format!("agent returned exit code {}", exit_code), + )); + } + String::from_utf8(stdout).map_err(|e| { + Error::agent( + "run container detached", + format!("invalid container ID in response: {}", e), + ) + }) + } + + /// Send stdin data to a running interactive command. + pub fn send_stdin(&mut self, data: &[u8]) -> Result<()> { + self.send(&AgentRequest::Stdin { + data: data.to_vec(), + }) + } + + /// Send a window resize event to a running interactive command. + pub fn send_resize(&mut self, cols: u16, rows: u16) -> Result<()> { + self.send(&AgentRequest::Resize { cols, rows }) + } + + /// Run an interactive session driven by channels instead of the process's + /// real stdin/stdout. Input events arrive on `input`; output is delivered to + /// `on_output`. This is the transport-agnostic counterpart to + /// [`Self::interactive_session`] — used to bridge a VM PTY to a remote + /// WebSocket terminal without touching the host's terminal. + /// + /// The loop polls only the vsock socket (input comes from the channel, not an + /// fd) and drains pending input each iteration. When `input` disconnects + /// (the remote peer hung up) it sends EOF once and keeps running until the + /// command exits — a shell reading its PTY exits on EOF. + fn interactive_session_io( + &mut self, + request: AgentRequest, + input: std::sync::mpsc::Receiver, + mut on_output: F, + op: &str, + ) -> Result + where + F: FnMut(InteractiveOutput), + { + use crate::agent::terminal::poll_io; + + // No socket read timeout — the poll loop handles readiness and the + // session runs until the command exits or the peer hangs up. + self.stream + .set_read_timeout(None) + .map_err(|e| Error::agent("set read timeout", e.to_string()))?; + + self.send(&request)?; + match self.receive()? { + AgentResponse::Started => {} + AgentResponse::Error { message, .. } => return Err(Error::agent(op, message)), + _ => return Err(Error::agent(op, "expected Started response")), + } + + let socket_fd = self.stream_raw_fd(); + let mut input_eof_sent = false; + + let exit_code = loop { + // stdin_fd = -1 → poll() ignores it; only the socket drives readiness. + let poll_result = poll_io(-1, socket_fd, POLL_TIMEOUT_MS) + .map_err(|e| Error::agent("poll", e.to_string()))?; + + // Drain agent output first (prevents deadlock when its send buffer fills). + if poll_result.socket_ready { + match self.receive() { + Ok(AgentResponse::Stdout { data }) => { + on_output(InteractiveOutput::Stdout(data)) + } + Ok(AgentResponse::Stderr { data }) => { + on_output(InteractiveOutput::Stderr(data)) + } + Ok(AgentResponse::Exited { exit_code }) => break exit_code, + Ok(AgentResponse::Error { message, .. }) => { + return Err(Error::agent(op, message)) + } + Ok(_) => {} + Err(e) => { + if e.is_io() + && matches!( + e.source_io_error_kind(), + Some(std::io::ErrorKind::WouldBlock) + ) + { + continue; + } + return Err(e); + } + } + } + + if poll_result.socket_hangup && !poll_result.socket_ready { + return Err(Error::agent(op, "connection to VM lost".to_string())); + } + + // Forward any pending input without blocking the output path. + loop { + match input.try_recv() { + Ok(InteractiveInput::Stdin(data)) => { + self.send(&AgentRequest::Stdin { data })? + } + Ok(InteractiveInput::Resize { cols, rows }) => { + self.send(&AgentRequest::Resize { cols, rows })? + } + Ok(InteractiveInput::Eof) => { + if !input_eof_sent { + self.send(&AgentRequest::Stdin { data: Vec::new() })?; + input_eof_sent = true; + } + } + Err(std::sync::mpsc::TryRecvError::Empty) => break, + Err(std::sync::mpsc::TryRecvError::Disconnected) => { + // Remote peer (WebSocket client) gone. Return immediately + // instead of waiting for the command to exit on its own. + // This method runs on a DEDICATED, disposable connection, + // so returning drops it; the agent's interactive loop then + // sees the closed peer and kills the PTY child. Waiting here + // would pin the connection (and, on the shared client, the + // per-machine lock) until a command that ignores stdin EOF + // — a `sleep`, a daemon — finally exits. + return Ok(DISCONNECT_EXIT_CODE); + } + } + } + }; + + Ok(exit_code) + } + + /// Interactive VM exec driven by channels (remote PTY). Counterpart to + /// [`Self::vm_exec_interactive`] that does not bind the host terminal. + pub fn vm_exec_interactive_io( + &mut self, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + tty: bool, + input: std::sync::mpsc::Receiver, + on_output: F, + ) -> Result + where + F: FnMut(InteractiveOutput), + { + self.interactive_session_io( + AgentRequest::VmExec { + command, + env, + workdir, + timeout_ms: None, + interactive: true, + tty, + background: false, + stdin_data: None, + }, + input, + on_output, + "vm exec interactive (io)", + ) + } + + /// Interactive container run driven by channels (remote PTY). Counterpart to + /// [`Self::run_interactive`] that does not bind the host terminal. + pub fn run_interactive_io( + &mut self, + config: RunConfig, + input: std::sync::mpsc::Receiver, + on_output: F, + ) -> Result + where + F: FnMut(InteractiveOutput), + { + let tty = config.tty; + self.interactive_session_io( + AgentRequest::Run { + image: config.image, + command: config.command, + env: config.env, + workdir: config.workdir, + user: config.user, + mounts: config.mounts, + timeout_ms: None, + interactive: true, + tty, + detached: false, + unprivileged: config.unprivileged, + persistent_overlay_id: config.persistent_overlay_id, + stdin_data: None, + background: false, + }, + input, + on_output, + "run interactive (io)", + ) + } + + // ======================================================================== + // File I/O + // ======================================================================== + + /// Write a file into the VM. + /// + /// Transparently dispatches between single-shot and streaming + /// based on `data.len()`: + /// + /// - Files ≤ [`FILE_WRITE_SINGLE_SHOT_MAX`] (1 MiB): one + /// [`AgentRequest::FileWrite`] message — the lowest-latency + /// path and what 99% of `cp` calls hit. + /// - Files larger than that: a sequence of + /// [`AgentRequest::FileWriteBegin`] + + /// [`AgentRequest::FileWriteChunk`] messages, each under + /// [`MAX_FRAME_SIZE`]. This is the only correct way to upload + /// files whose base64-encoded form would exceed the frame + /// limit — without it the send blocks the socket (EAGAIN + /// after write timeout) and risks OOMing the guest agent. + pub fn write_file(&mut self, path: &str, data: &[u8], mode: Option) -> Result<()> { + self.write_file_with_progress(path, data, mode, |_| {}) + } + + /// Write a file into the VM with a progress callback. + /// + /// `on_progress` is called after each chunk is acked by the + /// agent, with the running byte total. Single-shot writes (small + /// files) call it once at the end. Callers who don't need + /// progress should use [`Self::write_file`] which passes a no-op. + pub fn write_file_with_progress( + &mut self, + path: &str, + data: &[u8], + mode: Option, + mut on_progress: F, + ) -> Result<()> { + if data.len() <= FILE_WRITE_SINGLE_SHOT_MAX { + let resp = self.request(&AgentRequest::FileWrite { + path: path.to_string(), + data: data.to_vec(), + mode, + })?; + expect_ok(resp, "write file")?; + on_progress(data.len() as u64); + Ok(()) + } else { + self.write_file_streaming(path, data, mode, &mut on_progress) + } + } + + /// Streaming file upload from a `&[u8]` slice. + fn write_file_streaming( + &mut self, + path: &str, + data: &[u8], + mode: Option, + on_progress: &mut F, + ) -> Result<()> { + self.write_file_streaming_from_reader( + path, + &mut std::io::Cursor::new(data), + data.len() as u64, + mode, + on_progress, + ) + } + + /// Stream a file from a [`Read`] source into the VM. + /// + /// Reads `FILE_WRITE_CHUNK_SIZE` bytes at a time from `reader`, + /// sending each chunk over the protocol. Only one chunk is in + /// memory at a time — the caller doesn't need to buffer the + /// entire file. + pub fn write_file_from_reader( + &mut self, + path: &str, + reader: R, + total_size: u64, + mode: Option, + ) -> Result<()> { + self.write_file_from_reader_with_progress(path, reader, total_size, mode, |_| {}) + } + + /// Stream a file from a [`Read`] source with progress callback. + pub fn write_file_from_reader_with_progress( + &mut self, + path: &str, + reader: R, + total_size: u64, + mode: Option, + mut on_progress: F, + ) -> Result<()> { + if total_size <= FILE_WRITE_SINGLE_SHOT_MAX as u64 { + // Small file: read into memory and use single-shot path. + let mut data = Vec::with_capacity(total_size as usize); + std::io::Read::read_to_end(&mut std::io::Read::take(reader, total_size), &mut data) + .map_err(|e| Error::agent("read source file", e.to_string()))?; + return self.write_file_with_progress(path, &data, mode, on_progress); + } + self.write_file_streaming_from_reader( + path, + &mut { reader }, + total_size, + mode, + &mut on_progress, + ) + } + + /// Core streaming upload loop. Reads chunks from `reader` and + /// sends them over the protocol. Only one chunk buffer is live + /// at a time (~1 MiB). + fn write_file_streaming_from_reader( + &mut self, + path: &str, + reader: &mut R, + total_size: u64, + mode: Option, + on_progress: &mut F, + ) -> Result<()> { + let resp = self.request(&AgentRequest::FileWriteBegin { + path: path.to_string(), + mode, + total_size, + })?; + expect_ok(resp, "begin streaming write")?; + + let mut buf = vec![0u8; FILE_WRITE_CHUNK_SIZE]; + let mut bytes_sent = 0u64; + + loop { + // Fill the chunk buffer. + let mut filled = 0; + while filled < buf.len() { + match reader.read(&mut buf[filled..]) { + Ok(0) => break, + Ok(n) => filled += n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => return Err(Error::agent("read source file", e.to_string())), + } + } + + if filled == 0 { + // EOF — send final empty chunk to finalize. + let resp = self.request(&AgentRequest::FileWriteChunk { + data: Vec::new(), + done: true, + })?; + expect_ok(resp, "finalize streaming write")?; + break; + } + + bytes_sent += filled as u64; + let done = bytes_sent >= total_size; + + let resp = self.request(&AgentRequest::FileWriteChunk { + data: buf[..filled].to_vec(), + done, + })?; + expect_ok(resp, "stream write chunk")?; + on_progress(bytes_sent); + + if done { + break; + } + } + Ok(()) + } + + /// Read a file from the VM. + /// + /// Consumes the streamed `DataChunk` responses the agent emits + /// (see `handle_streaming_file_read` in the agent). The agent + /// sends one or more chunks, with `done: true` on the final + /// frame — possibly empty. This method concatenates chunks and + /// returns the full contents. + /// + /// Two safety bounds: + /// - Receive timeout extended to 600 s so large files don't + /// spuriously fail on slow storage; a 200 MB file at 10 MB/s + /// would exceed the default 30 s receive timeout otherwise. + /// - Total size capped at [`FILE_TRANSFER_MAX_TOTAL`] (4 GiB) — + /// symmetric with the write path. A misbehaving or compromised + /// guest can't OOM the host by streaming unbounded data. + pub fn read_file(&mut self, path: &str) -> Result> { + self.read_file_with_progress(path, |_| {}) + } + + /// Read a file from the VM with a progress callback. + /// + /// `on_progress` is called with the running byte total after + /// each `DataChunk` is received. Use [`Self::read_file`] if you + /// don't need progress. + pub fn read_file_with_progress( + &mut self, + path: &str, + on_progress: F, + ) -> Result> { + const FILE_READ_TIMEOUT: Duration = Duration::from_secs(600); + + let _timeout_guard = self.set_extended_read_timeout(FILE_READ_TIMEOUT)?; + self.send_raw(&AgentRequest::FileRead { + path: path.to_string(), + })?; + + consume_streamed_read_with_progress(|| self.recv_raw(), on_progress) + } + + /// Download a file from the VM directly to a local path. + /// + /// Unlike [`Self::read_file`] which accumulates the entire file + /// in memory, this writes each chunk to disk as it arrives — + /// only one 16 MiB chunk is in memory at a time. + pub fn read_file_to_path( + &mut self, + guest_path: &str, + local_path: &std::path::Path, + mut on_progress: F, + ) -> Result { + use std::io::Write; + const FILE_READ_TIMEOUT: Duration = Duration::from_secs(600); + + let _timeout_guard = self.set_extended_read_timeout(FILE_READ_TIMEOUT)?; + self.send_raw(&AgentRequest::FileRead { + path: guest_path.to_string(), + })?; + + let mut file = std::fs::File::create(local_path).map_err(|e| { + Error::agent( + "write local file", + format!("{}: {}", local_path.display(), e), + ) + })?; + + let mut total = 0u64; + loop { + match self.recv_raw()? { + AgentResponse::DataChunk { data, done } => { + let next_total = total.saturating_add(data.len() as u64); + if next_total > FILE_TRANSFER_MAX_TOTAL { + let _ = std::fs::remove_file(local_path); + return Err(Error::agent( + "read file", + format!( + "guest streamed {} bytes, exceeding the {} byte cap", + next_total, FILE_TRANSFER_MAX_TOTAL + ), + )); + } + if !data.is_empty() { + file.write_all(&data) + .map_err(|e| Error::agent("write local file", e.to_string()))?; + total = next_total; + on_progress(total); + } + if done { + file.flush() + .map_err(|e| Error::agent("flush local file", e.to_string()))?; + return Ok(total); + } + } + AgentResponse::Error { message, .. } => { + let _ = std::fs::remove_file(local_path); + return Err(Error::agent("read file", message)); + } + _ => { + let _ = std::fs::remove_file(local_path); + return Err(Error::agent("read file", "unexpected response")); + } + } + } + } + + // ======================================================================== + // Streaming Exec + // ======================================================================== + + /// Execute a command with streaming output. + /// + /// This compatibility wrapper buffers all events before returning. New + /// callers that need live output should use `vm_exec_streaming_with`. + /// + /// Sends a VmExec request with interactive=true, tty=false. Reads + /// Stdout/Stderr/Exited responses into a vector and blocks until the + /// command finishes — call from a blocking context (e.g., + /// `spawn_blocking`). + pub fn vm_exec_streaming( + &mut self, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + ) -> Result> { + let mut events = Vec::new(); + self.vm_exec_streaming_with(command, env, workdir, timeout, |event| { + events.push(event); + })?; + Ok(events) + } + + /// Execute a command with streaming output and handle events as they arrive. + /// + /// This is the live-output variant of `vm_exec_streaming`. + pub fn vm_exec_streaming_with( + &mut self, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + on_event: F, + ) -> Result<()> + where + F: FnMut(ExecEvent), + { + let timeout_ms = timeout.map(|t| t.as_millis() as u64); + + self.stream + .set_read_timeout(None) + .map_err(|e| Error::agent("set read timeout", e.to_string()))?; + + self.send(&AgentRequest::VmExec { + command, + env, + workdir, + timeout_ms, + interactive: true, + tty: false, + background: false, + stdin_data: None, + })?; + + collect_exec_events(self, "streaming exec", on_event) + } + + /// Low-level send without waiting for response (public). + pub fn send_raw(&mut self, request: &AgentRequest) -> Result<()> { + self.send(request) + } + + /// Low-level receive a single response (public). + pub fn recv_raw(&mut self) -> Result { + self.receive() + } + + /// Set a command-execution timeout and return a guard that resets it on drop. + /// + /// If `timeout` is Some, the socket deadline is `timeout + TIMEOUT_BUFFER_SECS`. + /// If None, the socket read timeout is disabled entirely — the command runs + /// until completion (or the VM dies, triggering EOF). This matches + /// `interactive_session`'s behavior and avoids any implicit ceiling on how + /// long a non-interactive command can run. The `ReadTimeoutGuard` restores + /// `DEFAULT_READ_TIMEOUT_SECS` on drop so subsequent operations get the + /// normal 30-second timeout. + fn set_exec_timeout(&self, timeout: Option) -> Result> { + match timeout { + Some(t) => { + self.set_read_timeout(t + Duration::from_secs(TIMEOUT_BUFFER_SECS))?; + } + None => { + self.stream.set_read_timeout(None).map_err(|e| { + Error::agent( + "set read timeout", + format!("failed to clear socket read timeout: {}", e), + ) + })?; + } + } + Ok(ReadTimeoutGuard::new(&self.stream)) + } + + /// Set an extended read timeout and return a guard that resets it on drop. + /// + /// Used for long-running streaming operations (e.g., layer export) where + /// individual chunks may take longer than the default 30s timeout. + pub fn set_extended_read_timeout(&self, timeout: Duration) -> Result> { + self.set_read_timeout(timeout)?; + Ok(ReadTimeoutGuard::new(&self.stream)) + } + + /// Low-level send without waiting for response. + fn send(&mut self, request: &AgentRequest) -> Result<()> { + let data = self.encode_traced(request)?; + self.stream.write_all(&data)?; + self.stream.flush()?; + Ok(()) + } + + /// Read exactly `buf.len()` bytes, retrying on EAGAIN/WouldBlock. + /// + /// Unlike `read_exact`, this never loses partially-read data on EAGAIN. + /// On macOS, vsock sockets can spuriously return WouldBlock even in + /// blocking mode, so we must handle it without corrupting the stream. + /// + /// If `propagate_initial_wouldblock` is true and WouldBlock occurs before + /// any bytes are read, the error is propagated (preserves read timeout + /// behavior). Once any bytes are consumed, EAGAIN is retried. + /// + /// # Stall protection + /// + /// When the socket has a read timeout configured, the retry loop is bounded + /// by a wall-clock *idle* deadline. Without it, a peer that writes a valid + /// length prefix and then stalls mid-frame (fewer body bytes than declared, + /// without closing) would spin this loop at 1ms forever, pinning the host + /// thread and the per-machine client lock and defeating every client-level + /// timeout. The idle deadline is reset on every byte of progress, so a + /// slow-but-steady body is never penalized — only a body that delivers *no* + /// bytes for a full read-timeout window fails, with a `TimedOut` error. + /// + /// When no read timeout is configured (interactive sessions), WouldBlock is + /// treated as the spurious macOS vsock EAGAIN it is meant to be, and the loop + /// retries indefinitely as before — there is no deadline to enforce. + fn read_exact_retry( + &mut self, + buf: &mut [u8], + propagate_initial_wouldblock: bool, + ) -> std::io::Result<()> { + // Idle window: how long we tolerate zero progress before declaring a + // stall. Derived from the socket's configured read timeout so it tracks + // the caller's intent; `None` means blocking mode (no deadline). + let idle_window = self.stream.read_timeout().ok().flatten(); + let mut deadline = idle_window.map(|w| std::time::Instant::now() + w); + + let mut pos = 0; + while pos < buf.len() { + match self.stream.read(&mut buf[pos..]) { + Ok(0) => { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "connection closed", + )); + } + Ok(n) => { + pos += n; + // Progress made — extend the idle deadline. + if let Some(w) = idle_window { + deadline = Some(std::time::Instant::now() + w); + } + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + if pos == 0 && propagate_initial_wouldblock { + // No data consumed yet and caller wants timeout errors — propagate + return Err(e); + } + // Mid-read (or caller wants full retry). Retry, but bail out + // if the idle deadline has passed so a stalled mid-frame body + // can't busy-spin forever. + if let Some(d) = deadline { + if std::time::Instant::now() >= d { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "timed out reading frame body: peer stalled mid-frame", + )); + } + } + std::thread::sleep(std::time::Duration::from_millis(1)); + continue; + } + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + } + } + Ok(()) + } + + /// Low-level receive a single response. + fn receive(&mut self) -> Result { + // Check if a read timeout is set — if so, WouldBlock before any data + // means a real timeout and should be propagated. If no timeout (interactive + // sessions), WouldBlock is always a spurious macOS vsock EAGAIN. + let has_timeout = self.stream.read_timeout().ok().flatten().is_some(); + + let mut header = [0u8; 4]; + self.read_exact_retry(&mut header, has_timeout)?; + let len = u32::from_be_bytes(header) as usize; + + // Validate frame size to prevent OOM from malicious/buggy responses + if len > MAX_FRAME_SIZE as usize { + // Header consumed but body not read — stream is desynchronized. + // Shut down the read half so all future reads fail immediately + // rather than interpreting body bytes as a frame header. + let _ = self.stream.shutdown(std::net::Shutdown::Read); + return Err(Error::agent( + "validate frame", + format!( + "frame too large: {} bytes (max: {} bytes)", + len, MAX_FRAME_SIZE + ), + )); + } + + let mut buf = vec![0u8; len]; + // Always retry body reads — header is already consumed so we can't + // propagate an error without corrupting the stream. + if let Err(e) = self.read_exact_retry(&mut buf, false) { + // Body read failed — stream is desynchronized. Shut down the + // read half so future reads fail cleanly. + let _ = self.stream.shutdown(std::net::Shutdown::Read); + return Err(e.into()); + } + + let resp: AgentResponse = serde_json::from_slice(&buf) + .map_err(|e| Error::agent("deserialize response", e.to_string()))?; + Ok(resp) + } +} + +/// Cumulative byte ceiling for the streaming/collect exec path. +/// +/// The buffered (`Completed`) exec path caps guest output inside the guest at +/// `smolvm_agent::process::MAX_EXEC_OUTPUT` (11 MiB). The streaming path, +/// however, relays frames one at a time and the SSE handler +/// (`api::handlers::exec::exec_stream`) buffers the *entire* event vector in +/// host RAM before responding. Without a cap, a chatty or infinite guest +/// command (`yes`, `cat /dev/zero`) that emits `Stdout` frames forever and +/// never sends `Exited` grows the host `serve` process without bound → host +/// OOM → every co-tenant VM on the node is killed (cross-tenant DoS). +/// +/// We mirror the buffered path's 11 MiB ceiling: once cumulative stdout+stderr +/// crosses it, we emit a truncation error event and stop relaying (terminating +/// the collect loop) instead of growing unbounded. +/// +/// Semantics: this is a per-exec-session cap on the BUFFERED/relayed output of +/// the streaming exec path. Interactive PTY sessions are long-lived by design +/// and do NOT flow through here — they use +/// [`AgentClient::interactive_session_io`], which streams frame-by-frame to the +/// WebSocket without accumulating — so a legitimate long interactive session is +/// unaffected by this cap. +const MAX_STREAMING_EXEC_OUTPUT: usize = 11 * 1024 * 1024; + +fn collect_exec_events(client: &mut AgentClient, op: &str, on_event: F) -> Result<()> +where + F: FnMut(ExecEvent), +{ + collect_exec_events_inner(|| client.receive(), op, MAX_STREAMING_EXEC_OUTPUT, on_event) +} + +/// Cap-parameterized core of [`collect_exec_events`], pulling responses from a +/// `next_response` closure so it can be unit-tested against synthetic frame +/// sequences with a small cap (instead of booting a VM and streaming 11 MiB). +fn collect_exec_events_inner( + mut next_response: N, + op: &str, + cap: usize, + mut on_event: F, +) -> Result<()> +where + N: FnMut() -> Result, + F: FnMut(ExecEvent), +{ + match next_response()? { + AgentResponse::Started => {} + AgentResponse::Error { message, .. } => { + return Err(Error::agent(op, message)); + } + _ => return Err(Error::agent(op, "expected Started")), + } + + // Cumulative stdout+stderr bytes relayed on this session. Bounds the host + // buffer so a guest that never sends `Exited` can't OOM the host. + let mut total: usize = 0; + loop { + match next_response() { + Ok(AgentResponse::Stdout { data }) => { + total = total.saturating_add(data.len()); + on_event(ExecEvent::Stdout(data)); + } + Ok(AgentResponse::Stderr { data }) => { + total = total.saturating_add(data.len()); + on_event(ExecEvent::Stderr(data)); + } + Ok(AgentResponse::Exited { exit_code }) => { + on_event(ExecEvent::Exit(exit_code)); + break; + } + Ok(AgentResponse::Error { message, .. }) => { + on_event(ExecEvent::Error(message)); + break; + } + Ok(_) => {} + Err(err) => { + on_event(ExecEvent::Error(err.to_string())); + break; + } + } + + // Cap check runs after relaying each frame (matching the buffered + // path's "send chunk, then break at the cap" order): we relay at most + // `cap` + one frame before terminating with a truncation signal. + if total >= cap { + on_event(ExecEvent::Error(format!( + "streaming output exceeded {cap} byte cap; exec terminated (output truncated)" + ))); + break; + } + } + Ok(()) +} + +/// Consume streamed `DataChunk` responses, enforcing the per-transfer +/// size cap and returning the concatenated bytes. +/// +/// Pulled out of `AgentClient::read_file` so it can be unit-tested +/// against synthetic response sequences without booting a VM, and +/// against a small cap so the test doesn't have to allocate 4 GiB +/// just to exercise the cap branch. +/// +/// Two small variants on the same loop: +/// - [`consume_streamed_read_with_progress`]: production cap, with +/// a progress callback (used by [`AgentClient::read_file`] + +/// [`AgentClient::read_file_with_progress`]). +/// - [`consume_streamed_read_with_cap`]: parameterized cap, no +/// progress (used by tests so they can exercise the cap branch +/// with kilobytes instead of gigabytes). +/// +/// Both delegate to [`consume_streamed_read_inner`]. +fn consume_streamed_read_with_progress(next_response: F, on_progress: P) -> Result> +where + F: FnMut() -> Result, + P: FnMut(u64), +{ + consume_streamed_read_inner(next_response, FILE_TRANSFER_MAX_TOTAL, on_progress) +} + +#[cfg(test)] +fn consume_streamed_read_with_cap(next_response: F, cap: u64) -> Result> +where + F: FnMut() -> Result, +{ + consume_streamed_read_inner(next_response, cap, |_| {}) +} + +fn consume_streamed_read_inner( + mut next_response: F, + cap: u64, + mut on_progress: P, +) -> Result> +where + F: FnMut() -> Result, + P: FnMut(u64), +{ + let mut out: Vec = Vec::new(); + let mut total: u64 = 0; + loop { + match next_response()? { + AgentResponse::DataChunk { data, done } => { + // Cap *before* extending so a single oversized chunk + // can't push us past the limit. + let next_total = total.saturating_add(data.len() as u64); + if next_total > cap { + return Err(Error::agent( + "read file", + format!( + "guest streamed {} bytes, exceeding the {} byte cap; \ + use a virtiofs mount for larger files", + next_total, cap + ), + )); + } + out.extend_from_slice(&data); + total = next_total; + on_progress(total); + if done { + return Ok(out); + } + } + AgentResponse::Error { message, .. } => { + return Err(Error::agent("read file", message)); + } + other => { + return Err(Error::agent( + "read file", + format!("unexpected response: {:?}", other), + )); + } + } + } +} + +#[cfg(test)] +mod read_cap_tests { + use super::*; + + /// Build a `DataChunk` response with `n` zero bytes. + fn chunk(n: usize, done: bool) -> AgentResponse { + AgentResponse::DataChunk { + data: vec![0u8; n], + done, + } + } + + /// Drive the consumer over a fixed list of responses with a + /// small (1 KiB) cap. Tests only need to exercise the size + /// arithmetic; the production cap of 4 GiB would need the test + /// to allocate a Vec that big to trip — wasteful and unnecessary. + /// The cap is parameterized on the internal helper precisely so + /// this test can scale down. + const TEST_CAP: u64 = 1024; + + fn drive(responses: Vec) -> Result> { + let mut iter = responses.into_iter(); + consume_streamed_read_with_cap( + || { + iter.next() + .ok_or_else(|| Error::agent("test", "no more responses")) + }, + TEST_CAP, + ) + } + + #[test] + fn read_cap_terminator_returns_full_buffer() { + let out = drive(vec![chunk(100, false), chunk(50, true)]).unwrap(); + assert_eq!(out.len(), 150); + } + + #[test] + fn read_cap_empty_terminator_is_valid_eof() { + let out = drive(vec![chunk(100, false), chunk(0, true)]).unwrap(); + assert_eq!(out.len(), 100); + } + + #[test] + fn read_cap_rejects_single_chunk_at_or_above_limit() { + // Single chunk that on its own pushes past the cap. We're at + // 1024-byte cap so this chunk is 1025 bytes — trivial to + // allocate in tests, exercises the same arithmetic that + // would catch a 4 GiB+ chunk in production. + let err = drive(vec![chunk(TEST_CAP as usize + 1, true)]).unwrap_err(); + let msg = format!("{}", err); + assert!( + msg.contains("exceeding") && msg.contains("byte cap"), + "expected size-cap error, got: {}", + msg + ); + } + + #[test] + fn read_cap_rejects_when_accumulated_chunks_exceed_limit() { + // Two chunks under the cap individually but exceeding it + // combined. This is the realistic exhaustion vector — a + // misbehaving guest streaming "fine-sized" chunks forever. + let half = (TEST_CAP / 2) as usize; + let err = drive(vec![ + chunk(half, false), + chunk(half, false), + chunk(half, false), // would push past the cap + ]) + .unwrap_err(); + assert!(format!("{}", err).contains("byte cap")); + } + + #[test] + fn read_cap_chunk_at_exactly_limit_is_accepted() { + // Boundary: a chunk that lands the accumulated total at + // exactly the cap is fine. Only > cap is rejected. + let out = drive(vec![chunk(TEST_CAP as usize, true)]).unwrap(); + assert_eq!(out.len(), TEST_CAP as usize); + } + + #[test] + fn read_cap_propagates_agent_error_response() { + let err = drive(vec![AgentResponse::Error { + message: "no such file".to_string(), + code: None, + }]) + .unwrap_err(); + assert!(format!("{}", err).contains("no such file")); + } + + #[test] + fn read_cap_rejects_unexpected_response_type() { + let err = drive(vec![AgentResponse::Pong { version: 1 }]).unwrap_err(); + assert!(format!("{}", err).contains("unexpected response")); + } +} + +#[cfg(test)] +mod collect_exec_cap_tests { + //! Regression coverage for the streaming-exec host-OOM guard. + //! + //! Proves the streaming/collect path (`collect_exec_events_inner`) stops + //! and emits a truncation signal once cumulative relayed output crosses the + //! cap, instead of buffering unbounded output from a guest that never sends + //! `Exited` (the `yes` / `cat /dev/zero` cross-tenant DoS). + use super::*; + + fn stdout(n: usize) -> AgentResponse { + AgentResponse::Stdout { data: vec![0u8; n] } + } + + /// Drive `collect_exec_events_inner` over a fixed response list with a + /// small cap, capturing the relayed events and the terminal result. + fn drive(cap: usize, responses: Vec) -> (Vec, Result<()>) { + let mut iter = responses.into_iter(); + let mut events = Vec::new(); + let res = collect_exec_events_inner( + || { + iter.next() + .ok_or_else(|| Error::agent("test", "no more responses")) + }, + "test exec", + cap, + |e| events.push(e), + ); + (events, res) + } + + const TEST_CAP: usize = 1024; + + #[test] + fn stops_at_cumulative_cap_with_truncation_signal() { + // An infinite chatty stream: `Started`, then far more stdout than the + // cap, and crucially NEVER `Exited`. If the cap did not fire the loop + // would drain all 1000 frames (and in production loop forever). Because + // it caps, only a handful of frames are consumed before termination. + let half = TEST_CAP / 2; + let mut responses = vec![AgentResponse::Started]; + for _ in 0..1000 { + responses.push(stdout(half)); // would total 500_000 bytes unbounded + } + let (events, res) = drive(TEST_CAP, responses); + res.expect("collect returns Ok after capping (not the iterator-drained error)"); + + // Terminated near the cap, nowhere near draining 1000 frames. + assert!( + events.len() < 10, + "expected termination near the cap, got {} events", + events.len() + ); + + // Final event is the truncation error. + match events.last().expect("at least one event") { + ExecEvent::Error(msg) => assert!( + msg.contains("byte cap") && msg.contains("truncated"), + "expected truncation error, got: {msg}" + ), + other => panic!("expected trailing truncation Error event, got {other:?}"), + } + + // Relayed bytes are bounded: at most cap + one frame. + let relayed: usize = events + .iter() + .map(|e| match e { + ExecEvent::Stdout(d) | ExecEvent::Stderr(d) => d.len(), + _ => 0, + }) + .sum(); + assert!( + relayed <= TEST_CAP + half, + "relayed {relayed} bytes exceeds the cap+one-frame bound" + ); + } + + #[test] + fn single_oversized_frame_trips_the_cap() { + // One frame that alone crosses the cap: relayed once, then truncated. + let (events, res) = drive(TEST_CAP, vec![AgentResponse::Started, stdout(TEST_CAP + 1)]); + res.unwrap(); + assert!(matches!(events.last().unwrap(), ExecEvent::Error(m) if m.contains("byte cap"))); + } + + #[test] + fn passes_through_under_cap_and_exits_cleanly() { + // Normal, well-behaved exec under the cap relays everything and ends on + // `Exit` with no truncation error injected. + let (events, res) = drive( + TEST_CAP, + vec![ + AgentResponse::Started, + stdout(100), + AgentResponse::Stderr { + data: b"warn".to_vec(), + }, + AgentResponse::Exited { exit_code: 0 }, + ], + ); + res.unwrap(); + assert_eq!(events.last().unwrap(), &ExecEvent::Exit(0)); + assert!( + !events.iter().any(|e| matches!(e, ExecEvent::Error(_))), + "clean exec must not inject a truncation error" + ); + } +} + +#[cfg(test)] +mod run_background_tests { + //! Regression test for image-backed `machine run -d`. + //! + //! The original bug: the CLI's image + `--detach` path pulled the image + //! and persisted the VM record but silently dropped the command. The + //! fix wires in `AgentClient::run_background`, which must send a + //! `Run { background: true }` over the wire and parse the returned PID. + //! + //! If this test fails, the detach path either lost its `background` + //! plumbing or stopped parsing the PID response — either way, the + //! original "command never runs" regression is back. + use super::*; + use smolvm_protocol::{encode_message, AgentRequest, AgentResponse, Envelope}; + use std::io::{Read, Write}; + use std::thread; + + #[test] + fn run_background_sends_background_true_and_returns_pid() { + let (client_stream, mut server_stream) = UdsStream::pair().unwrap(); + + // Fake agent: read one request, assert it's a background Run, respond + // with a Completed PID. Mirrors what the real agent does in + // `handle_run_background`. + let server = thread::spawn(move || { + let mut len_buf = [0u8; 4]; + server_stream.read_exact(&mut len_buf).unwrap(); + let len = u32::from_be_bytes(len_buf) as usize; + + let mut payload = vec![0u8; len]; + server_stream.read_exact(&mut payload).unwrap(); + + let envelope: Envelope = + serde_json::from_slice(&payload).expect("valid Envelope"); + + match envelope.body { + AgentRequest::Run { + image, + command, + persistent_overlay_id, + background, + interactive, + tty, + .. + } => { + assert!( + background, + "run_background must send background: true — the image+detach CLI path \ + depends on this field to dispatch the command inside the container" + ); + assert!(!interactive, "background runs are never interactive"); + assert!(!tty, "background runs never allocate a TTY"); + assert_eq!(image, "docker.io/library/alpine:3.19"); + assert_eq!(command, vec!["sh", "-c", "echo hi"]); + assert_eq!( + persistent_overlay_id, + Some("default".to_string()), + "background runs must use a persistent overlay so subsequent execs \ + see the same filesystem" + ); + } + other => panic!("expected AgentRequest::Run, got {:?}", other), + } + + let resp = AgentResponse::Completed { + exit_code: 0, + stdout: b"12345".to_vec(), + stderr: Vec::new(), + }; + let encoded = encode_message(&resp).expect("encode response"); + server_stream.write_all(&encoded).expect("write response"); + }); + + let mut client = AgentClient::from_stream(client_stream); + let config = RunConfig::new( + "alpine:3.19", + vec!["sh".to_string(), "-c".to_string(), "echo hi".to_string()], + ) + .with_persistent_overlay(Some("default".to_string())); + + let pid = client + .run_background(config) + .expect("run_background should succeed on a Completed response"); + + assert_eq!(pid, 12345, "client must parse the PID from stdout"); + server.join().expect("server thread joined cleanly"); + } + + #[test] + fn run_background_rejects_nonzero_exit_code() { + // If the agent fails to spawn the container, it returns a non-zero + // exit_code. The client must turn that into an error rather than + // silently returning a bogus PID. + let (client_stream, mut server_stream) = UdsStream::pair().unwrap(); + + let server = thread::spawn(move || { + let mut len_buf = [0u8; 4]; + server_stream.read_exact(&mut len_buf).unwrap(); + let len = u32::from_be_bytes(len_buf) as usize; + let mut payload = vec![0u8; len]; + server_stream.read_exact(&mut payload).unwrap(); + + let resp = AgentResponse::Completed { + exit_code: 1, + stdout: Vec::new(), + stderr: b"spawn failed".to_vec(), + }; + let encoded = encode_message(&resp).unwrap(); + server_stream.write_all(&encoded).unwrap(); + }); + + let mut client = AgentClient::from_stream(client_stream); + let config = RunConfig::new("alpine:3.19", vec!["true".to_string()]) + .with_persistent_overlay(Some("default".to_string())); + + let err = client + .run_background(config) + .expect_err("non-zero exit must surface as an error"); + assert!( + format!("{}", err).contains("spawn failed") + || format!("{}", err).contains("run background"), + "unexpected error: {}", + err + ); + server.join().unwrap(); + } +} + +#[cfg(test)] +mod run_streaming_tests { + //! Regression coverage for image-backed streaming exec. + //! + //! `machine exec --stream` must preserve the same execution target as + //! buffered `machine exec`. For image-backed machines that means sending a + //! `Run { interactive: true }` request to the agent, not `VmExec` against + //! the bare agent rootfs. + use super::*; + use smolvm_protocol::{encode_message, AgentRequest, AgentResponse, Envelope}; + use std::io::{Read, Write}; + use std::thread; + + #[test] + fn run_streaming_sends_interactive_run_and_collects_events() { + let (client_stream, mut server_stream) = UdsStream::pair().unwrap(); + + let server = thread::spawn(move || { + let mut len_buf = [0u8; 4]; + server_stream.read_exact(&mut len_buf).unwrap(); + let len = u32::from_be_bytes(len_buf) as usize; + let mut payload = vec![0u8; len]; + server_stream.read_exact(&mut payload).unwrap(); + + let envelope: Envelope = + serde_json::from_slice(&payload).expect("valid Envelope"); + match envelope.body { + AgentRequest::Run { + image, + command, + mounts, + interactive, + tty, + detached, + background, + persistent_overlay_id, + .. + } => { + assert_eq!(image, "docker.io/library/ubuntu:24.04"); + assert_eq!(command, vec!["/bin/bash", "-lc", "echo hi"]); + assert_eq!( + mounts, + vec![("work".to_string(), "/work".to_string(), false)] + ); + assert!(interactive, "streaming image exec must use interactive Run"); + assert!(!tty, "plain --stream must not allocate a TTY"); + assert!(!detached, "streaming exec must not detach"); + assert!(!background, "streaming exec must not run in background"); + assert_eq!(persistent_overlay_id, Some("dev".to_string())); + } + other => panic!("expected AgentRequest::Run, got {:?}", other), + } + + for response in [ + AgentResponse::Started, + AgentResponse::Stdout { + data: b"hi\n".to_vec(), + }, + AgentResponse::Stderr { + data: b"warn\n".to_vec(), + }, + AgentResponse::Exited { exit_code: 7 }, + ] { + let encoded = encode_message(&response).expect("encode response"); + server_stream.write_all(&encoded).expect("write response"); + } + }); + + let mut client = AgentClient::from_stream(client_stream); + let config = RunConfig::new( + "ubuntu:24.04", + vec![ + "/bin/bash".to_string(), + "-lc".to_string(), + "echo hi".to_string(), + ], + ) + .with_mounts(vec![("work".to_string(), "/work".to_string(), false)]) + .with_persistent_overlay(Some("dev".to_string())); + + let mut events = Vec::new(); + client + .run_streaming_with(config, |event| events.push(event)) + .expect("run_streaming_with should handle streamed events"); + + assert_eq!( + events, + vec![ + ExecEvent::Stdout(b"hi\n".to_vec()), + ExecEvent::Stderr(b"warn\n".to_vec()), + ExecEvent::Exit(7), + ] + ); + server.join().expect("server thread joined cleanly"); + } +} + +#[cfg(test)] +mod stalled_body_tests { + //! Regression for the busy-spin-forever on a stalled mid-frame body. + //! + //! A peer that writes a valid 4-byte length prefix and then delivers fewer + //! body bytes than declared (without closing the socket) used to pin the + //! host thread in a 1ms retry loop with no wall-clock bound, defeating every + //! client-level timeout. `read_exact_retry` now honors a wall-clock idle + //! deadline derived from the socket read timeout, so `receive()` must return + //! a timeout error promptly instead of hanging. + use super::*; + use std::io::Write; + use std::time::{Duration, Instant}; + + #[test] + fn receive_times_out_on_stalled_mid_frame_body() { + let (client_stream, mut server_stream) = UdsStream::pair().unwrap(); + + // Short read timeout so the test is fast: the idle deadline tracks this. + let read_timeout = Duration::from_millis(150); + client_stream + .set_read_timeout(Some(read_timeout)) + .expect("set client read timeout"); + + // Server: declare a 64-byte body, then send only 3 bytes and STALL — + // hold the socket open (never drop it) so the client never sees EOF. + let server = std::thread::spawn(move || { + let declared_len: u32 = 64; + server_stream + .write_all(&declared_len.to_be_bytes()) + .expect("write length prefix"); + server_stream + .write_all(&[1u8, 2, 3]) + .expect("write partial body"); + server_stream.flush().expect("flush"); + // Stall: keep the connection open well past the client's deadline so + // the client cannot rely on EOF to unblock. + std::thread::sleep(Duration::from_secs(3)); + drop(server_stream); + }); + + let mut client = AgentClient::from_stream(client_stream); + + let start = Instant::now(); + let result = client.receive(); + let elapsed = start.elapsed(); + + assert!( + result.is_err(), + "receive() must error on a stalled mid-frame body, not return Ok" + ); + // Must return within a small multiple of the read timeout — nowhere near + // the server's 3s hold, proving it did not busy-spin until EOF. + assert!( + elapsed < Duration::from_secs(2), + "receive() should time out promptly (got {elapsed:?}); it must not \ + spin until the peer closes" + ); + + server.join().expect("server thread joined"); + } + + #[test] + fn read_exact_retry_bounds_stalled_read_without_eof() { + // Drive read_exact_retry directly: header-style non-propagating read of a + // buffer larger than what the peer sends, with the peer stalling. + let (client_stream, mut server_stream) = UdsStream::pair().unwrap(); + client_stream + .set_read_timeout(Some(Duration::from_millis(100))) + .expect("set read timeout"); + + let server = std::thread::spawn(move || { + server_stream.write_all(&[0xAAu8]).expect("write one byte"); + server_stream.flush().expect("flush"); + std::thread::sleep(Duration::from_secs(3)); + drop(server_stream); + }); + + let mut client = AgentClient::from_stream(client_stream); + + let start = Instant::now(); + // Ask for 8 bytes but only 1 will ever arrive; propagate=false forces the + // retry path (the same path receive() uses for the body). + let mut buf = [0u8; 8]; + let err = client + .read_exact_retry(&mut buf, false) + .expect_err("stalled read must return a bounded error"); + let elapsed = start.elapsed(); + + assert_eq!( + err.kind(), + std::io::ErrorKind::TimedOut, + "stalled mid-buffer read must surface a TimedOut error" + ); + assert!( + elapsed < Duration::from_secs(2), + "read_exact_retry must not busy-spin until EOF (got {elapsed:?})" + ); + + server.join().expect("server thread joined"); + } +} diff --git a/src/agent/fork.rs b/src/agent/fork.rs new file mode 100644 index 0000000..b8a56f5 --- /dev/null +++ b/src/agent/fork.rs @@ -0,0 +1,531 @@ +//! Live fork mechanics shared by the CLI (`machine fork`) and the serve API +//! (`POST /api/v1/machines/{id}/fork`). +//! +//! A fork freezes a running, forkable golden machine — it stays paused as the +//! shared copy-on-write base — snapshots its memfd-backed RAM + device state, +//! gives the clone copy-on-write disk overlays, and lets the caller boot the +//! clone from that snapshot. The boot itself differs between callers (the CLI +//! uses `start_vm_named`; the API uses `AgentManager`), so it stays out of here; +//! everything up to and including the snapshot + disk clone is shared so the two +//! entry points can never silently diverge. + +use crate::agent::{resolve_disk_image, vm_data_dir, AgentClient}; +use crate::config::VmRecord; +use crate::data::validate_vm_name; +use crate::db::SmolvmDb; +use crate::{Error, Result}; +use std::path::{Path, PathBuf}; + +/// Path to a forkable machine's control socket (pause/resume/checkpoint/FORK). +pub fn control_socket_path(name: &str) -> PathBuf { + vm_data_dir(name).join("control.sock") +} + +/// Send a single line command to a VM control socket and return its reply line. +pub fn control_socket_cmd(sock: &Path, cmd: &str) -> Result { + use crate::platform::uds::UdsStream; + use std::io::{Read, Write}; + + let mut stream = UdsStream::connect(sock) + .map_err(|e| Error::agent("connect control socket", e.to_string()))?; + stream + .set_read_timeout(Some(std::time::Duration::from_secs(60))) + .ok(); + stream + .write_all(format!("{cmd}\n").as_bytes()) + .map_err(|e| Error::agent("write control socket", e.to_string()))?; + let mut reply = String::new(); + let mut byte = [0u8; 1]; + loop { + match stream.read(&mut byte) { + Ok(0) => break, + Ok(_) => { + if byte[0] == b'\n' { + break; + } + reply.push(byte[0] as char); + } + Err(e) => return Err(Error::agent("read control socket", e.to_string())), + } + } + Ok(reply) +} + +/// The result of preparing a fork: the golden is frozen + snapshotted and the +/// clone's DB record + copy-on-write disks exist on disk. The caller boots the +/// clone from `snapshot_dir`, then calls [`rejuvenate_clone`]. +pub struct PreparedFork { + /// Directory holding the golden's checkpoint + memfd manifest. Pass it as the + /// clone's `LaunchFeatures::snapshot_dir` to boot from it instead of cold. + pub snapshot_dir: PathBuf, + /// The clone's freshly-inserted DB record (golden's config, remapped ports). + pub clone_record: VmRecord, + /// Per-port inbound remap as `(golden_host, guest, clone_host)`, for the + /// caller to log. Empty when the golden has no forwards. When ports were + /// pinned, `golden_host == clone_host`. + pub port_remaps: Vec<(u16, u16, u16)>, +} + +/// Freeze a running, forkable `golden`, snapshot it, register `clone` in the DB +/// with copy-on-write disks, and return everything the caller needs to boot the +/// clone. Launch-agnostic: the actual boot is the caller's job (CLI via +/// `start_vm_named`, API via `AgentManager`), keyed off the returned +/// `snapshot_dir`. +/// +/// On any failure after the clone record is inserted, the record and its data +/// directory are cleaned up before returning the error, so a failed fork leaves +/// no half-registered clone behind. +pub fn prepare_fork( + db: &SmolvmDb, + golden: &str, + clone: &str, + pinned_ports: &[(u16, u16)], + clone_forkable: bool, +) -> Result { + validate_vm_name(clone, "clone name").map_err(|e| Error::config("clone name", e))?; + + // Nested fork is unsupported: a clone boots from a copy-on-write MAP_PRIVATE + // mapping of the golden's RAM, not a fresh memfd, so it cannot itself be + // re-forked (its FORK would fail with "no memfd-backed RAM"). Reject + // `forkable` up front instead of producing a clone that looks forkable but + // isn't. + if clone_forkable { + return Err(Error::agent( + "fork", + "nested fork is not supported: a clone cannot be re-forked, so `forkable` \ + on a fork has no effect (drop it)", + )); + } + + let golden_rec = db + .get_vm(golden)? + .ok_or_else(|| Error::vm_not_found(golden))?; + + // The golden must be alive and forkable. We probe the control socket rather + // than the vsock agent: after its first fork the golden is frozen (paused) + // as the shared base, so an agent ping would fail — but STATUS still answers + // (running or paused), and we can fork it again. + let ctl = control_socket_path(golden); + if !ctl.exists() { + return Err(Error::agent( + "fork", + format!("golden '{golden}' is not running forkable; start it with `machine start --forkable --name {golden}`"), + )); + } + let status = control_socket_cmd(&ctl, "STATUS").map_err(|e| { + Error::agent( + "fork", + format!("golden '{golden}' control socket not responding ({e}); start it with `machine start --forkable --name {golden}`"), + ) + })?; + if !status.starts_with("OK") { + return Err(Error::agent( + "fork", + format!("golden '{golden}' is not ready to fork: {status}"), + )); + } + if db.get_vm(clone)?.is_some() { + return Err(Error::agent( + "fork", + format!("machine '{clone}' already exists"), + )); + } + + // Clone dir + snapshot dir. A leftover data directory with no DB record is + // an orphan from a previously crashed fork; its stale qcow2 overlays would + // make `krun_create_disk_overlay` fail (rc=-5, it refuses to overwrite an + // existing target). The DB check above guarantees no live clone owns this + // name, so clearing the directory is safe. + let clone_dir = vm_data_dir(clone); + if clone_dir.exists() { + std::fs::remove_dir_all(&clone_dir) + .map_err(|e| Error::agent("clear orphan clone dir", e.to_string()))?; + } + std::fs::create_dir_all(&clone_dir) + .map_err(|e| Error::agent("create clone dir", e.to_string()))?; + + // The golden writes its frozen snapshot (checkpoint + memfd manifest) here. + // It lives under the GOLDEN's data dir, not the clone's: under Landlock the + // frozen golden VMM is confined to its own data dir, so it can write here but + // could not write into a separate clone's dir. The clone — which already needs + // read access to the golden's dir for its copy-on-write disk backing — reads + // the snapshot from the same place. See `internal_boot`'s Landlock grants. + let gdir = vm_data_dir(golden); + let snapshot_dir = gdir.join("fork-snapshots").join(clone); + std::fs::create_dir_all(&snapshot_dir) + .map_err(|e| Error::agent("create snapshot dir", e.to_string()))?; + // Under per-VM uid isolation (privileged launcher) the frozen golden VMM runs + // as its own unprivileged uid and writes the snapshot here via the FORK + // command below, so hand this dir to that uid. No-op unless privileged; if the + // drop is active the golden's uid lookup must succeed (fail closed). + if let Some(result) = + crate::process::vm_drop_ids(&crate::agent::vm_uid_registry_dir(), &gdir, None) + { + let (uid, gid) = + result.map_err(|e| Error::agent("fork: resolve golden uid", e.to_string()))?; + crate::process::chown_tree(&snapshot_dir, uid, gid) + .map_err(|e| Error::agent("fork: chown snapshot dir", e.to_string()))?; + } + + // Register the clone in the DB with the golden's config, no running-state, + // and its port forwards remapped to fresh host ports. With the default TSI + // backend outbound is proxied per-process (each clone gets it for free, no + // guest MAC/IP involved); only inbound host ports must be made distinct so + // the clone is reachable without colliding with the still-running golden or + // sibling clones. + let mut clone_rec = golden_rec.clone(); + clone_rec.name = clone.to_string(); + clone_rec.pid = None; + clone_rec.pid_start_time = None; + let mut port_remaps = Vec::new(); + if !pinned_ports.is_empty() { + // User pinned the clone's forwards explicitly — use them as-is. + clone_rec.ports = pinned_ports.to_vec(); + for (h, g) in &clone_rec.ports { + port_remaps.push((*h, *g, *h)); + } + } else if !clone_rec.ports.is_empty() { + let mut remapped = Vec::with_capacity(clone_rec.ports.len()); + for (golden_host, guest) in &clone_rec.ports { + match alloc_free_host_port() { + Some(h) => { + port_remaps.push((*golden_host, *guest, h)); + remapped.push((h, *guest)); + } + None => tracing::warn!( + guest, + "could not allocate a host port for fork clone; dropping forward" + ), + } + } + clone_rec.ports = remapped; + } + clone_rec.golden = Some(golden.to_string()); + db.insert_vm(clone, &clone_rec)?; + + // Freeze the golden and write its snapshot (checkpoint + memfd manifest). + let cleanup = || { + let _ = db.remove_vm(clone); + let _ = std::fs::remove_dir_all(&clone_dir); + let _ = std::fs::remove_dir_all(&snapshot_dir); + }; + let reply = match control_socket_cmd(&ctl, &format!("FORK {}", snapshot_dir.display())) { + Ok(r) => r, + Err(e) => { + cleanup(); + return Err(e); + } + }; + if !reply.starts_with("OK") { + cleanup(); + return Err(Error::agent("fork", format!("golden FORK failed: {reply}"))); + } + + if let Err(e) = clone_fork_disks(&gdir, &clone_dir) { + cleanup(); + return Err(e); + } + + Ok(PreparedFork { + snapshot_dir, + clone_record: clone_rec, + port_remaps, + }) +} + +/// Give the clone its own disks. The golden is frozen with its block workers +/// quiesced and flushed, so its images are a consistent backing. On Linux each +/// disk is a qcow2 copy-on-write overlay over the golden's — filesystem +/// independent, so the overlay starts near-empty and the fork is O(metadata) +/// regardless of how much data the golden holds. macOS clonefiles the disks +/// (APFS CoW). Either way the `.formatted` marker is copied so the clone never +/// reformats and wipes the inherited filesystem. +fn clone_fork_disks(gdir: &Path, clone_dir: &Path) -> Result<()> { + // The golden's actual disks that exist, resolved by file presence (`.qcow2` + // if the golden is itself a clone, else `.raw`) — the same single source of + // truth the agent manager uses. Each entry pairs the canonical `.raw` + // filename (for naming the clone's disk) with the golden's real backing file + // and its format. + let disks: Vec<(&str, PathBuf, crate::data::disk::DiskFormat)> = [ + crate::data::storage::STORAGE_DISK_FILENAME, + crate::data::storage::OVERLAY_DISK_FILENAME, + ] + .into_iter() + .map(|raw| { + let (src, fmt) = resolve_disk_image(gdir, raw); + (raw, src, fmt) + }) + .filter(|(_, src, _)| src.exists()) + .collect(); + + #[cfg(target_os = "linux")] + { + // Each clone disk is a qcow2 CoW overlay over the golden's disk. Build + // all overlay specs first so libkrun is loaded once for the batch + // (absolute backing path: it's written verbatim into the overlay + // header), then copy the `.formatted` markers so the clone never + // reformats and wipes the inherited filesystem. + let mut specs = Vec::with_capacity(disks.len()); + for (raw, src, fmt) in &disks { + let base = src + .canonicalize() + .map_err(|e| Error::agent("clone disk", format!("{}: {e}", src.display())))?; + let overlay = clone_dir.join(Path::new(raw).with_extension("qcow2")); + specs.push((overlay, base, *fmt)); + } + crate::agent::create_disk_overlays(&specs)?; + for (raw, _, _) in &disks { + // Marker basename is the disk stem + ".formatted" (same for the + // golden's `.raw`/`.qcow2` and the clone's `.qcow2`). + let marker = Path::new(raw).with_extension("formatted"); + let src_marker = gdir.join(&marker); + if src_marker.exists() { + let _ = std::fs::copy(&src_marker, clone_dir.join(&marker)); + } + } + } + #[cfg(target_os = "macos")] + { + // macOS uses clonefile (APFS CoW), keeping the golden's disk format. + for (_, src, _) in &disks { + let dst = clone_dir.join(src.file_name().unwrap()); + crate::disk_utils::clone_or_copy_file(src, &dst) + .map_err(|e| Error::agent("clone disk", format!("{}: {e}", src.display())))?; + let src_marker = src.with_extension("formatted"); + if src_marker.exists() { + let _ = std::fs::copy(&src_marker, dst.with_extension("formatted")); + } + } + } + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + { + // Fork-clone disk overlays rely on libkrun's qcow2 overlay (Linux) or + // APFS clonefile (macOS); neither is wired up on Windows. + let _ = (&disks, clone_dir); + return Err(Error::agent( + "clone disk", + "live fork is not supported on this platform", + )); + } + #[allow(unreachable_code)] + Ok(()) +} + +/// Number of times we try to confirm a clone's identity rejuvenation before +/// giving up and failing the fork. `connect_with_retry` already rides out the +/// agent's boot; these extra attempts cover a momentarily-busy agent whose +/// `vm_exec` errors or exits non-zero transiently. +const REJUVENATE_ATTEMPTS: usize = 3; + +/// Build the shell script that re-mints a clone's on-disk identity. Kept as a +/// pure function of `(clone, seed)` so the security-critical contents (fresh +/// machine-id, regenerated SSH host keys) are unit-tested without a live VM. +/// +/// `clone` is a validated machine name (alphanumeric + dashes) and `seed` is +/// hex, so single-quoting both is injection-safe. +/// +/// The script is fail-hard on the *unambiguously per-machine* identity material +/// (`set -e`): if a clone cannot get its own machine-id or SSH host keys, the +/// fork must fail rather than vend a clone that impersonates the golden. Steps +/// that are legitimately absent on minimal/library images (no sshd, no dbus, +/// no cloud-init) are guarded so they no-op instead of failing. +fn build_rejuvenation_script(clone: &str, seed: &str) -> String { + format!( + "set -e; \ + hostname '{c}' 2>/dev/null || true; \ + printf '%s\\n' '{c}' > /etc/hostname; \ + tr -d '-' < /proc/sys/kernel/random/uuid > /etc/machine-id; \ + if [ -f /var/lib/dbus/machine-id ] && [ ! -L /var/lib/dbus/machine-id ]; then \ + tr -d '-' < /proc/sys/kernel/random/uuid > /var/lib/dbus/machine-id; \ + fi; \ + if [ -d /etc/ssh ] && command -v ssh-keygen >/dev/null 2>&1; then \ + rm -f /etc/ssh/ssh_host_*_key /etc/ssh/ssh_host_*_key.pub; \ + ssh-keygen -A >/dev/null 2>&1; \ + fi; \ + rm -rf /var/lib/cloud/instance /var/lib/cloud/instances/* /var/lib/cloud/data/instance-id 2>/dev/null || true; \ + printf '%s' '{s}' > /dev/urandom 2>/dev/null || true; \ + true", + c = clone, + s = seed, + ) +} + +/// Per-clone identity rejuvenation after a fork. A fork CoW-clones the golden's +/// disks wholesale, so every per-machine on-disk secret (machine-id, SSH host +/// keys, dbus id, cloud-init instance state) is byte-identical in the clone — +/// and clones can belong to *different tenants*. Left unchanged, that is a +/// cross-tenant impersonation / MITM hole (identical SSH host keys) and a +/// duplicate-identity bug. This runs over the freshly-booted clone's agent to +/// give it a fresh hostname, machine-id, SSH host keys, and to stir the kernel +/// RNG with fresh host entropy so the random streams diverge. +/// +/// FAIL-CLOSED: this returns `Err` if the reset could not be *confirmed* (agent +/// unreachable, or the re-mint script exited non-zero) after +/// [`REJUVENATE_ATTEMPTS`] tries. Callers MUST treat that as a fork failure and +/// tear the clone down — a clone that still carries the golden's identity must +/// never be vended (see [`fail_closed_on_rejuvenation`]). +/// +/// RESIDUAL LIMITATION (out of scope, intentional): this rejuvenates only +/// *on-disk* identity. It cannot scrub the golden's *in-RAM* secrets — a +/// session token, JWT, or TLS private key held in a golden-resident process's +/// memory is CoW-inherited identically by every clone. That is intrinsic to +/// fork-from-warm and is not fixable here; the mitigation is a product +/// constraint (goldens must be prepacked library base images that mint no +/// per-instance boot secrets in RAM, and/or restart key daemons post-fork), not +/// disk rejuvenation. Likewise this stirs but does not *credit* entropy +/// (no `RNDADDENTROPY`/VMGENID yet) and does not re-address the network +/// (MAC/IP; safe under the default TSI backend) — both are follow-ups. +pub fn rejuvenate_clone(clone: &str) -> Result<()> { + let sock = vm_data_dir(clone).join("agent.sock"); + let seed = host_random_hex(64); + let script = build_rejuvenation_script(clone, &seed); + + let mut last_err = String::from("unknown error"); + for attempt in 1..=REJUVENATE_ATTEMPTS { + match rejuvenate_once(&sock, &script) { + Ok(()) => return Ok(()), + Err(e) => { + tracing::warn!( + clone, + attempt, + error = %e, + "clone rejuvenation attempt failed" + ); + last_err = e; + if attempt < REJUVENATE_ATTEMPTS { + std::thread::sleep(std::time::Duration::from_millis(500)); + } + } + } + } + Err(Error::agent( + "rejuvenate clone", + format!( + "identity reset could not be confirmed after {REJUVENATE_ATTEMPTS} attempts: {last_err}" + ), + )) +} + +/// One attempt: connect to the clone's agent and run the re-mint script. Any +/// connect error, exec error, or non-zero exit is a failure (fail-closed). +fn rejuvenate_once(sock: &Path, script: &str) -> std::result::Result<(), String> { + let mut client = + AgentClient::connect_with_retry(sock).map_err(|e| format!("agent connect: {e}"))?; + match client.vm_exec( + vec!["/bin/sh".into(), "-c".into(), script.to_string()], + vec![], + None, + Some(std::time::Duration::from_secs(10)), + None, + ) { + Ok((0, _, _)) => Ok(()), + Ok((code, _, stderr)) => Err(format!( + "re-mint script exited {code}: {}", + String::from_utf8_lossy(&stderr).trim() + )), + Err(e) => Err(format!("exec: {e}")), + } +} + +/// Fail-closed fork finalizer. A clone whose identity could not be rejuvenated +/// MUST NOT be vended (it would share the golden's machine-id/hostname/SSH host +/// keys across tenants), so on any rejuvenation `Err` this runs `teardown` +/// (stop + remove the clone) and propagates the error, turning a rejuvenation +/// failure into a fork failure. On `Ok` it does nothing and the caller proceeds +/// to mark the clone ready. Extracted as a pure decision so the fail-closed +/// behavior is unit-tested independently of the VM/agent machinery. +pub fn fail_closed_on_rejuvenation( + rejuvenation: Result<()>, + teardown: F, +) -> Result<()> { + match rejuvenation { + Ok(()) => Ok(()), + Err(e) => { + teardown(); + Err(e) + } + } +} + +/// Allocate a currently-free host TCP port by binding to port 0 and reading back +/// the OS-assigned port. Used to give each clone distinct inbound forwards. +fn alloc_free_host_port() -> Option { + std::net::TcpListener::bind(("127.0.0.1", 0)) + .ok() + .and_then(|l| l.local_addr().ok()) + .map(|addr| addr.port()) +} + +/// Read `hex_len/2` random bytes from the host RNG, hex-encoded. Used to seed +/// each clone's RNG with distinct host entropy. +fn host_random_hex(hex_len: usize) -> String { + use std::io::Read; + let mut buf = vec![0u8; hex_len / 2]; + if let Ok(mut f) = std::fs::File::open("/dev/urandom") { + let _ = f.read_exact(&mut buf); + } + buf.iter().map(|b| format!("{b:02x}")).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + // Fix 1: the re-mint script must regenerate the per-machine on-disk secrets + // that a wholesale CoW disk clone would otherwise share across tenants — + // above all the SSH host keys. + #[test] + fn rejuvenation_script_regenerates_per_machine_secrets() { + let script = build_rejuvenation_script("clone-a", "deadbeef"); + + // SSH host keys: delete the golden's, then regenerate fresh ones. + assert!( + script.contains("ssh_host_"), + "script must remove the golden's SSH host keys: {script}" + ); + assert!( + script.contains("ssh-keygen -A"), + "script must regenerate SSH host keys: {script}" + ); + // Fresh machine-id, hostname, and dbus id. + assert!(script.contains("> /etc/machine-id")); + assert!(script.contains("> /etc/hostname")); + assert!(script.contains("/var/lib/dbus/machine-id")); + // The clone name and RNG seed are threaded through. + assert!(script.contains("clone-a")); + assert!(script.contains("deadbeef")); + // Guarded so it fails hard on core identity but no-ops when sshd/dbus + // are absent (minimal library images). + assert!(script.contains("set -e")); + assert!(script.contains("command -v ssh-keygen")); + } + + // Fix 2 (fail-closed): an Err rejuvenation must tear the clone down and + // propagate the error — never leave it live/ready. + #[test] + fn rejuvenation_failure_tears_down_and_errors() { + let torn_down = Cell::new(false); + let result = fail_closed_on_rejuvenation( + Err(Error::agent("rejuvenate clone", "agent unreachable")), + || torn_down.set(true), + ); + assert!(result.is_err(), "a rejuvenation failure must fail the fork"); + assert!( + torn_down.get(), + "a rejuvenation failure must tear the clone down" + ); + } + + // Success path: the clone is kept (no teardown) and the fork proceeds. + #[test] + fn rejuvenation_success_keeps_clone_live() { + let torn_down = Cell::new(false); + let result = fail_closed_on_rejuvenation(Ok(()), || torn_down.set(true)); + assert!(result.is_ok()); + assert!( + !torn_down.get(), + "a successful rejuvenation must not tear the clone down" + ); + } +} diff --git a/src/agent/fsnotify_watch.rs b/src/agent/fsnotify_watch.rs new file mode 100644 index 0000000..e6d39d9 --- /dev/null +++ b/src/agent/fsnotify_watch.rs @@ -0,0 +1,338 @@ +//! Host → guest fsnotify propagation for `-v` mounts. +//! +//! virtiofs serves file *contents* to the guest, but it does not deliver +//! host-side change *notifications*. A file-watcher inside the guest (Vite, +//! webpack, nodemon, `inotifywait`) therefore never fires when a mounted file is +//! edited on the host — the classic reason "hot reload doesn't work in a +//! container on macOS". Because smolvm ships its own guest kernel (libkrunfw), +//! we can close this gap end to end: +//! +//! 1. This watcher runs on the host, watching each mounted source directory via +//! the OS-native mechanism (FSEvents on macOS, inotify on Linux). +//! 2. For every change it maps the host path to the guest-side virtiofs path and +//! sends it to the agent over a dedicated vsock connection +//! ([`AgentRequest::FsNotify`]). +//! 3. The agent writes it to `/proc/smolvm-fsnotify` (a libkrunfw kernel patch), +//! which fires the matching fsnotify event on the guest inode. +//! +//! Because the container's view of a `-v` mount is a bind of the same virtiofs +//! inode, a watcher inside the container wakes up exactly as if the change had +//! happened locally. +//! +//! The watcher is best-effort and self-contained: if the kernel lacks the patch, +//! or the connection drops when the VM exits, propagation simply stops — the +//! mount still serves reads. Dropping [`FsNotifyWatcher`] stops the thread. + +use crate::agent::AgentClient; +use crate::data::storage::HostMount; +use notify::{Event, EventKind, RecursiveMode, Watcher}; +use smolvm_protocol::{fsnotify_mask, FsNotifyEvent}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc; +use std::sync::Arc; +use std::thread::JoinHandle; +use std::time::Duration; +use tracing::{debug, info, warn}; + +/// Guest mountpoint root for virtiofs devices. MUST match the agent's +/// `paths::VIRTIOFS_MOUNT_ROOT` — the agent stages each `-v` device at +/// `/smolvm{index}` and binds it into the container, so events fired on +/// that path reach the container's bind of the same inode. +const GUEST_VIRTIOFS_ROOT: &str = "/mnt/virtiofs"; + +/// How long to coalesce a burst of change events before sending. An editor save +/// typically emits several events (write, close, attrib); batching de-dupes them +/// into one round-trip while keeping latency well under a human-perceptible +/// reload delay. +const COALESCE_WINDOW: Duration = Duration::from_millis(25); + +/// A single watched mount: host source directory → guest virtiofs staging base. +struct WatchTarget { + host_source: PathBuf, + /// e.g. `/mnt/virtiofs/smolvm0` + guest_base: String, +} + +/// Propagates host file changes under `-v` mounts into the guest as fsnotify +/// events for the lifetime of the value. Drop to stop. +pub struct FsNotifyWatcher { + stop: Arc, + handle: Option>, +} + +impl FsNotifyWatcher { + /// Start watching the source directory of every mount and replaying changes + /// into the guest. `mounts` must be in the same order passed to VM start, so + /// index `i` maps to virtiofs tag `smolvm{i}`. + /// + /// Returns `None` (non-fatal) when there is nothing to watch or the watcher + /// thread can't be spawned — the mount still works, only live change + /// notifications are unavailable. + pub fn start(socket_path: PathBuf, mounts: &[HostMount]) -> Option { + // Opt-out escape hatch: setting SMOL_NO_HOT_RELOAD disables host FS + // watching entirely (e.g. very large trees, or privacy preference). + if std::env::var_os("SMOL_NO_HOT_RELOAD").is_some() { + return None; + } + + // Only directories can be recursively watched; a file-target mount (rare) + // is skipped. Read-only mounts are still watched: the host may edit them + // (that is exactly the read-only-source hot-reload case). + let targets: Vec = mounts + .iter() + .enumerate() + .filter(|(_, m)| m.source.is_dir()) + .map(|(i, m)| WatchTarget { + host_source: m.source.clone(), + guest_base: format!("{GUEST_VIRTIOFS_ROOT}/smolvm{i}"), + }) + .collect(); + if targets.is_empty() { + return None; + } + + let stop = Arc::new(AtomicBool::new(false)); + let stop_thread = stop.clone(); + let handle = std::thread::Builder::new() + .name("fsnotify-watch".into()) + .spawn(move || run_watch(socket_path, targets, stop_thread)) + .ok()?; + + Some(Self { + stop, + handle: Some(handle), + }) + } +} + +impl Drop for FsNotifyWatcher { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} + +/// Watcher thread body: owns the OS watcher + a dedicated agent connection. +fn run_watch(socket_path: PathBuf, targets: Vec, stop: Arc) { + let (tx, rx) = mpsc::channel::>(); + + let mut watcher = match notify::recommended_watcher(move |res| { + // The receiver is dropped only when this thread exits, so a send error + // just means we're shutting down. + let _ = tx.send(res); + }) { + Ok(w) => w, + Err(e) => { + warn!(error = %e, "failed to create host fs watcher; hot-reload propagation disabled"); + return; + } + }; + + for t in &targets { + if let Err(e) = watcher.watch(&t.host_source, RecursiveMode::Recursive) { + warn!(path = %t.host_source.display(), error = %e, "failed to watch mount source"); + } + } + + // A dedicated connection so injected events never interleave with the + // command's own request/response stream on the primary connection. + let mut client = match AgentClient::connect_with_retry(&socket_path) { + Ok(c) => c, + Err(e) => { + debug!(error = %e, "fsnotify watcher could not connect to agent; disabled"); + return; + } + }; + + info!( + mounts = targets.len(), + "host→guest fsnotify propagation active (hot-reload)" + ); + + while !stop.load(Ordering::SeqCst) { + // Block briefly so we notice `stop` without a busy loop. + let first = match rx.recv_timeout(Duration::from_millis(200)) { + Ok(Ok(ev)) => ev, + Ok(Err(_)) => continue, // watcher-level error event; ignore + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + + let mut batch = Vec::new(); + collect_events(&first, &targets, &mut batch); + + // Coalesce the rest of the burst (a save fans out into several events). + let deadline = std::time::Instant::now() + COALESCE_WINDOW; + while let Some(remaining) = deadline.checked_duration_since(std::time::Instant::now()) { + match rx.recv_timeout(remaining) { + Ok(Ok(ev)) => collect_events(&ev, &targets, &mut batch), + Ok(Err(_)) => {} + Err(_) => break, + } + } + + if batch.is_empty() { + continue; + } + dedup(&mut batch); + + if let Err(e) = client.fsnotify(batch) { + // The VM has almost certainly gone away (the command exited). Stop + // quietly rather than spinning on a dead socket. + debug!(error = %e, "fsnotify inject failed; stopping watcher"); + break; + } + } +} + +/// Translate one host event into guest-side [`FsNotifyEvent`]s, appended to `out`. +fn collect_events(event: &Event, targets: &[WatchTarget], out: &mut Vec) { + for host_path in &event.paths { + let Some(t) = targets + .iter() + .find(|t| host_path.starts_with(&t.host_source)) + else { + continue; + }; + let Ok(rel) = host_path.strip_prefix(&t.host_source) else { + continue; + }; + // The watched root itself firing (empty rel) carries no useful child. + if rel.as_os_str().is_empty() { + continue; + } + + if host_path.exists() { + // Create/modify/attrib on an existing path: fire directly on it. + // fsnotify_dentry() in the guest propagates to the parent dir's + // watchers with the child name, so both file- and dir-watches fire. + out.push(FsNotifyEvent { + path: join_guest(&t.guest_base, rel), + mask: mask_for(&event.kind, host_path), + }); + } else if let Some(parent) = rel.parent() { + // Deleted / moved away: the exact path no longer resolves in the + // guest, so fire a MODIFY on the (still-present) parent directory. + // Directory watchers re-scan and observe the removal. + out.push(FsNotifyEvent { + path: join_guest(&t.guest_base, parent), + mask: fsnotify_mask::FS_MODIFY | fsnotify_mask::FS_ISDIR, + }); + } + } +} + +/// Join a guest base path with a host-relative path, normalizing separators. +fn join_guest(base: &str, rel: &Path) -> String { + let rel = rel.to_string_lossy(); + if rel.is_empty() { + base.to_string() + } else { + format!("{base}/{rel}") + } +} + +/// Map a notify [`EventKind`] to the closest `FS_*` mask. +fn mask_for(kind: &EventKind, path: &Path) -> u32 { + use notify::event::ModifyKind; + + let base = match kind { + EventKind::Create(_) => fsnotify_mask::FS_CREATE, + EventKind::Modify(ModifyKind::Metadata(_)) => fsnotify_mask::FS_ATTRIB, + // A rename whose destination exists reads as a create in the new dir. + EventKind::Modify(ModifyKind::Name(_)) => fsnotify_mask::FS_CREATE, + EventKind::Modify(_) => fsnotify_mask::FS_MODIFY, + // Anything else that left the file present is treated as a content change + // — the safe default that wakes content watchers. + _ => fsnotify_mask::FS_MODIFY, + }; + + if path.is_dir() { + base | fsnotify_mask::FS_ISDIR + } else { + base + } +} + +/// Sort + drop duplicate (path, mask) pairs produced within one burst. +fn dedup(batch: &mut Vec) { + batch.sort_by(|a, b| a.path.cmp(&b.path).then(a.mask.cmp(&b.mask))); + batch.dedup_by(|a, b| a.path == b.path && a.mask == b.mask); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn target() -> WatchTarget { + WatchTarget { + host_source: PathBuf::from("/host/project"), + guest_base: "/mnt/virtiofs/smolvm0".to_string(), + } + } + + #[test] + fn join_guest_maps_relative_paths() { + assert_eq!( + join_guest("/mnt/virtiofs/smolvm0", Path::new("src/app.js")), + "/mnt/virtiofs/smolvm0/src/app.js" + ); + assert_eq!( + join_guest("/mnt/virtiofs/smolvm0", Path::new("")), + "/mnt/virtiofs/smolvm0" + ); + } + + #[test] + fn deleted_path_fires_parent_dir_modify() { + // A path that does not exist on disk maps to a MODIFY on its parent. + let targets = vec![target()]; + let ev = Event { + kind: EventKind::Remove(notify::event::RemoveKind::File), + paths: vec![PathBuf::from("/host/project/src/gone.js")], + attrs: Default::default(), + }; + let mut out = Vec::new(); + collect_events(&ev, &targets, &mut out); + assert_eq!(out.len(), 1); + assert_eq!(out[0].path, "/mnt/virtiofs/smolvm0/src"); + assert_eq!( + out[0].mask & fsnotify_mask::FS_MODIFY, + fsnotify_mask::FS_MODIFY + ); + } + + #[test] + fn path_outside_any_mount_is_ignored() { + let targets = vec![target()]; + let ev = Event { + kind: EventKind::Modify(notify::event::ModifyKind::Data( + notify::event::DataChange::Any, + )), + paths: vec![PathBuf::from("/somewhere/else/x.js")], + attrs: Default::default(), + }; + let mut out = Vec::new(); + collect_events(&ev, &targets, &mut out); + assert!(out.is_empty()); + } + + #[test] + fn dedup_collapses_duplicate_events() { + let mut batch = vec![ + FsNotifyEvent { + path: "/a".into(), + mask: fsnotify_mask::FS_MODIFY, + }, + FsNotifyEvent { + path: "/a".into(), + mask: fsnotify_mask::FS_MODIFY, + }, + ]; + dedup(&mut batch); + assert_eq!(batch.len(), 1); + } +} diff --git a/src/agent/krun.rs b/src/agent/krun.rs new file mode 100644 index 0000000..8d015f3 --- /dev/null +++ b/src/agent/krun.rs @@ -0,0 +1,333 @@ +//! Runtime loader for libkrun. +//! +//! smolvm loads libkrun explicitly instead of relying on ELF/Mach-O +//! load-time linking. This lets packed Linux stubs start on hosts that do not +//! already have libkrun installed; packed mode can then extract bundled +//! libraries and load them from the cache. + +use crate::util::{libkrun_filename, libkrunfw_filename}; +use std::path::Path; + +/// Function pointers loaded from libkrun. +/// +/// Required symbols are loaded eagerly. Optional symbols are exposed as +/// `Option` so callers can report feature-specific errors. +#[allow(missing_docs)] +pub struct KrunFunctions { + // Keep the loaded libraries resident for the lifetime of the function + // table: the raw symbol pointers below borrow from these. `libloading` + // unloads (dlclose / FreeLibrary) on drop, so these must outlive the fns. + _handle: libloading::Library, + // Wrapped in `Option` so `Drop` can `take()` and `forget()` it, keeping + // libkrunfw resident (never unloaded) for any libkrun-owned references. + _fw_handle: Option, + pub set_log_level: unsafe extern "C" fn(u32) -> i32, + pub create_ctx: unsafe extern "C" fn() -> i32, + pub free_ctx: unsafe extern "C" fn(u32), + pub set_vm_config: unsafe extern "C" fn(u32, u8, u32) -> i32, + pub set_workdir: unsafe extern "C" fn(u32, *const libc::c_char) -> i32, + pub set_exec: unsafe extern "C" fn( + u32, + *const libc::c_char, + *const *const libc::c_char, + *const *const libc::c_char, + ) -> i32, + pub set_port_map: unsafe extern "C" fn(u32, *const *const libc::c_char) -> i32, + pub add_disk2: + unsafe extern "C" fn(u32, *const libc::c_char, *const libc::c_char, u32, bool) -> i32, + pub add_vsock_port2: unsafe extern "C" fn(u32, u32, *const libc::c_char, bool) -> i32, + pub add_virtiofs: unsafe extern "C" fn(u32, *const libc::c_char, *const libc::c_char) -> i32, + pub add_virtiofs3: Option< + unsafe extern "C" fn(u32, *const libc::c_char, *const libc::c_char, u64, bool) -> i32, + >, + pub start_enter: unsafe extern "C" fn(u32) -> i32, + pub add_vsock: unsafe extern "C" fn(u32, u32) -> i32, + /// Add a virtio-console device (the upstream replacement for the removed + /// `krun_set_console_output`). Unix: input/output/err file descriptors. + pub add_virtio_console_default: + unsafe extern "C" fn(u32, libc::c_int, libc::c_int, libc::c_int) -> i32, + pub set_egress_policy: Option< + unsafe extern "C" fn( + u32, + *const *const libc::c_char, + *const *const libc::c_char, + *const *const libc::c_char, + ) -> i32, + >, + pub add_net_unixstream: Option< + unsafe extern "C" fn(u32, *const libc::c_char, libc::c_int, *mut u8, u32, u32) -> i32, + >, + pub get_egress_handle: Option *mut libc::c_void>, + pub set_gpu_options2: Option i32>, + /// Retrieve guest RAM regions (`gpa_start, host_va, len` triples) for + /// zero-copy CUDA transfers. `None` on libkrun builds that predate the API. + pub get_guest_ram: Option i32>, + /// Register a Unix control socket for the VM (pause/resume/checkpoint/restore). + pub set_control_socket: Option i32>, + /// Boot the VM as a fork clone from a snapshot directory (CoW-map a golden + /// VM's RAM + restore state instead of cold-booting). + pub set_snapshot: Option i32>, + /// Create a qcow2 copy-on-write overlay backed by an existing disk image + /// (used for fork-clone block disks). Pure filesystem op; takes no ctx. + pub create_disk_overlay: + Option i32>, +} + +impl KrunFunctions { + /// Load libkrun from the given library directory. + /// + /// libkrunfw is preloaded with `RTLD_GLOBAL` because libkrun may resolve it + /// later by soname. + /// + /// # Safety + /// + /// Caller must ensure `lib_dir` contains compatible libkrun/libkrunfw + /// libraries for the current host. + pub unsafe fn load(lib_dir: &Path) -> Result { + #[cfg(target_os = "linux")] + preload_linux_gpu_dependencies(lib_dir); + + let fw_lib_path = lib_dir.join(libkrunfw_filename()); + // libkrunfw must be loaded with RTLD_GLOBAL on Unix so libkrun can + // resolve it later by soname; `libloading` only exposes that via the + // os-specific `Library::open`. On Windows there is no RTLD_GLOBAL + // concept, so a plain `Library::new` is used. + let fw_handle = load_library_global(&fw_lib_path) + .map_err(|e| format!("failed to load {}: {}", fw_lib_path.display(), e))?; + + let lib_path = lib_dir.join(libkrun_filename()); + // RTLD_LAZY (not RTLD_NOW) on Unix: a single libkrun built with the GPU + // feature references virglrenderer, but on Linux that NEEDED entry is + // stripped at package time so a host without virglrenderer can still + // load it. Lazy binding defers the virgl symbols until the GPU path + // actually calls them; preload_linux_gpu_dependencies() loads + // virglrenderer first when a GPU host has it. Non-GPU hosts never bind + // those symbols. + let handle = libloading::Library::new(&lib_path) + .map_err(|e| format!("failed to load {}: {}", lib_path.display(), e))?; + + macro_rules! load_sym { + ($name:ident) => {{ + let sym: libloading::Symbol<*mut std::ffi::c_void> = handle + .get(concat!(stringify!($name), "\0").as_bytes()) + .map_err(|_| format!("symbol not found: {}", stringify!($name)))?; + // The raw pointer outlives the `Symbol` borrow; the backing + // `Library` is kept resident in `_handle`/`_fw_handle`. + #[allow(clippy::missing_transmute_annotations)] + std::mem::transmute(*sym) + }}; + } + + macro_rules! load_optional_sym { + ($name:literal) => {{ + match handle.get::<*mut std::ffi::c_void>(concat!($name, "\0").as_bytes()) { + Ok(sym) => + { + #[allow(clippy::missing_transmute_annotations)] + Some(std::mem::transmute(*sym)) + } + Err(_) => None, + } + }}; + } + + // Resolve all symbols (borrowing `handle`) before moving the libraries + // into the struct, so the moves don't conflict with the borrows. + let set_log_level = load_sym!(krun_set_log_level); + let create_ctx = load_sym!(krun_create_ctx); + let free_ctx = load_sym!(krun_free_ctx); + let set_vm_config = load_sym!(krun_set_vm_config); + let set_workdir = load_sym!(krun_set_workdir); + let set_exec = load_sym!(krun_set_exec); + let set_port_map = load_sym!(krun_set_port_map); + let add_disk2 = load_sym!(krun_add_disk2); + let add_vsock_port2 = load_sym!(krun_add_vsock_port2); + let add_virtiofs = load_sym!(krun_add_virtiofs); + let add_virtiofs3 = load_optional_sym!("krun_add_virtiofs3"); + let start_enter = load_sym!(krun_start_enter); + let add_vsock = load_sym!(krun_add_vsock); + let add_virtio_console_default = load_sym!(krun_add_virtio_console_default); + let set_egress_policy = load_optional_sym!("krun_set_egress_policy"); + let add_net_unixstream = load_optional_sym!("krun_add_net_unixstream"); + let get_egress_handle = load_optional_sym!("krun_get_egress_handle"); + let set_gpu_options2 = load_optional_sym!("krun_set_gpu_options2"); + let get_guest_ram = load_optional_sym!("krun_get_guest_ram"); + let set_control_socket = load_optional_sym!("krun_set_control_socket"); + let set_snapshot = load_optional_sym!("krun_set_snapshot"); + let create_disk_overlay = load_optional_sym!("krun_create_disk_overlay"); + + Ok(Self { + _handle: handle, + _fw_handle: Some(fw_handle), + set_log_level, + create_ctx, + free_ctx, + set_vm_config, + set_workdir, + set_exec, + set_port_map, + add_disk2, + add_vsock_port2, + add_virtiofs, + add_virtiofs3, + start_enter, + add_vsock, + add_virtio_console_default, + set_egress_policy, + add_net_unixstream, + get_egress_handle, + set_gpu_options2, + get_guest_ram, + set_control_socket, + set_snapshot, + create_disk_overlay, + }) + } +} + +impl KrunFunctions { + /// Redirect the guest console output to `path`, using the upstream + /// virtio-console API (the replacement for the removed + /// `krun_set_console_output`). Returns libkrun's rc, or a negative value if + /// the file can't be opened. + /// + /// The opened fds are intentionally leaked: a console device's fds must stay + /// valid for the VM's lifetime, and `krun_start_enter` runs the VM in this + /// process, so the process owns them until it exits. + /// + /// # Safety + /// `ctx` must be a valid libkrun context that has not yet been started. + #[cfg(unix)] + pub unsafe fn console_output_to_file(&self, ctx: u32, path: &Path) -> i32 { + use std::os::fd::IntoRawFd; + let Ok(out) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + else { + return -1; + }; + let out_fd = out.into_raw_fd(); + // Console input comes from /dev/null (the agent talks over vsock, not the + // console); output and stderr both go to the log file. + let null_fd = unsafe { libc::open(c"/dev/null".as_ptr(), libc::O_RDONLY) }; + unsafe { (self.add_virtio_console_default)(ctx, null_fd, out_fd, out_fd) } + } + + /// Windows stub: the virtio-console redirection relies on POSIX file + /// descriptors passed to libkrun. The Windows libkrun ABI/console wiring is + /// not implemented here, so this is a no-op that reports failure. + /// + /// # Safety + /// `unsafe` only to match the Unix signature (which dereferences libkrun + /// function pointers). This stub touches nothing and is always sound to call. + #[cfg(not(unix))] + pub unsafe fn console_output_to_file(&self, _ctx: u32, _path: &Path) -> i32 { + -1 + } +} + +// libkrun's `_handle` is unloaded automatically when `KrunFunctions` is +// dropped (libloading calls dlclose / FreeLibrary). libkrunfw (`_fw_handle`) +// is intentionally kept resident for any libkrun-owned references, so we leak +// it rather than letting it unload. +impl Drop for KrunFunctions { + fn drop(&mut self) { + // Leak libkrunfw rather than unloading it: libkrun may still hold + // references resolved against it. `_handle` (libkrun itself) is dropped + // normally, unloading it. + if let Some(fw) = self._fw_handle.take() { + std::mem::forget(fw); + } + } +} + +/// Load a dynamic library with global symbol visibility where the platform +/// supports it (RTLD_GLOBAL on Unix). On Windows there is no equivalent flag, +/// so a normal load is performed. +unsafe fn load_library_global(path: &Path) -> Result { + #[cfg(unix)] + { + use libloading::os::unix::{Library, RTLD_GLOBAL, RTLD_NOW}; + Library::open(Some(path), RTLD_NOW | RTLD_GLOBAL) + .map(Into::into) + .map_err(|e| e.to_string()) + } + #[cfg(not(unix))] + { + libloading::Library::new(path).map_err(|e| e.to_string()) + } +} + +#[cfg(target_os = "linux")] +fn dlerror_message() -> String { + unsafe { + let err = libc::dlerror(); + if err.is_null() { + "unknown error".to_string() + } else { + std::ffi::CStr::from_ptr(err).to_string_lossy().to_string() + } + } +} + +#[cfg(target_os = "linux")] +fn preload_linux_gpu_dependencies(lib_dir: &Path) { + for lib_name in &["libepoxy.so.0", "libvirglrenderer.so.1"] { + let path = lib_dir.join(lib_name); + if path.exists() { + dlopen_global(&path); + } else { + // Not bundled: try the host's copy by soname. A GPU host has + // virglrenderer (and its X11/DRM/Mesa chain) installed system-wide; + // loading it RTLD_GLOBAL here lets libkrun's lazily-bound virgl + // symbols resolve when the GPU path runs. Best-effort — on a non-GPU + // host it simply isn't found, which is fine: those symbols are never + // called, and the libkrun NEEDED entry was stripped at package time. + dlopen_global_soname(lib_name); + } + } + + let server = lib_dir.join("virgl_render_server"); + if server.exists() && std::env::var("VIRGL_RENDER_SERVER_PATH").is_err() { + if let Some(s) = server.to_str() { + #[allow(deprecated)] + std::env::set_var("VIRGL_RENDER_SERVER_PATH", s); + } + } +} + +#[cfg(target_os = "linux")] +fn dlopen_global(path: &Path) -> bool { + use std::ffi::CString; + let Ok(path_c) = CString::new(path.to_string_lossy().as_bytes()) else { + return false; + }; + + unsafe { + let handle = libc::dlopen(path_c.as_ptr(), libc::RTLD_NOW | libc::RTLD_GLOBAL); + if handle.is_null() { + tracing::warn!( + path = %path.display(), + error = %dlerror_message(), + "failed to preload library" + ); + return false; + } + } + + true +} + +/// Load a library by soname (no path), letting the dynamic loader search the +/// host's standard library directories. Used to pick up a GPU host's +/// system-installed virglrenderer when it isn't bundled. Best-effort: on a +/// non-GPU host the library is absent, which is expected and not an error. +#[cfg(target_os = "linux")] +fn dlopen_global_soname(soname: &str) -> bool { + use std::ffi::CString; + let Ok(soname_c) = CString::new(soname) else { + return false; + }; + unsafe { !libc::dlopen(soname_c.as_ptr(), libc::RTLD_NOW | libc::RTLD_GLOBAL).is_null() } +} diff --git a/src/agent/launcher.rs b/src/agent/launcher.rs new file mode 100644 index 0000000..6f0aedf --- /dev/null +++ b/src/agent/launcher.rs @@ -0,0 +1,1703 @@ +//! Agent VM launcher. +//! +//! This module provides the low-level VM launching functionality. +//! All setup is done in the child process after fork, where +//! DYLD_LIBRARY_PATH is still available for dlopen. + +use crate::data::consts::{ENV_SMOLVM_KRUN_LOG_LEVEL, ENV_SMOLVM_LIB_DIR}; +use crate::data::disk::DiskFormat; +use crate::data::storage::HostMount; +use crate::error::{Error, Result}; +use crate::network::backend::COMPAT_NET_FEATURES; +use crate::network::backend::TSI_FEATURE_HIJACK_INET; +use crate::network::{plan_launch_network, EffectiveNetworkBackend}; +use crate::storage::{OverlayDisk, StorageDisk}; +use crate::util::{libkrun_filename, libkrunfw_filename}; + +use crate::agent::vsock_service; +use smolvm_network::PortMapping as VirtioPortMapping; +use smolvm_network::{start_virtio_network, GuestNetworkConfig, VirtioNetworkRuntime}; +use smolvm_protocol::{guest_env, ports}; +use socket2::Socket; +#[cfg(windows)] +use socket2::{Domain, SockAddr, Type}; +use std::ffi::CString; +#[cfg(unix)] +use std::os::fd::FromRawFd; +// `std::os::fd` does not exist on Windows. Keep the `RawFd` name working in +// signatures on both platforms via a portable alias. +#[cfg(unix)] +use std::os::fd::RawFd; +#[cfg(not(unix))] +#[allow(dead_code)] +type RawFd = std::os::raw::c_int; +use std::path::{Path, PathBuf}; + +use super::{KrunFunctions, PortMapping, VmResources}; + +/// Maximum number of CIDR entries held in the live egress allow-list. +/// Protects the muxer's per-packet O(n) scan from unbounded growth when +/// a host resolves to many IPs across many refresh cycles. +const EGRESS_CIDR_CAP: usize = 512; + +/// Hidden benchmark knob for root virtiofs DAX. +/// +/// Default configures the root virtiofs device with a 512 MB DAX window (the +/// same default the removed `krun_set_root` used). Set `SMOLVM_ROOTFS_DAX=0` to +/// use `krun_add_virtiofs3("/dev/root", ..., shm_size=0, read_only=false)`, +/// disabling the root DAX region for benchmarking. +const ENV_SMOLVM_ROOTFS_DAX: &str = "SMOLVM_ROOTFS_DAX"; + +/// Root virtiofs DAX window (512 MB), matching the default the removed +/// `krun_set_root` configured. DAX gives the host a coherent shared mapping of +/// the root fs so the guest agent's ready-marker write is visible to the host +/// immediately. Plain `krun_add_virtiofs` passes shm_size=0 (no DAX), dropping +/// virtiofs to writeback caching — the marker isn't seen until the multi-second +/// socket-probe grace, regressing boot time from ~hundreds of ms to ~5 s. +const ROOTFS_DAX_WINDOW: u64 = 1 << 29; + +/// The Arc type shared between the egress-refresh thread and libkrun's vsock muxer. +type EgressArc = std::sync::Arc>>; + +/// Disks to attach to the agent VM. +pub struct VmDisks<'a> { + /// Storage disk for OCI layers (/dev/vda in guest). + pub storage: &'a StorageDisk, + /// Optional overlay disk for persistent rootfs (/dev/vdb in guest). + pub overlay: Option<&'a OverlayDisk>, +} + +/// Find the directory containing libkrun/libkrunfw by checking explicit overrides and +/// paths relative to the current executable. +/// +/// Checks: +/// - `$SMOLVM_LIB_DIR` (explicit override for embedded runtimes) +/// - `/lib/` (distribution layout) +/// - `/../lib/` (alternative layout) +/// - `/../../lib/linux-/` (source tree dev builds) +pub fn find_lib_dir() -> Option { + let lib_names = [libkrun_filename(), libkrunfw_filename()]; + if let Ok(explicit_dir) = std::env::var(ENV_SMOLVM_LIB_DIR) { + let path = PathBuf::from(explicit_dir); + if lib_names.iter().all(|lib| path.join(lib).exists()) { + return path.canonicalize().ok().or(Some(path)); + } + + tracing::warn!( + path = %path.display(), + "{} does not contain the expected libkrun/libkrunfw libraries", ENV_SMOLVM_LIB_DIR + ); + } + + let exe = std::env::current_exe().ok()?; + let exe_dir = exe.parent()?; + + let candidates = [ + exe_dir.join("lib"), + // The Windows release ships krun.dll / libkrunfw.dll directly beside + // smolvm.exe (no lib/ subdir, no wrapper to set SMOLVM_LIB_DIR), matching + // the convention that Windows resolves DLLs from the executable's own + // directory. Harmless on Unix dists, where the libs live in lib/. + exe_dir.to_path_buf(), + exe_dir.join("../lib"), + exe_dir.join("../../lib"), + exe_dir.join(format!("../../lib/linux-{}", std::env::consts::ARCH)), + ]; + + for dir in &candidates { + if lib_names.iter().all(|lib| dir.join(lib).exists()) { + return dir.canonicalize().ok(); + } + } + + None +} + +/// A qcow2 copy-on-write overlay to create: `(overlay_path, base_path, base_format)`. +/// `base_path` must be absolute — it is written verbatim into the overlay header, +/// and imago resolves a relative backing path against the overlay's own directory. +pub type DiskOverlaySpec = (PathBuf, PathBuf, DiskFormat); + +/// Create the given qcow2 copy-on-write overlays, loading libkrun once for the +/// whole batch (overlay creation is a pure filesystem op, but the only place the +/// `krun_create_disk_overlay` symbol lives is libkrun). Stops at the first error. +pub fn create_disk_overlays(specs: &[DiskOverlaySpec]) -> Result<()> { + if specs.is_empty() { + return Ok(()); + } + let lib_dir = find_lib_dir().ok_or_else(|| { + Error::agent( + "create disk overlay", + "could not locate the libkrun library directory", + ) + })?; + let krun = unsafe { KrunFunctions::load(&lib_dir) } + .map_err(|e| Error::agent("create disk overlay", e))?; + let create = krun.create_disk_overlay.ok_or_else(|| { + Error::agent( + "create disk overlay", + "libkrun is missing krun_create_disk_overlay (rebuild libkrun)", + ) + })?; + + for (overlay, base, base_format) in specs { + let overlay_c = path_to_cstring(overlay)?; + let base_c = path_to_cstring(base)?; + let rc = unsafe { + create( + overlay_c.as_ptr(), + base_c.as_ptr(), + base_format.to_krun_u32(), + ) + }; + if rc < 0 { + return Err(Error::agent( + "create disk overlay", + format!( + "krun_create_disk_overlay failed (rc={rc}) for {} <- {}", + overlay.display(), + base.display() + ), + )); + } + } + Ok(()) +} + +/// Launch the agent VM (call in the forked child process). +/// +/// This function sets up and starts the VM in a single call. +/// It should be called in the child process after fork, where +/// DYLD_LIBRARY_PATH is still available for dlopen to find libkrunfw. +/// +/// Optional features for VM launch (SSH agent, DNS filtering, etc.). +/// +/// Groups optional capabilities that don't affect core VM operation. +/// New features should be added here rather than as additional parameters +/// on manager/launcher functions. +#[derive(Debug, Clone, Default)] +pub struct LaunchFeatures { + /// Host SSH agent socket path for forwarding into the guest. + pub ssh_agent_socket: Option, + /// Enable CUDA-over-vsock: smolvm starts a host CUDA server and the guest + /// remotes its CUDA Driver-API calls to the host GPU. + pub cuda: bool, + /// Hostnames for DNS filtering. When set, the host starts a DNS filter + /// listener and the guest agent proxies DNS queries through it. + pub dns_filter_hosts: Option>, + /// Pre-extracted OCI layer directory for machines created from .smolmachine. + /// When set, the launcher mounts this directory via virtiofs so the agent + /// can use pre-extracted layers instead of pulling from a registry. + pub packed_layers_dir: Option, + /// Root-owned shared pack copy's OCI layers dir (`_shared//layers`) + /// to present at `packed_layers_dir` via a per-VM idmapped bind mount. Set by + /// [`with_packed_layers`](LaunchFeatures::with_packed_layers) when create + /// wrote a shared pointer; the manager keeps it only when the per-VM uid drop + /// is active (else it collapses `packed_layers_dir` onto the shared copy). The + /// `layers/` subdir is presented (not the store root) so the guest stacks only + /// the image layers, never the sibling `agent-rootfs/`. + pub pack_idmap_source: Option, + /// Additional disk images to attach to the VM (path, read_only, format). + /// Appear as /dev/vdc, /dev/vdd, ... after the storage and overlay disks. + pub extra_disks: Vec<(std::path::PathBuf, bool, DiskFormat)>, + /// Start as a fork base: back guest RAM with a memfd (copy-on-write + /// cloneable) and expose `control_socket` so the machine can be forked. + pub forkable: bool, + /// Boot this VM as a fork clone, restoring from the golden's snapshot at + /// this directory (set on the clone; `None` for a normal cold boot). + pub snapshot_dir: Option, + /// Control socket path for a forkable machine (pause/resume/checkpoint/FORK). + pub control_socket: Option, + /// Override the parent-death watchdog. `None` = default (arm it iff a + /// separate boot binary is used, i.e. an in-process SDK embedder whose VM + /// must die with it). `Some(false)` forces it off — for a CLI that sets + /// `SMOLVM_BOOT_BINARY` (so `current_exe` need not handle `_boot-vm`) yet + /// DETACHES the VM to persist after the CLI exits (e.g. `smol start`/`fork`). + pub watch_parent: Option, +} + +impl LaunchFeatures { + /// Fold an image's own registry into the enforced DNS egress filter so a + /// scoped machine's in-guest base-image pull isn't blocked by its own + /// allow-list. + /// + /// An image-based machine pulls its base image inside the guest on first + /// boot, subject to `dns_filter_hosts` (applied at VM boot). Scoping egress + /// to only, say, an LLM API would otherwise fail the pull with a DNS-lookup + /// error for the registry. Call this only when a fresh remote pull will + /// happen (first boot, registry-sourced). No-op when the filter is unset or + /// empty, when layers are packed/local (`uses_packed_layers`), or when the + /// reference is a `local:` source — none of which pull over the network. + /// + /// Only the enforced launch policy is widened; the caller's stored + /// `dns_filter_hosts` on the record keeps exactly the hosts the user asked + /// for. Mirrors the control plane's `create_body` registry-fold. + pub fn allow_image_pull_egress(&mut self, image: Option<&str>, uses_packed_layers: bool) { + let Some(hosts) = self.dns_filter_hosts.as_mut() else { + return; + }; + if hosts.is_empty() || uses_packed_layers { + return; + } + let Some(image) = image else { + return; + }; + if crate::data::image_source::is_local_ref(image) { + return; + } + for host in crate::registry::registry_pull_hosts(image) { + if !hosts.iter().any(|h| h.eq_ignore_ascii_case(&host)) { + hosts.push(host); + } + } + } + + /// Wire pre-extracted OCI layers for a machine created from a `.smolmachine`. + /// + /// `layers_cache_dir` is the machine's OWN extraction directory (under its + /// [`vm_data_dir`](crate::agent::vm_data_dir), via + /// [`machine_layers_cache_dir`](crate::agent::machine_layers_cache_dir)), not + /// the shared content-addressed pack cache. The bundle is extracted there + /// once at create time, so every subsequent start is independent of the + /// original `.smolmachine` file. When `source_smolmachine` is `None` the + /// machine is image/registry-sourced and `self` is returned unchanged. + /// + /// Normal path: the layers are already extracted, so this only acquires a + /// lease (re-mounting the case-sensitive volume on macOS; a no-op on Linux) + /// and points `packed_layers_dir` at it — no dependency on the sidecar. + /// Fallback path: if the per-machine directory has no extracted layers (a + /// machine created before this layout, or an interrupted create), extract + /// from the `source_smolmachine` sidecar, which must still exist in that case. + /// + /// This is the single source of truth shared by every start path — the CLI + /// `machine start` and the API start/ensure/restart handlers — so they + /// cannot drift apart and silently drop the bundled layers. + /// + /// Performs blocking filesystem work; on async paths call it from within a + /// `spawn_blocking` context. + pub fn with_packed_layers( + mut self, + layers_cache_dir: &Path, + source_smolmachine: Option<&str>, + ) -> Result { + let Some(sidecar_path) = source_smolmachine else { + return Ok(self); + }; + + // Shared pack store: if create extracted the pack into the node's shared + // content-addressed store and dropped a pointer beside this machine, the + // per-machine `pack` dir is an empty mountpoint. Point `packed_layers_dir` + // at it and carry the shared copy as the idmap source; the manager keeps + // the idmap only when the per-VM uid drop is active (else it collapses + // `packed_layers_dir` onto the shared copy directly). No lease — the + // shared copy is never the macOS case-sensitive volume (Linux-only path). + // + // The pointer is the store ROOT (`_shared/`), which holds the + // sidecar's whole tree — `agent-rootfs/`, `layers/`, `storage.ext4` as + // siblings. The guest stacks OCI image layers from the `layers/` subdir + // (its `layer-order` index + digest dirs live there), exactly like the + // per-machine path's `/layers`. Present that subdir, NOT the root: + // otherwise the guest name-sorts the root's entries and mis-stacks + // `agent-rootfs/` as an image layer alongside `layers/`, surfacing the + // pack's internals (`/`, `.tar`, `layer-order`) at the + // container `/` and running the agent rootfs instead of the real image. + if let Some(shared) = super::read_shared_pack_pointer(layers_cache_dir) { + let layers = shared.join("layers"); + self.packed_layers_dir = Some(layers_cache_dir.to_path_buf()); + self.pack_idmap_source = Some(if layers.is_dir() { layers } else { shared }); + return Ok(self); + } + + if !smolvm_pack::extract::is_extracted(layers_cache_dir) { + // Fallback: layers not yet extracted into this machine's own dir + // (pre-this-layout machine, or an interrupted create). Extract from + // the source bundle, which must still be present in that case. + let sidecar = Path::new(sidecar_path); + if !sidecar.exists() { + return Err(Error::agent( + "start machine", + format!( + "packed layers are not extracted for this machine and its \ + source .smolmachine is missing: {}\nRe-create the machine \ + from the bundle.", + sidecar_path + ), + )); + } + let footer = smolvm_pack::packer::read_footer_from_sidecar(sidecar) + .map_err(|e| Error::agent("read sidecar footer", e.to_string()))?; + smolvm_pack::extract::extract_sidecar(sidecar, layers_cache_dir, &footer, false, false) + .map_err(|e| Error::agent("extract sidecar", e.to_string()))?; + } + + let layers_lease = smolvm_pack::extract::acquire_layers_lease(layers_cache_dir, false) + .map_err(|e| Error::agent("acquire layers lease", e.to_string()))?; + self.packed_layers_dir = Some(layers_lease.path.clone()); + // Leak the lease so the case-sensitive layers volume stays mounted for + // the VM's lifetime (macOS only; a no-op on Linux). Unlike the previous + // shared-cache design, this volume is owned 1:1 by the machine: the stop + // and delete handlers detach it unconditionally via + // `force_detach_layers_volume`, so no co-tenant can be relying on it and + // no lease outlives the machine. + std::mem::forget(layers_lease); + + Ok(self) + } +} + +/// Configuration for launching an agent VM. +pub struct LaunchConfig<'a> { + /// Path to the agent rootfs directory. + pub rootfs_path: &'a Path, + /// Storage and overlay disk handles. + pub disks: &'a VmDisks<'a>, + /// Path to the vsock Unix socket for the control channel. + pub vsock_socket: &'a Path, + /// Optional path to write console output. + pub console_log: Option<&'a Path>, + /// Host directory mounts to expose to the guest. + pub mounts: &'a [HostMount], + /// Port mappings (host:guest). + pub port_mappings: &'a [PortMapping], + /// VM resources (CPU, memory, network, disk sizes). + pub resources: VmResources, + /// Host SSH agent socket path for forwarding into the guest. + pub ssh_agent_socket: Option<&'a Path>, + /// Host DNS filter socket path. When set, the guest DNS proxy forwards + /// queries over vsock to this socket for filtering. + pub dns_filter_socket: Option<&'a Path>, + /// Host CUDA-over-vsock server socket (experimental). When set, the guest + /// CUDA client connects out to this AF_UNIX path and the host server runs + /// the calls on the NVIDIA GPU. Resolved at the boot-config boundary (the + /// subprocess reads `SMOLVM_CUDA_SOCK`) so the launcher stays policy-free. + pub cuda_socket: Option<&'a Path>, + /// Pre-extracted OCI layers directory for .smolmachine-sourced machines. + /// Mounted via virtiofs as "smolvm_layers" so the agent uses packed layers. + pub packed_layers_dir: Option<&'a Path>, + /// Additional disk images (path, read_only, format). Appear as /dev/vdc, /dev/vdd, ... + pub extra_disks: &'a [(std::path::PathBuf, bool, DiskFormat)], + /// Whether DNS filtering was configured for this launch, even if the + /// host-side proxy socket could not be created. + pub dns_filter_enabled: bool, + /// Hostnames to periodically re-resolve for the live egress policy. + /// When set, a background thread re-resolves these every 5 minutes and + /// atomically replaces the CIDR list via the Arc handle obtained from + /// libkrun. This keeps the egress allow-list accurate for long-running VMs + /// hitting CDN-backed hosts whose IPs rotate. + pub egress_refresh_hosts: Option>, + /// Where to flush this VM's cumulative egress byte count (virtio-net only). + /// A background thread writes it here every few seconds; serve reads it to + /// surface `egressBytes` in the machine info, the same per-VM-dir bridge the + /// vsock/console paths use. `None` disables egress telemetry (e.g. TSI). + pub egress_telemetry: Option<&'a Path>, +} + +/// Launch the agent VM using libkrun. +/// +/// This function never returns on success. +pub fn launch_agent_vm(config: &LaunchConfig<'_>) -> Result<()> { + let t0 = std::time::Instant::now(); + + // Emit boot timing to stderr (captured in the startup error log by the + // subprocess's stdio redirect) when INFO logging is enabled. + // tracing_subscriber writes to stdout by default, but the subprocess has + // stdout=/dev/null; stderr is the only channel that reaches the log file. + macro_rules! boot_timing { + ($label:expr) => { + if tracing::enabled!(tracing::Level::INFO) { + eprintln!("[boot] {:25} {}ms", $label, t0.elapsed().as_millis()); + } + }; + } + + // `egress_telemetry` is consumed only by the unix-only virtio-net path. + #[cfg_attr(not(unix), allow(unused_variables))] + let LaunchConfig { + rootfs_path, + disks, + vsock_socket, + console_log, + mounts, + port_mappings, + resources, + ssh_agent_socket, + dns_filter_socket, + cuda_socket, + packed_layers_dir, + extra_disks, + dns_filter_enabled, + egress_refresh_hosts, + egress_telemetry, + } = config; + + crate::network::validate_requested_network_backend(resources, None, port_mappings.len())?; + + // Raise file descriptor limits + raise_fd_limits(); + + let lib_dir = find_lib_dir().ok_or_else(|| { + Error::agent( + "find libraries", + "libkrun/libkrunfw not found. Install smolvm with bundled libraries or set SMOLVM_LIB_DIR.", + ) + })?; + let krun = + unsafe { KrunFunctions::load(&lib_dir) }.map_err(|e| Error::agent("load libkrun", e))?; + boot_timing!("dylib loaded"); + + // Pre-read the agent binary into the OS page cache so the virtiofs thread + // can serve the guest's first exec without waiting for disk I/O. + // Runs concurrently with krun context setup below — by the time + // krun_start_enter is called, the file is already in page cache. + { + let agent_bin = rootfs_path.join("usr/local/bin/smolvm-agent"); + if agent_bin.exists() { + let _ = std::thread::Builder::new() + .name("agent-preread".into()) + .spawn(move || { + let _ = std::fs::read(&agent_bin); + }); + } + } + + unsafe { + let krun_set_log_level = krun.set_log_level; + let krun_create_ctx = krun.create_ctx; + let krun_free_ctx = krun.free_ctx; + let krun_set_vm_config = krun.set_vm_config; + let krun_set_workdir = krun.set_workdir; + let krun_set_exec = krun.set_exec; + let krun_add_disk2 = krun.add_disk2; + let krun_add_vsock_port2 = krun.add_vsock_port2; + let krun_set_port_map = krun.set_port_map; + let krun_add_virtiofs = krun.add_virtiofs; + let krun_add_virtiofs3 = krun.add_virtiofs3; + let krun_start_enter = krun.start_enter; + let krun_add_vsock = krun.add_vsock; + let krun_get_guest_ram = krun.get_guest_ram; + + // Set log level (0 = off, 1 = error, 2 = warn, 3 = info, 4 = debug) + // Enable debug logging to trace vsock timing issues + let log_level = std::env::var(ENV_SMOLVM_KRUN_LOG_LEVEL) + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + krun_set_log_level(log_level); + + // Create VM context + let ctx = krun_create_ctx(); + if ctx < 0 { + return Err(Error::agent("create vm context", "krun_create_ctx failed")); + } + let ctx = ctx as u32; + boot_timing!("ctx created"); + + // Set VM config + if krun_set_vm_config(ctx, resources.cpus, resources.memory_mib) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent("configure vm", "krun_set_vm_config failed")); + } + + // Enable GPU if requested (virgl for OpenGL + Venus for Vulkan via virtio-gpu). + // Requires libkrun built with `gpu` feature and host virglrenderer. + // On macOS, also requires MoltenVK (Vulkan → Metal translation). + if resources.gpu { + let virgl_flags = super::gpu_virgl_flags(); + // Size the GPU shared-memory region. Caller may override + // via `--gpu-vram ` (CLI) or `gpu_vram = N` (Smolfile); + // default is `DEFAULT_GPU_VRAM_MIB`. + let vram_mib = resources.effective_gpu_vram_mib(); + let vram_bytes: u64 = (vram_mib as u64) * crate::data::consts::BYTES_PER_MIB; + + // Resolve krun_set_gpu_options2 dynamically — it may not exist + // if libkrun was built without the `gpu` feature. + let set_gpu = match krun.set_gpu_options2 { + Some(f) => f, + None => { + krun_free_ctx(ctx); + return Err(Error::agent( + "configure gpu", + "libkrun was built without GPU support (krun_set_gpu_options2 not found). \ + Rebuild libkrun with GPU=1 — see project README for details.", + )); + } + }; + + let ret = set_gpu(ctx, virgl_flags, vram_bytes); + if ret < 0 { + krun_free_ctx(ctx); + return Err(Error::agent( + "configure gpu", + format!("krun_set_gpu_options2 failed (ret={}). Check that virglrenderer is installed.", ret), + )); + } + tracing::info!("GPU enabled (Venus/Vulkan via virtio-gpu)"); + } + + // Helper: evaluate a fallible expression, freeing ctx if it fails. + // Replaces bare `?` which would leak the libkrun context. + macro_rules! try_or_free_ctx { + ($expr:expr, $op:expr, $msg:expr) => { + match $expr { + Ok(val) => val, + Err(_) => { + krun_free_ctx(ctx); + return Err(Error::agent($op, $msg)); + } + } + }; + } + + // Set root filesystem via the root virtiofs tag ("/dev/root"). + // + // Upstream libkrun removed krun_set_root in favor of krun_add_virtiofs* + // with KRUN_FS_ROOT_TAG. Default path: krun_add_virtiofs, preserving the + // established rootfs DAX defaults. Benchmark path: SMOLVM_ROOTFS_DAX=0 + // uses krun_add_virtiofs3 with shm_size=0, disabling the root DAX region + // while keeping the root read-write. + let root = try_or_free_ctx!( + path_to_cstring(rootfs_path), + "set rootfs", + "path contains null byte" + ); + let root_tag = cstr("/dev/root"); + if rootfs_dax_disabled() { + let Some(add_virtiofs3) = krun_add_virtiofs3 else { + krun_free_ctx(ctx); + return Err(Error::agent( + "set rootfs", + "SMOLVM_ROOTFS_DAX=0 requires libkrun with krun_add_virtiofs3", + )); + }; + + if add_virtiofs3(ctx, root_tag.as_ptr(), root.as_ptr(), 0, false) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent( + "set rootfs", + "krun_add_virtiofs3 failed for root filesystem", + )); + } + tracing::info!("rootfs configured via virtiofs without DAX"); + } else { + // Default: restore the 512 MB root DAX window the removed krun_set_root + // configured. Plain krun_add_virtiofs passes shm_size=0 (no DAX), which + // drops virtiofs to writeback caching and hides the guest's ready-marker + // write from the host until the socket-probe grace — a boot-time regression. + let Some(add_virtiofs3) = krun_add_virtiofs3 else { + krun_free_ctx(ctx); + return Err(Error::agent( + "set rootfs", + "root DAX requires libkrun with krun_add_virtiofs3", + )); + }; + if add_virtiofs3( + ctx, + root_tag.as_ptr(), + root.as_ptr(), + ROOTFS_DAX_WINDOW, + false, + ) < 0 + { + krun_free_ctx(ctx); + return Err(Error::agent( + "set rootfs", + "krun_add_virtiofs3 failed for root filesystem", + )); + } + } + + let network_plan = select_network_plan(resources, *dns_filter_enabled, port_mappings.len()); + + // `mut` is only needed on unix (the VirtioNet arm assigns it); on + // Windows the runtime is owned by the accept thread, so the launcher's + // binding stays `None`. + #[cfg_attr(not(unix), allow(unused_mut))] + let mut virtio_network_runtime: Option = None; + let guest_network: Option = match network_plan.backend { + EffectiveNetworkBackend::None => { + // Upstream libkrun no longer creates an implicit vsock (the old + // krun_disable_implicit_vsock is gone), so just add it explicitly. + if krun_add_vsock(ctx, 0) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent("configure vsock", "krun_add_vsock failed")); + } + + tracing::debug!("configured vsock without guest networking"); + None + } + EffectiveNetworkBackend::Tsi => { + if krun_add_vsock(ctx, TSI_FEATURE_HIJACK_INET) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent( + "configure vsock", + "krun_add_vsock with TSI failed", + )); + } + + let port_cstrings: Vec = port_mappings + .iter() + .map(|p| { + CString::new(format!("{}:{}", p.host, p.guest)) + .expect("port mapping format cannot contain null bytes") + }) + .collect(); + let mut port_ptrs: Vec<*const libc::c_char> = + port_cstrings.iter().map(|s| s.as_ptr()).collect(); + port_ptrs.push(std::ptr::null()); + + if krun_set_port_map(ctx, port_ptrs.as_ptr()) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent("set port mapping", "krun_set_port_map failed")); + } + + // Egress policy: static CIDRs plus DNS allow-host filtering + // enforced inside libkrun. When allow-hosts are set, the guest's + // UDP DNS queries to port 53 are intercepted and forwarded only + // to the host-trusted resolver; A/AAAA answers are learned as + // temporary allowed IPs. The guest-side DNS proxy is left off + // (see below) so those queries leave as real UDP datagrams. + let egress_hosts = egress_refresh_hosts.clone().unwrap_or_default(); + if resources.allowed_cidrs.is_some() || !egress_hosts.is_empty() { + let Some(set_egress) = krun.set_egress_policy else { + krun_free_ctx(ctx); + return Err(Error::agent( + "set egress policy", + "libkrun does not support egress policy (krun_set_egress_policy not found). \ + Update libkrun or remove --allow-cidr/--allow-host flags.", + )); + }; + + // CIDRs (plus the resolver IP via ensure_dns_in_cidrs) — a + // null-terminated array. + let mut all_cidrs = resources.allowed_cidrs.clone().unwrap_or_default(); + crate::data::network::ensure_dns_in_cidrs(&mut all_cidrs); + let cidr_cstrings: Vec = all_cidrs + .iter() + .map(|c| CString::new(c.as_str()).expect("CIDR cannot contain null bytes")) + .collect(); + let mut cidr_ptrs: Vec<*const libc::c_char> = + cidr_cstrings.iter().map(|s| s.as_ptr()).collect(); + cidr_ptrs.push(std::ptr::null()); + + // Allow-host list + trusted resolver, only when hosts are set. + let host_cstrings: Vec = egress_hosts + .iter() + .map(|h| CString::new(h.as_str()).expect("host cannot contain null bytes")) + .collect(); + let mut host_ptrs: Vec<*const libc::c_char> = + host_cstrings.iter().map(|s| s.as_ptr()).collect(); + host_ptrs.push(std::ptr::null()); + + let resolver_cstring = + CString::new(crate::data::network::default_dns_addr().to_string()) + .expect("resolver IP has no null bytes"); + let resolver_ptrs: Vec<*const libc::c_char> = + vec![resolver_cstring.as_ptr(), std::ptr::null()]; + + let (host_arg, resolver_arg) = if egress_hosts.is_empty() { + (std::ptr::null(), std::ptr::null()) + } else { + (host_ptrs.as_ptr(), resolver_ptrs.as_ptr()) + }; + + if set_egress(ctx, cidr_ptrs.as_ptr(), host_arg, resolver_arg) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent( + "set egress policy", + "krun_set_egress_policy failed", + )); + } + } + + tracing::info!("network backend: tsi"); + None + } + EffectiveNetworkBackend::VirtioNet => { + let add_net_unixstream = krun.add_net_unixstream.ok_or_else(|| { + Error::agent( + "configure virtio-net", + "libkrun does not expose krun_add_net_unixstream; update libkrun or use --net-backend tsi", + ) + })?; + // virtio-net carries guest networking, but the host-guest control + // channel still rides vsock. Upstream libkrun no longer creates an + // implicit vsock, so add it explicitly (no TSI hijacking — virtio-net + // owns the network path); otherwise krun_add_vsock_port2 below fails + // with ENODEV. + if krun_add_vsock(ctx, 0) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent("configure vsock", "krun_add_vsock failed")); + } + + let mut guest_network = GuestNetworkConfig::default(); + // A custom resolver (--dns) becomes the gateway's upstream: the + // guest still points at the gateway (100.96.0.1), which forwards + // queries to this address instead of the default. + if let Some(dns) = resources.dns { + guest_network.upstream_dns = dns; + } + let mut guest_mac = guest_network.guest_mac; + + let virtio_port_mappings: Vec = port_mappings + .iter() + .map(|mapping| VirtioPortMapping::new(mapping.host, mapping.guest)) + .collect(); + let egress = smolvm_network::EgressPolicy::new( + resources.allowed_cidrs.as_deref(), + egress_refresh_hosts.as_deref(), + ); + let egress_path = egress_telemetry.map(|p| p.to_path_buf()); + + // The host and guest ends of the virtio-net channel are an AF_UNIX + // stream. On Unix we hand libkrun one end of a socketpair fd and run + // the gateway on the other immediately. Windows has no socketpair + // for AF_UNIX, so we bind a listener on a per-VM path, hand libkrun + // the path, and accept its connection (made when the VM boots inside + // the blocking `krun_start_enter`) on a background thread. + #[cfg(unix)] + { + let (host_fd, guest_fd) = create_unix_stream_pair().map_err(|e| { + Error::agent("configure virtio-net", format!("socketpair failed: {e}")) + })?; + // SAFETY: ownership of the host-side socketpair fd transfers + // here (already inside the function's outer `unsafe` block). + let host_stream = Socket::from_raw_fd(host_fd); + let runtime = match start_virtio_network( + host_stream, + guest_network, + &virtio_port_mappings, + egress, + ) { + Ok(runtime) => runtime, + Err(err) => { + libc::close(guest_fd); + krun_free_ctx(ctx); + return Err(Error::agent( + "configure virtio-net", + format!("failed to start virtio network runtime: {err}"), + )); + } + }; + + if add_net_unixstream( + ctx, + std::ptr::null(), + guest_fd, + guest_mac.as_mut_ptr(), + COMPAT_NET_FEATURES, + 0, + ) < 0 + { + libc::close(guest_fd); + krun_free_ctx(ctx); + return Err(Error::agent( + "configure virtio-net", + "krun_add_net_unixstream failed", + )); + } + + // Flush this NIC's egress counter to the per-VM dir so serve can + // bill it (parity with how disk size reaches the node API). + if let Some(path) = egress_path { + crate::agent::manager::spawn_egress_flush(path, runtime.egress_counter()); + } + virtio_network_runtime = Some(runtime); + } + #[cfg(windows)] + { + // Per-VM AF_UNIX path for the net channel, a sibling of the + // agent-control vsock socket (already a working AF_UNIX path). + let net_sock_path = vsock_socket.with_extension("net"); + let listener = bind_unix_listener(&net_sock_path).map_err(|e| { + krun_free_ctx(ctx); + Error::agent( + "configure virtio-net", + format!("failed to bind virtio-net socket: {e}"), + ) + })?; + let path_c = try_or_free_ctx!( + path_to_cstring(&net_sock_path), + "configure virtio-net", + "virtio-net socket path contains null byte" + ); + if add_net_unixstream( + ctx, + path_c.as_ptr(), + -1, + guest_mac.as_mut_ptr(), + COMPAT_NET_FEATURES, + 0, + ) < 0 + { + krun_free_ctx(ctx); + return Err(Error::agent( + "configure virtio-net", + "krun_add_net_unixstream failed", + )); + } + + // libkrun connects to the path while the VM boots inside the + // blocking krun_start_enter, so accept on a background thread. + // The accepted runtime owns its worker threads and parks here + // until libkrun closes the stream (VM exit) for a clean teardown. + let spawn = std::thread::Builder::new() + .name("smolvm-net-accept".into()) + .spawn(move || match listener.accept() { + Ok((sock, _)) => match start_virtio_network( + sock, + guest_network, + &virtio_port_mappings, + egress, + ) { + Ok(runtime) => { + if let Some(path) = egress_path { + crate::agent::manager::spawn_egress_flush( + path, + runtime.egress_counter(), + ); + } + runtime.block_until_shutdown(); + } + Err(err) => { + tracing::error!(error = %err, "virtio-net runtime failed to start"); + } + }, + Err(err) => { + tracing::warn!(error = %err, "virtio-net accept failed"); + } + }); + if let Err(e) = spawn { + krun_free_ctx(ctx); + return Err(Error::agent( + "configure virtio-net", + format!("failed to spawn virtio-net accept thread: {e}"), + )); + } + } + + tracing::info!("network backend: virtio-net"); + Some(guest_network) + } + }; + + // Add storage disk (critical - VM needs storage to function) + // This is the first disk → /dev/vda in guest + let block_id = cstr("storage"); + let disk_path = try_or_free_ctx!( + path_to_cstring(disks.storage.path()), + "add storage disk", + "path contains null byte" + ); + let storage_format = disks.storage.format().to_krun_u32(); + if krun_add_disk2( + ctx, + block_id.as_ptr(), + disk_path.as_ptr(), + storage_format, + false, + ) < 0 + { + krun_free_ctx(ctx); + return Err(Error::agent( + "add storage disk", + "krun_add_disk2 failed - VM cannot function without storage", + )); + } + + // Add overlay disk for persistent rootfs changes (optional) + // This is the second disk → /dev/vdb in guest + if let Some(overlay) = disks.overlay { + let overlay_id = cstr("overlay"); + let overlay_path = try_or_free_ctx!( + path_to_cstring(overlay.path()), + "add overlay disk", + "path contains null byte" + ); + let overlay_format = overlay.format().to_krun_u32(); + if krun_add_disk2( + ctx, + overlay_id.as_ptr(), + overlay_path.as_ptr(), + overlay_format, + false, + ) < 0 + { + krun_free_ctx(ctx); + return Err(Error::agent( + "add overlay disk", + "krun_add_disk2 failed for rootfs overlay", + )); + } + } + + // Add extra disks (e.g., source VM storage for --from-vm export) + // These appear as /dev/vdc, /dev/vdd, ... after storage and overlay + for (i, (disk_path, read_only, format)) in extra_disks.iter().enumerate() { + let block_id_str = format!("extra{}", i); + let block_id = try_or_free_ctx!( + CString::new(block_id_str.as_str()), + "add extra disk", + "block id contains null byte" + ); + let path = try_or_free_ctx!( + path_to_cstring(disk_path), + "add extra disk", + "path contains null byte" + ); + if krun_add_disk2( + ctx, + block_id.as_ptr(), + path.as_ptr(), + format.to_krun_u32(), + *read_only, + ) < 0 + { + krun_free_ctx(ctx); + return Err(Error::agent( + "add extra disk", + format!("krun_add_disk2 failed for extra disk {}", i), + )); + } + tracing::debug!(disk = i, path = %disk_path.display(), read_only, "added extra disk"); + } + + // Add vsock port for control channel (critical - host-guest communication) + let socket_path = try_or_free_ctx!( + path_to_cstring(vsock_socket), + "add vsock port", + "path contains null byte" + ); + if krun_add_vsock_port2(ctx, ports::AGENT_CONTROL, socket_path.as_ptr(), true) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent( + "add vsock port", + "krun_add_vsock_port2 failed - control channel required for host-guest communication", + )); + } + + // Guest↔host vsock services (SSH agent, DNS filter, CUDA, …). The set + // and its wiring live in `vsock_service`; here we just register the + // port for each one enabled for this launch. Adding a capability needs + // no new control flow in this function. The `active_vsock` set is reused + // below to inject each service's guest-side activation env vars, so the + // host and guest sides cannot drift. + let vsock_inputs = vsock_service::VsockServiceInputs { + ssh_agent_socket: ssh_agent_socket.as_deref(), + dns_filter_socket: dns_filter_socket.as_deref(), + cuda_socket: cuda_socket.as_deref(), + }; + let active_vsock: Vec<_> = vsock_service::registry() + .iter() + .filter_map(|svc| svc.resolve(&vsock_inputs)) + .collect(); + for svc in &active_vsock { + // An egress port must never shadow the required control channel. + debug_assert_ne!( + svc.port, + ports::AGENT_CONTROL, + "{} would shadow the agent control channel", + svc.name + ); + let sock_path = try_or_free_ctx!( + path_to_cstring(svc.socket), + "add vsock port", + "path contains null byte" + ); + if krun_add_vsock_port2(ctx, svc.port, sock_path.as_ptr(), svc.listen) < 0 { + tracing::warn!( + "failed to add {} vsock port {} — disabled", + svc.name, + svc.port + ); + } else { + tracing::info!("{} enabled on vsock port {}", svc.name, svc.port); + } + } + + // Redirect console output to a file if specified, via the upstream + // virtio-console API (krun_set_console_output was removed). + if let Some(log_path) = console_log { + if krun.console_output_to_file(ctx, log_path) < 0 { + // Expected on Windows (fd-based console redirection is a no-op + // there); don't let a benign WARN mask the real boot failure the + // readiness monitor reports. See `boot_failure_reason`. + #[cfg(windows)] + tracing::debug!( + "guest console not captured on Windows (fd redirection unsupported)" + ); + #[cfg(not(windows))] + tracing::warn!("failed to set console output"); + } + } + + // Register a control socket (pause/resume/checkpoint/restore) when + // requested via SMOLVM_CONTROL_SOCKET. Best-effort: a missing symbol + // (older libkrun) or a failure just leaves the VM without a control + // channel rather than aborting the boot. + if let Ok(ctl_path) = std::env::var("SMOLVM_CONTROL_SOCKET") { + if !ctl_path.is_empty() { + match krun.set_control_socket { + Some(set_control_socket) => match CString::new(ctl_path.clone()) { + Ok(ctl_c) => { + let ret = set_control_socket(ctx, ctl_c.as_ptr()); + if ret < 0 { + tracing::warn!("krun_set_control_socket failed: {ret}"); + } else { + tracing::info!(socket = %ctl_path, "control socket enabled"); + } + } + Err(_) => tracing::warn!("control socket path contains null byte"), + }, + None => tracing::warn!( + "SMOLVM_CONTROL_SOCKET set but libkrun lacks krun_set_control_socket" + ), + } + } + } + + // Fork clone: boot from a snapshot dir (CoW-map a golden VM's RAM + + // restore state) instead of cold-booting, when SMOLVM_SNAPSHOT_DIR is set. + if let Ok(snap_dir) = std::env::var("SMOLVM_SNAPSHOT_DIR") { + if !snap_dir.is_empty() { + match krun.set_snapshot { + Some(set_snapshot) => match CString::new(snap_dir.clone()) { + Ok(dir_c) => { + let ret = set_snapshot(ctx, dir_c.as_ptr()); + if ret < 0 { + tracing::error!("krun_set_snapshot failed: {ret}"); + } else { + tracing::info!(dir = %snap_dir, "booting as fork clone from snapshot"); + } + } + Err(_) => tracing::warn!("snapshot dir contains null byte"), + }, + None => tracing::warn!( + "SMOLVM_SNAPSHOT_DIR set but libkrun lacks krun_set_snapshot" + ), + } + } + } + + // Add virtiofs mounts + // Each mount gets a tag like "smolvm0", "smolvm1", etc. + // The guest must mount these manually (or via the agent) + for (i, mount) in mounts.iter().enumerate() { + let mount_tag = HostMount::mount_tag(i); + let tag = try_or_free_ctx!( + CString::new(mount_tag.clone()), + "configure mount", + "mount tag contains null byte" + ); + let host_path = try_or_free_ctx!( + path_to_cstring(&mount.source), + "configure mount", + "mount path contains null byte" + ); + + tracing::debug!( + tag = %mount_tag, + host = %mount.source.display(), + guest = %mount.target.display(), + read_only = mount.read_only, + "adding virtiofs mount" + ); + + // Read-only mounts must be enforced host-side by the virtiofs + // device (krun_add_virtiofs3's read_only flag), not only by the + // guest's bind-remount: a root process in the guest can undo a + // guest-side remount, but it cannot make the host server accept + // writes. Fail closed if the symbol is missing rather than + // silently attaching the mount writable. + if mount.read_only { + let Some(add_virtiofs3) = krun_add_virtiofs3 else { + krun_free_ctx(ctx); + return Err(Error::agent( + "add virtiofs mount", + "read-only mounts require libkrun with krun_add_virtiofs3", + )); + }; + if add_virtiofs3(ctx, tag.as_ptr(), host_path.as_ptr(), 0, true) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent( + "add virtiofs mount", + format!( + "krun_add_virtiofs3 failed for '{}' - requested mount cannot be attached", + mount.source.display() + ), + )); + } + } else if krun_add_virtiofs(ctx, tag.as_ptr(), host_path.as_ptr()) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent( + "add virtiofs mount", + format!( + "krun_add_virtiofs failed for '{}' - requested mount cannot be attached", + mount.source.display() + ), + )); + } + } + + // Mount pre-extracted OCI layers for .smolmachine-sourced machines. + // The agent detects this via SMOLVM_PACKED_LAYERS and uses the layers + // as container overlay lowerdirs instead of pulling from a registry. + if let Some(layers_dir) = packed_layers_dir { + if layers_dir.exists() { + let tag = cstr("smolvm_layers"); + let host_path = path_to_cstring(layers_dir)?; + if krun_add_virtiofs(ctx, tag.as_ptr(), host_path.as_ptr()) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent( + "add packed layers virtiofs", + "krun_add_virtiofs failed for packed layers", + )); + } + } else { + // packed_layers_dir was set — which only happens after + // `with_packed_layers` acquired the lease — but the directory is + // not on disk at mount time. On macOS that means the per-machine + // case-sensitive layers volume isn't mounted (e.g. a concurrent + // stop/delete detached it). Mounting nothing would silently fall + // the guest back to a registry pull and break offline runs, and the + // launcher has no path to re-extract or re-mount here. Rather than + // boot a VM that is doomed to fail offline, free the context and + // fail fast with an actionable error. + krun_free_ctx(ctx); + return Err(Error::agent( + "add packed layers virtiofs", + format!( + "packed layers directory not found at {}: this machine's \ + layers volume is not mounted, so the guest cannot use its \ + bundled image and an offline run would fail. Restart the \ + machine, or re-create it from the .smolmachine bundle.", + layers_dir.display() + ), + )); + } + } + + // Attach the Rosetta 2 Linux runtime as a virtiofs mount when requested + // AND available on this host, so a stray `--rosetta` on a non-Rosetta host + // degrades to a no-op rather than a dangling tag the guest can't mount. + // The guest agent (gated on guest_env::ROSETTA, set below) mounts it at + // ROSETTA_GUEST_PATH and registers the binfmt_misc wrapper. + let rosetta_enabled = resources.rosetta && crate::vm::rosetta::is_available(); + if rosetta_enabled { + if let Some(runtime) = crate::vm::rosetta::runtime_path() { + let tag = cstr(smolvm_protocol::ROSETTA_TAG); + let host_path = cstr(runtime); + if krun_add_virtiofs(ctx, tag.as_ptr(), host_path.as_ptr()) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent( + "add rosetta virtiofs", + "krun_add_virtiofs failed for Rosetta runtime", + )); + } + } + } + + boot_timing!("devices configured"); + + // Set working directory + let workdir = cstr("/"); + krun_set_workdir(ctx, workdir.as_ptr()); + + // Build environment + let mut env_strings = vec![ + cstr("HOME=/root"), + cstr("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"), + cstr("TERM=xterm-256color"), + ]; + + // Host wall-clock at launch, so the agent can seed the guest clock on + // hypervisors without a guest-readable paravirt clock (WHP/Windows). The + // agent ignores it unless its own clock looks obviously wrong. + if let Ok(now) = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) { + env_strings.push(cstr(&format!( + "{}={}", + guest_env::HOST_TIME_NS, + now.as_nanos() + ))); + } + + // Pass mount info to the agent via environment + // Format: SMOLVM_MOUNT_0=tag:guest_path:ro + for (i, mount) in mounts.iter().enumerate() { + let mount_tag = HostMount::mount_tag(i); + let ro_flag = if mount.read_only { "ro" } else { "rw" }; + let env_val = format!( + "SMOLVM_MOUNT_{}={}:{}:{}", + i, + mount_tag, + mount.target.display(), + ro_flag + ); + if let Ok(cstr) = CString::new(env_val) { + env_strings.push(cstr); + } + } + + // Pass mount count + if !mounts.is_empty() { + if let Ok(cstr) = CString::new(format!("SMOLVM_MOUNT_COUNT={}", mounts.len())) { + env_strings.push(cstr); + } + } + + // Activate the guest side of each enabled vsock service (e.g. tell the + // agent to start the SSH agent bridge). The env pairs come from the same + // registry that wired the ports above, so the two sides cannot diverge. + for svc in &active_vsock { + for (key, value) in svc.guest_env { + env_strings.push(cstr(&format!("{key}={value}"))); + } + } + + // Tell the agent GPU was requested so it can sanity-check the + // virtio-gpu device actually appeared in the guest. libkrun + // happily accepts `krun_set_gpu_options2` even if the embedded + // kernel lacks the driver; without this check the user sees + // "VM started" and discovers missing GPU only when their + // workload hits a rendering call. + if resources.gpu { + let gpu_env = format!("{}={}", guest_env::GPU, guest_env::VALUE_ON); + if let Ok(cs) = CString::new(gpu_env) { + env_strings.push(cs); + } + } + + // Signal the guest agent to set up Rosetta (mount the runtime + register + // binfmt_misc). Gated on the same host-availability check as the mount. + if rosetta_enabled { + let rosetta_env = format!("{}={}", guest_env::ROSETTA, guest_env::VALUE_ON); + if let Ok(cs) = CString::new(rosetta_env) { + env_strings.push(cs); + } + } + + // Forward this VM's per-VM readiness-marker name into the guest env (the + // manager set it on this boot subprocess) so the agent writes the marker + // the host pre-created and polls. See manager::ready_marker_name. + if let Ok(marker) = std::env::var(guest_env::READY_MARKER) { + env_strings.push(cstr(&format!("{}={}", guest_env::READY_MARKER, marker))); + } + + // DNS allow-host filtering is now enforced inside libkrun (see the + // egress policy above). The guest-side DNS proxy is intentionally NOT + // started: the guest keeps its default resolv.conf (1.1.1.1/8.8.8.8) so + // its UDP DNS queries leave as real datagrams and are intercepted at the + // TSI layer. The DNS-filter vsock port is still registered (above, via + // the service registry) for the host-side proxy. + + // Guest-network env vars — virtio-net interface config plus the TSI + // `--dns` override — are built in one shared place so the static and + // dynamic launchers can't diverge (see `agent::guest_network_env`). + env_strings.extend(crate::agent::guest_network_env( + guest_network, + resources.dns, + )); + + // Tell the agent about pre-extracted packed layers + if packed_layers_dir.is_some_and(|d| d.exists()) { + env_strings.push(cstr("SMOLVM_PACKED_LAYERS=smolvm_layers:/packed_layers")); + } + + let mut envp: Vec<*const libc::c_char> = env_strings.iter().map(|s| s.as_ptr()).collect(); + envp.push(std::ptr::null()); + + // Set exec command (/sbin/init) + let exec_path = cstr("/sbin/init"); + let argv_strings = [cstr("/sbin/init")]; + let mut argv: Vec<*const libc::c_char> = argv_strings.iter().map(|s| s.as_ptr()).collect(); + argv.push(std::ptr::null()); + + if krun_set_exec(ctx, exec_path.as_ptr(), argv.as_ptr(), envp.as_ptr()) < 0 { + krun_free_ctx(ctx); + return Err(Error::agent("set exec command", "krun_set_exec failed")); + } + + // Egress CIDR live-refresh thread. + // + // Re-resolves DNS filter hostnames every SMOLVM_EGRESS_REFRESH_SECS + // (default 5 min) and atomically replaces the Arc>> + // that the vsock muxer reads on every packet. The Arc is borrowed from + // libkrun via `krun_get_egress_handle` — see libkrun/src/libkrun/src/lib.rs. + // + // Each cycle: resolve all hosts → build fresh list → single write-lock + // swap. If all hosts fail to resolve, the previous list is kept intact. + if let Some(hosts) = egress_refresh_hosts.as_ref().filter(|h| !h.is_empty()) { + if let Some(krun_get_egress_handle) = krun.get_egress_handle { + let raw_handle = krun_get_egress_handle(ctx); + + if !raw_handle.is_null() { + let arc: EgressArc = *Box::from_raw(raw_handle as *mut EgressArc); + let hosts_copy = hosts.clone(); + if let Err(e) = std::thread::Builder::new() + .name("egress-refresh".into()) + .spawn(move || { + let refresh_secs: u64 = std::env::var("SMOLVM_EGRESS_REFRESH_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(5 * 60); + let refresh_interval = std::time::Duration::from_secs(refresh_secs); + loop { + std::thread::sleep(refresh_interval); + // Resolve all hosts into a fresh list, then swap + // the shared Vec in a single write-lock acquisition. + // This ensures old rotated-away IPs are removed. + let mut fresh: Vec<(std::net::IpAddr, u8)> = Vec::new(); + 'hosts: for host in &hosts_copy { + match resolve_host_subprocess(host) { + Ok(new_cidrs) => { + for cidr_str in new_cidrs { + if fresh.len() >= EGRESS_CIDR_CAP { + break 'hosts; + } + if let Some((ip_str, prefix_str)) = + cidr_str.split_once('/') + { + if let (Ok(ip), Ok(prefix)) = ( + ip_str.parse::(), + prefix_str.parse::(), + ) { + if !fresh.contains(&(ip, prefix)) { + fresh.push((ip, prefix)); + } + } + } + } + } + Err(e) => { + tracing::warn!( + host = %host, + error = %e, + "egress-refresh: resolve failed" + ); + } + } + } + // Only replace if at least one host resolved + // successfully; keeps the old list on total failure. + if !fresh.is_empty() { + let mut guard = arc.write().unwrap_or_else(|e| e.into_inner()); + *guard = fresh; + } + } + }) + { + tracing::warn!(error = %e, "egress-refresh spawn failed"); + } + } + } + } + + // Enable guest-RAM zero-copy for the CUDA server: publish a provider it + // can query (lazily, once the VM is up) for the guest-RAM host mapping. + // The guest side is armed via SMOLVM_CUDA_ZEROCOPY (see vsock_service). + if cuda_socket.is_some() { + if let Some(get_ram) = krun_get_guest_ram { + crate::cuda_host::set_guest_ram_provider(Box::new(move || { + let mut count = 0u64; + // SAFETY: FFI into libkrun; valid after the VM is built. + if get_ram(ctx, std::ptr::null_mut(), 0, &mut count) != 0 || count == 0 { + return None; + } + let mut buf = vec![0u64; count as usize * 3]; + if get_ram(ctx, buf.as_mut_ptr(), count as u32, &mut count) != 0 { + return None; + } + Some( + buf.chunks_exact(3) + .map(|c| (c[0], c[1], c[2])) + .collect::>(), + ) + })); + } else { + tracing::info!("cuda-host: libkrun has no krun_get_guest_ram — zero-copy disabled"); + } + } + + // Start VM (this replaces the process on success) + boot_timing!("entering vm"); + let ret = krun_start_enter(ctx); + + // If we get here, something went wrong — free the context before returning + krun_free_ctx(ctx); + drop(virtio_network_runtime); + Err(Error::agent( + "start vm", + format!("krun_start_enter returned: {}", ret), + )) + } +} + +/// Create a CString from a static string that is known not to contain NUL bytes. +fn cstr(s: &str) -> CString { + CString::new(s).expect("string literal must not contain NUL bytes") +} + +/// Convert a Path to a CString. +fn path_to_cstring(path: &Path) -> Result { + CString::new(path.to_string_lossy().as_bytes()) + .map_err(|_| Error::agent("convert path", "path contains null byte")) +} + +fn rootfs_dax_disabled() -> bool { + std::env::var(ENV_SMOLVM_ROOTFS_DAX) + .map(|value| { + matches!( + value.as_str(), + "0" | "false" | "False" | "FALSE" | "no" | "off" + ) + }) + .unwrap_or(false) +} + +// Unix-only: virtio-net is the sole caller and is itself unix-gated. +#[cfg(unix)] +fn create_unix_stream_pair() -> std::io::Result<(RawFd, RawFd)> { + let mut fds = [0; 2]; + // SAFETY: `socketpair` initializes both descriptors on success. + let result = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; + if result < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok((fds[0], fds[1])) +} + +// Windows has no AF_UNIX `socketpair`, so the virtio-net host end binds a +// listener on a per-VM path and accepts the connection libkrun makes to it. +#[cfg(windows)] +pub(crate) fn bind_unix_listener(path: &Path) -> std::io::Result { + // A leftover socket file from a previous run would make bind fail with + // EADDRINUSE, so clear it first (ignore "not found"). + let _ = std::fs::remove_file(path); + let listener = Socket::new(Domain::UNIX, Type::STREAM, None)?; + listener.bind(&SockAddr::unix(path)?)?; + listener.listen(1)?; + Ok(listener) +} + +fn select_network_plan( + resources: &VmResources, + dns_filter_enabled: bool, + port_count: usize, +) -> crate::network::LaunchNetworkPlan { + let dns_filter_placeholder = [String::from("configured")]; + let dns_filter_hosts = dns_filter_enabled.then_some(dns_filter_placeholder.as_slice()); + plan_launch_network(resources, dns_filter_hosts, port_count) +} + +/// Resolve a hostname to /32 CIDR strings for the egress-refresh thread. +/// +/// ## Why not `getaddrinfo`? +/// +/// The `egress-refresh` thread runs inside the `_boot-vm` subprocess. Before +/// `krun_start_enter` is called, `internal_boot.rs` closes every inherited FD +/// from 3 up to `max_fd`. Apple's Network framework maps shared memory at +/// process launch and accesses it via FD-derived handles. After the mass close, +/// those handles are invalid, so any call to `getaddrinfo` (which routes +/// through the Network framework on macOS) crashes with SIGBUS at +/// `_os_log_preferences_refresh` inside `nw_path_libinfo_path_check`. +/// +/// Spawning an external `dig` process sidesteps this: `exec()` gives the child +/// a completely fresh address space, so it never touches the broken inherited +/// shared memory. On non-macOS platforms `getaddrinfo` via glibc is safe and +/// is used directly. +#[cfg(target_os = "macos")] +#[inline(never)] +fn resolve_host_subprocess(host: &str) -> std::result::Result, String> { + // `/usr/bin/dig` is always present on macOS (part of BIND-tools in the + // base system). `+short` prints one result per line (IPs and CNAMEs); + // `+timeout=5 +tries=2` keeps the refresh loop from stalling the VM on + // a flaky network. + let output = std::process::Command::new("/usr/bin/dig") + .args(["+short", "+timeout=5", "+tries=2", host]) + .output() + .map_err(|e| format!("dig subprocess failed for '{}': {}", host, e))?; + + let stdout = String::from_utf8_lossy(&output.stdout); + // `+short` emits CNAMEs (ending in '.') interleaved with IPs; parse:: + // silently skips the CNAME lines, leaving only valid addresses. + let cidrs: Vec = stdout + .lines() + .filter_map(|line| { + line.trim() + .parse::() + .ok() + .map(|ip| format!("{}/32", ip)) + }) + .collect(); + + if cidrs.is_empty() { + return Err(format!("dig resolved '{}' to no IP addresses", host)); + } + Ok(cidrs) +} + +/// On non-macOS (Linux), `getaddrinfo` is safe to call from background threads +/// in child processes — glibc does not use shared-memory handles that become +/// invalid after a mass FD close. Delegate directly to the standard resolver. +#[cfg(not(target_os = "macos"))] +#[inline(never)] +fn resolve_host_subprocess(host: &str) -> std::result::Result, String> { + crate::smolfile::resolve_host_to_cidrs(host) +} + +/// Raise file descriptor limits (required by libkrun). +fn raise_fd_limits() { + // rlimit is a unix concept; no-op on Windows. The function stays callable + // on all platforms so its (unconditional) call sites need no gating. + #[cfg(unix)] + unsafe { + let mut limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + + if libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) == 0 { + limit.rlim_cur = limit.rlim_max; + libc::setrlimit(libc::RLIMIT_NOFILE, &limit); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn scoped(hosts: &[&str]) -> LaunchFeatures { + LaunchFeatures { + dns_filter_hosts: Some(hosts.iter().map(|h| h.to_string()).collect()), + ..Default::default() + } + } + + #[test] + fn allow_image_pull_egress_folds_registry_into_scope() { + let mut f = scoped(&["api.anthropic.com"]); + f.allow_image_pull_egress(Some("ghcr.io/acme/app:1"), false); + assert_eq!( + f.dns_filter_hosts.unwrap(), + vec!["api.anthropic.com".to_string(), "ghcr.io".to_string()] + ); + } + + #[test] + fn allow_image_pull_egress_dockerhub_adds_both_apexes_without_dupes() { + // docker.io already listed by the user; the fold must add docker.com but + // not re-add docker.io. + let mut f = scoped(&["docker.io", "pypi.org"]); + f.allow_image_pull_egress(Some("alpine"), false); + assert_eq!( + f.dns_filter_hosts.unwrap(), + vec![ + "docker.io".to_string(), + "pypi.org".to_string(), + "docker.com".to_string() + ] + ); + } + + #[test] + fn allow_image_pull_egress_noop_when_unscoped_or_packed_or_local() { + // No filter set → unscoped machine, nothing to widen. + let mut f = LaunchFeatures::default(); + f.allow_image_pull_egress(Some("alpine"), false); + assert!(f.dns_filter_hosts.is_none()); + + // Packed layers (.smolmachine / local dir) → no in-guest pull. + let mut f = scoped(&["api.anthropic.com"]); + f.allow_image_pull_egress(Some("alpine"), true); + assert_eq!(f.dns_filter_hosts.unwrap(), vec!["api.anthropic.com"]); + + // A `local:` reference is host-assembled, never pulled. + let mut f = scoped(&["api.anthropic.com"]); + f.allow_image_pull_egress(Some("local:abc123"), false); + assert_eq!(f.dns_filter_hosts.unwrap(), vec!["api.anthropic.com"]); + + // Bare VM (no image) → nothing to fold. + let mut f = scoped(&["api.anthropic.com"]); + f.allow_image_pull_egress(None, false); + assert_eq!(f.dns_filter_hosts.unwrap(), vec!["api.anthropic.com"]); + } + + /// Build a machine `pack` dir plus a `.pack-shared` pointer in its parent that + /// names `shared`, mirroring what create writes when the pack lands in the + /// node's content-addressed store. Returns the layers cache dir to pass to + /// [`LaunchFeatures::with_packed_layers`]. + fn machine_with_pointer(root: &Path, shared: &Path) -> PathBuf { + let layers_cache_dir = root.join("vm").join("pack"); + fs::create_dir_all(&layers_cache_dir).unwrap(); + let pointer = super::super::shared_pack_pointer_path(&layers_cache_dir); + fs::write(&pointer, shared.to_string_lossy().as_bytes()).unwrap(); + layers_cache_dir + } + + // Regression: the shared-store branch must present the `layers/` SUBDIR of the + // shared copy as the idmap source, not the store root. Carrying the root let + // the guest name-sort `agent-rootfs/` + `layers/` and mis-stack the agent + // rootfs as an image layer, surfacing pack internals (`/`, `.tar`, + // `layer-order`) at the container `/`. + #[test] + fn shared_pack_idmap_source_targets_layers_subdir() { + let tmp = tempfile::tempdir().unwrap(); + let shared = tmp.path().join("_shared").join("ea92da8fcheck"); + fs::create_dir_all(shared.join("layers").join("25f1d6b1951a")).unwrap(); + fs::create_dir_all(shared.join("agent-rootfs")).unwrap(); + let layers_cache_dir = machine_with_pointer(tmp.path(), &shared); + + let features = LaunchFeatures::default() + .with_packed_layers(&layers_cache_dir, Some("dummy.smolmachine")) + .unwrap(); + + // The guest mounts the per-machine `pack` mountpoint... + assert_eq!( + features.packed_layers_dir.as_deref(), + Some(layers_cache_dir.as_path()) + ); + // ...backed by the shared copy's `layers/` subdir — NOT the store root. + assert_eq!( + features.pack_idmap_source.as_deref(), + Some(shared.join("layers").as_path()) + ); + } + + // A shared copy with no `layers/` subdir (hypothetical/legacy layout) falls + // back to the store root so boot still has a valid idmap source rather than + // pointing at a path that doesn't exist (internal_boot fails closed on a + // missing source). + #[test] + fn shared_pack_idmap_falls_back_to_root_without_layers_subdir() { + let tmp = tempfile::tempdir().unwrap(); + let shared = tmp.path().join("_shared").join("deadbeefcheck"); + fs::create_dir_all(&shared).unwrap(); + let layers_cache_dir = machine_with_pointer(tmp.path(), &shared); + + let features = LaunchFeatures::default() + .with_packed_layers(&layers_cache_dir, Some("dummy.smolmachine")) + .unwrap(); + + assert_eq!( + features.pack_idmap_source.as_deref(), + Some(shared.as_path()) + ); + } + + // No pointer + no source bundle: this is not the shared-store path, so the + // function must not invent an idmap source. (A bare VM with no packed layers.) + #[test] + fn no_source_smolmachine_leaves_idmap_unset() { + let tmp = tempfile::tempdir().unwrap(); + let layers_cache_dir = tmp.path().join("vm").join("pack"); + fs::create_dir_all(&layers_cache_dir).unwrap(); + + let features = LaunchFeatures::default() + .with_packed_layers(&layers_cache_dir, None) + .unwrap(); + + assert!(features.pack_idmap_source.is_none()); + assert!(features.packed_layers_dir.is_none()); + } +} diff --git a/src/agent/launcher_dynamic.rs b/src/agent/launcher_dynamic.rs new file mode 100644 index 0000000..8ee33b7 --- /dev/null +++ b/src/agent/launcher_dynamic.rs @@ -0,0 +1,696 @@ +//! Dynamic (dlopen-based) libkrun launcher for packed/sidecar mode. +//! +//! This module provides a `KrunFunctions` struct that loads libkrun via `dlopen` +//! at runtime, enabling the main smolvm binary to boot VMs using libraries +//! extracted from a `.smolmachine` sidecar file. +//! +//! The static FFI path in `launcher.rs` remains untouched for normal operations. + +use crate::network::backend::COMPAT_NET_FEATURES; +use crate::network::backend::TSI_FEATURE_HIJACK_INET; +use crate::network::{plan_launch_network, EffectiveNetworkBackend}; +use smolvm_network::PortMapping as VirtioPortMapping; +use smolvm_network::{start_virtio_network, GuestNetworkConfig, VirtioNetworkRuntime}; +use smolvm_protocol::{guest_env, ports}; +#[cfg(unix)] +use socket2::Socket; +use std::ffi::CString; +#[cfg(unix)] +use std::os::fd::FromRawFd; +// `std::os::fd` does not exist on Windows. Keep the `RawFd` name working in +// signatures on both platforms via a portable alias. +#[cfg(unix)] +use std::os::fd::RawFd; +#[cfg(not(unix))] +#[allow(dead_code)] +type RawFd = std::os::raw::c_int; +use std::path::{Path, PathBuf}; + +pub use super::krun::KrunFunctions; +use super::VmResources; + +/// Volume mount for packed binaries. +#[derive(Debug, Clone)] +pub struct PackedMount { + /// Virtiofs tag (e.g., "smolvm0"). + pub tag: String, + /// Host source path (passed to `krun_add_virtiofs`). + pub host_path: String, + /// Guest mount path (passed to agent via `SMOLVM_MOUNT_*` env). + pub guest_path: String, + /// Whether the mount is read-only. + pub read_only: bool, +} + +/// Configuration for launching a packed VM. +pub struct PackedLaunchConfig<'a> { + /// Path to agent rootfs directory. + pub rootfs_path: &'a Path, + /// Path to storage disk. + pub storage_path: &'a Path, + /// Path to vsock Unix socket. + pub vsock_socket: &'a Path, + /// Path to layers directory (for virtiofs). + pub layers_dir: &'a Path, + /// Volume mounts. + pub mounts: &'a [PackedMount], + /// Port mappings (host, guest). + pub port_mappings: &'a [(u16, u16)], + /// VM resources. + pub resources: VmResources, + /// Debug logging. + pub debug: bool, + /// Path to overlay disk (VM mode only, mounted as /dev/vdb). + pub overlay_path: Option<&'a Path>, + /// Path to redirect VM console output (prevents libkrun from putting + /// the inherited terminal into raw mode). + pub console_log: PathBuf, +} + +/// The `krun_add_disk2` image-format code for a disk file: `1` (qcow2) when it +/// begins with the qcow2 magic (`QFI\xfb`), else `0` (raw). Matches +/// `DiskFormat::to_krun_u32`. Reading the bytes keeps the format in sync with +/// what libkrun will parse, regardless of the file's extension. +fn krun_disk_format(path: &Path) -> u32 { + use std::io::Read; + const QCOW2_MAGIC: [u8; 4] = [0x51, 0x46, 0x49, 0xfb]; + let mut magic = [0u8; 4]; + let is_qcow2 = std::fs::File::open(path) + .and_then(|mut f| f.read_exact(&mut magic)) + .is_ok() + && magic == QCOW2_MAGIC; + if is_qcow2 { + 1 + } else { + 0 + } +} + +/// Launch VM using dynamically loaded libkrun (for packed/sidecar mode). +/// +/// This mirrors the setup logic in `launcher.rs:launch_agent_vm()` but calls +/// through `KrunFunctions` instead of static `extern "C"` symbols. +/// +/// # Safety +/// +/// Must be called in a forked child process. Never returns on success. +pub fn launch_agent_vm_dynamic( + krun: &KrunFunctions, + config: &PackedLaunchConfig, +) -> Result<(), String> { + crate::network::validate_requested_network_backend( + &config.resources, + None, + config.port_mappings.len(), + ) + .map_err(|e| e.to_string())?; + + // Raise file descriptor limits + raise_fd_limits(); + + // Set library path so libkrun can find libkrunfw. Only consumed by the + // macos/linux env-var blocks below, so unused on other targets (Windows). + #[cfg_attr( + not(any(target_os = "macos", target_os = "linux")), + allow(unused_variables) + )] + let lib_dir = config + .rootfs_path + .parent() + .unwrap_or(Path::new(".")) + .join("lib"); + #[cfg(target_os = "macos")] + { + let lib_path = lib_dir.to_string_lossy(); + unsafe { std::env::set_var("DYLD_LIBRARY_PATH", lib_path.as_ref()) }; + } + #[cfg(target_os = "linux")] + { + let lib_path = lib_dir.to_string_lossy(); + unsafe { std::env::set_var("LD_LIBRARY_PATH", lib_path.as_ref()) }; + } + + // SAFETY: Each FFI call below is individually wrapped in unsafe. + // All CString/pointer construction is safe Rust outside the unsafe blocks. + + // Set log level + let log_level = if config.debug { 3 } else { 0 }; + // SAFETY: set_log_level is a valid function pointer loaded from libkrun + unsafe { (krun.set_log_level)(log_level) }; + + // Create VM context + // SAFETY: create_ctx is a valid function pointer loaded from libkrun + let ctx = unsafe { (krun.create_ctx)() }; + if ctx < 0 { + return Err("krun_create_ctx failed".to_string()); + } + let ctx = ctx as u32; + + // Helper: clean up context on error (string message) + macro_rules! free_ctx_on_err { + ($msg:expr) => {{ + // SAFETY: ctx is a valid context from krun_create_ctx + unsafe { (krun.free_ctx)(ctx) }; + return Err($msg.to_string()); + }}; + } + + // Helper: evaluate a fallible expression, freeing ctx if it fails. + // Replaces bare `?` which would leak the libkrun context. + macro_rules! try_or_free_ctx { + ($expr:expr, $msg:expr) => { + match $expr { + Ok(val) => val, + Err(_) => free_ctx_on_err!($msg), + } + }; + } + + // Set VM config + // SAFETY: ctx is valid, cpus and mem are primitive values + if unsafe { (krun.set_vm_config)(ctx, config.resources.cpus, config.resources.memory_mib) } < 0 + { + free_ctx_on_err!("krun_set_vm_config failed"); + } + + // Enable GPU (virtio-gpu / Venus Vulkan) when requested by the manifest. + // Flag logic lives in super::gpu_virgl_flags() — see mod.rs for the full + // explanation of each flag's purpose on Linux vs macOS. + if config.resources.gpu { + let virgl_flags = super::gpu_virgl_flags(); + let vram_mib = config.resources.effective_gpu_vram_mib(); + let vram_bytes: u64 = (vram_mib as u64) * crate::data::consts::BYTES_PER_MIB; + + match krun.set_gpu_options2 { + Some(set_gpu) => { + let ret = unsafe { set_gpu(ctx, virgl_flags, vram_bytes) }; + if ret < 0 { + free_ctx_on_err!(format!( + "krun_set_gpu_options2 failed (ret={}). Check that virglrenderer is installed.", + ret + )); + } + tracing::info!("GPU enabled (Venus/Vulkan via virtio-gpu)"); + } + None => { + free_ctx_on_err!( + "libkrun was built without GPU support (krun_set_gpu_options2 not found). \ + Rebuild libkrun with GPU=1 — see project README for details." + ); + } + } + } + + // Set root filesystem via the root virtiofs tag (upstream removed + // krun_set_root in favor of krun_add_virtiofs with KRUN_FS_ROOT_TAG). + let root = try_or_free_ctx!( + path_to_cstring(config.rootfs_path), + "rootfs path contains null byte" + ); + let root_tag = cstr("/dev/root"); + // Default root with a 512 MB DAX window (matches the removed krun_set_root). + // Plain krun_add_virtiofs passes shm_size=0 (no DAX), dropping virtiofs to + // writeback caching so the guest's ready-marker write isn't visible to the + // host until the socket-probe grace — a multi-second boot-time regression. + let Some(add_virtiofs3) = krun.add_virtiofs3 else { + free_ctx_on_err!("root DAX requires libkrun with krun_add_virtiofs3"); + }; + // SAFETY: ctx is valid; root_tag/root are valid null-terminated C strings. + if unsafe { add_virtiofs3(ctx, root_tag.as_ptr(), root.as_ptr(), 1 << 29, false) } < 0 { + free_ctx_on_err!("krun_add_virtiofs3 failed for root filesystem"); + } + + let network_plan = plan_launch_network(&config.resources, None, config.port_mappings.len()); + + // `mut` is only needed on unix (the VirtioNet arm assigns it); on Windows + // the runtime is owned by the accept thread, so the binding stays `None`. + #[cfg_attr(not(unix), allow(unused_mut))] + let mut virtio_network_runtime: Option = None; + let guest_network: Option = match network_plan.backend { + EffectiveNetworkBackend::None => { + // Upstream libkrun no longer creates an implicit vsock; add explicitly. + if unsafe { (krun.add_vsock)(ctx, 0) } < 0 { + free_ctx_on_err!("krun_add_vsock failed"); + } + tracing::debug!("configured vsock without guest networking"); + None + } + EffectiveNetworkBackend::Tsi => { + if unsafe { (krun.add_vsock)(ctx, TSI_FEATURE_HIJACK_INET) } < 0 { + free_ctx_on_err!("krun_add_vsock with TSI failed"); + } + + let port_cstrings: Vec = config + .port_mappings + .iter() + .map(|(host, guest)| { + CString::new(format!("{}:{}", host, guest)) + .expect("port mapping cannot contain null bytes") + }) + .collect(); + let mut port_ptrs: Vec<*const libc::c_char> = + port_cstrings.iter().map(|s| s.as_ptr()).collect(); + port_ptrs.push(std::ptr::null()); + + if unsafe { (krun.set_port_map)(ctx, port_ptrs.as_ptr()) } < 0 { + free_ctx_on_err!("krun_set_port_map failed"); + } + + if let Some(ref cidrs) = config.resources.allowed_cidrs { + if !cidrs.is_empty() { + let set_egress = krun.set_egress_policy.ok_or_else(|| { + "libkrun does not support egress policy (krun_set_egress_policy not found). \ + Update libkrun or remove --allow-cidr flags." + .to_string() + })?; + + let mut all_cidrs = cidrs.clone(); + crate::data::network::ensure_dns_in_cidrs(&mut all_cidrs); + + let cidr_cstrings: Vec = all_cidrs + .iter() + .map(|c| CString::new(c.as_str()).expect("CIDR cannot contain null bytes")) + .collect(); + let mut cidr_ptrs: Vec<*const libc::c_char> = + cidr_cstrings.iter().map(|s| s.as_ptr()).collect(); + cidr_ptrs.push(std::ptr::null()); + + // The dynamic path enforces CIDR-only egress; DNS allow-host + // filtering (hosts + resolver args) is wired in the main + // launcher path. + if unsafe { + (set_egress)(ctx, cidr_ptrs.as_ptr(), std::ptr::null(), std::ptr::null()) + } < 0 + { + free_ctx_on_err!("krun_set_egress_policy failed"); + } + } + } + + tracing::info!("network backend: tsi"); + None + } + EffectiveNetworkBackend::VirtioNet => { + let add_net_unixstream = krun.add_net_unixstream.ok_or_else(|| { + "libkrun does not expose krun_add_net_unixstream; update libkrun or use --net-backend tsi" + .to_string() + })?; + + // virtio-net carries guest networking, but the host-guest control + // channel still rides vsock. Upstream libkrun no longer creates an + // implicit vsock, so add it explicitly (no TSI hijacking — virtio-net + // owns the network path); otherwise krun_add_vsock_port2 below fails + // with ENODEV. + if unsafe { (krun.add_vsock)(ctx, 0) } < 0 { + free_ctx_on_err!("krun_add_vsock failed"); + } + + let guest_network = GuestNetworkConfig::default(); + let mut guest_mac = guest_network.guest_mac; + let port_mappings: Vec = config + .port_mappings + .iter() + .map(|(host, guest)| VirtioPortMapping::new(*host, *guest)) + .collect(); + let egress = smolvm_network::EgressPolicy::from_allowed_cidrs( + config.resources.allowed_cidrs.as_deref(), + ); + + // The host/guest ends of the virtio-net channel are an AF_UNIX + // stream: a socketpair fd on Unix, a per-VM path listener libkrun + // connects to on Windows. Mirrors the static launcher's VirtioNet arm. + #[cfg(unix)] + { + let (host_fd, guest_fd) = + create_unix_stream_pair().map_err(|e| format!("socketpair failed: {e}"))?; + // SAFETY: ownership of the host-side socketpair fd transfers here. + let host_stream = unsafe { Socket::from_raw_fd(host_fd) }; + let runtime = match start_virtio_network( + host_stream, + guest_network, + &port_mappings, + egress, + ) { + Ok(runtime) => runtime, + Err(err) => { + // SAFETY: guest_fd was created by socketpair above and not moved elsewhere. + unsafe { libc::close(guest_fd) }; + return Err(format!("failed to start virtio network runtime: {err}")); + } + }; + + if unsafe { + (add_net_unixstream)( + ctx, + std::ptr::null(), + guest_fd, + guest_mac.as_mut_ptr(), + COMPAT_NET_FEATURES, + 0, + ) + } < 0 + { + // SAFETY: guest_fd was created by socketpair above and not moved elsewhere. + unsafe { libc::close(guest_fd) }; + free_ctx_on_err!("krun_add_net_unixstream failed"); + } + + virtio_network_runtime = Some(runtime); + } + #[cfg(windows)] + { + let net_sock_path = config.vsock_socket.with_extension("net"); + let listener = match super::launcher::bind_unix_listener(&net_sock_path) { + Ok(listener) => listener, + Err(e) => free_ctx_on_err!(format!("failed to bind virtio-net socket: {e}")), + }; + let path_c = match path_to_cstring(&net_sock_path) { + Ok(path_c) => path_c, + Err(_) => free_ctx_on_err!("virtio-net socket path contains null byte"), + }; + if unsafe { + (add_net_unixstream)( + ctx, + path_c.as_ptr(), + -1, + guest_mac.as_mut_ptr(), + COMPAT_NET_FEATURES, + 0, + ) + } < 0 + { + free_ctx_on_err!("krun_add_net_unixstream failed"); + } + + // libkrun connects to the path while the VM boots inside the + // blocking krun_start_enter, so accept on a background thread; the + // runtime parks there until libkrun closes the stream (VM exit). + let spawn = std::thread::Builder::new() + .name("smolvm-net-accept".into()) + .spawn(move || match listener.accept() { + Ok((sock, _)) => { + match start_virtio_network(sock, guest_network, &port_mappings, egress) { + Ok(runtime) => runtime.block_until_shutdown(), + Err(err) => { + tracing::error!(error = %err, "virtio-net runtime failed to start") + } + } + } + Err(err) => tracing::warn!(error = %err, "virtio-net accept failed"), + }); + if let Err(e) = spawn { + free_ctx_on_err!(format!("failed to spawn virtio-net accept thread: {e}")); + } + } + + tracing::info!("network backend: virtio-net"); + Some(guest_network) + } + }; + + // Add storage disk. The format MUST match the on-disk image: a machine's + // storage/overlay can be an instant qcow2 CoW overlay, and telling libkrun a + // qcow2 file is raw makes it expose the tiny overlay file (~256 KiB) as the + // whole device, so the guest formats a tiny ext4 and image writes fail with + // "no space left on device". Detect the format from the file's magic so it + // can't drift from what libkrun parses (mirrors `launcher.rs`, which passes + // the disk object's `format()`). + let block_id = cstr("storage"); + let disk_path = try_or_free_ctx!( + path_to_cstring(config.storage_path), + "storage path contains null byte" + ); + let storage_format = krun_disk_format(config.storage_path); + // SAFETY: ctx is valid, block_id and disk_path are valid C strings + if unsafe { + (krun.add_disk2)( + ctx, + block_id.as_ptr(), + disk_path.as_ptr(), + storage_format, + false, + ) + } < 0 + { + free_ctx_on_err!("krun_add_disk2 failed"); + } + + // Add overlay disk as 2nd disk (/dev/vdb) for VM mode + if let Some(overlay) = config.overlay_path { + let overlay_id = cstr("overlay"); + let overlay_disk = + try_or_free_ctx!(path_to_cstring(overlay), "overlay path contains null byte"); + let overlay_format = krun_disk_format(overlay); + // SAFETY: ctx is valid, overlay_id and overlay_disk are valid C strings + if unsafe { + (krun.add_disk2)( + ctx, + overlay_id.as_ptr(), + overlay_disk.as_ptr(), + overlay_format, + false, + ) + } < 0 + { + free_ctx_on_err!("krun_add_disk2 failed for overlay disk"); + } + } + + // Add vsock port for control channel + let socket_path = try_or_free_ctx!( + path_to_cstring(config.vsock_socket), + "vsock socket path contains null byte" + ); + // SAFETY: ctx is valid, socket_path is a valid C string + if unsafe { (krun.add_vsock_port2)(ctx, ports::AGENT_CONTROL, socket_path.as_ptr(), true) } < 0 + { + free_ctx_on_err!("krun_add_vsock_port2 failed"); + } + + // Redirect console output to a log file so libkrun doesn't put the + // inherited terminal into raw mode (which would break terminal echo + // if the child is killed before exit observers can restore it). Uses the + // upstream virtio-console API (krun_set_console_output was removed). + // SAFETY: ctx is a valid, not-yet-started libkrun context. + if unsafe { krun.console_output_to_file(ctx, &config.console_log) } < 0 { + // On Windows the fd-based virtio-console redirection isn't wired (the + // wrapper is a known no-op that always returns < 0), so this is expected + // — NOT a boot failure. Keep it out of the startup error log at WARN so a + // benign line can't become what the readiness monitor surfaces as "the + // error" when the boot later fails for a real reason (see + // `boot_failure_reason`). + #[cfg(windows)] + tracing::debug!("guest console not captured on Windows (fd redirection unsupported)"); + #[cfg(not(windows))] + tracing::warn!("failed to set console output"); + } + + // Set working directory + let workdir = cstr("/"); + // SAFETY: ctx is valid, workdir is a valid C string + unsafe { (krun.set_workdir)(ctx, workdir.as_ptr()) }; + + // Build environment (all safe Rust) + let mut env_strings = vec![ + cstr("HOME=/root"), + cstr("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"), + cstr("TERM=xterm-256color"), + ]; + + // Tell agent about packed layers mount + if config.layers_dir.exists() { + env_strings.push(cstr("SMOLVM_PACKED_LAYERS=smolvm_layers:/packed_layers")); + } + + // Pass mount info to the agent via environment + for (i, mount) in config.mounts.iter().enumerate() { + let ro_flag = if mount.read_only { "ro" } else { "rw" }; + let env_val = format!( + "SMOLVM_MOUNT_{}={}:{}:{}", + i, mount.tag, mount.guest_path, ro_flag + ); + if let Ok(cstr) = CString::new(env_val) { + env_strings.push(cstr); + } + } + + if !config.mounts.is_empty() { + if let Ok(cstr) = CString::new(format!("SMOLVM_MOUNT_COUNT={}", config.mounts.len())) { + env_strings.push(cstr); + } + } + + // Tell the agent GPU was requested so it creates /dev/dri nodes and starts + // seatd after pivot_root. Keep this in sync with the normal launcher. + if config.resources.gpu { + let gpu_env = format!("{}={}", guest_env::GPU, guest_env::VALUE_ON); + if let Ok(cstr) = CString::new(gpu_env) { + env_strings.push(cstr); + } + } + + // Enable Rosetta only when requested AND actually available on this host, so + // a stray `--rosetta` on a non-Rosetta host degrades to a no-op rather than a + // dangling virtiofs tag the guest would fail to mount. The guest agent reads + // guest_env::ROSETTA and mounts the runtime + registers binfmt_misc. + let rosetta_enabled = config.resources.rosetta && crate::vm::rosetta::is_available(); + if rosetta_enabled { + let rosetta_env = format!("{}={}", guest_env::ROSETTA, guest_env::VALUE_ON); + if let Ok(cstr) = CString::new(rosetta_env) { + env_strings.push(cstr); + } + } + + // Guest-network env vars — virtio-net interface config plus the TSI `--dns` + // override — are built in one shared place so the static and dynamic + // launchers can't diverge (see `agent::guest_network_env`). The dynamic + // launcher having open-coded this is exactly how it once dropped `--dns` on + // TSI (PR #466). + env_strings.extend(crate::agent::guest_network_env( + guest_network, + config.resources.dns, + )); + + let mut envp: Vec<*const libc::c_char> = env_strings.iter().map(|s| s.as_ptr()).collect(); + envp.push(std::ptr::null()); + + // Set exec command (MUST be before add_virtiofs) + let exec_path = cstr("/sbin/init"); + let argv_strings = [cstr("/sbin/init")]; + let mut argv: Vec<*const libc::c_char> = argv_strings.iter().map(|s| s.as_ptr()).collect(); + argv.push(std::ptr::null()); + + // SAFETY: ctx is valid, all pointers are valid null-terminated C strings/arrays + if unsafe { (krun.set_exec)(ctx, exec_path.as_ptr(), argv.as_ptr(), envp.as_ptr()) } < 0 { + free_ctx_on_err!("krun_set_exec failed"); + } + + // Every virtiofs mount gets a DAX window (like the root fs above): without + // DAX, virtiofs falls back to writeback caching where each file access is a + // FUSE round-trip over the virtio queue — pathological for read-heavy mounts + // with many files (a multi-GB Python venv took minutes just to import, and a + // single file larger than the window stalled entirely). 2 GiB exceeds any + // realistic single mapped file; the window is virtual host address space + // backed on demand, so oversizing costs nothing until touched. + const VIRTIOFS_DAX_WINDOW: u64 = 1 << 31; + + // Add virtiofs mount for packed layers (AFTER set_exec) + if config.layers_dir.exists() { + let layers_tag = cstr("smolvm_layers"); + let layers_path = try_or_free_ctx!( + path_to_cstring(config.layers_dir), + "layers dir path contains null byte" + ); + // SAFETY: ctx is valid, tag and path are valid C strings + if unsafe { + add_virtiofs3( + ctx, + layers_tag.as_ptr(), + layers_path.as_ptr(), + VIRTIOFS_DAX_WINDOW, + false, + ) + } < 0 + { + free_ctx_on_err!("krun_add_virtiofs failed for packed layers"); + } + } + + // Add user-specified virtiofs mounts + for mount in config.mounts.iter() { + let tag = try_or_free_ctx!( + CString::new(mount.tag.as_str()), + "mount tag contains null byte" + ); + let host_path = try_or_free_ctx!( + CString::new(mount.host_path.as_str()), + "mount path contains null byte" + ); + + // SAFETY: ctx is valid, tag and host_path are valid C strings + if unsafe { + add_virtiofs3( + ctx, + tag.as_ptr(), + host_path.as_ptr(), + VIRTIOFS_DAX_WINDOW, + mount.read_only, + ) + } < 0 + { + free_ctx_on_err!(format!( + "krun_add_virtiofs failed for '{}' - requested mount cannot be attached", + mount.tag + )); + } + } + + // Attach the Rosetta 2 Linux runtime (read-write is unnecessary but the plain + // add_virtiofs is what the guest expects for this tag; the runtime dir is + // never written). runtime_path() is Some iff is_available() held above. + if rosetta_enabled { + if let Some(runtime) = crate::vm::rosetta::runtime_path() { + let rosetta_tag = cstr(smolvm_protocol::ROSETTA_TAG); + let rosetta_path = try_or_free_ctx!( + CString::new(runtime), + "rosetta runtime path contains null byte" + ); + // SAFETY: ctx is valid, tag and path are valid C strings + if unsafe { (krun.add_virtiofs)(ctx, rosetta_tag.as_ptr(), rosetta_path.as_ptr()) } < 0 + { + free_ctx_on_err!("krun_add_virtiofs failed for Rosetta runtime"); + } + } + } + + // Start VM (never returns on success) + // SAFETY: ctx is valid, all configuration has been set + let ret = unsafe { (krun.start_enter)(ctx) }; + + // If we get here, something went wrong — free the context before returning + // SAFETY: ctx is a valid context from krun_create_ctx + unsafe { (krun.free_ctx)(ctx) }; + drop(virtio_network_runtime); + Err(format!("krun_start_enter returned: {}", ret)) +} + +/// Create a CString from a static string that is known not to contain NUL bytes. +fn cstr(s: &str) -> CString { + CString::new(s).expect("string literal must not contain NUL bytes") +} + +/// Convert a Path to a CString. +fn path_to_cstring(path: &Path) -> Result { + CString::new(path.to_string_lossy().as_bytes()) + .map_err(|_| "path contains null byte".to_string()) +} + +// Unix-only: virtio-net is the sole caller and is itself unix-gated. +#[cfg(unix)] +fn create_unix_stream_pair() -> std::io::Result<(RawFd, RawFd)> { + let mut fds = [0; 2]; + // SAFETY: `socketpair` initializes both descriptors on success. + let result = unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_STREAM, 0, fds.as_mut_ptr()) }; + if result < 0 { + return Err(std::io::Error::last_os_error()); + } + Ok((fds[0], fds[1])) +} + +/// Raise file descriptor limits (required by libkrun). +fn raise_fd_limits() { + // rlimit is a unix concept; no-op on Windows. The function stays callable + // on all platforms so its (unconditional) call sites need no gating. + #[cfg(unix)] + unsafe { + let mut limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + + if libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) == 0 { + limit.rlim_cur = limit.rlim_max; + libc::setrlimit(libc::RLIMIT_NOFILE, &limit); + } + } +} diff --git a/src/agent/manager.rs b/src/agent/manager.rs new file mode 100644 index 0000000..c59eb51 --- /dev/null +++ b/src/agent/manager.rs @@ -0,0 +1,2842 @@ +//! Agent VM lifecycle management. +//! +//! The AgentManager is responsible for starting and stopping the agent VM, +//! which runs the smolvm-agent for OCI image management and command execution. + +use crate::data::validate_vm_name; +use crate::error::{Error, Result}; +use crate::process::{self, ChildProcess}; +use crate::storage::{DiskFormat, OverlayDisk, StorageDisk}; +use parking_lot::Mutex; +use smolvm_protocol::AGENT_READY_MARKER; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use super::launcher; +use super::{HostMount, PortMapping, VmResources}; + +// ============================================================================ +// Configuration Constants +// ============================================================================ + +/// Timeout for the agent to become ready after starting. +const AGENT_READY_TIMEOUT: Duration = Duration::from_secs(30); + +// Re-use shared polling constants from process module. +use crate::process::FAST_POLL_INTERVAL; + +/// Timeout for agent to stop gracefully before force kill. +/// Reduced from 5s - VMs typically exit within 100ms after shutdown signal. +const AGENT_STOP_TIMEOUT: Duration = Duration::from_secs(2); + +/// Timeout when waiting for agent to stop. +const WAIT_FOR_STOP_TIMEOUT: Duration = Duration::from_secs(10); + +/// Running VM configuration persisted to disk so new CLI invocations +/// can restore the actual config of a detached VM. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +struct RunningVmConfig { + /// Schema version for forward compatibility. + #[serde(default = "RunningVmConfig::default_version")] + version: u32, + mounts: Vec, + ports: Vec, + resources: VmResources, +} + +impl RunningVmConfig { + const CURRENT_VERSION: u32 = 1; + + fn default_version() -> u32 { + 1 + } +} + +/// Whether the in-memory VM config is trustworthy. +#[derive(Debug, Clone)] +enum ConfigState { + /// Config was never populated (fresh manager, no reconnect yet). + Unknown, + /// Config was set during VM start or restored from disk on reconnect. + Known, + /// Config file was missing or corrupt on reconnect — cannot trust defaults. + LoadFailed(String), +} + +/// State of the agent VM. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AgentState { + /// Agent is not running. + Stopped, + /// Agent is starting up. + Starting, + /// Agent is running and ready. + Running, + /// Agent is shutting down. + Stopping, +} + +impl std::fmt::Display for AgentState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AgentState::Stopped => write!(f, "stopped"), + AgentState::Starting => write!(f, "starting"), + AgentState::Running => write!(f, "running"), + AgentState::Stopping => write!(f, "stopping"), + } + } +} + +/// Get the Docker config directory path. +/// +/// Checks DOCKER_CONFIG environment variable first, then falls back to ~/.docker/ +pub fn docker_config_dir() -> Option { + // Check DOCKER_CONFIG env var first + if let Ok(docker_config) = std::env::var("DOCKER_CONFIG") { + let path = PathBuf::from(docker_config); + if path.exists() { + return Some(path); + } + tracing::debug!( + path = %path.display(), + "DOCKER_CONFIG path does not exist" + ); + } + + // Fall back to ~/.docker/ + if let Some(home) = dirs::home_dir() { + let docker_dir = home.join(".docker"); + if docker_dir.exists() { + return Some(docker_dir); + } + } + + None +} + +/// Create a HostMount for Docker config directory. +/// +/// Returns Some(mount) if the Docker config directory exists, +/// None otherwise. +pub fn docker_config_mount() -> Option { + let docker_dir = docker_config_dir()?; + + tracing::info!( + path = %docker_dir.display(), + "mounting Docker config directory" + ); + + // Mount to /root/.docker which is where crane looks by default + // Use read-only mount to prevent modification + Some(HostMount { + source: docker_dir, + target: PathBuf::from("/root/.docker"), + read_only: true, + }) +} + +/// Internal state shared between threads. +struct AgentInner { + state: AgentState, + /// Child process (if running). + child: Option, + /// Currently configured mounts. + mounts: Vec, + /// Currently configured port mappings. + ports: Vec, + /// Currently configured VM resources. + resources: VmResources, + /// Whether the in-memory config is trustworthy. + config_state: ConfigState, + /// If true, the agent has been detached and should not be stopped on drop. + detached: bool, + /// True for the most recent launch via a fork snapshot. A clone resumes past + /// boot and never (re)writes the `.smolvm-ready` marker, so `wait_for_ready` + /// must detect readiness by pinging the restored agent instead. Set per-launch + /// in `start_via_subprocess` from `LaunchFeatures.snapshot_dir` — this carries + /// the flag without a process-global env var (unsafe in the multithreaded + /// `serve` process where concurrent forks would race). + is_clone: bool, + /// Held while the VM is running. Released on stop/Drop to allow other + /// processes to start the VM. The kernel releases the lock automatically + /// if the process crashes. + #[cfg(unix)] + vm_lock_handle: Option, +} + +/// Get the data directory for a named VM. +/// +/// Uses a fixed-length hash of the name as the directory name so the socket +/// path length is constant regardless of the name. This lets us support +/// arbitrary-length VM names portably across hosts — the kernel's +/// `sockaddr_un.sun_path` limit (~104 bytes) applies to the full socket +/// path, and a 16-char hash keeps that path bounded. +/// +/// Layout: `/smolvm/vms//` +/// - `` = first 16 hex chars (8 bytes) of SHA-256 of the name +/// - A plaintext `name` file inside the directory records the original +/// name. This is load-bearing: [`ensure_vm_dir`] reads it to detect +/// hash collisions. External tooling can use it for debugging too. +/// +/// **No legacy fallback, no migration**: smolvm is alpha. VMs created under +/// any older layout scheme are not readable by this version — users recreate +/// them. Dual-path support would silently expire VMs when their legacy +/// name-path exceeds the kernel socket budget, so we don't offer it. +pub fn vm_data_dir(name: &str) -> PathBuf { + vm_cache_root().join(vm_dir_hash(name)) +} + +/// Node-shared, content-addressed pack store: `/_shared`. Each +/// build-constant pack is extracted here once per node under `/` +/// (root-owned, read-only) and presented to every machine via a per-VM idmapped +/// bind mount — instead of a private per-machine extraction + chown. The `_` +/// prefix cannot collide with a 16-hex [`vm_dir_hash`], so it sits safely beside +/// the per-machine data dirs on the same filesystem. +pub fn shared_pack_cache_root() -> PathBuf { + vm_cache_root().join("_shared") +} + +/// Actual host disk consumed by a machine's data dir, in MiB. Sums *real blocks* +/// (`st_blocks × 512`), not apparent file lengths — the disk images are sparse, so +/// a 20 GiB image that the guest has barely written to consumes only a few MiB. +/// This is the gauge the control integrates over time for active-disk billing. +/// `None` if the dir can't be read (machine gone / not yet created). +#[cfg(target_os = "linux")] +pub fn disk_used_mb(name: &str) -> Option { + use std::os::unix::fs::MetadataExt; + fn walk_blocks(dir: &Path) -> u64 { + let Ok(entries) = std::fs::read_dir(dir) else { + return 0; + }; + let mut total = 0u64; + for entry in entries.flatten() { + let Ok(meta) = entry.metadata() else { continue }; + if meta.is_dir() { + total = total.saturating_add(walk_blocks(&entry.path())); + } else { + // st_blocks is in 512-byte units regardless of the fs block size. + total = total.saturating_add(meta.blocks().saturating_mul(512)); + } + } + total + } + let dir = vm_data_dir(name); + if !dir.exists() { + return None; + } + Some(walk_blocks(&dir) / (1024 * 1024)) +} + +/// macOS host has no VMs (dev stub) — no disk to measure. +#[cfg(not(target_os = "linux"))] +pub fn disk_used_mb(_name: &str) -> Option { + None +} + +/// Resolve the on-disk image for a `.raw` disk filename in `dir`. A fork clone +/// has a `.qcow2` copy-on-write overlay in place of the raw disk, so prefer that +/// when present; otherwise fall back to the raw disk. The file on disk is the +/// single source of truth for the format (no format is stored in the record). +pub fn resolve_disk_image(dir: &Path, raw_filename: &str) -> (PathBuf, DiskFormat) { + let qcow2 = dir.join(Path::new(raw_filename).with_extension("qcow2")); + if qcow2.exists() { + (qcow2, DiskFormat::Qcow2) + } else { + (dir.join(raw_filename), DiskFormat::Raw) + } +} + +/// Per-machine extraction directory for a `.smolmachine` bundle's OCI layers. +/// +/// Unlike the shared content-addressed pack cache (`smolvm-pack/`), +/// this lives *under* the machine's own [`vm_data_dir`], which means: +/// - it is reclaimed for free when the data dir is removed on delete; +/// - it is outside `pack prune`'s scope (never reaped while the machine exists); +/// - the macOS case-sensitive layers volume is owned 1:1 by the machine, so the +/// stop/delete paths can detach it unconditionally with no co-tenant risk. +/// +/// The subdir is `pack` (deliberately not `layers`) so it cannot collide with +/// the `layers/` subtree that `extract_sidecar` creates *inside* this directory. +pub fn machine_layers_cache_dir(name: &str) -> PathBuf { + vm_data_dir(name).join("pack") +} + +/// Filename of the shared-pack pointer dropped beside a machine's +/// [`machine_layers_cache_dir`] when create extracted the pack into the node's +/// shared content-addressed store (`_shared/`) instead of a private +/// per-machine copy. Its contents are the absolute path of that shared copy. +pub const SHARED_PACK_POINTER: &str = ".pack-shared"; + +/// Path of the shared-pack pointer for a machine, given its layers cache dir +/// (`/pack`). The pointer sits in the parent (``) so it +/// is not shadowed when the `pack` mountpoint is idmap-bound at boot. +pub fn shared_pack_pointer_path(layers_cache_dir: &std::path::Path) -> PathBuf { + layers_cache_dir + .parent() + .unwrap_or(layers_cache_dir) + .join(SHARED_PACK_POINTER) +} + +/// Read the shared-pack pointer for a machine, returning the shared copy's path +/// iff the pointer exists and names an existing directory. A stale pointer (the +/// shared copy was evicted) reads as `None`, so callers fall back to the +/// per-machine extraction path. +pub fn read_shared_pack_pointer(layers_cache_dir: &std::path::Path) -> Option { + let raw = std::fs::read_to_string(shared_pack_pointer_path(layers_cache_dir)).ok()?; + let shared = PathBuf::from(raw.trim()); + shared.is_dir().then_some(shared) +} + +/// Per-VM egress telemetry file: `/egress`. The launcher (running +/// in the VM subprocess) periodically writes the NIC's cumulative egress byte +/// count here; serve (the parent) reads it when building `MachineInfo`, so +/// egress reaches the node API through the same per-VM dir that already bridges +/// sockets and console between the two processes. Resolved from the name on both +/// sides, so no path needs to be threaded across the process boundary. +pub fn egress_telemetry_file(name: &str) -> PathBuf { + vm_data_dir(name).join("egress") +} + +/// How often the VM subprocess flushes its egress counter to disk. The control +/// plane's egress rollup runs on a multi-minute cadence, so a value this small +/// keeps the file comfortably fresh while writing only a few bytes. +// Used only by the (Unix-only) virtio-net launch path's egress flusher. +#[cfg_attr(not(unix), allow(dead_code))] +const EGRESS_FLUSH_SECS: u64 = 15; + +/// Spawn a detached thread (in the VM subprocess) that periodically writes the +/// NIC's cumulative egress byte count to the per-VM telemetry file. serve reads +/// that file when building `MachineInfo`, so egress reaches the node API the +/// same way disk size does. The thread exits when the subprocess does; the last +/// value persists in the file even after exit, so a stopped machine's final +/// egress is still readable. Best-effort: a write error never affects the VM. +#[cfg_attr(not(unix), allow(dead_code))] +pub fn spawn_egress_flush( + path: std::path::PathBuf, + counter: std::sync::Arc, +) { + std::thread::spawn(move || loop { + let bytes = counter.load(std::sync::atomic::Ordering::Relaxed); + if let Err(e) = std::fs::write(&path, bytes.to_string()) { + tracing::debug!(path = ?path, error = %e, "egress telemetry flush failed"); + } + std::thread::sleep(std::time::Duration::from_secs(EGRESS_FLUSH_SECS)); + }); +} + +/// Read the per-VM egress telemetry file written by [`spawn_egress_flush`]. +/// Returns `None` if the file is absent (TSI VM, or not yet flushed) or +/// unparseable — egress is simply unavailable for that machine. +pub fn read_egress_telemetry(name: &str) -> Option { + std::fs::read_to_string(egress_telemetry_file(name)) + .ok()? + .trim() + .parse() + .ok() +} + +/// Cache root: `/smolvm/vms/`. +pub fn vm_cache_root() -> PathBuf { + dirs::cache_dir() + .or_else(dirs::data_local_dir) + .unwrap_or_else(|| PathBuf::from("/tmp")) + .join("smolvm") + .join("vms") +} + +/// Per-node registry for the collision-free per-VM uid allocator +/// (`/smolvm/uids/`), a sibling of the VM data dirs. Root-managed; the +/// dropped VMMs never touch it. See `process::allocate_vm_uid`. +pub fn vm_uid_registry_dir() -> PathBuf { + vm_cache_root() + .parent() + .map(|p| p.join("uids")) + .unwrap_or_else(|| PathBuf::from("/tmp/smolvm-uids")) +} + +/// Compute the 16-hex-char directory name for a VM. +/// +/// Uses SHA-256 truncated to 8 bytes. The specific hash function doesn't +/// matter much — we need stability (same input → same output across runs +/// and hosts) and collision resistance; SHA-256 was already in the dep +/// tree via smolvm-pack and smolvm-registry. +/// +/// **Threat model**: 8 bytes = 64 bits. Accidental collisions among +/// non-adversarial names become likely around 2^32 distinct VMs — not a +/// concern. Adversarial collisions (an attacker picking a name that +/// hashes to the same directory as an existing VM) take ~2^32 work, a +/// few hours on a laptop. This is acceptable for single-user smolvm. A +/// future multi-tenant deployment (smolfleet) should add per-tenant +/// namespacing or a longer hash. +pub fn vm_dir_hash(name: &str) -> String { + use sha2::{Digest, Sha256}; + let digest = Sha256::digest(name.as_bytes()); + hex::encode(&digest[..8]) +} + +/// Sweep stale readiness markers out of the shared agent rootfs. +/// +/// Every VM writes a per-VM marker `.smolvm-ready.` into the *shared* +/// agent rootfs, where `` is its data-dir name. `cleanup_marker_files` +/// removes a VM's own marker on clean teardown, but a crash / SIGKILL / external +/// reap leaves it behind — and `delete_vm` removes the VM's data dir, not the +/// marker in the shared rootfs — so the rootfs accumulates one stale marker per +/// VM ever booted. Under uid isolation those markers are foreign-owned `0600`, +/// which also broke `pack create` (BUG-151). Remove any marker whose VM data dir +/// (`vm_cache_root()/`) no longer exists. The host owns the rootfs +/// directory, so it can unlink the markers regardless of their file owner. +/// Best-effort: I/O errors are ignored. +pub fn prune_orphaned_ready_markers() { + if let Ok(rootfs) = AgentManager::default_rootfs_path() { + prune_orphaned_ready_markers_in(&rootfs, &vm_cache_root()); + } +} + +/// Whether a readiness-marker file at `path` can be created (pre-created +/// empty as a side effect on success, which the Landlock carve-out relies +/// on). False on read-only rootfs installs — see +/// `AgentManager::ready_marker_writable`. +fn marker_creatable(path: &Path) -> bool { + // truncate(false): a fast guest may already have written the marker by the + // time this probe runs — truncating would erase the readiness signal and + // force the boot onto the socket path it was trying to avoid. + std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(path) + .is_ok() +} + +/// Path-injectable core of [`prune_orphaned_ready_markers`] (unit-testable). +fn prune_orphaned_ready_markers_in(rootfs: &Path, vm_cache_root: &Path) { + let prefix = format!("{}.", AGENT_READY_MARKER); + let Ok(entries) = std::fs::read_dir(rootfs) else { + return; + }; + for entry in entries.flatten() { + let file_name = entry.file_name(); + let Some(name) = file_name.to_str() else { + continue; + }; + let Some(hash) = name.strip_prefix(&prefix) else { + continue; + }; + // A marker with no hash suffix is the shared/legacy `.smolvm-ready`; leave + // it. Otherwise the marker is stale iff its VM data dir is gone. + if !hash.is_empty() && !vm_cache_root.join(hash).exists() { + let _ = std::fs::remove_file(entry.path()); + } + } +} + +/// Create the VM data directory and commit the `name → hash` binding. +/// +/// Writes (or verifies) a plaintext `name` file inside the hash directory. +/// The file is the ground truth for collision detection: if we open a hash +/// directory whose `name` file doesn't match the requested name, it means +/// two distinct VMs have hashed to the same directory — a hard error. +/// +/// Returns the created/verified directory path. +/// +/// Called from the same paths that create VM storage (manager construction, +/// agent launch setup). Safe to call repeatedly: the `name` file is written +/// once and verified on subsequent calls. +pub fn ensure_vm_dir(name: &str) -> std::io::Result { + ensure_vm_dir_at(&vm_data_dir(name), name) +} + +/// Lower-level form of [`ensure_vm_dir`] that operates on an explicit +/// directory path. Factored out for testability — callers in production +/// should use [`ensure_vm_dir`]. +pub fn ensure_vm_dir_at(dir: &std::path::Path, name: &str) -> std::io::Result { + std::fs::create_dir_all(dir)?; + + let name_file = dir.join("name"); + match std::fs::read_to_string(&name_file) { + Ok(existing) if existing == name => { + // Already committed — no-op. + } + Ok(existing) => { + // Collision: the hash directory already belongs to a different + // name. Refuse with a clear error; silent sharing would corrupt + // both VMs' storage. + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!( + "VM directory hash collision: requested name '{}' hashes \ + to the same directory as existing VM '{}' at {}. \ + Rename one of them.", + name, + existing.trim_end(), + dir.display(), + ), + )); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // First time — write the binding. Done once, never overwritten. + std::fs::write(&name_file, name.as_bytes())?; + } + Err(e) => return Err(e), + } + Ok(dir.to_path_buf()) +} + +/// Agent VM manager. +/// +/// Manages the lifecycle of the agent VM which handles OCI image operations +/// and command execution. +/// +/// Each VM gets its own agent with isolated paths under +/// `~/.cache/smolvm/vms/{name}/` (socket, PID file, storage, overlay). +pub struct AgentManager { + /// VM name (None only for low-level `new()` callers; CLI always sets a name). + name: Option, + /// Path to the agent rootfs. + rootfs_path: PathBuf, + /// Storage disk for OCI layers. + storage_disk: StorageDisk, + /// Overlay disk for persistent rootfs changes. + overlay_disk: OverlayDisk, + /// vsock socket path for control channel. + vsock_socket: PathBuf, + /// PID file path for tracking the VM process across CLI invocations. + pid_file: PathBuf, + /// Config file path for persisting running VM config across CLI invocations. + config_file: PathBuf, + /// Console log path (optional). + console_log: Option, + /// Startup error log path written by the child if machine launch fails before readiness + startup_error_log: PathBuf, + /// Per-VM lock file for cross-process coordination. + /// + /// Acquired with flock(LOCK_EX) before spawn and held through PID file + /// write. Prevents two processes from starting the same VM simultaneously. + /// The kernel releases the lock on process exit (crash-safe). + #[cfg(unix)] + vm_lock: PathBuf, + /// Internal state. + inner: Arc>, +} + +impl AgentManager { + /// Create a new agent manager with explicit paths (low-level). + /// + /// # Arguments + /// + /// * `rootfs_path` - Path to the agent VM rootfs + /// * `storage_disk` - Storage disk for OCI layers + /// * `overlay_disk` - Overlay disk for persistent rootfs changes + pub fn new( + rootfs_path: impl Into, + storage_disk: StorageDisk, + overlay_disk: OverlayDisk, + ) -> Result { + Self::new_internal(None, rootfs_path.into(), storage_disk, overlay_disk) + } + + /// Create a new agent manager for a named VM. + /// + /// Each named VM gets isolated paths for socket, storage, and logs. + pub fn new_named( + name: impl Into, + rootfs_path: impl Into, + storage_disk: StorageDisk, + overlay_disk: OverlayDisk, + ) -> Result { + Self::new_internal( + Some(name.into()), + rootfs_path.into(), + storage_disk, + overlay_disk, + ) + } + + /// Internal constructor. + fn new_internal( + name: Option, + rootfs_path: PathBuf, + storage_disk: StorageDisk, + overlay_disk: OverlayDisk, + ) -> Result { + if let Some(ref vm_name) = name { + validate_vm_name(vm_name, "machine name") + .map_err(|e| Error::config("validate machine name", e))?; + } + + // Named VMs colocate runtime artifacts (sockets, logs, pid, config) in + // their hash-derived data directory — matching where `storage_disk` + // lives via `ensure_vm_dir` and what `vm_data_dir` / `machine data-dir` + // report. Using the hash path bounds socket paths under the + // `sockaddr_un.sun_path` budget (104 bytes macOS / 108 Linux) for any + // VM name length. + // + // Unnamed VMs (ephemeral) don't have a data dir, so they fall back to + // the platform runtime dir (`/run/user//smolvm` on Linux, + // `~/Library/Caches/smolvm` on macOS) — shared across ephemeral runs. + let smolvm_runtime = if let Some(ref vm_name) = name { + vm_data_dir(vm_name) + } else { + dirs::runtime_dir() + .or_else(dirs::cache_dir) + .unwrap_or_else(|| PathBuf::from("/tmp")) + .join("smolvm") + }; + std::fs::create_dir_all(&smolvm_runtime)?; + + let vsock_socket = smolvm_runtime.join("agent.sock"); + let pid_file = smolvm_runtime.join("agent.pid"); + let config_file = smolvm_runtime.join("agent.config.json"); + let console_log = Some(smolvm_runtime.join("agent-console.log")); + let startup_error_log: PathBuf = smolvm_runtime.join("agent-startup-error.log"); + #[cfg(unix)] + let vm_lock = smolvm_runtime.join("vm.lock"); + + Ok(Self { + name, + rootfs_path, + storage_disk, + overlay_disk, + vsock_socket, + pid_file, + config_file, + console_log, + startup_error_log, + #[cfg(unix)] + vm_lock, + inner: Arc::new(Mutex::new(AgentInner { + state: AgentState::Stopped, + child: None, + mounts: Vec::new(), + ports: Vec::new(), + resources: VmResources::default(), + config_state: ConfigState::Unknown, + detached: false, + is_clone: false, + #[cfg(unix)] + vm_lock_handle: None, + })), + }) + } + + /// Get the default agent manager. + /// + /// Uses default paths for rootfs and storage. + /// `storage_gb` and `overlay_gb` override the default disk sizes (20 GiB / 10 GiB). + /// + /// Canonicalized to `for_vm_with_sizes("default", ...)` so that all + /// lifecycle commands (start/stop/exec/status) use consistent paths. + pub fn new_default_with_sizes( + storage_gb: Option, + overlay_gb: Option, + ) -> Result { + Self::for_vm_with_sizes("default", storage_gb, overlay_gb) + } + + /// Get the default agent manager with default sizes. + /// + /// Canonicalized to `for_vm("default")` so that all lifecycle commands + /// use consistent socket/PID/storage paths. + pub fn new_default() -> Result { + Self::for_vm("default") + } + + /// Get an agent manager for a named VM. + /// + /// Each named VM gets its own isolated storage and socket. + /// `storage_gb` and `overlay_gb` override the default disk sizes (20 GiB / 10 GiB). + pub fn for_vm_with_sizes( + name: impl Into, + storage_gb: Option, + overlay_gb: Option, + ) -> Result { + let name = name.into(); + let rootfs_path = Self::default_rootfs_path()?; + let sg = storage_gb.unwrap_or(crate::storage::DEFAULT_STORAGE_SIZE_GIB); + let og = overlay_gb.unwrap_or(crate::storage::DEFAULT_OVERLAY_SIZE_GIB); + + // Named VMs get their own storage disk. `ensure_vm_dir` commits the + // name→hash binding on first call and detects collisions on + // subsequent calls (refusing to open a hash dir that belongs to a + // different name). + let storage_dir = ensure_vm_dir(&name)?; + + // A fork clone has a `.qcow2` copy-on-write overlay in place of the + // `.raw` disk; detect it by file presence (the on-disk file is the + // source of truth) and open it as-is rather than creating/formatting. + let (storage_path, storage_format) = + resolve_disk_image(&storage_dir, crate::storage::STORAGE_DISK_FILENAME); + let storage_disk = match storage_format { + DiskFormat::Qcow2 => { + StorageDisk::open_existing_with_format(&storage_path, storage_format)? + } + // Fresh disk: prefer an instant qcow2 CoW overlay over the template + // (Linux, default size) instead of a raw copy, to avoid per-boot + // host-disk thrash under concurrency. Falls back to raw otherwise. + DiskFormat::Raw => StorageDisk::open_or_overlay_at(&storage_path, sg)?, + }; + + let (overlay_path, overlay_format) = + resolve_disk_image(&storage_dir, crate::storage::OVERLAY_DISK_FILENAME); + let overlay_disk = match overlay_format { + DiskFormat::Qcow2 => { + OverlayDisk::open_existing_with_format(&overlay_path, overlay_format)? + } + DiskFormat::Raw => OverlayDisk::open_or_overlay_at(&overlay_path, og)?, + }; + + Self::new_named(name, rootfs_path, storage_disk, overlay_disk) + } + + /// Get an agent manager for a named VM with default sizes. + pub fn for_vm(name: impl Into) -> Result { + Self::for_vm_with_sizes(name, None, None) + } + + /// Get the VM name if this is a named agent. + pub fn name(&self) -> Option<&str> { + self.name.as_deref() + } + + /// Names of VMs forked from this one. Their block disks are copy-on-write + /// overlays backed by this VM's disks, so it must not be re-launched with + /// writable disks while they exist. Best-effort: on a registry read error, + /// returns empty rather than blocking the launch. + fn dependent_clones(&self) -> Vec { + let Some(name) = self.name() else { + return Vec::new(); // the unnamed/default manager is never a fork base + }; + match crate::db::SmolvmDb::open().and_then(|db| db.dependent_clones(name)) { + Ok(clones) => clones, + Err(e) => { + tracing::warn!(vm = name, error = %e, "could not check for dependent clones"); + Vec::new() + } + } + } + + /// Get the default path for the agent rootfs. + /// + /// Checks `SMOLVM_AGENT_ROOTFS` env var first, then falls back to the + /// platform data directory (`~/.local/share/smolvm/agent-rootfs` on Linux, + /// `~/Library/Application Support/smolvm/agent-rootfs` on macOS). + pub fn default_rootfs_path() -> Result { + if let Ok(path) = std::env::var("SMOLVM_AGENT_ROOTFS") { + return Ok(PathBuf::from(path)); + } + + // SDKs bundle the rootfs as a tarball (they can't ship a dir tree with + // symlinks/modes through a wheel) and point us at it. Extract it once to a + // cache dir and use that, so `npm i` / `pip install` is self-contained + // with no separate engine install. Re-extracts when the tarball changes + // (a new SDK version ships a newer agent). + if let Some(tar) = std::env::var_os("SMOLVM_AGENT_ROOTFS_TAR") { + return Self::ensure_extracted_rootfs(Path::new(&tar)); + } + + // Distribution layout: the binary sits next to its rootfs. The macOS / + // Linux tarballs use a wrapper script that sets SMOLVM_AGENT_ROOTFS, but + // the Windows release ships `smolvm.exe` with no wrapper, so resolve the + // rootfs relative to the executable: prefer an already-extracted + // `agent-rootfs/` dir, else extract a bundled `agent-rootfs.tar[.gz]` + // once (a `.zip` can't carry the Linux dir tree with its symlinks/modes). + if let Some(exe_dir) = std::env::current_exe() + .ok() + .and_then(|p| p.parent().map(Path::to_path_buf)) + { + let dir = exe_dir.join("agent-rootfs"); + if dir.is_dir() { + return Ok(dir); + } + for name in ["agent-rootfs.tar.gz", "agent-rootfs.tar"] { + let tar = exe_dir.join(name); + if tar.is_file() { + return Self::ensure_extracted_rootfs(&tar); + } + } + } + + let data_dir = dirs::data_local_dir() + .or_else(dirs::data_dir) + .ok_or_else(|| Error::storage("resolve path", "could not determine data directory"))?; + + Ok(data_dir.join("smolvm").join("agent-rootfs")) + } + + /// Extract a bundled agent-rootfs tarball to a cache dir (idempotent) and + /// return that dir. Keyed by the tarball's size+mtime so a newer SDK build + /// re-extracts; extraction is staged in a temp dir then atomically renamed so + /// concurrent SDK processes never see a half-extracted rootfs. + fn ensure_extracted_rootfs(tar: &Path) -> Result { + let meta = + std::fs::metadata(tar).map_err(|e| Error::storage("stat rootfs tar", e.to_string()))?; + let mtime = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs()) + .unwrap_or(0); + let key = format!("{:x}-{:x}", meta.len(), mtime); + + let base = dirs::cache_dir() + .ok_or_else(|| Error::storage("resolve cache dir", "no cache directory"))? + .join("smolvm") + .join("rootfs"); + let dest = base.join(&key); + if dest.join(".extracted").exists() { + return Ok(dest); + } + + std::fs::create_dir_all(&base) + .map_err(|e| Error::storage("create rootfs cache", e.to_string()))?; + let tmp = base.join(format!(".tmp-{}-{}", std::process::id(), key)); + let _ = std::fs::remove_dir_all(&tmp); + std::fs::create_dir_all(&tmp) + .map_err(|e| Error::storage("create rootfs staging", e.to_string()))?; + + // Use the system `tar` — it preserves symlinks (e.g. /sbin/init) and modes + // (the executable agent), which the tar crate handling here would not need + // to reimplement. Extraction runs on the host before VM boot (unsandboxed). + let status = std::process::Command::new("tar") + .arg("-xpf") + .arg(tar) + .arg("-C") + .arg(&tmp) + .status() + .map_err(|e| Error::storage("extract rootfs tar", e.to_string()))?; + if !status.success() { + // On Windows, `tar` can't recreate the rootfs's busybox symlinks + // without symlink privilege (Developer Mode / elevation) and exits + // non-zero with warnings — but the real files (notably the agent) + // still extract. Treat the result as usable as long as the agent + // binary landed; otherwise it's a genuine extraction failure. + let agent_present = tmp.join("usr/local/bin/smolvm-agent").exists(); + if !agent_present { + let _ = std::fs::remove_dir_all(&tmp); + return Err(Error::storage( + "extract rootfs tar", + format!("tar exited with {status} and the agent binary was not extracted"), + )); + } + tracing::warn!( + "rootfs tar extraction exited with {status} (host could not create some \ + symlinks); continuing — the agent binary extracted successfully" + ); + } + let _ = std::fs::write(tmp.join(".extracted"), b""); + // Atomic publish; if another process won the race, just use theirs. + match std::fs::rename(&tmp, &dest) { + Ok(()) => {} + Err(_) if dest.join(".extracted").exists() => { + let _ = std::fs::remove_dir_all(&tmp); + } + Err(e) => { + let _ = std::fs::remove_dir_all(&tmp); + return Err(Error::storage("publish extracted rootfs", e.to_string())); + } + } + Ok(dest) + } + + /// Get the current state of the agent. + pub fn state(&self) -> AgentState { + self.inner.lock().state + } + + /// Check if the agent is running. + pub fn is_running(&self) -> bool { + self.state() == AgentState::Running + } + + /// If cached state is Running but the process is not actually alive, + /// reset to Stopped so that start paths can proceed. This handles the + /// case where a VM crashed without going through `stop()`. + fn reset_stale_running_state(&self) { + let mut inner = self.inner.lock(); + if inner.state == AgentState::Running && !self.is_process_alive_inner(&inner) { + tracing::info!("resetting stale Running state to Stopped (VM process is dead)"); + inner.state = AgentState::Stopped; + inner.child = None; + #[cfg(unix)] + { + inner.vm_lock_handle = None; + } + } + } + + /// Force the in-memory state back to `Stopped` and release the per-VM lock, + /// after the VM process has already been stopped out-of-band (the HTTP stop + /// path kills the recorded PID directly rather than via this manager). + /// + /// The serve process holds the `vm.lock` flock for the lifetime of the + /// registry's `AgentManager`; if `vm_lock_handle` is not dropped here, the + /// serve process keeps holding the lock and a subsequent start fails to + /// re-acquire it ("another process is already starting or running this VM"). + /// Only call once the process is confirmed dead. + pub fn mark_stopped(&self) { + let mut inner = self.inner.lock(); + inner.state = AgentState::Stopped; + inner.child = None; + #[cfg(unix)] + { + inner.vm_lock_handle = None; + } + } + + /// Return consistent (state, pid) for API status responses. + /// + /// Clears the PID when effective state is `Stopped`, so clients never + /// see a stale PID paired with a stopped state. + pub fn effective_status(&self) -> (AgentState, Option) { + let inner = self.inner.lock(); + let state = if inner.state == AgentState::Running && !self.is_process_alive_inner(&inner) { + AgentState::Stopped + } else { + inner.state + }; + let pid = if state == AgentState::Stopped { + None + } else { + inner.child.as_ref().map(|c| c.pid()) + }; + (state, pid) + } + + /// Get the vsock socket path. + pub fn vsock_socket(&self) -> &Path { + &self.vsock_socket + } + + /// Get the console log path. + pub fn console_log(&self) -> Option<&Path> { + self.console_log.as_deref() + } + + /// Get the storage disk path. + pub fn storage_path(&self) -> &Path { + self.storage_disk.path() + } + + /// Get the overlay disk path. + pub fn overlay_path(&self) -> &Path { + self.overlay_disk.path() + } + + /// Check if an agent is already running (socket exists + responds to ping). + /// + /// Returns Some(()) if agent is running and reachable, None otherwise. + /// This also updates the internal state to Running if successful. + pub fn try_connect_existing(&self) -> Option<()> { + self.try_connect_existing_with_pid(None) + } + + /// Try to reconnect to an existing agent with a known PID. + /// + /// If the PID is provided and the process is alive, sets the child process. + /// Falls back to reading the PID file if no PID is provided. + /// Returns Some(()) if agent is running and reachable, None otherwise. + pub fn try_connect_existing_with_pid(&self, pid: Option) -> Option<()> { + self.try_connect_existing_with_pid_and_start_time(pid, None) + } + + /// Try to reconnect to an existing agent with a known PID and expected start time. + /// + /// The `expected_start_time` is the start time stored when the VM was originally + /// launched. If provided, it is used to verify the PID hasn't been recycled by the OS. + pub fn try_connect_existing_with_pid_and_start_time( + &self, + pid: Option, + expected_start_time: Option, + ) -> Option<()> { + if !self.vsock_socket.exists() { + return None; + } + + // Resolve PID and start time. + // If caller provides expected_start_time, use it (DB source of truth). + // Otherwise fall back to PID file which stores both PID and start time. + let (effective_pid, pid_start_time) = if let Some(p) = pid { + ( + Some(p), + expected_start_time.or_else(|| { + // Caller didn't provide start time — try PID file as fallback + self.read_pid_file_with_start_time() + .and_then(|(file_pid, st)| if file_pid == p { st } else { None }) + }), + ) + } else { + match self.read_pid_file_with_start_time() { + Some((p, st)) => (Some(p), st), + None => (None, None), + } + }; + + // Try to ping the agent + if let Ok(mut client) = super::AgentClient::connect(&self.vsock_socket) { + if client.ping().is_ok() { + // Update internal state to reflect running + let mut inner = self.inner.lock(); + inner.state = AgentState::Running; + // Only store child PID if identity is verified via start time. + // Without verification, stop() could signal the wrong process. + if let Some(p) = effective_pid { + if process::is_our_process_strict(p, pid_start_time) { + inner.child = Some(ChildProcess::new(p)); + } else { + tracing::debug!( + pid = p, + "skipping child PID storage: identity not verified" + ); + } + } + // Restore the running VM config from disk so that + // ensure_running_with_full_config can accurately compare + // the requested config against the actual running config. + if matches!(inner.config_state, ConfigState::Unknown) { + match self.load_running_config() { + Ok(config) => { + inner.mounts = config.mounts; + inner.ports = config.ports; + inner.resources = config.resources; + inner.config_state = ConfigState::Known; + } + Err(reason) => { + tracing::warn!( + reason = %reason, + "could not restore running VM config; \ + config changes will force restart" + ); + inner.config_state = ConfigState::LoadFailed(reason); + } + } + } + return Some(()); + } + } + + None + } + + /// Read PID and start time from the PID file. + fn read_pid_file_with_start_time(&self) -> Option<(i32, Option)> { + let content = std::fs::read_to_string(&self.pid_file).ok()?; + let mut lines = content.lines(); + let pid = lines.next()?.trim().parse::().ok()?; + let start_time = lines.next().and_then(|s| s.trim().parse::().ok()); + Some((pid, start_time)) + } + + /// Save the running VM config to disk so future CLI invocations can + /// restore the actual config of a detached VM on reconnect. + /// + /// Uses atomic write (tmp + rename) to avoid partial/corrupt reads. + fn save_running_config( + &self, + mounts: &[HostMount], + ports: &[PortMapping], + resources: &VmResources, + ) { + let config = RunningVmConfig { + version: RunningVmConfig::CURRENT_VERSION, + mounts: mounts.to_vec(), + ports: ports.to_vec(), + resources: resources.clone(), + }; + match serde_json::to_string(&config) { + Ok(json) => { + let tmp = self.config_file.with_extension("json.tmp"); + if let Err(e) = std::fs::write(&tmp, &json) { + tracing::warn!(error = %e, "failed to write VM config tmp file"); + return; + } + if let Err(e) = std::fs::rename(&tmp, &self.config_file) { + tracing::warn!(error = %e, "failed to rename VM config file"); + } + } + Err(e) => { + tracing::warn!(error = %e, "failed to serialize VM config"); + } + } + } + + /// Load the running VM config from disk. + /// + /// Returns an error string describing why the load failed, so callers + /// can log it and treat the config as unknown (fail-closed). + fn load_running_config(&self) -> std::result::Result { + let content = std::fs::read_to_string(&self.config_file) + .map_err(|e| format!("config file {}: {}", self.config_file.display(), e))?; + serde_json::from_str(&content) + .map_err(|e| format!("invalid JSON in {}: {}", self.config_file.display(), e)) + } + + /// Get the child PID if known. + pub fn child_pid(&self) -> Option { + self.inner.lock().child.as_ref().map(|c| c.pid()) + } + + /// Get the VM process ID and its captured start time for verified external cleanup. + /// + /// Prefers the in-memory child handle (start time captured at spawn). + /// Falls back to the PID file if no in-memory handle is present. + /// Returns `None` if neither source has a PID. + pub fn pid_and_start_time(&self) -> Option<(i32, Option)> { + { + let inner = self.inner.lock(); + if let Some(child) = &inner.child { + return Some((child.pid(), child.start_time())); + } + } + self.read_pid_file_with_start_time() + } + + /// Check if the VM process is actually alive using start-time-aware + /// verification. + /// + /// Checks in-memory child handle first, then falls back to PID file. + /// Returns `false` when neither source provides a PID (fail-closed). + /// Uses `is_our_process` (lenient) so that a live process without + /// start-time data is assumed to be ours rather than silently ignored. + pub fn is_process_alive(&self) -> bool { + let inner = self.inner.lock(); + self.is_process_alive_inner(&inner) + } + + /// Inner liveness check that accepts a lock guard to avoid double-locking. + fn is_process_alive_inner(&self, inner: &AgentInner) -> bool { + // Try in-memory child handle first (has stored start time) + if let Some(child) = inner.child.as_ref() { + return crate::process::is_our_process(child.pid(), child.start_time()); + } + + // Fall back to PID file (covers orphan/reconnect paths) + if let Some((pid, start_time)) = self.read_pid_file_with_start_time() { + return crate::process::is_our_process(pid, start_time); + } + + // No PID source — fail closed + false + } + + /// Connect to the running agent and return a client. + /// + /// Uses retry logic to handle transient connection failures. + pub fn connect(&self) -> crate::error::Result { + super::AgentClient::connect_with_retry(&self.vsock_socket) + } + + /// Get the currently configured mounts. + pub fn mounts(&self) -> Vec { + self.inner.lock().mounts.clone() + } + + /// Check if the given mounts match the currently running agent's mounts. + pub fn mounts_match(&self, mounts: &[HostMount]) -> bool { + let inner = self.inner.lock(); + inner.mounts == mounts + } + + /// Check if the given resources match the currently running agent's resources. + pub fn resources_match(&self, resources: VmResources) -> bool { + let inner = self.inner.lock(); + inner.resources == resources + } + + /// Check if the given port mappings match the currently running agent's ports. + pub fn ports_match(&self, ports: &[PortMapping]) -> bool { + let inner = self.inner.lock(); + inner.ports == ports + } + + /// Ensure the agent is running with the specified mounts. + /// + /// If the agent is running with different mounts, it will be restarted. + pub fn ensure_running_with_mounts(&self, mounts: Vec) -> Result { + self.ensure_running_with_full_config( + mounts, + Vec::new(), + VmResources::default(), + Default::default(), + ) + } + + /// Ensure the agent is running with the specified mounts and resources. + /// + /// If the agent is running with different mounts or resources, it will be restarted. + pub fn ensure_running_with_config( + &self, + mounts: Vec, + resources: VmResources, + ) -> Result { + self.ensure_running_with_full_config(mounts, Vec::new(), resources, Default::default()) + } + + /// Re-attach this machine's pre-extracted packed layers if the caller did + /// not already wire them. + /// + /// The implicit-start preflight (`ensure_machine_running`) passes + /// `default()` features when the VM is already up, to skip the macOS hdiutil + /// mount on the exec hot path. If that preflight then detects a config + /// change and restarts, the relaunch would otherwise drop the packed layers + /// and the guest would fall back to a registry pull (broken offline). The + /// layers are discoverable from the machine name alone — derive the + /// per-machine directory, re-acquire the lease, and set `packed_layers_dir`. + /// A no-op when layers are already wired (an explicit-start path set + /// `packed_layers_dir` and its mount is still live), when this is not a named + /// machine, or when nothing was extracted (image/registry-sourced machine). + /// macOS mount cost only; a compile-time no-op on Linux. + fn rewire_packed_layers_if_extracted( + &self, + features: &mut launcher::LaunchFeatures, + ) -> Result<()> { + if features.packed_layers_dir.is_some() { + return Ok(()); + } + let Some(name) = self.name.as_deref() else { + return Ok(()); + }; + let cache_dir = machine_layers_cache_dir(name); + if !smolvm_pack::extract::is_extracted(&cache_dir) { + return Ok(()); + } + // This runs only on the relaunch path, after stop()/reset_stale_running_state, + // so no live VM is using the volume. The original start leaked a lease to keep + // it mounted; drop that stale mount (and its lease files) before acquiring a + // fresh one, so a config-change restart doesn't stack a second lease/mount on + // the first. Gated by the same conditions as the re-acquire below, so the + // explicit-start paths (features already `Some`, mount still live) return + // early above and never detach. macOS hdiutil detach; a no-op on Linux. + smolvm_pack::extract::force_detach_layers_volume(&cache_dir); + match smolvm_pack::extract::acquire_layers_lease(&cache_dir, false) { + Ok(lease) => { + features.packed_layers_dir = Some(lease.path.clone()); + // Keep the volume mounted for the VM's lifetime; the stop/delete + // handlers detach it (see `machine_layers_cache_dir`). + std::mem::forget(lease); + } + Err(e) => { + return Err(Error::agent("re-attach packed layers", e.to_string())); + } + } + Ok(()) + } + + /// Ensure the agent is running with the specified mounts, ports, and resources. + /// + /// If the agent is running with different configuration, it will be restarted. + /// Returns `true` if the VM was freshly started/restarted, `false` if reused. + pub fn ensure_running_with_full_config( + &self, + mounts: Vec, + ports: Vec, + resources: VmResources, + mut features: launcher::LaunchFeatures, + ) -> Result { + // Check if agent is already running with the same configuration. + // try_connect_existing restores config from disk on reconnect, + // so the comparison below is accurate even for detached VMs. + if self.try_connect_existing().is_some() { + let inner = self.inner.lock(); + match &inner.config_state { + ConfigState::Known => { + if inner.mounts == mounts + && inner.ports == ports + && inner.resources == resources + { + return Ok(false); + } + // Config is known but doesn't match — fall through to restart. + } + ConfigState::LoadFailed(reason) => { + // Fail-closed: cannot verify running config matches requested, + // so force restart to ensure correct isolation/network settings. + tracing::info!( + reason = %reason, + "forcing VM restart: running config unknown" + ); + } + ConfigState::Unknown => { + // This shouldn't happen (try_connect_existing always resolves + // Unknown to Known or LoadFailed), but fail-closed just in case. + tracing::info!("forcing VM restart: config state still unknown"); + } + } + } + + // If running with different/unknown config, we need to restart + let needs_restart = { + let inner = self.inner.lock(); + inner.state == AgentState::Running + }; + + if needs_restart { + tracing::info!("restarting agent VM due to configuration change"); + self.stop()?; + } else { + // try_connect_existing failed but state may still be Running (crashed VM). + // Reset to Stopped so start_with_full_config can proceed. + self.reset_stale_running_state(); + } + + // Re-attach packed layers if a config-change restart dropped them + // (see `rewire_packed_layers_if_extracted`). + self.rewire_packed_layers_if_extracted(&mut features)?; + + // Start with new config + self.start_with_full_config(mounts, ports, resources, features)?; + Ok(true) + } + + /// Ensure the agent is running. + /// + /// If the agent is not running, this starts it. + /// If the agent is already running, this is a no-op. + /// Returns `true` if the VM was freshly started, `false` if reused. + pub fn ensure_running(&self) -> Result { + // First, check if an agent is already running (from a previous invocation) + if self.try_connect_existing().is_some() { + return Ok(false); + } + + // try_connect_existing failed — if state is stale Running (crashed VM), + // reset to Stopped so we can start fresh. + self.reset_stale_running_state(); + + // Check internal state + let state = self.state(); + + match state { + AgentState::Running => Ok(false), // shouldn't reach here after reset, but safe + AgentState::Starting => { + self.wait_for_ready()?; + Ok(true) + } + AgentState::Stopped => { + self.start()?; + Ok(true) + } + AgentState::Stopping => { + self.wait_for_stop()?; + self.start()?; + Ok(true) + } + } + } + + /// Start the agent VM. + pub fn start(&self) -> Result<()> { + self.start_with_full_config( + Vec::new(), + Vec::new(), + VmResources::default(), + Default::default(), + ) + } + + /// Start the agent VM with specified mounts. + pub fn start_with_mounts(&self, mounts: Vec) -> Result<()> { + self.start_with_full_config( + mounts, + Vec::new(), + VmResources::default(), + Default::default(), + ) + } + + /// Start the agent VM with specified mounts and resources. + pub fn start_with_config(&self, mounts: Vec, resources: VmResources) -> Result<()> { + self.start_with_full_config(mounts, Vec::new(), resources, Default::default()) + } + + /// This VM's readiness-marker filename — **per VM** (`.`), not + /// the shared protocol constant. A per-VM marker means concurrent boots can't + /// race on one shared file, and under uid isolation each marker is pre-created + /// 0600 owned by that VM's uid instead of world-writable. The host passes this + /// name to the guest agent via the `SMOLVM_READY_MARKER` guest env var so both + /// sides agree. + fn ready_marker_name(&self) -> String { + let hash = self + .storage_disk + .path() + .parent() + .and_then(|p| p.file_name()) + .and_then(|n| n.to_str()) + .unwrap_or("default"); + format!("{}.{}", AGENT_READY_MARKER, hash) + } + + /// Host path of this VM's per-VM readiness marker. + fn ready_marker_path(&self) -> PathBuf { + self.rootfs_path.join(self.ready_marker_name()) + } + + /// Whether this VM's ready marker can actually be created. + /// + /// The marker lives inside the agent rootfs (the virtiofs share), so on + /// system installs where that directory is root-owned (/opt, /usr/lib) + /// and smolvm runs unprivileged, neither the host pre-create nor the + /// guest's virtiofs write can ever succeed — the marker "never appears" + /// and every boot silently pays the full socket-probe grace (#590). + /// Probing this up front lets `wait_for_ready` fall back to socket + /// polling immediately instead. The probe leaves the marker pre-created + /// empty on success, which is also what the Landlock carve-out expects + /// (readiness requires non-empty content, so this cannot false-positive). + fn ready_marker_writable(&self) -> bool { + marker_creatable(&self.ready_marker_path()) + } + + /// Common pre-launch setup: validate state, pre-format disks, clean markers. + /// + /// Called by both `start_with_full_config` (fork) and `start_via_subprocess`. + /// Sets internal state to `Starting` and stores config. Returns error if + /// the agent is not in the `Stopped` state. + fn prepare_for_launch( + &self, + mounts: &[HostMount], + ports: &[PortMapping], + resources: VmResources, + ) -> Result<()> { + // Refuse to (re)launch a fork base while clones depend on it. Clones + // CoW-read this VM's disks by path; re-running it would reopen them + // writable and silently corrupt every clone. Clones don't need the base + // process alive, so refusing is safe — delete the clones first to reuse + // the name. Covers every launch path (CLI fork + subprocess) since both + // funnel through here. + let clones = self.dependent_clones(); + if !clones.is_empty() { + return Err(Error::agent( + "start agent", + format!( + "'{}' is a fork base for {} live clone(s) ({}); their disks are \ + copy-on-write overlays backed by its disks, so it cannot be \ + re-launched while they exist — delete the clones first", + self.name().unwrap_or_default(), + clones.len(), + clones.join(", ") + ), + )); + } + + // Validate resources before doing anything else. + resources.validate()?; + + // Acquire the per-VM file lock BEFORE checking state. This serializes + // concurrent start attempts across OS processes. The lock is held + // until stop/Drop releases it. If another process already holds the + // lock (VM is running), we block briefly then re-check state. + #[cfg(unix)] + let lock_handle = { + use std::os::unix::io::AsRawFd; + let lock_file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(&self.vm_lock) + .map_err(|e| Error::agent("acquire VM lock", e.to_string()))?; + let ret = unsafe { libc::flock(lock_file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if ret != 0 { + let err = std::io::Error::last_os_error(); + if err.raw_os_error() == Some(libc::EWOULDBLOCK) { + return Err(Error::agent( + "start agent", + "another process is already starting or running this VM", + )); + } + return Err(Error::agent("acquire VM lock", err.to_string())); + } + lock_file + }; + + // Check and update state + { + let mut inner = self.inner.lock(); + if inner.state != AgentState::Stopped { + return Err(Error::agent( + "start agent", + "agent already starting or running", + )); + } + inner.state = AgentState::Starting; + inner.mounts = mounts.to_vec(); + inner.ports = ports.to_vec(); + inner.resources = resources; + inner.config_state = ConfigState::Known; + #[cfg(unix)] + { + inner.vm_lock_handle = Some(lock_handle); + } + } + + tracing::info!( + rootfs = %self.rootfs_path.display(), + storage = %self.storage_disk.path().display(), + socket = %self.vsock_socket.display(), + mount_count = mounts.len(), + "preparing agent VM launch" + ); + + // Check KVM availability on Linux + #[cfg(target_os = "linux")] + { + if let Err(e) = crate::platform::linux::check_kvm_available() { + let mut inner = self.inner.lock(); + inner.state = AgentState::Stopped; + return Err(e); + } + } + + // Validate rootfs exists + if !self.rootfs_path.exists() { + let mut inner = self.inner.lock(); + inner.state = AgentState::Stopped; + return Err(Error::agent( + "verify rootfs", + format!("agent rootfs not found: {}", self.rootfs_path.display()), + )); + } + + // Pre-format storage and overlay disks in parallel + { + let storage_disk = &self.storage_disk; + let overlay_disk = &self.overlay_disk; + std::thread::scope(|s| { + let storage_handle = s.spawn(|| storage_disk.ensure_formatted()); + let overlay_result = overlay_disk.ensure_formatted(); + if let Err(e) = storage_handle.join().unwrap_or_else(|_| { + Err(crate::Error::storage("format storage", "thread panicked")) + }) { + tracing::warn!( + error = %e, + "failed to pre-format disk on host" + ); + } + if let Err(e) = overlay_result { + tracing::warn!( + error = %e, + "failed to pre-format overlay disk on host" + ); + } + }); + } + + // Clean up old socket and this VM's stale (per-VM) readiness marker. + let _ = std::fs::remove_file(&self.vsock_socket); + let _ = std::fs::remove_file(self.ready_marker_path()); + let _ = std::fs::remove_file(&self.startup_error_log); + + Ok(()) + } + + /// Common post-launch bookkeeping: store child PID, write config/PID files, + /// wait for agent ready. + /// + /// Called by both `start_with_full_config` (fork) and `start_via_subprocess`. + fn finalize_launch( + &self, + child_pid: i32, + mounts: &[HostMount], + ports: &[PortMapping], + resources: &VmResources, + ) -> Result<()> { + let boot_start = std::time::Instant::now(); + + // Store child process handle + { + let mut inner = self.inner.lock(); + inner.child = Some(ChildProcess::new(child_pid)); + } + + // Write running config (for future CLI invocations to detect config changes) + self.save_running_config(mounts, ports, resources); + + // Write PID file with start time for PID reuse detection + let pid_content = match process::process_start_time(child_pid) { + Some(t) => format!("{}\n{}", child_pid, t), + None => child_pid.to_string(), + }; + if let Err(e) = std::fs::write(&self.pid_file, pid_content) { + tracing::warn!(error = %e, "failed to write PID file"); + } + + // Wait for the agent to be ready + match self.wait_for_ready() { + Ok(_) => { + let mut inner = self.inner.lock(); + inner.state = AgentState::Running; + let boot_secs = boot_start.elapsed().as_secs_f64(); + metrics::histogram!("smolvm_vm_boot_seconds").record(boot_secs); + metrics::gauge!("smolvm_machines_running").increment(1.0); + tracing::info!( + pid = child_pid, + boot_ms = boot_secs * 1000.0, + "agent VM is ready" + ); + Ok(()) + } + Err(e) => { + // The _boot-vm child may be stuck inside krun_start_enter() + // where SIGTERM alone may not kill it (the VM run loop can + // mask signals). Use the full SIGTERM -> wait -> SIGKILL + // sequence so the child is reliably dead before we return, + // preventing an orphaned process from holding ports/sockets + // and making every subsequent start attempt fail permanently. + if let Err(kill_err) = process::stop_vm_process( + child_pid, + AGENT_STOP_TIMEOUT, + process::VM_SIGKILL_TIMEOUT, + ) { + tracing::warn!( + pid = child_pid, + error = %kill_err, + "failed to kill _boot-vm child after start failure; \ + process may be orphaned" + ); + } + // Remove the PID file written earlier in this function so a + // stale PID doesn't confuse future reconnect attempts. + let _ = std::fs::remove_file(&self.pid_file); + let mut inner = self.inner.lock(); + inner.state = AgentState::Stopped; + inner.child = None; + #[cfg(unix)] + { + inner.vm_lock_handle = None; + } + Err(e) + } + } + } + + /// Start the agent VM with specified mounts, ports, and resources. + /// + /// Spawns a fresh subprocess (`smolvm _boot-vm`) via `posix_spawn` to run + /// the VM. This gives the child a completely clean process with no inherited + /// Hypervisor.framework state, preventing VM context leaks when the child + /// crashes (e.g., during GPU device setup). + /// + /// Previously used `fork()` which inherited parent state and caused + /// unreliable GPU launches on macOS. + pub fn start_with_full_config( + &self, + mounts: Vec, + ports: Vec, + resources: VmResources, + features: launcher::LaunchFeatures, + ) -> Result<()> { + // Delegate to subprocess launch — safe for both single-threaded (CLI) + // and multi-threaded (API server) callers. Required for GPU support + // (Hypervisor.framework detects forked multi-threaded state). + self.start_via_subprocess(mounts, ports, resources, features) + } + + /// Start the VM by spawning a fresh subprocess instead of fork(). + /// + /// On macOS, fork() in a multi-threaded process (e.g., from within the + /// tokio-based API server) creates unstable children: Apple frameworks + /// like Hypervisor.framework detect the forked multi-threaded state and + /// abort the child ~2 seconds after boot. + /// + /// This method avoids fork entirely by spawning a fresh `smolvm _boot-vm` + /// process via `Command::new()` (which uses `posix_spawn` on macOS). + /// The subprocess is single-threaded and runs `krun_start_enter` safely. + pub fn start_via_subprocess( + &self, + mounts: Vec, + ports: Vec, + resources: VmResources, + mut features: launcher::LaunchFeatures, + ) -> Result<()> { + use super::boot_config::BootConfig; + + let t_launch = Instant::now(); + + let resources_for_config = resources.clone(); + // Per-boot disk-prep (template copy). Formerly bounded by a process-wide + // boot gate to avoid host-disk thrash on slow/shared storage; removed + // after metal measurements showed disk-prep at ~1ms (NVMe + qcow2 thin + // clone) with flat boot latency from 8 → 32 concurrent boots — the gate + // was non-binding and only serialized needlessly. + self.prepare_for_launch(&mounts, &ports, resources)?; + tracing::info!( + elapsed_ms = t_launch.elapsed().as_millis(), + "boot: disks ready" + ); + + let storage_size_gb = resources_for_config + .storage_gib + .unwrap_or(crate::storage::DEFAULT_STORAGE_SIZE_GIB); + let overlay_size_gb = resources_for_config + .overlay_gib + .unwrap_or(crate::storage::DEFAULT_OVERLAY_SIZE_GIB); + + // Forkable / fork-clone launch params are carried PER-PROCESS — set on + // the boot subprocess's own env below (and on `self.is_clone`), never via + // `std::env::set_var`. A process-global env var is a data race in the + // multithreaded `serve` process, where concurrent forks would clobber + // each other (and `set_var` is `unsafe` in edition 2024 for that reason). + let fork_env: Vec<(&str, String)> = { + let mut v = Vec::new(); + if features.forkable { + v.push(("SMOLVM_FORKABLE", "1".to_string())); + } + if let Some(ref ctl) = features.control_socket { + v.push(("SMOLVM_CONTROL_SOCKET", ctl.to_string_lossy().into_owned())); + } + if let Some(ref snap) = features.snapshot_dir { + v.push(("SMOLVM_SNAPSHOT_DIR", snap.to_string_lossy().into_owned())); + } + v + }; + self.inner.lock().is_clone = features.snapshot_dir.is_some(); + + // Per-VM uid isolation: when running privileged (root `serve`), give this + // VMM its own dedicated, collision-free unprivileged uid so a guest→VMM + // escape is contained to one VM. We're still root here, so chown the VM's + // data dir (disks, sockets, logs) to that uid and tighten it to 0700 (so + // a sibling VM's uid — or a Landlock-exempt clone — can't read its disks); + // `internal_boot` does the actual setuid drop from the inherited + // SMOLVM_VM_UID. A fork clone shares its golden's uid (resolved from the + // snapshot path) so it can map the golden's memfd. No-op unless privileged; + // opt out with SMOLVM_VM_UID_DROP=off. See process::vm_drop_ids. + let data_dir = self.storage_disk.path().parent().map(|p| p.to_path_buf()); + let registry = vm_uid_registry_dir(); + let mut uid_env: Vec<(&str, String)> = Vec::new(); + // Shared pack store: `with_packed_layers` sets `pack_idmap_source` when + // create wrote a `_shared/` pointer, leaving `packed_layers_dir` + // an empty per-VM mountpoint. Ensure that mountpoint exists BEFORE the + // uid-drop chown_tree below, so it's owned by the VM's uid (harmless — the + // idmap bind covers it at boot) rather than left for chown to miss. + if features.pack_idmap_source.is_some() { + if let Some(ref mountpoint) = features.packed_layers_dir { + let _ = std::fs::create_dir_all(mountpoint); + } + } + // Whether the per-VM uid drop is active for this boot. The idmapped pack + // bind mount needs CAP_SYS_ADMIN (root) — the same precondition as the + // drop — so we only keep `pack_idmap_source` when the drop is active; + // otherwise the VMM stays root and reads the shared copy directly. + let mut uid_drop_active = false; + if let Some(d) = data_dir.as_deref() { + if let Some(result) = + crate::process::vm_drop_ids(®istry, d, features.snapshot_dir.as_deref()) + { + // The drop is active — allocation MUST succeed or we refuse to boot + // (fail closed; never silently run the VMM over-privileged). + let (uid, gid) = result.map_err(|e| { + Error::agent( + "allocate per-VM uid (refusing to boot over-privileged)", + e.to_string(), + ) + })?; + crate::process::chown_tree(d, uid, gid) + .map_err(|e| Error::agent("chown vm data dir for uid drop", e.to_string()))?; + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::PermissionsExt; + // 0700: a sibling VM's uid — or a Landlock-exempt clone — must + // not be able to read this VM's disks. + let _ = std::fs::set_permissions(d, std::fs::Permissions::from_mode(0o700)); + } + // The dropped uid must traverse to its 0700 data dir, the shared + // rootfs, and the disk templates. The data dir itself stays 0700; + // widen only the ancestor chains (execute-only). Resilient to a + // restrictive umask on the runtime-created intermediates. + crate::process::ensure_traversable(®istry); + if let Some(parent) = d.parent() { + crate::process::ensure_traversable(parent); + } + crate::process::ensure_traversable(&self.rootfs_path); + if let Some(home) = dirs::home_dir() { + crate::process::ensure_traversable(&home.join(".smolvm")); + } + // Pre-create this VM's per-VM readiness marker owned by its uid + // (0600): the dropped guest can overwrite it but couldn't create a + // file in the shared, host-user-owned rootfs. Per-VM name + 0600 = + // no shared-marker race and no world-writable file. + #[cfg(target_os = "linux")] + { + use std::os::unix::ffi::OsStrExt; + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + let marker = self.ready_marker_path(); + if std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .mode(0o600) + .open(&marker) + .is_ok() + { + let _ = std::fs::set_permissions( + &marker, + std::fs::Permissions::from_mode(0o600), + ); + if let Ok(c) = std::ffi::CString::new(marker.as_os_str().as_bytes()) { + unsafe { libc::lchown(c.as_ptr(), uid, uid) }; + } + } + } + tracing::info!(uid, vm_dir = %d.display(), "per-VM uid isolation enabled"); + uid_env = vec![ + ("SMOLVM_VM_UID", uid.to_string()), + ("SMOLVM_VM_GID", gid.to_string()), + ]; + uid_drop_active = true; + } + } + + // Resolve how the shared pack is presented to the guest. With the uid + // drop active, keep the idmap source so `internal_boot` (still root, in a + // private mount namespace) idmap-binds the root-owned shared copy onto the + // empty `packed_layers_dir` mountpoint, mapping on-disk uid 0 -> vm_uid. + // Without the drop (non-root `serve`, or SMOLVM_VM_UID_DROP=off), there is + // no second uid to isolate from, so collapse the indirection: point + // `packed_layers_dir` straight at the shared copy and read it as root. + let pack_idmap_source = if uid_drop_active { + features.pack_idmap_source.take() + } else { + if let Some(shared) = features.pack_idmap_source.take() { + features.packed_layers_dir = Some(shared); + } + None + }; + + // Write boot config to a file the subprocess will read + let config = BootConfig { + rootfs_path: self.rootfs_path.clone(), + storage_disk_path: self.storage_disk.path().to_path_buf(), + overlay_disk_path: self.overlay_disk.path().to_path_buf(), + vsock_socket: self.vsock_socket.clone(), + console_log: self.console_log.clone(), + startup_error_log: self.startup_error_log.clone(), + storage_size_gb, + overlay_size_gb, + mounts: mounts.clone(), + ports: ports.clone(), + resources: resources_for_config.clone(), + ssh_agent_socket: features.ssh_agent_socket, + cuda: features.cuda, + dns_filter_hosts: features.dns_filter_hosts, + packed_layers_dir: features.packed_layers_dir, + pack_idmap_source, + extra_disks: features.extra_disks, + }; + let config_path = self + .storage_disk + .path() + .parent() + .unwrap_or_else(|| std::path::Path::new("/tmp")) + .join("boot-config.json"); + let config_json = serde_json::to_vec(&config) + .map_err(|e| Error::agent("serialize boot config", e.to_string()))?; + std::fs::write(&config_path, &config_json) + .map_err(|e| Error::agent("write boot config", e.to_string()))?; + tracing::info!( + elapsed_ms = t_launch.elapsed().as_millis(), + "boot: config written" + ); + + // Spawn fresh subprocess (posix_spawn on macOS — safe for multi-threaded parents) + let exe = std::env::current_exe() + .map_err(|e| Error::agent("find smolvm binary", e.to_string()))?; + let spawn_start = Instant::now(); + // Embedders (e.g. the Node SDK, where current_exe is `node`) can point the + // boot subprocess at a `_boot-vm`-capable, signed helper binary instead of self. + let boot_binary = std::env::var_os("SMOLVM_BOOT_BINARY"); + // An in-process embedder (the Node/Python SDK) sets SMOLVM_BOOT_BINARY and + // owns the VM's lifetime — when that host process dies, the VM must die + // too, or it leaks as an orphan holding the VM's full RAM. The CLI (which + // detaches the VM on purpose) and `serve` (which reconnects to surviving + // VMs) don't set it, so they keep today's behavior. We pass the resulting + // flag down so the boot subprocess only arms its parent-death watchdog in + // the embedder case. See `cli/internal_boot::run`. + // + // A CLI that sets SMOLVM_BOOT_BINARY (because its own `current_exe` can't + // serve `_boot-vm`, e.g. `smol`) but DETACHES the VM must opt out via + // `features.watch_parent = Some(false)` — otherwise its persistent + // machines die the moment the CLI process exits. + let watch_parent = features.watch_parent.unwrap_or(boot_binary.is_some()); + let boot_exe = boot_binary + .map(std::path::PathBuf::from) + .unwrap_or_else(|| exe.clone()); + let mut cmd = std::process::Command::new(&boot_exe); + cmd.args(["_boot-vm", &config_path.to_string_lossy()]) + .env( + "SMOLVM_BOOT_WATCH_PARENT", + if watch_parent { "1" } else { "0" }, + ) + // Forkable / fork-clone vars set explicitly on the child (not via + // inherited process-global env) — see fork_env above. + .envs(fork_env) + // Per-VM uid drop (privileged launcher only) — see uid_env above. + .envs(uid_env) + // Per-VM readiness marker name — forwarded by the launcher into the + // guest env so the agent writes this VM's own marker (no shared-marker + // race). The host polls the same path. + .env( + smolvm_protocol::guest_env::READY_MARKER, + self.ready_marker_name(), + ) + .stdin(std::process::Stdio::null()) + // SMOLVM_BOOT_DEBUG=1 surfaces the boot subprocess's stdout/stderr so + // embedded-host launch failures can be diagnosed (normally silenced). + .stdout(if std::env::var_os("SMOLVM_BOOT_DEBUG").is_some() { + std::process::Stdio::inherit() + } else { + std::process::Stdio::null() + }) + .stderr(if std::env::var_os("SMOLVM_BOOT_DEBUG").is_some() { + std::process::Stdio::inherit() + } else { + std::process::Stdio::null() + }); + // Own process group (pgid = child pid) so the VM is immune to SIGHUP from + // the parent's terminal closing, without making it a session leader. + // POSIX-only; Windows process groups have different semantics and the + // SIGHUP concern does not apply. + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + cmd.process_group(0); + } + // Windows analogue: detach the VM from the launching process's console so + // it survives that shell closing (a closing console delivers + // CTRL_CLOSE_EVENT to attached processes, which would kill a detached + // persistent machine the moment `machine start` returns), and give it its + // own process group so a Ctrl-C in the launcher isn't forwarded. + // + // NB: an OpenSSH session wraps its processes in a job object with + // KILL_ON_JOB_CLOSE, so a machine started over `ssh ... cmd /c` still dies + // on disconnect — that's an SSH artifact, not present in a normal terminal + // or under the `serve` supervisor. + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + const DETACHED_PROCESS: u32 = 0x0000_0008; + const CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200; + cmd.creation_flags(DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP); + } + let child = cmd + .spawn() + .map_err(|e| Error::agent("spawn boot subprocess", e.to_string()))?; + + let child_pid = child.id() as i32; + // Register the detached VM PID for the serve supervisor's selective + // reaper. The boot subprocess owns its process group and is never + // `wait()`ed, so it would zombie on exit; the supervisor tick reaps it. + // (In non-serve callers without a supervisor this is a harmless no-op — + // the sweep is only driven from serve.) The `child` handle drops without + // waiting (Rust `Child::drop` is a no-op), leaving the PID for the sweep. + crate::process::register_vm_child(child_pid); + tracing::info!( + pid = child_pid, + spawn_ms = spawn_start.elapsed().as_millis(), + "boot: subprocess spawned" + ); + + // Lossless-restart placement: on a systemd host, adopt the just-forked VM + // into its own `smolvm-vm-.scope` (sibling unit, owned by PID1) so a + // later `serve` restart can't kill or `219/CGROUP`-crash on it. Serve set + // SMOLVM_VM_USE_SCOPE at startup and did NOT set SMOLVM_CGROUP_ROOT, so the + // boot subprocess skipped self-placement and the VM is still in serve's + // cgroup for this microsecond window — the adopt moves it out. Caps mirror + // process::place_in_cgroup (CGROUP_MEM_OVERHEAD_MIB=768, CGROUP_PIDS_MAX + // =1024) as scope properties. Best-effort: on failure the VM keeps running + // (just not restart-safe), same as an uncapped cgroup join. + #[cfg(target_os = "linux")] + if std::env::var_os("SMOLVM_VM_USE_SCOPE").is_some() { + if let Some(name) = self.name() { + let caps = crate::systemd_scope::ScopeCaps { + memory_max_bytes: Some( + (resources_for_config.memory_mib as u64 + 768) * 1024 * 1024, + ), + cpu_quota_usec_per_sec: Some( + (resources_for_config.cpus.max(1) as u64) * 1_000_000, + ), + tasks_max: Some(1024), + }; + if let Err(e) = crate::systemd_scope::adopt_into_scope(name, child_pid, &caps) { + // Only reachable when is_available() said yes (root + systemd + + // busctl) but the bus call still failed — effectively a broken + // D-Bus. The VM keeps running but stays in serve's cgroup, + // uncapped and not restart-safe. Loud so the operator notices. + tracing::warn!( + error = %e, pid = child_pid, + "failed to adopt VM into systemd scope; VM left in service cgroup — uncapped and NOT restart-safe" + ); + } + } + } + + self.finalize_launch(child_pid, &mounts, &ports, &resources_for_config) + } + + /// Like `ensure_running_with_full_config` but uses subprocess launch. + /// + /// Use this from multi-threaded contexts (API server) where fork() is + /// unsafe on macOS. See `start_via_subprocess` for details. + pub fn ensure_running_via_subprocess( + &self, + mounts: Vec, + ports: Vec, + resources: VmResources, + mut features: launcher::LaunchFeatures, + ) -> Result { + // Check if agent is already running (same logic as ensure_running_with_full_config) + if self.try_connect_existing().is_some() { + let inner = self.inner.lock(); + match &inner.config_state { + ConfigState::Known => { + if inner.mounts == mounts + && inner.ports == ports + && inner.resources == resources + { + return Ok(false); + } + } + ConfigState::LoadFailed(reason) => { + tracing::info!( + reason = %reason, + "forcing VM restart: running config unknown" + ); + } + ConfigState::Unknown => { + tracing::info!("forcing VM restart: config state still unknown"); + } + } + } + + let needs_restart = { + let inner = self.inner.lock(); + inner.state == AgentState::Running + }; + + if needs_restart { + tracing::info!("restarting agent VM due to configuration change"); + self.stop()?; + } else { + self.reset_stale_running_state(); + } + + // Re-attach packed layers if a config-change restart dropped them + // (see `rewire_packed_layers_if_extracted`). + self.rewire_packed_layers_if_extracted(&mut features)?; + + self.start_via_subprocess(mounts, ports, resources, features)?; + Ok(true) + } + + /// Verify identity of a VM process and kill it. + /// + /// Uses two methods to confirm the PID belongs to our VM: + /// 1. **Vsock shutdown** — if the guest agent acknowledges, it's our VM + /// 2. **PID start-time** — strict comparison guards against PID reuse + /// + /// If either method confirms identity, sends SIGTERM (then SIGKILL on timeout). + /// Path to this VM's `boot-config.json` — the unique per-VM argument passed to + /// the `_boot-vm` subprocess. Must mirror the construction at launch (next to + /// the storage disk) so a teardown can identify the live VM process by argv. + fn boot_config_path(&self) -> std::path::PathBuf { + self.storage_disk + .path() + .parent() + .unwrap_or_else(|| std::path::Path::new("/tmp")) + .join("boot-config.json") + } + + /// Returns `Ok(())` if the process is confirmed dead, `Err` if still alive + /// or identity could not be verified. + fn stop_vm_process(&self, pid: crate::process::Pid, start_time: Option) -> Result<()> { + // Use short timeout — the agent may already be gone (ephemeral run exited). + // A 100ms connect timeout avoids blocking the exit path. + let shutdown_acked = if let Ok(mut client) = + super::AgentClient::connect_with_short_timeout(&self.vsock_socket) + { + client.shutdown().is_ok() + } else { + false + }; + + // Identity check: vsock acknowledgement OR strict PID start-time match OR + // an argv match on this VM's unique boot-config path. We intentionally do + // NOT use the lenient is_our_process() here because it treats any alive + // PID as "ours" when start_time is None — which risks killing an unrelated + // process if the OS reused the PID. The cmdline fallback is safe for the + // same reason it's specific: only our `_boot-vm /boot-config.json` + // process carries that exact path, so it recovers the case where the agent + // vsock is wedged (no ack) AND the start-time record is missing — the exact + // combination that otherwise leaks a live orphan the control can never see. + let identity_ok = shutdown_acked + || process::is_our_process_strict(pid, start_time) + || process::cmdline_contains(pid, &self.boot_config_path().to_string_lossy()); + + if identity_ok { + if !process::is_our_process_strict(pid, start_time) { + tracing::debug!( + pid, + "PID start-time not verified, identity confirmed via vsock" + ); + } + let _ = process::stop_vm_process(pid, AGENT_STOP_TIMEOUT, process::VM_SIGKILL_TIMEOUT); + } + + if process::is_alive(pid) { + if !identity_ok { + // Kill was skipped (no vsock ack, start-time unverifiable) AND the + // process is genuinely still alive — a real orphan/leak risk. + tracing::warn!( + pid, + "skipping kill: PID identity not verified and vsock shutdown \ + failed; process still alive" + ); + } + Err(Error::agent( + "stop agent", + format!("process {} still alive after stop attempts", pid), + )) + } else { + if !identity_ok { + // The vsock connect failed because the VMM was already exiting, so + // the skipped SIGKILL was a no-op against an already-dead process. + // Benign teardown race (not a leak) — log at debug, not warn. + tracing::debug!( + pid, + "skipped kill (identity unverified, vsock shutdown failed); \ + process already exited" + ); + } + Ok(()) + } + } + + /// Remove PID file, config file, and vsock socket marker files. + /// + /// Only call after the VM process is confirmed dead. + fn cleanup_marker_files(&self) { + // Include the per-VM readiness marker so the shared rootfs doesn't + // accumulate one stale marker per VM ever booted. + let ready_marker = self.ready_marker_path(); + for path in [ + &self.pid_file, + &self.config_file, + &self.vsock_socket, + &ready_marker, + ] { + if let Err(e) = std::fs::remove_file(path) { + if e.kind() != std::io::ErrorKind::NotFound { + tracing::debug!(error = %e, path = %path.display(), "failed to remove marker file"); + } + } + } + } + + /// Kill the VM process immediately with SIGKILL. No graceful shutdown. + /// + /// Used for ephemeral `machine run` where the command has already finished + /// and there's no state to preserve. Much faster than `stop()` which + /// attempts a graceful vsock shutdown + SIGTERM + poll. + pub fn kill(&self) { + // Two PID sources with very different PID-reuse risk: + // - the in-memory child: a direct child we still own, so the kernel + // cannot recycle its PID until we reap it → safe to SIGKILL by PID. + // - the pid-file: a process we did NOT spawn as our child (recovered + // after a re-attach), so between the pid-file write and now the OS + // may have reused the PID. Verify the recorded start-time before + // SIGKILL (`kill_verified`) so we never signal an unrelated process. + let owned_child = { + let inner = self.inner.lock(); + inner.child.as_ref().map(|c| c.pid()) + }; + + let killed_pid = match owned_child { + Some(pid) => { + if process::is_alive(pid) { + process::kill(pid); + } + Some(pid) + } + None => match self.read_pid_file_with_start_time() { + Some((pid, start_time)) => { + if process::kill_verified(pid, start_time) { + Some(pid) + } else if process::cmdline_contains( + pid, + &self.boot_config_path().to_string_lossy(), + ) { + // Start-time unverifiable, but the live PID's argv carries + // this VM's unique boot-config path — unambiguously ours, + // so kill it rather than leak a live orphan. + process::kill(pid); + Some(pid) + } else { + if process::is_alive(pid) { + tracing::warn!( + pid, + "skipping kill: pid-file PID is alive but neither \ + start-time nor argv identify it as ours (possible \ + PID reuse)" + ); + } + None + } + } + None => None, + }, + }; + + if let Some(pid) = killed_pid { + // Brief wait for the kernel to reap (SIGKILL is near-instant). + // try_wait reaps zombie children; is_alive catches non-children + // that have been reparented to init/launchd. + for _ in 0..10 { + if process::try_wait(pid).is_some() || !process::is_alive(pid) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + } + self.cleanup_marker_files(); + } + + /// Remove the VM's entire data directory (storage, overlay, socket, logs). + /// + /// Only safe for ephemeral VMs after the process is confirmed dead. + pub fn cleanup_data_dir(&self) { + if let Some(ref name) = self.name { + let dir = vm_data_dir(name); + // Release this VM's per-VM uid (if any) back to the allocator before + // the dir — which holds the `.vm-uid` record — is removed. A fork + // clone has no uid of its own (it shares its golden's), so this is a + // no-op for it. See process::free_vm_uid. + crate::process::free_vm_uid(&vm_uid_registry_dir(), &dir); + if dir.exists() { + if let Err(e) = std::fs::remove_dir_all(&dir) { + tracing::debug!( + error = %e, + path = %dir.display(), + "failed to remove ephemeral VM data directory" + ); + } + } + } + } + + /// Stop the agent VM. + pub fn stop(&self) -> Result<()> { + let state = { + let inner = self.inner.lock(); + inner.state + }; + + if state == AgentState::Stopped { + // Even if internal state is Stopped, check PID file for orphan processes + // from previous CLI invocations that weren't properly cleaned up. + if let Some((pid, start_time)) = self.read_pid_file_with_start_time() { + if let Err(e) = self.stop_vm_process(pid, start_time) { + tracing::warn!( + pid, + "orphan process still alive, preserving PID/socket files" + ); + return Err(e); + } + self.cleanup_marker_files(); + } + return Ok(()); + } + + { + let mut inner = self.inner.lock(); + inner.state = AgentState::Stopping; + } + + tracing::info!("stopping agent VM"); + + // Get the child PID and start time — try in-memory first, then PID file. + // The PID file fallback is critical for default VMs where a fresh + // AgentManager doesn't know the PID from a previous CLI invocation. + let (child_pid, pid_start_time) = { + let inner = self.inner.lock(); + if let Some(child) = inner.child.as_ref() { + // Use the start time captured when the child handle was created, + // not recomputed from the PID (which would be self-fulfilling + // if the PID was recycled by the OS). + (Some(child.pid()), child.start_time()) + } else { + match self.read_pid_file_with_start_time() { + Some((pid, start_time)) => (Some(pid), start_time), + None => (None, None), + } + } + }; + + if let Some(pid) = child_pid { + if let Err(e) = self.stop_vm_process(pid, pid_start_time) { + // Revert to Running — don't lie about state or delete markers + { + let mut inner = self.inner.lock(); + inner.state = AgentState::Running; + } + return Err(e); + } + } + + // Defense in depth: sync host's view of the disk files + // This catches any writes that made it to the host buffer but weren't flushed + // Combined with agent-side sync(), this provides robust data integrity + for (label, path) in [ + ("storage", self.storage_disk.path()), + ("overlay", self.overlay_disk.path()), + ] { + if let Ok(file) = std::fs::File::open(path) { + if file.sync_all().is_ok() { + tracing::debug!("{} disk synced to host", label); + } + } + } + + // Clean up — safe now that process is confirmed dead + { + let mut inner = self.inner.lock(); + inner.state = AgentState::Stopped; + inner.child = None; + // Release the per-VM file lock so other processes can start this VM. + #[cfg(unix)] + { + inner.vm_lock_handle = None; + } + } + + self.cleanup_marker_files(); + metrics::gauge!("smolvm_machines_running").decrement(1.0); + + Ok(()) + } + + /// Wait for the agent to be ready. + /// + /// Polls the virtiofs file marker `.smolvm-ready` written by the agent + /// after completing initialization. Includes a vsock control-channel ping + /// fallback for agents too old to write the marker. + /// + /// Measured latency (macOS 26, warm boots): ~135ms total from subprocess spawn. + /// + /// Instrumented trace findings (May 2026): + /// - Ready marker appears on host fs within ~1ms of the virtiofs FUSE write. + /// No visibility gap exists; polling interval is the only noise source. + /// - hv_gic_set_spi() costs 0–15µs per call — SPI injection is free. + /// - Bottleneck is setup_persistent_rootfs() block I/O on /dev/vdb: + /// 246 requests × 83–131ms = 48ms (37% of total boot time). Skipping + /// the overlay disk for ephemeral runs is the highest-impact optimization. + /// - Guest /proc/uptime runs at half real speed (CNTFRQ_EL0 2× counter rate); + /// this explains why guest logs show "70ms" while host measures "131ms". + /// It is a display artifact only and does not cause any actual delay. + /// + /// Polling at 1ms for the first second to give sub-poll-interval resolution + /// for boot timing experiments. Falls back to 5ms after 1 second. + fn wait_for_ready(&self) -> Result<()> { + let timeout = AGENT_READY_TIMEOUT; + let start = Instant::now(); + // Marker-first readiness only works when the marker file (inside the + // agent-rootfs virtiofs share) is writable at all. On read-only system + // installs it never appears; skip the grace and poll the socket from + // the start instead of stalling every boot for the full grace (#590). + let marker_expected = self.ready_marker_writable(); + let socket_probe_grace = if marker_expected { + Duration::from_secs(5) + } else { + tracing::info!( + rootfs = %self.rootfs_path.display(), + "agent-rootfs is not writable by this process; ready-marker fast \ + path unavailable, using socket readiness probe. For the fastest \ + boots make the rootfs dir writable by the running user or point \ + SMOLVM_AGENT_ROOTFS at a writable copy" + ); + Duration::ZERO + }; + + tracing::debug!("waiting for agent to be ready"); + + // Fork clone: the guest resumes past boot, so it never (re)writes the + // `.smolvm-ready` marker. Detect readiness by pinging the restored agent + // directly (it is already in its accept loop) — no marker, no grace. + let is_clone = self.inner.lock().is_clone; + if is_clone { + while start.elapsed() < timeout { + { + let mut inner = self.inner.lock(); + if let Some(ref mut child) = inner.child { + if !child.is_running() { + return Err(Error::agent( + "monitor agent", + "clone agent process exited during startup".to_string(), + )); + } + } + } + if self.vsock_socket.exists() { + if let Ok(mut client) = + super::AgentClient::connect_with_boot_probe_timeout(&self.vsock_socket) + { + if client.ping().is_ok() { + tracing::info!( + elapsed_ms = start.elapsed().as_millis(), + "clone agent ready (ping)" + ); + return Ok(()); + } + } + } + std::thread::sleep(Duration::from_millis(20)); + } + return Err(Error::agent( + "wait for ready", + "clone agent did not respond to ping within timeout".to_string(), + )); + } + + let ready_marker = self.ready_marker_path(); + + while start.elapsed() < timeout { + // Check if child process is still alive + { + let mut inner = self.inner.lock(); + if let Some(ref mut child) = inner.child { + if !child.is_running() { + let exit_code = child.exit_code(); + let log = std::fs::read_to_string(&self.startup_error_log) + .ok() + .map(|content| content.trim().to_string()) + .filter(|content| !content.is_empty()); + return Err(Error::agent( + "monitor agent", + boot_failure_reason(exit_code, log.as_deref()), + )); + } + } + } + + // Ready when the marker is present AND non-empty. The guest writes + // its uptime (always non-empty) into it. Under SMOLVM_LANDLOCK the + // confined VMM pre-creates this file empty so Landlock can grant + // write on just this one file (see internal_boot.rs) — so existence + // alone would false-positive; require content. + if std::fs::metadata(&ready_marker) + .map(|m| m.len() > 0) + .unwrap_or(false) + { + let elapsed = start.elapsed(); + tracing::info!(elapsed_ms = elapsed.as_millis(), "agent ready (marker)"); + return Ok(()); + } + + // After long grace, fall back to socket ping. This is the intended + // path only for pre-marker agents; for a current agent the marker + // never appearing is a real fault (e.g. a Landlock rule denying the + // marker write — see internal_boot.rs) that silently cost us the + // whole probe grace. Logged at WARN so the degradation can't hide. + if start.elapsed() >= socket_probe_grace && self.vsock_socket.exists() { + if let Ok(mut client) = + super::AgentClient::connect_with_boot_probe_timeout(&self.vsock_socket) + { + if client.ping().is_ok() { + let elapsed = start.elapsed(); + if marker_expected { + let landlock = std::env::var("SMOLVM_LANDLOCK").unwrap_or_default(); + tracing::warn!( + elapsed_ms = elapsed.as_millis(), + landlock = %landlock, + "agent ready via SOCKET FALLBACK — ready marker never appeared; \ + boot was delayed by the probe grace. If a current agent, this is a \ + fault (under SMOLVM_LANDLOCK=enforce the ready-marker carve-out in \ + internal_boot.rs is likely broken)" + ); + } else { + // Read-only rootfs: socket IS the readiness path; no + // grace was paid (socket_probe_grace == 0). + tracing::info!( + elapsed_ms = elapsed.as_millis(), + "agent ready (socket; marker unavailable on read-only rootfs)" + ); + } + return Ok(()); + } + } + } else { + // 1ms polling during first second for sub-interval boot timing resolution; + // 5ms thereafter to avoid burning CPU while waiting on slow starts. + let poll_ms = if start.elapsed() < Duration::from_secs(1) { + 1 + } else { + 5 + }; + std::thread::sleep(Duration::from_millis(poll_ms)); + } + } + + Err(Error::agent( + "wait for ready", + format!( + "agent did not become ready within {} seconds", + timeout.as_secs() + ), + )) + } + + /// Wait for the agent to stop. + fn wait_for_stop(&self) -> Result<()> { + let timeout = WAIT_FOR_STOP_TIMEOUT; + let start = Instant::now(); + + while start.elapsed() < timeout { + if self.state() == AgentState::Stopped { + return Ok(()); + } + std::thread::sleep(FAST_POLL_INTERVAL); + } + + Err(Error::agent( + "shutdown agent", + "timeout waiting for agent to stop", + )) + } + + /// Check if agent process is still running. + pub fn check_alive(&self) -> bool { + let mut inner = self.inner.lock(); + + if let Some(ref mut child) = inner.child { + child.is_running() + } else { + false + } + } + + /// Detach the agent manager, preventing cleanup on drop. + /// + /// Call this when you want the agent VM to continue running after + /// this manager instance is dropped (e.g., for persistent VMs). + /// + /// This is preferred over `std::mem::forget` because: + /// - Intent is explicit and documented + /// - Other resources (non-child-process) are still properly cleaned up + /// - The manager can still be used after detaching + pub fn detach(&self) { + let mut inner = self.inner.lock(); + inner.detached = true; + tracing::debug!("agent manager detached, VM will continue running"); + } + + /// Check if the agent manager has been detached. + pub fn is_detached(&self) -> bool { + let inner = self.inner.lock(); + inner.detached + } +} + +impl Drop for AgentManager { + fn drop(&mut self) { + let inner = self.inner.lock(); + let detached = inner.detached; + let has_child = inner.child.is_some(); + drop(inner); + + if detached { + return; + } + + // Only stop the VM if this manager actually owns the child process. + // Managers created as observers (e.g., API read handlers, monitor + // loop iterations) have no child handle and must NOT kill VMs they + // didn't start. Without this guard, dropping an observer manager + // triggers the orphan-cleanup path in stop(), which reads the PID + // file and kills whatever VM another manager is running. + if has_child { + if let Err(e) = self.stop() { + tracing::debug!(error = %e, "failed to stop agent in drop"); + } + } + } +} + +/// Build a diagnostic reason when the boot subprocess exits before the agent +/// signals ready. Prefers a real error the launcher logged (a clean +/// `krun_start_enter` failure, a panic, an `Error:`); otherwise reports the exit +/// code. A native-crash exit code (NTSTATUS `0xCxxxxxxx` — e.g. `0xC0000005` +/// access violation or `0xC0000135` DLL-not-found on Windows) points at a +/// mismatched/corrupt `krun.dll`/`libkrunfw.dll` or unavailable WHP, which is +/// far more useful than whatever benign WARN happened to be logged last (on +/// Windows the guest console isn't captured, so the log is often just that). +fn boot_failure_reason(exit_code: Option, startup_log: Option<&str>) -> String { + let real_error = startup_log.and_then(|log| { + log.lines().rev().find_map(|line| { + let lower = line.to_ascii_lowercase(); + if lower.contains("error") + || lower.contains("panic") + || lower.contains("krun_start_enter returned") + { + Some(line.trim().to_string()) + } else { + None + } + }) + }); + + let code_note = match exit_code { + Some(code) => { + let unsigned = code as u32; + // NTSTATUS-style crash codes are 0xCxxxxxxx — a native crash, not a + // clean exit (Unix exit codes never fall in this range). + if unsigned & 0xF000_0000 == 0xC000_0000 { + format!( + "boot process crashed (exit 0x{unsigned:08X}) before the agent was ready \ + — usually a mismatched or corrupt krun.dll / libkrunfw.dll, or Windows \ + Hypervisor Platform (WHP) is unavailable; verify the DLLs match smolvm \ + (checksums.txt) and that WHP is enabled" + ) + } else { + format!("boot process exited (code {code}) before the agent was ready") + } + } + None => "agent process exited during startup".to_string(), + }; + + match real_error { + Some(err) => format!("{err} ({code_note})"), + None => code_note, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vm_dir_hash_is_deterministic() { + // Stability guarantee: the same name always maps to the same hash. + // Callers rely on this to locate existing VM data across processes. + assert_eq!(vm_dir_hash("sandbox-1"), vm_dir_hash("sandbox-1")); + assert_eq!(vm_dir_hash("default"), vm_dir_hash("default")); + } + + #[test] + #[cfg(unix)] + fn marker_creatable_detects_readonly_rootfs() { + use std::os::unix::fs::PermissionsExt; + let tmp = tempfile::tempdir().unwrap(); + + // Writable rootfs: probe succeeds and pre-creates the marker empty. + let marker = tmp.path().join(format!("{}.abc123", AGENT_READY_MARKER)); + assert!(marker_creatable(&marker)); + assert_eq!(std::fs::metadata(&marker).unwrap().len(), 0); + + // Read-only rootfs (root-owned system install, #590): probe fails so + // wait_for_ready can skip the socket-probe grace instead of stalling. + let ro = tmp.path().join("ro-rootfs"); + std::fs::create_dir(&ro).unwrap(); + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o555)).unwrap(); + let denied = ro.join(format!("{}.abc123", AGENT_READY_MARKER)); + if !nix_is_root() { + assert!(!marker_creatable(&denied)); + } + std::fs::set_permissions(&ro, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + + /// chmod-based denial doesn't apply to root (e.g. docker CI); skip there. + #[cfg(unix)] + fn nix_is_root() -> bool { + // SAFETY: geteuid has no preconditions and cannot fail. + unsafe { libc::geteuid() == 0 } + } + + #[test] + fn boot_failure_native_crash_gets_dll_hint() { + // 0xC0000005 (access violation) — a mismatched/corrupt DLL, not whatever + // benign WARN was logged last. The hint must name the DLLs + WHP. + let r = boot_failure_reason(Some(0xC000_0005u32 as i32), None); + assert!(r.contains("0xC0000005"), "{r}"); + assert!(r.contains("krun.dll") && r.contains("WHP"), "{r}"); + } + + #[test] + fn boot_failure_prefers_real_error_over_warn() { + // A benign WARN must never be surfaced when the log also has a real error. + let log = "WARN failed to set console output\nError: kernel not found"; + let r = boot_failure_reason(Some(1), Some(log)); + assert!(r.contains("kernel not found"), "{r}"); + assert!(!r.starts_with("WARN"), "{r}"); + } + + #[test] + fn boot_failure_clean_exit_and_unknown() { + assert!(boot_failure_reason(Some(1), None).contains("code 1")); + assert_eq!( + boot_failure_reason(None, None), + "agent process exited during startup" + ); + } + + #[test] + fn prune_removes_only_orphaned_ready_markers() { + let tmp = tempfile::tempdir().unwrap(); + let rootfs = tmp.path().join("agent-rootfs"); + let cache = tmp.path().join("vms"); + std::fs::create_dir_all(&rootfs).unwrap(); + std::fs::create_dir_all(&cache).unwrap(); + + // A live VM (its data dir exists) and markers in the shared rootfs. + std::fs::create_dir_all(cache.join("aaaa")).unwrap(); + let live = rootfs.join(format!("{AGENT_READY_MARKER}.aaaa")); // VM exists -> keep + let orphan = rootfs.join(format!("{AGENT_READY_MARKER}.bbbb")); // VM gone -> remove + let legacy = rootfs.join(AGENT_READY_MARKER); // no hash suffix -> keep + let real_file = rootfs.join("bin"); // not a marker -> keep + for p in [&live, &orphan, &legacy, &real_file] { + std::fs::write(p, b"1").unwrap(); + } + + prune_orphaned_ready_markers_in(&rootfs, &cache); + + assert!(live.exists(), "marker for a live VM must be kept"); + assert!(!orphan.exists(), "marker for a deleted VM must be removed"); + assert!(legacy.exists(), "the hash-less shared marker must be kept"); + assert!(real_file.exists(), "non-marker files must be untouched"); + } + + #[test] + fn vm_dir_hash_is_16_hex_chars() { + let h = vm_dir_hash("anything"); + assert_eq!(h.len(), 16, "expected 16 hex chars, got {}: {}", h.len(), h); + assert!( + h.chars().all(|c| c.is_ascii_hexdigit()), + "hash contains non-hex chars: {}", + h + ); + } + + #[test] + fn vm_dir_hash_differs_for_different_names() { + assert_ne!(vm_dir_hash("a"), vm_dir_hash("b")); + assert_ne!(vm_dir_hash("sandbox-1"), vm_dir_hash("sandbox-2")); + } + + #[test] + fn vm_data_dir_path_length_is_bounded_regardless_of_name() { + // Core correctness property: socket-path overflow is impossible + // because the variable section is fixed at 16 chars. A 200-char name + // produces the same-length path as a 1-char name. No legacy fallback + // means this holds deterministically, regardless of filesystem state. + let short = vm_data_dir("x"); + let long = vm_data_dir(&"a".repeat(200)); + assert_eq!( + short.as_os_str().len(), + long.as_os_str().len(), + "path length must be independent of name length" + ); + } + + #[test] + fn ensure_vm_dir_writes_name_file_on_first_call() { + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("abc123"); + let result = ensure_vm_dir_at(&dir, "my-vm").unwrap(); + assert_eq!(result, dir); + assert_eq!(std::fs::read_to_string(dir.join("name")).unwrap(), "my-vm"); + } + + #[test] + fn ensure_vm_dir_is_idempotent_for_matching_name() { + // Second call with the same name must succeed (every machine start, + // exec, etc. re-enters this path). Must not touch the name file. + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("abc123"); + ensure_vm_dir_at(&dir, "my-vm").unwrap(); + + // Tamper with the mtime semantics: if we were rewriting, we'd clobber + // any user edit. Write a sentinel and confirm it survives. + let name_file = dir.join("name"); + let before = std::fs::metadata(&name_file).unwrap().modified().unwrap(); + std::thread::sleep(std::time::Duration::from_millis(10)); + + ensure_vm_dir_at(&dir, "my-vm").unwrap(); + let after = std::fs::metadata(&name_file).unwrap().modified().unwrap(); + assert_eq!( + before, after, + "name file must not be rewritten on repeat calls" + ); + } + + #[test] + fn ensure_vm_dir_rejects_hash_collision() { + // Simulate two distinct VM names hashing to the same directory. + // ensure_vm_dir_at is parameterized on the directory so we can + // exercise this without needing a real SHA-256 collision. + let tmp = tempfile::tempdir().unwrap(); + let dir = tmp.path().join("collision-dir"); + + ensure_vm_dir_at(&dir, "first-vm").unwrap(); + + let err = ensure_vm_dir_at(&dir, "second-vm") + .expect_err("expected collision error for different name at same dir"); + let msg = err.to_string(); + assert!( + msg.contains("hash collision"), + "error should identify collision: {msg}" + ); + assert!( + msg.contains("first-vm") && msg.contains("second-vm"), + "error should name both VMs: {msg}" + ); + + // The name file must still point to the first VM — we must NOT have + // clobbered it during the failed attempt. + assert_eq!( + std::fs::read_to_string(dir.join("name")).unwrap(), + "first-vm", + ); + } + + #[cfg(target_os = "macos")] + #[test] + fn rewire_packed_layers_returns_lease_failure() { + let temp = tempfile::tempdir().unwrap(); + + let name = format!( + "review-rewire-lease-failure-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + struct VmDataCleanup(String); + impl Drop for VmDataCleanup { + fn drop(&mut self) { + smolvm_pack::extract::force_detach_layers_volume(&machine_layers_cache_dir( + &self.0, + )); + let _ = std::fs::remove_dir_all(vm_data_dir(&self.0)); + } + } + let _cleanup = VmDataCleanup(name.clone()); + + let storage = StorageDisk::open_or_create_at(&temp.path().join("storage.img"), 1).unwrap(); + let overlay = OverlayDisk::open_or_create_at(&temp.path().join("overlay.img"), 1).unwrap(); + let manager = + AgentManager::new_named(&name, temp.path().join("rootfs"), storage, overlay).unwrap(); + + let cache_dir = machine_layers_cache_dir(&name); + std::fs::create_dir_all(&cache_dir).unwrap(); + std::fs::write(cache_dir.join(".smolvm-extracted"), "").unwrap(); + std::fs::write(cache_dir.join("layers-cs.sparseimage"), b"not a disk image").unwrap(); + + let mut features = launcher::LaunchFeatures::default(); + let err = manager + .rewire_packed_layers_if_extracted(&mut features) + .expect_err("restart must fail when packed layers cannot be reattached"); + + assert!( + err.to_string().contains("re-attach packed layers"), + "unexpected error: {err}" + ); + assert!(features.packed_layers_dir.is_none()); + } +} diff --git a/src/agent/mod.rs b/src/agent/mod.rs new file mode 100644 index 0000000..3f5992f --- /dev/null +++ b/src/agent/mod.rs @@ -0,0 +1,114 @@ +//! Agent VM management. +//! +//! This module manages the agent VM lifecycle and provides a client +//! for communicating with the smolvm-agent via vsock. + +pub mod boot_config; +mod client; +pub mod fork; +mod fsnotify_watch; +mod krun; +mod launcher; +pub mod launcher_dynamic; +mod manager; +pub mod state_probe; +pub mod terminal; +mod vsock_service; + +pub use crate::data::network::PortMapping; +pub use crate::data::resources::VmResources; +pub use crate::data::storage::HostMount; +pub use client::{ + AgentClient, ExecEvent, InteractiveInput, InteractiveOutput, PullOptions, RunConfig, +}; +pub use fsnotify_watch::FsNotifyWatcher; +pub use krun::KrunFunctions; +pub use launcher::{ + create_disk_overlays, find_lib_dir, launch_agent_vm, DiskOverlaySpec, LaunchConfig, + LaunchFeatures, VmDisks, +}; +pub use manager::{ + disk_used_mb, docker_config_dir, docker_config_mount, ensure_vm_dir, machine_layers_cache_dir, + prune_orphaned_ready_markers, read_egress_telemetry, read_shared_pack_pointer, + resolve_disk_image, shared_pack_cache_root, shared_pack_pointer_path, vm_cache_root, + vm_data_dir, vm_dir_hash, vm_uid_registry_dir, AgentManager, AgentState, SHARED_PACK_POINTER, +}; + +/// Agent VM name. +pub const AGENT_VM_NAME: &str = "smolvm-agent"; + +/// Compute the `virgl_flags` bitmask for `krun_set_gpu_options2`. +/// +/// Shared by both the static (`launcher.rs`) and dynamic (`launcher_dynamic.rs`) +/// launchers so they can never silently diverge. +/// +/// Flag values from `libkrun/include/libkrun.h` virglrenderer bindings: +/// bit 0 — VIRGLRENDERER_USE_EGL (Linux): EGL context for GPU rendering +/// bit 3 — VIRGLRENDERER_USE_SURFACELESS (Linux): no display server required +/// bit 6 — VIRGLRENDERER_VENUS (both): Vulkan-over-virtio-gpu (Venus ICD) +/// bit 7 — VIRGLRENDERER_NO_VIRGL (macOS): skip OpenGL (vrend) init — without +/// EGL, vrend_renderer_init crashes on null platform function pointers +/// bit 9 — VIRGLRENDERER_RENDER_SERVER (Linux): REQUIRED for render-server mode. +/// Enables virglrenderer to call the get_server_fd callback and use an +/// external render server. Without this bit, virglrenderer attempts +/// in-process Venus which fails (version stays 0). With get_server_fd +/// provided in the callbacks struct, virglrenderer uses the externally +/// spawned virgl_render_server instead of fork/exec-ing its own process. +fn gpu_virgl_flags() -> u32 { + #[cfg(target_os = "linux")] + { + (1 << 0) | (1 << 3) | (1 << 6) | (1 << 9) + } + #[cfg(not(target_os = "linux"))] + { + (1 << 6) | (1 << 7) + } +} + +/// Build the guest-network environment variables handed to the agent at boot. +/// +/// Shared by both the static (`launcher.rs`) and dynamic (`launcher_dynamic.rs`) +/// launchers — like [`gpu_virgl_flags`] — so the two can never silently diverge. +/// They previously each open-coded this block, and the dynamic one drifted: it +/// omitted the TSI `--dns` override, silently dropping `--dns` on the default +/// backend (PR #466). With one source of truth, the guest's resolver is decided +/// in exactly one place and the agent remains the sole writer of resolv.conf. +/// +/// `guest_network` is `Some` for virtio-net (the agent derives its resolver from +/// `dns_server`, the gateway address). For TSI it is `None`: there is no host +/// gateway to route a resolver through, so a `--dns` override must be passed +/// straight through for the agent to write into resolv.conf. +pub(crate) fn guest_network_env( + guest_network: Option, + dns_override: Option, +) -> Vec { + use smolvm_protocol::guest_env; + let mut env: Vec = Vec::new(); + let mut push = |key: &str, val: String| { + env.push(std::ffi::CString::new(format!("{key}={val}")).expect("env var contains NUL")); + }; + if let Some(n) = guest_network { + push( + guest_env::BACKEND, + guest_env::BACKEND_VIRTIO_NET.to_string(), + ); + push(guest_env::GUEST_IP, n.guest_ip.to_string()); + push(guest_env::GATEWAY, n.gateway_ip.to_string()); + push(guest_env::PREFIX_LEN, n.prefix_len.to_string()); + push(guest_env::GUEST_MAC, format_mac(n.guest_mac)); + push(guest_env::GUEST_IP6, n.guest_ip6.to_string()); + push(guest_env::GATEWAY6, n.gateway_ip6.to_string()); + push(guest_env::PREFIX_LEN6, n.prefix_len6.to_string()); + push(guest_env::DNS, n.dns_server.to_string()); + } else if let Some(dns) = dns_override { + push(guest_env::DNS, dns.to_string()); + } + env +} + +fn format_mac(mac: [u8; 6]) -> String { + format!( + "{:02x}:{:02x}:{:02x}:{:02x}:{:02x}:{:02x}", + mac[0], mac[1], mac[2], mac[3], mac[4], mac[5] + ) +} diff --git a/src/agent/state_probe.rs b/src/agent/state_probe.rs new file mode 100644 index 0000000..6a8901a --- /dev/null +++ b/src/agent/state_probe.rs @@ -0,0 +1,299 @@ +//! Resolve the actual state of a VM by reconciling DB record, +//! process liveness, and agent reachability. +//! +//! Three signals can disagree: +//! +//! - `record.state` — what we last persisted (e.g., `Running` after +//! a successful start). +//! - `is_process_alive(pid)` — whether the libkrun VMM process is +//! still around. Cheap (`kill(pid, 0)` + start-time match). +//! - vsock ping — whether the guest agent inside the VM responds. +//! The only signal that proves the VM is *useful*; everything +//! else is approximation. +//! +//! The combinations and their resolutions: +//! +//! | record.state | PID alive | ping ok | resolved state | +//! |---|---|---|---| +//! | Running | yes | yes | Running | +//! | Running | yes | no | **Unreachable** ← the bug 2 case | +//! | Running | no | yes | Running (macOS PID-not-visible edge case) | +//! | Running | no | no | Stopped | +//! | Stopped/Created/Failed | * | * | record.state (no probe) | +//! +//! Used by: +//! - CLI `machine list` — so the column shows the truth. +//! - HTTP API list/get/exec handlers — same reason. +//! - CLI `machine start --name` — to detect the Unreachable state +//! and recover by killing the zombie VMM. + +use crate::agent::{AgentClient, AgentManager}; +use crate::config::{RecordState, VmRecord}; +use crate::db::SmolvmDb; + +/// Compute the resolved state for a VM record. May perform a single +/// short-timeout vsock ping; all other paths are stat-only. +/// +/// `name` is the machine name (used to locate the agent socket). +/// Pings use a 250 ms timeout — long enough to tolerate a busy host +/// but short enough that a `machine list` over a handful of stale +/// records doesn't take seconds. +pub fn resolve_state(name: &str, record: &VmRecord) -> RecordState { + if record.state != RecordState::Running { + return record.state.clone(); + } + + let pid_alive = record.is_process_alive(); + + if pid_alive { + // A fork base with live clones was snapshot-frozen by `machine + // fork`: its guest is paused and never answers a vsock ping. + // Report it as Frozen *without* probing — this distinguishes it + // from a genuine zombie (so the reaper and supervisor leave it + // alone) and avoids a multi-second ping timeout on a VM we know + // won't respond (the `machine status`/`ls` hang). + if has_live_clones(name) { + return RecordState::Frozen; + } + // PID alive: confirm the agent responds. If it doesn't, the + // VMM is zombied — agent crashed but libkrun stayed up. This + // is the "machine list says running, exec says not running" + // divergence the operator sees in the wild. + if probe_agent(name) { + RecordState::Running + } else { + RecordState::Unreachable + } + } else { + // PID not visible. On macOS the session-leader VMM may not + // be reachable via `kill(pid, 0)` from a different shell — + // try the agent socket as a fallback before declaring stopped. + if probe_agent(name) { + RecordState::Running + } else { + RecordState::Stopped + } + } +} + +/// Cheap (no vsock probe) check for a snapshot-frozen fork base: its +/// record says `Running`, its VMM PID is alive, and it has dependent +/// clones. Such a VM's guest agent is paused and must not be connected to +/// or probed — doing so blocks for the full vsock timeout. Callers report +/// it as [`RecordState::Frozen`] directly. Shares the same condition +/// `resolve_state` uses, so the two never disagree. +pub fn is_frozen_fork_base(name: &str, record: &VmRecord) -> bool { + record.state == RecordState::Running && record.is_process_alive() && has_live_clones(name) +} + +/// True if `name` is a fork base — at least one VM record's disk overlays +/// are copy-on-write backed by this machine's disks (`golden == Some(name)`). +/// Such a machine is snapshot-frozen and must not be probed, reaped, or +/// auto-restarted; it has to outlive its clones. Best-effort: a DB error +/// resolves to `false` (treat as not-a-fork-base) so a transient failure +/// can't wedge state resolution. +fn has_live_clones(name: &str) -> bool { + SmolvmDb::open() + .and_then(|db| db.dependent_clones(name)) + .map(|clones| !clones.is_empty()) + .unwrap_or(false) +} + +/// Tear down a zombie libkrun VMM (live PID, dead agent) and clear +/// the record's PID/state so the caller can proceed to a clean +/// fresh start — or simply leave the VM Stopped. +/// +/// Kills via verified SIGTERM (PID + start-time match, so a recycled +/// PID is never harmed), waits up to 2 s for the process to exit, +/// then escalates to verified SIGKILL. +/// +/// Infallible by design: if the kill or DB write fails, downstream +/// code will surface a clear error (libkrun "address already in +/// use" on restart, or the next `list` will re-probe and converge +/// on Stopped since the PID ends up dead either way). The caller +/// doesn't need to branch on success. +/// +/// This is the shared teardown used by both the CLI (`machine start +/// | stop | delete --stop`) and the HTTP API (`POST /machines/X/start`) +/// so all surfaces recover from the Unreachable state the same way. +pub fn recover_unreachable_machine(record: &VmRecord) { + use std::time::{Duration, Instant}; + + if let Some(pid) = record.pid { + if crate::process::terminate_verified(pid, record.pid_start_time) { + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + if !crate::process::is_alive(pid) { + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + if crate::process::is_alive(pid) { + crate::process::kill_verified(pid, record.pid_start_time); + } + } + } + + // Best-effort DB clear. We write via SmolvmDb (not SmolvmConfig) + // to avoid pulling the full in-memory config load; the db update + // is targeted to the one record. + if let Ok(db) = SmolvmDb::open() { + let _ = db.update_vm(&record.name, |r| { + r.state = RecordState::Stopped; + r.pid = None; + r.pid_start_time = None; + }); + } +} + +/// If the named VM resolves to `Unreachable`, recover it (kill the +/// zombie VMM, clear the record). No-op for any other state. +/// +/// Loads the record via `SmolvmDb` so callers on either side of the +/// CLI/API boundary can share the helper without coupling to +/// `SmolvmConfig`. +/// +/// Returns `true` when recovery actually ran, so callers can print +/// a user-facing notice (start/stop/delete all want to mention the +/// teardown happened — the alternative is a silent kill, which is +/// surprising when the operator didn't know a zombie existed). +pub fn recover_if_unreachable(name: &str) -> bool { + let Ok(db) = SmolvmDb::open() else { + return false; + }; + let Ok(Some(record)) = db.get_vm(name) else { + return false; + }; + // Only a genuine zombie resolves to `Unreachable`. A frozen fork base + // resolves to `Frozen` (see `resolve_state`) and is left alone here — + // reaping it would kill the CoW backing out from under live clones. + // A force-delete that genuinely wants to break the chain calls + // `recover_unreachable_machine` directly, bypassing this path. + if resolve_state(name, &record) != RecordState::Unreachable { + return false; + } + recover_unreachable_machine(&record); + true +} + +/// Return true if a short-timeout vsock ping to the agent for this +/// machine succeeds. Used as the liveness ground-truth. +fn probe_agent(name: &str) -> bool { + let Ok(manager) = AgentManager::for_vm(name) else { + return false; + }; + // Detach immediately — this manager is only used to locate the vsock + // socket. Without detach, Drop sends Shutdown and kills the VM. + manager.detach(); + // Use a moderate 3s timeout (not the 100ms boot probe) so a busy agent + // — e.g., processing a Run request's overlayfs setup or a container + // operation — isn't falsely reported as Unreachable. The 100ms timeout + // caused `machine ls` / `machine status` to flap to Unreachable any + // time the agent happened to be processing another request. + let Ok(mut client) = AgentClient::connect_for_state_probe(manager.vsock_socket()) else { + return false; + }; + client.ping().is_ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a record in the given state. We don't need to populate + /// every field — most of them are irrelevant to the resolver. + fn record(state: RecordState, pid: Option) -> VmRecord { + let mut r = VmRecord::new("test".to_string(), 1, 256, vec![], vec![], false); + r.state = state; + r.pid = pid; + r + } + + #[test] + fn non_running_states_pass_through_without_probing() { + // The resolver short-circuits on any state other than + // Running — never touches the agent socket. This matters + // both for performance (no per-record probe on big lists) + // and for correctness (Stopped should never randomly + // become Unreachable just because some unrelated socket + // happens to be reachable). + for s in [ + RecordState::Created, + RecordState::Stopped, + RecordState::Failed, + RecordState::Unreachable, // already-unreachable stays unreachable + ] { + let r = record(s.clone(), Some(99999)); + // Pass a name that won't match any real agent socket on + // disk, so any accidental probe would fail fast. + let resolved = resolve_state("does-not-exist", &r); + assert_eq!(resolved, s, "state {:?} should pass through", s); + } + } + + #[test] + fn running_with_no_pid_and_no_agent_returns_stopped() { + // No PID + no reachable agent socket → not running. The + // macOS PID-fallback path probes anyway (in case the PID + // check missed a session-leader process), but with a name + // that has no agent socket, the probe fails and we + // correctly return Stopped. + // + // Avoiding `Some(0)` here because PID 0 is a magic value + // (`kill(0, 0)` targets the current process group and + // returns success), which would mis-classify as "alive". + let r = record(RecordState::Running, None); + let resolved = resolve_state("ghost-does-not-exist", &r); + assert_eq!(resolved, RecordState::Stopped); + } + + /// The cases that require a live agent socket (Running with PID + /// alive + ping ok, Running with PID alive + ping fail) are + /// exercised by the manual repro in `docs/file-transfer-fixes-plan.md` + /// because they need a real VM. Layer A's contract is otherwise + /// nailed down by the cases above. + #[test] + fn unreachable_variant_round_trips_through_serde() { + // The Unreachable variant must serialize cleanly so a record + // marked Unreachable in memory persists to the database correctly. + let r = record(RecordState::Unreachable, Some(12345)); + let json = serde_json::to_string(&r).unwrap(); + assert!(json.contains("\"state\":\"unreachable\"")); + let back: VmRecord = serde_json::from_str(&json).unwrap(); + assert_eq!(back.state, RecordState::Unreachable); + } + + #[test] + fn unreachable_displays_as_unreachable() { + // The state shown in the CLI list comes from Display. + // Operator must see "unreachable" verbatim, not the debug + // form or a fallback. + assert_eq!(format!("{}", RecordState::Unreachable), "unreachable"); + } + + #[test] + fn frozen_displays_as_frozen() { + // A snapshot-frozen fork base must show as "frozen" in `ls`/`status`, + // not the misleading "unreachable" (which reads as a fault). + assert_eq!(format!("{}", RecordState::Frozen), "frozen"); + } + + #[test] + fn frozen_variant_round_trips_through_serde() { + let r = record(RecordState::Frozen, Some(12345)); + let json = serde_json::to_string(&r).unwrap(); + assert!(json.contains("\"state\":\"frozen\"")); + let back: VmRecord = serde_json::from_str(&json).unwrap(); + assert_eq!(back.state, RecordState::Frozen); + } + + #[test] + fn non_running_fork_base_is_not_frozen() { + // `is_frozen_fork_base` requires record.state == Running. A golden + // that was force-reaped to Stopped (clones dangling) must NOT report + // Frozen — Frozen implies a live, paused VMM. (No clones exist for + // "does-not-exist" so this also covers has_live_clones == false.) + let r = record(RecordState::Stopped, Some(99999)); + assert!(!is_frozen_fork_base("does-not-exist", &r)); + } +} diff --git a/src/agent/terminal.rs b/src/agent/terminal.rs new file mode 100644 index 0000000..02bbebe --- /dev/null +++ b/src/agent/terminal.rs @@ -0,0 +1,624 @@ +//! Terminal handling for interactive sessions. +//! +//! Provides raw mode control and I/O multiplexing for bidirectional +//! communication between local terminal and remote VM. +//! +//! The interactive TTY machinery (raw mode, `poll()`-based I/O multiplexing, +//! SIGWINCH handling, non-blocking stdin) is POSIX-specific. On Windows the +//! same public API is provided as compile-only stubs: the host control plane +//! builds, but interactive exec sessions are not wired up there. See the +//! `windows_stub` module below. + +/// Portable file-descriptor handle used by the interactive poll loop. A real +/// `RawFd` on Unix; an opaque placeholder on Windows where the poll loop is a +/// stub. +#[cfg(unix)] +pub type Fd = std::os::unix::io::RawFd; +/// Portable file-descriptor handle (Windows): an opaque placeholder, since the +/// `poll()`-based interactive loop is a stub there. +#[cfg(not(unix))] +pub type Fd = i64; + +#[cfg(unix)] +mod unix_impl { + use std::io; + use std::os::unix::io::{AsRawFd, RawFd}; + use std::sync::atomic::{AtomicBool, Ordering}; + + /// Atomic flag set by the SIGWINCH signal handler. + static SIGWINCH_RECEIVED: AtomicBool = AtomicBool::new(false); + + /// Install a SIGWINCH handler that sets an atomic flag. + /// + /// Call this before entering an interactive loop that needs resize detection. + /// The handler is process-global; re-installing is safe and idempotent. + pub fn install_sigwinch_handler() { + extern "C" fn handler(_: libc::c_int) { + SIGWINCH_RECEIVED.store(true, Ordering::Relaxed); + } + // SAFETY: handler only touches an atomic — async-signal-safe. + unsafe { + libc::signal(libc::SIGWINCH, handler as *const () as libc::sighandler_t); + } + } + + /// Check and clear the SIGWINCH flag. + /// + /// Returns `true` if a terminal resize occurred since the last check. + pub fn check_sigwinch() -> bool { + SIGWINCH_RECEIVED.swap(false, Ordering::Relaxed) + } + + /// RAII guard for terminal raw mode. + /// + /// Saves the original terminal settings and restores them on drop, + /// even if the program panics. + pub struct RawModeGuard { + fd: RawFd, + original: libc::termios, + } + + impl RawModeGuard { + /// Enable raw mode on the given file descriptor (usually stdin). + /// + /// Returns `None` if the fd is not a TTY. + pub fn new(fd: RawFd) -> Option { + // Check if it's a TTY + if unsafe { libc::isatty(fd) } != 1 { + return None; + } + + // Get current terminal settings + let mut original: libc::termios = unsafe { std::mem::zeroed() }; + if unsafe { libc::tcgetattr(fd, &mut original) } != 0 { + return None; + } + + // Create raw mode settings + let mut raw = original; + + // Input: disable BREAK, CR-to-NL, parity, strip, flow control + raw.c_iflag &= !(libc::BRKINT | libc::ICRNL | libc::INPCK | libc::ISTRIP | libc::IXON); + + // Output: disable post-processing + raw.c_oflag &= !libc::OPOST; + + // Control: 8-bit chars + raw.c_cflag |= libc::CS8; + + // Local: disable echo, canonical mode, signals, extended + raw.c_lflag &= !(libc::ECHO | libc::ICANON | libc::IEXTEN | libc::ISIG); + + // Read returns immediately with whatever is available + raw.c_cc[libc::VMIN] = 1; + raw.c_cc[libc::VTIME] = 0; + + // Apply raw mode + if unsafe { libc::tcsetattr(fd, libc::TCSAFLUSH, &raw) } != 0 { + return None; + } + + Some(Self { fd, original }) + } + } + + impl Drop for RawModeGuard { + fn drop(&mut self) { + // Restore original terminal settings + unsafe { + libc::tcsetattr(self.fd, libc::TCSAFLUSH, &self.original); + } + } + } + + /// Get the current terminal size. + pub fn get_terminal_size() -> Option<(u16, u16)> { + let mut size: libc::winsize = unsafe { std::mem::zeroed() }; + let fd = io::stdin().as_raw_fd(); + + if unsafe { libc::ioctl(fd, libc::TIOCGWINSZ, &mut size) } == 0 { + Some((size.ws_col, size.ws_row)) + } else { + None + } + } + + /// Poll result indicating which sources have data available. + pub struct PollResult { + /// True if stdin has data available to read. + pub stdin_ready: bool, + /// True if the socket has data available to read. + pub socket_ready: bool, + /// True if the socket has hung up (peer closed connection). + pub socket_hangup: bool, + } + + /// Poll stdin and a socket for readability. + /// + /// Returns which file descriptors are ready for reading. + /// Timeout is in milliseconds, -1 for infinite. + pub fn poll_io(stdin_fd: RawFd, socket_fd: RawFd, timeout_ms: i32) -> io::Result { + let mut fds = [ + libc::pollfd { + fd: stdin_fd, + events: libc::POLLIN, + revents: 0, + }, + libc::pollfd { + fd: socket_fd, + events: libc::POLLIN, + revents: 0, + }, + ]; + + let result = unsafe { libc::poll(fds.as_mut_ptr(), 2, timeout_ms) }; + + if result < 0 { + let err = io::Error::last_os_error(); + // EINTR is not an error - just means we got a signal + if err.kind() == io::ErrorKind::Interrupted { + return Ok(PollResult { + stdin_ready: false, + socket_ready: false, + socket_hangup: false, + }); + } + return Err(err); + } + + Ok(PollResult { + // POLLHUP: the pipe's write end was closed (e.g., `echo | smolvm exec -i`). + // Without this, piped stdin EOF is never detected and the host hangs + // forever waiting for more input. + stdin_ready: fds[0].revents & (libc::POLLIN | libc::POLLHUP | libc::POLLERR) != 0, + socket_ready: fds[1].revents & libc::POLLIN != 0, + socket_hangup: fds[1].revents & (libc::POLLHUP | libc::POLLERR) != 0, + }) + } + + /// Check if stdin is a TTY. + pub fn stdin_is_tty() -> bool { + unsafe { libc::isatty(io::stdin().as_raw_fd()) == 1 } + } + + /// Write all bytes to a writer, retrying on WouldBlock. + /// + /// When stdin is set to non-blocking via `O_NONBLOCK`, the flag propagates + /// to stdout/stderr on terminals (they share the same kernel file description). + /// This helper retries writes that fail with WouldBlock. + pub fn write_all_retry(writer: &mut impl io::Write, data: &[u8]) -> io::Result<()> { + let mut pos = 0; + while pos < data.len() { + match writer.write(&data[pos..]) { + Ok(0) => { + return Err(io::Error::new(io::ErrorKind::WriteZero, "failed to write")); + } + Ok(n) => pos += n, + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + Ok(()) + } + + /// Flush a writer, retrying on WouldBlock. + pub fn flush_retry(writer: &mut impl io::Write) -> io::Result<()> { + loop { + match writer.flush() { + Ok(()) => return Ok(()), + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + } + + /// RAII guard for non-blocking stdin mode. + /// + /// Sets stdin to non-blocking on creation, restores on drop. + pub struct NonBlockingStdin { + fd: RawFd, + original_flags: libc::c_int, + } + + impl NonBlockingStdin { + /// Set stdin to non-blocking mode. + pub fn new() -> io::Result { + let fd = io::stdin().as_raw_fd(); + let flags = unsafe { libc::fcntl(fd, libc::F_GETFL) }; + if flags < 0 { + return Err(io::Error::last_os_error()); + } + + if unsafe { libc::fcntl(fd, libc::F_SETFL, flags | libc::O_NONBLOCK) } < 0 { + return Err(io::Error::last_os_error()); + } + + Ok(Self { + fd, + original_flags: flags, + }) + } + } + + impl Drop for NonBlockingStdin { + fn drop(&mut self) { + unsafe { + libc::fcntl(self.fd, libc::F_SETFL, self.original_flags); + } + } + } +} // mod unix_impl + +#[cfg(unix)] +pub use unix_impl::{ + check_sigwinch, flush_retry, get_terminal_size, install_sigwinch_handler, poll_io, + stdin_is_tty, write_all_retry, NonBlockingStdin, PollResult, RawModeGuard, +}; + +/// Windows implementation of the interactive-terminal API. +/// +/// Mirrors the POSIX `unix_impl` module using Win32 console + WinSock APIs so +/// `machine exec -it` / `machine shell` drive a real raw-mode session on +/// Windows. The shared poll loop in `agent::client` is platform-agnostic and +/// is reused unchanged; only these primitives differ. +#[cfg(not(unix))] +#[allow(missing_docs)] +mod windows_impl { + use super::Fd; + use std::io; + use std::sync::Mutex; + + use windows_sys::Win32::Foundation::{HANDLE, WAIT_FAILED}; + use windows_sys::Win32::Networking::WinSock::{ + ioctlsocket, WSACloseEvent, WSACreateEvent, WSAEnumNetworkEvents, WSAEventSelect, FD_CLOSE, + FD_READ, FIONBIO, SOCKET, WSAEVENT, WSANETWORKEVENTS, WSA_INVALID_EVENT, + }; + use windows_sys::Win32::System::Console::{ + GetConsoleMode, GetConsoleScreenBufferInfo, GetNumberOfConsoleInputEvents, GetStdHandle, + PeekConsoleInputW, ReadConsoleInputW, SetConsoleMode, CONSOLE_SCREEN_BUFFER_INFO, + DISABLE_NEWLINE_AUTO_RETURN, ENABLE_ECHO_INPUT, ENABLE_LINE_INPUT, ENABLE_PROCESSED_INPUT, + ENABLE_VIRTUAL_TERMINAL_INPUT, ENABLE_VIRTUAL_TERMINAL_PROCESSING, INPUT_RECORD, KEY_EVENT, + STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, + }; + use windows_sys::Win32::System::Threading::WaitForMultipleObjects; + + /// Convert the portable `Fd` (an `i64` carrying a console `HANDLE`) into a + /// Win32 `HANDLE`. + fn fd_to_handle(fd: Fd) -> HANDLE { + fd as usize as HANDLE + } + + /// Convert the portable `Fd` (an `i64` carrying a WinSock `SOCKET`) into a + /// `SOCKET`. + fn fd_to_socket(fd: Fd) -> SOCKET { + fd as usize as SOCKET + } + + fn std_output_handle() -> HANDLE { + // SAFETY: GetStdHandle has no preconditions. + unsafe { GetStdHandle(STD_OUTPUT_HANDLE) } + } + + /// Last terminal size observed, used to emulate SIGWINCH via polling. + static LAST_SIZE: Mutex> = Mutex::new(None); + + /// Seed the resize cache with the current terminal size. + /// + /// The Windows console has no SIGWINCH; resize is detected by comparing the + /// current size against this cache in [`check_sigwinch`]. + pub fn install_sigwinch_handler() { + let current = get_terminal_size(); + if let Ok(mut guard) = LAST_SIZE.lock() { + *guard = current; + } + } + + /// Return `true` if the terminal has been resized since the last check. + pub fn check_sigwinch() -> bool { + let current = get_terminal_size(); + let mut guard = match LAST_SIZE.lock() { + Ok(g) => g, + Err(_) => return false, + }; + if *guard != current { + *guard = current; + true + } else { + false + } + } + + /// RAII guard for console raw mode. + /// + /// Saves the input handle's mode (and the output handle's VT mode), switches + /// the console into raw VT-input mode, and restores both on drop. + pub struct RawModeGuard { + input: HANDLE, + original_input_mode: u32, + output: HANDLE, + original_output_mode: u32, + restore_output: bool, + } + + impl RawModeGuard { + /// Enable raw mode on the given console input handle. + /// + /// Returns `None` if the handle is not a console. + pub fn new(fd: Fd) -> Option { + let input = fd_to_handle(fd); + + let mut original_input_mode: u32 = 0; + // SAFETY: `input` is the process's console input handle; GetConsoleMode + // fails (returns 0) if it is not a console, which we treat as "no TTY". + if unsafe { GetConsoleMode(input, &mut original_input_mode) } == 0 { + return None; + } + + // Clear cooked-mode input flags, enable VT input so the guest sees + // raw key/escape sequences. + let raw_input_mode = (original_input_mode + & !(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT | ENABLE_PROCESSED_INPUT)) + | ENABLE_VIRTUAL_TERMINAL_INPUT; + // SAFETY: setting a derived mode on a valid console handle. + if unsafe { SetConsoleMode(input, raw_input_mode) } == 0 { + return None; + } + + // Enable VT processing on output so ANSI escapes from the guest render. + let output = std_output_handle(); + let mut original_output_mode: u32 = 0; + let restore_output = + // SAFETY: querying the output handle's mode. + if unsafe { GetConsoleMode(output, &mut original_output_mode) } != 0 { + let out_mode = original_output_mode + | ENABLE_VIRTUAL_TERMINAL_PROCESSING + | DISABLE_NEWLINE_AUTO_RETURN; + // SAFETY: setting a derived mode on a valid console output handle. + unsafe { + SetConsoleMode(output, out_mode); + } + true + } else { + false + }; + + Some(Self { + input, + original_input_mode, + output, + original_output_mode, + restore_output, + }) + } + } + + impl Drop for RawModeGuard { + fn drop(&mut self) { + // SAFETY: restoring previously-saved console modes on valid handles. + unsafe { + SetConsoleMode(self.input, self.original_input_mode); + if self.restore_output { + SetConsoleMode(self.output, self.original_output_mode); + } + } + } + } + + /// Get the current terminal size from the console window rectangle. + pub fn get_terminal_size() -> Option<(u16, u16)> { + let output = std_output_handle(); + let mut info: CONSOLE_SCREEN_BUFFER_INFO = unsafe { std::mem::zeroed() }; + // SAFETY: `output` is the console output handle; the call fails (0) if it + // is not a console. + if unsafe { GetConsoleScreenBufferInfo(output, &mut info) } == 0 { + return None; + } + let w = info.srWindow; + let cols = (w.Right - w.Left + 1).max(0) as u16; + let rows = (w.Bottom - w.Top + 1).max(0) as u16; + Some((cols, rows)) + } + + /// Poll result indicating which sources have data available. + pub struct PollResult { + pub stdin_ready: bool, + pub socket_ready: bool, + pub socket_hangup: bool, + } + + /// Poll the console input handle and the agent socket for readability. + /// + /// `timeout_ms` is in milliseconds; -1 means infinite. A `stdin_fd` of -1 + /// (the loop's EOF sentinel) suppresses waiting on console input. + pub fn poll_io(stdin_fd: Fd, socket_fd: Fd, timeout_ms: i32) -> io::Result { + let socket = fd_to_socket(socket_fd); + + // Register the socket with a WSA event for FD_READ | FD_CLOSE. This also + // puts the socket into non-blocking mode; we restore blocking before + // returning so the client's blocking receive() behaves like Unix. + // SAFETY: WSACreateEvent has no preconditions. + let event: WSAEVENT = unsafe { WSACreateEvent() }; + if event == WSA_INVALID_EVENT { + return Err(io::Error::last_os_error()); + } + + // Restore the socket to blocking and free the event on every exit path. + struct SocketCleanup { + socket: SOCKET, + event: WSAEVENT, + } + impl Drop for SocketCleanup { + fn drop(&mut self) { + // SAFETY: deregister the event selection, then put the socket back + // into blocking mode and close the event. + unsafe { + WSAEventSelect(self.socket, 0 as WSAEVENT, 0); + let mut nonblocking: u32 = 0; + ioctlsocket(self.socket, FIONBIO, &mut nonblocking); + WSACloseEvent(self.event); + } + } + } + let _cleanup = SocketCleanup { socket, event }; + + // SAFETY: registering FD_READ|FD_CLOSE notifications on the socket. + if unsafe { WSAEventSelect(socket, event, (FD_READ | FD_CLOSE) as i32) } != 0 { + return Err(io::Error::last_os_error()); + } + + let wait_console = stdin_fd != -1; + let console = fd_to_handle(stdin_fd); + + // Build the wait handle set: optionally the console input handle, then + // the WSA event (which is HANDLE-compatible). + let mut handles: [HANDLE; 2] = [0 as HANDLE; 2]; + let mut count: u32 = 0; + if wait_console { + handles[count as usize] = console; + count += 1; + } + handles[count as usize] = event as usize as HANDLE; + count += 1; + + let dw_timeout: u32 = if timeout_ms < 0 { + u32::MAX // INFINITE + } else { + timeout_ms as u32 + }; + + // SAFETY: `handles[..count]` are valid waitable handles. + let wait = unsafe { WaitForMultipleObjects(count, handles.as_ptr(), 0, dw_timeout) }; + + if wait == WAIT_FAILED { + return Err(io::Error::last_os_error()); + } + + let mut stdin_ready = false; + let mut socket_ready = false; + let mut socket_hangup = false; + + // The console input handle signals for non-key events too (focus, mouse, + // resize). Peek and only report stdin readiness for actual key/char input; + // drain non-key records so we don't busy-loop. + if wait_console { + stdin_ready = console_input_has_key(console); + } + + // Read which socket events fired regardless of which handle woke the wait — + // WSAEnumNetworkEvents reflects all pending notifications and resets them. + let mut net_events: WSANETWORKEVENTS = unsafe { std::mem::zeroed() }; + // SAFETY: `socket` and `event` are valid; net_events is writable. + if unsafe { WSAEnumNetworkEvents(socket, event, &mut net_events) } == 0 { + if net_events.lNetworkEvents & FD_READ as i32 != 0 { + socket_ready = true; + } + if net_events.lNetworkEvents & FD_CLOSE as i32 != 0 { + socket_hangup = true; + } + } + + Ok(PollResult { + stdin_ready, + socket_ready, + socket_hangup, + }) + } + + /// Inspect pending console input records: return `true` if a key/char event + /// is queued. Non-key events (focus/mouse/buffer-resize) are drained so the + /// wait doesn't busy-loop, but key events are left in the queue so the + /// client's subsequent `Stdin::read` (ReadFile/ReadConsole) consumes them. + fn console_input_has_key(console: HANDLE) -> bool { + loop { + let mut available: u32 = 0; + // SAFETY: querying the count of pending input records. + if unsafe { GetNumberOfConsoleInputEvents(console, &mut available) } == 0 + || available == 0 + { + return false; + } + + // Peek (non-destructive) at the next record. + let mut record: INPUT_RECORD = unsafe { std::mem::zeroed() }; + let mut read: u32 = 0; + // SAFETY: peeks one record into `record` without removing it. + if unsafe { PeekConsoleInputW(console, &mut record, 1, &mut read) } == 0 || read == 0 { + return false; + } + + if record.EventType as u32 == KEY_EVENT { + // Real input — leave it queued for the loop's stdin read. + return true; + } + + // Non-key event: remove just this record so the wait doesn't keep + // re-signaling on it, then look again for a real key event. + // SAFETY: removes exactly one (the peeked non-key) record. + if unsafe { ReadConsoleInputW(console, &mut record, 1, &mut read) } == 0 || read == 0 { + return false; + } + } + } + + /// Check if stdin is a console (TTY). + pub fn stdin_is_tty() -> bool { + // SAFETY: GetStdHandle + GetConsoleMode; the latter fails for non-consoles. + unsafe { + let h = GetStdHandle(STD_INPUT_HANDLE); + let mut mode: u32 = 0; + GetConsoleMode(h, &mut mode) != 0 + } + } + + /// Write all bytes to a writer. + pub fn write_all_retry(writer: &mut impl io::Write, data: &[u8]) -> io::Result<()> { + writer.write_all(data) + } + + /// Flush a writer. + pub fn flush_retry(writer: &mut impl io::Write) -> io::Result<()> { + writer.flush() + } + + /// RAII guard placeholder for non-blocking stdin. + /// + /// On Windows the interactive loop only reads stdin after `poll_io` reports + /// readiness, and reads through `std::io::Stdin::read` (which yields UTF-8 + /// from the VT-input console), so no fd-level non-blocking flag is needed. + pub struct NonBlockingStdin; + + impl NonBlockingStdin { + pub fn new() -> io::Result { + Ok(Self) + } + } +} + +#[cfg(not(unix))] +pub use windows_impl::{ + check_sigwinch, flush_retry, get_terminal_size, install_sigwinch_handler, poll_io, + stdin_is_tty, write_all_retry, NonBlockingStdin, PollResult, RawModeGuard, +}; + +#[cfg(all(test, unix))] +mod tests { + use super::*; + + #[test] + fn test_stdin_is_tty_returns_bool() { + // Just verify it doesn't panic - actual value depends on test environment + let _ = stdin_is_tty(); + } + + #[test] + fn test_get_terminal_size_returns_option() { + // Just verify it doesn't panic + let _ = get_terminal_size(); + } +} diff --git a/src/agent/vsock_service.rs b/src/agent/vsock_service.rs new file mode 100644 index 0000000..616d197 --- /dev/null +++ b/src/agent/vsock_service.rs @@ -0,0 +1,109 @@ +//! Registry of guest↔host vsock-backed services. +//! +//! Each entry bridges a guest vsock port to a host `AF_UNIX` socket via +//! `krun_add_vsock_port2` — a deliberate opening in guest↔host isolation. A +//! service is *enabled* purely by whether its opt-in input (a socket path) is +//! present for the launch: enabling it IS the decision, so there is no separate +//! gate. Every socket path here is host-derived (the per-VM dir or host config), +//! never guest-influenced. +//! +//! This module is the single place the launcher branches on these services. +//! Adding a new vsock-backed capability is one `impl VsockService` plus one line +//! in [`registry`] — no new control flow in the launcher. + +use smolvm_protocol::ports; +use std::path::Path; + +/// The optional, host-derived socket paths a service resolves itself from. +/// +/// One field per opt-in source; a service reads the one it owns. All paths are +/// produced by smolvm, never by the guest. +pub struct VsockServiceInputs<'a> { + /// Host SSH agent socket to forward into the guest. + pub ssh_agent_socket: Option<&'a Path>, + /// Host DNS-filter proxy socket. + pub dns_filter_socket: Option<&'a Path>, + /// Host CUDA-over-vsock server socket (experimental). + pub cuda_socket: Option<&'a Path>, +} + +/// A service resolved to "on" for a launch: the concrete wiring the launcher +/// applies. +pub struct ActiveVsockService<'a> { + /// Human-readable name for log lines. + pub name: &'static str, + /// Guest-visible vsock port. Must be reserved in [`ports`] and must not + /// collide with [`ports::AGENT_CONTROL`]. + pub port: u32, + /// `false` → guest connects *out* to the host endpoint at `socket` + /// (SSH/DNS/CUDA); `true` → guest serves and the host connects in. + pub listen: bool, + /// Host `AF_UNIX` socket the port bridges to. + pub socket: &'a Path, + /// Env vars injected into the guest agent to activate the guest side of the + /// service (e.g. the SSH agent bridge). Empty when the guest side needs no + /// signal (it connects out on its own, like the DNS and CUDA clients). + pub guest_env: &'static [(&'static str, &'static str)], +} + +/// One guest↔host vsock-backed capability. Implementors are stateless unit +/// structs registered in [`registry`]. +pub trait VsockService: Sync { + /// Resolve to an active service when enabled for this launch, else `None`. + fn resolve<'a>(&self, inputs: &VsockServiceInputs<'a>) -> Option>; +} + +/// SSH agent forwarding: the host's `SSH_AUTH_SOCK` bridged into the guest. The +/// guest agent is told to start its bridge via `SMOLVM_SSH_AGENT=1`. +struct SshAgentService; +impl VsockService for SshAgentService { + fn resolve<'a>(&self, inputs: &VsockServiceInputs<'a>) -> Option> { + inputs.ssh_agent_socket.map(|socket| ActiveVsockService { + name: "SSH agent forwarding", + port: ports::SSH_AGENT, + listen: false, + socket, + guest_env: &[("SMOLVM_SSH_AGENT", "1")], + }) + } +} + +/// DNS filtering proxy: the guest forwards DNS queries to a host listener that +/// enforces the egress allow-list. +struct DnsFilterService; +impl VsockService for DnsFilterService { + fn resolve<'a>(&self, inputs: &VsockServiceInputs<'a>) -> Option> { + inputs.dns_filter_socket.map(|socket| ActiveVsockService { + name: "DNS filtering", + port: ports::DNS_FILTER, + listen: false, + socket, + guest_env: &[], + }) + } +} + +/// CUDA-over-vsock (experimental): the guest CUDA client connects out to a host +/// server that loads `nvcuda.dll` / `libcuda.so.1` and runs the calls on the +/// host NVIDIA GPU. +struct CudaService; +impl VsockService for CudaService { + fn resolve<'a>(&self, inputs: &VsockServiceInputs<'a>) -> Option> { + inputs.cuda_socket.map(|socket| ActiveVsockService { + name: "CUDA-over-vsock", + port: ports::CUDA, + listen: false, + socket, + // Arm guest-RAM zero-copy for `cudaMallocHost` buffers. The shim + // only uses it when it can read `/proc/self/pagemap` (guest is root) + // and the host published the mapping; otherwise it falls back to the + // byte-shipping path transparently. + guest_env: &[("SMOLVM_CUDA_ZEROCOPY", "1")], + }) + } +} + +/// All known vsock services. Add a capability by appending one entry. +pub fn registry() -> &'static [&'static dyn VsockService] { + &[&SshAgentService, &DnsFilterService, &CudaService] +} diff --git a/src/api/errors.rs b/src/api/errors.rs new file mode 100644 index 0000000..427e7a8 --- /dev/null +++ b/src/api/errors.rs @@ -0,0 +1,152 @@ +//! API error types with HTTP status mapping. + +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use serde::Serialize; +use std::fmt::Display; + +/// API error type with HTTP status code mapping. +#[derive(Debug)] +pub enum ApiError { + /// Resource not found (404). + NotFound(String), + /// Conflict - resource already exists or invalid state (409). + Conflict(String), + /// A published host port could not be bound because it is already in use + /// (409). Distinct from `Conflict` so the control plane can recognize this + /// specific, retryable failure (reallocate a different port and retry) + /// instead of treating it as a generic 500/409. + PortConflict(String), + /// Bad request - invalid input (400). + BadRequest(String), + /// Request timeout (408). + Timeout, + /// Internal server error (500). + Internal(String), +} + +impl ApiError { + /// Convert any displayable error to an internal API error. + pub fn internal(err: impl Display) -> Self { + Self::Internal(err.to_string()) + } + + /// Wrap a database-layer error with a consistent message prefix. + pub fn database(err: impl Display) -> Self { + Self::Internal(format!("database error: {}", err)) + } +} + +/// JSON error response body. +#[derive(Serialize)] +struct ErrorResponse { + error: String, + code: &'static str, +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let (status, code, message) = match self { + ApiError::NotFound(msg) => (StatusCode::NOT_FOUND, "NOT_FOUND", msg), + ApiError::Conflict(msg) => (StatusCode::CONFLICT, "CONFLICT", msg), + ApiError::PortConflict(msg) => (StatusCode::CONFLICT, "PORT_IN_USE", msg), + ApiError::BadRequest(msg) => (StatusCode::BAD_REQUEST, "BAD_REQUEST", msg), + ApiError::Timeout => ( + StatusCode::REQUEST_TIMEOUT, + "TIMEOUT", + "request timed out".to_string(), + ), + ApiError::Internal(msg) => (StatusCode::INTERNAL_SERVER_ERROR, "INTERNAL_ERROR", msg), + }; + + let body = Json(ErrorResponse { + error: message, + code, + }); + + (status, body).into_response() + } +} + +impl From for ApiError { + fn from(err: crate::error::Error) -> Self { + match &err { + crate::error::Error::VmNotFound { name } => { + ApiError::NotFound(format!("machine not found: {}", name)) + } + crate::error::Error::InvalidState { expected, actual } => ApiError::Conflict(format!( + "invalid state: expected {}, got {}", + expected, actual + )), + // Handle structured Agent errors using kind for HTTP status mapping + crate::error::Error::Agent { reason, kind, .. } => match kind { + crate::error::AgentErrorKind::NotFound => ApiError::NotFound(reason.clone()), + crate::error::AgentErrorKind::Conflict => ApiError::Conflict(reason.clone()), + crate::error::AgentErrorKind::Other => ApiError::Internal(reason.clone()), + }, + _ => ApiError::Internal(err.to_string()), + } + } +} + +/// Classify errors from `ensure_machine_running` into proper HTTP status codes. +/// +/// Mount validation errors are 400 (Bad Request), everything else uses the +/// standard `Error -> ApiError` mapping (500 for startup failures, etc.). +pub fn classify_ensure_running_error(err: crate::Error) -> ApiError { + match &err { + crate::Error::Mount { .. } + | crate::Error::InvalidMountPath { .. } + | crate::Error::MountSourceNotFound { .. } => { + ApiError::BadRequest(format!("mount validation failed: {}", err)) + } + _ => ApiError::from(err), + } +} + +impl From for ApiError { + fn from(err: tokio::task::JoinError) -> Self { + ApiError::Internal(format!("task failed: {}", err)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::http::StatusCode; + + #[test] + fn test_api_error_status_codes() { + let cases = [ + (ApiError::NotFound("x".into()), StatusCode::NOT_FOUND), + (ApiError::Conflict("x".into()), StatusCode::CONFLICT), + (ApiError::BadRequest("x".into()), StatusCode::BAD_REQUEST), + (ApiError::Timeout, StatusCode::REQUEST_TIMEOUT), + ( + ApiError::Internal("x".into()), + StatusCode::INTERNAL_SERVER_ERROR, + ), + ]; + for (error, expected) in cases { + assert_eq!(error.into_response().status(), expected); + } + } + + #[test] + fn test_agent_error_kind_mapping() { + // NotFound kind -> NotFound + let err = crate::error::Error::agent_not_found("lookup", "container not found"); + assert!(matches!(ApiError::from(err), ApiError::NotFound(_))); + + // Conflict kind -> Conflict + let err = crate::error::Error::agent_conflict("create", "already exists"); + assert!(matches!(ApiError::from(err), ApiError::Conflict(_))); + + // Default (Other) kind -> Internal + let err = crate::error::Error::agent("connect", "connection refused"); + assert!(matches!(ApiError::from(err), ApiError::Internal(_))); + } +} diff --git a/src/api/handlers/exec.rs b/src/api/handlers/exec.rs new file mode 100644 index 0000000..53046b7 --- /dev/null +++ b/src/api/handlers/exec.rs @@ -0,0 +1,793 @@ +//! Command execution handlers. + +use axum::{ + extract::{ + ws::{Message, WebSocketUpgrade}, + Path, Query, State, + }, + response::sse::{Event, KeepAlive, Sse}, + Json, +}; +use futures_util::{SinkExt, StreamExt}; +use std::convert::Infallible; +use std::io::{BufRead, BufReader, Seek, SeekFrom}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use crate::api::error::{classify_ensure_running_error, ApiError}; +use crate::api::state::{ensure_running_and_persist, with_machine_client_traced, ApiState}; +use crate::api::types::{ + ApiErrorResponse, EnvVar, ExecRequest, ExecResponse, LogsQuery, RunRequest, +}; +use crate::api::validate_command; +use crate::api::TraceId; +use crate::data::consts::BYTES_PER_MIB; +use crate::data::storage::HostMount; +use tokio::sync::Semaphore; + +/// Execute a command in a machine. +/// +/// This executes directly in the VM (not in a container). +#[utoipa::path( + post, + path = "/api/v1/machines/{id}/exec", + tag = "Execution", + params( + ("id" = String, Path, description = "Machine name") + ), + request_body = ExecRequest, + responses( + (status = 200, description = "Command executed", body = ExecResponse), + (status = 400, description = "Invalid request", body = ApiErrorResponse), + (status = 404, description = "Machine not found", body = ApiErrorResponse), + (status = 500, description = "Execution failed", body = ApiErrorResponse) + ) +)] +pub async fn exec_command( + State(state): State>, + Path(id): Path, + trace_id: Option>, + Json(req): Json, +) -> Result, ApiError> { + let tid = trace_id.map(|t| t.0 .0.clone()); + validate_command(&req.command)?; + + let entry = state.get_machine(&id)?; + + // Ensure machine is running and persist state to DB + ensure_running_and_persist(&state, &id, &entry) + .await + .map_err(classify_ensure_running_error)?; + + // Resolve secrets ONCE, before the background/foreground split, so a + // detached workload gets them too (a long-lived daemon usually needs its + // credentials more than a one-shot exec does). Env precedence (low → high): + // req.env (caller-plaintext) → record.secret_refs (persisted by a + // TrustedLocal actor) → req.secrets (ad-hoc, Untrusted). Validation runs + // before resolution so structural/scope violations surface as 400 without + // the resolution audit firing. + crate::api::handlers::validate_request_secrets(&req.secrets)?; + let record_env = crate::api::handlers::record_secret_refs_env(&entry)?; + let req_env = crate::api::handlers::resolve_request_secrets(&req.secrets)?; + let mut env = EnvVar::to_tuples(&req.env); + env.extend(crate::secrets::expose_into_env(record_env)); + env.extend(crate::secrets::expose_into_env(req_env)); + + // Detached/background: spawn the process and return its PID immediately, so a + // long-lived daemon (dev server, agent runner) keeps running after the + // request returns. Image machines run it in their container (persistent + // overlay); plain machines run it in the VM. + if req.background { + let command = req.command.clone(); + let workdir = req.workdir.clone(); + let machine_image = state.lookup_vm(&id).await?.and_then(|r| r.image); + let pid = if let Some(image) = machine_image { + let mounts_config = { + let e = entry.lock(); + e.mounts + .iter() + .enumerate() + .map(|(i, m)| (HostMount::mount_tag(i), m.target.clone(), m.readonly)) + .collect::>() + }; + let overlay_id = id.clone(); + with_machine_client_traced(&entry, tid, move |c| { + if c.query(&image)?.is_none() { + c.pull_with_registry_config(&image)?; + } + let config = crate::agent::RunConfig::new(image, command) + .with_env(env) + .with_workdir(workdir) + .with_mounts(mounts_config) + .with_persistent_overlay(Some(overlay_id)); + c.run_background(config) + }) + .await? + } else { + with_machine_client_traced(&entry, tid, move |c| { + c.vm_exec_background(command, env, workdir) + }) + .await? + }; + let stdout = format!("pid={pid}\n"); + return Ok(Json(ExecResponse { + exit_code: 0, + stdout_b64: stdout.clone().into_bytes(), + stdout, + stderr_b64: Vec::new(), + stderr: String::new(), + })); + } + + // Secrets already resolved into `env` above (shared with the background + // path); env precedence is req.env < record.secret_refs < req.secrets. + let command = req.command.clone(); + let workdir = req.workdir.clone(); + let timeout = req.timeout_secs.map(Duration::from_secs); + let stdin_data = req.stdin.clone(); + + // Image-based machines exec INSIDE a container from their image, with a + // per-machine persistent overlay so filesystem changes persist across exec + // sessions. Without this, exec runs in the bare agent VM (no `python3`, + // etc.) — the image is never entered. Plain machines exec in the VM + // directly via `vm_exec`. + let machine_image = state.lookup_vm(&id).await?.and_then(|r| r.image); + + let start = std::time::Instant::now(); + let (exit_code, stdout, stderr) = if let Some(image) = machine_image { + let mounts_config = { + let e = entry.lock(); + e.mounts + .iter() + .enumerate() + .map(|(i, m)| (HostMount::mount_tag(i), m.target.clone(), m.readonly)) + .collect::>() + }; + let overlay_id = id.clone(); + let stdin_data = stdin_data.clone(); + with_machine_client_traced(&entry, tid, move |c| { + // Pull only if the image isn't already present — avoids a registry + // round-trip on every exec, and works once cached even on + // network-restricted machines. + if c.query(&image)?.is_none() { + c.pull_with_registry_config(&image)?; + } + let config = crate::agent::RunConfig::new(image, command) + .with_env(env) + .with_workdir(workdir) + .with_mounts(mounts_config) + .with_timeout(timeout) + .with_persistent_overlay(Some(overlay_id)) + .with_stdin(stdin_data); + c.run_non_interactive(config) + }) + .await? + } else { + with_machine_client_traced(&entry, tid, move |c| { + c.vm_exec(command, env, workdir, timeout, stdin_data) + }) + .await? + }; + metrics::histogram!("smolvm_exec_seconds").record(start.elapsed().as_secs_f64()); + + Ok(Json(ExecResponse { + exit_code, + stdout: String::from_utf8_lossy(&stdout).into_owned(), + stderr: String::from_utf8_lossy(&stderr).into_owned(), + stdout_b64: stdout, + stderr_b64: stderr, + })) +} + +/// Execute a command with streaming output (Server-Sent Events). +/// +/// Returns real-time stdout/stderr as SSE events. Useful for long-running +/// commands where buffering the entire output is impractical. +#[utoipa::path( + post, + path = "/api/v1/machines/{id}/exec/stream", + tag = "Execution", + params( + ("id" = String, Path, description = "Machine name") + ), + request_body = ExecRequest, + responses( + (status = 200, description = "Streaming output (SSE)", content_type = "text/event-stream"), + (status = 404, description = "Machine not found", body = ApiErrorResponse), + (status = 500, description = "Execution failed", body = ApiErrorResponse) + ) +)] +pub async fn exec_stream( + State(state): State>, + Path(id): Path, + trace_id: Option>, + Json(req): Json, +) -> Result>>, ApiError> { + let tid = trace_id.map(|t| t.0 .0.clone()); + validate_command(&req.command)?; + + let entry = state.get_machine(&id)?; + ensure_running_and_persist(&state, &id, &entry) + .await + .map_err(classify_ensure_running_error)?; + + crate::api::handlers::validate_request_secrets(&req.secrets)?; + let record_env = crate::api::handlers::record_secret_refs_env(&entry)?; + let req_env = crate::api::handlers::resolve_request_secrets(&req.secrets)?; + + let command = req.command.clone(); + let mut env = EnvVar::to_tuples(&req.env); + env.extend(crate::secrets::expose_into_env(record_env)); + env.extend(crate::secrets::expose_into_env(req_env)); + let workdir = req.workdir.clone(); + let timeout = req.timeout_secs.map(Duration::from_secs); + + // Image-based machines stream from a container in their image (persistent + // overlay keyed by machine name); plain machines stream from the VM + // directly. Without this, streaming exec on an image machine produces no + // output (the agent-base streaming path doesn't enter the container). + let machine_image = state.lookup_vm(&id).await?.and_then(|r| r.image); + + // Run streaming exec via the machine client (vsock is synchronous) + let start = std::time::Instant::now(); + let events = if let Some(image) = machine_image { + let mounts_config = { + let e = entry.lock(); + e.mounts + .iter() + .enumerate() + .map(|(i, m)| (HostMount::mount_tag(i), m.target.clone(), m.readonly)) + .collect::>() + }; + let overlay_id = id.clone(); + with_machine_client_traced(&entry, tid, move |c| { + if c.query(&image)?.is_none() { + c.pull_with_registry_config(&image)?; + } + let config = crate::agent::RunConfig::new(image, command) + .with_env(env) + .with_workdir(workdir) + .with_mounts(mounts_config) + .with_timeout(timeout) + .with_persistent_overlay(Some(overlay_id)); + let mut evs = Vec::new(); + c.run_streaming_with(config, |e| evs.push(e))?; + Ok(evs) + }) + .await? + } else { + with_machine_client_traced(&entry, tid, move |c| { + c.vm_exec_streaming(command, env, workdir, timeout) + }) + .await? + }; + metrics::histogram!("smolvm_exec_seconds").record(start.elapsed().as_secs_f64()); + + // Convert events to SSE stream + let stream = futures_util::stream::iter(events.into_iter().map(|event| { + let sse_event = match event { + crate::agent::ExecEvent::Stdout(data) => Event::default() + .event("stdout") + .data(String::from_utf8_lossy(&data)), + crate::agent::ExecEvent::Stderr(data) => Event::default() + .event("stderr") + .data(String::from_utf8_lossy(&data)), + crate::agent::ExecEvent::Exit(code) => Event::default() + .event("exit") + .data(format!("{{\"exitCode\":{}}}", code)), + crate::agent::ExecEvent::Error(msg) => Event::default() + .event("error") + .data(format!("{{\"message\":\"{}\"}}", msg)), + }; + Ok(sse_event) + })); + + Ok(Sse::new(stream).keep_alive(KeepAlive::default())) +} + +/// Run a command in an image. +/// +/// This creates a temporary overlay from the image and runs the command. +#[utoipa::path( + post, + path = "/api/v1/machines/{id}/run", + tag = "Execution", + params( + ("id" = String, Path, description = "Machine name") + ), + request_body = RunRequest, + responses( + (status = 200, description = "Command executed", body = ExecResponse), + (status = 400, description = "Invalid request", body = ApiErrorResponse), + (status = 404, description = "Machine not found", body = ApiErrorResponse), + (status = 500, description = "Execution failed", body = ApiErrorResponse) + ) +)] +pub async fn run_command( + State(state): State>, + Path(id): Path, + trace_id: Option>, + Json(req): Json, +) -> Result, ApiError> { + let tid = trace_id.map(|t| t.0 .0.clone()); + validate_command(&req.command)?; + + let entry = state.get_machine(&id)?; + + // Ensure machine is running and persist state to DB + ensure_running_and_persist(&state, &id, &entry) + .await + .map_err(classify_ensure_running_error)?; + + crate::api::handlers::validate_request_secrets(&req.secrets)?; + let record_env = crate::api::handlers::record_secret_refs_env(&entry)?; + let req_env = crate::api::handlers::resolve_request_secrets(&req.secrets)?; + + let image = req.image.clone(); + let command = req.command.clone(); + let mut env = EnvVar::to_tuples(&req.env); + env.extend(crate::secrets::expose_into_env(record_env)); + env.extend(crate::secrets::expose_into_env(req_env)); + let workdir = req.workdir.clone(); + let timeout = req.timeout_secs.map(Duration::from_secs); + + // Get mounts from machine config (converted to protocol format) + let mounts_config = { + let entry = entry.lock(); + entry + .mounts + .iter() + .enumerate() + .map(|(i, m)| { + let tag = HostMount::mount_tag(i); + (tag, m.target.clone(), m.readonly) + }) + .collect::>() + }; + + let start = std::time::Instant::now(); + let (exit_code, stdout, stderr) = with_machine_client_traced(&entry, tid, move |c| { + let config = crate::agent::RunConfig::new(image, command) + .with_env(env) + .with_workdir(workdir) + .with_mounts(mounts_config) + .with_timeout(timeout); + c.run_non_interactive(config) + }) + .await?; + metrics::histogram!("smolvm_exec_seconds").record(start.elapsed().as_secs_f64()); + + Ok(Json(ExecResponse { + exit_code, + stdout: String::from_utf8_lossy(&stdout).into_owned(), + stderr: String::from_utf8_lossy(&stderr).into_owned(), + stdout_b64: stdout, + stderr_b64: stderr, + })) +} + +/// Query parameters for an interactive PTY session. +#[derive(Debug, serde::Deserialize)] +pub struct InteractiveQuery { + /// Program to run (single argv[0]); defaults to `/bin/sh`. + pub cmd: Option, + /// Initial terminal width in columns. + pub cols: Option, + /// Initial terminal height in rows. + pub rows: Option, +} + +/// Interactive PTY session over a WebSocket. +/// +/// The client connects a WebSocket; binary frames are forwarded to the +/// command's stdin and the PTY's output is sent back as binary frames. A JSON +/// text frame `{"type":"resize","cols":N,"rows":N}` resizes the terminal. When +/// the command exits, a final text frame `{"type":"exit","code":N}` is sent +/// before the socket closes. +/// +/// Image machines run the program in their persistent-overlay container (the +/// same filesystem `exec` uses); plain machines run it directly in the VM. +pub async fn exec_interactive( + State(state): State>, + Path(id): Path, + Query(q): Query, + _trace_id: Option>, + ws: WebSocketUpgrade, +) -> Result { + let entry = state.get_machine(&id)?; + ensure_running_and_persist(&state, &id, &entry) + .await + .map_err(classify_ensure_running_error)?; + + let machine_image = state.lookup_vm(&id).await?.and_then(|r| r.image); + + let command = vec![q.cmd.clone().unwrap_or_else(|| "/bin/sh".to_string())]; + let init_size = (q.cols.unwrap_or(80), q.rows.unwrap_or(24)); + + // Snapshot mounts now (used only for image runs) so the upgrade closure + // doesn't need to re-lock the entry. + let mounts_config = { + let e = entry.lock(); + e.mounts + .iter() + .enumerate() + .map(|(i, m)| (HostMount::mount_tag(i), m.target.clone(), m.readonly)) + .collect::>() + }; + + Ok(ws.on_upgrade(move |socket| async move { + let (mut ws_tx, mut ws_rx) = socket.split(); + + // Input channel: WS task → blocking session (sync mpsc; session try_recv's it). + let (in_tx, in_rx) = std::sync::mpsc::channel::(); + // Output channel: blocking session → WS task (tokio mpsc; blocking_send from session). + let (out_tx, mut out_rx) = + tokio::sync::mpsc::channel::(256); + + // Seed the initial PTY size before any input. + let _ = in_tx.send(crate::agent::InteractiveInput::Resize { + cols: init_size.0, + rows: init_size.1, + }); + + // Run the interactive session on a DEDICATED agent connection — NOT the + // shared per-machine client. A PTY can outlive its usefulness (a client + // that disconnects while a `sleep` or daemon keeps running), and holding + // the shared client lock for the whole session would block every other + // operation on that machine until the command exits. A fresh connection + // also lets the agent kill the PTY child the moment we drop it on + // disconnect. We lock the entry only briefly, to dial. + let session_entry = entry.clone(); + let session = tokio::spawn(async move { + let connect = { session_entry.lock().manager.connect() }; + let mut client = match connect { + Ok(c) => c, + Err(e) => { + tracing::warn!(error = ?e, "pty: failed to open dedicated agent connection"); + return -1; + } + }; + tokio::task::spawn_blocking(move || { + let on_output = move |o| { + // If the WS side is gone, the receiver is dropped; ignore. + let _ = out_tx.blocking_send(o); + }; + if let Some(image) = machine_image { + match client.query(&image) { + Ok(Some(_)) => {} + Ok(None) => { + if let Err(e) = client.pull_with_registry_config(&image) { + tracing::warn!(error = ?e, "pty: image pull failed"); + return -1; + } + } + Err(e) => { + tracing::warn!(error = ?e, "pty: image query failed"); + return -1; + } + } + let config = crate::agent::RunConfig::new(image, command) + .with_mounts(mounts_config) + .with_tty(true) + .with_persistent_overlay(Some(id)); + client + .run_interactive_io(config, in_rx, on_output) + .unwrap_or_else(|e| { + tracing::warn!(error = ?e, "pty: interactive run failed"); + -1 + }) + } else { + client + .vm_exec_interactive_io(command, Vec::new(), None, true, in_rx, on_output) + .unwrap_or_else(|e| { + tracing::warn!(error = ?e, "pty: interactive vm_exec failed"); + -1 + }) + } + }) + .await + .unwrap_or(-1) + }); + + // Pump WS → session input. Dropping `in_tx` (when this task ends) signals + // EOF to the session via channel disconnect. + let input_pump = tokio::spawn(async move { + while let Some(Ok(msg)) = ws_rx.next().await { + match msg { + Message::Binary(b) + if in_tx + .send(crate::agent::InteractiveInput::Stdin(b.to_vec())) + .is_err() => + { + break; + } + Message::Binary(_) => {} + Message::Text(t) => { + // Control frames are JSON; anything else is treated as raw stdin. + match serde_json::from_str::(t.as_str()) { + Ok(v) if v["type"] == "resize" => { + let cols = v["cols"].as_u64().unwrap_or(80) as u16; + let rows = v["rows"].as_u64().unwrap_or(24) as u16; + let _ = in_tx + .send(crate::agent::InteractiveInput::Resize { cols, rows }); + } + Ok(v) if v["type"] == "stdin" => { + if let Some(d) = v["data"].as_str() { + let _ = in_tx.send(crate::agent::InteractiveInput::Stdin( + d.as_bytes().to_vec(), + )); + } + } + _ => { + let _ = in_tx.send(crate::agent::InteractiveInput::Stdin( + t.as_bytes().to_vec(), + )); + } + } + } + Message::Close(_) => { + let _ = in_tx.send(crate::agent::InteractiveInput::Eof); + break; + } + _ => {} + } + } + }); + + // Pump session output → WS. Ends when the session drops `out_tx` (command exit). + while let Some(o) = out_rx.recv().await { + let bytes = match o { + crate::agent::InteractiveOutput::Stdout(d) + | crate::agent::InteractiveOutput::Stderr(d) => d, + }; + if ws_tx.send(Message::Binary(bytes.into())).await.is_err() { + break; + } + } + + // Report the exit code and close. The session task returns the command's + // exit code (or a sentinel: -1 on internal error, 130 on disconnect). + let code = session.await.unwrap_or(-1); + let _ = ws_tx + .send(Message::Text( + format!("{{\"type\":\"exit\",\"code\":{code}}}").into(), + )) + .await; + let _ = ws_tx.send(Message::Close(None)).await; + input_pump.abort(); + })) +} + +/// Maximum number of concurrent log-follow SSE streams. +/// Each follower polls via `spawn_blocking` every 100ms, so capping concurrency +/// prevents blocking-pool saturation under high follower counts. +static LOG_FOLLOW_SEMAPHORE: std::sync::LazyLock = + std::sync::LazyLock::new(|| Semaphore::new(16)); + +/// Stream machine console logs via SSE. +#[utoipa::path( + get, + path = "/api/v1/machines/{id}/logs", + tag = "Logs", + params( + ("id" = String, Path, description = "Machine name"), + ("follow" = Option, Query, description = "Follow the logs (like tail -f)"), + ("tail" = Option, Query, description = "Number of lines to show from the end") + ), + responses( + (status = 200, description = "Log stream (SSE)", content_type = "text/event-stream"), + (status = 404, description = "Machine or log file not found", body = ApiErrorResponse) + ) +)] +pub async fn stream_logs( + State(state): State>, + Path(id): Path, + Query(query): Query, +) -> Result>>, ApiError> { + let entry = state.get_machine(&id)?; + + // Get console log path + let log_path: PathBuf = { + let entry = entry.lock(); + entry + .manager + .console_log() + .ok_or_else(|| ApiError::NotFound("console log not configured".into()))? + .to_path_buf() + }; + + // Check if file exists (blocking check is acceptable here since it's fast) + let path_check = log_path.clone(); + let exists = tokio::task::spawn_blocking(move || path_check.exists()) + .await + .map_err(ApiError::internal)?; + + if !exists { + return Err(ApiError::NotFound(format!( + "log file not found: {}", + log_path.display() + ))); + } + + let follow = query.follow; + let tail = query.tail; + let json_only = query.format.as_deref() == Some("json"); + + // Validate tail value upfront + const MAX_TAIL_LINES: usize = 10_000; + if let Some(n) = tail { + if n > MAX_TAIL_LINES { + return Err(ApiError::BadRequest(format!( + "tail value {} exceeds maximum of {}", + n, MAX_TAIL_LINES, + ))); + } + } + + // Acquire a follow permit if the client wants to follow. This limits + // concurrent long-lived polling streams to prevent blocking-pool saturation. + // The permit is moved into the stream so it's held for the stream's lifetime. + let follow_permit = if follow { + Some( + LOG_FOLLOW_SEMAPHORE + .try_acquire() + .map_err(|_| ApiError::Conflict("too many concurrent log followers".into()))?, + ) + } else { + None + }; + + // For tail, read last N lines upfront using spawn_blocking with bounded memory + let (initial_lines, start_pos) = if let Some(n) = tail { + let path = log_path.clone(); + tokio::task::spawn_blocking(move || read_last_n_lines_bounded(&path, n)) + .await + .map_err(ApiError::internal)? + .map_err(ApiError::internal)? + } else { + (Vec::new(), 0) + }; + + // Create the SSE stream + let stream = async_stream::stream! { + // Hold the follow permit for the stream's lifetime so it's released + // when the client disconnects or the stream ends. + let _permit = follow_permit; + + // Emit initial tail lines first + for line in initial_lines { + if json_only && serde_json::from_str::(&line).is_err() { + continue; // skip non-JSON lines in json mode + } + yield Ok(Event::default().data(line)); + } + + if tail.is_some() && !follow { + return; + } + + // For following or full read, poll the file for new content + let mut pos = if tail.is_some() { start_pos } else { 0 }; + let mut partial_line = String::new(); + + loop { + // Read new content in spawn_blocking + let path = log_path.clone(); + let current_pos = pos; + + let result = tokio::task::spawn_blocking(move || { + read_from_position(&path, current_pos) + }) + .await + .unwrap_or_else(|e| Err(std::io::Error::other(e))); + + match result { + Ok((new_data, new_pos)) => { + pos = new_pos; + if !new_data.is_empty() { + partial_line.push_str(&new_data); + // Yield complete lines + while let Some(newline_pos) = partial_line.find('\n') { + let line = partial_line[..newline_pos].trim_end_matches('\r').to_string(); + partial_line = partial_line[newline_pos + 1..].to_string(); + if json_only && serde_json::from_str::(&line).is_err() { + continue; // skip non-JSON lines in json mode + } + yield Ok(Event::default().data(line)); + } + // Flush partial line if it exceeds the safety cap + if partial_line.len() > MAX_PARTIAL_LINE { + yield Ok(Event::default().data(partial_line.clone())); + partial_line.clear(); + } + } + } + Err(e) => { + yield Ok(Event::default().data(format!("error: {}", e))); + break; + } + } + + if !follow { + // Yield any remaining partial line + if !partial_line.is_empty() { + yield Ok(Event::default().data(partial_line.clone())); + } + break; + } + + // Wait before polling again + tokio::time::sleep(Duration::from_millis(100)).await; + } + }; + + Ok(Sse::new(stream).keep_alive(KeepAlive::default())) +} + +/// Read the last N lines from a file using a bounded ring buffer. +/// Returns (lines, file_position_at_end) for follow mode. +fn read_last_n_lines_bounded( + path: &std::path::Path, + n: usize, +) -> std::io::Result<(Vec, u64)> { + use std::collections::VecDeque; + + let file = std::fs::File::open(path)?; + let metadata = file.metadata()?; + let file_len = metadata.len(); + + // n == 0 means "no tail lines" — skip reading the file entirely + if n == 0 { + return Ok((Vec::new(), file_len)); + } + + let reader = BufReader::new(file); + + // Use a ring buffer to keep only the last N lines in memory + let mut ring: VecDeque = VecDeque::with_capacity(n + 1); + + for line in reader.lines() { + let line = line?; + if ring.len() == n { + ring.pop_front(); + } + ring.push_back(line); + } + + Ok((ring.into_iter().collect(), file_len)) +} + +/// Maximum bytes to read per poll cycle (64 KiB). +/// Bounds memory usage per follower and prevents a single large write from +/// blocking the async runtime. +const MAX_READ_CHUNK: u64 = 64 * 1024; + +/// Maximum size of the partial (incomplete) line buffer (1 MiB). +/// If a log produces data without newlines beyond this limit, the partial +/// buffer is flushed as-is to prevent unbounded memory growth. +const MAX_PARTIAL_LINE: usize = BYTES_PER_MIB as usize; + +/// Read new content from a file starting at a given position. +/// Reads at most `MAX_READ_CHUNK` bytes per call. +fn read_from_position(path: &std::path::Path, pos: u64) -> std::io::Result<(String, u64)> { + use std::io::Read as _; + + let mut file = std::fs::File::open(path)?; + let metadata = file.metadata()?; + let file_len = metadata.len(); + + if pos >= file_len { + // No new content + return Ok((String::new(), pos)); + } + + file.seek(SeekFrom::Start(pos))?; + let to_read = std::cmp::min(file_len - pos, MAX_READ_CHUNK) as usize; + let mut buf = vec![0u8; to_read]; + file.read_exact(&mut buf)?; + let new_pos = pos + to_read as u64; + + let text = String::from_utf8_lossy(&buf).into_owned(); + Ok((text, new_pos)) +} diff --git a/src/api/handlers/files.rs b/src/api/handlers/files.rs new file mode 100644 index 0000000..528796b --- /dev/null +++ b/src/api/handlers/files.rs @@ -0,0 +1,145 @@ +//! File I/O handlers — upload and download files to/from a running machine. + +use axum::{ + body::Bytes, + extract::{Path, State}, + Json, +}; +use serde::Serialize; +use std::sync::Arc; +use utoipa::ToSchema; + +use crate::api::error::{classify_ensure_running_error, ApiError}; +use crate::api::state::{ensure_running_and_persist, with_machine_client_traced, ApiState}; +use crate::api::TraceId; + +/// Response from file upload. +#[derive(Debug, Serialize, ToSchema)] +pub struct FileUploadResponse { + /// Path where the file was written. + pub path: String, + /// Size of the file in bytes. + pub size: u64, +} + +/// Upload a file to a machine. +/// +/// Writes the request body as a file at the specified path inside the VM. +/// Creates parent directories automatically. +#[utoipa::path( + put, + path = "/api/v1/machines/{id}/files/{path}", + tag = "Files", + params( + ("id" = String, Path, description = "Machine name"), + ("path" = String, Path, description = "File path inside the VM (e.g., workspace/script.py)") + ), + request_body(content = Vec, content_type = "application/octet-stream"), + responses( + (status = 200, description = "File uploaded", body = FileUploadResponse), + (status = 404, description = "Machine not found"), + (status = 500, description = "Write failed") + ) +)] +pub async fn upload_file( + State(state): State>, + Path((id, file_path)): Path<(String, String)>, + trace_id: Option>, + body: Bytes, +) -> Result, ApiError> { + let tid = trace_id.map(|t| t.0 .0.clone()); + let entry = state.get_machine(&id)?; + ensure_running_and_persist(&state, &id, &entry) + .await + .map_err(classify_ensure_running_error)?; + + let machine_image = state.lookup_vm(&id).await?.and_then(|r| r.image); + + let file_path = file_path.trim_start_matches('/'); + let guest_path = format!("/{}", file_path); + let size = body.len() as u64; + + let overlay_id = id.clone(); + with_machine_client_traced(&entry, tid, move |c| { + // For image machines, mount the per-machine persistent container overlay + // (same id exec uses) so the file lands INSIDE the container, not the + // read-only agent base. Pull the image first if it isn't present yet. + if let Some(ref image) = machine_image { + if c.query(image)?.is_none() { + c.pull_with_registry_config(image)?; + } + // Activate the per-machine container overlay so the file op targets + // the image container, not the read-only agent base. `prepare_overlay` + // mounts but doesn't make it the active fs for write_file/read_file; + // a no-op container run (same path exec takes) does. + c.run_non_interactive( + crate::agent::RunConfig::new(image.clone(), vec!["/bin/true".to_string()]) + .with_persistent_overlay(Some(overlay_id.clone())), + )?; + } + c.write_file(&guest_path, &body, None) + }) + .await?; + + Ok(Json(FileUploadResponse { + path: format!("/{}", file_path), + size, + })) +} + +/// Download a file from a machine. +/// +/// Returns the file contents as a raw byte stream. +#[utoipa::path( + get, + path = "/api/v1/machines/{id}/files/{path}", + tag = "Files", + params( + ("id" = String, Path, description = "Machine name"), + ("path" = String, Path, description = "File path inside the VM") + ), + responses( + (status = 200, description = "File contents", content_type = "application/octet-stream"), + (status = 404, description = "Machine or file not found"), + (status = 500, description = "Read failed") + ) +)] +pub async fn download_file( + State(state): State>, + Path((id, file_path)): Path<(String, String)>, + trace_id: Option>, +) -> Result { + let tid = trace_id.map(|t| t.0 .0.clone()); + let entry = state.get_machine(&id)?; + ensure_running_and_persist(&state, &id, &entry) + .await + .map_err(classify_ensure_running_error)?; + + let machine_image = state.lookup_vm(&id).await?.and_then(|r| r.image); + + let file_path = file_path.trim_start_matches('/'); + let guest_path = format!("/{}", file_path); + + let overlay_id = id.clone(); + let data = with_machine_client_traced(&entry, tid, move |c| { + // Read from inside the container overlay for image machines (matching + // upload + exec), not the agent base. + if let Some(ref image) = machine_image { + if c.query(image)?.is_none() { + c.pull_with_registry_config(image)?; + } + // Activate the per-machine container overlay so the file op targets + // the image container, not the read-only agent base. `prepare_overlay` + // mounts but doesn't make it the active fs for write_file/read_file; + // a no-op container run (same path exec takes) does. + c.run_non_interactive( + crate::agent::RunConfig::new(image.clone(), vec!["/bin/true".to_string()]) + .with_persistent_overlay(Some(overlay_id.clone())), + )?; + } + c.read_file(&guest_path) + }) + .await?; + + Ok(Bytes::from(data)) +} diff --git a/src/api/handlers/health.rs b/src/api/handlers/health.rs new file mode 100644 index 0000000..65e45ca --- /dev/null +++ b/src/api/handlers/health.rs @@ -0,0 +1,93 @@ +//! Health check endpoint. + +use axum::http::StatusCode; +use axum::{extract::State, Json}; +use std::sync::Arc; +use std::time::Duration; + +use crate::api::state::ApiState; +use crate::api::types::{HealthResponse, MachineCountsResponse}; + +/// Deadline for the `/readyz` blocking-pool round-trip. Generous vs. a no-op task +/// on a healthy pool (microseconds); only a genuinely saturated pool misses it. +const READYZ_BUDGET: Duration = Duration::from_secs(2); + +/// Server start time for uptime calculation. +static SERVER_START: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Record the server start time. Call once at startup. +pub fn mark_server_start() { + let _ = SERVER_START.set(std::time::Instant::now()); +} + +/// Health check endpoint. +#[utoipa::path( + get, + path = "/health", + tag = "Health", + responses( + (status = 200, description = "Server is healthy", body = HealthResponse) + ) +)] +pub async fn health(State(state): State>) -> Json { + // Count from the authoritative DB (off-reactor), the same source `/machines` + // reads. The in-memory machine map retains entries for VMs removed + // out-of-band — an ephemeral cleanup or a CLI/other-process delete — which + // made `/health.machines.total` report a stale count that diverged from + // `/machines` and the DB (a monitoring/scheduling consumer then saw ghost + // machines). Falls back to `None` if the DB read fails. + let machines = state.list_vm_records().await.ok().map(|vms| { + let total = vms.len(); + let running = vms.iter().filter(|(_, r)| r.is_process_alive()).count(); + MachineCountsResponse { total, running } + }); + let uptime = SERVER_START.get().map(|t| t.elapsed().as_secs()); + + Json(HealthResponse { + status: "ok", + version: crate::VERSION, + machines, + uptime_seconds: uptime, + }) +} + +/// Readiness probe that exercises the **blocking pool** — the resource VM +/// start/exec/stop dispatch depends on. +/// +/// `/health` is pure-async and keeps returning 200 even when the blocking pool is +/// exhausted, so the control's data-plane probe couldn't distinguish a wedged node +/// (start/exec timing out) from a healthy one, and never auto-cordoned it (the +/// 2026-07-05 worker-1 wedge). This round-trips a no-op `spawn_blocking` task under +/// a deadline: on a healthy pool it returns 200 in microseconds; if the pool can't +/// service it in [`READYZ_BUDGET`] the node is dispatch-wedged and it returns 503, +/// which the control treats as a data-plane failure and cordons. +#[utoipa::path( + get, + path = "/readyz", + tag = "Health", + responses( + (status = 200, description = "Dispatch path (blocking pool) is responsive"), + (status = 503, description = "Blocking pool saturated — node is dispatch-wedged") + ) +)] +pub async fn readyz() -> StatusCode { + let probe = tokio::task::spawn_blocking(|| ()); + match tokio::time::timeout(READYZ_BUDGET, probe).await { + Ok(Ok(())) => StatusCode::OK, + // Timed out (pool saturated) or the task panicked/was cancelled — either way + // the dispatch path is not servicing work; report unready. + _ => StatusCode::SERVICE_UNAVAILABLE, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // On a healthy runtime the blocking-pool round-trip completes well inside the + // budget, so readiness reports OK. + #[tokio::test] + async fn readyz_ok_when_pool_responsive() { + assert_eq!(readyz().await, StatusCode::OK); + } +} diff --git a/src/api/handlers/images.rs b/src/api/handlers/images.rs new file mode 100644 index 0000000..a18166e --- /dev/null +++ b/src/api/handlers/images.rs @@ -0,0 +1,130 @@ +//! Image management handlers. + +use axum::{ + extract::{Path, State}, + Json, +}; +use std::sync::Arc; + +use crate::agent::PullOptions; +use crate::api::error::{classify_ensure_running_error, ApiError}; +use crate::api::state::{ensure_running_and_persist, with_machine_client_traced, ApiState}; +use crate::api::types::{ + ApiErrorResponse, ImageInfo, ListImagesResponse, PullImageRequest, PullImageResponse, +}; +use crate::api::TraceId; + +/// List images in a machine. +#[utoipa::path( + get, + path = "/api/v1/machines/{id}/images", + tag = "Images", + params( + ("id" = String, Path, description = "Machine name") + ), + responses( + (status = 200, description = "List of images", body = ListImagesResponse), + (status = 404, description = "Machine not found", body = ApiErrorResponse) + ) +)] +pub async fn list_images( + State(state): State>, + Path(machine_id): Path, + trace_id: Option>, +) -> Result, ApiError> { + let tid = trace_id.map(|t| t.0 .0.clone()); + let entry = state.get_machine(&machine_id)?; + + // Check if machine VM is actually alive, return empty list if not + { + let entry = entry.lock(); + if !entry.manager.is_process_alive() { + return Ok(Json(ListImagesResponse { images: Vec::new() })); + } + } + + let images = with_machine_client_traced(&entry, tid, |c| c.list_images()).await?; + + let images = images + .into_iter() + .map(|i| ImageInfo { + reference: i.reference, + digest: i.digest, + size: i.size, + architecture: i.architecture, + os: i.os, + layer_count: i.layer_count, + }) + .collect(); + + Ok(Json(ListImagesResponse { images })) +} + +/// Pull an image into a machine. +#[utoipa::path( + post, + path = "/api/v1/machines/{id}/images/pull", + tag = "Images", + params( + ("id" = String, Path, description = "Machine name") + ), + request_body = PullImageRequest, + responses( + (status = 200, description = "Image pulled", body = PullImageResponse), + (status = 400, description = "Invalid request", body = ApiErrorResponse), + (status = 404, description = "Machine not found", body = ApiErrorResponse), + (status = 500, description = "Failed to pull image", body = ApiErrorResponse) + ) +)] +pub async fn pull_image( + State(state): State>, + Path(machine_id): Path, + trace_id: Option>, + Json(req): Json, +) -> Result, ApiError> { + let tid = trace_id.map(|t| t.0 .0.clone()); + if req.image.is_empty() { + return Err(ApiError::BadRequest( + "image reference cannot be empty".into(), + )); + } + + let entry = state.get_machine(&machine_id)?; + + // Ensure machine is running and persist state to DB + ensure_running_and_persist(&state, &machine_id, &entry) + .await + .map_err(classify_ensure_running_error)?; + + let image = req.image.clone(); + let oci_platform = req.oci_platform.clone(); + let proxy = req.proxy.clone(); + let no_proxy = req.no_proxy.clone(); + let start = std::time::Instant::now(); + let image_info = with_machine_client_traced(&entry, tid, move |c| { + let mut opts = PullOptions::new().use_registry_config(true); + if let Some(p) = oci_platform { + opts = opts.oci_platform(p); + } + if let Some(p) = proxy { + opts = opts.proxy(p); + } + if let Some(np) = no_proxy { + opts = opts.no_proxy(np); + } + c.pull(&image, opts) + }) + .await?; + metrics::histogram!("smolvm_image_pull_seconds").record(start.elapsed().as_secs_f64()); + + Ok(Json(PullImageResponse { + image: ImageInfo { + reference: image_info.reference, + digest: image_info.digest, + size: image_info.size, + architecture: image_info.architecture, + os: image_info.os, + layer_count: image_info.layer_count, + }, + })) +} diff --git a/src/api/handlers/machines.rs b/src/api/handlers/machines.rs new file mode 100644 index 0000000..6742153 --- /dev/null +++ b/src/api/handlers/machines.rs @@ -0,0 +1,2120 @@ +//! Machine lifecycle handlers. +//! +//! These handlers manage persistent machines via the shared database, +//! accessible to both API and CLI commands. +//! +//! ## Limitations +//! +//! ### Name Length Limit +//! +//! Machine name length is bounded by the kernel's `sockaddr_un.sun_path` +//! limit (104 bytes on macOS, 108 on Linux). The full socket path is: +//! +//! ```text +//! ~/Library/Caches/smolvm/vms/{name}/agent.sock +//! ``` +//! +//! Maximum usable name length therefore depends on the user's home directory. +//! For a typical macOS home (`/Users//`, ~20 chars), names can be +//! 50+ characters. The actual socket path is validated at create time via +//! [`crate::data::validate_socket_path_fits`] so overly-long names are +//! rejected with a clear error up front. +//! +//! Recommended: keep names short and descriptive (e.g., "dev-vm", "test-1"). + +use axum::{ + extract::{Path, Query, State}, + Json, +}; +use std::sync::Arc; +use std::time::Duration; + +use crate::agent::{vm_data_dir, AgentClient, AgentManager, HostMount}; +use crate::api::error::ApiError; +use crate::api::state::{ + vm_resources_to_spec, with_machine_client_traced, ApiState, MachineEntry, MachineRegistration, + ReservationGuard, +}; +use crate::api::types::{ + ApiErrorResponse, CreateMachineRequest, DeleteResponse, EnvVar, ExecResponse, ExportRequest, + ExportResponse, ForkRequest, ListMachinesResponse, MachineExecRequest, MachineInfo, MountInfo, + MountSpec, PortSpec, ResizeMachineRequest, ResourceSpec, StartMachineQuery, +}; +use crate::api::validate_command; +use crate::api::TraceId; +use crate::config::{RecordState, RestartConfig, VmRecord}; +use crate::data::disk::{Overlay, Storage}; +use crate::data::validate_vm_name; +use crate::process::{ + is_alive, is_our_process_strict, process_start_time, stop_vm_process, VM_SIGKILL_TIMEOUT, + VM_SIGTERM_TIMEOUT, +}; +use crate::storage::{expand_disk, DEFAULT_OVERLAY_SIZE_GIB, DEFAULT_STORAGE_SIZE_GIB}; +use crate::util::generate_machine_name; +use crate::Error as SmolvmError; + +/// Re-export of the shared resolver. The CLI and API list endpoints +/// must compute state the same way, otherwise `machine list` (CLI) +/// and `GET /api/v1/machines` (API) can disagree about whether a VM +/// is `Running`, `Stopped`, or `Unreachable`. Single source of truth +/// lives in `agent::state_probe`. +use crate::agent::state_probe::resolve_state as resolve_machine_state; + +/// Convert VmRecord to MachineInfo (pure mapping, no I/O). +fn record_to_info(name: &str, record: &VmRecord) -> MachineInfo { + let actual_state = resolve_machine_state(name, record); + // Clear stale PID when the process is not actually running, so clients + // never see state=stopped paired with a PID. + let pid = if actual_state == RecordState::Stopped { + None + } else { + record.pid + }; + MachineInfo { + name: name.to_string(), + state: actual_state.to_string(), + cpus: record.cpus, + mem: record.mem, + pid, + mounts: record + .mounts + .iter() + .enumerate() + .map(|(i, (source, target, readonly))| MountInfo { + tag: HostMount::mount_tag(i), + source: source.clone(), + target: target.clone(), + readonly: *readonly, + }) + .collect(), + ports: record + .ports + .iter() + .map(|(host, guest)| PortSpec { + host: *host, + guest: *guest, + }) + .collect(), + network: record.network, + network_backend: record.network_backend, + allowed_cidrs: record.allowed_cidrs.clone(), + allowed_hosts: record.dns_filter_hosts.clone(), + // Report the RESOLVED provisioned disk sizes, not the request echo: a + // machine created without an explicit size still gets a real disk at the + // node default, and billing/telemetry need the actual allocated GiB, not + // `None`. `open_or_create` provisions every VM a storage disk at + // `DEFAULT_STORAGE_SIZE_GIB` (and an overlay at `DEFAULT_OVERLAY_SIZE_GIB`) + // when unset. + storage_gb: Some(record.storage_gb.unwrap_or(DEFAULT_STORAGE_SIZE_GIB)), + overlay_gb: Some(record.overlay_gb.unwrap_or(DEFAULT_OVERLAY_SIZE_GIB)), + // Cumulative egress, read from the per-VM telemetry file the subprocess + // flushes. Surfaced here so the control plane reads it from the machine + // list exactly like disk size — no bespoke endpoint. + egress_bytes: crate::agent::read_egress_telemetry(name), + // Live consumed CPU-seconds for the VMM child, sampled from the host + // (user+system CPU time). Resets on restart — the control plane treats it + // as a monotonic-with-resets counter and accumulates the durable total. + // `None` when stopped (pid cleared) or the process vanished mid-sample. + cpu_seconds: pid + .and_then(crate::process::process_stats) + .map(|s| s.cpu_time_ns / 1_000_000_000), + // Same consumed CPU in milliseconds — sub-second precision so consumers + // don't quantize a barely-busy process up to a whole second. + cpu_millis: pid + .and_then(crate::process::process_stats) + .map(|s| s.cpu_time_ns / 1_000_000), + // Current RSS (MiB) of the VMM process — an instantaneous gauge the + // control plane integrates over time for active-memory billing. + rss_mb: pid + .and_then(crate::process::process_stats) + .map(|s| s.rss_bytes / (1024 * 1024)), + // Actual used disk (sparse-image blocks) — a gauge for active-disk billing, + // measured from the data dir regardless of whether the VMM is running. + disk_used_mb: crate::agent::disk_used_mb(name), + created_at: record.created_at, + } +} + +/// Build a MachineEntry from a VmRecord and AgentManager. +/// +/// Used by `start_machine` to register a machine in ApiState after boot +/// or during registry repair. Centralizes the record→entry conversion +/// so the two branches don't drift. +fn machine_entry_from_record(record: &VmRecord, manager: AgentManager) -> MachineEntry { + let mounts = record + .mounts + .iter() + .map(|(s, t, ro)| MountSpec { + source: s.clone(), + target: t.clone(), + readonly: *ro, + }) + .collect(); + let ports = record + .ports + .iter() + .map(|(h, g)| PortSpec { + host: *h, + guest: *g, + }) + .collect(); + MachineEntry { + manager, + mounts, + ports, + resources: ResourceSpec { + // VmResources carries no hostname allow-list, so graft it back from the + // record — otherwise a reloaded machine would silently lose allowed_hosts. + allowed_hosts: record.dns_filter_hosts.clone(), + ..vm_resources_to_spec(record.vm_resources()) + }, + restart: record.restart.clone(), + network: record.network, + secret_refs: record.secret_refs.clone(), + source_smolmachine: record.source_smolmachine.clone(), + } +} + +/// Attempt graceful shutdown, then force-terminate if still running. +/// +/// Uses verified signals to prevent killing an unrelated process if the +/// PID was recycled by the OS. Returns true if the process is confirmed +/// dead (or was never running), false if it may still be alive. +/// `graceful`: when true (stop), give the guest a SIGTERM grace period to flush +/// to its persistent overlay before SIGKILL. When false (delete), the machine's +/// disks are discarded immediately after, so there is nothing to flush — SIGKILL +/// at once instead of waiting out the guest's graceful shutdown (the bulk of the +/// ~1.9s DELETE latency on metal). +fn shutdown_machine_process( + name: &str, + pid: Option, + pid_start_time: Option, + graceful: bool, +) -> bool { + // Try graceful shutdown via vsock first. + // If vsock connects, this confirms the process is our VM (identity verification). + let manager = AgentManager::for_vm(name).ok(); + let mut vsock_confirmed = false; + if let Some(ref manager) = manager { + if let Ok(mut client) = AgentClient::connect(manager.vsock_socket()) { + vsock_confirmed = true; + let _ = client.shutdown(); + } + } + + // PID-based signal handling. + if let Some(pid) = pid { + // Identity check: vsock acknowledgement OR strict PID start-time match. + // We intentionally do NOT use the lenient is_our_process() here because + // it treats any alive PID as "ours" when start_time is None — which risks + // killing an unrelated process if the OS reused the PID. + let identity_ok = vsock_confirmed || is_our_process_strict(pid, pid_start_time); + + if identity_ok { + // On delete the disks are removed right after, so skip the SIGTERM + // grace and SIGKILL immediately (ZERO grace). On stop keep the grace + // so the guest can flush to its persistent overlay first. + let sigterm = if graceful { + VM_SIGTERM_TIMEOUT + } else { + Duration::ZERO + }; + let _ = stop_vm_process(pid, sigterm, VM_SIGKILL_TIMEOUT); + } else { + tracing::debug!(pid, name, "PID already dead"); + } + + // Post-check: verify the process is actually gone. If it outlived the + // pid-targeted SIGKILL (or the recorded pid is wrong), fall back to + // killing the systemd transient scope — its cgroup owns every process the + // VM spawned — then wait briefly for the SIGKILL to land. Only give up if + // STILL alive. + if is_alive(pid) { + let _ = crate::systemd_scope::kill_scope(name); + for _ in 0..10 { + if !is_alive(pid) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(300)); + } + if is_alive(pid) { + tracing::warn!(pid, name, "process still alive after shutdown + scope kill"); + return false; + } + } + } else { + // No recorded pid — the pid-based kill can't run at all, which is exactly + // how a stuck/crash-looping VM becomes an un-deletable orphan (delete 500s + // "still alive; not removing" while the node keeps running the VM). Kill + // the transient scope's cgroup directly and confirm via vsock that the VM + // is actually gone. + let _ = crate::systemd_scope::kill_scope(name); + for _ in 0..10 { + let reachable = manager + .as_ref() + .and_then(|m| AgentClient::connect(m.vsock_socket()).ok()) + .map(|mut c| c.ping().is_ok()) + .unwrap_or(false); + if !reachable { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(300)); + } + tracing::warn!( + name, + "VM still reachable via vsock after scope kill; no PID to signal" + ); + return false; + } + + true +} + +/// Disks to restore for a VM-mode (`--from-vm`) pack. Unlike an image pack (OCI +/// layers), a VM-mode `.smolmachine` carries the source VM's overlay + storage +/// DISKS — the actual rootfs (`/bin/sh`, files written before packing). They must +/// be seeded onto the new machine's disks or it boots with only the bare +/// agent-rootfs. `pack run` does this; the API create path must too. +struct VmModeSeed { + overlay_template: Option, + storage_template: Option, + /// Original (pre-truncation) virtual size of the overlay disk. The packed + /// template has its trailing zero extent stripped, so the disk must be + /// ftruncated back to this before boot or it isn't a valid full filesystem. + overlay_logical_size: Option, + /// Requested disk sizes (GiB) from the create request, honored as a lower + /// bound on the seeded disks (the guest grows the inherited fs with resize2fs). + storage_gb: Option, + overlay_gb: Option, +} + +/// Create a new machine. +#[utoipa::path( + post, + path = "/api/v1/machines", + tag = "Machines", + request_body = CreateMachineRequest, + responses( + (status = 200, description = "Machine created", body = MachineInfo), + (status = 400, description = "Invalid request", body = ApiErrorResponse), + (status = 409, description = "Machine already exists", body = ApiErrorResponse) + ) +)] +pub async fn create_machine( + State(state): State>, + Json(req): Json, +) -> Result, ApiError> { + // Validate: registry_ref, from, and image are mutually exclusive + let source_count = [ + req.registry_ref.is_some(), + req.from.is_some(), + req.image.is_some(), + ] + .iter() + .filter(|&&b| b) + .count(); + if source_count > 1 { + return Err(ApiError::BadRequest( + "'registryRef', 'from', and 'image' are mutually exclusive".to_string(), + )); + } + + // Published ports need the inbound path that only virtio-net has. With an + // UNSET backend the launcher auto-selects virtio-net when ports are present + // (see `plan_launch_network`), so ports "just work" without per-request + // wiring — mirroring the CLI and `validate_requested_network_backend`. Only + // an EXPLICIT TSI choice alongside ports is a misconfig (TSI is + // outbound-only and would silently never accept connections). + if !req.ports.is_empty() && req.network_backend == Some(crate::network::NetworkBackend::Tsi) { + return Err(ApiError::BadRequest( + "published ports require networkBackend 'virtio-net' (TSI is outbound-only); \ + omit networkBackend or set it to 'virtio-net'" + .to_string(), + )); + } + + // If registry_ref is set, pull the artifact from the registry and treat as `from` + let mut req = req; + if let Some(ref registry_ref) = req.registry_ref.clone() { + let pulled_path = pull_from_registry( + registry_ref, + req.registry_identity_token.as_deref(), + &req.blob_peers, + ) + .await?; + req.from = Some(pulled_path); + req.registry_ref = None; + } + + // Generate name if not provided, then validate. The on-disk layout uses + // a hash-derived directory (see `vm_data_dir`) so name length doesn't + // affect the socket path — only character sanity + a generous length + // cap are needed. + let name = req.name.clone().unwrap_or_else(generate_machine_name); + validate_vm_name(&name, "machine name").map_err(ApiError::BadRequest)?; + + // Validate mount paths + for mount_spec in &req.mounts { + HostMount::try_from(mount_spec).map_err(|e| ApiError::BadRequest(e.to_string()))?; + } + + // If --from is set, read manifest and extract sidecar + let ( + image, + source_smolmachine, + entrypoint, + cmd, + env, + workdir, + manifest_cpus, + manifest_mem, + manifest_net, + manifest_secret_refs, + vm_seed, + ) = if let Some(ref sidecar_path) = req.from { + let path = std::path::Path::new(sidecar_path); + if !path.exists() { + return Err(ApiError::BadRequest(format!( + "sidecar file not found: {}", + sidecar_path + ))); + } + let manifest = smolvm_pack::packer::read_manifest_from_sidecar(path) + .map_err(|e| ApiError::internal(format!("read .smolmachine: {}", e)))?; + // Reject a cross-architecture artifact up front (400, not a mid-boot 500): + // a packed VM/image carries native binaries that cannot run under a + // different-arch guest kernel. Guest arch must match; host OS need not. + crate::platform::ensure_artifact_arch_matches_host(&manifest.platform) + .map_err(|e| ApiError::BadRequest(e.to_string()))?; + // Extraction happens after the agent manager creates this machine's data + // dir (below), so the layers land in the machine's own dir, not here. + let canonical = path + .canonicalize() + .unwrap_or_else(|_| path.to_path_buf()) + .to_string_lossy() + .into_owned(); + let env_parsed: Vec<(String, String)> = manifest + .env + .iter() + .filter_map(|e| { + e.split_once('=') + .map(|(k, v)| (k.to_string(), v.to_string())) + }) + .collect(); + // A .smolmachine is an untrusted, portable artifact: validate its secret + // refs Untrusted, which rejects every source kind, so a packed + // from_env/from_file can't read this host's env/files at exec time. + // Reject rather than carry/exfil. + for (key, r) in &manifest.secret_refs { + crate::secrets::validate_ref(r, crate::secrets::ResolutionScope::Untrusted).map_err( + |e| { + ApiError::BadRequest(format!( + "packed secret '{}': {} (packs may not carry secret refs)", + key, e + )) + }, + )?; + } + // VM-mode packs carry disks, not layers — capture the templates so the + // machine's overlay/storage disks can be seeded from them below. + let vm_seed = if manifest.mode == smolvm_pack::format::PackMode::Vm { + Some(VmModeSeed { + overlay_template: manifest + .assets + .overlay_template + .as_ref() + .map(|t| t.path.clone()), + storage_template: manifest + .assets + .storage_template + .as_ref() + .map(|t| t.path.clone()), + overlay_logical_size: manifest.assets.overlay_logical_size, + storage_gb: req.storage_gb, + overlay_gb: req.overlay_gb, + }) + } else { + None + }; + // A VM-mode pack is NOT a container/image machine: its `image` is the + // synthetic `vm://` label, not a pullable ref. `record.image.is_some()` + // is the universal "container machine" signal (exec routing, workload + // launch, pull-on-start, re-pack), so storing the vm:// label would make + // exec run `crun` over a nonexistent image instead of `vm_exec` in the VM + // (the /bin/sh-not-found bug). Store None so every consumer treats it as a + // VM; provenance lives in `source_smolmachine`. + let image = if vm_seed.is_some() { + None + } else { + Some(manifest.image) + }; + ( + image, + Some(canonical), + manifest.entrypoint, + manifest.cmd, + env_parsed, + manifest.workdir, + manifest.cpus, + manifest.mem, + manifest.network, + manifest.secret_refs, + vm_seed, + ) + } else { + ( + req.image.clone(), + None, + vec![], + vec![], + vec![], + None, + crate::data::resources::DEFAULT_MICROVM_CPU_COUNT, + crate::data::resources::DEFAULT_MICROVM_MEMORY_MIB, + req.network, + Default::default(), + None, + ) + }; + + // Use explicit API resources when provided. Otherwise, preserve packed + // artifact manifest defaults, or the high VM defaults for non-artifact + // machines. Memory is ballooned, so a generous default does not imply + // immediate host commitment. + let (cpus, mem) = resolve_create_resources(&req, manifest_cpus, manifest_mem); + let network = req.network || manifest_net; + + // Reserve the name atomically (prevents concurrent creation) + let guard = ReservationGuard::new(&state, name.clone())?; + + // Create manager (does not boot the VM) + let manager = tokio::task::spawn_blocking({ + let name = name.clone(); + let storage_gb = req.storage_gb; + let overlay_gb = req.overlay_gb; + move || { + AgentManager::for_vm_with_sizes(&name, storage_gb, overlay_gb) + .map_err(|e| ApiError::internal(format!("failed to create agent manager: {}", e))) + } + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))??; + + // Extract the bundle's OCI layers into this machine's own data dir (created + // by the manager above) rather than the shared pack cache, so every start is + // independent of the .smolmachine file surviving and the macOS layers volume + // is owned 1:1 by the machine. Extraction mounts the case-sensitive volume on + // macOS; detach it immediately so a created-but-unstarted machine leaves + // nothing mounted (invariant: the per-machine layers volume is mounted iff + // the VM is running). The name was reserved above, so this never clobbers + // another machine's layers. + if let Some(ref sidecar_path) = source_smolmachine { + let name = name.clone(); + let sidecar_path = sidecar_path.clone(); + tokio::task::spawn_blocking(move || -> Result<(), ApiError> { + let path = std::path::Path::new(&sidecar_path); + let cache_dir = crate::agent::machine_layers_cache_dir(&name); + let result = (|| { + let footer = smolvm_pack::packer::read_footer_from_sidecar(path) + .map_err(|e| ApiError::internal(format!("read sidecar footer: {}", e)))?; + if smolvm_pack::extract::shared_extract_enabled() { + // Shared content-addressed store: extract the build-constant + // pack ONCE per node into `_shared/` (root-owned, + // read-only) instead of a private per-machine copy, and drop a + // pointer beside this machine. The per-machine `pack` dir is + // left an empty mountpoint that the boot path idmap-binds the + // shared copy onto (mapping on-disk uid 0 -> the VM's dropped + // uid), so a 28.6 MB / 362-file agent-rootfs decodes once per + // node rather than once per machine — the cold-start tax this + // removes — with the per-VM uid isolation (#456) preserved. + let shared_root = crate::agent::shared_pack_cache_root(); + let shared_dir = smolvm_pack::extract::extract_sidecar_shared( + path, + &shared_root, + &footer, + false, + ) + .map_err(|e| ApiError::internal(format!("extract sidecar (shared): {}", e)))?; + std::fs::create_dir_all(&cache_dir).map_err(|e| { + ApiError::internal(format!("create pack mountpoint: {}", e)) + })?; + let pointer = crate::agent::shared_pack_pointer_path(&cache_dir); + std::fs::write(&pointer, shared_dir.to_string_lossy().as_bytes()).map_err( + |e| ApiError::internal(format!("write shared pack pointer: {}", e)), + )?; + Ok(()) + } else { + // Per-machine extraction: macOS case-sensitive layers volume + // (owned 1:1 by the machine), or the `SMOLVM_DISABLE_SHARED_EXTRACT` + // kill-switch. Wipe any prior cache first for a clean slate. + smolvm_pack::extract::force_detach_layers_volume(&cache_dir); + match std::fs::remove_dir_all(&cache_dir) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + return Err(ApiError::internal(format!( + "clear packed layers cache: {}", + e + ))); + } + } + smolvm_pack::extract::extract_sidecar(path, &cache_dir, &footer, false, false) + .map_err(|e| ApiError::internal(format!("extract sidecar: {}", e))) + } + })(); + // Detach the case-sensitive volume mounted during extraction so a + // created-but-unstarted machine leaves nothing mounted, and so the + // rollback below can remove the data dir cleanly (macOS; no-op on Linux). + smolvm_pack::extract::force_detach_layers_volume(&cache_dir); + if let Err(e) = result { + // Extraction failed after the manager created the machine's data + // dir. guard.complete() will not run, so no DB record persists and + // the name is released on drop — but the on-disk dir would be left + // orphaned. Roll it back so a retry starts clean. Best-effort: a + // remove failure only leaves the orphan, never a worse state. + // cache_dir is /pack, so its parent is the data dir. + if let Some(vm_dir) = cache_dir.parent() { + let _ = std::fs::remove_dir_all(vm_dir); + } + return Err(e); + } + Ok(()) + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))??; + } + + // VM-mode pack: seed this machine's overlay + storage disks from the packed + // templates (extracted above) so a start boots the source VM's rootfs rather + // than the bare agent-rootfs (the /bin/sh-missing bug). `open_or_create_at` + // reuses an existing disk, so seeding once at create persists across starts. + // Mirrors `pack_run`'s VM-mode disk restore (`setup_vm_overlay` + + // `create_or_copy_storage_disk`). + if let Some(seed) = vm_seed { + let name2 = name.clone(); + let disk_dir = manager + .storage_path() + .parent() + .map(std::path::Path::to_path_buf) + .unwrap_or_else(|| vm_data_dir(&name)); + let seed_result = tokio::task::spawn_blocking(move || -> Result<(), ApiError> { + let cache_dir = crate::agent::machine_layers_cache_dir(&name2); + // With the shared store, the pack contents live in `_shared/` + // (the per-machine `pack` dir is an empty mountpoint), so seed the + // VM-mode disk templates from the shared copy. Falls back to the + // per-machine dir when no pointer was written (macOS / kill-switch). + let pack_content_dir = + crate::agent::read_shared_pack_pointer(&cache_dir).unwrap_or(cache_dir); + crate::storage::seed_vm_mode_disks( + &disk_dir, + &pack_content_dir, + seed.overlay_template.as_deref(), + seed.storage_template.as_deref(), + seed.overlay_logical_size, + seed.overlay_gb, + seed.storage_gb, + ) + .map_err(|e| ApiError::internal(format!("seed VM-mode disks: {}", e))) + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))?; + // On failure roll back the data dir the manager created, so a retry starts + // clean (the reservation guard releases the name but leaves the dir). + if let Err(e) = seed_result { + let _ = std::fs::remove_dir_all(vm_data_dir(&name)); + return Err(e); + } + } + + let resources = ResourceSpec { + cpus: Some(cpus), + memory_mb: Some(mem), + network: Some(network), + gpu: Some(req.gpu), + storage_gb: req.storage_gb, + overlay_gb: req.overlay_gb, + allowed_cidrs: req.allowed_cidrs.clone(), + allowed_hosts: req.allowed_hosts.clone(), + network_backend: req.network_backend, + }; + + // Validate request-body secret refs before persisting. Untrusted + // scope rejects every source kind, so any non-empty `secrets` map on + // the API surface is refused regardless of server binding — secrets + // must be configured locally via the CLI. + crate::api::handlers::validate_request_secrets(&req.secrets)?; + + // Complete registration: persists to DB + registers in ApiState + let complete_result = guard.complete(MachineRegistration { + manager, + mounts: req.mounts.clone(), + ports: req.ports.clone(), + resources: resources.clone(), + restart: match req.restart { + Some(ref spec) => { + let policy = spec + .policy + .as_deref() + .unwrap_or("never") + .parse() + .map_err(|e: String| ApiError::BadRequest(e))?; + RestartConfig { + policy, + max_retries: spec.max_retries.unwrap_or(0), + ..Default::default() + } + } + None => RestartConfig::default(), + }, + network, + image, + source_smolmachine, + entrypoint, + cmd, + env, + workdir, + // Record secrets = packed refs from --from (validated Untrusted above) + // merged with request refs (validated Untrusted at ~line 333); request + // refs win on key collision. Both sources are store-only, so RecordReplay + // resolution at exec time stays safe. + secret_refs: { + let mut s = manifest_secret_refs; + s.extend(req.secrets.clone()); + s + }, + }); + if let Err(e) = complete_result { + let data_dir = vm_data_dir(&name); + smolvm_pack::extract::force_detach_layers_volume(&crate::agent::machine_layers_cache_dir( + &name, + )); + if let Err(remove_err) = std::fs::remove_dir_all(&data_dir) { + if remove_err.kind() != std::io::ErrorKind::NotFound { + tracing::warn!( + machine = %name, + dir = %data_dir.display(), + error = %remove_err, + "failed to remove machine data dir after create commit failure" + ); + } + } + return Err(e); + } + + // Fetch the persisted record for the response (off the reactor). + let record = state + .lookup_vm(&name) + .await? + .ok_or_else(|| ApiError::internal("machine disappeared after creation".to_string()))?; + + Ok(Json(record_to_info(&name, &record))) +} + +/// List all machines. +#[utoipa::path( + get, + path = "/api/v1/machines", + tag = "Machines", + responses( + (status = 200, description = "List of machines", body = ListMachinesResponse), + (status = 500, description = "Database error", body = ApiErrorResponse) + ) +)] +pub async fn list_machines( + State(state): State>, +) -> Result, ApiError> { + // Read off the reactor: an inline synchronous `list_vms()` here let a stalled + // write park the worker pool and wedge the liveness probes (this is the path + // the control plane polls every reconcile). See tests/reactor_wedge.rs. + let vms = state.list_vm_records().await?; + let machines: Vec = vms + .iter() + .map(|(name, record)| record_to_info(name, record)) + .collect(); + + Ok(Json(ListMachinesResponse { machines })) +} + +/// Get machine status. +#[utoipa::path( + get, + path = "/api/v1/machines/{name}", + tag = "Machines", + params( + ("name" = String, Path, description = "Machine name") + ), + responses( + (status = 200, description = "Machine details", body = MachineInfo), + (status = 404, description = "Machine not found", body = ApiErrorResponse) + ) +)] +pub async fn get_machine( + State(state): State>, + Path(name): Path, +) -> Result, ApiError> { + let record = state + .lookup_vm(&name) + .await? + .ok_or_else(|| ApiError::NotFound(format!("machine '{}' not found", name)))?; + + Ok(Json(record_to_info(&name, &record))) +} + +/// Classify a VM launch/boot failure. A published host-port bind conflict — the +/// virtio-net runtime couldn't bind `0.0.0.0:` because something +/// (typically an orphaned VMM) still holds it — is surfaced as `PortConflict` +/// (409 `PORT_IN_USE`), which the control plane recognizes and retries on a +/// freshly-allocated port. Everything else stays a 500. Matching is scoped to +/// the virtio-net path so an unrelated AddrInUse can't be mistaken for it. +fn classify_launch_error(e: String) -> ApiError { + let lc = e.to_ascii_lowercase(); + if lc.contains("address already in use") && lc.contains("virtio") { + ApiError::PortConflict(e) + } else { + ApiError::Internal(e) + } +} + +/// Start a machine. +#[utoipa::path( + post, + path = "/api/v1/machines/{name}/start", + tag = "Machines", + params( + ("name" = String, Path, description = "Machine name"), + ("forkable" = Option, Query, description = "Start as a fork base (memfd RAM + control socket)") + ), + responses( + (status = 200, description = "Machine started", body = MachineInfo), + (status = 404, description = "Machine not found", body = ApiErrorResponse), + (status = 409, description = "A published host port is already in use (PORT_IN_USE)", body = ApiErrorResponse), + (status = 500, description = "Failed to start", body = ApiErrorResponse) + ) +)] +pub async fn start_machine( + State(state): State>, + Path(name): Path, + Query(query): Query, +) -> Result, ApiError> { + // Hold the per-machine lifecycle lock across the whole start so a concurrent + // stop/delete cannot detach the macOS layers volume between our acquire+mount + // and the launch, nor launch a guest into the launcher's missing-dir error + // (review finding #3). Acquired before the DB read and resolve_state probe + // below so the "is it running?" decision and the launch happen under one held + // lock; it is the outermost lock (the entry mutex is taken later, inside the + // spawn_blocking). Linux: the guarded detach/mount are no-ops. + let lifecycle = state.lifecycle_lock(&name); + let _guard = lifecycle.lock().await; + + // Get VM record from database (off the reactor) + let record = state + .lookup_vm(&name) + .await? + .ok_or_else(|| ApiError::NotFound(format!("machine '{}' not found", name)))?; + + // Resolve via the shared probe (PID + vsock ping) so we don't + // mistake a zombie VMM (live PID, dead agent) for Running — the + // CLI's `start --name` handles this same case; the API must + // match or a REST caller ends up with "start succeeded" followed + // by every subsequent /exec failing. + // + // `resolve_state` does a short vsock ping, so run it on the + // blocking pool rather than in the async task. + let name_probe = name.clone(); + let record_probe = record.clone(); + let resolved = tokio::task::spawn_blocking(move || { + crate::agent::state_probe::resolve_state(&name_probe, &record_probe) + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))?; + + if resolved == RecordState::Running { + if !state.machine_exists(&name) { + // Running in DB but not in registry (startup recovery case). + let name_for_repair = name.clone(); + let storage_gb = record.storage_gb; + let overlay_gb = record.overlay_gb; + let manager = tokio::task::spawn_blocking(move || { + AgentManager::for_vm_with_sizes(&name_for_repair, storage_gb, overlay_gb) + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))? + .map_err(|e| { + ApiError::internal(format!( + "machine '{}' is running but registry repair failed: {}", + name, e + )) + })?; + + state.insert_machine(&name, machine_entry_from_record(&record, manager)); + } + return Ok(Json(record_to_info(&name, &record))); + } + + if resolved == RecordState::Unreachable { + // Zombie: verified-kill the VMM and clear the DB record + // before falling through to a clean fresh start. Any stale + // in-memory registry entry gets overwritten by the + // `insert_machine` call later in this handler. + let name_recover = name.clone(); + tokio::task::spawn_blocking(move || { + crate::agent::state_probe::recover_if_unreachable(&name_recover); + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))?; + } + + let mounts = record.host_mounts(); + let ports = record.port_mappings(); + let resources = record.vm_resources(); + + // Start agent VM in blocking task. + // Uses subprocess launch to avoid macOS fork-in-multithreaded-process issue. + let name_clone = name.clone(); + let storage_gb = record.storage_gb; + let overlay_gb = record.overlay_gb; + let source_smolmachine = record.source_smolmachine.clone(); + let dns_filter_hosts = record.dns_filter_hosts.clone(); + let forkable = query.forkable; + let (manager, pid) = tokio::task::spawn_blocking(move || { + let manager = AgentManager::for_vm_with_sizes(&name_clone, storage_gb, overlay_gb) + .map_err(|e| format!("failed to create agent manager: {}", e))?; + + // Wire pre-extracted layers if this machine was created from a .smolmachine. + let mut features = crate::api::state::build_launch_features( + Some(&name_clone), + source_smolmachine.as_deref(), + dns_filter_hosts, + ) + .map_err(|e| format!("failed to prepare packed layers: {}", e))?; + // Forkable start: memfd-back guest RAM and expose a control socket at the + // machine's known path so it can later be forked via the fork endpoint. + if forkable { + features.forkable = true; + features.control_socket = Some(crate::agent::fork::control_socket_path(&name_clone)); + } + let _ = manager + .ensure_running_via_subprocess(mounts, ports, resources, features) + .map_err(|e| format!("failed to start machine: {}", e))?; + + let pid = manager.child_pid(); + Ok::<_, String>((manager, pid)) + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))? + .map_err(classify_launch_error)?; + + // Register in ApiState so exec/run/container endpoints can find it + state.insert_machine(&name, machine_entry_from_record(&record, manager)); + + // Image machines: launch the image's workload (its ENTRYPOINT+CMD) as a + // detached container now that the VM is up — mirroring the CLI start path + // (`vm_common.rs`). Without this, an image machine started via the API boots + // only the bare agent VM and never runs its server, so a published port + // forwards to a guest socket nothing is listening on (connection reset → + // proxy 502). An empty command lets the agent resolve the image's own + // ENTRYPOINT+CMD. This runs once per fresh start: the handler returns early + // above when the machine is already Running, so the container is never + // double-launched. Best-effort: a launch failure leaves a reachable VM + // (Running, exec-able) rather than failing the start and stranding a retry + // on the early-return path where the workload would never get launched. + if let Some(image) = record.image.clone() { + let entry = state.get_machine(&name)?; + let mut command = record.entrypoint.clone(); + command.extend(record.cmd.clone()); + let mut env = record.env.clone(); + env.extend(crate::secrets::expose_into_env( + super::record_secret_refs_env(&entry)?, + )); + let workdir = record.workdir.clone(); + let user = record.user.clone(); + let mounts_config = { + let e = entry.lock(); + e.mounts + .iter() + .enumerate() + .map(|(i, m)| (HostMount::mount_tag(i), m.target.clone(), m.readonly)) + .collect::>() + }; + let overlay_id = name.clone(); + let launch = with_machine_client_traced(&entry, None, move |c| { + if c.query(&image)?.is_none() { + c.pull_with_registry_config(&image)?; + } + let config = crate::agent::RunConfig::new(image, command) + .with_env(env) + .with_workdir(workdir) + .with_user(user) + .with_mounts(mounts_config) + .with_persistent_overlay(Some(overlay_id)); + c.run_container_detached(config).map(|_| ()) + }) + .await; + if let Err(e) = launch { + tracing::warn!( + machine = %name, + error = ?e, + "failed to launch image workload after start; VM is up but its server is not running" + ); + } + } + + // Capture start time for PID verification + let pid_start_time = pid.and_then(process_start_time); + + // Persist state to database (off the reactor) + let record = state + .update_vm(&name, move |r| { + r.state = RecordState::Running; + r.pid = pid; + r.pid_start_time = pid_start_time; + }) + .await? + .ok_or_else(|| { + ApiError::NotFound(format!( + "machine '{}' disappeared from database during start", + name + )) + })?; + + // Build response directly with state=running. We just confirmed the VM + // is running (wait_for_ready passed), so we bypass actual_state() which + // may falsely report "stopped" on macOS due to setsid/session-leader + // PID visibility issues. + let mut info = record_to_info(&name, &record); + info.state = "running".to_string(); + info.pid = pid; + Ok(Json(info)) +} + +/// Classify a fork-preparation failure into the right HTTP status. The golden +/// missing is a 404; the golden not being forkable / not yet ready, or the clone +/// name already being taken, is a 409; a nested-fork request is a 400; anything +/// else is a 500. +fn classify_fork_error(e: SmolvmError) -> ApiError { + let msg = e.to_string(); + let lc = msg.to_ascii_lowercase(); + if lc.contains("nested fork") { + ApiError::BadRequest(msg) + } else if lc.contains("already exists") + || lc.contains("not running forkable") + || lc.contains("control socket not responding") + || lc.contains("not ready to fork") + { + // Clone name taken, or the golden isn't a ready fork base — both 409. + ApiError::Conflict(msg) + } else if lc.contains("not found") { + ApiError::NotFound(msg) + } else { + ApiError::Internal(msg) + } +} + +/// Fork a running, forkable golden machine into a new clone (copy-on-write +/// memory + disks). +#[utoipa::path( + post, + path = "/api/v1/machines/{name}/fork", + tag = "Machines", + params( + ("name" = String, Path, description = "Golden (source) machine name") + ), + request_body = ForkRequest, + responses( + (status = 200, description = "Clone forked and running", body = MachineInfo), + (status = 400, description = "Invalid request (e.g. nested fork)", body = ApiErrorResponse), + (status = 404, description = "Golden machine not found", body = ApiErrorResponse), + (status = 409, description = "Golden not forkable, or clone name already exists", body = ApiErrorResponse), + (status = 500, description = "Fork failed", body = ApiErrorResponse) + ) +)] +pub async fn fork_machine( + State(state): State>, + Path(golden): Path, + Json(req): Json, +) -> Result, ApiError> { + let clone = req.name.clone(); + let pinned_ports: Vec<(u16, u16)> = req.ports.iter().map(|p| (p.host, p.guest)).collect(); + + // Serialize lifecycle on the CLONE name so a concurrent start/stop/delete of + // the same clone can't race the fork's register + boot. The golden is only + // read + frozen via its control socket, which tolerates concurrent forks. + let lifecycle = state.lifecycle_lock(&clone); + let _guard = lifecycle.lock().await; + + // Phase 1: freeze + snapshot the golden, register the clone with CoW disks. + // This is unix-socket IO + disk work, so it runs on the blocking pool. Its + // failures carry precondition semantics (404/409/400), mapped distinctly + // from the boot failures below. + let prep = { + let db = state.db().clone(); + let golden_b = golden.clone(); + let clone_b = clone.clone(); + let ports = pinned_ports.clone(); + tokio::task::spawn_blocking(move || { + crate::agent::fork::prepare_fork( + &db, &golden_b, &clone_b, &ports, /* clone_forkable */ false, + ) + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))? + .map_err(classify_fork_error)? + }; + + // Phase 2: boot the clone from the golden's in-memory snapshot (warm — its + // processes are already running in the restored RAM, so unlike a cold start + // there is no image workload to launch), then rejuvenate its identity. + let clone_b = clone.clone(); + let db = state.db().clone(); + let (manager, pid, clone_record) = tokio::task::spawn_blocking(move || { + let record = prep.clone_record; + let mounts = record.host_mounts(); + let ports = record.port_mappings(); + let resources = record.vm_resources(); + + let manager = + AgentManager::for_vm_with_sizes(&clone_b, record.storage_gb, record.overlay_gb) + .map_err(|e| format!("failed to create agent manager: {}", e))?; + + let mut features = crate::api::state::build_launch_features( + Some(&clone_b), + record.source_smolmachine.as_deref(), + record.dns_filter_hosts.clone(), + ) + .map_err(|e| format!("failed to prepare packed layers: {}", e))?; + // Boot from the golden's snapshot instead of cold-booting. + features.snapshot_dir = Some(prep.snapshot_dir); + + if let Err(e) = manager.ensure_running_via_subprocess(mounts, ports, resources, features) { + // Boot failed: roll back the clone registration so a failed fork + // leaves nothing half-created. + let _ = db.remove_vm(&clone_b); + let _ = std::fs::remove_dir_all(vm_data_dir(&clone_b)); + return Err(format!("failed to boot clone: {}", e)); + } + + // Give the clone a fresh on-disk identity (hostname, machine-id, SSH + // host keys, RNG) so it does not carry the golden's per-machine secrets + // into a (possibly different) tenant. FAIL-CLOSED: if the reset can't be + // confirmed, tear the booted clone down and fail the fork rather than + // vend a clone that impersonates the golden. + crate::agent::fork::fail_closed_on_rejuvenation( + crate::agent::fork::rejuvenate_clone(&clone_b), + || { + manager.kill(); + manager.cleanup_data_dir(); + let _ = db.remove_vm(&clone_b); + }, + ) + .map_err(|e| format!("clone identity rejuvenation failed: {}", e))?; + + let pid = manager.child_pid(); + Ok::<_, String>((manager, pid, record)) + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))? + .map_err(classify_launch_error)?; + + // Register the clone so exec/run endpoints can reach it. + state.insert_machine(&clone, machine_entry_from_record(&clone_record, manager)); + + // Persist the running state. + let pid_start_time = pid.and_then(process_start_time); + let record = state + .update_vm(&clone, move |r| { + r.state = RecordState::Running; + r.pid = pid; + r.pid_start_time = pid_start_time; + }) + .await? + .ok_or_else(|| { + ApiError::NotFound(format!( + "clone '{}' disappeared from database during fork", + clone + )) + })?; + + let mut info = record_to_info(&clone, &record); + info.state = "running".to_string(); + info.pid = pid; + Ok(Json(info)) +} + +/// Stop a machine. +#[utoipa::path( + post, + path = "/api/v1/machines/{name}/stop", + tag = "Machines", + params( + ("name" = String, Path, description = "Machine name") + ), + responses( + (status = 200, description = "Machine stopped", body = MachineInfo), + (status = 404, description = "Machine not found", body = ApiErrorResponse), + (status = 500, description = "Failed to stop", body = ApiErrorResponse) + ) +)] +pub async fn stop_machine( + State(state): State>, + Path(name): Path, +) -> Result, ApiError> { + // Hold the per-machine lifecycle lock across the whole stop so the layers + // volume detach below cannot race a concurrent start's acquire+mount+launch + // (review finding #3). Acquired before the DB read and actual_state() probe + // so the liveness check and the detach act on the same held lock — without + // it, stop could decide "running" off a snapshot a concurrent start has + // already superseded, then detach a volume that start just mounted. Outermost + // lock; the entry mutex is not taken here. Linux: detach is a no-op. + let lifecycle = state.lifecycle_lock(&name); + let _guard = lifecycle.lock().await; + + // Get VM record from database (off the reactor) + let record = state + .lookup_vm(&name) + .await? + .ok_or_else(|| ApiError::NotFound(format!("machine '{}' not found", name)))?; + + // Check state + let actual_state = record.actual_state(); + if actual_state != RecordState::Running { + // Not running. If a prior start mounted the layers volume but the VM + // then failed to boot (or the server crashed while running), the volume + // could still be mounted — detach it so a stopped machine never holds a + // mount (invariant: the per-machine layers volume is mounted iff the VM + // is running). Safe: actual_state() probed liveness, so the process is + // confirmed dead and nothing is using the volume. macOS hdiutil detach; + // a no-op on Linux. + if record.source_smolmachine.is_some() { + let name_clone = name.clone(); + tokio::task::spawn_blocking(move || { + smolvm_pack::extract::force_detach_layers_volume( + &crate::agent::machine_layers_cache_dir(&name_clone), + ); + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))?; + } + return Ok(Json(record_to_info(&name, &record))); + } + + // Get PID and start time from database record - this is the source of truth + let pid = record.pid; + let pid_start_time = record.pid_start_time; + + // Stop VM — prefer using the registered manager (which holds the flock) + // over creating a throwaway one. This ensures the flock is released so + // a subsequent start can re-acquire it. + let entry = state.get_machine(&name).ok(); + let name_clone = name.clone(); + let stopped = tokio::task::spawn_blocking(move || { + let ok = if let Some(ref entry) = entry { + let e = entry.lock(); + match e.manager.stop() { + Ok(()) => true, + Err(err) => { + tracing::warn!(name = %name_clone, error = %err, "manager.stop() failed, falling back to process kill"); + shutdown_machine_process(&name_clone, pid, pid_start_time, true) + } + } + } else { + shutdown_machine_process(&name_clone, pid, pid_start_time, true) + }; + if ok { + // Process is gone — detach this machine's case-sensitive layers + // volume (macOS hdiutil mount; no-op on Linux). The volume lives + // under the machine's own data dir and is owned 1:1 by it, so the + // detach is unconditional and re-acquired on the next start. + smolvm_pack::extract::force_detach_layers_volume( + &crate::agent::machine_layers_cache_dir(&name_clone), + ); + } + ok + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))?; + + if !stopped { + return Err(ApiError::Internal(format!( + "machine '{}' process may still be running after stop attempt", + name + ))); + } + + // The VM process is confirmed dead, but the long-lived registry manager for + // this machine still holds the per-VM `vm.lock` flock in this serve process. + // Release it so a subsequent start can re-acquire the lock; otherwise start + // fails with "another process is already starting or running this VM". + if let Ok(entry) = state.get_machine(&name) { + entry.lock().manager.mark_stopped(); + } + + // Persist state to database and get updated record — only after confirmed stop + let record = state + .update_vm(&name, |r| { + r.state = RecordState::Stopped; + r.pid = None; + r.pid_start_time = None; + }) + .await? + .ok_or_else(|| { + ApiError::NotFound(format!( + "machine '{}' disappeared from database during stop", + name + )) + })?; + + Ok(Json(record_to_info(&name, &record))) +} + +/// `POST /drain` — explicit, control-initiated node drain (decommission). +/// +/// Once serve restarts are lossless (per-VM systemd scopes + detach), drain is no +/// longer a side-effect of process shutdown — it's a deliberate decommission step. +/// The control plane (autoscaler scale-in) calls this BEFORE terminating the host +/// so VMs flush cleanly. Control-only by construction: the serve listener is mTLS- +/// gated, and the loopback door is localhost. See docs/lossless-serve-restart.md. +pub async fn drain_node(State(state): State>) -> axum::http::StatusCode { + tracing::info!("drain requested via API (node decommission)"); + drain_machines(&state).await; + axum::http::StatusCode::OK +} + +/// Gracefully stop every running VM. Two callers: the opt-in shutdown path +/// (`SMOLVM_DRAIN_ON_SHUTDOWN`, legacy — being retired now that restart is +/// lossless) and the explicit `POST /drain` decommission endpoint ([`drain_node`]). +/// Draining stops VMs cleanly — flushing disk state and marking them stopped so +/// the control plane can reschedule. Best-effort, concurrent, and bounded so it +/// fits inside the host's termination grace period. +pub async fn drain_machines(state: &Arc) { + let running: Vec<(String, Option, Option)> = match state.list_vm_records().await { + Ok(vms) => vms + .into_iter() + .filter(|(_, r)| r.actual_state() == RecordState::Running && r.is_process_alive()) + .map(|(name, r)| (name, r.pid, r.pid_start_time)) + .collect(), + Err(e) => { + tracing::error!(error = ?e, "drain: failed to list machines"); + return; + } + }; + if running.is_empty() { + return; + } + tracing::info!( + count = running.len(), + "draining running machines before shutdown" + ); + + let mut handles = Vec::with_capacity(running.len()); + for (name, pid, pid_start_time) in running { + let state = state.clone(); + handles.push(tokio::spawn(async move { + let name_for_kill = name.clone(); + let entry = state.get_machine(&name).ok(); + let stopped = tokio::task::spawn_blocking(move || { + // Prefer the registered manager (holds the flock); fall back to a + // PID-verified signal — same path as the stop handler. + let via_manager = entry + .as_ref() + .map(|e| e.lock().manager.stop().is_ok()) + .unwrap_or(false); + via_manager || shutdown_machine_process(&name_for_kill, pid, pid_start_time, true) + }) + .await + .unwrap_or(false); + if let Ok(entry) = state.get_machine(&name) { + entry.lock().manager.mark_stopped(); + } + let _ = state + .update_vm(&name, |r| { + r.state = RecordState::Stopped; + r.pid = None; + r.pid_start_time = None; + }) + .await; + tracing::info!(machine = %name, stopped, "drain: machine stopped"); + })); + } + + let drain_all = async { + for h in handles { + let _ = h.await; + } + }; + if tokio::time::timeout(std::time::Duration::from_secs(25), drain_all) + .await + .is_err() + { + tracing::warn!("drain: deadline reached before all machines stopped"); + } +} + +/// Delete a machine. +#[utoipa::path( + delete, + path = "/api/v1/machines/{name}", + tag = "Machines", + params( + ("name" = String, Path, description = "Machine name") + ), + responses( + (status = 200, description = "Machine deleted", body = DeleteResponse), + (status = 404, description = "Machine not found", body = ApiErrorResponse), + (status = 500, description = "Failed to delete", body = ApiErrorResponse) + ) +)] +pub async fn delete_machine( + State(state): State>, + Path(name): Path, +) -> Result, ApiError> { + // Hold the per-machine lifecycle lock across the whole delete so the layers + // volume detach (before the data-dir removal) cannot race a concurrent + // start's acquire+mount+launch (review finding #3). Acquired before the DB + // read so the existence check, shutdown, detach, and removal all happen under + // one held lock. Outermost lock; the entry mutex is not taken here. Linux: + // detach is a no-op. + let lifecycle = state.lifecycle_lock(&name); + let _guard = lifecycle.lock().await; + + // Check if VM exists and get its state (off the reactor) + let record = state + .lookup_vm(&name) + .await? + .ok_or_else(|| ApiError::NotFound(format!("machine '{}' not found", name)))?; + + // Get PID and start time from database record + let pid = record.pid; + let pid_start_time = record.pid_start_time; + + // Stop if running (in blocking task) + let name_clone = name.clone(); + let stopped = tokio::task::spawn_blocking(move || { + let ok = shutdown_machine_process(&name_clone, pid, pid_start_time, false); + if ok { + // Process is gone — detach this machine's case-sensitive layers + // volume (macOS hdiutil mount; no-op on Linux) before the data dir is + // removed below, otherwise `rm -rf` fails with "Resource busy". The + // volume is owned 1:1 by this machine, so the detach is unconditional. + smolvm_pack::extract::force_detach_layers_volume( + &crate::agent::machine_layers_cache_dir(&name_clone), + ); + } + ok + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))?; + + if !stopped { + return Err(ApiError::Internal(format!( + "machine '{}' process (pid {}) is still alive after shutdown; not removing", + name, + pid.map(|p| p.to_string()) + .unwrap_or_else(|| "unknown".into()), + ))); + } + + // Remove from registry (in-memory + database) in a blocking task: the DB + // delete is synchronous disk I/O and must not run on an async worker thread, + // where it would starve the small per-node reactor under delete churn. + let state_rm = state.clone(); + let name_rm = name.clone(); + tokio::task::spawn_blocking(move || -> Result<(), ApiError> { + match state_rm.remove_machine(&name_rm) { + Ok(_) => Ok(()), + Err(ApiError::NotFound(_)) => { + // Machine exists in DB but not in registry (startup recovery case). + // Remove directly from DB. + let removed = state_rm + .db() + .remove_vm(&name_rm) + .map_err(ApiError::database)?; + if removed.is_none() { + return Err(ApiError::NotFound(format!( + "machine '{}' not found", + name_rm + ))); + } + Ok(()) + } + Err(e) => Err(e), + } + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))??; + + // Remove VM data directory (disk images, sockets, etc.) + let data_dir = vm_data_dir(&name); + if data_dir.exists() { + // Release this VM's per-VM uid (if any) back to the allocator before the + // dir holding its `.vm-uid` record is removed, so a high-churn cloud node + // doesn't leak the uid range. A fork clone has no uid of its own (it + // shares its golden's). See process::free_vm_uid. + crate::process::free_vm_uid(&crate::agent::vm_uid_registry_dir(), &data_dir); + if let Err(e) = std::fs::remove_dir_all(&data_dir) { + tracing::warn!(error = %e, "failed to remove VM data directory: {}", data_dir.display()); + } + } + + Ok(Json(DeleteResponse { deleted: name })) +} + +/// Execute a command in a machine. +#[utoipa::path( + post, + path = "/api/v1/machines/{name}/exec", + tag = "Machines", + params( + ("name" = String, Path, description = "Machine name") + ), + request_body = MachineExecRequest, + responses( + (status = 200, description = "Command executed", body = ExecResponse), + (status = 400, description = "Invalid request", body = ApiErrorResponse), + (status = 404, description = "Machine not found", body = ApiErrorResponse), + (status = 409, description = "Machine not running", body = ApiErrorResponse), + (status = 500, description = "Execution failed", body = ApiErrorResponse) + ) +)] +pub async fn exec_machine( + State(state): State>, + Path(name): Path, + trace_id: Option>, + Json(req): Json, +) -> Result, ApiError> { + let tid = trace_id.map(|t| t.0 .0.clone()); + validate_command(&req.command)?; + + // Load the in-memory machine entry; its `secret_refs` were + // populated at create time and updated via start/stop handlers. + // This avoids a second DB read per request. + let entry = state.get_machine(&name)?; + crate::api::handlers::validate_request_secrets(&req.secrets)?; + let record_env = crate::api::handlers::record_secret_refs_env(&entry)?; + let req_env = crate::api::handlers::resolve_request_secrets(&req.secrets)?; + + let name_clone = name.clone(); + let command = req.command.clone(); + let mut env = EnvVar::to_tuples(&req.env); + env.extend(crate::secrets::expose_into_env(record_env)); + env.extend(crate::secrets::expose_into_env(req_env)); + let workdir = req.workdir.clone(); + let timeout = req.timeout_secs.map(Duration::from_secs); + let stdin_data = req.stdin.clone(); + + let result = tokio::task::spawn_blocking(move || { + // Get manager and check if running + let manager = AgentManager::for_vm(&name_clone) + .map_err(|e| SmolvmError::agent("create agent manager", e.to_string()))?; + + if manager.try_connect_existing().is_none() { + return Err(SmolvmError::InvalidState { + expected: "running".into(), + actual: "stopped".into(), + }); + } + + // Execute command + let mut client = manager + .connect() + .map_err(|e| SmolvmError::agent("connect", e.to_string()))?; + if let Some(tid) = tid { + client.set_trace_id(tid); + } + let (exit_code, stdout, stderr) = client + .vm_exec(command, env, workdir, timeout, stdin_data) + .map_err(|e| SmolvmError::agent("exec", e.to_string()))?; + + // Keep VM running (persistent) + manager.detach(); + + Ok(ExecResponse { + exit_code, + stdout: String::from_utf8_lossy(&stdout).into_owned(), + stderr: String::from_utf8_lossy(&stderr).into_owned(), + stdout_b64: stdout, + stderr_b64: stderr, + }) + }) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))?; + + result.map(Json).map_err(ApiError::from) +} + +/// Resize a machine's disk resources. +#[utoipa::path( + post, + path = "/api/v1/machines/{name}/resize", + tag = "Machines", + params( + ("name" = String, Path, description = "Machine name") + ), + request_body = ResizeMachineRequest, + responses( + (status = 200, description = "Machine resized", body = MachineInfo), + (status = 400, description = "Invalid request", body = ApiErrorResponse), + (status = 404, description = "Machine not found", body = ApiErrorResponse), + (status = 409, description = "Machine is running", body = ApiErrorResponse), + (status = 500, description = "Resize failed", body = ApiErrorResponse) + ) +)] +pub async fn resize_machine( + State(state): State>, + Path(name): Path, + Json(req): Json, +) -> Result, ApiError> { + let record = state + .lookup_vm(&name) + .await? + .ok_or_else(|| ApiError::NotFound(format!("machine '{}' not found", name)))?; + + let actual_state = record.actual_state(); + match actual_state { + RecordState::Stopped | RecordState::Created => {} + _ => { + return Err(ApiError::Conflict(format!( + "machine '{}' must be stopped before resizing. Current state: {:?}", + name, actual_state + ))); + } + } + + let current_storage_gb = record.storage_gb.unwrap_or(DEFAULT_STORAGE_SIZE_GIB); + let current_overlay_gb = record.overlay_gb.unwrap_or(DEFAULT_OVERLAY_SIZE_GIB); + + if req.storage_gb.unwrap_or(current_storage_gb) < current_storage_gb { + return Err(ApiError::BadRequest(format!( + "storageGb cannot be smaller than current size ({} GiB)", + current_storage_gb + ))); + } + if req.overlay_gb.unwrap_or(current_overlay_gb) < current_overlay_gb { + return Err(ApiError::BadRequest(format!( + "overlayGb cannot be smaller than current size ({} GiB)", + current_overlay_gb + ))); + } + + if req.storage_gb.is_none() && req.overlay_gb.is_none() { + return Err(ApiError::BadRequest( + "at least one of storageGb or overlayGb must be specified".into(), + )); + } + + let manager = AgentManager::for_vm(&name) + .map_err(|e| ApiError::internal(format!("failed to get agent manager: {}", e)))?; + + if let Some(storage_gb) = req.storage_gb { + if storage_gb > current_storage_gb { + let storage_path = manager.storage_path(); + expand_disk::(storage_path, storage_gb) + .map_err(|e| ApiError::internal(format!("failed to expand storage: {}", e)))?; + } + } + + if let Some(overlay_gb) = req.overlay_gb { + if overlay_gb > current_overlay_gb { + let overlay_path = manager.overlay_path(); + expand_disk::(overlay_path, overlay_gb) + .map_err(|e| ApiError::internal(format!("failed to expand overlay: {}", e)))?; + } + } + + let (storage_gb, overlay_gb) = (req.storage_gb, req.overlay_gb); + let record = state + .update_vm(&name, move |r| { + if let Some(s) = storage_gb { + r.storage_gb = Some(s); + } + if let Some(o) = overlay_gb { + r.overlay_gb = Some(o); + } + }) + .await? + .ok_or_else(|| { + ApiError::NotFound(format!("machine '{}' disappeared during resize", name)) + })?; + + Ok(Json(record_to_info(&name, &record))) +} + +/// Export a stopped machine to a `.smolmachine` and push it directly to a +/// registry. +/// +/// The machine must be stopped: exporting a running VM would snapshot an +/// inconsistent overlay. The `.smolmachine` is produced by subprocessing this +/// same binary's `pack create --from-vm ` (the tested path that boots a +/// helper VM to export the container overlay), then streamed to the registry +/// with the control-plane-minted, pre-scoped OCI bearer. +#[utoipa::path( + post, + path = "/api/v1/machines/{name}/export", + tag = "Machines", + params( + ("name" = String, Path, description = "Machine name") + ), + request_body = ExportRequest, + responses( + (status = 200, description = "Machine exported and pushed", body = ExportResponse), + (status = 404, description = "Machine not found", body = ApiErrorResponse), + (status = 409, description = "Machine is not stopped", body = ApiErrorResponse), + (status = 500, description = "Export or push failed", body = ApiErrorResponse) + ) +)] +pub async fn export_machine( + State(state): State>, + Path(id): Path, + Json(req): Json, +) -> Result, ApiError> { + // Resolve the machine record; the path id is the machine name in this API. + let record = state + .lookup_vm(&id) + .await? + .ok_or_else(|| ApiError::NotFound(format!("machine '{}' not found", id)))?; + let name = id; + + // Require STOPPED via the shared probe so a running VM (whose overlay is + // still being written) can't be snapshotted into an inconsistent image. + let name_probe = name.clone(); + let record_probe = record.clone(); + let resolved = + tokio::task::spawn_blocking(move || resolve_machine_state(&name_probe, &record_probe)) + .await + .map_err(|e| ApiError::internal(format!("task error: {}", e)))?; + if resolved != RecordState::Stopped { + return Err(ApiError::Conflict( + "machine must be stopped to export".to_string(), + )); + } + + // Build the .smolmachine by subprocessing this binary's tested export path. + // The serve handlers and the pack CLI share the same on-disk SmolvmDb, so + // `pack create --from-vm ` sees the serve-managed machine. + let tmp = tempfile::Builder::new() + .suffix(".smolmachine") + .tempfile() + .map_err(|e| ApiError::internal(format!("create temp file: {}", e)))?; + let tmp_path = tmp.path().to_path_buf(); + let exe = + std::env::current_exe().map_err(|e| ApiError::internal(format!("current_exe: {}", e)))?; + + let output = tokio::process::Command::new(&exe) + .args([ + "pack", + "create", + "--from-vm", + &name, + "-o", + &tmp_path.to_string_lossy(), + ]) + .output() + .await + .map_err(|e| ApiError::internal(format!("spawn pack export: {}", e)))?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(ApiError::internal(format!( + "pack export failed: {}", + stderr + ))); + } + + // Read back the PackManifest from the sidecar footer for the response. + let manifest = smolvm_pack::read_manifest_from_sidecar(&tmp_path) + .map_err(|e| ApiError::internal(format!("read exported manifest: {}", e)))?; + let manifest_json = serde_json::to_string(&manifest) + .map_err(|e| ApiError::internal(format!("serialize manifest: {}", e)))?; + + // Push directly to the registry using the pre-scoped bearer token. The + // control mints a tenant-scoped OCI bearer, so use the raw token path + // (.with_token), not /v2/auth. + let base_url = if smolvm_registry::is_local_registry(&req.reference_host) { + format!("http://{}", req.reference_host) + } else { + format!("https://{}", req.reference_host) + }; + let client = smolvm_registry::RegistryClient::new(base_url).with_token(req.push_token.clone()); + + let result = smolvm_registry::push(&client, &req.repo, &req.tag, &tmp_path) + .await + .map_err(|e| ApiError::internal(format!("registry push failed: {}", e)))?; + + // tmp drops here, deleting the sidecar. + Ok(Json(ExportResponse { + digest: result.manifest_digest, + size_bytes: result.layer_size, + platform: result.platform, + manifest: manifest_json, + })) +} + +async fn pull_from_registry( + registry_ref: &str, + identity_token: Option<&str>, + blob_peers: &[String], +) -> Result { + let parsed = crate::registry::Reference::parse(registry_ref) + .map_err(|e| ApiError::BadRequest(format!("invalid registry reference: {}", e)))?; + + let settings = crate::settings::SmolSettings::load() + .map_err(|e| ApiError::internal(format!("load settings: {}", e)))?; + + let effective_registry = settings + .machines + .get_mirror(&parsed.registry) + .unwrap_or(&parsed.registry); + let api_host = match effective_registry { + "docker.io" => "registry-1.docker.io", + h => h, + }; + let base_url = if smolvm_registry::is_local_registry(api_host) { + format!("http://{}", api_host) + } else { + format!("https://{}", api_host) + }; + + let mut client = smolvm_registry::RegistryClient::new(base_url); + + // A request-supplied identity token (the control plane's short-lived, + // tenant-scoped pull token) takes precedence over any persisted credential. + if let Some(token) = identity_token { + client = client.with_identity_token(token.to_string()); + } else if let Some(entry) = settings.machines.registries.get(effective_registry) { + if let Some(ref token) = entry.identity_token { + client = client.with_identity_token(token.clone()); + } + } + + let cache = smolvm_registry::BlobCache::open_default() + .map_err(|e| ApiError::internal(format!("blob cache: {}", e)))?; + + let repo = parsed.repository(); + let tag_or_digest = registry_reference_tag_or_digest(&parsed); + + tracing::info!( + registry_ref = %registry_ref, + repo = %repo, + reference = %tag_or_digest, + "pulling .smolmachine from registry" + ); + + let result = smolvm_registry::pull(&client, &repo, tag_or_digest, None, &cache, blob_peers) + .await + .map_err(|e| match &e { + // A missing image/manifest is the caller's mistake (typo'd ref, or a + // bare name that resolved to an empty repo) — surface it as 404, not a + // 500. A 500 here misreports a client error as a server fault and + // pollutes the fleet error-rate SLO on every bad reference. + smolvm_registry::RegistryError::BlobNotFound(_) + | smolvm_registry::RegistryError::ApiError { status: 404, .. } => { + ApiError::NotFound(format!("image not found in registry: {}", e)) + } + _ => ApiError::internal(format!("registry pull failed: {}", e)), + })?; + + tracing::info!(path = %result.path.display(), cached = result.cached, "pull complete"); + + Ok(result.path.to_string_lossy().into_owned()) +} + +fn registry_reference_tag_or_digest(parsed: &crate::registry::Reference) -> &str { + parsed + .digest + .as_deref() + .or(parsed.tag.as_deref()) + .unwrap_or("latest") +} + +fn resolve_create_resources( + req: &CreateMachineRequest, + manifest_cpus: u8, + manifest_mem: u32, +) -> (u8, u32) { + ( + req.cpus.unwrap_or(manifest_cpus), + req.mem.unwrap_or(manifest_mem), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::SmolvmDb; + use tempfile::TempDir; + + #[test] + fn classify_launch_error_flags_virtio_port_conflict() { + // The real virtio-net host-port bind failure → retryable PortConflict. + let e = "agent operation failed: configure virtio-net: failed to start virtio network \ + runtime: Address already in use (os error 98)" + .to_string(); + assert!(matches!( + classify_launch_error(e), + ApiError::PortConflict(_) + )); + } + + #[test] + fn classify_launch_error_keeps_others_internal() { + // An unrelated AddrInUse (no virtio context) must NOT be treated as a + // published-port conflict — reallocating a port wouldn't help. + assert!(matches!( + classify_launch_error("bind vsock: Address already in use".to_string()), + ApiError::Internal(_) + )); + // A generic boot failure stays a 500. + assert!(matches!( + classify_launch_error("failed to start machine: kernel panic".to_string()), + ApiError::Internal(_) + )); + } + + #[test] + fn test_record_to_info() { + let record = VmRecord::new( + "test-vm".to_string(), + 2, + 1024, + vec![ + ("/host/path".to_string(), "/guest/path".to_string(), false), + ("/host/ro".to_string(), "/guest/ro".to_string(), true), + ], + vec![(8080, 80), (3000, 3000)], + false, + ); + + let info = record_to_info("test-vm", &record); + + assert_eq!(info.name, "test-vm"); + assert_eq!(info.state, "created"); + assert_eq!(info.cpus, 2); + assert_eq!(info.mem, 1024); + assert_eq!(info.mounts.len(), 2); + assert_eq!(info.ports.len(), 2); + assert!(!info.network); + assert!(info.pid.is_none()); + } + + #[test] + fn test_record_to_info_with_running_state() { + let mut record = VmRecord::new("running-vm".to_string(), 1, 512, vec![], vec![], false); + record.state = RecordState::Running; + record.pid = Some(12345); + + let info = record_to_info("running-vm", &record); + + assert_eq!(info.name, "running-vm"); + // Note: actual_state() checks if process is alive, which won't be true in test + // So it will show as "stopped" even though record state is Running + assert_eq!(info.cpus, 1); + assert_eq!(info.mem, 512); + assert_eq!(info.mounts.len(), 0); + assert_eq!(info.ports.len(), 0); + } + + #[test] + fn test_record_to_info_default_values() { + let record = VmRecord::new("minimal-vm".to_string(), 1, 512, vec![], vec![], false); + + let info = record_to_info("minimal-vm", &record); + + assert_eq!(info.name, "minimal-vm"); + assert_eq!(info.state, "created"); + assert_eq!(info.cpus, 1); + assert_eq!(info.mem, 512); + assert_eq!(info.mounts.len(), 0); + assert_eq!(info.ports.len(), 0); + assert!(!info.network); + assert!(info.pid.is_none()); + assert!(info.created_at > 0); + // A machine created without explicit disk sizes still reports the RESOLVED + // provisioned sizes (the node default), not None — billing/telemetry need + // the actual allocated GiB. + assert_eq!(info.storage_gb, Some(DEFAULT_STORAGE_SIZE_GIB)); + assert_eq!(info.overlay_gb, Some(DEFAULT_OVERLAY_SIZE_GIB)); + } + + #[test] + fn test_record_to_info_with_network() { + let record = VmRecord::new("network-vm".to_string(), 1, 512, vec![], vec![], true); + + let info = record_to_info("network-vm", &record); + + assert_eq!(info.name, "network-vm"); + assert!(info.network); + } + + #[test] + fn test_record_to_info_echoes_backend_and_cidrs() { + let mut record = VmRecord::new("policy-vm".to_string(), 1, 512, vec![], vec![], true); + record.network_backend = Some(crate::network::NetworkBackend::VirtioNet); + record.allowed_cidrs = Some(vec!["10.0.0.0/8".to_string()]); + + let info = record_to_info("policy-vm", &record); + + assert_eq!( + info.network_backend, + Some(crate::network::NetworkBackend::VirtioNet) + ); + assert_eq!( + info.allowed_cidrs.as_deref(), + Some(["10.0.0.0/8".to_string()].as_slice()) + ); + + // Unset config stays absent so the JSON omits the fields entirely. + let bare = VmRecord::new("bare-vm".to_string(), 1, 512, vec![], vec![], false); + let bare_info = record_to_info("bare-vm", &bare); + assert!(bare_info.network_backend.is_none()); + assert!(bare_info.allowed_cidrs.is_none()); + } + + #[test] + fn registry_reference_uses_digest_before_tag_or_latest() { + let digest = "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + + let digest_ref = + crate::registry::Reference::parse(&format!("python-dev@{digest}")).unwrap(); + assert_eq!(registry_reference_tag_or_digest(&digest_ref), digest); + + let tagged_ref = crate::registry::Reference::parse("python-dev:v1").unwrap(); + assert_eq!(registry_reference_tag_or_digest(&tagged_ref), "v1"); + + let latest_ref = crate::registry::Reference::parse("python-dev").unwrap(); + assert_eq!(registry_reference_tag_or_digest(&latest_ref), "latest"); + } + + fn minimal_create_request() -> CreateMachineRequest { + CreateMachineRequest { + name: Some("test-vm".to_string()), + cpus: None, + mem: None, + mounts: vec![], + ports: vec![], + network: false, + gpu: false, + storage_gb: None, + overlay_gb: None, + allowed_cidrs: None, + allowed_hosts: None, + network_backend: None, + restart: None, + image: None, + from: None, + registry_ref: None, + registry_identity_token: None, + blob_peers: vec![], + secrets: Default::default(), + } + } + + #[test] + fn create_resources_use_high_defaults_when_omitted() { + let req = minimal_create_request(); + + assert_eq!( + resolve_create_resources( + &req, + crate::data::resources::DEFAULT_MICROVM_CPU_COUNT, + crate::data::resources::DEFAULT_MICROVM_MEMORY_MIB, + ), + ( + crate::data::resources::DEFAULT_MICROVM_CPU_COUNT, + crate::data::resources::DEFAULT_MICROVM_MEMORY_MIB, + ) + ); + } + + #[test] + fn create_resources_preserve_manifest_defaults_when_omitted() { + let req = minimal_create_request(); + + assert_eq!(resolve_create_resources(&req, 6, 12_288), (6, 12_288)); + } + + #[test] + fn create_resources_explicit_api_values_override_manifest_defaults() { + let mut req = minimal_create_request(); + req.cpus = Some(2); + req.mem = Some(2048); + + assert_eq!(resolve_create_resources(&req, 6, 12_288), (2, 2048)); + } + + #[test] + fn create_request_deserialization_keeps_resource_omission_distinct() { + let req: CreateMachineRequest = serde_json::from_value(serde_json::json!({ + "name": "api-vm" + })) + .unwrap(); + + assert_eq!(req.cpus, None); + assert_eq!(req.mem, None); + + let req: CreateMachineRequest = serde_json::from_value(serde_json::json!({ + "name": "api-vm", + "cpus": 2, + "memoryMb": 2048 + })) + .unwrap(); + + assert_eq!(req.cpus, Some(2)); + assert_eq!(req.mem, Some(2048)); + } + + /// Helper to create a test database and API state. + #[allow(dead_code)] + fn setup_test_state() -> (TempDir, Arc) { + let dir = TempDir::new().expect("failed to create temp dir"); + let db_path = dir.path().join("test.db"); + let db = SmolvmDb::open_at(&db_path).expect("failed to open test db"); + let state = Arc::new(ApiState::with_db(db)); + (dir, state) + } + + #[tokio::test] + async fn test_resize_validation_shrink_storage_rejected() { + let (_dir, state) = setup_test_state(); + let db = state.db(); + create_test_vm(db, "test-vm", Some(20), Some(5)); + + let req = ResizeMachineRequest { + storage_gb: Some(10), + overlay_gb: None, + }; + let result = resize_machine(State(state), Path("test-vm".to_string()), Json(req)).await; + assert!(matches!(result.unwrap_err(), ApiError::BadRequest(_))); + } + + #[tokio::test] + async fn test_resize_validation_no_params_rejected() { + let (_dir, state) = setup_test_state(); + let db = state.db(); + create_test_vm(db, "test-vm", Some(20), Some(5)); + + let req = ResizeMachineRequest { + storage_gb: None, + overlay_gb: None, + }; + let result = resize_machine(State(state), Path("test-vm".to_string()), Json(req)).await; + assert!(matches!(result.unwrap_err(), ApiError::BadRequest(_))); + } + + #[tokio::test] + async fn test_resize_not_found() { + let (_dir, state) = setup_test_state(); + let req = ResizeMachineRequest { + storage_gb: Some(30), + overlay_gb: None, + }; + let result = resize_machine(State(state), Path("nonexistent".to_string()), Json(req)).await; + assert!(matches!(result.unwrap_err(), ApiError::NotFound(_))); + } + + /// Helper to create a VM record in the database. + fn create_test_vm(db: &SmolvmDb, name: &str, storage_gb: Option, overlay_gb: Option) { + let mut record = VmRecord::new(name.to_string(), 1, 512, vec![], vec![], false); + record.storage_gb = storage_gb; + record.overlay_gb = overlay_gb; + db.insert_vm(name, &record) + .expect("failed to insert test vm"); + } +} diff --git a/src/api/handlers/mod.rs b/src/api/handlers/mod.rs new file mode 100644 index 0000000..97ed388 --- /dev/null +++ b/src/api/handlers/mod.rs @@ -0,0 +1,297 @@ +//! HTTP request handlers. + +pub mod exec; +pub mod files; +pub mod health; +pub mod images; +pub mod machines; +pub mod node; +pub mod p2p; +pub mod volumes; + +use crate::api::error::ApiError; +use crate::api::state::MachineEntry; +use crate::secrets::ResolutionError; + +/// Maximum number of ad-hoc secret refs in a single API request body. +/// Bounds the per-request resolution work and blocks trivial DOS +/// attempts that flood `secrets: {...}` with thousands of entries. +pub(crate) const MAX_REQ_SECRETS_PER_REQUEST: usize = 64; + +/// Resolve a [`MachineEntry`]'s persisted `secret_refs` under +/// `RecordReplay` scope. In-memory access — no DB hit per request. +/// +/// Returns the resolved `(key, value)` tuples ready to extend into an +/// env vector. Failures map to structured [`ApiError`]s via +/// [`classify_resolution_error`] so HTTP status codes are consistent +/// across exec/run handlers. +pub(crate) fn record_secret_refs_env( + entry: &std::sync::Arc>, +) -> Result, ApiError> { + let refs = { + let guard = entry.lock(); + guard.secret_refs.clone() + }; + if refs.is_empty() { + return Ok(Vec::new()); + } + crate::secrets::resolve_refs_to_env_classified( + &refs, + crate::secrets::ResolutionScope::RecordReplay, + ) + .map_err(classify_resolution_error) +} + +/// Map a classified [`ResolutionError`] to an [`ApiError`] per the +/// status-code table in `docs/secrets-api-pack-plan.md` §4.3. +/// +/// Client-fixable failures (`EnvUnset`, `FileReadFailed`, +/// `FileTooLarge`) → 400; server-state failures (`Internal`) → 500. +/// Body always includes the secret key and the +/// failure class, never the raw error message (which may include the +/// `from_file` path or `from_env` variable name). +pub(crate) fn classify_resolution_error(e: ResolutionError) -> ApiError { + let ResolutionError { key, kind } = e; + let body = format!("secret '{}': {}", key, kind.as_str()); + if kind.is_client_error() { + ApiError::BadRequest(body) + } else { + ApiError::internal(body) + } +} + +/// Maximum length of a secret key (guest-side env var name). +/// +/// Aligned with the agent's env-var validation so API and agent agree. +pub(crate) const MAX_SECRET_KEY_LEN: usize = 256; + +/// Check that a request-supplied secret key is a valid POSIX-style env +/// var name. This is the same rule the agent applies to the final env +/// list; enforcing it at API ingress converts a deep-in-agent confused +/// error into a clear 400 at the boundary. +/// +/// Rules: +/// - non-empty +/// - ≤ `MAX_SECRET_KEY_LEN` bytes +/// - first char is ASCII letter or `_` +/// - all chars are ASCII alphanumeric or `_` +fn check_env_key_shape(key: &str) -> Result<(), String> { + if key.is_empty() { + return Err("env key must not be empty".into()); + } + if key.len() > MAX_SECRET_KEY_LEN { + return Err(format!("env key exceeds {}-byte limit", MAX_SECRET_KEY_LEN)); + } + let first = key.chars().next().expect("non-empty"); + if !first.is_ascii_alphabetic() && first != '_' { + return Err("env key must start with an ASCII letter or underscore".into()); + } + if !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + return Err("env key must contain only ASCII alphanumeric and underscore".into()); + } + Ok(()) +} + +/// Validate an incoming `req.secrets` map against: +/// +/// - the per-request size cap (`MAX_REQ_SECRETS_PER_REQUEST`), +/// - the POSIX env-var key rule for the map key, and +/// - the untrusted-source ref policy: an HTTP caller is `Untrusted`, and +/// no ref source kind (`from_env`/`from_file`) is resolvable in that +/// scope, so any non-empty `secrets` map is rejected. Secrets must be +/// configured locally via the CLI instead. +/// +/// On failure, produces a `BadRequest` response body naming the +/// specific key and rule. Used by exec/run/create handlers before +/// resolution is attempted. +pub(crate) fn validate_request_secrets( + refs: &std::collections::BTreeMap, +) -> Result<(), ApiError> { + if refs.len() > MAX_REQ_SECRETS_PER_REQUEST { + return Err(ApiError::BadRequest(format!( + "request `secrets` map has {} entries; maximum is {}", + refs.len(), + MAX_REQ_SECRETS_PER_REQUEST + ))); + } + for (name, r) in refs { + // Validate the *key* before the ref. A malformed key can't + // safely be included in an error message back to the caller + // (could contain control chars or huge strings), so we only + // echo its byte length when it's malformed. + check_env_key_shape(name).map_err(|rule| { + ApiError::BadRequest(format!( + "secrets entry with {}-byte key rejected: {}", + name.len(), + rule + )) + })?; + crate::secrets::validate_ref(r, crate::secrets::ResolutionScope::Untrusted) + .map_err(|e| ApiError::BadRequest(format!("secret '{}': {}", name, e)))?; + } + Ok(()) +} + +/// Resolve request-body `req.secrets` under `Untrusted` scope. Caller +/// must have already called [`validate_request_secrets`]. +pub(crate) fn resolve_request_secrets( + refs: &std::collections::BTreeMap, +) -> Result, ApiError> { + if refs.is_empty() { + return Ok(Vec::new()); + } + // `Untrusted` here is defense-in-depth — validate_request_secrets + // has already enforced allowed source kinds and size caps. + // resolve_refs_to_env_classified maps failures to ResolutionError + // which we then map to structured ApiError via the status-code + // table above. + // + // Note: we deliberately pass the ref map through twice (validate + // then resolve) instead of doing one combined pass, so validation + // failures are distinguishable in the audit log from resolution + // failures — the first emits a `SecretRefError` trail via the + // handler's own logs, the second emits `secrets::audit` records + // with classified `error_kind`. + crate::secrets::resolve_refs_to_env_classified(refs, crate::secrets::ResolutionScope::Untrusted) + .map_err(classify_resolution_error) +} + +#[cfg(test)] +mod tests { + use super::*; + use smolvm_protocol::SecretRef; + use std::collections::BTreeMap; + + fn env_ref(name: &str) -> SecretRef { + SecretRef { + from_env: Some(name.to_string()), + from_file: None, + } + } + + #[test] + fn validate_request_secrets_accepts_empty_map() { + // The HTTP API can no longer carry resolvable secret refs (an + // untrusted caller must not read this host's env/files), so the + // only request that passes secret validation is one with none. + let refs = BTreeMap::new(); + assert!(validate_request_secrets(&refs).is_ok()); + } + + #[test] + fn validate_request_secrets_rejects_from_env() { + let mut refs = BTreeMap::new(); + refs.insert("X".to_string(), env_ref("HOST_VAR")); + let err = validate_request_secrets(&refs).unwrap_err(); + match err { + ApiError::BadRequest(msg) => { + assert!(msg.contains("X"), "body must name the bad key: {}", msg); + assert!( + msg.contains("env") || msg.contains("trusted local host"), + "body must explain rule: {}", + msg + ); + } + other => panic!("expected BadRequest, got {:?}", other), + } + } + + #[test] + fn validate_request_secrets_rejects_from_file() { + let mut refs = BTreeMap::new(); + refs.insert( + "X".to_string(), + SecretRef { + from_env: None, + from_file: Some("/absolute/path".into()), + }, + ); + let err = validate_request_secrets(&refs).unwrap_err(); + assert!(matches!(err, ApiError::BadRequest(_))); + } + + #[test] + fn validate_request_secrets_rejects_bad_keys() { + // Every failing key shape should be rejected at ingress with a + // BadRequest, never allowed through to the agent where it would + // become an opaque error. + let bad_keys = [ + "", // empty + "1FOO", // leading digit + "FOO BAR", // space + "FOO=BAR", // equals sign + "FOO-BAR", // hyphen + "FOO.BAR", // dot + "FOO\0BAR", // NUL + "FOO\nBAR", // control char + "ünicöde", // non-ASCII + ]; + for bad in bad_keys { + let mut refs = BTreeMap::new(); + refs.insert(bad.to_string(), env_ref("K")); + let err = validate_request_secrets(&refs) + .expect_err(&format!("key '{}' must be rejected", bad.escape_default())); + assert!( + matches!(err, ApiError::BadRequest(_)), + "key '{}' should be 400, got {:?}", + bad.escape_default(), + err + ); + } + } + + #[test] + fn validate_request_secrets_rejects_oversized_keys() { + let huge_key = "A".repeat(MAX_SECRET_KEY_LEN + 1); + let mut refs = BTreeMap::new(); + refs.insert(huge_key, env_ref("K")); + let err = validate_request_secrets(&refs).unwrap_err(); + assert!(matches!(err, ApiError::BadRequest(_))); + } + + #[test] + fn validate_request_secrets_enforces_size_cap() { + let mut refs = BTreeMap::new(); + for i in 0..=MAX_REQ_SECRETS_PER_REQUEST { + refs.insert(format!("K{}", i), env_ref(&format!("K{}", i))); + } + let err = validate_request_secrets(&refs).unwrap_err(); + match err { + ApiError::BadRequest(msg) => { + assert!(msg.contains(&MAX_REQ_SECRETS_PER_REQUEST.to_string())); + } + other => panic!("expected BadRequest, got {:?}", other), + } + } + + #[test] + fn classify_resolution_error_maps_to_status() { + use crate::secrets::{ResolutionError, ResolutionFailure}; + + let e = ResolutionError { + key: "MY_KEY".to_string(), + kind: ResolutionFailure::EnvUnset, + }; + match classify_resolution_error(e) { + ApiError::BadRequest(body) => { + assert!(body.contains("MY_KEY")); + assert!(body.contains("env_unset")); + } + other => panic!("EnvUnset must be 4xx, got {:?}", other), + } + + let e = ResolutionError { + key: "MY_KEY".to_string(), + kind: ResolutionFailure::Internal, + }; + // Internal is a server-state issue → 5xx. We can't easily + // pattern-match ApiError's internal variant name, but we can + // verify it's not a client error. + let mapped = classify_resolution_error(e); + assert!( + !matches!(mapped, ApiError::BadRequest(_)), + "Internal must not map to 4xx: {:?}", + mapped + ); + } +} diff --git a/src/api/handlers/node.rs b/src/api/handlers/node.rs new file mode 100644 index 0000000..a10be14 --- /dev/null +++ b/src/api/handlers/node.rs @@ -0,0 +1,114 @@ +//! Node-level introspection endpoints. + +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use axum::{extract::State, Json}; +use std::sync::Arc; + +use crate::api::state::ApiState; +use crate::api::types::CapacityResponse; + +/// Id minted once per serve process (pid + startup nanos — unique across a restart, +/// dependency-free). Constant for the process lifetime; a change tells the control +/// the serve restarted and wiped its warm pool. Lazily initialized on first read. +fn boot_id() -> &'static str { + static BOOT_ID: std::sync::OnceLock = std::sync::OnceLock::new(); + BOOT_ID.get_or_init(|| { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + format!("{}-{}", std::process::id(), nanos) + }) +} + +/// Report live node capacity — current allocations + real utilization across +/// all running machines on this host. +/// +/// Read-only and runtime-agnostic: it exposes only what the runtime knows +/// (allocated vs. used), leaving totals/reserved policy to the caller. A fleet +/// node-agent polls this over HTTP and forwards it to the control plane in its +/// heartbeat, which keeps the runtime itself free of any cloud coupling. +#[utoipa::path( + get, + path = "/capacity", + tag = "Node", + responses( + (status = 200, description = "Live node capacity", body = CapacityResponse), + (status = 503, description = "Main runtime stalled — node is unschedulable") + ) +)] +pub async fn capacity(State(state): State>) -> Response { + // If the main runtime stopped heartbeating, the lifecycle path (boot/stop/ + // exec) is wedged even though this loopback door — on its own runtime — can + // still answer. Report 503 so the node-agent's existing self-cordon drains + // this node instead of the control scheduling onto a black hole. The agent + // treats 503 like any poll failure and self-heals once heartbeats resume. + if state.runtime_stalled() { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "main runtime stalled; node unschedulable", + ) + .into_response(); + } + + let (allocated_cpus, allocated_memory_mb) = state.allocated_resources(); + let (used_cpus, used_memory_mb, used_disk_gb) = state.real_utilization(); + + Json(CapacityResponse { + allocated_cpus, + allocated_memory_mb, + used_cpus, + used_memory_mb, + used_disk_gb, + boot_id: boot_id().to_string(), + }) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::SmolvmDb; + + use axum::body::to_bytes; + + #[tokio::test] + async fn capacity_reports_zero_on_an_idle_node() { + let dir = tempfile::tempdir().unwrap(); + let db = SmolvmDb::open_at(&dir.path().join("test.db")).unwrap(); + let state = Arc::new(ApiState::with_db(db)); + // A fresh state has heartbeat 0 == "now", so it is not yet stalled. + + // No running machines → nothing allocated and nothing in use. + let resp = capacity(State(state)).await; + assert_eq!(resp.status(), StatusCode::OK); + let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); + let cap: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(cap["allocated_cpus"], 0); + assert_eq!(cap["allocated_memory_mb"], 0); + assert_eq!(cap["used_cpus"], 0.0); + assert_eq!(cap["used_memory_mb"], 0); + assert_eq!(cap["used_disk_gb"], 0); + } + + #[tokio::test] + async fn capacity_returns_503_when_runtime_heartbeat_is_stale() { + // Force an immediate stall window so the missing heartbeat counts as stalled. + std::env::set_var("SMOLVM_RUNTIME_STALE_SECS", "1"); + let dir = tempfile::tempdir().unwrap(); + let db = SmolvmDb::open_at(&dir.path().join("test.db")).unwrap(); + let state = Arc::new(ApiState::with_db(db)); + + // Let the 1s stall window elapse with no supervisor heartbeat. + tokio::time::sleep(std::time::Duration::from_millis(1_100)).await; + let resp = capacity(State(state.clone())).await; + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + + // A heartbeat clears the stall and capacity answers 200 again. + state.beat_runtime_heartbeat(); + let resp = capacity(State(state)).await; + assert_eq!(resp.status(), StatusCode::OK); + std::env::remove_var("SMOLVM_RUNTIME_STALE_SECS"); + } +} diff --git a/src/api/handlers/p2p.rs b/src/api/handlers/p2p.rs new file mode 100644 index 0000000..be97168 --- /dev/null +++ b/src/api/handlers/p2p.rs @@ -0,0 +1,141 @@ +//! Brokered peer-to-peer blob serving. +//! +//! `GET /p2p/blob/{digest}` streams a content-addressed layer blob straight from +//! this node's local blob cache to a sibling fleet node, so a `create` on +//! another node can pull a hot `.smolmachine` layer from a peer instead of the +//! registry. Read-only and content-addressed. +//! +//! ## Security +//! +//! This endpoint is mTLS-gated by the serve listener by construction (like +//! `/drain`, see the router comment): only a client whose cert chains to the +//! fleet node-CA can reach it. It is NOT tenant-scoped, so on a multi-tenant +//! fleet it is a cross-tenant read oracle for private blobs, bounded only by +//! mTLS + sha256 digest entropy. See [`smolvm_registry::peer`] for the full +//! trust model and the scoped-token requirement before enabling P2P on a +//! multi-tenant fleet that hosts private artifacts. + +use axum::{ + body::Body, + extract::Path, + http::{header, StatusCode}, + response::{IntoResponse, Response}, +}; +use smolvm_registry::BlobCache; +use tokio_util::io::ReaderStream; + +use crate::api::error::ApiError; + +/// Serve a cached layer blob by digest to a peer node. +/// +/// Opens the default blob cache and delegates to [`serve_blob_from`], which does +/// the digest validation, lookup, and streaming. +pub async fn serve_blob(Path(digest): Path) -> Result { + let cache = BlobCache::open_default() + .map_err(|e| ApiError::internal(format!("open blob cache: {e}")))?; + serve_blob_from(&cache, &digest).await +} + +/// Validate the digest, look it up in `cache`, and stream it. +/// +/// The digest is validated first — the path segment is attacker-influenced and +/// is used to build a cache filesystem path — then looked up: a miss is `404`, a +/// hit streams the file from disk. The body is streamed (never buffered) because +/// a `.smolmachine` layer can be hundreds of MB / multiple GB. +async fn serve_blob_from(cache: &BlobCache, digest: &str) -> Result { + // Reject a malformed / path-traversing digest before it is used to build any + // cache path (mirrors the check `pull` runs before touching the cache). + smolvm_registry::validate_digest(digest) + .map_err(|e| ApiError::BadRequest(format!("invalid blob digest: {e}")))?; + + let Some(path) = cache.get(digest) else { + return Err(ApiError::NotFound(format!("blob not cached: {digest}"))); + }; + + let file = tokio::fs::File::open(&path) + .await + .map_err(|e| ApiError::internal(format!("open cached blob: {e}")))?; + let len = file + .metadata() + .await + .map(|m| m.len()) + .map_err(|e| ApiError::internal(format!("stat cached blob: {e}")))?; + + let body = Body::from_stream(ReaderStream::new(file)); + Ok(( + StatusCode::OK, + [ + (header::CONTENT_TYPE, "application/octet-stream".to_string()), + (header::CONTENT_LENGTH, len.to_string()), + ], + body, + ) + .into_response()) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::to_bytes; + use sha2::{Digest, Sha256}; + + fn temp_cache() -> (tempfile::TempDir, BlobCache) { + let tmp = tempfile::tempdir().unwrap(); + let cache = BlobCache::open(tmp.path().to_path_buf(), 1024 * 1024).unwrap(); + (tmp, cache) + } + + fn digest_of(data: &[u8]) -> String { + format!("sha256:{}", hex::encode(Sha256::digest(data))) + } + + #[tokio::test] + async fn present_blob_streams_with_content_length() { + let data = b"p2p-served-blob-bytes".to_vec(); + let digest = digest_of(&data); + let (_tmp, cache) = temp_cache(); + cache.put(&digest, &data).unwrap(); + + let resp = serve_blob_from(&cache, &digest) + .await + .expect("present blob must serve 200") + .into_response(); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers() + .get(header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()), + Some(data.len().to_string().as_str()) + ); + let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); + assert_eq!(body.as_ref(), data.as_slice()); + } + + #[tokio::test] + async fn absent_blob_is_404() { + let (_tmp, cache) = temp_cache(); + let digest = digest_of(b"never-cached"); + let err = serve_blob_from(&cache, &digest) + .await + .expect_err("absent blob must be an error"); + assert!(matches!(err, ApiError::NotFound(_))); + } + + #[tokio::test] + async fn malformed_digest_is_400() { + let (_tmp, cache) = temp_cache(); + + // Malformed digest is rejected before any cache access. + let err = serve_blob_from(&cache, "sha256:not-a-real-digest") + .await + .expect_err("malformed digest must be rejected"); + assert!(matches!(err, ApiError::BadRequest(_))); + + // A path-traversal attempt is likewise a 400, never a filesystem read. + let err = serve_blob_from(&cache, "../../etc/passwd") + .await + .expect_err("path-traversal digest must be rejected"); + assert!(matches!(err, ApiError::BadRequest(_))); + } +} diff --git a/src/api/handlers/volumes.rs b/src/api/handlers/volumes.rs new file mode 100644 index 0000000..730f48d --- /dev/null +++ b/src/api/handlers/volumes.rs @@ -0,0 +1,116 @@ +//! Volume provisioning endpoints — node-side storage for the control plane. +//! +//! smolfleet's `create_volume`/`delete_volume` call these to materialize a `local` +//! volume ON the worker (not on the control plane's own disk, which the worker +//! can't see): a directory under the smolvm data dir that the worker +//! virtiofs-shares into a guest at mount time. +//! +//! Persistent-disk (`pd`) volumes are NOT handled here — disk create/format/attach +//! run on the control plane (which has cloud credentials and no `serve` syscall +//! reaper), and the node-agent does the final mount out-of-band. See smolfleet +//! docs/d3-replicated-volumes.md. + +use axum::{extract::Path, http::StatusCode, Json}; +use serde::{Deserialize, Serialize}; + +use crate::api::error::ApiError; + +/// Request body for `POST /api/v1/volumes`. +#[derive(Deserialize)] +pub struct ProvisionVolumeRequest { + /// Control-plane volume id (e.g. `vol-`); names the on-node directory. + pub id: String, + /// Requested size. Advisory for the `local` backend — a plain directory has no + /// hard quota. + #[serde(default)] + pub size_gb: u64, + /// Storage backend. Only `local` (a worker dir, the default) is handled on the + /// node; `pd` volumes are driven from the control plane, not here. + #[serde(default)] + pub backend: Option, +} + +/// Response body for `POST /api/v1/volumes`. +#[derive(Serialize)] +pub struct ProvisionVolumeResponse { + /// Host path the volume is mounted at on this node — becomes a workload mount + /// `source`. + pub node_path: String, +} + +/// Base directory for node-local volumes: `/smolvm/volumes`. +fn volumes_base() -> std::path::PathBuf { + dirs::data_local_dir() + .unwrap_or_else(|| std::path::PathBuf::from(".")) + .join("smolvm") + .join("volumes") +} + +/// Reject ids that could escape the volumes base (path traversal / separators). +/// The control plane generates `vol-`; this is defense in depth. +fn safe_volume_id(id: &str) -> Result<(), ApiError> { + let ok = !id.is_empty() + && id.len() <= 128 + && id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'); + if ok { + Ok(()) + } else { + Err(ApiError::BadRequest(format!("invalid volume id: {id:?}"))) + } +} + +/// `POST /api/v1/volumes` — create the backing storage and return its host path. +pub async fn provision_volume( + Json(req): Json, +) -> Result, ApiError> { + safe_volume_id(&req.id)?; + match req.backend.as_deref().unwrap_or("local") { + "local" => { + let path = volumes_base().join(&req.id); + std::fs::create_dir_all(&path).map_err(ApiError::internal)?; + let node_path = path.to_string_lossy().to_string(); + tracing::info!(volume_id = %req.id, node_path = %node_path, size_gb = req.size_gb, "provisioned local volume"); + Ok(Json(ProvisionVolumeResponse { node_path })) + } + other => Err(ApiError::BadRequest(format!( + "unsupported node volume backend: {other:?} (pd volumes are driven from the control plane)" + ))), + } +} + +/// `DELETE /api/v1/volumes/{id}` — tear down the backing storage. Idempotent: +/// deleting an already-absent volume succeeds. +pub async fn deprovision_volume(Path(id): Path) -> Result { + safe_volume_id(&id)?; + let path = volumes_base().join(&id); + match std::fs::remove_dir_all(&path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} // already gone + Err(e) => return Err(ApiError::internal(e)), + } + tracing::info!(volume_id = %id, "deprovisioned local volume"); + Ok(StatusCode::NO_CONTENT) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_path_traversal_ids() { + assert!(safe_volume_id("vol-abc123").is_ok()); + assert!(safe_volume_id("vol_1").is_ok()); + assert!(safe_volume_id("../etc").is_err()); + assert!(safe_volume_id("a/b").is_err()); + assert!(safe_volume_id("").is_err()); + assert!(safe_volume_id(&"x".repeat(200)).is_err()); + } + + #[test] + fn volumes_base_is_under_smolvm() { + let b = volumes_base(); + assert!(b.ends_with("smolvm/volumes"), "got {b:?}"); + } +} diff --git a/src/api/mod.rs b/src/api/mod.rs new file mode 100644 index 0000000..461edb6 --- /dev/null +++ b/src/api/mod.rs @@ -0,0 +1,367 @@ +//! HTTP API server for smolvm. +//! +//! This module provides an HTTP API for managing machines, containers, and images +//! without CLI overhead. +//! +//! # Example +//! +//! ```bash +//! # Start the server on the default Unix socket +//! smolvm serve start +//! +//! # Or start the server on TCP explicitly +//! smolvm serve start --listen 127.0.0.1:8080 +//! +//! # Create a machine +//! curl -X POST http://localhost:8080/api/v1/machines \ +//! -H "Content-Type: application/json" \ +//! -d '{"name": "test"}' +//! ``` + +#[path = "errors.rs"] +pub mod error; +pub mod handlers; +pub mod state; +pub mod supervisor; +pub mod types; + +use axum::{ + extract::Request, + http::{HeaderValue, StatusCode}, + middleware::{self, Next}, + response::Response, + routing::{delete, get, post, put}, + Router, +}; +use std::sync::Arc; +use std::time::Duration; +use tower_http::{ + cors::{AllowOrigin, CorsLayer}, + timeout::TimeoutLayer, + trace::TraceLayer, +}; +use utoipa::OpenApi; +use utoipa_swagger_ui::SwaggerUi; + +use self::error::ApiError; +use state::ApiState; + +/// OpenAPI documentation for the smolvm API. +#[derive(OpenApi)] +#[openapi( + info( + title = "smolvm API", + version = "0.5.2", + description = "smolvm API for managing machines and images.", + license(name = "Apache-2.0", url = "https://www.apache.org/licenses/LICENSE-2.0") + ), + tags( + (name = "Health", description = "Health check endpoints"), + (name = "Node", description = "Node capacity introspection"), + (name = "Machines", description = "Machine lifecycle management"), + (name = "Execution", description = "Command execution in machines"), + (name = "Logs", description = "Log streaming"), + (name = "Images", description = "OCI image management"), + (name = "Files", description = "File upload and download") + ), + paths( + // Health + handlers::health::health, + // Node + handlers::node::capacity, + // Execution + handlers::exec::exec_command, + handlers::exec::exec_stream, + handlers::exec::run_command, + handlers::exec::stream_logs, + // Files + handlers::files::upload_file, + handlers::files::download_file, + // Images + handlers::images::list_images, + handlers::images::pull_image, + // Machines + handlers::machines::create_machine, + handlers::machines::list_machines, + handlers::machines::get_machine, + handlers::machines::start_machine, + handlers::machines::fork_machine, + handlers::machines::stop_machine, + handlers::machines::delete_machine, + handlers::machines::exec_machine, + handlers::machines::resize_machine, + handlers::machines::export_machine, + ), + components(schemas( + // Request types + types::CreateMachineRequest, + types::RestartSpec, + types::MountSpec, + types::PortSpec, + types::ResourceSpec, + types::ExecRequest, + types::RunRequest, + types::EnvVar, + types::PullImageRequest, + types::DeleteQuery, + types::LogsQuery, + types::MachineExecRequest, + types::ResizeMachineRequest, + types::ForkRequest, + types::ExportRequest, + types::StartMachineQuery, + // Response types + types::HealthResponse, + types::CapacityResponse, + types::MachineInfo, + types::MountInfo, + types::ListMachinesResponse, + types::ExecResponse, + types::ExportResponse, + types::ImageInfo, + types::ListImagesResponse, + types::PullImageResponse, + types::StartResponse, + types::StopResponse, + types::DeleteResponse, + types::ApiErrorResponse, + )) +)] +pub struct ApiDoc; + +/// Default timeout for API requests (5 minutes). +/// Most operations (start, stop, exec) complete within this time. +/// Long-running operations like image pulls may need longer, but this +/// provides a reasonable upper bound for most requests. +const API_REQUEST_TIMEOUT_SECS: u64 = 300; + +/// Validate that an API command payload is not empty. +pub fn validate_command(cmd: &[String]) -> Result<(), ApiError> { + if cmd.is_empty() { + return Err(ApiError::BadRequest("command cannot be empty".into())); + } + Ok(()) +} + +/// Create the API router with all endpoints. +/// +/// `cors_origins` specifies allowed CORS origins. If empty, defaults to +/// localhost:8080 and localhost:3000 (both http and 127.0.0.1 variants). +pub fn create_router(state: Arc, cors_origins: Vec) -> Router { + // Health check route. `/health` is liveness (pure-async, always answers); + // `/readyz` is a dispatch-path readiness probe that exercises the blocking pool + // so the control can detect — and cordon — a node whose start/exec are wedged + // even while `/health` still returns 200. + let health_route = Router::new() + .route("/health", get(handlers::health::health)) + .route("/readyz", get(handlers::health::readyz)); + + // Node capacity introspection (polled by a fleet node-agent over HTTP). + let capacity_route = Router::new().route("/capacity", get(handlers::node::capacity)); + + // Explicit, control-initiated node drain (decommission). Stops all VMs + // cleanly. Control-only by construction (the listener is mTLS-gated; the + // loopback door is localhost). See docs/lossless-serve-restart.md. + let drain_route = Router::new().route("/drain", post(handlers::machines::drain_node)); + + // Brokered P2P blob serving: hand a cached layer blob to a sibling node so a + // create on another node can pull it from a peer instead of the registry. + // Read-only, content-addressed, and mTLS-gated by the listener by + // construction (like /drain, see the comment above). No request timeout: + // blobs can be large. + let p2p_route = Router::new().route("/p2p/blob/{digest}", get(handlers::p2p::serve_blob)); + + // Long-lived streaming routes (no request timeout): SSE logs and the + // interactive PTY WebSocket both outlive the 5-minute API timeout. + let logs_route = Router::new() + .route("/{id}/logs", get(handlers::exec::stream_logs)) + .route( + "/{id}/exec/interactive", + get(handlers::exec::exec_interactive), + ); + + // Machine routes with timeout + let machine_routes_with_timeout = Router::new() + .route("/", post(handlers::machines::create_machine)) + .route("/", get(handlers::machines::list_machines)) + .route("/{id}", get(handlers::machines::get_machine)) + .route("/{id}/start", post(handlers::machines::start_machine)) + .route("/{id}/fork", post(handlers::machines::fork_machine)) + .route("/{id}/stop", post(handlers::machines::stop_machine)) + .route("/{id}/export", post(handlers::machines::export_machine)) + .route("/{id}", delete(handlers::machines::delete_machine)) + // Exec routes + .route("/{id}/exec", post(handlers::exec::exec_command)) + .route("/{id}/exec/stream", post(handlers::exec::exec_stream)) + .route("/{id}/run", post(handlers::exec::run_command)) + // File I/O routes + .route("/{id}/files/{*path}", put(handlers::files::upload_file)) + .route("/{id}/files/{*path}", get(handlers::files::download_file)) + // Image routes + .route("/{id}/images", get(handlers::images::list_images)) + .route("/{id}/images/pull", post(handlers::images::pull_image)) + // Apply timeout only to these routes + .layer(TimeoutLayer::with_status_code( + StatusCode::REQUEST_TIMEOUT, + Duration::from_secs(API_REQUEST_TIMEOUT_SECS), + )); + + // Machine routes + let machine_routes = Router::new() + .merge(logs_route) + .merge(machine_routes_with_timeout); + + // Volume provisioning (node-side storage for the control plane): create the + // backing storage on THIS worker and return its host path. See + // handlers::volumes. + let volume_routes = Router::new() + .route("/", post(handlers::volumes::provision_volume)) + .route("/{id}", delete(handlers::volumes::deprovision_volume)); + + // API v1 routes + let api_v1 = Router::new() + .nest("/machines", machine_routes) + .nest("/volumes", volume_routes); + + // CORS: Use configured origins, or default to localhost for security. + let default_origins = || { + vec![ + "http://localhost:8080" + .parse() + .expect("hardcoded CORS origin"), + "http://127.0.0.1:8080" + .parse() + .expect("hardcoded CORS origin"), + "http://localhost:3000" + .parse() + .expect("hardcoded CORS origin"), + "http://127.0.0.1:3000" + .parse() + .expect("hardcoded CORS origin"), + ] + }; + let origins: Vec = if cors_origins.is_empty() { + default_origins() + } else { + let mut valid = Vec::new(); + for origin in &cors_origins { + match origin.parse() { + Ok(v) => valid.push(v), + Err(e) => { + tracing::warn!(origin = %origin, error = %e, "invalid CORS origin, skipping"); + } + } + } + if valid.is_empty() { + tracing::warn!("no valid CORS origins provided, falling back to defaults"); + default_origins() + } else { + valid + } + }; + + let cors = CorsLayer::new() + .allow_origin(AllowOrigin::list(origins)) + .allow_methods([ + axum::http::Method::GET, + axum::http::Method::POST, + axum::http::Method::DELETE, + ]) + .allow_headers([axum::http::header::CONTENT_TYPE]); + + // Prometheus metrics + let metrics_route = Router::new().route("/metrics", get(serve_metrics)); + + // Combine all routes + Router::new() + .merge(health_route) + .merge(capacity_route) + .merge(drain_route) + .merge(p2p_route) + .merge(metrics_route) + .nest("/api/v1", api_v1) + .merge(SwaggerUi::new("/swagger-ui").url("/api-docs/openapi.json", ApiDoc::openapi())) + .layer(middleware::from_fn(trace_id_middleware)) + .layer(TraceLayer::new_for_http()) + .layer(cors) + .with_state(state) +} + +/// Install the global Prometheus metrics recorder. +/// Returns None if a recorder is already installed (e.g., in tests). +pub fn install_metrics_recorder() -> Option { + metrics_exporter_prometheus::PrometheusBuilder::new() + .install_recorder() + .ok() +} + +/// Serve Prometheus metrics as text. +async fn serve_metrics() -> String { + METRICS_HANDLE.get().map(|h| h.render()).unwrap_or_default() +} + +/// Global handle to the Prometheus recorder, set once at startup. +/// Only accessed by serve.rs (startup) and serve_metrics (handler). +pub static METRICS_HANDLE: std::sync::OnceLock = + std::sync::OnceLock::new(); + +/// Normalize a request path for Prometheus labels. +/// Replaces machine IDs with `:id` to prevent cardinality explosion. +fn normalize_metrics_path(path: &str) -> String { + let parts: Vec<&str> = path.split('/').collect(); + if parts.len() >= 4 && parts[1] == "api" && parts[3] == "machines" { + if let Some(id_pos) = parts.get(4) { + if !id_pos.is_empty() { + let mut normalized = parts[..4].to_vec(); + normalized.push(":id"); + normalized.extend_from_slice(&parts[5..]); + return normalized.join("/"); + } + } + } + path.to_string() +} + +/// Trace ID for correlating API requests to agent operations. +#[derive(Clone, Debug)] +pub struct TraceId(pub String); + +/// Middleware that generates a unique trace ID for each request and returns it +/// in the `X-Trace-Id` response header. +async fn trace_id_middleware(mut req: Request, next: Next) -> Response { + use std::sync::atomic::{AtomicU64, Ordering}; + use tracing::Instrument; + + static REQUEST_SEQ: AtomicU64 = AtomicU64::new(0); + let seq = REQUEST_SEQ.fetch_add(1, Ordering::Relaxed); + let trace_id = format!("{:08x}{:08x}", seq, std::process::id()); + + req.extensions_mut().insert(TraceId(trace_id.clone())); + + let method = req.method().to_string(); + // Normalize path to template to avoid cardinality explosion from machine IDs + let path_template = normalize_metrics_path(req.uri().path()); + + let span = tracing::info_span!("request", trace_id = %trace_id); + let mut response = next.run(req).instrument(span).await; + + let status = response.status().as_u16().to_string(); + metrics::counter!("smolvm_api_requests_total", "method" => method, "status" => status, "path" => path_template).increment(1); + + if let Ok(val) = HeaderValue::from_str(&trace_id) { + response.headers_mut().insert("x-trace-id", val); + } + response +} + +#[cfg(test)] +mod tests { + use super::validate_command; + + #[test] + fn test_validate_command() { + assert!(validate_command(&[]).is_err()); + assert!(validate_command(&["echo".to_string()]).is_ok()); + assert!(validate_command(&["echo".to_string(), "hello".to_string()]).is_ok()); + } +} diff --git a/src/api/state.rs b/src/api/state.rs new file mode 100644 index 0000000..693ae14 --- /dev/null +++ b/src/api/state.rs @@ -0,0 +1,1672 @@ +//! API server state management. + +use crate::agent::{AgentManager, HostMount, PortMapping, VmResources}; +use crate::api::error::ApiError; +use crate::api::types::{MachineInfo, MountSpec, PortSpec, ResourceSpec, RestartSpec}; +use crate::config::{RecordState, RestartConfig, RestartPolicy, VmRecord}; +use crate::data::resources::{DEFAULT_MICROVM_CPU_COUNT, DEFAULT_MICROVM_MEMORY_MIB}; +use crate::db::SmolvmDb; +use parking_lot::RwLock; +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::sync::Arc; + +/// Per-PID sample for computing CPU rate across heartbeat intervals. +#[derive(Debug, Clone, Copy)] +struct CpuSample { + /// When the sample was taken. + at: std::time::Instant, + /// Cumulative CPU time at that moment (nanoseconds). + cpu_time_ns: u64, +} + +/// Shared API server state. +pub struct ApiState { + /// Registry of machine managers by name. + machines: RwLock>>>, + /// Reserved machine names (creation in progress). + /// This prevents race conditions during machine creation. + reserved_names: RwLock>, + /// Per-machine lifecycle locks serializing start/stop/delete/restart. + /// + /// On macOS, stop/delete `hdiutil`-detach a machine's case-sensitive + /// packed-layers volume while a concurrent start acquires+mounts+launches + /// against it. Without exclusion a detach can pull the volume out from under + /// a freshly-launched VM serving it over virtiofs, or a guest launched in the + /// detach window hits the launcher's missing-dir error (review finding #3). + /// Each lifecycle handler `.lock().await`s this per-name async mutex as its + /// outermost lock — before any DB read or `MachineEntry` mutex — and holds it + /// for the whole operation, so mount and detach can never interleave for one + /// machine. Lock order is lifecycle (tokio, async) → entry (parking_lot, + /// inside `spawn_blocking`); no entry-holding path ever takes lifecycle, so + /// there is no inversion. On Linux the guarded detach/mount are compile-time + /// no-ops, making this harmless serialization. Scope: the API server only — + /// the embedded (`control.rs`) and CLI (`vm_common.rs`) paths are separate + /// processes with no shared in-process lock. + /// + /// Entries are created on first use and never removed: the map is bounded by + /// the number of distinct machine names, so a retained `Arc>` per + /// deleted name is negligible, and never removing avoids handing two callers + /// different mutexes for the same name (which would defeat the exclusion). + lifecycle_locks: RwLock>>>, + /// Database for persistent state. + db: SmolvmDb, + /// Previous CPU samples per VM PID, used to compute the fractional-CPU + /// rate as a delta over wall time. Pruned on each sample to drop dead PIDs. + cpu_samples: parking_lot::Mutex>, + /// Monotonic base for the runtime-liveness heartbeat. All heartbeat values + /// are millis elapsed since this instant, so they're a single lock-free + /// `AtomicU64` instead of a mutex-guarded `Instant`. + started_at: std::time::Instant, + /// Milliseconds (since `started_at`) of the supervisor's last tick. The + /// supervisor bumps this every `CHECK_INTERVAL` from the main runtime, so a + /// stale value means the main runtime's timer wheel stopped being driven + /// (a reactor stall) or the supervisor task itself wedged. The loopback + /// `/capacity` listener — which runs on its OWN runtime and so keeps + /// answering even when the main runtime is stuck — reads this to report the + /// node as unschedulable (HTTP 503) the moment the main runtime stops + /// making progress, turning a silent wedge into a fast, honest drain signal. + runtime_heartbeat_ms: std::sync::atomic::AtomicU64, +} + +/// Internal machine entry with manager and configuration. +pub struct MachineEntry { + /// The agent manager for this machine. + pub manager: AgentManager, + /// Host mounts configured for this machine. + pub mounts: Vec, + /// Port mappings configured for this machine. + pub ports: Vec, + /// VM resources configured for this machine. + pub resources: ResourceSpec, + /// Restart configuration for this machine. + pub restart: RestartConfig, + /// Whether outbound network access is enabled. + pub network: bool, + /// Secret refs persisted on the VM record, cached in memory so + /// exec handlers don't need a second DB read per request. Exec + /// handlers resolve these under `RecordReplay` scope. + pub secret_refs: std::collections::BTreeMap, + /// Path to the `.smolmachine` sidecar this machine was created from, if any. + /// When set, the start paths mount the bundle's pre-extracted OCI layers via + /// virtiofs instead of having the guest pull from a registry. `None` for + /// image/registry-sourced machines. Mirrors `VmRecord::source_smolmachine`. + pub source_smolmachine: Option, +} + +/// Parameters for registering a new machine. +pub struct MachineRegistration { + /// The agent manager for this machine. + pub manager: AgentManager, + /// Host mounts to configure. + pub mounts: Vec, + /// Port mappings to configure. + pub ports: Vec, + /// VM resources to configure. + pub resources: ResourceSpec, + /// Restart configuration. + pub restart: RestartConfig, + /// Whether outbound network access is enabled. + pub network: bool, + /// OCI image reference (e.g., "alpine:latest"). + pub image: Option, + /// Path to .smolmachine sidecar this machine was created from. + pub source_smolmachine: Option, + /// Container entrypoint (from manifest). + pub entrypoint: Vec, + /// Container cmd (from manifest). + pub cmd: Vec, + /// Environment variables (from manifest). + pub env: Vec<(String, String)>, + /// Working directory (from manifest). + pub workdir: Option, + /// Secret refs to attach to this machine (from a Smolfile or + /// `CreateMachineRequest.secrets`). + pub secret_refs: std::collections::BTreeMap, +} + +/// RAII guard for machine name reservation. +/// +/// Automatically releases reservation on drop unless consumed by `complete()`. +/// This ensures reservations are always cleaned up, even on panic. +/// +/// # Example +/// +/// ```ignore +/// let guard = ReservationGuard::new(&state, "my-machine".to_string())?; +/// +/// // Create the machine manager... +/// let manager = AgentManager::for_vm(guard.name())?; +/// +/// // Complete registration, consuming the guard +/// guard.complete(MachineRegistration { manager, mounts, ports, resources, restart, network })?; +/// ``` +pub struct ReservationGuard<'a> { + state: &'a ApiState, + name: String, + token: String, + completed: bool, +} + +impl<'a> ReservationGuard<'a> { + /// Reserve a machine name. Returns a guard that auto-releases on drop. + pub fn new(state: &'a ApiState, name: String) -> Result { + let token = SmolvmDb::create_reservation_token(); + state.reserve_machine_name(&name, &token)?; + Ok(Self { + state, + name, + token, + completed: false, + }) + } + + /// Get the reserved name. + pub fn name(&self) -> &str { + &self.name + } + + /// Complete registration, consuming the guard without releasing. + /// + /// This transfers ownership of the name to the machine registry. + pub fn complete(mut self, registration: MachineRegistration) -> Result<(), ApiError> { + self.state + .complete_machine_registration(self.name.clone(), &self.token, registration)?; + self.completed = true; + Ok(()) + } +} + +impl Drop for ReservationGuard<'_> { + fn drop(&mut self) { + if !self.completed { + self.state + .release_machine_reservation(&self.name, &self.token); + tracing::debug!(machine = %self.name, "reservation guard released on drop"); + } + } +} + +/// Whether `s` is a VM data-dir name — i.e. the output shape of `vm_dir_hash`: +/// exactly 16 lowercase hex chars. Used to confine the dangling-dir sweep to VM +/// dirs, never the shared pack store (`_shared`) or other cache entries. +fn is_vm_dir_hash(s: &str) -> bool { + s.len() == 16 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// Pure reaping decision for the dangling-dir sweep: a cache entry is dangling +/// iff it has the VM-dir-hash shape AND no live machine record maps to it. +fn is_dangling_vm_dir(dir_name: &str, valid_hashes: &std::collections::HashSet) -> bool { + is_vm_dir_hash(dir_name) && !valid_hashes.contains(dir_name) +} + +impl ApiState { + /// Create a new API state, opening the database. + /// + /// Returns an error if the database cannot be opened. + pub fn new() -> Result { + let db = SmolvmDb::open() + .map_err(|e| ApiError::internal(format!("failed to open database: {}", e)))?; + // Ensure tables exist at server startup (CLI paths handle this lazily). + db.init_tables().map_err(|e| { + ApiError::internal(format!("failed to initialize database tables: {}", e)) + })?; + Ok(Self { + machines: RwLock::new(HashMap::new()), + reserved_names: RwLock::new(HashSet::new()), + lifecycle_locks: RwLock::new(HashMap::new()), + db, + cpu_samples: parking_lot::Mutex::new(HashMap::new()), + started_at: std::time::Instant::now(), + runtime_heartbeat_ms: std::sync::atomic::AtomicU64::new(0), + }) + } + + /// Create a new API state with a specific database. + /// + /// Useful for testing with temporary databases. + pub fn with_db(db: SmolvmDb) -> Self { + Self { + machines: RwLock::new(HashMap::new()), + reserved_names: RwLock::new(HashSet::new()), + lifecycle_locks: RwLock::new(HashMap::new()), + db, + cpu_samples: parking_lot::Mutex::new(HashMap::new()), + started_at: std::time::Instant::now(), + runtime_heartbeat_ms: std::sync::atomic::AtomicU64::new(0), + } + } + + /// How long the main runtime may go without a supervisor heartbeat before + /// the loopback `/capacity` door reports the node as stalled. Defaults to + /// 4× the supervisor's 5s tick (`SMOLVM_RUNTIME_STALE_SECS` overrides), so + /// a few slow ticks never flap the node but a genuine reactor wedge drains + /// it within ~20s + the node-agent's own cordon grace. + fn runtime_stale_after_ms() -> u64 { + std::env::var("SMOLVM_RUNTIME_STALE_SECS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&s| s > 0) + .map(|s| s * 1000) + .unwrap_or(20_000) + } + + /// Record that the main runtime is making progress. Called from the + /// supervisor's tick — a value only advances if the runtime's timer wheel + /// fired the tick, so it is a true liveness signal for the main reactor. + pub fn beat_runtime_heartbeat(&self) { + let elapsed = self.started_at.elapsed().as_millis() as u64; + self.runtime_heartbeat_ms + .store(elapsed, std::sync::atomic::Ordering::Relaxed); + } + + /// Whether the main runtime has gone too long without a heartbeat — i.e. the + /// supervisor tick stopped firing, which on a multi-thread runtime means the + /// IO/timer driver is no longer being driven. Read from the loopback door's + /// dedicated runtime so it stays accurate even when the main runtime is + /// wedged. The window before the first heartbeat is treated as healthy + /// (startup), since `elapsed - 0 = elapsed` only crosses the threshold once + /// the runtime has actually been up longer than the stall window without a + /// single supervisor tick — which is itself a real stall. + pub fn runtime_stalled(&self) -> bool { + let now = self.started_at.elapsed().as_millis() as u64; + let last = self + .runtime_heartbeat_ms + .load(std::sync::atomic::Ordering::Relaxed); + now.saturating_sub(last) > Self::runtime_stale_after_ms() + } + + /// Load existing machines from persistent database. + /// Call this on server startup to reconnect to running VMs. + pub fn load_persisted_machines(&self) -> Vec { + let vms = match self.db.list_vms() { + Ok(vms) => vms, + Err(e) => { + tracing::warn!(error = %e, "failed to load VMs from database"); + return Vec::new(); + } + }; + + let mut loaded = Vec::new(); + + for (name, record) in vms { + // Only clean up machines that have a PID (were started) but whose + // process is no longer alive. Machines in "created" state (pid=None) + // have never been started and must be preserved — they are valid + // configs waiting for a start call. + if record.pid.is_some() && !record.is_process_alive() { + tracing::info!(machine = %name, "cleaning up dead machine from database"); + if let Err(e) = self.db.remove_vm(&name) { + tracing::warn!(machine = %name, error = %e, "failed to remove dead machine from database"); + } + // Reclaim the data dir too. Removing only the DB record leaks the + // machine's storage + overlay images (multi-GB sparse files) — and + // since the record is gone, nothing will ever clean them up later. + // On a long-lived node that churns/crashes machines this is a slow + // disk-fill across server restarts. The `pid.is_some()` guard above + // means we only touch machines that were started and then died, not + // intentionally-stopped (pid=None) machines whose disks must persist. + let dir = crate::agent::vm_data_dir(&name); + if dir.exists() { + if let Err(e) = std::fs::remove_dir_all(&dir) { + tracing::warn!(machine = %name, error = %e, "failed to remove dead machine data dir"); + } + } + continue; + } + + // Convert VmRecord to MachineEntry + let mounts: Vec = record + .mounts + .iter() + .map(|(source, target, readonly)| MountSpec { + source: source.clone(), + target: target.clone(), + readonly: *readonly, + }) + .collect(); + + let ports: Vec = record + .ports + .iter() + .map(|(host, guest)| PortSpec { + host: *host, + guest: *guest, + }) + .collect(); + + let resources = ResourceSpec { + cpus: Some(record.cpus), + memory_mb: Some(record.mem), + network: Some(record.network), + gpu: record.gpu, + storage_gb: record.storage_gb, + overlay_gb: record.overlay_gb, + allowed_cidrs: record.allowed_cidrs.clone(), + allowed_hosts: record.dns_filter_hosts.clone(), + network_backend: record.network_backend, + }; + + // Create AgentManager and try to reconnect + match AgentManager::for_vm_with_sizes(&name, record.storage_gb, record.overlay_gb) { + Ok(manager) => { + // Try to reconnect to existing running VM + let reconnected = manager + .try_connect_existing_with_pid_and_start_time( + record.pid, + record.pid_start_time, + ) + .is_some(); + + if reconnected { + tracing::info!(machine = %name, pid = ?record.pid, "reconnected to machine"); + } else { + // Process is alive but agent isn't reachable yet (transient + // boot/socket timing). Register the machine anyway so it's + // visible via APIs and the supervisor can manage it. Keep + // the DB record for future reconnect attempts. + tracing::info!(machine = %name, pid = ?record.pid, "machine alive but not yet reachable, registering for later reconnect"); + } + + let mut machines = self.machines.write(); + machines.insert( + name.clone(), + Arc::new(parking_lot::Mutex::new(MachineEntry { + manager, + mounts, + ports, + resources, + restart: record.restart.clone(), + network: record.network, + secret_refs: record.secret_refs.clone(), + source_smolmachine: record.source_smolmachine.clone(), + })), + ); + loaded.push(name.clone()); + } + Err(e) => { + // Process is alive but manager creation failed (transient + // filesystem/env issue). Preserve the DB record so the VM + // isn't orphaned — next server restart can retry. + tracing::warn!(machine = %name, error = %e, "failed to create manager for alive machine, preserving DB record"); + } + } + } + + loaded + } + + /// Remove VM data dirs under the cache root that no current machine record + /// references — a filesystem-level GC for the dirs the per-record path can't + /// see (a dir whose record was already gone: a legacy leak, or a CLI ephemeral + /// orphan that resolved to the wrong hash). + /// + /// Deliberately NOT called from `load_persisted_machines`: this is a global + /// sweep keyed off the *current* DB, so it must only run in the real `smolvm + /// serve` process (which owns this node's cache), never from a unit test or an + /// embedded library user whose DB does not describe the host's dirs — there it + /// would wipe live machines. The caller invokes it at server startup, before + /// requests are served, so it cannot race VM creation. Returns the count + /// removed. + /// + /// Safe by construction: only entries whose name is a VM data-dir hash (16 + /// lowercase hex chars, the `vm_dir_hash` shape) are candidates, so the shared + /// pack store (`_shared`) and marker files are never touched, and every hash + /// backing a live DB record is skipped. + pub fn reclaim_dangling_vm_dirs(&self) -> usize { + let valid: std::collections::HashSet = match self.db.list_vms() { + Ok(vms) => vms + .iter() + .map(|(name, _)| crate::agent::vm_dir_hash(name)) + .collect(), + Err(_) => return 0, + }; + let root = crate::agent::vm_cache_root(); + let entries = match std::fs::read_dir(&root) { + Ok(e) => e, + Err(_) => return 0, + }; + let mut removed = 0; + for entry in entries.flatten() { + let dir_name = entry.file_name().to_string_lossy().into_owned(); + if !is_dangling_vm_dir(&dir_name, &valid) { + continue; + } + if entry.path().is_dir() { + tracing::info!(dir = %dir_name, "removing dangling VM data dir (no machine record)"); + match std::fs::remove_dir_all(entry.path()) { + Ok(()) => removed += 1, + Err(e) => { + tracing::warn!(dir = %dir_name, error = %e, "failed to remove dangling VM data dir") + } + } + } + } + removed + } + + /// Get a machine entry by name. + pub fn get_machine( + &self, + name: &str, + ) -> Result>, ApiError> { + let machines = self.machines.read(); + machines + .get(name) + .cloned() + .ok_or_else(|| ApiError::NotFound(format!("machine '{}' not found", name))) + } + + /// Get the per-machine lifecycle lock for `name`, creating it on first use. + /// + /// Callers `.lock().await` the returned mutex at the top of + /// start/stop/delete/restart and hold the guard for the whole operation so + /// the macOS layers-volume mount (start) and detach (stop/delete) can never + /// interleave for one machine. See the `lifecycle_locks` field docs for the + /// lock-ordering and scope contract. + pub fn lifecycle_lock(&self, name: &str) -> Arc> { + // Fast path: the lock already exists (the common case after first use). + if let Some(lock) = self.lifecycle_locks.read().get(name) { + return lock.clone(); + } + // Slow path: create it under the write lock. `or_insert_with` collapses + // the race where two callers reach here for the same name concurrently — + // both end up with the same `Arc`, preserving mutual exclusion. + self.lifecycle_locks + .write() + .entry(name.to_string()) + .or_insert_with(|| Arc::new(tokio::sync::Mutex::new(()))) + .clone() + } + + /// Remove a machine from the registry (also removes from database). + /// + /// Must NOT hold the registry write lock across the DB delete: `db.remove_vm` + /// is synchronous disk I/O (SQLite, which under churn waits on `busy_timeout`), + /// and spanning it with the write lock blocks every reader — including the + /// `/health` probe (`machine_counts` takes `machines.read()`). Under delete + /// churn on a small (2-core) worker that wedges the whole node: the reactor's + /// threads park on the registry lock and can no longer `accept()`. The real + /// per-machine mutual exclusion is the caller's `lifecycle_lock` (held across + /// the entire start/stop/delete), so releasing the registry lock between the + /// existence check, the DB delete, and the in-memory remove is safe. + pub fn remove_machine( + &self, + name: &str, + ) -> Result>, ApiError> { + // Existence check under a brief read lock (released immediately). + if !self.machines.read().contains_key(name) { + return Err(ApiError::NotFound(format!("machine '{}' not found", name))); + } + + // Remove from the database first, WITHOUT the registry lock held — if this + // fails, in-memory state stays consistent (the entry is still in the map). + match self.db.remove_vm(name) { + Ok(Some(_)) => {} // expected: row existed and was deleted + Ok(None) => { + // Row was already gone from DB (concurrent delete or manual cleanup). + // Log and continue — we still need to clean up in-memory state. + tracing::warn!( + machine = name, + "machine not found in database during remove (already deleted?)" + ); + } + Err(e) => { + tracing::error!(error = %e, machine = name, "failed to remove machine from database"); + return Err(ApiError::Internal(format!("database error: {}", e))); + } + } + + // Brief write lock: swap the entry out of the registry (O(1)). If a + // concurrent path already removed it, degrade to NotFound rather than + // panicking — correctness for the same name is guaranteed by the caller's + // lifecycle lock, so this only fires on misuse. + self.machines + .write() + .remove(name) + .ok_or_else(|| ApiError::NotFound(format!("machine '{}' not found", name))) + } + + /// Update machine state in database (call after start/stop). + /// + /// Returns an error if the database write fails. Callers in API handlers + /// should propagate this error; the supervisor can log and continue. + pub fn update_machine_state( + &self, + name: &str, + state: RecordState, + pid: Option, + ) -> std::result::Result<(), crate::Error> { + let pid_start_time = pid.and_then(crate::process::process_start_time); + let result = self.db.update_vm(name, |record| { + record.state = state; + record.pid = pid; + record.pid_start_time = pid_start_time; + })?; + match result { + Some(_) => Ok(()), + None => Err(crate::Error::database( + "update machine state", + format!("machine '{}' not found in database", name), + )), + } + } + + /// List all machines. + pub fn list_machines(&self) -> Vec { + let machines = self.machines.read(); + machines + .iter() + .map(|(name, entry)| { + let entry = entry.lock(); + machine_entry_to_info(name.clone(), &entry) + }) + .collect() + } + + /// Check if a machine exists. + pub fn machine_exists(&self, name: &str) -> bool { + self.machines.read().contains_key(name) + } + + /// Return (total, running) machine counts for health endpoint. + /// Uses try_lock to avoid blocking on contended machine entries. + pub fn machine_counts(&self) -> (usize, usize) { + let machines = self.machines.read(); + let total = machines.len(); + let running = machines + .values() + .filter(|e| { + e.try_lock() + .map(|entry| entry.manager.is_process_alive()) + .unwrap_or(true) // assume running if locked (active operation) + }) + .count(); + (total, running) + } + + /// Disarm every machine manager's `Drop` so a non-draining `serve` shutdown + /// (e.g. a binary-upgrade restart) leaves running VMs alive for the next + /// process to reconnect to, instead of `AgentManager::drop` tearing each one + /// down via `stop()`. Mirrors the CLI's detach-before-exit (`SigintGuard` + /// disarm + `manager.detach()`). The drain path does the opposite — it stops + /// VMs cleanly — so this is invoked ONLY on the survive-and-reconnect path. + /// Uses a blocking lock per entry: missing one would let its VM be killed. + pub fn detach_all(&self) { + let machines = self.machines.read(); + let mut detached = 0usize; + for entry in machines.values() { + entry.lock().manager.detach(); + detached += 1; + } + if detached > 0 { + tracing::info!( + count = detached, + "detached machine managers on shutdown; VMs left running for reconnect" + ); + } + } + + /// Compute total allocated resources across all running machines. + /// Returns (allocated_cpus, allocated_memory_mb). + pub fn allocated_resources(&self) -> (u32, u64) { + let machines = self.machines.read(); + let mut cpus: u32 = 0; + let mut memory_mb: u64 = 0; + for entry in machines.values() { + if let Some(e) = entry.try_lock() { + if e.manager.is_process_alive() { + cpus += e.resources.cpus.unwrap_or(1) as u32; + memory_mb += e.resources.memory_mb.unwrap_or(256) as u64; + } + } + } + (cpus, memory_mb) + } + + /// Sample real CPU + memory + disk utilization across all running VM processes. + /// + /// Returns `(used_cpus, used_memory_mb, used_disk_gb)`: + /// - CPU is fractional CPUs (e.g., 2.5 = 2.5 CPUs of load), computed as + /// `Δcpu_time / Δwall_time` since the previous sample per PID. First sample + /// for a new PID returns 0 CPU; subsequent samples return the real rate. + /// - Memory is the sum of resident set sizes across VM processes. + /// - Disk is the sum of VM storage + overlay disk file sizes on disk. + pub fn real_utilization(&self) -> (f64, u64, u64) { + let now = std::time::Instant::now(); + let mut total_cpus: f64 = 0.0; + let mut total_rss_bytes: u64 = 0; + let mut total_disk_bytes: u64 = 0; + + let pid_and_paths: Vec<(Option, std::path::PathBuf, std::path::PathBuf)> = { + let machines = self.machines.read(); + machines + .values() + .filter_map(|entry| { + entry.try_lock().and_then(|e| { + if !e.manager.is_process_alive() { + return None; + } + Some(( + e.manager.child_pid(), + e.manager.storage_path().to_path_buf(), + e.manager.overlay_path().to_path_buf(), + )) + }) + }) + .collect() + }; + + let mut samples = self.cpu_samples.lock(); + let mut still_alive: HashSet = HashSet::with_capacity(pid_and_paths.len()); + + for (pid_opt, storage, overlay) in pid_and_paths { + // Disk: stat the storage + overlay files. Cheap (one stat() each). + if let Ok(meta) = std::fs::metadata(&storage) { + total_disk_bytes = total_disk_bytes.saturating_add(meta.len()); + } + if let Ok(meta) = std::fs::metadata(&overlay) { + total_disk_bytes = total_disk_bytes.saturating_add(meta.len()); + } + + // CPU + memory: only if we have a PID + let Some(pid) = pid_opt else { continue }; + still_alive.insert(pid); + let Some(stats) = crate::process::process_stats(pid) else { + continue; + }; + total_rss_bytes = total_rss_bytes.saturating_add(stats.rss_bytes); + + if let Some(prev) = samples.get(&pid).copied() { + let dt_ns = now.duration_since(prev.at).as_nanos() as u64; + if dt_ns > 0 { + let dcpu_ns = stats.cpu_time_ns.saturating_sub(prev.cpu_time_ns); + total_cpus += dcpu_ns as f64 / dt_ns as f64; + } + } + samples.insert( + pid, + CpuSample { + at: now, + cpu_time_ns: stats.cpu_time_ns, + }, + ); + } + + // Drop samples for PIDs that no longer exist (avoids leaking memory + // as VMs come and go over the lifetime of the smolvm serve process). + samples.retain(|pid, _| still_alive.contains(pid)); + + ( + total_cpus, + total_rss_bytes / (1024 * 1024), + total_disk_bytes / (1024 * 1024 * 1024), + ) + } + + // ======================================================================== + // Atomic Machine Creation (Reservation Pattern) + // ======================================================================== + + /// Reserve a machine name atomically. + /// + /// This prevents race conditions where two concurrent requests try to create + /// a machine with the same name. The name is reserved until either: + /// - `complete_machine_registration()` is called (success) + /// - `release_machine_reservation()` is called (failure/cleanup) + /// + /// Returns `Err(Conflict)` if the name is already taken or reserved. + pub fn reserve_machine_name(&self, name: &str, token: &str) -> Result<(), ApiError> { + // First check: machine existence (early exit for common case). + // Use separate scope to release read lock before acquiring write lock. + // This prevents lock-order inversion with complete_machine_registration. + { + let machines = self.machines.read(); + if machines.contains_key(name) { + return Err(ApiError::Conflict(format!( + "machine '{}' already exists", + name + ))); + } + } + + // Acquire reservation lock + let mut reserved = self.reserved_names.write(); + + // Double-check machine existence (could have been added while we + // didn't hold the machines lock). This is necessary for correctness. + if self.machines.read().contains_key(name) { + return Err(ApiError::Conflict(format!( + "machine '{}' already exists", + name + ))); + } + + // Check if name is already reserved (creation in progress) + if reserved.contains(name) { + return Err(ApiError::Conflict(format!( + "machine '{}' is being created by another request", + name + ))); + } + + reserved.insert(name.to_string()); + + match self.db.reserve_vm_create(name, token) { + Ok(true) => {} + Ok(false) => { + reserved.remove(name); + return Err(ApiError::Conflict(format!( + "machine '{}' already exists or is being created", + name + ))); + } + Err(e) => { + reserved.remove(name); + return Err(ApiError::database(e)); + } + } + + tracing::debug!(machine = %name, "reserved machine name"); + Ok(()) + } + + /// Release a machine name reservation. + /// + /// Call this if machine creation fails after `reserve_machine_name()`. + pub fn release_machine_reservation(&self, name: &str, token: &str) { + let mut reserved = self.reserved_names.write(); + if reserved.remove(name) { + tracing::debug!(machine = %name, "released machine name reservation"); + } + if let Err(e) = self.db.release_vm_create_reservation(name, token) { + tracing::warn!(machine = %name, error = %e, "failed to release DB create reservation"); + } + } + + /// Complete machine registration after successful creation. + /// + /// This converts a reserved name into a fully registered machine. + /// The reservation is released and the machine entry is added. + pub fn complete_machine_registration( + &self, + name: String, + token: &str, + reg: MachineRegistration, + ) -> Result<(), ApiError> { + // Persist to database (with conflict detection) + let mut record = VmRecord::new_with_restart( + name.clone(), + reg.resources.cpus.unwrap_or(DEFAULT_MICROVM_CPU_COUNT), + reg.resources + .memory_mb + .unwrap_or(DEFAULT_MICROVM_MEMORY_MIB), + reg.mounts + .iter() + .map(|m| (m.source.clone(), m.target.clone(), m.readonly)) + .collect(), + reg.ports.iter().map(|p| (p.host, p.guest)).collect(), + reg.network, + reg.restart.clone(), + ); + record.storage_gb = reg.resources.storage_gb; + record.overlay_gb = reg.resources.overlay_gb; + // Persist egress policy + backend selection from the request (previously + // dropped here, so API-created machines silently lost both). + record.allowed_cidrs = reg.resources.allowed_cidrs.clone(); + record.dns_filter_hosts = reg.resources.allowed_hosts.clone(); + record.network_backend = reg.resources.network_backend; + record.image = reg.image; + record.source_smolmachine = reg.source_smolmachine.clone(); + record.entrypoint = reg.entrypoint; + record.cmd = reg.cmd; + record.env = reg.env; + record.workdir = reg.workdir; + record.secret_refs = reg.secret_refs.clone(); + + // Complete the cross-process create reservation and insert the VM row + // atomically. Only after that succeeds do we publish the in-memory entry. + match self.db.commit_reserved_vm(&name, token, &record) { + Ok(true) => { + { + let mut reserved = self.reserved_names.write(); + if !reserved.remove(&name) { + // Name wasn't reserved - this is a programming error + tracing::warn!(machine = %name, "completing registration for non-reserved name"); + } + } + // Successfully inserted, now add to in-memory registry + let mut machines = self.machines.write(); + machines.insert( + name, + Arc::new(parking_lot::Mutex::new(MachineEntry { + manager: reg.manager, + mounts: reg.mounts, + ports: reg.ports, + resources: reg.resources, + restart: reg.restart, + network: reg.network, + secret_refs: reg.secret_refs, + source_smolmachine: reg.source_smolmachine, + })), + ); + Ok(()) + } + Ok(false) => { + // Name already exists or the DB reservation was lost. + Err(ApiError::Conflict(format!( + "machine '{}' already exists or is no longer reserved", + name + ))) + } + Err(e) => { + tracing::error!(error = %e, machine = %name, "database error during registration"); + Err(ApiError::database(e)) + } + } + } + + /// Get the underlying database handle. + /// + /// Prefer the async helpers below (`lookup_vm`/`list_vm_records`/`update_vm`) + /// from async request handlers: `SmolvmDb`'s methods are synchronous SQLite + /// I/O, and calling them directly on the tokio reactor lets a stalled write + /// (under create/delete churn) park the worker pool and wedge the liveness + /// probes. The helpers run the I/O on the blocking pool. `db()` itself is for + /// synchronous contexts (CLI, embedded runtime, inside an existing + /// `spawn_blocking`). See `tests/reactor_wedge.rs`. + pub fn db(&self) -> &SmolvmDb { + &self.db + } + + /// Off-reactor VM lookup. Runs the blocking SQLite read on the blocking pool. + pub async fn lookup_vm(&self, name: &str) -> Result, ApiError> { + let db = self.db.clone(); + let name = name.to_string(); + tokio::task::spawn_blocking(move || db.get_vm(&name)) + .await + .map_err(|e| ApiError::internal(format!("db lookup_vm task join: {e}")))? + .map_err(ApiError::database) + } + + /// Off-reactor full VM listing. Runs the blocking SQLite scan on the blocking + /// pool (reads use the connection-pool, so this never serializes behind a write). + pub async fn list_vm_records(&self) -> Result, ApiError> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || db.list_vms()) + .await + .map_err(|e| ApiError::internal(format!("db list_vm_records task join: {e}")))? + .map_err(ApiError::database) + } + + /// Off-reactor read-modify-write of a VM record. Runs the synchronous + /// transaction on the blocking pool so the write — which holds the single + /// writer connection and may wait out `busy_timeout` under contention — never + /// parks a reactor worker thread. + pub async fn update_vm(&self, name: &str, f: F) -> Result, ApiError> + where + F: FnOnce(&mut VmRecord) + Send + 'static, + { + let db = self.db.clone(); + let name = name.to_string(); + tokio::task::spawn_blocking(move || db.update_vm(&name, f)) + .await + .map_err(|e| ApiError::internal(format!("db update_vm task join: {e}")))? + .map_err(ApiError::database) + } + + /// Insert a machine entry directly into the in-memory registry. + /// + /// Used by start_machine to register a booted VM so that exec/run/container + /// endpoints can find it without server restart. + pub fn insert_machine(&self, name: &str, entry: MachineEntry) { + let mut machines = self.machines.write(); + machines.insert(name.to_string(), Arc::new(parking_lot::Mutex::new(entry))); + } + + // ======================================================================== + // Restart Management Methods + // ======================================================================== + + /// List all machine names. + pub fn list_machine_names(&self) -> Vec { + self.machines.read().keys().cloned().collect() + } + + /// Get restart config for a machine from the in-memory registry. + pub fn get_restart_config(&self, name: &str) -> Option { + let machines = self.machines.read(); + machines.get(name).map(|entry| { + let entry = entry.lock(); + entry.restart.clone() + }) + } + + /// Best-effort update to the VM database record. Logs warnings on + /// `Ok(None)` (row not found) and `Err` without propagating. + fn update_vm_best_effort(&self, name: &str, op_label: &str, f: impl FnOnce(&mut VmRecord)) { + match self.db.update_vm(name, f) { + Ok(Some(_)) => {} + Ok(None) => { + tracing::warn!(machine = %name, op = op_label, "machine not found in database"); + } + Err(e) => { + tracing::warn!(error = %e, machine = %name, op = op_label, "failed to persist update"); + } + } + } + + /// Increment restart count for a machine. + pub fn increment_restart_count(&self, name: &str) { + if let Some(entry) = self.machines.read().get(name) { + entry.lock().restart.restart_count += 1; + } + self.update_vm_best_effort(name, "increment_restart_count", |r| { + r.restart.restart_count += 1; + }); + } + + /// Mark machine as user-stopped. + pub fn mark_user_stopped(&self, name: &str, stopped: bool) { + if let Some(entry) = self.machines.read().get(name) { + entry.lock().restart.user_stopped = stopped; + } + self.update_vm_best_effort(name, "mark_user_stopped", |r| { + r.restart.user_stopped = stopped; + }); + } + + /// Reset restart count (on successful start). + pub fn reset_restart_count(&self, name: &str) { + if let Some(entry) = self.machines.read().get(name) { + entry.lock().restart.restart_count = 0; + } + self.update_vm_best_effort(name, "reset_restart_count", |r| { + r.restart.restart_count = 0; + }); + } + + /// Update last exit code for a machine. + pub fn set_last_exit_code(&self, name: &str, exit_code: Option) { + self.update_vm_best_effort(name, "set_last_exit_code", |r| { + r.last_exit_code = exit_code; + }); + } + + /// Get last exit code for a machine. + pub fn get_last_exit_code(&self, name: &str) -> Option { + self.db + .get_vm(name) + .ok() + .flatten() + .and_then(|r| r.last_exit_code) + } + + /// Check if a machine process is alive. + /// + /// Delegates to `AgentManager::is_process_alive()` which checks the + /// in-memory child handle (with stored start time) and falls back to the + /// PID file. This is start-time-aware to avoid false positives from PID + /// reuse, and covers orphan processes not tracked in-memory. + pub fn is_machine_alive(&self, name: &str) -> bool { + if let Some(entry) = self.machines.read().get(name) { + // This runs on the supervisor's heartbeat path, so it must never + // block. A `MachineEntry` whose lock is currently held is, by + // definition, actively in use (mid agent I/O) — i.e. alive — so we + // treat lock contention as "alive" rather than parking the + // supervisor on it. Blocking here behind a single busy/stuck + // machine would stall the runtime heartbeat and mark the entire + // node unschedulable. + match entry.try_lock() { + Some(entry) => entry.manager.is_process_alive(), + None => true, + } + } else { + false + } + } +} + +/// Run a blocking operation against a machine's agent client. +/// +/// Handles the common pattern: clone entry → spawn_blocking → lock → connect → op → map errors. +/// Propagates an optional trace ID to the agent for request correlation. +pub async fn with_machine_client_traced( + entry: &Arc>, + trace_id: Option, + op: F, +) -> Result +where + T: Send + 'static, + F: FnOnce(&mut crate::agent::AgentClient) -> crate::Result + Send + 'static, +{ + let entry_clone = entry.clone(); + tokio::task::spawn_blocking(move || { + // Acquire a connected client under the per-machine lock, then RELEASE + // the lock before running the (potentially long, unbounded-blocking) + // agent operation. `connect()` returns an owned `AgentClient` over its + // own socket, so `op` needs no lock once connected. Holding the + // `MachineEntry` lock across blocking agent I/O lets a hung guest agent + // pin the lock indefinitely, which blocks the supervisor's liveness + // probe (`is_machine_alive`) and stalls the whole runtime heartbeat — + // marking the node unschedulable behind a single stuck exec. + let mut client = { + let entry = entry_clone.lock(); + let mut client = entry.manager.connect()?; + if let Some(tid) = trace_id { + client.set_trace_id(tid); + } + client + }; + op(&mut client) + }) + .await? + .map_err(ApiError::internal) +} + +// ============================================================================ +// Shared Machine Helpers +// ============================================================================ + +/// Build `LaunchFeatures` for an API-driven machine start. +/// +/// Thin wrapper over [`crate::agent::LaunchFeatures::with_packed_layers`] +/// — the single source of truth shared with the CLI and embedded start paths. +/// When the machine was created from a `.smolmachine` artifact +/// (`source_smolmachine` is set), its pre-extracted OCI layers — extracted into +/// the machine's own data dir at create time, keyed by `machine_name` — are +/// mounted via virtiofs so the guest uses them instead of pulling from a +/// registry; otherwise default features are returned. Without it the three API +/// start entrypoints (`start_machine`, `ensure_machine_running`, supervisor +/// restart) would pass `packed_layers_dir = None` and the guest would fall back +/// to a network pull. Performs blocking filesystem work — call from within a +/// `spawn_blocking` context. +pub fn build_launch_features( + machine_name: Option<&str>, + source_smolmachine: Option<&str>, + dns_filter_hosts: Option>, +) -> crate::Result { + let features = crate::agent::LaunchFeatures::default(); + let mut features = match machine_name { + Some(name) => features.with_packed_layers( + &crate::agent::machine_layers_cache_dir(name), + source_smolmachine, + )?, + None => features, + }; + // Carry the egress hostname allow-list into the boot config; `internal_boot` + // starts the DNS filter for these names and learns their answers into the + // egress allow-list (parity with the CLI `--allow-host` path). + features.dns_filter_hosts = dns_filter_hosts; + Ok(features) +} + +/// Ensure a machine is running, starting it if needed. +/// +/// This is the shared preflight check used by exec, container, and image handlers. +/// It converts the machine's mount/port/resource config and calls +/// `ensure_running_with_full_config` in a blocking task. +/// +/// On the not-already-up path this mounts the machine's macOS layers volume, so +/// callers MUST hold the per-machine lifecycle lock (see `ensure_running_and_persist`, +/// the only caller) for the duration, or the mount can race a concurrent +/// stop/delete detach (review finding #3). +pub async fn ensure_machine_running( + entry: &Arc>, +) -> crate::Result<()> { + let entry_clone = entry.clone(); + tokio::task::spawn_blocking(move || { + let entry = entry_clone.lock(); + let mounts: Vec<_> = entry + .mounts + .iter() + .map(HostMount::try_from) + .collect::>>()?; + let ports: Vec<_> = entry.ports.iter().map(PortMapping::from).collect(); + let resources = resource_spec_to_vm_resources(&entry.resources, entry.network); + + // Use subprocess launch to avoid macOS fork-in-multithreaded-process issue. + // + // Build the packed-layers features only when the VM is not already up. + // This preflight runs on every implicit-start request (exec/run/files/ + // images); when the VM is already running `ensure_running_via_subprocess` + // returns early (discarding `features`) as long as the mount/port/resource + // config is unchanged. Acquiring the layers lease on that hot path is + // wasted work — on macOS it re-mounts the case-sensitive volume via + // hdiutil — so gate it on the same already-running check. + // + // The gated `default()` is safe across the relaunch branch too: if the + // preflight detects a mount/port/resource change and restarts the VM, + // `ensure_running_via_subprocess` re-attaches this machine's pre-extracted + // packed layers itself (see `rewire_packed_layers_if_extracted`), so the + // restart keeps using them instead of falling back to a registry pull. + let features = if entry.manager.try_connect_existing().is_some() { + crate::agent::LaunchFeatures::default() + } else { + build_launch_features( + entry.manager.name(), + entry.source_smolmachine.as_deref(), + entry.resources.allowed_hosts.clone(), + )? + }; + entry + .manager + .ensure_running_via_subprocess(mounts, ports, resources, features)?; + Ok(()) + }) + .await + .map_err(|e| crate::Error::agent("ensure running", e.to_string()))? +} + +/// Ensure a machine is running and persist the Running state to the database. +/// +/// Used by handlers that implicitly start VMs (containers, exec, images). +/// State persistence is best-effort — a DB write failure is logged but does +/// not fail the request, matching the supervisor's error-handling pattern. +pub async fn ensure_running_and_persist( + state: &ApiState, + name: &str, + entry: &Arc>, +) -> crate::Result<()> { + // Hold the per-machine lifecycle lock across the implicit-start preflight. + // ensure_machine_running mounts the macOS layers volume (via with_packed_layers) + // when the VM is not already up — exactly like the explicit start_machine — so + // it must exclude against a concurrent stop/delete detach the same way (review + // finding #3 covers "start/ensure"). Acquired before ensure_machine_running's + // spawn_blocking takes the entry mutex, preserving the lifecycle → entry order; + // released before the caller's actual exec/file/image op, which neither mounts + // nor detaches. On Linux the guarded mount is a no-op, so this is harmless. + let lifecycle = state.lifecycle_lock(name); + let _guard = lifecycle.lock().await; + + ensure_machine_running(entry).await?; + + let pid = { + let entry = entry.lock(); + entry.manager.child_pid() + }; + if let Err(e) = state.update_machine_state(name, RecordState::Running, pid) { + tracing::warn!(machine = %name, error = %e, "failed to persist Running state after implicit start"); + } + + Ok(()) +} + +// ============================================================================ +// Type Conversions +// ============================================================================ + +impl TryFrom<&MountSpec> for HostMount { + type Error = crate::Error; + + /// Validate and canonicalize a MountSpec into a HostMount. + /// + /// API mount specs require absolute source paths even though CLI parsing + /// allows relative host paths that are canonicalized against the current + /// working directory. + fn try_from(spec: &MountSpec) -> Result { + let source = Path::new(&spec.source); + if !source.is_absolute() { + return Err(crate::Error::mount( + "validate source", + format!("path must be absolute: {}", source.display()), + )); + } + + HostMount::new(&spec.source, &spec.target, spec.readonly) + } +} + +impl From<&HostMount> for MountSpec { + fn from(mount: &HostMount) -> Self { + MountSpec { + source: mount.source.to_string_lossy().to_string(), + target: mount.target.to_string_lossy().to_string(), + readonly: mount.read_only, + } + } +} + +impl From<&PortSpec> for PortMapping { + fn from(spec: &PortSpec) -> Self { + PortMapping::new(spec.host, spec.guest) + } +} + +impl From<&PortMapping> for PortSpec { + fn from(mapping: &PortMapping) -> Self { + PortSpec { + host: mapping.host, + guest: mapping.guest, + } + } +} + +/// Convert multiple MountSpecs to HostMount values. +/// +/// Returns an error if any mount fails validation. +pub fn mounts_to_host_mounts(specs: &[MountSpec]) -> Result, ApiError> { + specs + .iter() + .map(|s| HostMount::try_from(s).map_err(|e| ApiError::BadRequest(e.to_string()))) + .collect() +} + +/// Convert ResourceSpec to VmResources. +pub fn resource_spec_to_vm_resources(spec: &ResourceSpec, network: bool) -> VmResources { + VmResources { + cpus: spec.cpus.unwrap_or(DEFAULT_MICROVM_CPU_COUNT), + memory_mib: spec.memory_mb.unwrap_or(DEFAULT_MICROVM_MEMORY_MIB), + network, + network_backend: spec.network_backend, + gpu: spec.gpu.unwrap_or(false), + // gpu_vram_mib not currently on ResourceSpec — API callers + // inherit the default. Add to ResourceSpec if the API ever + // needs to expose it. + gpu_vram_mib: None, + rosetta: false, + storage_gib: spec.storage_gb, + overlay_gib: spec.overlay_gb, + allowed_cidrs: spec.allowed_cidrs.clone(), + // Custom DNS is a local-CLI feature for now; the cloud ResourceSpec + // does not expose it, so API-launched VMs inherit the backend default. + dns: None, + } +} + +/// Convert VmResources to ResourceSpec. +pub fn vm_resources_to_spec(res: VmResources) -> ResourceSpec { + ResourceSpec { + cpus: Some(res.cpus), + memory_mb: Some(res.memory_mib), + network: Some(res.network), + gpu: Some(res.gpu), + storage_gb: res.storage_gib, + overlay_gb: res.overlay_gib, + allowed_cidrs: res.allowed_cidrs, + // VmResources has no hostname allow-list; callers that need it graft it + // back from the source record (see the MachineEntry reload path). + allowed_hosts: None, + network_backend: res.network_backend, + } +} + +/// Convert RestartSpec to RestartConfig. +pub fn restart_spec_to_config(spec: Option<&RestartSpec>) -> RestartConfig { + match spec { + Some(spec) => { + let policy = spec + .policy + .as_ref() + .and_then(|p| p.parse::().ok()) + .unwrap_or_default(); + RestartConfig { + policy, + max_retries: spec.max_retries.unwrap_or(0), + ..Default::default() + } + } + None => RestartConfig::default(), + } +} + +/// Convert a MachineEntry (in-memory state) to MachineInfo (API response). +pub fn machine_entry_to_info(name: String, entry: &MachineEntry) -> MachineInfo { + let state = if entry.manager.try_connect_existing().is_some() { + "running" + } else { + "stopped" + }; + let egress_bytes = crate::agent::read_egress_telemetry(&name); + // Live consumed CPU-seconds for the VMM child (host-sampled, resets on + // restart); the control plane accumulates the durable total. None when there + // is no live process to sample. + let stats = entry + .manager + .child_pid() + .and_then(crate::process::process_stats); + let cpu_seconds = stats.map(|s| s.cpu_time_ns / 1_000_000_000); + let cpu_millis = stats.map(|s| s.cpu_time_ns / 1_000_000); + let rss_mb = stats.map(|s| s.rss_bytes / (1024 * 1024)); + // Actual used disk (sparse-image blocks) — a gauge the control integrates for + // active-disk billing. Independent of whether there's a live VMM process. + let disk_used_mb = crate::agent::disk_used_mb(&name); + + MachineInfo { + name, + state: state.to_string(), + cpus: entry.resources.cpus.unwrap_or(1), + mem: entry.resources.memory_mb.unwrap_or(512), + pid: entry.manager.child_pid(), + mounts: entry + .mounts + .iter() + .enumerate() + .map(|(i, m)| crate::api::types::MountInfo { + tag: crate::data::storage::HostMount::mount_tag(i), + source: m.source.clone(), + target: m.target.clone(), + readonly: m.readonly, + }) + .collect(), + ports: entry.ports.clone(), + network: entry.network, + network_backend: entry.resources.network_backend, + allowed_cidrs: entry.resources.allowed_cidrs.clone(), + allowed_hosts: entry.resources.allowed_hosts.clone(), + storage_gb: entry.resources.storage_gb, + overlay_gb: entry.resources.overlay_gb, + egress_bytes, + cpu_seconds, + cpu_millis, + rss_mb, + disk_used_mb, + created_at: 0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + /// Create an ApiState with a temporary database for testing. + fn temp_api_state() -> (TempDir, ApiState) { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("test.db"); + let db = SmolvmDb::open_at(&path).unwrap(); + (dir, ApiState::with_db(db)) + } + + #[test] + fn test_type_conversions() { + // MountSpec -> HostMount preserves readonly flag (use /tmp which exists) + let spec = MountSpec { + source: "/tmp".into(), + target: "/guest".into(), + readonly: true, + }; + assert!(HostMount::try_from(&spec).unwrap().read_only); + + let spec = MountSpec { + source: "/tmp".into(), + target: "/guest".into(), + readonly: false, + }; + assert!(!HostMount::try_from(&spec).unwrap().read_only); + + // ResourceSpec with None uses defaults + let spec = ResourceSpec { + cpus: None, + memory_mb: None, + network: None, + gpu: None, + storage_gb: None, + overlay_gb: None, + allowed_cidrs: None, + allowed_hosts: None, + network_backend: None, + }; + let res = resource_spec_to_vm_resources(&spec, false); + assert_eq!(res.cpus, DEFAULT_MICROVM_CPU_COUNT); + assert_eq!(res.memory_mib, DEFAULT_MICROVM_MEMORY_MIB); + assert!(!res.network); + + // Test with network enabled + let res = resource_spec_to_vm_resources(&spec, true); + assert!(res.network); + } + + #[test] + fn build_launch_features_carries_allowed_hosts() { + // The serve-API launch path must forward the egress hostname allow-list + // into the boot config, so `internal_boot` starts the DNS filter for it. + let hosts = vec!["api.anthropic.com".to_string(), "pypi.org".to_string()]; + let features = build_launch_features(None, None, Some(hosts.clone())).unwrap(); + assert_eq!(features.dns_filter_hosts, Some(hosts)); + + // No hostname policy stays None (unrestricted egress, unchanged behavior). + let features = build_launch_features(None, None, None).unwrap(); + assert_eq!(features.dns_filter_hosts, None); + } + + #[test] + fn test_machine_not_found() { + let (_dir, state) = temp_api_state(); + assert!(matches!( + state.get_machine("nope"), + Err(ApiError::NotFound(_)) + )); + assert!(matches!( + state.remove_machine("nope"), + Err(ApiError::NotFound(_)) + )); + } + + // remove_machine must clear BOTH the DB row and the in-memory registry entry + // even though it no longer holds the registry write lock across the DB delete + // (the reorder that keeps `/health` from wedging under delete churn). + #[test] + fn test_remove_machine_clears_db_and_registry() { + let (_dir, state) = temp_api_state(); + + // Put a machine in BOTH the DB and the in-memory registry. + let record = VmRecord::new("remove-test-m1".into(), 1, 512, vec![], vec![], false); + state.db.insert_vm("remove-test-m1", &record).unwrap(); + let manager = AgentManager::for_vm("remove-test-m1").unwrap(); + state.insert_machine( + "remove-test-m1", + MachineEntry { + manager, + mounts: vec![], + ports: vec![], + resources: ResourceSpec { + cpus: None, + memory_mb: None, + network: None, + gpu: None, + storage_gb: None, + overlay_gb: None, + allowed_cidrs: None, + allowed_hosts: None, + network_backend: None, + }, + restart: RestartConfig::default(), + network: false, + secret_refs: Default::default(), + source_smolmachine: None, + }, + ); + + // Remove succeeds and returns the entry. + assert!(state.remove_machine("remove-test-m1").is_ok()); + + // Gone from BOTH stores. + assert!( + state.db.get_vm("remove-test-m1").unwrap().is_none(), + "DB row should be deleted" + ); + assert!( + matches!( + state.get_machine("remove-test-m1"), + Err(ApiError::NotFound(_)) + ), + "registry entry should be removed" + ); + + // Second remove → NotFound, not a panic. + assert!(matches!( + state.remove_machine("remove-test-m1"), + Err(ApiError::NotFound(_)) + )); + } + + // REGRESSION (runtime-wedge): the supervisor's liveness probe must NEVER + // block on the per-machine `MachineEntry` lock. A machine that is mid + // agent-I/O (e.g. a stuck exec) holds that lock; if `is_machine_alive` + // blocked on it, one wedged machine would stall the supervisor heartbeat + // and mark the whole node unschedulable (the 503/black-hole wedge observed + // under concurrent boots). With the entry lock held, `is_machine_alive` + // must return promptly, reporting the in-use machine as alive. + #[test] + fn is_machine_alive_does_not_block_when_entry_locked() { + use std::time::Duration; + + let (_dir, state) = temp_api_state(); + let record = VmRecord::new("busy-m1".into(), 1, 512, vec![], vec![], false); + state.db.insert_vm("busy-m1", &record).unwrap(); + let manager = AgentManager::for_vm("busy-m1").unwrap(); + state.insert_machine( + "busy-m1", + MachineEntry { + manager, + mounts: vec![], + ports: vec![], + resources: ResourceSpec { + cpus: None, + memory_mb: None, + network: None, + gpu: None, + storage_gb: None, + overlay_gb: None, + allowed_cidrs: None, + allowed_hosts: None, + network_backend: None, + }, + restart: RestartConfig::default(), + network: false, + secret_refs: Default::default(), + source_smolmachine: None, + }, + ); + + // Hold the entry lock to simulate an in-flight agent op pinning it. + let entry = state.get_machine("busy-m1").unwrap(); + let held = entry.lock(); + + // The probe must finish without waiting on the held lock. If it blocked, + // the scoped thread would still be running after the grace period. + std::thread::scope(|s| { + let h = s.spawn(|| state.is_machine_alive("busy-m1")); + std::thread::sleep(Duration::from_millis(200)); + assert!( + h.is_finished(), + "is_machine_alive blocked on a held MachineEntry lock — would stall the supervisor" + ); + assert!( + h.join().unwrap(), + "a locked (actively in-use) machine must read as alive" + ); + }); + drop(held); + } + + // ======================================================================== + // Startup reconciliation tests + // ======================================================================== + + #[test] + fn test_load_persisted_machines_removes_dead_records() { + let (_dir, state) = temp_api_state(); + + // Insert a record with a PID that doesn't exist (dead process) + let mut record = VmRecord::new("dead-machine".into(), 1, 512, vec![], vec![], false); + record.pid = Some(i32::MAX); // PID that certainly doesn't exist + record.state = RecordState::Running; + state.db.insert_vm("dead-machine", &record).unwrap(); + + // Give it a data dir with a marker file — reconciliation must reclaim it, + // not just the DB record (otherwise the disks leak across restarts). + let data_dir = crate::agent::vm_data_dir("dead-machine"); + std::fs::create_dir_all(&data_dir).unwrap(); + std::fs::write(data_dir.join("storage.raw"), b"x").unwrap(); + + // Verify record exists before load + assert!(state.db.get_vm("dead-machine").unwrap().is_some()); + + // Load should detect dead process and clean up DB record + let loaded = state.load_persisted_machines(); + assert!(loaded.is_empty(), "dead machine should not be loaded"); + + // DB record should be cleaned up + assert!( + state.db.get_vm("dead-machine").unwrap().is_none(), + "dead machine DB record should be removed" + ); + + // Data dir should be reclaimed too (no disk leak). + assert!( + !data_dir.exists(), + "dead machine data dir should be removed, not leaked" + ); + + // Name should be available for reuse + let token = SmolvmDb::create_reservation_token(); + assert!(state.reserve_machine_name("dead-machine", &token).is_ok()); + } + + #[test] + fn dangling_vm_dir_reaping_policy() { + use std::collections::HashSet; + // VM-dir-hash shape = exactly 16 lowercase hex chars. + assert!(is_vm_dir_hash("0213f25389658451")); + assert!(!is_vm_dir_hash("_shared")); // shared pack store — never a candidate + assert!(!is_vm_dir_hash("uids")); + assert!(!is_vm_dir_hash("0213F25389658451")); // uppercase isn't our shape + assert!(!is_vm_dir_hash("0213f253896584")); // too short + assert!(!is_vm_dir_hash("0213f25389658451aa")); // too long + assert!(!is_vm_dir_hash("0213f2538965845z")); // non-hex + + // A hash backing a live record is kept; an unbacked hash is dangling; + // non-VM entries are never dangling regardless of the valid set. + let valid: HashSet = ["aaaaaaaaaaaaaaaa".to_string()].into_iter().collect(); + assert!(is_dangling_vm_dir("bbbbbbbbbbbbbbbb", &valid)); // hash, no record → reap + assert!(!is_dangling_vm_dir("aaaaaaaaaaaaaaaa", &valid)); // hash, has record → keep + assert!(!is_dangling_vm_dir("_shared", &valid)); // not a hash → keep + } + + #[test] + fn test_load_persisted_machines_preserves_created_no_pid() { + let (_dir, state) = temp_api_state(); + + // Insert a record with no PID (created but never started). + // These must be preserved — they are valid configs waiting for + // a start call. + let record = VmRecord::new("ghost".into(), 1, 512, vec![], vec![], false); + state.db.insert_vm("ghost", &record).unwrap(); + + // Load should NOT remove it — no PID means "never started", not "dead". + let _loaded = state.load_persisted_machines(); + assert!( + state.db.get_vm("ghost").unwrap().is_some(), + "created (no-PID) machine must be preserved across server restart" + ); + } + + #[test] + fn test_load_persisted_machines_preserves_alive_unreachable_records() { + let (_dir, state) = temp_api_state(); + + // Use our own PID (always alive and owned by us, so kill(pid,0)==0). + // AgentManager::for_vm will create a VM directory but reconnect + // will fail (no socket/agent), so it hits the "alive but unreachable" + // path. The DB record should be preserved. + let our_pid = std::process::id() as i32; + let mut record = VmRecord::new("alive-vm".into(), 1, 512, vec![], vec![], false); + record.pid = Some(our_pid); + record.state = RecordState::Running; + state.db.insert_vm("alive-vm", &record).unwrap(); + + // Load — reconnect will fail (no agent socket), but record should + // be preserved in DB since process is alive + let _loaded = state.load_persisted_machines(); + + // DB record should still exist (not deleted) + assert!( + state.db.get_vm("alive-vm").unwrap().is_some(), + "alive machine DB record should be preserved when reconnect fails" + ); + } +} diff --git a/src/api/supervisor.rs b/src/api/supervisor.rs new file mode 100644 index 0000000..33e2455 --- /dev/null +++ b/src/api/supervisor.rs @@ -0,0 +1,452 @@ +//! Machine supervisor for health monitoring and restart policies. +//! +//! The supervisor runs as a background task that periodically checks machine health +//! and automatically restarts machines based on their restart policies. + +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::watch; + +use crate::api::state::ApiState; +use crate::config::RecordState; + +/// Interval between health checks. +const CHECK_INTERVAL: Duration = Duration::from_secs(5); + +/// Minimum delay between restart attempts. +const MIN_RESTART_DELAY: Duration = Duration::from_secs(1); + +/// Outcome of [`Supervisor::restart_timing`] for a dead, restart-eligible machine. +#[derive(Debug, PartialEq, Eq)] +enum RestartTiming { + /// Restart now (fast-path backoff, or the armed time has arrived). + Fire, + /// Just armed the backoff timer this tick; do not restart yet. + Armed, + /// Timer armed on an earlier tick and not yet due; keep waiting. + Waiting, +} + +/// Machine supervisor for health monitoring and automatic restarts. +pub struct Supervisor { + state: Arc, + shutdown_rx: watch::Receiver, + /// Per-machine instant at which a scheduled restart becomes due. Lets the + /// supervisor honor restart backoff WITHOUT sleeping inside the shared + /// health-check loop: a crash-looping machine with a long exponential + /// backoff must not stall every other machine's liveness check, gauge + /// reconcile, and log rotation. The supervisor is the single task that owns + /// the loop, so a plain map needs no synchronization. Entries are cleared + /// when a machine is alive again, its restart fires, or its policy stops it. + next_restart_at: std::collections::HashMap, +} + +impl Supervisor { + /// Create a new supervisor. + pub fn new(state: Arc, shutdown_rx: watch::Receiver) -> Self { + Self { + state, + shutdown_rx, + next_restart_at: std::collections::HashMap::new(), + } + } + + /// Run the supervisor loop. + /// + /// This method blocks until shutdown is signaled. + pub async fn run(mut self) { + let mut ticker = tokio::time::interval(CHECK_INTERVAL); + // Don't catch up on missed ticks + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + tracing::info!("supervisor started"); + + loop { + tokio::select! { + _ = ticker.tick() => { + // The tick firing proves the main runtime's timer wheel is + // still being driven — record it so the loopback `/capacity` + // door (on its own runtime) can report the node stalled if + // these ever stop. Bump BEFORE the work below so a slow + // health check doesn't itself register as a stall. + self.state.beat_runtime_heartbeat(); + // Reap exited VM boot subprocesses (selective, per registered + // PID) BEFORE the health check, so a just-crashed VM's zombie + // is gone and `is_alive` reports it crashed this same tick. + crate::process::reap_vm_children(); + self.check_all_machines().await; + } + _ = self.shutdown_rx.changed() => { + if *self.shutdown_rx.borrow() { + tracing::info!("supervisor shutting down"); + break; + } + } + } + } + } + + /// Check all machines and restart any that need it. + async fn check_all_machines(&mut self) { + let machine_names = self.state.list_machine_names(); + + // Drop schedule entries for machines that no longer exist so the map + // can't grow without bound across deletes. + self.next_restart_at + .retain(|name, _| machine_names.iter().any(|n| n == name)); + + for name in machine_names { + if let Err(e) = self.check_machine(&name).await { + tracing::warn!(machine = %name, error = %e, "failed to check machine"); + } + } + + // Reconcile the running gauge with actual state (handles crashed VMs + // that never went through stop(), preventing gauge drift). + let (total, running) = self.state.machine_counts(); + metrics::gauge!("smolvm_machines_running").set(running as f64); + metrics::gauge!("smolvm_machines_total").set(total as f64); + + // Also rotate logs for all machines + self.rotate_logs_if_needed().await; + } + + /// Check a single machine and restart if needed. + async fn check_machine(&mut self, name: &str) -> crate::Result<()> { + // Check if machine is alive + let is_alive = self.state.is_machine_alive(name); + + if is_alive { + // Running again (recovered, or a prior restart took hold) — drop any + // pending restart schedule so a future death re-arms from scratch. + self.next_restart_at.remove(name); + return Ok(()); + } + + // Machine is dead — try to retrieve its exit code via waitpid + // and persist it so the restart policy can use it. + if let Ok(Some(record)) = self.state.db().get_vm(name) { + if let Some(pid) = record.pid { + let exit_code = crate::process::try_wait(pid); + self.state.set_last_exit_code(name, exit_code); + } + } + + let last_exit_code = self.state.get_last_exit_code(name); + + // Machine is dead, check restart policy + let restart_config = match self.state.get_restart_config(name) { + Some(config) => config, + None => { + self.next_restart_at.remove(name); + return Ok(()); // Machine doesn't exist anymore + } + }; + + // Determine if we should restart (delegate to RestartConfig) + if !restart_config.should_restart(last_exit_code) { + self.next_restart_at.remove(name); + tracing::debug!(machine = %name, policy = %restart_config.policy, "machine dead, not restarting per policy"); + // Update state to stopped (best-effort in supervisor) + if let Err(e) = self + .state + .update_machine_state(name, RecordState::Stopped, None) + { + tracing::warn!(machine = %name, error = %e, "failed to persist stopped state"); + } + return Ok(()); + } + + // Honor the backoff by SCHEDULING, not sleeping: blocking here would + // stall every other machine's check this tick. See `restart_timing`. + let backoff = restart_config.backoff_duration(); + let now = tokio::time::Instant::now(); + match Self::restart_timing(&mut self.next_restart_at, name, backoff, now) { + RestartTiming::Waiting => return Ok(()), + RestartTiming::Armed => { + tracing::info!( + machine = %name, + restart_count = restart_config.restart_count, + backoff_secs = backoff.as_secs(), + "machine dead, scheduling restart" + ); + return Ok(()); + } + RestartTiming::Fire => {} + } + + // Increment restart count + self.state.increment_restart_count(name); + + // Attempt restart + self.restart_machine(name).await + } + + /// Decide, for a dead machine that should restart, whether its restart fires + /// on this tick — without ever sleeping. Mutates the schedule map in place: + /// + /// - backoff ≤ [`MIN_RESTART_DELAY`]: [`RestartTiming::Fire`] immediately + /// (the previous fast-path; nothing to schedule). + /// - first dead tick with a longer backoff: arm `now + backoff` and return + /// [`RestartTiming::Armed`]. + /// - a later tick before the armed time: [`RestartTiming::Waiting`]. + /// - a tick at/after the armed time: clear the entry and [`RestartTiming::Fire`]. + /// + /// Pulled out as a pure function (no `ApiState`) so the backoff scheduling is + /// unit-testable; the live ticker's resolution bounds how promptly `Armed` + /// transitions to `Fire`. + fn restart_timing( + schedule: &mut std::collections::HashMap, + name: &str, + backoff: Duration, + now: tokio::time::Instant, + ) -> RestartTiming { + if backoff <= MIN_RESTART_DELAY { + schedule.remove(name); + return RestartTiming::Fire; + } + match schedule.get(name).copied() { + Some(due) if now < due => RestartTiming::Waiting, + Some(_) => { + schedule.remove(name); + RestartTiming::Fire + } + None => { + schedule.insert(name.to_string(), now + backoff); + RestartTiming::Armed + } + } + } + + /// Attempt to restart a machine. + async fn restart_machine(&self, name: &str) -> crate::Result<()> { + // Hold the per-machine lifecycle lock across the acquire+mount+launch so a + // concurrent user-initiated stop/delete cannot detach the macOS layers + // volume out from under this restart (review finding #3). Acquired before + // get_machine and the entry-mutex lock taken inside the spawn_blocking + // below, keeping the lifecycle → entry lock order that start/stop/delete + // also follow. If a user op holds it, this restart blocks until that op + // completes (operations are bounded), then re-derives state from the DB. + // Linux: the guarded mount is a no-op. + let lifecycle = self.state.lifecycle_lock(name); + let _guard = lifecycle.lock().await; + + // Re-check liveness now that we hold the lock. We may have queued this + // restart while the machine was mid-boot (an API start or an earlier + // restart held the lock); by the time we acquire it the boot may have + // completed, so re-launching would kill+reboot a healthy machine and — + // under concurrent boots — double the disk-prep load. If it's alive + // again, there's nothing to restart. + if self.state.is_machine_alive(name) { + tracing::debug!(machine = %name, "machine came back alive before restart; skipping"); + return Ok(()); + } + + let entry = match self.state.get_machine(name) { + Ok(entry) => entry, + Err(_) => { + tracing::warn!(machine = %name, "machine no longer exists, skipping restart"); + return Ok(()); + } + }; + + // Load the authoritative config from the database record rather than + // the in-memory MachineEntry, which may have lost fields (e.g., + // network_backend, gpu_vram_mib) during the ResourceSpec round-trip. + let record = match self.state.db().get_vm(name) { + Ok(Some(r)) => r, + Ok(None) => { + tracing::warn!(machine = %name, "machine not found in database, skipping restart"); + return Ok(()); + } + Err(e) => { + tracing::warn!(machine = %name, error = %e, "failed to read machine record, skipping restart"); + return Ok(()); + } + }; + + // Never auto-restart a fork base: clones CoW-read its disks by path, so + // relaunching it writable would corrupt them. Clones keep working + // without the base process, so skipping is safe (and avoids thrashing, + // since `prepare_for_launch` would refuse the launch anyway). + match self.state.db().dependent_clones(name) { + Ok(clones) if !clones.is_empty() => { + tracing::warn!( + machine = %name, + clones = %clones.join(", "), + "not auto-restarting: machine is a fork base with live clones" + ); + return Ok(()); + } + Ok(_) => {} + Err(e) => { + tracing::warn!(machine = %name, error = %e, "could not check dependent clones; proceeding") + } + } + + let mounts = record.host_mounts(); + let ports = record.port_mappings(); + let resources = record.vm_resources(); + let source_smolmachine = record.source_smolmachine.clone(); + let dns_filter_hosts = record.dns_filter_hosts.clone(); + let name_for_features = name.to_string(); + + let entry_clone = entry.clone(); + let start_result = tokio::task::spawn_blocking(move || { + let entry = entry_clone.lock(); + // Wire pre-extracted layers if this machine was created from a .smolmachine. + let features = crate::api::state::build_launch_features( + Some(&name_for_features), + source_smolmachine.as_deref(), + dns_filter_hosts, + )?; + entry + .manager + .ensure_running_via_subprocess(mounts, ports, resources, features) + }) + .await + .map_err(|e| crate::Error::agent("ensure running", e.to_string()))?; + + // Handle start result + match start_result { + Ok(_) => { + // Get updated PID and persist state + let pid = { + let entry = entry.lock(); + entry.manager.child_pid() + }; + if let Err(e) = self + .state + .update_machine_state(name, RecordState::Running, pid) + { + tracing::warn!(machine = %name, error = %e, "failed to persist running state"); + } + tracing::info!(machine = %name, pid = ?pid, "machine restarted successfully"); + Ok(()) + } + Err(e) => { + if let Err(db_err) = + self.state + .update_machine_state(name, RecordState::Failed, None) + { + tracing::warn!(machine = %name, error = %db_err, "failed to persist failed state"); + } + tracing::error!(machine = %name, error = %e, "failed to restart machine"); + Err(e) + } + } + } + + /// Rotate logs for all machines if they exceed the size limit. + async fn rotate_logs_if_needed(&self) { + let machine_names = self.state.list_machine_names(); + + for name in machine_names { + if let Some(log_path) = self.get_machine_log_path(&name) { + if let Err(e) = crate::log_rotation::rotate_if_needed(&log_path) { + tracing::debug!(machine = %name, error = %e, "failed to rotate logs"); + } + } + } + } + + /// Get the console log path for a machine. + /// + /// Resolves to the VM's hash-derived data directory — the canonical + /// layout used by `AgentManager::new_internal` and exposed via + /// `vm_data_dir` / the `machine data-dir` CLI command. + fn get_machine_log_path(&self, name: &str) -> Option { + if crate::data::validate_vm_name(name, "machine name").is_err() { + tracing::warn!(machine = %name, "skipping invalid machine name when resolving log path"); + return None; + } + + let log_path = crate::agent::vm_data_dir(name).join("agent-console.log"); + if log_path.exists() { + Some(log_path) + } else { + None + } + } +} + +// Tests for should_restart and backoff_duration live in src/config.rs +// since the logic now lives on RestartConfig directly. + +#[cfg(test)] +mod restart_timing_tests { + use super::{RestartTiming, Supervisor, MIN_RESTART_DELAY}; + use std::collections::HashMap; + use std::time::Duration; + use tokio::time::Instant; + + // A backoff at or below the minimum restarts immediately and schedules nothing. + #[test] + fn fast_path_fires_immediately() { + let mut sched = HashMap::new(); + let now = Instant::now(); + assert_eq!( + Supervisor::restart_timing(&mut sched, "m", MIN_RESTART_DELAY, now), + RestartTiming::Fire + ); + assert!(sched.is_empty(), "fast-path must not arm a timer"); + } + + // A longer backoff arms on the first dead tick and waits on the next, but does + // NOT fire — proving one crash-looping machine never blocks the loop here. + #[test] + fn long_backoff_arms_then_waits_then_fires() { + let mut sched = HashMap::new(); + let backoff = Duration::from_secs(8); + let t0 = Instant::now(); + + // First dead tick: arm, don't fire. + assert_eq!( + Supervisor::restart_timing(&mut sched, "m", backoff, t0), + RestartTiming::Armed + ); + assert!(sched.contains_key("m")); + + // A later tick still before the due time: keep waiting, schedule intact. + let before_due = t0 + Duration::from_secs(5); + assert_eq!( + Supervisor::restart_timing(&mut sched, "m", backoff, before_due), + RestartTiming::Waiting + ); + assert!(sched.contains_key("m")); + + // A tick at/after the due time: fire and clear the schedule. + let after_due = t0 + Duration::from_secs(9); + assert_eq!( + Supervisor::restart_timing(&mut sched, "m", backoff, after_due), + RestartTiming::Fire + ); + assert!( + !sched.contains_key("m"), + "firing must clear the schedule so a later death re-arms" + ); + } + + // Independent machines don't interfere: a long-backoff machine waiting must + // not stop a fast-path machine from firing on the same tick. + #[test] + fn machines_are_scheduled_independently() { + let mut sched = HashMap::new(); + let now = Instant::now(); + // Arm "slow" with a long backoff. + assert_eq!( + Supervisor::restart_timing(&mut sched, "slow", Duration::from_secs(30), now), + RestartTiming::Armed + ); + // "fast" still fires immediately despite "slow" being parked. + assert_eq!( + Supervisor::restart_timing(&mut sched, "fast", MIN_RESTART_DELAY, now), + RestartTiming::Fire + ); + // "slow" is still parked, not due. + assert_eq!( + Supervisor::restart_timing(&mut sched, "slow", Duration::from_secs(30), now), + RestartTiming::Waiting + ); + } +} diff --git a/src/api/types.rs b/src/api/types.rs new file mode 100644 index 0000000..c5afac7 --- /dev/null +++ b/src/api/types.rs @@ -0,0 +1,697 @@ +//! JSON request and response types for the API. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use utoipa::ToSchema; + +/// Map of guest-side env var names to secret refs. +/// +/// Present on every exec-like endpoint and on `CreateMachineRequest`. +/// Every entry is validated under `ResolutionScope::Untrusted` before +/// it's acted on — the HTTP API treats every caller as untrusted +/// regardless of where the server is bound, so no ref source kind is +/// accepted — `from_env` and `from_file` are both rejected with 400. +/// Configure secrets locally via the CLI instead. +/// +/// Capped at `MAX_REQ_SECRETS_PER_REQUEST` entries per request. +pub type RequestSecretRefs = BTreeMap; + +// ============================================================================ +// Machine Types +// ============================================================================ + +/// Restart policy specification for machine creation. +#[derive(Debug, Clone, Deserialize, Serialize, Default, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct RestartSpec { + /// Restart policy: "never", "always", "on-failure", "unless-stopped". + #[serde(default)] + pub policy: Option, + /// Maximum restart attempts (0 = unlimited). + #[serde(default)] + pub max_retries: Option, +} + +/// Mount specification (for requests). +#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)] +pub struct MountSpec { + /// Host path to mount. + #[schema(example = "/Users/me/code")] + pub source: String, + /// Path inside the machine. + #[schema(example = "/workspace")] + pub target: String, + /// Read-only mount. + #[serde(default)] + pub readonly: bool, +} + +/// Mount information (for responses, includes virtiofs tag). +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct MountInfo { + /// Virtiofs tag (e.g., "smolvm0"). Use this in container mounts. + #[schema(example = "smolvm0")] + pub tag: String, + /// Host path. + #[schema(example = "/Users/me/code")] + pub source: String, + /// Path inside the machine. + #[schema(example = "/workspace")] + pub target: String, + /// Read-only mount. + pub readonly: bool, +} + +/// Port mapping specification. +#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)] +pub struct PortSpec { + /// Port on the host. + #[schema(example = 8080)] + pub host: u16, + /// Port inside the machine. + #[schema(example = 80)] + pub guest: u16, +} + +/// VM resource specification. +#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ResourceSpec { + /// Number of vCPUs. + #[serde(default)] + #[schema(example = 2)] + pub cpus: Option, + /// Memory in MiB. + #[serde(default)] + #[schema(example = 1024)] + pub memory_mb: Option, + /// Enable outbound network access (TSI). + /// Note: Only TCP/UDP supported, not ICMP (ping). + #[serde(default)] + pub network: Option, + /// Enable GPU acceleration (Vulkan via virtio-gpu). + #[serde(default)] + pub gpu: Option, + /// Storage disk size in GiB (default: 20). + #[serde(default)] + #[schema(example = 20)] + pub storage_gb: Option, + /// Overlay disk size in GiB (default: 10). + #[serde(default)] + #[schema(example = 10)] + pub overlay_gb: Option, + /// Allowed egress CIDR ranges. When set, only these IP ranges are reachable. + /// Omit for unrestricted egress. Empty list denies all egress. + #[serde(default)] + pub allowed_cidrs: Option>, + /// Allowed egress hostnames. When set, DNS answers for these names (and their + /// subdomains) are learned into the egress allow-list so the machine can reach + /// them by name. Combine with `allowed_cidrs` to also permit fixed ranges. + #[serde(default)] + pub allowed_hosts: Option>, + /// Network backend: `tsi` (default, outbound-only) or `virtio-net` + /// (required for published `ports`). Omit for the default (TSI). + #[serde(default)] + pub network_backend: Option, +} + +// ============================================================================ +// Exec Types +// ============================================================================ + +/// Request to execute a command in a machine. +#[derive(Debug, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ExecRequest { + /// Command and arguments. + #[schema(example = json!(["echo", "hello"]))] + pub command: Vec, + /// Environment variables. + #[serde(default)] + pub env: Vec, + /// Ad-hoc secret refs. Rejected unless empty: an untrusted HTTP + /// caller cannot read this host's env/files. See `RequestSecretRefs`. + #[serde(default)] + #[schema(value_type = Object)] + pub secrets: RequestSecretRefs, + /// Working directory. + #[serde(default)] + #[schema(example = "/workspace")] + pub workdir: Option, + /// Timeout in seconds. + #[serde(default)] + #[schema(example = 30)] + pub timeout_secs: Option, + /// Data to pipe to the command's stdin. + #[serde(default)] + pub stdin: Option, + /// Run the command detached: spawn it in the background and return its PID + /// immediately instead of waiting. The process keeps running (a long-lived + /// daemon — dev server, agent runner) until it exits or the machine stops. + #[serde(default)] + pub background: bool, +} + +/// Environment variable. +#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)] +pub struct EnvVar { + /// Variable name. + #[schema(example = "MY_VAR")] + pub name: String, + /// Variable value. + #[schema(example = "my_value")] + pub value: String, +} + +impl EnvVar { + /// Convert a slice of EnvVar to (name, value) tuples for the agent protocol. + pub fn to_tuples(env: &[EnvVar]) -> Vec<(String, String)> { + env.iter() + .map(|e| (e.name.clone(), e.value.clone())) + .collect() + } +} + +/// Command execution result. +/// +/// **Encoding note**: `stdout`/`stderr` are a lossy UTF-8 view (non-UTF-8 bytes +/// become U+FFFD) kept for older clients. `stdoutB64`/`stderrB64` carry the raw, +/// byte-exact output (base64) and should be preferred by callers that need +/// binary-safe results (image bytes, tarballs, etc.) — the agent preserves +/// bytes end-to-end and these fields do too. +#[derive(Debug, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ExecResponse { + /// Exit code. + #[schema(example = 0)] + pub exit_code: i32, + /// Standard output, lossy UTF-8 (non-UTF-8 bytes → U+FFFD). Prefer `stdoutB64`. + #[schema(example = "hello\n")] + pub stdout: String, + /// Standard error, lossy UTF-8 (non-UTF-8 bytes → U+FFFD). Prefer `stderrB64`. + #[schema(example = "")] + pub stderr: String, + /// Raw stdout bytes, base64-encoded — byte-exact, binary-safe. + #[serde(with = "smolvm_protocol::base64_bytes")] + #[schema(value_type = String)] + pub stdout_b64: Vec, + /// Raw stderr bytes, base64-encoded — byte-exact, binary-safe. + #[serde(with = "smolvm_protocol::base64_bytes")] + #[schema(value_type = String)] + pub stderr_b64: Vec, +} + +/// Request to export a stopped machine to a `.smolmachine` and push it to a +/// registry. The control plane mints a pre-scoped OCI bearer (`push_token`) +/// that authorizes the write against `reference_host`. +#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ExportRequest { + /// Repository to push into (e.g. `tenant/my-machine`). + #[schema(example = "tenant/my-machine")] + pub repo: String, + /// Tag to push under (e.g. `latest`). + #[schema(example = "latest")] + pub tag: String, + /// Pre-scoped OCI bearer token minted by the control plane. + pub push_token: String, + /// Registry host to push to (e.g. `registry.smolmachines.com`). + #[schema(example = "registry.smolmachines.com")] + pub reference_host: String, +} + +/// Result of exporting a machine to a registry. +#[derive(Debug, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ExportResponse { + /// Digest of the pushed OCI manifest (reference as `repo@`). + #[schema(example = "sha256:abc123")] + pub digest: String, + /// Size of the `.smolmachine` sidecar blob in bytes. + #[schema(example = 104857600)] + pub size_bytes: u64, + /// Host platform the artifact targets (e.g. `linux/amd64`). + #[schema(example = "linux/amd64")] + pub platform: String, + /// The `PackManifest` JSON carried in the sidecar footer. + pub manifest: String, +} + +/// Request to run a command in an image. +#[derive(Debug, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct RunRequest { + /// Image to run in. + #[schema(example = "python:3.12-alpine")] + pub image: String, + /// Command and arguments. + #[schema(example = json!(["python", "-c", "print('hello')"]))] + pub command: Vec, + /// Environment variables. + #[serde(default)] + pub env: Vec, + /// Ad-hoc secret refs. Rejected unless empty (untrusted scope). + #[serde(default)] + #[schema(value_type = Object)] + pub secrets: RequestSecretRefs, + /// Working directory. + #[serde(default)] + pub workdir: Option, + /// Timeout in seconds. + #[serde(default)] + pub timeout_secs: Option, +} + +// ============================================================================ +// Image Types +// ============================================================================ + +/// Image information. +#[derive(Debug, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ImageInfo { + /// Image reference. + #[schema(example = "alpine:latest")] + pub reference: String, + /// Image digest. + #[schema(example = "sha256:abc123...")] + pub digest: String, + /// Size in bytes. + #[schema(example = 7500000)] + pub size: u64, + /// Architecture. + #[schema(example = "arm64")] + pub architecture: String, + /// OS. + #[schema(example = "linux")] + pub os: String, + /// Number of layers. + #[schema(example = 3)] + pub layer_count: usize, +} + +/// List images response. +#[derive(Debug, Serialize, ToSchema)] +pub struct ListImagesResponse { + /// List of images. + pub images: Vec, +} + +/// Request to pull an image. +#[derive(Debug, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct PullImageRequest { + /// Image reference. + #[schema(example = "python:3.12-alpine")] + pub image: String, + /// OCI platform for multi-arch images (e.g., "linux/arm64"). + #[serde(default)] + #[schema(example = "linux/arm64")] + pub oci_platform: Option, + /// Proxy URL applied to the in-VM registry client + /// (sets HTTP_PROXY and HTTPS_PROXY). + #[serde(default)] + #[schema(example = "http://192.168.127.254:3128")] + pub proxy: Option, + /// Comma-separated NO_PROXY list of hosts/CIDRs that bypass the proxy. + #[serde(default)] + #[schema(example = "127.0.0.1,localhost,.internal")] + pub no_proxy: Option, +} + +/// Pull image response. +#[derive(Debug, Serialize, ToSchema)] +pub struct PullImageResponse { + /// Information about the pulled image. + pub image: ImageInfo, +} + +// ============================================================================ +// Logs Types +// ============================================================================ + +/// Query parameters for logs endpoint. +#[derive(Debug, Deserialize, ToSchema)] +pub struct LogsQuery { + /// If true, follow the logs (like tail -f). Default: false. + #[serde(default)] + pub follow: bool, + /// Number of lines to show from the end (like tail -n). Default: all. + #[serde(default)] + #[schema(example = 100)] + pub tail: Option, + /// Output format: "raw" (default) or "json" (only emit valid JSON lines). + #[serde(default)] + pub format: Option, +} + +// ============================================================================ +// Delete Types +// ============================================================================ + +/// Query parameters for delete machine endpoint. +#[derive(Debug, Default, Deserialize, ToSchema)] +pub struct DeleteQuery { + /// If true, force delete even if stop fails and VM is still running. + /// This may orphan the VM process. Default: false. + #[serde(default)] + pub force: bool, +} + +// ============================================================================ +// Health Types +// ============================================================================ + +/// Health check response. +#[derive(Debug, Serialize, ToSchema)] +pub struct HealthResponse { + /// Health status (e.g., "ok"). + #[schema(example = "ok")] + pub status: &'static str, + /// Server version. + #[schema(example = "0.5.2")] + pub version: &'static str, + /// Machine counts. + #[serde(skip_serializing_if = "Option::is_none")] + pub machines: Option, + /// Server uptime in seconds. + #[serde(skip_serializing_if = "Option::is_none")] + pub uptime_seconds: Option, +} + +/// Machine counts for health response. +#[derive(Debug, Serialize, ToSchema)] +pub struct MachineCountsResponse { + /// Total machines in the database. + pub total: usize, + /// Currently running machines. + pub running: usize, +} + +/// Live node capacity: current allocations and real utilization across all +/// running machines on this host. Read-only introspection — a fleet control +/// plane (or any operator) polls this to gauge node load. The reporter owns +/// totals/reserved; this endpoint reports only what the runtime itself knows. +#[derive(Debug, Clone, Serialize, ToSchema)] +pub struct CapacityResponse { + /// CPUs allocated to running machines (sum of per-machine cpu requests). + pub allocated_cpus: u32, + /// Memory (MB) allocated to running machines. + pub allocated_memory_mb: u64, + /// Real fractional CPU load across VM processes (e.g. 2.5 = 2.5 CPUs). + pub used_cpus: f64, + /// Real resident memory (MB) across VM processes. + pub used_memory_mb: u64, + /// Real disk (GB) consumed by VM storage + overlay files. + pub used_disk_gb: u64, + /// Opaque id minted once per serve process. It changes iff the serve restarts + /// — the signal the control uses to detect that this node's warm pool (and any + /// in-memory VM state) was wiped, so it can prune the now-stale pool records. + #[serde(default)] + pub boot_id: String, +} + +// ============================================================================ +// Error Types +// ============================================================================ + +/// API error response. +#[derive(Debug, Serialize, ToSchema)] +pub struct ApiErrorResponse { + /// Error message. + #[schema(example = "machine 'test' not found")] + pub error: String, + /// Error code. + #[schema(example = "NOT_FOUND")] + pub code: String, +} + +// ============================================================================ +// Machine Types +// ============================================================================ + +/// Request to create a new machine. +#[derive(Debug, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct CreateMachineRequest { + /// Machine name. Auto-generated if omitted. + #[serde(default)] + #[schema(example = "my-vm")] + pub name: Option, + /// Number of vCPUs. + #[serde(default)] + #[schema(example = 4)] + pub cpus: Option, + /// Memory in MiB. + #[serde(default, rename = "memoryMb")] + #[schema(example = 8192)] + pub mem: Option, + /// Host mounts to attach. + #[serde(default)] + pub mounts: Vec, + /// Port mappings (host:guest). + #[serde(default)] + pub ports: Vec, + /// Enable outbound network access (TSI). + /// Note: Only TCP/UDP supported, not ICMP (ping). + #[serde(default)] + pub network: bool, + /// Enable GPU acceleration (Vulkan via virtio-gpu). + #[serde(default)] + pub gpu: bool, + /// Storage disk size in GiB (default: 20). + #[serde(default)] + pub storage_gb: Option, + /// Overlay disk size in GiB (default: 10). + #[serde(default)] + pub overlay_gb: Option, + /// Allowed egress CIDR ranges. + #[serde(default)] + pub allowed_cidrs: Option>, + /// Allowed egress hostnames (and their subdomains); DNS answers for these + /// names are learned into the egress allow-list. + #[serde(default)] + pub allowed_hosts: Option>, + /// Network backend: `tsi` (default, outbound-only) or `virtio-net`. + /// Published `ports` require `virtio-net` (TSI has no inbound path). + #[serde(default)] + pub network_backend: Option, + /// Restart policy configuration. + #[serde(default)] + pub restart: Option, + /// OCI image reference (e.g., "alpine:latest"). Mutually exclusive with `from`. + #[serde(default)] + pub image: Option, + /// Path to a .smolmachine sidecar file. Creates the machine from pre-packed + /// layers instead of pulling from a registry. Mutually exclusive with `image`. + #[serde(default)] + pub from: Option, + /// Registry reference to a .smolmachine artifact (e.g., "myapp:v1"). + /// Pulls from the registry before creating the VM. + /// Mutually exclusive with `image` and `from`. + #[serde(default)] + pub registry_ref: Option, + /// Bearer credential (an OCI Distribution `identity_token`) to present when + /// pulling `registry_ref`. The control plane supplies a short-lived, + /// tenant-scoped token here so a node can fetch a tenant's private + /// `.smolmachine`. Takes precedence over any persisted registry credential. + #[serde(default)] + pub registry_identity_token: Option, + /// Brokered P2P blob peers: node base URLs (`https://:`) supplied + /// by the control plane. On a cache miss the layer blob is fetched from a + /// peer's `GET /p2p/blob/` (over node→node mTLS) before the registry. + /// Empty (the default) ⇒ registry-only, byte-for-byte as before. + #[serde(default)] + pub blob_peers: Vec, + /// Secret refs attached to the machine. Resolved at every + /// subsequent exec against the host's env/files. Rejected unless empty; + /// accepted — `from_env`/`from_file` on the API surface would let + /// an untrusted caller exfiltrate the server process's env or + /// read arbitrary host files; use the CLI `machine create` path + /// for those source kinds. + #[serde(default)] + #[schema(value_type = Object)] + pub secrets: RequestSecretRefs, +} + +/// Request to execute a command in a machine. +#[derive(Debug, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct MachineExecRequest { + /// Command and arguments. + #[schema(example = json!(["echo", "hello"]))] + pub command: Vec, + /// Environment variables. + #[serde(default)] + pub env: Vec, + /// Ad-hoc secret refs. Rejected unless empty (untrusted scope). + #[serde(default)] + #[schema(value_type = Object)] + pub secrets: RequestSecretRefs, + /// Working directory. + #[serde(default)] + pub workdir: Option, + /// Timeout in seconds. + #[serde(default)] + pub timeout_secs: Option, + /// Data to pipe to the command's stdin. + #[serde(default)] + pub stdin: Option, +} + +/// Machine status information. +#[derive(Debug, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct MachineInfo { + /// Machine name. + #[schema(example = "my-vm")] + pub name: String, + /// Current state ("created", "running", "stopped"). + #[schema(example = "running")] + pub state: String, + /// Number of vCPUs. + #[schema(example = 2)] + pub cpus: u8, + /// Memory in MiB. + #[serde(rename = "memoryMb")] + #[schema(example = 1024)] + pub mem: u32, + /// Process ID (if running). + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(example = 12345)] + pub pid: Option, + /// Configured mounts (with virtiofs tags for container use). + pub mounts: Vec, + /// Configured port mappings. + pub ports: Vec, + /// Whether outbound network access is enabled. + pub network: bool, + /// Network backend the machine runs (`tsi` or `virtio-net`). Omitted when + /// unset (the default TSI). Echoes back what `create` accepted. + #[serde(skip_serializing_if = "Option::is_none")] + pub network_backend: Option, + /// Allowed egress CIDRs. Omitted when unrestricted; an empty list denies all. + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_cidrs: Option>, + /// Allowed egress hostnames. Omitted when unset. Echoes back what `create` accepted. + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_hosts: Option>, + /// Storage disk size in GiB. + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(example = 20)] + pub storage_gb: Option, + /// Overlay disk size in GiB. + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(example = 2)] + pub overlay_gb: Option, + /// Cumulative guest-outbound (egress) bytes since boot, for billing. Present + /// only for virtio-net machines that have reported a value; omitted for TSI + /// or machines that haven't flushed yet. Surfaced the same way `storage_gb` + /// is, so the control plane reads both from the machine list. + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(example = 1048576)] + pub egress_bytes: Option, + /// Consumed CPU-seconds (user+system) of the machine's CURRENT VMM process, + /// sampled live from the host. Resets to 0 on a VM restart — it's a stateless + /// snapshot; the control plane accumulates a durable total from it. Omitted + /// for stopped machines (no live process to sample). + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(example = 42)] + pub cpu_seconds: Option, + /// Same consumed CPU but in MILLISECONDS — sub-second precision so consumers + /// integrating this don't quantize a barely-busy process up to a whole second. + /// Derived from the same nanosecond sample as `cpu_seconds`. Omitted for + /// stopped machines (no live process to sample). + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(example = 42830)] + pub cpu_millis: Option, + /// Current resident memory (RSS) of the machine's VMM process, in MiB, sampled + /// live from the host. Unlike CPU this is an instantaneous gauge (not a + /// counter); the control plane integrates it over time for active-memory + /// billing. Omitted for stopped machines (no live process to sample). + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(example = 128)] + pub rss_mb: Option, + /// Actual host disk consumed by this machine's data dir, in MiB (real blocks of + /// the sparse disk images, not provisioned capacity). An instantaneous gauge the + /// control integrates over time for active-disk billing. Omitted when the data + /// dir can't be read. + #[serde(skip_serializing_if = "Option::is_none")] + #[schema(example = 256)] + pub disk_used_mb: Option, + /// Creation timestamp (seconds since Unix epoch). + pub created_at: u64, +} + +/// List machines response. +#[derive(Debug, Serialize, ToSchema)] +pub struct ListMachinesResponse { + /// List of machines. + pub machines: Vec, +} + +/// Generic delete response. +#[derive(Debug, Serialize, ToSchema)] +pub struct DeleteResponse { + /// Name of deleted resource. + #[schema(example = "my-machine")] + pub deleted: String, +} + +/// Generic start response. +#[derive(Debug, Serialize, ToSchema)] +pub struct StartResponse { + /// Identifier of started resource. + #[schema(example = "abc123")] + pub started: String, +} + +/// Generic stop response. +#[derive(Debug, Serialize, ToSchema)] +pub struct StopResponse { + /// Identifier of stopped resource. + #[schema(example = "abc123")] + pub stopped: String, +} + +// ============================================================================ +// Resize Types +// ============================================================================ + +/// Request to resize a machine's disk resources. +#[derive(Debug, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ResizeMachineRequest { + /// Storage disk size in GiB (expand only, optional). + #[serde(default)] + #[schema(example = 50)] + pub storage_gb: Option, + /// Overlay disk size in GiB (expand only, optional). + #[serde(default)] + #[schema(example = 20)] + pub overlay_gb: Option, +} + +/// Query string for `POST /machines/{name}/start`. +#[derive(Debug, Default, Deserialize, ToSchema)] +pub struct StartMachineQuery { + /// Start as a fork base: back the guest RAM with a memfd (copy-on-write + /// cloneable) and expose a control socket so the machine can later be forked + /// with `POST /machines/{name}/fork`. The golden freezes after its first fork. + #[serde(default)] + pub forkable: bool, +} + +/// Request to fork a running, forkable golden machine into a new clone. +#[derive(Debug, Clone, Deserialize, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ForkRequest { + /// Name for the new clone machine. + #[schema(example = "clone-1")] + pub name: String, + /// Pin the clone's inbound port forwards. Without this, the golden's + /// forwards are remapped to freshly-allocated host ports so the clone does + /// not collide with the still-running golden or sibling clones. + #[serde(default)] + pub ports: Vec, +} diff --git a/src/cli/cleanup_ephemeral.rs b/src/cli/cleanup_ephemeral.rs new file mode 100644 index 0000000..82bef24 --- /dev/null +++ b/src/cli/cleanup_ephemeral.rs @@ -0,0 +1,271 @@ +//! Detached cleanup helper for ephemeral `machine run` VMs. +//! +//! Invoked as `smolvm _cleanup-ephemeral ` +//! after the parent CLI has flushed output and is about to exit. Running out of +//! process lets the parent return the guest's exit code immediately while disk +//! removal and process teardown happen asynchronously. +//! +//! Safety guarantees: +//! - SIGKILL is sent only after strict PID+start_time verification (no fallback). +//! - Directory removal only happens after the VM process is confirmed dead. +//! - DB deregistration runs only after the directory is confirmed gone. +//! - All three steps are gated: a failure at any step leaves state for orphan sweep. + +use crate::cli::vm_common::deregister_ephemeral_vm; +use smolvm::agent::{vm_cache_root, vm_data_dir}; +use std::path::Path; + +/// Entry point for the `_cleanup-ephemeral` subcommand. +pub fn run(vm_name: &str, pid: i32, start_time: u64) { + let data_dir = vm_data_dir(vm_name); + let cache_root = vm_cache_root(); + // The ephemeral DB record is keyed by the VM's own name (see `RunCmd::run`), + // so deregistration and dir removal target the same name. + let record_name = vm_name.to_owned(); + run_inner( + pid, + start_time, + smolvm::process::kill_verified, + smolvm::process::is_alive, + move || { + // Validate the path is inside the smolvm cache root — never delete + // /, home, empty paths, or symlink-rooted paths. + if !is_safe_cache_path(&data_dir, &cache_root) { + return false; + } + if data_dir.is_dir() { + // Release this VM's per-VM uid before its `.vm-uid` record goes + // with the dir. `machine run` is the highest-churn path, so + // freeing it here keeps the uid registry small (the allocator + // self-heals stale entries too, but proactively freeing avoids + // buildup). + smolvm::process::free_vm_uid(&smolvm::agent::vm_uid_registry_dir(), &data_dir); + std::fs::remove_dir_all(&data_dir).is_ok() + } else { + !data_dir.exists() + } + }, + move || deregister_ephemeral_vm(&record_name), + ); +} + +/// Returns `true` only when `data_dir` is a safe path to delete: +/// non-empty, strictly inside `cache_root`, not equal to `cache_root`, +/// and not a symlink (prevents escaping via redirected root). +fn is_safe_cache_path(data_dir: &Path, cache_root: &Path) -> bool { + !data_dir.as_os_str().is_empty() + && data_dir.starts_with(cache_root) + && data_dir != cache_root + && !data_dir.is_symlink() +} + +/// Core cleanup logic with injectable side-effect functions. +/// +/// Separated from `run` to enable unit testing of the failure-ordering invariants +/// without needing a real process, filesystem, or database. +/// +/// - `remove_dir`: called when the process is confirmed dead; returns `true` on +/// success. The caller is responsible for path-safety checks inside this closure. +/// - `deregister`: called only after `remove_dir` succeeds; the caller binds the +/// ephemeral record name in the closure. +fn run_inner( + pid: i32, + start_time: u64, + kill_verified: FKill, + is_alive: FAlive, + remove_dir: FRemove, + deregister: FDeregister, +) where + FKill: Fn(i32, Option) -> bool, + FAlive: Fn(i32) -> bool, + FRemove: Fn() -> bool, + FDeregister: Fn(), +{ + let start_time_opt = if start_time > 0 { + Some(start_time) + } else { + None + }; + + // Send SIGKILL using strict verification: refuses to signal if start_time + // is missing or the PID has been reused by an unrelated process. + let kill_sent = kill_verified(pid, start_time_opt); + + let vm_confirmed_dead = if kill_sent { + // Poll until the process is gone. The VM is reparented to launchd/init + // after our parent exited, so try_wait returns ECHILD — use is_alive. + let mut dead = false; + for _ in 0..500 { + if !is_alive(pid) { + dead = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + dead + } else { + // kill_verified refused — either start_time was absent/mismatched (PID + // reuse) or the process is already dead. Safe to proceed only if dead. + !is_alive(pid) + }; + + if !vm_confirmed_dead { + // VM is still alive (SIGKILL timed out) or a different process now holds + // the PID. Leave the data directory and DB record intact so the orphan + // sweep can retry. + return; + } + + // Remove the data directory. The caller's closure handles path-safety + // validation. Failure here leaves the DB record for the orphan sweep. + if !remove_dir() { + return; + } + + // Deregister the ephemeral DB record. Done last so cleanup_orphaned_ephemeral_vms() + // can find and recover this record if the helper is killed before reaching here. + deregister(); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + use std::path::PathBuf; + + /// Build a safe (cache_root, data_dir) pair inside a real temp directory. + fn fake_dirs() -> (tempfile::TempDir, PathBuf) { + let cache_root = tempfile::TempDir::new().unwrap(); + let data_dir = cache_root.path().join("vm-abc123"); + (cache_root, data_dir) + } + + /// start_time = 0 → kill is skipped; proceeds if the process is already dead. + #[test] + fn missing_start_time_skips_kill_and_cleans_up_dead_process() { + let (_cache_tmp, data_dir) = fake_dirs(); + std::fs::create_dir(&data_dir).unwrap(); + let kill_called = Cell::new(false); + let deregistered = Cell::new(false); + + run_inner( + 9999, + 0, // start_time = 0 → start_time_opt = None + |_pid, st| { + assert_eq!(st, None, "no start_time → kill_verified must receive None"); + kill_called.set(true); + false + }, + |_pid| false, // process already dead + || std::fs::remove_dir_all(&data_dir).is_ok(), + || deregistered.set(true), + ); + + assert!(kill_called.get(), "kill_verified should still be called"); + assert!(!data_dir.exists(), "data dir should be removed"); + assert!( + deregistered.get(), + "ephemeral record should be deregistered" + ); + } + + /// PID reuse: start_time present but mismatched — kill refused, process alive → abort. + /// + /// Invariant: when a different process now holds the PID, neither the data + /// directory nor the DB record should be touched. + #[test] + fn reused_pid_refuses_all_cleanup_when_process_alive() { + let (_cache_tmp, data_dir) = fake_dirs(); + std::fs::create_dir(&data_dir).unwrap(); + let remove_called = Cell::new(false); + let deregistered = Cell::new(false); + + run_inner( + 9999, + 12345, + |_pid, _st| false, // kill_verified refuses (start_time mismatch / PID reused) + |_pid| true, // is_alive: a different process holds this PID + || { + remove_called.set(true); + false + }, + || deregistered.set(true), + ); + + assert!(data_dir.exists(), "data dir must be preserved on PID reuse"); + assert!(!remove_called.get(), "remove must not be called"); + assert!(!deregistered.get(), "DB record must be preserved"); + } + + /// Failed directory removal must prevent DB deregistration. + /// + /// This is the critical ordering invariant: the orphan sweep relies on the + /// DB record surviving so it can retry cleanup on the next startup. + #[test] + fn failed_dir_removal_preserves_db_record() { + let deregistered = Cell::new(false); + + run_inner( + 9999, + 12345, + |_pid, _st| true, // kill_verified succeeds + |_pid| false, // process confirmed dead after kill + || false, // remove_dir fails (e.g. permission error, busy mount) + || deregistered.set(true), + ); + + assert!( + !deregistered.get(), + "DB record must NOT be removed when directory deletion fails" + ); + } + + /// Already-dead PID: kill refused because process is gone; cleanup proceeds normally. + #[test] + fn already_dead_pid_proceeds_to_full_cleanup() { + let (_cache_tmp, data_dir) = fake_dirs(); + std::fs::create_dir(&data_dir).unwrap(); + let deregistered = Cell::new(false); + + run_inner( + 9999, + 12345, + |_pid, _st| false, // kill_verified: process already dead, refuses to signal + |_pid| false, // is_alive: confirmed dead + || std::fs::remove_dir_all(&data_dir).is_ok(), + || deregistered.set(true), + ); + + assert!(!data_dir.exists(), "data dir should be removed"); + assert!( + deregistered.get(), + "ephemeral record should be deregistered" + ); + } + + /// Path safety check: only paths strictly inside cache_root are safe. + #[test] + fn safe_cache_path_checks() { + let cache = tempfile::TempDir::new().unwrap(); + let data_dir = cache.path().join("vm-abc123"); + assert!( + is_safe_cache_path(&data_dir, cache.path()), + "child of cache root is safe" + ); + assert!( + !is_safe_cache_path(cache.path(), cache.path()), + "cache root itself is not safe" + ); + assert!( + !is_safe_cache_path(Path::new(""), cache.path()), + "empty path is not safe" + ); + + let other = tempfile::TempDir::new().unwrap(); + let outside = other.path().join("vm-abc123"); + assert!( + !is_safe_cache_path(&outside, cache.path()), + "path outside cache root is not safe" + ); + } +} diff --git a/src/cli/config.rs b/src/cli/config.rs new file mode 100644 index 0000000..65e17c9 --- /dev/null +++ b/src/cli/config.rs @@ -0,0 +1,294 @@ +//! Configuration CLI commands. +//! +//! Commands for managing smolvm configuration, including registry settings. + +use clap::{Args, Subcommand}; +use smolvm::settings::SmolSettings; +use smolvm::Result; + +/// Configuration commands +#[derive(Subcommand, Debug)] +pub enum ConfigCmd { + /// Show current configuration + Show(ShowCmd), + + /// Manage registry configuration + #[command(subcommand)] + Registries(RegistriesCmd), +} + +impl ConfigCmd { + pub fn run(self) -> Result<()> { + match self { + ConfigCmd::Show(cmd) => cmd.run(), + ConfigCmd::Registries(cmd) => cmd.run(), + } + } +} + +// ============================================================================ +// Show Command +// ============================================================================ + +/// Show current configuration +#[derive(Args, Debug)] +pub struct ShowCmd {} + +impl ShowCmd { + pub fn run(self) -> Result<()> { + // Load and display global config + let config = smolvm::SmolvmConfig::load()?; + println!("Global Configuration:"); + println!(" Default CPUs: {}", config.default_cpus); + println!(" Default Memory: {} MiB", config.default_mem); + println!(" Default DNS: {}", config.default_dns); + + // Load and display unified settings + let settings = SmolSettings::load().unwrap_or_default(); + println!(); + println!("Settings:"); + if let Ok(path) = SmolSettings::config_path() { + println!(" Config file: {}", path.display()); + if path.exists() { + println!(" Status: configured"); + } else { + println!(" Status: not configured (using defaults)"); + } + } + + println!(); + println!(" [machines] (smolmachine artifact registries):"); + println!(" Default: {}", settings.machines.default_registry()); + println!(" Configured: {}", settings.machines.registries.len()); + for (name, entry) in &settings.machines.registries { + let auth_status = format_auth_status(entry); + println!(" {}: {}", name, auth_status); + } + + println!(); + println!(" [images] (container image registries):"); + println!(" Default: {}", settings.images.default_registry()); + println!(" Configured: {}", settings.images.registries.len()); + for (name, entry) in &settings.images.registries { + let auth_status = format_auth_status(entry); + let mirror_status = entry + .mirror + .as_ref() + .map(|m| format!(" -> {}", m)) + .unwrap_or_default(); + println!(" {}: {}{}", name, auth_status, mirror_status); + } + + Ok(()) + } +} + +fn format_auth_status(entry: &smolvm::registry::RegistryEntry) -> &'static str { + if entry.username.is_some() { + if entry.password_env.is_some() { + "auth (env var)" + } else if entry.password.is_some() { + "auth (direct)" + } else { + "no password" + } + } else { + "no auth" + } +} + +// ============================================================================ +// Registries Commands +// ============================================================================ + +/// Registry configuration commands +#[derive(Subcommand, Debug)] +pub enum RegistriesCmd { + /// Show the path to the registries configuration file + Path, + + /// Edit the registries configuration file in your default editor + Edit, + + /// Show current registries configuration + Show, + + /// Create an example configuration file + Init, +} + +impl RegistriesCmd { + pub fn run(self) -> Result<()> { + match self { + RegistriesCmd::Path => { + let path = SmolSettings::config_path()?; + println!("{}", path.display()); + Ok(()) + } + RegistriesCmd::Edit => { + let path = SmolSettings::config_path()?; + + // Create parent directory if needed + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + + // Create file with example content if it doesn't exist + if !path.exists() { + std::fs::write(&path, EXAMPLE_CONFIG)?; + println!("Created example configuration at {}", path.display()); + } + + // Open in editor + let editor = std::env::var("EDITOR").unwrap_or_else(|_| "vi".to_string()); + let status = std::process::Command::new(&editor).arg(&path).status()?; + + if !status.success() { + return Err(smolvm::Error::config( + "edit config", + format!("editor '{}' exited with non-zero status", editor), + )); + } + + // Validate the config after editing + match SmolSettings::load() { + Ok(settings) => { + println!( + "Configuration valid: {} machine registries, {} image registries configured", + settings.machines.registries.len(), + settings.images.registries.len(), + ); + } + Err(e) => { + eprintln!("Warning: Configuration file has errors: {}", e); + } + } + + Ok(()) + } + RegistriesCmd::Show => { + let settings = SmolSettings::load().unwrap_or_default(); + let has_machines = !settings.machines.registries.is_empty(); + let has_images = !settings.images.registries.is_empty(); + + if !has_machines && !has_images { + println!("No registries configured."); + if let Ok(path) = SmolSettings::config_path() { + println!(); + println!("To configure registries, create: {}", path.display()); + println!("Or run: smolvm config registries init"); + } + return Ok(()); + } + + if has_machines { + println!("[machines] (smolmachine artifact registries):"); + println!(); + print_registry_entries(&settings.machines.registries); + } + + if has_images { + if has_machines { + println!(); + } + println!("[images] (container image registries):"); + println!(); + print_registry_entries(&settings.images.registries); + } + + Ok(()) + } + RegistriesCmd::Init => { + let path = SmolSettings::config_path()?; + + if path.exists() { + eprintln!("Configuration file already exists at {}", path.display()); + eprintln!("Use 'smolvm config registries edit' to modify it."); + return Ok(()); + } + + // Create parent directory if needed + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + + std::fs::write(&path, EXAMPLE_CONFIG)?; + println!("Created example configuration at {}", path.display()); + println!(); + println!("Edit the file to add your registry credentials."); + println!("Run 'smolvm config registries edit' to open in your editor."); + + Ok(()) + } + } + } +} + +fn print_registry_entries( + registries: &std::collections::HashMap, +) { + for (name, entry) in registries { + println!(" [{}]", name); + + if let Some(ref username) = entry.username { + println!(" username: {}", username); + } + + if let Some(ref password_env) = entry.password_env { + let status = if std::env::var(password_env).is_ok() { + " (set)" + } else { + " (NOT SET)" + }; + println!(" password_env: {}{}", password_env, status); + } else if entry.password.is_some() { + println!(" password: "); + } + + if let Some(ref mirror) = entry.mirror { + println!(" mirror: {}", mirror); + } + + println!(); + } +} + +/// Example configuration file content +const EXAMPLE_CONFIG: &str = r#"# smolvm Configuration +# +# Location: ~/.config/smolvm/config.toml +# +# [cloud] — smolfleet API settings +# [machines] — credentials for .smolmachine artifact registries +# [images] — credentials for container image registries (base images for VMs) +# +# For security, use password_env to reference environment variables +# instead of storing passwords directly in this file. + +# [cloud] +# endpoint = "https://api.smolmachines.com" +# api_key = "smk_..." + +# [machines.defaults] +# registry = "registry.smolmachines.com" + +# [machines.registries."registry.smolmachines.com"] +# username = "token" +# password = "your-jwt-token" + +# [images.defaults] +# registry = "docker.io" + +# [images.registries."docker.io"] +# username = "your-username" +# password_env = "DOCKER_HUB_TOKEN" # Set: export DOCKER_HUB_TOKEN="your-token" + +# [images.registries."ghcr.io"] +# username = "your-github-username" +# password_env = "GHCR_TOKEN" # Set: export GHCR_TOKEN="your-pat" + +# [images.registries."registry.example.com"] +# username = "user" +# password_env = "REGISTRY_PASSWORD" +# mirror = "mirror.example.com" # Optional: pull from mirror instead +"#; diff --git a/src/cli/internal_boot.rs b/src/cli/internal_boot.rs new file mode 100644 index 0000000..c736b02 --- /dev/null +++ b/src/cli/internal_boot.rs @@ -0,0 +1,535 @@ +//! Internal boot subprocess for the API server. +//! +//! This command is NOT for direct user invocation. It's spawned by the API +//! server to launch a VM in a fresh single-threaded process, avoiding the +//! macOS fork-in-multithreaded-process issue. +//! +//! Usage: smolvm _boot-vm + +use smolvm::agent::boot_config::BootConfig; +use smolvm::agent::{launch_agent_vm, LaunchConfig, VmDisks}; +use smolvm::data::disk::{DiskFormat, DiskType}; +use smolvm::storage::VmDisk; +use std::path::{Path, PathBuf}; + +/// Open a boot disk honoring its on-disk format. A fork clone's disk path ends +/// in `.qcow2` (a copy-on-write overlay, opened as-is over its backing image); +/// every other disk is a raw image that may need creating/formatting. The path +/// is the single source of truth for the format — see `agent::resolve_disk_image`. +fn open_boot_disk(path: &Path, size_gb: u64) -> smolvm::Result> { + if path.extension().and_then(|e| e.to_str()) == Some("qcow2") { + VmDisk::::open_existing_with_format(path, DiskFormat::Qcow2) + } else { + VmDisk::::open_or_create_at(path, size_gb) + } +} + +/// Run the boot subprocess. +/// +/// Reads the boot config from the given path, sets up libkrun, and calls +/// `krun_start_enter` which blocks forever (or until the VM exits). +pub fn run(config_path: PathBuf) -> smolvm::Result<()> { + let t_proc = std::time::Instant::now(); + + // --- Parent-death watchdog --------------------------------------------- + // This VMM (`smol-vmm _boot-vm`) is always spawned by an embedding parent: + // the API server, a fleet node, or — for the SDKs — the Node/Python + // in-process runtime. If that parent dies *abnormally* (panic, uncaught + // exception, `process.exit`/`os._exit`, crash, or SIGKILL) it never runs + // teardown, so without this watchdog the VMM would be reparented to init and + // keep running forever, holding the VM's full RAM — an orphaned-process leak. + // + // Detection is by reparenting: when the real parent dies the kernel reparents + // this process (to init/launchd, or a subreaper), so `getppid()` changes from + // the value captured here at startup. The watcher polls for that change and + // exits; the OS then tears the VM down with us. Polling (rather than an + // inherited-pipe EOF) needs no fd plumbing, survives the + // `close_inherited_fds_from(3)` call below, and uses only syscalls already in + // the seccomp allowlist (`getppid`, `nanosleep`). Catches every death mode, + // including SIGKILL, which no parent-side exit handler can. Started first so + // it also covers a parent dying mid-boot. + // + // Armed only when an in-process embedder (the SDK) owns the VM's lifetime — + // the manager signals this via SMOLVM_BOOT_WATCH_PARENT=1. The CLI detaches + // its VM on purpose and `serve` reconnects to surviving VMs, so for those the + // parent exiting is normal and the watchdog stays off. + // The getppid()-based reparenting check is POSIX-specific; on Windows the + // parent-death watchdog is not wired up (the SDK in-process embedder path is + // a Unix concern here). + #[cfg(unix)] + if std::env::var_os("SMOLVM_BOOT_WATCH_PARENT").as_deref() == Some(std::ffi::OsStr::new("1")) { + let original_ppid = unsafe { libc::getppid() }; + let _ = std::thread::Builder::new() + .name("parent-death-watch".into()) + .spawn(move || loop { + std::thread::sleep(std::time::Duration::from_millis(500)); + if unsafe { libc::getppid() } != original_ppid { + smolvm::process::exit_child(0); + } + }); + } + + // Read boot config + let config_data = std::fs::read(&config_path) + .map_err(|e| smolvm::Error::agent("read boot config", e.to_string()))?; + let config: BootConfig = serde_json::from_slice(&config_data) + .map_err(|e| smolvm::Error::agent("parse boot config", e.to_string()))?; + + // Clean up the config file — it's no longer needed + let _ = std::fs::remove_file(&config_path); + + // Redirect stdio. When SMOLVM_GPU_DEBUG=1, keep stderr pointed at a + // debug log file so virglrenderer/MoltenVK errors are captured. + // The GPU-debug stdio redirection uses POSIX fd dup2 and is part of the + // (Unix-only) GPU path; on Windows it falls through to the portable stderr + // log redirection below. + #[cfg(unix)] + let gpu_debug = std::env::var_os("SMOLVM_GPU_DEBUG").is_some(); + #[cfg(not(unix))] + let gpu_debug = false; + if gpu_debug { + #[cfg(unix)] + { + if let Some(ref log) = config.console_log { + let debug_path = log.with_file_name("gpu-debug.log"); + if let Ok(cpath) = std::ffi::CString::new(debug_path.to_string_lossy().as_bytes()) { + unsafe { + let fd = libc::open( + cpath.as_ptr(), + libc::O_WRONLY | libc::O_CREAT | libc::O_TRUNC, + 0o644, + ); + if fd >= 0 { + libc::dup2(fd, 2); + libc::close(fd); + } + } + } + } + // Detach stdin/stdout only — keep stderr for GPU debug output + unsafe { + let devnull = libc::open(c"/dev/null".as_ptr(), libc::O_RDWR); + if devnull >= 0 { + libc::dup2(devnull, 0); + libc::dup2(devnull, 1); + libc::close(devnull); + } + } + } + } else if let Err(e) = smolvm::process::detach_stdio_to_stderr_file(&config.startup_error_log) { + let _ = std::fs::write( + &config.startup_error_log, + format!("failed to redirect stdio: {}", e), + ); + smolvm::process::exit_child(1); + } + + // Close inherited file descriptors from the parent (server). + // Without this, the subprocess holds database locks, network sockets, etc. + // that can interfere with libkrun's operation. Keep stdin/stdout/stderr (0-2) + // which now point to /dev/null. + smolvm::process::close_inherited_fds_from(3); + + // Defense-in-depth before this process becomes the VMM host for an untrusted + // guest: block setuid privilege escalation and core dumps (which would leak + // guest RAM). See docs/runtime-isolation-hardening.md for the full roadmap. + smolvm::process::harden_self(); + + // If the supervisor delegated a cgroup v2 root (via SMOLVM_CGROUP_ROOT), + // place this VMM in a per-VM cgroup with cpu/pids/memory caps so an untrusted + // guest can't peg host CPU, fork-bomb the host, or balloon VMM memory. Inert + // when the env var is unset (no delegation) — never blocks boot. + #[cfg(target_os = "linux")] + if let Some(cgroup_root) = std::env::var_os("SMOLVM_CGROUP_ROOT") { + smolvm::process::place_in_cgroup( + std::path::Path::new(&cgroup_root), + config.resources.cpus, + config.resources.memory_mib, + ); + } + + // Shared pack store: present the node's root-owned shared pack copy at this + // VM's `packed_layers_dir` via a per-VM idmapped bind mount (on-disk uid 0 -> + // this VM's uid), in a private mount namespace that's torn down on exit. Done + // here while still privileged (needs CAP_SYS_ADMIN) and BEFORE the uid drop + // and Landlock/seccomp. The manager only sets `pack_idmap_source` when the uid + // drop is active, so SMOLVM_VM_UID is guaranteed present; fail closed if not. + #[cfg(target_os = "linux")] + if let Some(ref shared) = config.pack_idmap_source { + let target = match config.packed_layers_dir { + Some(ref t) => t, + None => { + eprintln!("[pack-idmap] idmap source set without a mountpoint; refusing to boot"); + smolvm::process::exit_child(1); + } + }; + let uid: u32 = std::env::var("SMOLVM_VM_UID") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or_else(|| { + eprintln!("[pack-idmap] idmap source set without SMOLVM_VM_UID; refusing to boot"); + smolvm::process::exit_child(1); + }); + let gid: u32 = std::env::var("SMOLVM_VM_GID") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(uid); + if let Err(e) = smolvm::process::setup_pack_idmap_mount(shared, target, uid, gid) { + eprintln!("[pack-idmap] failed to mount shared pack, refusing to boot: {e}"); + smolvm::process::exit_child(1); + } + } + + // Drop to an unprivileged uid before touching the guest, so a guest→VMM + // escape can't signal/ptrace the supervisor or neighbor VMs nor reach + // root-owned host files. Gated by SMOLVM_VM_UID (+ optional SMOLVM_VM_GID, + // default = uid). Requires a privileged supervisor and this VM's data + // dir/disks owned by the uid (see docs/runtime-isolation-hardening.md). + // Fails closed. Placed AFTER cgroup placement (needs privilege), BEFORE + // Landlock/seccomp (work unprivileged once no_new_privs is set). + #[cfg(target_os = "linux")] + if let Some(uid_str) = std::env::var_os("SMOLVM_VM_UID") { + let uid: u32 = match uid_str.to_str().and_then(|s| s.parse().ok()) { + Some(u) => u, + None => { + eprintln!("[uid-drop] invalid SMOLVM_VM_UID; refusing to boot"); + smolvm::process::exit_child(1); + } + }; + let gid: u32 = std::env::var("SMOLVM_VM_GID") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(uid); + if let Err(e) = smolvm::process::drop_privileges(uid, gid) { + eprintln!("[uid-drop] failed, refusing to boot over-privileged: {e}"); + smolvm::process::exit_child(1); + } + // The setuid above clears the dumpable flag. A forkable golden's clones + // map its guest-RAM memfd via /proc//fd, which ptrace_may_access + // denies on a non-dumpable target even at a uid match — so re-assert + // dumpable here (after the drop) for a forkable VM, or fork breaks under + // per-VM uid isolation. See process::set_dumpable. + #[cfg(target_os = "linux")] + if std::env::var_os("SMOLVM_FORKABLE").is_some() { + smolvm::process::set_dumpable(true); + } + } + + // A fork clone restores by mapping the golden's guest-RAM memfd, opened via + // /proc//fd/ — an anonymous inode with no filesystem path. + // Landlock is path-based and cannot grant access to a pathless object, so a + // Landlock-confined clone can never open the memfd (EACCES → can't boot). + // Clones therefore skip Landlock; they stay confined by seccomp, the per-VM + // uid drop, and the cgroup. Goldens and normal VMs are unaffected. + #[cfg(target_os = "linux")] + let is_fork_clone = std::env::var_os("SMOLVM_SNAPSHOT_DIR").is_some(); + + // Confine the VMM's filesystem view via Landlock — BEFORE seccomp (whose + // allowlist omits the landlock_* syscalls) and before libkrun loads. Granted: + // read+exec on rootfs/libs/system dirs, read-write on this VM's own data dir + // and the device nodes a VMM needs; the rest of the host fs is denied so a + // guest→VMM escape can't read other tenants' data or host secrets. Paths are + // derived per-VM from the boot config. Gated by SMOLVM_LANDLOCK=enforce + // (unset = off); fails closed. See docs/runtime-isolation-hardening.md. + #[cfg(target_os = "linux")] + if std::env::var("SMOLVM_LANDLOCK").as_deref() == Ok("enforce") && is_fork_clone { + eprintln!( + "[landlock] fork clone skips Landlock (must map the golden's pathless \ + memfd); still confined by seccomp + uid drop + cgroup" + ); + } + #[cfg(target_os = "linux")] + if std::env::var("SMOLVM_LANDLOCK").as_deref() == Ok("enforce") && !is_fork_clone { + let mut read_exec: Vec = [ + "/usr", "/lib", "/lib64", "/bin", "/sbin", "/etc", "/opt", "/proc", "/sys", + ] + .iter() + .map(std::path::PathBuf::from) + .collect(); + read_exec.push(config.rootfs_path.clone()); + if let Some(ref d) = config.packed_layers_dir { + read_exec.push(d.clone()); + } + // Grant read+exec on the directory libkrun/libkrunfw are actually + // dlopen'd from, resolved EXACTLY like the loader (`find_lib_dir`): + // `SMOLVM_LIB_DIR` if it holds the libs, else exe-relative bundle paths + // (`lib/`, `../../lib/linux-`, …). Consulting only `SMOLVM_LIB_DIR` + // denied the bundled/dev layout — where the libs live in an exe-relative + // `lib/` dir and the env var is unset — making `libkrunfw.so` fail to + // load under enforce ("cannot open shared object file: Permission denied"). + if let Some(lib_dir) = smolvm::agent::find_lib_dir() { + read_exec.push(lib_dir); + } + if let Some(libdir) = std::env::var_os("SMOLVM_LIB_DIR") { + read_exec.push(std::path::PathBuf::from(libdir)); + } + // A fresh VM's storage/overlay disk may be a qcow2 copy-on-write overlay + // backed by a read-only disk template in ~/.smolvm; the confined VMM must + // be able to open that backing file. Grant the template directory + // read-only — it is the install dir (same trust level as SMOLVM_LIB_DIR, + // which lives under it) and holds no secrets. A no-op for the copy path. + if let Some(home) = dirs::home_dir() { + read_exec.push(home.join(".smolvm")); + } + + let mut read_write: Vec = [ + "/dev/kvm", + "/dev/null", + "/dev/zero", + "/dev/full", + "/dev/urandom", + "/dev/random", + "/dev/dri", + "/dev/ptmx", + "/tmp", + ] + .iter() + .map(std::path::PathBuf::from) + .collect(); + for p in [ + &config.storage_disk_path, + &config.overlay_disk_path, + &config.vsock_socket, + &config.startup_error_log, + ] { + if let Some(parent) = p.parent() { + read_write.push(parent.to_path_buf()); + } + } + if let Some(parent) = config.console_log.as_ref().and_then(|c| c.parent()) { + read_write.push(parent.to_path_buf()); + } + if let Some(parent) = config.ssh_agent_socket.as_ref().and_then(|s| s.parent()) { + read_write.push(parent.to_path_buf()); + } + for (path, read_only, _format) in &config.extra_disks { + if *read_only { + read_exec.push(path.clone()); + } else { + read_write.push(path.clone()); + } + } + for m in &config.mounts { + if m.read_only { + read_exec.push(m.source.clone()); + } else { + read_write.push(m.source.clone()); + } + } + + // The guest agent signals boot-readiness by writing a marker file into + // the virtiofs rootfs, which the host polls. The rootfs is granted + // read-exec above, so that FUSE write would be denied here. Carve out + // write on JUST that one marker file — never the rootfs dir, which + // holds the shared agent binary/init/libs a guest→VMM escape must not + // be able to tamper with. Landlock needs an existing path to build the + // rule, so pre-create it empty; the guest overwrites it with content + // and the host treats non-empty as ready (see manager.rs wait loop). + // The marker name is per-VM (the host passes it via SMOLVM_READY_MARKER so + // concurrent boots don't share one file); fall back to the shared constant + // if unset. Granting/pre-creating the WRONG name would leave the agent's + // real (per-VM) write Landlock-denied → readiness limps to the vsock grace. + let marker_name = std::env::var(smolvm_protocol::guest_env::READY_MARKER) + .unwrap_or_else(|_| smolvm_protocol::AGENT_READY_MARKER.to_string()); + let ready_marker = config.rootfs_path.join(marker_name); + if let Err(e) = std::fs::File::create(&ready_marker) { + // Read-only rootfs (e.g. root-owned system install): the marker + // can never be written, by us or by the guest. Not fatal — the + // manager detects this and uses socket readiness instead (#590) — + // but don't hide it. + eprintln!( + "[boot] ready marker pre-create failed ({e}); readiness will use \ + the vsock socket probe: {}", + ready_marker.display() + ); + } + read_write.push(ready_marker); + + if let Err(e) = smolvm::process::restrict_filesystem(&read_exec, &read_write) { + eprintln!("[landlock] restriction failed, refusing to boot unconfined: {e}"); + smolvm::process::exit_child(1); + } + } + + // Confine this VMM to a syscall allowlist before it loads libkrun and + // enters the guest run loop, so a guest→VMM escape can't reach dangerous host + // syscalls. Gated by SMOLVM_SECCOMP=audit|enforce (unset = off). Installed + // while single-threaded so libkrun's vCPU/worker threads inherit the filter. + // Enforce mode fails closed (a filter that won't install must not silently run + // unconfined). See docs/runtime-isolation-hardening.md. + // + // The call site is gated for BOTH x86_64 and aarch64 (AWS Graviton / GCP + // Axion). The allowlist is arch-neutral (`libc::SYS_*` names resolve per + // arch) with the x86_64-only legacy syscalls cfg-gated in build_seccomp_program; + // `enforce` is validated to boot cleanly on aarch64 — bare VM, image-based + // container (crun), and networked image pull all run under enforce with zero + // SIGSYS on a Graviton-class host. + #[cfg(all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + ))] + match std::env::var("SMOLVM_SECCOMP").as_deref() { + Ok("enforce") => { + if let Err(e) = smolvm::process::install_seccomp_filter(true) { + eprintln!("[seccomp] enforce install failed, refusing to boot unconfined: {e}"); + smolvm::process::exit_child(1); + } + } + Ok("audit") => { + if let Err(e) = smolvm::process::install_seccomp_filter(false) { + eprintln!("[seccomp] audit install failed: {e}"); + } + } + _ => {} + } + + // Emit subprocess startup timing to the startup error log (stderr after + // the stdio redirect above). These lines decompose the dark window between + // parent's spawn() returning and launch_agent_vm() being called. + // Emit when RUST_LOG contains "info" — check env var directly since the + // tracing dispatch may be unusable after close_inherited_fds_from invalidates + // macOS framework file descriptors held by the parent process. + let rust_log = std::env::var("RUST_LOG").unwrap_or_default(); + let proc_timing_on = rust_log.contains("info"); + macro_rules! proc_timing { + ($label:expr) => { + if proc_timing_on { + eprintln!("[proc] {:25} {}ms", $label, t_proc.elapsed().as_millis()); + } + }; + } + proc_timing!("fds closed"); + + // Open storage and overlay disks, honoring qcow2 fork-clone overlays. + let storage_disk = match open_boot_disk::( + &config.storage_disk_path, + config.storage_size_gb, + ) { + Ok(d) => d, + Err(e) => { + let _ = std::fs::write( + &config.startup_error_log, + format!("failed to open storage disk: {}", e), + ); + smolvm::process::exit_child(1); + } + }; + proc_timing!("storage opened"); + + let overlay_disk = match open_boot_disk::( + &config.overlay_disk_path, + config.overlay_size_gb, + ) { + Ok(d) => d, + Err(e) => { + let _ = std::fs::write( + &config.startup_error_log, + format!("failed to open overlay disk: {}", e), + ); + smolvm::process::exit_child(1); + } + }; + proc_timing!("overlay opened"); + + // Launch the VM (never returns on success) + let disks = VmDisks { + storage: &storage_disk, + overlay: Some(&overlay_disk), + }; + + // Start DNS filter listener if configured + let dns_filter_socket_path = if let Some(ref hosts) = config.dns_filter_hosts { + if !hosts.is_empty() { + let socket_path = config + .vsock_socket + .parent() + .unwrap_or(std::path::Path::new("/tmp")) + .join("dns-filter.sock"); + if let Err(e) = smolvm::dns_filter_listener::start(&socket_path, hosts.clone()) { + tracing::warn!(error = %e, "failed to start DNS filter listener"); + None + } else { + Some(socket_path) + } + } else { + None + } + } else { + None + }; + + // CUDA-over-vsock opt-in, resolved once here at the boot-config boundary so + // the launcher receives a typed socket path (it never reads the environment). + // Two ways in, both inherited by this boot subprocess from the manager: + // * SMOLVM_CUDA_SOCK= — attach to an externally-managed host server + // at that AF_UNIX path (smolvm does not spawn one). + // * the machine's `cuda` flag (--cuda at create) — smolvm owns the + // lifecycle: derive a per-VM socket in this machine's dir and start the + // in-tree host server on it. + let cuda_socket: Option = if let Some(p) = std::env::var("SMOLVM_CUDA_SOCK") + .ok() + .filter(|s| !s.is_empty()) + { + Some(std::path::PathBuf::from(p)) + } else if config.cuda { + let path = config + .vsock_socket + .parent() + .unwrap_or_else(|| std::path::Path::new("/tmp")) + .join("cuda.sock"); + match smolvm::cuda_host::start(&path) { + Ok(()) => { + tracing::info!(path = %path.display(), "CUDA host server started"); + Some(path) + } + Err(e) => { + tracing::warn!(error = %e, "failed to start CUDA host server — CUDA disabled"); + None + } + } + } else { + None + }; + + proc_timing!("ready to launch"); + + // Egress telemetry lands in the per-VM dir (the vsock socket's parent), the + // same dir serve resolves from the machine name — so no name needs threading + // across the process boundary. + let egress_telemetry_path = config.vsock_socket.parent().map(|dir| dir.join("egress")); + + let result = launch_agent_vm(&LaunchConfig { + rootfs_path: &config.rootfs_path, + disks: &disks, + vsock_socket: &config.vsock_socket, + console_log: config.console_log.as_deref(), + egress_telemetry: egress_telemetry_path.as_deref(), + mounts: &config.mounts, + port_mappings: &config.ports, + resources: config.resources, + ssh_agent_socket: config.ssh_agent_socket.as_deref(), + dns_filter_socket: dns_filter_socket_path.as_deref(), + cuda_socket: cuda_socket.as_deref(), + packed_layers_dir: config.packed_layers_dir.as_deref(), + extra_disks: &config.extra_disks, + dns_filter_enabled: config + .dns_filter_hosts + .as_ref() + .is_some_and(|hosts| !hosts.is_empty()), + egress_refresh_hosts: config.dns_filter_hosts.clone(), + }); + + // If we get here, launch_agent_vm returned (should only happen on error) + if let Err(ref e) = result { + let _ = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&config.startup_error_log) + .and_then(|mut file| { + use std::io::Write; + writeln!(file, "{e}") + }); + } + + smolvm::process::exit_child(1); +} diff --git a/src/cli/machine.rs b/src/cli/machine.rs new file mode 100644 index 0000000..f053e08 --- /dev/null +++ b/src/cli/machine.rs @@ -0,0 +1,3844 @@ +//! Machine management commands. +//! +//! All VM-related commands are under the `machine` subcommand: +//! - exec: Persistent execution (machine keeps running) +//! - create: Create named VM configuration +//! - start: Start a machine (named or default) +//! - stop: Stop a machine (named or default) +//! - delete: Delete a named VM configuration +//! - status: Show machine status +//! - ls: List all named VMs + +use crate::cli::flush_output; +use crate::cli::format_bytes; +use crate::cli::parsers::{ + mounts_to_virtiofs_bindings, parse_cidr, parse_duration, parse_env_list, parse_image, +}; +use crate::cli::vm_common::{self, DeleteVmOptions}; +use clap::{Args, Subcommand}; +use sha2::{Digest, Sha256}; +use smolvm::agent::{docker_config_mount, AgentClient, AgentManager, RunConfig, VmResources}; +use smolvm::data::network::PortMapping; +use smolvm::data::resources::{DEFAULT_MICROVM_CPU_COUNT, DEFAULT_MICROVM_MEMORY_MIB}; +use smolvm::data::storage::HostMount; +use smolvm::network::{validate_requested_network_backend, NetworkBackend}; +use smolvm::{DEFAULT_IDLE_CMD, DEFAULT_SHELL_CMD}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +/// How many orphaned ephemeral VMs `machine run` reaps per boot. Small so the +/// hot path never stalls on a large backlog; a heavier backlog drains over +/// successive runs (and other commands sweep without a cap). +const EPHEMERAL_RUN_SWEEP_CAP: usize = 8; + +/// Resolve `--allow-cidr`, `--allow-host`, and `--outbound-localhost-only` into a CIDR list, +/// net flag, and the original hostname list (for DNS filtering). +/// +/// Resolution failure for `--allow-host` is a hard error — a typo or DNS outage +/// should not silently weaken the security policy. +/// Returns true when `s` structurally looks like an OCI image reference +/// rather than an executable name or path. +/// +/// Catches the common mistake of writing `smolvm machine run ubuntu:22.04 -- +/// bash` instead of `smolvm machine run --image ubuntu:22.04 -- bash`. +/// Only unambiguous structural signals are checked: +/// - `image:tag` form — colons are not valid in executable names +/// - `registry/image` or `namespace/image` form (non-absolute slash path) +/// +/// Bare names like `alpine` or `nginx` are intentionally not flagged here +/// because they are indistinguishable from valid bare commands. +fn is_likely_image_ref(s: &str) -> bool { + if s.contains(':') { + return true; + } + s.contains('/') && !s.starts_with('/') && !s.starts_with("./") && !s.starts_with("../") +} + +fn resolve_egress_flags( + mut allow_cidr: Vec, + allow_host: Vec, + outbound_localhost_only: bool, + net: bool, +) -> smolvm::Result<(Vec, bool, Option>)> { + // Resolve hostnames to CIDRs — fail hard on resolution errors + for host in &allow_host { + let cidrs = crate::cli::parsers::resolve_host_to_cidrs(host) + .map_err(|e| smolvm::Error::config("--allow-host", e))?; + tracing::info!(host, ?cidrs, "resolved hostname for egress policy"); + allow_cidr.extend(cidrs); + } + + if outbound_localhost_only { + allow_cidr.push("127.0.0.0/8".to_string()); + allow_cidr.push("::1/128".to_string()); + } + let net = net || !allow_cidr.is_empty(); + + // Preserve original hostnames for DNS filtering (None if no --allow-host was used) + let dns_filter_hosts = if allow_host.is_empty() { + None + } else { + Some(allow_host) + }; + + Ok((allow_cidr, net, dns_filter_hosts)) +} + +/// Parse `--secret-env KEY=HOST_VAR` and `--secret-file KEY=PATH` flag values +/// into validated [`SecretRef`]s keyed by the guest-side env var name. +/// +/// CLI-supplied refs are `TrustedLocal` (the host user invoked the command), so +/// both source kinds are allowed; `validate_ref` still enforces structure and +/// absolute `from_file` paths. A key that appears more than once — across or +/// within the two flags — is a hard error, since silently keeping the last +/// occurrence would mask a typo. +fn parse_cli_secret_refs( + secret_env: &[String], + secret_file: &[String], +) -> smolvm::Result> { + use smolvm::secrets::{env_ref, file_ref, validate_ref, ResolutionScope, SecretRef}; + use std::collections::BTreeMap; + + let mut out: BTreeMap = BTreeMap::new(); + + let mut add = + |flag: &str, spec: &str, make: &dyn Fn(&str) -> SecretRef| -> smolvm::Result<()> { + let (key, value) = spec.split_once('=').ok_or_else(|| { + smolvm::Error::config(flag, format!("expected KEY=VALUE, got '{}'", spec)) + })?; + if key.is_empty() { + return Err(smolvm::Error::config( + flag, + format!("empty secret name in '{}'", spec), + )); + } + let r = make(value); + validate_ref(&r, ResolutionScope::TrustedLocal) + .map_err(|e| smolvm::Error::config(flag, format!("secret '{}': {}", key, e)))?; + if out.insert(key.to_string(), r).is_some() { + return Err(smolvm::Error::config( + flag, + format!("secret '{}' specified more than once", key), + )); + } + Ok(()) + }; + + for spec in secret_env { + add("--secret-env", spec, &|v| env_ref(v))?; + } + for spec in secret_file { + add("--secret-file", spec, &|v| file_ref(v))?; + } + Ok(out) +} + +/// Spawn a detached `smolvm _cleanup-ephemeral` helper process so the parent +/// CLI can exit immediately after flushing output. +/// +/// Returns `true` if the helper was spawned successfully. The caller must then +/// call `std::process::exit(exit_code)` without doing any further cleanup. +/// +/// Returns `false` if spawn fails (binary not found, exec error, etc.). +/// The caller falls back to synchronous cleanup in that case. +fn try_spawn_detached_cleanup(vm_name: &str, pid: i32, start_time: Option) -> bool { + // Require a verified start time so the helper can use is_our_process_strict + // before sending SIGKILL. Without it, fall back to synchronous cleanup. + let start_time_val = match start_time { + Some(t) => t, + None => return false, + }; + let exe = match std::env::current_exe() { + Ok(p) => p, + Err(_) => return false, + }; + let mut cmd = std::process::Command::new(exe); + cmd.arg("_cleanup-ephemeral") + .arg(vm_name) + .arg(pid.to_string()) + .arg(start_time_val.to_string()) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + // New process group so the helper is immune to SIGHUP when the parent + // terminal closes (pgid = child pid). POSIX-only; no Windows equivalent. + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + cmd.process_group(0); + } + let result = cmd.spawn(); + // Drop the Child handle without waiting — we exit immediately after this. + // The OS will not create a zombie because the helper outlives us and its + // real parent (launchd/init) reaps it when it exits. + result.is_ok() +} + +/// Manage machines +#[derive(Subcommand, Debug)] +pub enum MachineCmd { + /// Run a container image in an ephemeral machine + Run(RunCmd), + + /// Run a command directly in the VM (not in a container) + Exec(ExecCmd), + + /// Create a new named machine configuration + Create(CreateCmd), + + /// Start a machine + Start(StartCmd), + + /// Fork a running forkable machine into a new clone (CoW memory + disks) + Fork(ForkCmd), + + /// Stop a running machine + Stop(StopCmd), + + /// Delete a machine configuration + #[command(visible_alias = "rm")] + Delete(DeleteCmd), + + /// Show machine status + Status(StatusCmd), + + /// List all machines + #[command(visible_alias = "list")] + Ls(LsCmd), + + /// Resize a machine's disk resources (use `update` instead) + #[command(hide = true)] + Resize(ResizeCmd), + + /// Modify settings on a stopped machine (mounts, ports, resources, disks) + Update(UpdateCmd), + + /// List cached images and storage usage + Images(ImagesCmd), + + /// Remove unused images and layers to free disk space + Prune(PruneCmd), + + /// Open an interactive shell in a machine (starts it if stopped) + #[command(visible_alias = "sh")] + Shell(ShellCmd), + + /// Copy files between host and machine + Cp(CpCmd), + + /// Monitor a machine with health checks and restart policy + Monitor(MonitorCmd), + + /// Test network connectivity from inside the VM + #[command(hide = true)] + NetworkTest(NetworkTestCmd), + + /// Print the on-disk data directory path for a named machine. + /// + /// Useful for scripting and debugging — returns the path where the VM's + /// storage disk, overlay disk, and agent socket live. The path is + /// hash-derived, not name-derived. + #[command(name = "data-dir")] + DataDir(DataDirCmd), +} + +impl MachineCmd { + pub fn run(self) -> smolvm::Result<()> { + // Reclaim orphaned ephemeral VMs before doing work. `machine run` uses a + // BOUNDED sweep: a workflow that only ever calls `machine run` would + // otherwise never reclaim a data dir left by a run whose detached cleanup + // helper didn't finish (Ctrl-C / SIGKILL / host sleep mid-run). The cap + // keeps the boot hot path from stalling on a large backlog — it drains + // over successive runs. Other commands sweep everything. + if matches!(self, MachineCmd::Run(_)) { + super::vm_common::cleanup_orphaned_ephemeral_vms_bounded(EPHEMERAL_RUN_SWEEP_CAP); + } else { + super::vm_common::cleanup_orphaned_ephemeral_vms(); + } + + match self { + MachineCmd::Run(cmd) => cmd.run(), + MachineCmd::Exec(cmd) => cmd.run(), + MachineCmd::Create(cmd) => cmd.run(), + MachineCmd::Start(cmd) => cmd.run(), + MachineCmd::Fork(cmd) => cmd.run(), + MachineCmd::Stop(cmd) => cmd.run(), + MachineCmd::Delete(cmd) => cmd.run(), + MachineCmd::Status(cmd) => cmd.run(), + MachineCmd::Ls(cmd) => cmd.run(), + MachineCmd::Resize(cmd) => cmd.run(), + MachineCmd::Update(cmd) => cmd.run(), + MachineCmd::Images(cmd) => cmd.run(), + MachineCmd::Prune(cmd) => cmd.run(), + MachineCmd::Shell(cmd) => cmd.run(), + MachineCmd::Cp(cmd) => cmd.run(), + MachineCmd::Monitor(cmd) => cmd.run(), + MachineCmd::NetworkTest(cmd) => cmd.run(), + MachineCmd::DataDir(cmd) => cmd.run(), + } + } +} + +// ============================================================================ +// Run Command (Ephemeral) +// ============================================================================ + +/// Run a container image in an ephemeral machine. +/// +/// By default, runs in ephemeral mode (machine cleaned up after exit). +/// Use -d/--detach to keep the machine running for later interaction. +/// +/// Examples: +/// smolvm machine run --image alpine -- echo "hello" +/// smolvm machine run -it -I alpine +/// smolvm machine run -d --net -I ubuntu +/// smolvm machine run --net -v ./src:/app --image node -- npm start +#[derive(Args, Debug)] +pub struct RunCmd { + /// Container image (e.g., alpine, ubuntu:22.04, ghcr.io/org/image). + /// Optional when a Smolfile provides the image, or for bare VM mode. + #[arg(short = 'I', long, value_name = "IMAGE", value_parser = parse_image)] + pub image: Option, + + /// Raise the max accepted local image-archive size (e.g. 16GiB, 512M, or a + /// raw byte count); default 8GiB. For legitimately large images — sets + /// SMOLVM_MAX_IMAGE_BYTES for this run. + #[arg(long = "max-image-size", value_name = "SIZE", + value_parser = crate::cli::parsers::parse_size_bytes, help_heading = "Execution")] + pub max_image_size: Option, + + /// Run a packed `.smolmachine` artifact ephemerally (the VM is discarded on + /// exit) — the one-shot equivalent of `machine create --from … + start`. + /// CPU/memory fall back to the artifact's baked manifest unless overridden. + #[arg( + long, + value_name = "PATH", + conflicts_with_all = ["image", "smolfile", "detach", "name", "gpu", "gpu_vram_mib", "oci_platform", "allow_cidr", "allow_host", "outbound_localhost_only", "secret_env", "secret_file"], + help_heading = "Machine source" + )] + pub from: Option, + + /// Name a persistent machine when used with --detach. + /// Matches the --name flag on start/stop/exec/status/resize. In foreground + /// mode (no -d), --name is ignored with a warning. + #[arg(short = 'n', long, value_name = "NAME", help_heading = "Execution")] + pub name: Option, + + /// Command and arguments to run (default: image entrypoint or /bin/sh) + #[arg(trailing_var_arg = true, value_name = "COMMAND")] + pub command: Vec, + + /// Start the command in the background and detach, leaving the VM + /// running. Use `machine exec` to run further commands against the VM + /// and `machine stop` to tear it down. + #[arg(short = 'd', long, help_heading = "Execution")] + pub detach: bool, + + /// Keep stdin open for interactive input + #[arg(short = 'i', long, help_heading = "Execution")] + pub interactive: bool, + + /// Allocate a pseudo-TTY (use with -i for interactive shells) + #[arg(short = 't', long, help_heading = "Execution")] + pub tty: bool, + + /// Kill command after duration (e.g., "30s", "5m", "1h") + #[arg(long, value_parser = parse_duration, value_name = "DURATION", help_heading = "Execution")] + pub timeout: Option, + + /// Set working directory inside container + #[arg(short = 'w', long, value_name = "DIR", help_heading = "Container")] + pub workdir: Option, + + /// Set environment variable (can be used multiple times) + #[arg( + short = 'e', + long = "env", + value_name = "KEY=VALUE", + help_heading = "Container" + )] + pub env: Vec, + + /// Target OCI platform for multi-arch images + #[arg( + long = "oci-platform", + value_name = "OS/ARCH", + help_heading = "Container" + )] + pub oci_platform: Option, + + /// Mount host directory into container (can be used multiple times) + #[arg( + short = 'v', + long = "volume", + value_name = "HOST:CONTAINER[:ro]", + help_heading = "Container" + )] + pub volume: Vec, + + /// Expose port from container to host (can be used multiple times) + #[arg(short = 'p', long = "port", value_parser = PortMapping::parse, value_name = "HOST:GUEST", help_heading = "Network")] + pub port: Vec, + + /// Enable outbound network access + #[arg(long, help_heading = "Network")] + pub net: bool, + + /// Select the networking backend. + #[arg(long = "net-backend", value_enum, help_heading = "Network")] + pub net_backend: Option, + + /// Custom DNS resolver for the guest (implies --net). Use this when the + /// default public resolvers (8.8.8.8/1.1.1.1) are blocked on your network. + #[arg(long, value_name = "IP", help_heading = "Network")] + pub dns: Option, + + /// Allow egress to specific CIDR range (can be used multiple times, implies --net) + #[arg(long = "allow-cidr", value_parser = parse_cidr, value_name = "CIDR", help_heading = "Network")] + pub allow_cidr: Vec, + + /// Allow egress to specific hostname, resolved at VM start (can be used multiple times, implies --net) + #[arg(long = "allow-host", value_name = "HOSTNAME", help_heading = "Network")] + pub allow_host: Vec, + + /// Restrict outbound to localhost only (implies --net) + #[arg(long, help_heading = "Network")] + pub outbound_localhost_only: bool, + + /// Enable GPU acceleration (Vulkan via virtio-gpu) + #[arg(long, help_heading = "Resources")] + pub gpu: bool, + + /// GPU shared-memory region size in MiB. Ignored without --gpu. + /// Default 4096 (4 GiB). Must be > 0. + #[arg( + long = "gpu-vram", + value_name = "MiB", + help_heading = "Resources", + value_parser = crate::cli::parsers::parse_gpu_vram_mib, + )] + pub gpu_vram_mib: Option, + + /// Enable Rosetta 2 for x86_64 binary translation on Apple Silicon + #[arg(long, help_heading = "Resources")] + pub rosetta: bool, + + /// Number of virtual CPUs + #[arg(long, default_value_t = DEFAULT_MICROVM_CPU_COUNT, value_name = "N", help_heading = "Resources")] + pub cpus: u8, + + /// Memory allocation in MiB + #[arg(long, default_value_t = DEFAULT_MICROVM_MEMORY_MIB, value_name = "MiB", help_heading = "Resources")] + pub mem: u32, + + /// Storage disk size in GiB + #[arg(long, value_name = "GiB", help_heading = "Resources")] + pub storage: Option, + + /// Overlay disk size in GiB + #[arg(long, value_name = "GiB", help_heading = "Resources")] + pub overlay: Option, + + /// Load VM configuration from a Smolfile (TOML) + #[arg( + long = "smolfile", + visible_short_alias = 's', + value_name = "PATH", + help_heading = "Resources" + )] + pub smolfile: Option, + + /// Forward host SSH agent into the VM (enables git/ssh without exposing keys) + #[arg(long, help_heading = "Security")] + pub ssh_agent: bool, + + /// Remote guest CUDA Driver-API calls to the host NVIDIA GPU over vsock + #[arg(long, help_heading = "Hardware")] + pub cuda: bool, + + /// Mount ~/.docker/ config into VM for registry authentication + #[arg(long, help_heading = "Registry")] + pub docker_config: bool, + + /// Inject a secret from a host env var (GUEST_VAR=HOST_VAR), resolved at + /// launch. The value is never persisted to the machine record or a pack. + #[arg( + long = "secret-env", + value_name = "GUEST_VAR=HOST_VAR", + help_heading = "Security" + )] + pub secret_env: Vec, + + /// Inject a secret from a host file (GUEST_VAR=/abs/path), resolved at + /// launch. The value is never persisted to the machine record or a pack. + #[arg( + long = "secret-file", + value_name = "GUEST_VAR=PATH", + help_heading = "Security" + )] + pub secret_file: Vec, + + /// Skip the init-layer cache: re-run `init` on every ephemeral run instead of + /// baking `image + init` once into a cached, reusable artifact. Use this when + /// `init` depends on live volume contents (and so cannot be safely cached). + #[arg(long, help_heading = "Resources")] + pub no_init_cache: bool, + + /// Rebuild the cached init layer even if a matching one already exists. + #[arg(long, help_heading = "Resources")] + pub rebuild_init_cache: bool, + + /// Run the workload as an unprivileged container: restricted capabilities, + /// read-only cgroup, and no extra tmpfs. By default the workload is "VM-grade" + /// (the microVM is the isolation boundary, so it gets full privileges and any + /// image — incl. systemd — boots). Use this for defense-in-depth with untrusted + /// code. `init` always runs VM-grade (it needs privileges for apt/mounts). + #[arg(long, help_heading = "Security")] + pub unprivileged: bool, + + #[command(flatten, next_help_heading = "Network")] + pub proxy_opts: crate::cli::proxy_opts::ProxyOpts, +} + +/// Cache directory for baked init layers: a sibling of the per-VM cache +/// (`/smolvm/init-layers`), derived from the same canonical root as +/// [`smolvm::agent::vm_cache_root`] so it shares the install's cache location. +fn init_layer_cache_dir() -> PathBuf { + smolvm::agent::vm_cache_root() + .parent() + .map(|smolvm_root| smolvm_root.join("init-layers")) + .unwrap_or_else(|| std::path::PathBuf::from("/tmp/smolvm-init-layers")) +} + +/// Max real bytes the init-layer cache may occupy before eviction; override via +/// `SMOLVM_INIT_CACHE_MAX_BYTES`. Default 10 GiB (~tens of layers). +fn init_cache_max_bytes() -> u64 { + const DEFAULT: u64 = 10 * 1024 * 1024 * 1024; + std::env::var("SMOLVM_INIT_CACHE_MAX_BYTES") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|&n| n > 0) + .unwrap_or(DEFAULT) +} + +/// Evict least-recently-modified cached layers until the cache is at or below +/// `max_bytes`, never evicting `keep` (the layer just published). Best-effort: +/// per-entry errors are skipped. The `.smolmachine` sidecars are zstd-compressed +/// (dense) files, so apparent length is an accurate size. +fn prune_init_cache(dir: &Path, max_bytes: u64, keep: &Path) { + let mut layers: Vec<(PathBuf, std::time::SystemTime, u64)> = Vec::new(); + let Ok(rd) = std::fs::read_dir(dir) else { + return; + }; + for entry in rd.flatten() { + let path = entry.path(); + if path.extension().and_then(|x| x.to_str()) != Some("smolmachine") { + continue; + } + let Ok(meta) = entry.metadata() else { + continue; + }; + let mtime = meta.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH); + layers.push((path, mtime, meta.len())); + } + let total: u64 = layers.iter().map(|(_, _, s)| *s).sum(); + if total <= max_bytes { + return; + } + layers.sort_by_key(|(_, mtime, _)| *mtime); // oldest first + let mut over = total - max_bytes; + for (path, _, size) in layers { + if over == 0 { + break; + } + if path == keep { + continue; + } + if std::fs::remove_file(&path).is_ok() { + over = over.saturating_sub(size); + } + } +} + +/// Best-effort sweep of leftover `init-bake-*` temp machines from crashed bakes. +/// Age is taken from the DB record's `created_at` (always present — unlike the data +/// dir, which a create-then-crash leaves absent) and gated on a generous threshold +/// so an in-flight concurrent bake — which finishes in seconds — is never touched. +/// Override the threshold (seconds) with `SMOLVM_INIT_BAKE_GC_SECS`. +fn gc_stale_bake_machines(exe: &Path) { + let stale_after = std::env::var("SMOLVM_INIT_BAKE_GC_SECS") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .unwrap_or(30 * 60); + let Ok(cfg) = smolvm::config::SmolvmConfig::load() else { + return; + }; + let now = smolvm::util::current_timestamp(); + let stale: Vec = cfg + .list_vms() + .filter(|(name, _)| name.starts_with("init-bake-")) + .filter(|(_, record)| now.saturating_sub(record.created_at) >= stale_after) + .map(|(name, _)| name.clone()) + .collect(); + for name in stale { + let _ = run_smolvm(exe, &["machine", "delete", "--name", &name, "-f"]); + } +} + +/// Content key for an init layer: a hash of the image + init commands + env, so the +/// cache rebuilds exactly when those inputs change. +/// +/// PROTOTYPE LIMITATION: if `init` runs a script that lives on a mounted volume +/// (e.g. `bash /project/init.sh`), the script's CONTENTS are not part of the key — +/// inline the init steps into the Smolfile, or pass `--no-init-cache`. +fn init_layer_key(image: Option<&str>, init: &[String], env: &[String]) -> String { + let mut h = Sha256::new(); + h.update(image.unwrap_or("").as_bytes()); + h.update([0u8]); + for c in init { + h.update(c.as_bytes()); + h.update([0u8]); + } + h.update([0u8]); + for e in env { + h.update(e.as_bytes()); + h.update([0u8]); + } + hex::encode(h.finalize())[..16].to_string() +} + +/// Whether a `--image` value is an init-cache-bakeable source. Only registry +/// refs are: the bake snapshots via `pack create --from-vm`, which sources base +/// layers by pulling the image's registry manifest. A local archive or rootfs +/// dir has no registry manifest (it is flattened on boot), so it takes the +/// direct, uncached init path instead of a broken bake (#459). +fn image_bakeable(image: Option<&str>) -> bool { + matches!( + image.map(smolvm::data::image_source::classify), + Some(smolvm::data::image_source::ImageSource::Registry(_)) + ) +} + +/// Bake `image + init` into a cached `.smolmachine` (or reuse an existing one) and +/// return its path. Runs the well-tested `machine create/start/stop` + `pack create +/// --from-vm` flow as subprocesses of this same binary: create a temp machine from +/// the Smolfile with the workload replaced by a `/bin/true` no-op so only `init` +/// runs, snapshot it, and delete the temp machine. The real workload command is +/// supplied at run time against the resulting artifact. +fn ensure_init_layer( + params: &vm_common::CreateVmParams, + smolfile: Option<&Path>, + rebuild: bool, +) -> smolvm::Result { + // The bake here only ever receives a registry image: `ensure_init_layer` is + // gated on `image_bakeable()` (local archives/dirs take the direct path), + // because the `pack create --from-vm` snapshot below cannot source a local + // image's layers (they're flattened, with no registry manifest to pull). + let key = init_layer_key(params.image.as_deref(), ¶ms.init, ¶ms.env); + let dir = init_layer_cache_dir(); + std::fs::create_dir_all(&dir) + .map_err(|e| smolvm::Error::config("init-layer cache", e.to_string()))?; + let cached = dir.join(format!("{key}.smolmachine")); + if cached.exists() && !rebuild { + println!("Using cached init layer {key}"); + return Ok(cached); + } + + let smolfile = smolfile.ok_or_else(|| { + smolvm::Error::config( + "init-layer cache", + "init caching requires a --smolfile (the init source); pass --no-init-cache otherwise", + ) + })?; + println!( + "Baking init layer (one-time; reused on later runs) [{key}, {} init step(s)]", + params.init.len() + ); + let started = std::time::Instant::now(); + + let exe = std::env::current_exe() + .map_err(|e| smolvm::Error::config("init-layer cache", e.to_string()))?; + // Reap any temp machines orphaned by previously crashed bakes (age-gated so it + // never touches a concurrent in-flight bake). + gc_stale_bake_machines(&exe); + let pid = std::process::id(); + let tmp = format!("init-bake-{key}-{pid}"); + let sf = smolfile.to_string_lossy().to_string(); + + // Bake into a per-process staging dir, then atomically rename the sidecar into + // its final cache path. This makes an interrupted bake leave nothing usable (no + // truncated `.smolmachine` a later run would treat as valid), and makes two + // concurrent bakes of the same key safe — each stages independently and the last + // rename wins. The staging dir also absorbs `pack`'s stub binary, discarded when + // the dir is removed (the cache only needs the sidecar). + let staging = dir.join(format!(".staging-{key}-{pid}")); + let _ = std::fs::remove_dir_all(&staging); + std::fs::create_dir_all(&staging) + .map_err(|e| smolvm::Error::config("init-layer cache", e.to_string()))?; + let staged_out = staging.join("layer").to_string_lossy().to_string(); + let staged_sidecar = staging.join("layer.smolmachine"); + + // Clear any temp machine left by a prior failed bake (best-effort; the common + // case is "doesn't exist", whose error is captured and discarded by run_smolvm). + let _ = run_smolvm(&exe, &["machine", "delete", "--name", &tmp, "-f"]); + + let bake = (|| -> smolvm::Result<()> { + // Create from the Smolfile (init + volumes) but replace the workload with + // `/bin/true` so `start` runs init only. Forward the RESOLVED image and env + // (CLI overrides included) so the baked rootfs matches the cache key, which + // is derived from those same resolved params. + let mut create: Vec = ["machine", "create", "--name", &tmp, "--smolfile", &sf] + .iter() + .map(|s| s.to_string()) + .collect(); + if let Some(image) = ¶ms.image { + create.push("--image".into()); + create.push(image.clone()); + } + for e in ¶ms.env { + create.push("-e".into()); + create.push(e.clone()); + } + create.push("--".into()); + create.push("/bin/true".into()); + let create: Vec<&str> = create.iter().map(String::as_str).collect(); + + println!(" · pulling image and running init..."); + run_smolvm(&exe, &create)?; + run_smolvm(&exe, &["machine", "start", "--name", &tmp])?; + run_smolvm(&exe, &["machine", "stop", "--name", &tmp])?; + println!(" · snapshotting..."); + run_smolvm( + &exe, + &["pack", "create", "--from-vm", &tmp, "-o", &staged_out], + )?; + if !staged_sidecar.exists() { + return Err(smolvm::Error::config( + "init-layer cache", + format!("bake did not produce {}", staged_sidecar.display()), + )); + } + Ok(()) + })(); + let _ = run_smolvm(&exe, &["machine", "delete", "--name", &tmp, "-f"]); + + // Publish atomically only on success; always clear staging (drops the stub + any + // partial output) so a failed bake leaves the existing cache untouched. + let publish = bake.and_then(|_| { + std::fs::rename(&staged_sidecar, &cached).map_err(|e| { + smolvm::Error::config("init-layer cache", format!("publish cached layer: {e}")) + }) + }); + let _ = std::fs::remove_dir_all(&staging); + publish?; + + // Bound the cache: evict oldest layers if we're over the cap (keeping the one + // just baked). Best-effort — failure to prune never fails the run. + prune_init_cache(&dir, init_cache_max_bytes(), &cached); + + println!(" ✓ baked in {}s", started.elapsed().as_secs()); + Ok(cached) +} + +/// Run this same smolvm binary as a subprocess for one bake step. Output is +/// CAPTURED (not inherited) so the bake's internal create/pull/pack chatter — and +/// the harmless "vm not found" from the best-effort pre-clean — never reach the +/// user's terminal; on failure the captured stderr tail is surfaced in the error. +fn run_smolvm(exe: &Path, args: &[&str]) -> smolvm::Result<()> { + let out = std::process::Command::new(exe) + .args(args) + .output() + .map_err(|e| smolvm::Error::config("init-layer bake", e.to_string()))?; + if !out.status.success() { + let stderr = String::from_utf8_lossy(&out.stderr); + let tail = stderr + .lines() + .filter(|l| !l.trim().is_empty()) + .rev() + .take(12) + .collect::>() + .into_iter() + .rev() + .collect::>() + .join("\n"); + return Err(smolvm::Error::config( + "init-layer bake", + format!( + "`smolvm {}` failed ({}):\n{tail}", + args.join(" "), + out.status + ), + )); + } + Ok(()) +} + +impl RunCmd { + pub fn run(self) -> smolvm::Result<()> { + use smolvm::Error; + + // --max-image-size raises the archive cap for this invocation by setting + // the env var the resolver reads (image_source::max_archive_bytes). + if let Some(bytes) = self.max_image_size { + std::env::set_var("SMOLVM_MAX_IMAGE_BYTES", bytes.to_string()); + } + + // `--from`: run a packed .smolmachine artifact ephemerally, reusing the + // proven pack-run path. Resource flags fall back to the artifact's baked + // manifest values (matching `machine create --from`); the remaining run + // flags pass through. Flags the sidecar runner can't honor are rejected + // at parse time via `conflicts_with_all` on `from`. + if let Some(from) = self.from { + return crate::cli::pack_run::PackRunCmd { + sidecar: Some(from), + command: self.command, + interactive: self.interactive, + tty: self.tty, + timeout: self.timeout, + workdir: self.workdir, + env: self.env, + volume: self.volume, + port: self.port, + net: self.net, + net_backend: self.net_backend, + cpus: (self.cpus != DEFAULT_MICROVM_CPU_COUNT).then_some(self.cpus), + mem: (self.mem != DEFAULT_MICROVM_MEMORY_MIB).then_some(self.mem), + storage: self.storage, + overlay: self.overlay, + force_extract: false, + info: false, + debug: false, + } + .run(); + } + + let requested_name = self.name.clone(); + let vm_name = if self.detach { + requested_name.unwrap_or_else(|| "default".to_string()) + } else { + smolvm::util::generate_machine_name() + }; + + if self.name.is_some() && vm_name != "default" && self.detach { + let config = smolvm::config::SmolvmConfig::load()?; + if config.get_vm(&vm_name).is_some() { + return Err(Error::config( + "machine run -d --name", + format!( + "a machine named '{}' already exists. Use 'machine start --name {}' to start it, or 'machine delete --name {} -f' to remove it.", + vm_name, vm_name, vm_name + ), + )); + } + } + + let (cli_allow_cidrs, net, cli_dns_filter_hosts) = resolve_egress_flags( + self.allow_cidr, + self.allow_host, + self.outbound_localhost_only, + self.net, + )?; + + let params = crate::cli::smolfile::build_create_params( + vm_name.clone(), + self.image.clone(), + None, + self.command.clone(), + self.cpus, + self.mem, + self.volume, + self.port, + net, + self.net_backend, + self.dns, + vec![], + self.env, + self.workdir, + self.smolfile.clone(), + self.storage, + self.overlay, + cli_allow_cidrs, + )?; + + let mut params = params; + params.dns_filter_hosts = match (params.dns_filter_hosts.take(), cli_dns_filter_hosts) { + (Some(mut from_smolfile), Some(mut from_cli)) => { + from_smolfile.append(&mut from_cli); + Some(from_smolfile) + } + (Some(from_smolfile), None) => Some(from_smolfile), + (None, some) => some, + }; + // CLI `--secret-env`/`--secret-file` refs merge over any Smolfile + // `[secrets]` of the same name (CLI wins). + for (key, r) in parse_cli_secret_refs(&self.secret_env, &self.secret_file)? { + params.secret_refs.insert(key, r); + } + + // Init-layer cache (prototype): for an ephemeral run of an IMAGE with `init` + // commands, bake `image + init` once into a cached `.smolmachine` and run from + // that artifact, so init's cost (e.g. `apt install`) is paid once and reused + // on every later run instead of re-running on each ephemeral boot. Skipped for + // detached/persistent runs (`-d`) and when `--no-init-cache` is set. + // + // Only REGISTRY images are baked. A local image (`--image -` / `--image + // file.tar` / a rootfs dir) is flattened with no registry manifest, so the + // bake's `pack create --from-vm` snapshot can't source its layers; and a + // `--image -` archive can't be re-read by the bake's child subprocess + // (null stdin) anyway. Local images take the direct path below, which + // stages the archive once in this process and runs init inline (#459). + if !self.no_init_cache + && !self.detach + && image_bakeable(params.image.as_deref()) + && !params.init.is_empty() + { + let cached = + ensure_init_layer(¶ms, self.smolfile.as_deref(), self.rebuild_init_cache)?; + // The real workload: CLI trailing args win, else the Smolfile's + // entrypoint+cmd (the baked artifact's own command is a `/bin/true` no-op). + let command = if !self.command.is_empty() { + self.command.clone() + } else { + let mut c = params.entrypoint.clone(); + c.extend(params.cmd.clone()); + c + }; + return crate::cli::pack_run::PackRunCmd { + sidecar: Some(cached), + command, + interactive: self.interactive, + tty: self.tty, + timeout: self.timeout, + workdir: params.workdir.clone(), + env: params.env.clone(), + volume: params.volume.clone(), + port: params.port.clone(), + net: params.net, + net_backend: params.network_backend, + cpus: (params.cpus != DEFAULT_MICROVM_CPU_COUNT).then_some(params.cpus), + mem: (params.mem != DEFAULT_MICROVM_MEMORY_MIB).then_some(params.mem), + storage: params.storage_gb, + overlay: params.overlay_gb, + force_extract: false, + info: false, + debug: false, + } + .run(); + } + + let mut mounts = HostMount::parse(¶ms.volume)?; + let ports = params.port.clone(); + PortMapping::check_duplicates(&ports) + .map_err(|e| smolvm::Error::config("validate ports", e))?; + + if self.docker_config { + if let Some(docker_mount) = docker_config_mount() { + mounts.push(docker_mount); + } else { + tracing::warn!("Docker config directory not found"); + } + } + + // Require an explicit command, -it flag, or Smolfile entrypoint/cmd. + // Without any of these, /bin/sh hangs waiting for input — confusing UX. + if self.detach && (self.interactive || self.tty) { + eprintln!("warning: -i/-t flags are ignored in detached mode (-d)"); + } + + let has_smolfile_command = !params.entrypoint.is_empty() || !params.cmd.is_empty(); + let (interactive, tty) = if !self.interactive + && !self.tty + && !self.detach + && self.command.is_empty() + && !has_smolfile_command + { + return Err(smolvm::Error::config( + "machine run", + "no command specified.\n\ + Use: smolvm machine run -- \n\ + Or: smolvm machine run -it", + )); + } else { + (self.interactive, self.tty) + }; + + // `--image -` consumes stdin to read the archive; `-i`/`-t` also bind + // stdin to the guest. They cannot both own stdin. + if self.image.as_deref() == Some("-") && (interactive || tty) { + return Err(smolvm::Error::config( + "machine run", + "`--image -` reads the image archive from stdin and cannot be \ + combined with -i/-t, which also use stdin.\n\ + Pipe the archive from a file instead: --image ./image.tar", + )); + } + + // Detect the common mistake of passing an image reference as a positional + // argument instead of using --image. clap's trailing_var_arg captures any + // positional before "--" into `command`, so `smolvm machine run ubuntu:22.04 + // -- bash` silently puts "ubuntu:22.04" into command[0] and fails with a + // confusing ENOENT after the VM boots. Catching the unambiguous cases + // (image:tag, registry/image) here avoids an unnecessary boot round-trip. + { + let resolved_image = self.image.as_deref().or(params.image.as_deref()); + if resolved_image.is_none() + && !self.command.is_empty() + && is_likely_image_ref(&self.command[0]) + { + let cmd0 = &self.command[0]; + // Strip the "--" separator that trailing_var_arg includes + // in the vec so the suggestion doesn't show a double "--". + let rest: Vec<&str> = self.command[1..] + .iter() + .filter(|s| s.as_str() != "--") + .map(|s| s.as_str()) + .collect(); + let suggestion = if rest.is_empty() { + format!("smolvm machine run --image {cmd0}") + } else { + format!("smolvm machine run --image {cmd0} -- {}", rest.join(" ")) + }; + return Err(Error::config( + "machine run", + format!( + "'{cmd0}' looks like a container image reference, not a command.\n\ + To run a container, use --image:\n {suggestion}" + ), + )); + } + } + + let resources = VmResources { + cpus: params.cpus, + memory_mib: params.mem, + network: params.net, + network_backend: params.network_backend, + dns: params.dns, + // CLI --gpu wins; Smolfile gpu = true also enables it. + gpu: self.gpu || params.gpu, + gpu_vram_mib: self.gpu_vram_mib.or(params.gpu_vram_mib), + rosetta: self.rosetta || params.rosetta, + storage_gib: params.storage_gb, + overlay_gib: params.overlay_gb, + allowed_cidrs: params.allowed_cidrs.clone(), + }; + validate_requested_network_backend( + &resources, + params.dns_filter_hosts.as_deref(), + params.port.len(), + )?; + + let manager = + AgentManager::for_vm_with_sizes(&vm_name, params.storage_gb, params.overlay_gb) + .map_err(|e| Error::agent("create agent manager", e.to_string()))?; + + if self.detach { + eprintln!("Starting persistent machine..."); + } else { + eprintln!("Starting ephemeral machine ({})...", vm_name); + } + + let ssh_agent_socket = if self.ssh_agent || params.ssh_agent { + match std::env::var("SSH_AUTH_SOCK") { + Ok(path) => Some(std::path::PathBuf::from(path)), + Err(_) => { + return Err(Error::config( + "--ssh-agent", + "SSH_AUTH_SOCK is not set. Start an SSH agent with: eval $(ssh-agent) && ssh-add", + )); + } + } + } else { + None + }; + + // Resolve the image source on the host before launch: registry refs + // pass through to the guest pull; a local `docker save` archive or an + // unpacked rootfs directory is staged/validated and mounted via + // virtiofs (the `.smolmachine` packed-layers path), so no pull happens. + let raw_image = self.image.clone().or(params.image.clone()); + let mut packed_layers_dir = None; + let image = match raw_image.as_deref() { + Some(img) => { + use smolvm::data::image_source::{classify, resolve, ResolvedImage}; + match resolve(classify(img))? { + ResolvedImage::Registry(reference) => Some(reference), + ResolvedImage::Local { + reference, + packed_layers_dir: dir, + } => { + packed_layers_dir = Some(dir); + Some(reference) + } + } + } + None => None, + }; + let uses_packed_layers = packed_layers_dir.is_some(); + + let mut features = smolvm::agent::LaunchFeatures { + ssh_agent_socket, + cuda: self.cuda || params.cuda, + dns_filter_hosts: params.dns_filter_hosts.clone(), + packed_layers_dir, + extra_disks: Vec::new(), + ..Default::default() + }; + + // This launch pulls a registry image in-guest, subject to the egress + // filter — fold its registry into the enforced policy so a hostname + // scope doesn't block its own pull. + features.allow_image_pull_egress(image.as_deref(), uses_packed_layers); + + let freshly_started = manager + .ensure_running_with_full_config(mounts.clone(), ports, resources, features) + .map_err(|e| Error::agent("start machine", e.to_string()))?; + + // Register the ephemeral VM for tracking (machine list, orphan cleanup), + // keyed by the VM's OWN name. The orphan sweep only has the DB record and + // locates the disks via `vm_data_dir(record.name)`, so the record name + // MUST be the VM name — a separate generated name would hash to a + // different, nonexistent dir and the sweep would delete the record but + // leak the real (multi-GB) data dir. + // + // Detached runs are tracked via persist_named_running instead — skip + // ephemeral registration so the detach path does not leave an + // unreachable orphan record after persist_named_running succeeds. + if !self.detach { + vm_common::register_ephemeral_vm( + &vm_name, + manager.child_pid(), + params.cpus, + params.mem, + params.net, + image.clone(), + ); + } + + let mut client = AgentClient::connect_with_retry(manager.vsock_socket())?; + + // Install SIGINT guard so Ctrl+C during pull kills the VM process + // instead of orphaning it. The guard is disarmed before interactive + // exec (which has its own SIGINT handling). + let sigint_guard = manager.child_pid().map(smolvm::process::SigintGuard::new); + + // Resolve image: CLI > Smolfile > None (bare VM) + // When Rosetta is enabled, default the image pull to linux/amd64 so there + // is an x86_64 binary to translate; an explicit --oci-platform still wins. + // Without this, a multi-arch image resolves to the guest-native arm64 + // variant and Rosetta has nothing to do. + let rosetta_requested = self.rosetta || params.rosetta; + let effective_platform: Option = self + .oci_platform + .clone() + .or_else(|| rosetta_requested.then(|| "linux/amd64".to_string())); + + // Pull only registry images; a local source's layers are already + // mounted via virtiofs and the guest assembles its rootfs from them. + let image_info = if uses_packed_layers { + None + } else if let Some(ref img) = image { + match crate::cli::pull_with_progress( + &mut client, + img, + effective_platform.as_deref(), + self.proxy_opts.proxy(), + self.proxy_opts.no_proxy(), + ) { + Ok(info) => Some(info), + Err(e) if !params.net => { + // Add a hint when pull fails and networking is disabled — + // this is the most common user error. + return Err(smolvm::Error::agent( + "pull image", + format!( + "{}\n\nHint: networking is disabled. Add --net to enable image pulls:\n smolvm machine run --net --image {} ...", + e, img + ), + )); + } + Err(e) => return Err(e), + } + } else { + None + }; + + // Resolve Smolfile [secrets] for this launch. Tuples are plaintext; + // do not log them. Zeroizing buffers were scrubbed inside the helper. + // These are merged into `env`/`init_env` below but never flow into + // `params.env`, so the plaintext values never touch the persisted + // VM record — only the refs are stored (via DefaultVmOverrides), and + // they get re-resolved at each subsequent `machine start`. + let resolved_secrets = vm_common::resolve_secret_refs_for_env(¶ms.secret_refs)?; + + if freshly_started && !params.init.is_empty() { + // Route through `run_init_commands` so init runs inside the + // container when an image is set (so package managers like + // pacman/apt/dnf resolve against the image's rootfs), and + // in the bare agent otherwise. The persistent `start_*` + // paths use the same helper — keep parity. + // + // Convert the parsed HostMount list into the record-shape + // tuples the runner expects. This is a thin local conversion; + // the runner does its own tag assignment internally so call + // sites don't have to track which form the agent wants. + let record_mounts: Vec<(String, String, bool)> = mounts + .iter() + .map(|m| { + ( + m.source.to_string_lossy().into_owned(), + m.target.to_string_lossy().into_owned(), + m.read_only, + ) + }) + .collect(); + let mut init_env = parse_env_list(¶ms.env); + init_env.extend(resolved_secrets.iter().cloned()); + // Use the machine name as the overlay ID so any rootfs changes + // init makes (e.g. `pacman -S git`) are visible to a + // subsequent `machine exec`. The exec path resolves the + // overlay from the machine name, falling back to "default", + // so matching that name here is what makes init's effects + // observable to the user. + if let Err(e) = vm_common::run_init_commands( + &mut client, + ¶ms.init, + vm_common::InitRunContext { + image: image.as_deref(), + image_info: image_info.as_ref(), + env: &init_env, + workdir: params.workdir.as_deref(), + record_mounts: &record_mounts, + overlay_id: &vm_name, + }, + ) { + // Ephemeral VMs have no state to preserve — `kill()` + // matches the success path's lifetime semantics + // (manager.kill() at line ~563/655) and avoids the + // graceful-shutdown latency `stop()` adds when no one + // is going to use this VM again. + vm_common::deregister_ephemeral_vm(&vm_name); + manager.kill(); + return Err(e); + } + } + + // Resolve command: CLI trailing args > Smolfile entrypoint+cmd > image metadata > defaults + let command = if !self.command.is_empty() { + self.command.clone() + } else if !params.entrypoint.is_empty() || !params.cmd.is_empty() { + let mut cmd = params.entrypoint.clone(); + cmd.extend(params.cmd.clone()); + cmd + } else if let Some(ref info) = image_info { + let mut cmd = info.entrypoint.clone(); + cmd.extend(info.cmd.clone()); + if cmd.is_empty() { + if self.detach { + DEFAULT_IDLE_CMD.iter().map(|s| s.to_string()).collect() + } else { + vec![DEFAULT_SHELL_CMD.to_string()] + } + } else { + cmd + } + } else if self.detach { + DEFAULT_IDLE_CMD.iter().map(|s| s.to_string()).collect() + } else { + vec![DEFAULT_SHELL_CMD.to_string()] + }; + + let mut env = parse_env_list(¶ms.env); + env.extend(resolved_secrets.iter().cloned()); + let mount_bindings = mounts_to_virtiofs_bindings(&mounts); + + // Two modes: with image or bare VM (no image) + if let Some(ref img) = image { + let defaults = vm_common::resolve_image_runtime_defaults( + image_info.as_ref(), + &env, + params.workdir.as_deref(), + ); + if self.detach { + // Start the main workload container first. If this fails, the + // VM is stopped and no DB record is written — a retry won't + // hit "machine already exists." + { + let run_config = smolvm::agent::RunConfig::new(img.clone(), command.clone()) + .with_env(defaults.env.clone()) + .with_workdir(defaults.workdir.clone()) + .with_user(defaults.user.clone()) + .with_mounts(mount_bindings.clone()) + .with_persistent_overlay(Some(vm_name.clone())) + .with_unprivileged(self.unprivileged); + client.run_container_detached(run_config)?; + } + + // Container started — persist the DB record. If this fails, + // stop the VM to avoid an orphan that lifecycle commands can't find. + { + use smolvm::config::SmolvmConfig; + use vm_common::DefaultVmOverrides; + let mount_tuples: Vec<(String, String, bool)> = mounts + .iter() + .map(|m| { + ( + m.source.to_string_lossy().to_string(), + m.target.to_string_lossy().to_string(), + m.read_only, + ) + }) + .collect(); + let port_tuples: Vec<(u16, u16)> = + params.port.iter().map(|p| (p.host, p.guest)).collect(); + let persist_result = SmolvmConfig::load().and_then(|mut config| { + vm_common::persist_named_running( + &mut config, + &vm_name, + manager.child_pid(), + Some(DefaultVmOverrides { + // Persist the REFS (re-resolved at each start via + // record_env_with_secrets), never the resolved + // plaintext — see `env` below. + secret_refs: params.secret_refs.clone(), + cpus: params.cpus, + mem: params.mem, + mounts: mount_tuples, + ports: port_tuples, + network: params.net, + network_backend: params.network_backend, + dns: params.dns, + storage_gb: params.storage_gb, + overlay_gb: params.overlay_gb, + allowed_cidrs: params.allowed_cidrs.clone(), + init: params.init.clone(), + // Strip resolved secret values so plaintext never + // reaches the DB/pack record. defaults.env still + // carries them for RUNNING the container above; the + // record keeps only refs + non-secret env. + env: defaults + .env + .iter() + .filter(|(k, _)| !params.secret_refs.contains_key(k)) + .cloned() + .collect(), + workdir: defaults.workdir.clone(), + user: defaults.user.clone(), + image: Some(img.clone()), + entrypoint: Vec::new(), + cmd: command.clone(), + ssh_agent: self.ssh_agent || params.ssh_agent, + cuda: self.cuda || params.cuda, + dns_filter_hosts: params.dns_filter_hosts.clone(), + gpu: self.gpu || params.gpu, + gpu_vram_mib: self.gpu_vram_mib.or(params.gpu_vram_mib), + rosetta: self.rosetta || params.rosetta, + }), + ) + }); + if let Err(e) = persist_result { + let _ = manager.stop(); + return Err(Error::config( + "persist machine record", + format!("VM started but record could not be saved: {}. VM stopped to avoid orphan.", e), + )); + } + } + + // Disarm SIGINT guard — detaching, VM stays running. + drop(sigint_guard); + + if vm_name == "default" { + println!("Machine running in background"); + println!("\nTo interact:"); + println!(" smolvm machine exec -- "); + println!("\nTo stop:"); + println!(" smolvm machine stop"); + } else { + println!("Machine '{}' running in background", vm_name); + println!("\nTo interact:"); + println!(" smolvm machine exec --name {} -- ", vm_name); + println!("\nTo stop:"); + println!(" smolvm machine stop --name {}", vm_name); + } + + manager.detach(); + Ok(()) + } else { + // Disarm SIGINT guard — exec phase has its own signal handling. + if let Some(guard) = sigint_guard { + guard.disarm(); + } + + // Use the machine's persistent overlay so the foreground workload + // sees init's filesystem changes (init ran with the same overlay id). + // Without this the workload runs in a fresh overlay and init appears + // to "do nothing" (e.g. `apt install`ed binaries are missing). + let exit_code = if interactive || tty { + let config = RunConfig::new(img, command) + .with_env(defaults.env.clone()) + .with_workdir(defaults.workdir.clone()) + .with_user(defaults.user.clone()) + .with_mounts(mount_bindings) + .with_timeout(self.timeout) + .with_tty(tty) + .with_persistent_overlay(Some(vm_name.clone())) + .with_unprivileged(self.unprivileged); + client.run_interactive(config)? + } else { + let config = RunConfig::new(img, command) + .with_env(defaults.env) + .with_workdir(defaults.workdir) + .with_user(defaults.user) + .with_mounts(mount_bindings) + .with_timeout(self.timeout) + .with_persistent_overlay(Some(vm_name.clone())) + .with_unprivileged(self.unprivileged); + let (exit_code, stdout, stderr) = client.run_non_interactive(config)?; + if !stdout.is_empty() { + let _ = std::io::stdout().write_all(&stdout); + } + if !stderr.is_empty() { + let _ = std::io::stderr().write_all(&stderr); + } + flush_output(); + exit_code + }; + + // Ephemeral run — tear down VM and its data directory. + // Spawn a detached helper so the parent exits immediately after + // flushing output. Falls back to synchronous cleanup if spawn fails. + let (pid, start_time) = manager.pid_and_start_time().unwrap_or((0, None)); + if pid > 0 && try_spawn_detached_cleanup(&vm_name, pid, start_time) { + std::process::exit(exit_code); + } + // Fallback: synchronous cleanup (helper spawn failed). + vm_common::deregister_ephemeral_vm(&vm_name); + manager.kill(); + manager.cleanup_data_dir(); + std::process::exit(exit_code); + } + } else { + // Bare VM mode (no image) — disarm SIGINT guard before exec. + if let Some(guard) = sigint_guard { + guard.disarm(); + } + + if self.detach { + // Run entrypoint+cmd in background if present + let is_idle = command.is_empty() + || command + == DEFAULT_IDLE_CMD + .iter() + .map(|s| s.to_string()) + .collect::>(); + if !is_idle { + let pid = client.vm_exec_background(command, env, params.workdir.clone())?; + tracing::info!(pid = pid, "background workload started"); + } + + // Persist the VM state so it survives stop/start. + { + use smolvm::config::SmolvmConfig; + use vm_common::DefaultVmOverrides; + let mount_tuples: Vec<(String, String, bool)> = mounts + .iter() + .map(|m| { + ( + m.source.to_string_lossy().to_string(), + m.target.to_string_lossy().to_string(), + m.read_only, + ) + }) + .collect(); + let port_tuples: Vec<(u16, u16)> = + params.port.iter().map(|p| (p.host, p.guest)).collect(); + let mut config = SmolvmConfig::load()?; + vm_common::persist_named_running( + &mut config, + &vm_name, + manager.child_pid(), + Some(DefaultVmOverrides { + // Persist the refs so secrets re-resolve on restart + // (env below is already secret-free: parse_env_list). + secret_refs: params.secret_refs.clone(), + cpus: params.cpus, + mem: params.mem, + mounts: mount_tuples, + ports: port_tuples, + network: params.net, + network_backend: params.network_backend, + dns: params.dns, + storage_gb: params.storage_gb, + overlay_gb: params.overlay_gb, + allowed_cidrs: params.allowed_cidrs.clone(), + init: params.init.clone(), + env: parse_env_list(¶ms.env), + workdir: params.workdir.clone(), + user: None, + image: None, + entrypoint: params.entrypoint.clone(), + cmd: params.cmd.clone(), + ssh_agent: self.ssh_agent || params.ssh_agent, + cuda: self.cuda || params.cuda, + dns_filter_hosts: params.dns_filter_hosts.clone(), + gpu: self.gpu || params.gpu, + gpu_vram_mib: self.gpu_vram_mib.or(params.gpu_vram_mib), + rosetta: false, + }), + )?; + } + + if vm_name == "default" { + println!( + "Machine running (PID: {})", + manager.child_pid().unwrap_or(0) + ); + println!("\nTo interact:"); + println!(" smolvm machine exec -- "); + println!("\nTo stop:"); + println!(" smolvm machine stop"); + } else { + println!( + "Machine '{}' running (PID: {})", + vm_name, + manager.child_pid().unwrap_or(0) + ); + println!("\nTo interact:"); + println!(" smolvm machine exec --name {} -- ", vm_name); + println!("\nTo stop:"); + println!(" smolvm machine stop --name {}", vm_name); + } + + manager.detach(); + Ok(()) + } else { + let exit_code = if interactive || tty { + client.vm_exec_interactive( + command, + env, + params.workdir.clone(), + self.timeout, + tty, + )? + } else { + // Capture for error context before command is moved into vm_exec. + let cmd0 = command.first().cloned().unwrap_or_default(); + let (exit_code, stdout, stderr) = client + .vm_exec(command, env, params.workdir.clone(), self.timeout, None) + .map_err(|e| { + // In bare VM mode a spawn ENOENT often means the user + // forgot --image and passed the image name as a positional. + // Name the command that wasn't found so the hint is actionable. + let msg = e.to_string(); + if image.is_none() + && (msg.contains("No such file or directory") + || msg.contains("os error 2")) + && !cmd0.starts_with('/') + && !cmd0.starts_with('.') + { + Error::agent( + "vm exec", + format!( + "{msg}\n\nNote: '{cmd0}' was not found in the VM. \ + If you meant to run a container image, use --image:\n \ + smolvm machine run --image {cmd0} -- " + ), + ) + } else { + e + } + })?; + if !stdout.is_empty() { + let _ = std::io::stdout().write_all(&stdout); + } + if !stderr.is_empty() { + let _ = std::io::stderr().write_all(&stderr); + } + flush_output(); + exit_code + }; + // Ephemeral run — tear down VM and its data directory. + // Spawn a detached helper so the parent exits immediately after + // flushing output. Falls back to synchronous cleanup if spawn fails. + let (pid, start_time) = manager.pid_and_start_time().unwrap_or((0, None)); + if pid > 0 && try_spawn_detached_cleanup(&vm_name, pid, start_time) { + std::process::exit(exit_code); + } + // Fallback: synchronous cleanup (helper spawn failed). + vm_common::deregister_ephemeral_vm(&vm_name); + manager.kill(); + manager.cleanup_data_dir(); + std::process::exit(exit_code); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use clap::Parser; + + #[test] + fn init_layer_key_is_stable_and_input_sensitive() { + let init = vec!["apt-get install -y jq".to_string()]; + let env = vec!["FOO=bar".to_string()]; + let base = init_layer_key(Some("ubuntu:noble"), &init, &env); + // Deterministic for identical inputs. + assert_eq!(base, init_layer_key(Some("ubuntu:noble"), &init, &env)); + assert_eq!(base.len(), 16); + // Sensitive to each input: image, init, env. + assert_ne!(base, init_layer_key(Some("ubuntu:jammy"), &init, &env)); + assert_ne!( + base, + init_layer_key(Some("ubuntu:noble"), &["other".to_string()], &env) + ); + assert_ne!( + base, + init_layer_key(Some("ubuntu:noble"), &init, &["FOO=baz".to_string()]) + ); + // Order of init steps matters (different layer). + let init_rev = vec!["b".to_string(), "a".to_string()]; + let init_fwd = vec!["a".to_string(), "b".to_string()]; + assert_ne!( + init_layer_key(Some("x"), &init_fwd, &[]), + init_layer_key(Some("x"), &init_rev, &[]) + ); + } + + #[test] + fn parse_cli_secret_refs_builds_env_and_file_refs() { + let refs = parse_cli_secret_refs( + &["GUEST_TOKEN=HOST_TOKEN".to_string()], + &["GUEST_KEY=/abs/key".to_string()], + ) + .unwrap(); + assert_eq!(refs["GUEST_TOKEN"].from_env.as_deref(), Some("HOST_TOKEN")); + assert_eq!( + refs["GUEST_KEY"] + .from_file + .as_ref() + .map(|p| p.to_string_lossy().into_owned()), + Some("/abs/key".to_string()) + ); + } + + #[test] + fn parse_cli_secret_refs_rejects_bad_specs() { + // Missing '='. + assert!(parse_cli_secret_refs(&["NO_EQUALS".to_string()], &[]).is_err()); + // Empty key. + assert!(parse_cli_secret_refs(&["=HOST".to_string()], &[]).is_err()); + // Relative from_file path (validate_ref under TrustedLocal). + assert!(parse_cli_secret_refs(&[], &["K=relative/path".to_string()]).is_err()); + // Duplicate key across the two flags. + assert!( + parse_cli_secret_refs(&["DUP=HOST".to_string()], &["DUP=/abs/path".to_string()]) + .is_err() + ); + } + + #[derive(Parser, Debug)] + #[command(name = "machine")] + struct TestMachineCli { + #[command(subcommand)] + command: MachineCmd, + } + + #[test] + fn run_detach_accepts_name_flag() { + let cli = TestMachineCli::parse_from([ + "machine", "run", "-d", "--name", "foo", "--image", "alpine", + ]); + + let MachineCmd::Run(cmd) = cli.command else { + panic!("expected machine run command"); + }; + assert_eq!(cmd.name, Some("foo".to_string())); + assert!(cmd.detach); + } + + // Documents the clap parsing behaviour: positionals before "--" land in + // `command`, not `image`. is_likely_image_ref() catches the unambiguous + // cases before a VM is booted. + #[test] + fn run_image_ref_as_positional_lands_in_command_vec() { + let cli = TestMachineCli::parse_from(["machine", "run", "ubuntu:22.04", "--", "bash"]); + let MachineCmd::Run(cmd) = cli.command else { + panic!("expected machine run command"); + }; + assert_eq!(cmd.image, None); + // With trailing_var_arg, clap includes the "--" separator in the vec. + assert_eq!(cmd.command, ["ubuntu:22.04", "--", "bash"]); + // is_likely_image_ref catches this before the VM starts + assert!(is_likely_image_ref(&cmd.command[0])); + } + + #[test] + fn create_accepts_trailing_workload_command() { + let cli = TestMachineCli::parse_from([ + "machine", "create", "--name", "golden", "--image", "alpine", "--", "echo", "hi", + ]); + let MachineCmd::Create(cmd) = cli.command else { + panic!("expected machine create command"); + }; + assert_eq!(cmd.name, Some("golden".to_string())); + assert_eq!(cmd.image, Some("alpine".to_string())); + // The trailing command is captured (clap may include the "--" separator). + let words: Vec<&str> = cmd + .command + .iter() + .map(String::as_str) + .filter(|s| *s != "--") + .collect(); + assert_eq!(words, ["echo", "hi"]); + } + + #[test] + fn create_without_command_leaves_command_empty() { + // Regression: adding the trailing COMMAND arg must not break the common + // no-command form `machine create --name --net`. + let cli = TestMachineCli::parse_from(["machine", "create", "--name", "golden", "--net"]); + let MachineCmd::Create(cmd) = cli.command else { + panic!("expected machine create command"); + }; + assert_eq!(cmd.name, Some("golden".to_string())); + assert!(cmd.command.is_empty()); + assert!(cmd.net); + } + + #[test] + fn create_rejects_bare_positional_name() { + // Machine names are flags everywhere (issue #370). A bare positional — + // the old `machine create myvm` habit — must error, not be silently + // captured as the workload command. + assert!(TestMachineCli::try_parse_from(["machine", "create", "myvm"]).is_err()); + } + + #[test] + fn is_likely_image_ref_classifies_correctly() { + // Unambiguous image references + assert!(is_likely_image_ref("ubuntu:22.04")); // image:tag + assert!(is_likely_image_ref("ghcr.io/org/image")); // registry/path + assert!(is_likely_image_ref("library/alpine")); // namespace/image + + // Bare names are not flagged — indistinguishable from commands at parse time + assert!(!is_likely_image_ref("alpine")); + assert!(!is_likely_image_ref("bash")); + + // Absolute and relative paths are always commands + assert!(!is_likely_image_ref("/bin/sh")); + assert!(!is_likely_image_ref("./script.sh")); + } +} + +// ============================================================================ +// Exec Command (Persistent) - Direct VM Execution +// ============================================================================ + +/// Execute a command directly in the VM's Alpine rootfs. +/// +/// This runs commands at the VM level, not inside a container. Useful for +/// debugging, inspecting the VM environment, or running VM-level operations. +/// +/// Examples: +/// smolvm machine exec -- uname -a +/// smolvm machine exec --name myvm -- df -h +/// smolvm machine exec -it -- /bin/sh +#[derive(Args, Debug)] +pub struct ExecCmd { + /// Command and arguments to execute + #[arg(trailing_var_arg = true, required = true, value_name = "COMMAND")] + pub command: Vec, + + /// Target machine (default: "default") + #[arg(long, value_name = "NAME")] + pub name: Option, + + /// Set working directory in the VM + #[arg(short = 'w', long, value_name = "DIR")] + pub workdir: Option, + + /// Set environment variable (can be used multiple times) + #[arg(short = 'e', long = "env", value_name = "KEY=VALUE")] + pub env: Vec, + + /// Inject a secret from a host env var (GUEST_VAR=HOST_VAR) for this exec, + /// resolved on the host. The value never persists to the record. + #[arg(long = "secret-env", value_name = "GUEST_VAR=HOST_VAR")] + pub secret_env: Vec, + + /// Inject a secret from a host file (GUEST_VAR=/abs/path) for this exec, + /// resolved on the host. The value never persists to the record. + #[arg(long = "secret-file", value_name = "GUEST_VAR=PATH")] + pub secret_file: Vec, + + /// Kill command after duration (e.g., "30s", "5m") + #[arg(long, value_parser = parse_duration, value_name = "DURATION")] + pub timeout: Option, + + /// Keep stdin open for interactive input + #[arg(short = 'i', long)] + pub interactive: bool, + + /// Allocate a pseudo-TTY (use with -i for shells) + #[arg(short = 't', long)] + pub tty: bool, + + /// Stream output in real-time (prints as it arrives) + #[arg(long)] + pub stream: bool, + + /// Detach: spawn the command in the background and return its PID + /// immediately. The process keeps running (it is not killed when this + /// command returns), so it can host long-lived services — e.g. a server + /// bound to a published port. Incompatible with -i/-t and --stream. + #[arg(short = 'd', long, conflicts_with_all = ["interactive", "tty", "stream"])] + pub detach: bool, +} + +impl ExecCmd { + pub fn run(self) -> smolvm::Result<()> { + let (manager, mut client) = vm_common::ensure_running_and_connect(&self.name)?; + + // Detach immediately — exec never owns the VM lifecycle. Without this, + // any early return (failed exec, timeout, client signal) triggers + // AgentManager::Drop which calls stop() and kills the VM. + manager.detach(); + + let env = parse_env_list(&self.env); + + // Load machine record for workdir and image info + let name = self.name.clone().unwrap_or_else(|| "default".to_string()); + let record = smolvm::db::SmolvmDb::open() + .ok() + .and_then(|db| db.get_vm(&name).ok().flatten()); + + // Resolve workdir: CLI --workdir flag takes priority over Smolfile/machine config + let workdir = self + .workdir + .clone() + .or_else(|| record.as_ref().and_then(|r| r.workdir.clone())); + let record_image = record.as_ref().and_then(|r| r.image.clone()); + + // Check if this machine has an image — if so, exec inside the image's + // rootfs via client.run_interactive()/run_non_interactive() instead of bare vm_exec(). + let mount_bindings = record + .as_ref() + .map(|r| mounts_to_virtiofs_bindings(&r.host_mounts())) + .unwrap_or_default(); + + // Base env for the exec: the record's persisted `env` plus its + // `secret_refs` resolved to plaintext on the host (RecordReplay scope). + // CLI `--env` flags are layered on top via `merge_env_overrides`. The + // resolved plaintext lives only in this local for the exec's duration — + // it is never written back to the record or the DB. + let mut record_env: Vec<(String, String)> = match record.as_ref() { + Some(r) => vm_common::record_env_with_secrets(r)?, + None => Vec::new(), + }; + // Ad-hoc `--secret-env`/`--secret-file` refs for this exec only. The CLI + // user is TrustedLocal; resolved plaintext lives only in this local and + // is layered under any explicit `--env` overrides below. + let exec_secret_refs = parse_cli_secret_refs(&self.secret_env, &self.secret_file)?; + record_env.extend(smolvm::secrets::expose_into_env( + smolvm::secrets::resolve_refs_to_env( + &exec_secret_refs, + smolvm::secrets::ResolutionScope::TrustedLocal, + )?, + )); + + if let Some(ref image) = record_image { + let image_info = match client.query(image) { + Ok(info) => info, + Err(e) => { + tracing::debug!( + error = %e, + image = %image, + "failed to query local image metadata" + ); + None + } + }; + let configured_env = vm_common::merge_env_overrides(&record_env, &env); + let defaults = vm_common::resolve_image_runtime_defaults( + image_info.as_ref(), + &configured_env, + workdir.as_deref(), + ); + // Image-based machine: exec inside the image's rootfs via crun. + // Use machine name as persistent overlay ID so filesystem changes + // (e.g. package installs) survive across exec sessions. + let machine_name = name.clone(); + if self.detach { + let config = smolvm::agent::RunConfig::new(image, self.command.clone()) + .with_env(defaults.env) + .with_workdir(defaults.workdir) + .with_user(defaults.user) + .with_mounts(mount_bindings) + .with_persistent_overlay(Some(machine_name)); + let pid = client.run_background(config)?; + println!("{pid}"); + return Ok(()); + } + if self.interactive || self.tty { + let config = smolvm::agent::RunConfig::new(image, self.command.clone()) + .with_env(defaults.env.clone()) + .with_workdir(defaults.workdir.clone()) + .with_user(defaults.user.clone()) + .with_mounts(mount_bindings) + .with_timeout(self.timeout) + .with_tty(self.tty) + .with_persistent_overlay(Some(machine_name.clone())); + let exit_code = client.run_interactive(config)?; + std::process::exit(exit_code); + } + + if self.stream { + let config = smolvm::agent::RunConfig::new(image, self.command.clone()) + .with_env(defaults.env.clone()) + .with_workdir(defaults.workdir.clone()) + .with_user(defaults.user.clone()) + .with_mounts(mount_bindings) + .with_timeout(self.timeout) + .with_persistent_overlay(Some(machine_name.clone())); + let mut printer = ExecEventPrinter::default(); + client.run_streaming_with(config, |event| printer.handle(event))?; + std::process::exit(printer.exit_code); + } + + let config = smolvm::agent::RunConfig::new(image, self.command.clone()) + .with_env(defaults.env) + .with_workdir(defaults.workdir) + .with_user(defaults.user) + .with_mounts(mount_bindings) + .with_timeout(self.timeout) + .with_persistent_overlay(Some(machine_name)); + let (exit_code, stdout, stderr) = client.run_non_interactive(config)?; + vm_common::print_output_and_exit(&manager, exit_code, &stdout, &stderr); + } else { + // Bare VM: exec directly in the VM rootfs. + // Merge record env + resolved secrets with CLI env, same as image path. + let env = vm_common::merge_env_overrides(&record_env, &env); + if self.detach { + // Spawn detached in the guest root netns — no setsid/killpg, so + // a daemon (e.g. a server on a published port) survives. TSI sees + // the listen() and opens the host-side forward. + let pid = client.vm_exec_background(self.command.clone(), env, workdir.clone())?; + println!("{pid}"); + return Ok(()); + } + if self.interactive || self.tty { + let exit_code = client.vm_exec_interactive( + self.command.clone(), + env.clone(), + workdir.clone(), + self.timeout, + self.tty, + )?; + std::process::exit(exit_code); + } + + if self.stream { + let mut printer = ExecEventPrinter::default(); + client.vm_exec_streaming_with( + self.command.clone(), + env.clone(), + workdir.clone(), + self.timeout, + |event| printer.handle(event), + )?; + std::process::exit(printer.exit_code); + } + + let (exit_code, stdout, stderr) = client.vm_exec( + self.command.clone(), + env, + workdir.clone(), + self.timeout, + None, + )?; + vm_common::print_output_and_exit(&manager, exit_code, &stdout, &stderr); + } + } +} + +#[derive(Default)] +struct ExecEventPrinter { + exit_code: i32, +} + +impl ExecEventPrinter { + fn handle(&mut self, event: smolvm::agent::ExecEvent) { + match event { + smolvm::agent::ExecEvent::Stdout(data) => { + let _ = std::io::stdout().write_all(&data); + let _ = std::io::stdout().flush(); + } + smolvm::agent::ExecEvent::Stderr(data) => { + let _ = std::io::stderr().write_all(&data); + let _ = std::io::stderr().flush(); + } + smolvm::agent::ExecEvent::Exit(code) => { + self.exit_code = code; + } + smolvm::agent::ExecEvent::Error(msg) => { + eprintln!("error: {}", msg); + self.exit_code = 1; + } + } + } +} + +// ============================================================================ +// Shell Command +// ============================================================================ + +/// Open an interactive shell in a machine. +/// +/// Shortcut for `machine exec -it -- /bin/sh`. Starts the machine if stopped. +/// +/// Examples: +/// smolvm machine shell +/// smolvm machine shell --name myvm +/// smolvm machine sh --name myvm +#[derive(Args, Debug)] +pub struct ShellCmd { + /// Target machine (default: "default") + #[arg(long, short = 'n', value_name = "NAME")] + pub name: Option, +} + +impl ShellCmd { + pub fn run(self) -> smolvm::Result<()> { + // Delegate to exec with -it -- /bin/sh + ExecCmd { + command: vec!["/bin/sh".to_string()], + name: self.name, + workdir: None, + env: vec![], + secret_env: vec![], + secret_file: vec![], + timeout: None, + interactive: true, + tty: true, + stream: false, + detach: false, + } + .run() + } +} + +// ============================================================================ +// Create Command +// ============================================================================ + +/// Create a named machine configuration. +/// +/// Creates a persistent VM configuration that can be started later. +/// Use `smolvm machine start --name ` to start, then +/// `smolvm machine exec --name -- ` to run commands inside. +/// +/// Examples: +/// smolvm machine create --name myvm +/// smolvm machine create --name webserver --cpus 2 --mem 1024 -p 80:80 +#[derive(Args, Debug)] +pub struct CreateCmd { + /// Name for the machine (auto-generated if omitted) + #[arg(short = 'n', long, value_name = "NAME")] + pub name: Option, + + /// Container image (e.g., alpine, python:3.12-alpine) + #[arg(short = 'I', long, value_name = "IMAGE", value_parser = parse_image)] + pub image: Option, + + /// Raise the max accepted local image-archive size (e.g. 16GiB, 512M, or a + /// raw byte count); default 8GiB. For legitimately large images — sets + /// SMOLVM_MAX_IMAGE_BYTES for this run. + #[arg(long = "max-image-size", value_name = "SIZE", + value_parser = crate::cli::parsers::parse_size_bytes)] + pub max_image_size: Option, + + /// Number of virtual CPUs + #[arg(long, default_value_t = DEFAULT_MICROVM_CPU_COUNT, value_name = "N")] + pub cpus: u8, + + /// Memory allocation in MiB + #[arg(long, default_value_t = DEFAULT_MICROVM_MEMORY_MIB, value_name = "MiB")] + pub mem: u32, + + /// Storage disk size in GiB (for OCI layers and container data) + #[arg(long, value_name = "GiB")] + pub storage: Option, + + /// Overlay disk size in GiB (for persistent rootfs changes) + #[arg(long, value_name = "GiB")] + pub overlay: Option, + + /// Mount host directory (can be used multiple times) + #[arg(short = 'v', long = "volume", value_name = "HOST:GUEST[:ro]")] + pub volume: Vec, + + /// Expose port from VM to host (can be used multiple times) + #[arg(short = 'p', long = "port", value_parser = PortMapping::parse, value_name = "HOST:GUEST")] + pub port: Vec, + + /// Enable outbound network access + #[arg(long)] + pub net: bool, + + /// Select the networking backend. + #[arg(long = "net-backend", value_enum)] + pub net_backend: Option, + + /// Custom DNS resolver for the guest (implies --net). Use this when the + /// default public resolvers (8.8.8.8/1.1.1.1) are blocked on your network. + #[arg(long, value_name = "IP")] + pub dns: Option, + + /// Allow egress to specific CIDR range (can be used multiple times, implies --net) + #[arg(long = "allow-cidr", value_parser = parse_cidr, value_name = "CIDR")] + pub allow_cidr: Vec, + + /// Allow egress to specific hostname, resolved at VM start (can be used multiple times, implies --net) + #[arg(long = "allow-host", value_name = "HOSTNAME")] + pub allow_host: Vec, + + /// Restrict outbound to localhost only (implies --net) + #[arg(long)] + pub outbound_localhost_only: bool, + + /// Enable GPU acceleration (Vulkan via virtio-gpu) + #[arg(long)] + pub gpu: bool, + + /// GPU shared-memory region size in MiB. Ignored without --gpu. + /// Default 4096 (4 GiB). Must be > 0. + #[arg( + long = "gpu-vram", + value_name = "MiB", + value_parser = crate::cli::parsers::parse_gpu_vram_mib, + )] + pub gpu_vram_mib: Option, + + /// Enable Rosetta 2 for x86_64 binary translation on Apple Silicon + #[arg(long)] + pub rosetta: bool, + + /// Run command on every VM start (can be used multiple times) + #[arg(long = "init", value_name = "COMMAND")] + pub init: Vec, + + /// Set environment variable (can be used multiple times) + #[arg(short = 'e', long = "env", value_name = "KEY=VALUE")] + pub env: Vec, + + /// Set working directory inside the machine + #[arg(short = 'w', long = "workdir", value_name = "DIR")] + pub workdir: Option, + + /// Forward host SSH agent into the VM (enables git/ssh without exposing keys) + #[arg(long)] + pub ssh_agent: bool, + + /// Remote guest CUDA Driver-API calls to the host NVIDIA GPU over vsock + #[arg(long)] + pub cuda: bool, + + /// Inject a secret from a host env var (GUEST_VAR=HOST_VAR), resolved at + /// each launch. Only the reference is persisted, never the value. + #[arg(long = "secret-env", value_name = "GUEST_VAR=HOST_VAR")] + pub secret_env: Vec, + + /// Inject a secret from a host file (GUEST_VAR=/abs/path), resolved at + /// each launch. Only the reference is persisted, never the value. + #[arg(long = "secret-file", value_name = "GUEST_VAR=PATH")] + pub secret_file: Vec, + + /// Load configuration from a Smolfile (TOML) + #[arg(long = "smolfile", visible_short_alias = 's', value_name = "PATH")] + pub smolfile: Option, + + /// Create machine from a packed .smolmachine artifact. + /// Uses pre-extracted layers instead of pulling from a registry. + #[arg(long, value_name = "PATH", conflicts_with_all = ["image", "smolfile"])] + pub from: Option, + + /// Command to run as the machine's persistent workload (image machines). + /// Launched as a detached container on every `start`, so it stays running + /// (e.g. a pre-warmed browser to be forked). Without this, an image machine + /// boots to a bare agent and the image's CMD is not run. + /// + /// `last = true` requires the `--` separator. With the machine name now a + /// flag, a bare positional (an old-style `machine create myvm`) must fail + /// loudly instead of being silently captured as the workload command. + #[arg(last = true, value_name = "COMMAND")] + pub command: Vec, +} + +impl CreateCmd { + pub fn run(self) -> smolvm::Result<()> { + // --max-image-size raises the archive cap for this invocation by setting + // the env var the resolver reads (image_source::max_archive_bytes). + if let Some(bytes) = self.max_image_size { + std::env::set_var("SMOLVM_MAX_IMAGE_BYTES", bytes.to_string()); + } + // Branch for --from: create machine from .smolmachine artifact. + if let Some(ref sidecar_path) = self.from { + return self.run_from_smolmachine(sidecar_path); + } + + let (cli_allow_cidrs, net, cli_dns_filter_hosts) = resolve_egress_flags( + self.allow_cidr, + self.allow_host, + self.outbound_localhost_only, + self.net, + )?; + + let name = self + .name + .unwrap_or_else(smolvm::util::generate_machine_name); + + // Resolve a local image source (archive/dir) on the host now: stage it + // into the content-addressed cache and persist the resulting `local:…` + // reference, so `start` re-derives the mount dir without a registry + // pull. Registry refs pass through unchanged. + let image = match self.image.as_deref() { + Some(img) => { + use smolvm::data::image_source::{classify, resolve, ResolvedImage}; + Some(match resolve(classify(img))? { + ResolvedImage::Registry(reference) => reference, + ResolvedImage::Local { reference, .. } => reference, + }) + } + None => None, + }; + + let params = crate::cli::smolfile::build_create_params( + name, + image, + None, // entrypoint: from Smolfile only + self.command, // persistent-workload command (detached container on start) + self.cpus, + self.mem, + self.volume, + self.port, + net, + self.net_backend, + self.dns, + self.init, + self.env, + self.workdir, + self.smolfile.clone(), + self.storage, + self.overlay, + cli_allow_cidrs, + )?; + let mut params = params; + params.dns_filter_hosts = match (params.dns_filter_hosts.take(), cli_dns_filter_hosts) { + (Some(mut from_smolfile), Some(mut from_cli)) => { + from_smolfile.append(&mut from_cli); + Some(from_smolfile) + } + (Some(from_smolfile), None) => Some(from_smolfile), + (None, some) => some, + }; + // CLI `--secret-env`/`--secret-file` refs merge over any Smolfile + // `[secrets]` of the same name (CLI wins). Only refs are persisted. + for (key, r) in parse_cli_secret_refs(&self.secret_env, &self.secret_file)? { + params.secret_refs.insert(key, r); + } + let resources = VmResources { + cpus: params.cpus, + memory_mib: params.mem, + network: params.net, + network_backend: params.network_backend, + dns: params.dns, + gpu: params.gpu, + gpu_vram_mib: params.gpu_vram_mib, + rosetta: params.rosetta, + storage_gib: params.storage_gb, + overlay_gib: params.overlay_gb, + allowed_cidrs: params.allowed_cidrs.clone(), + }; + // Reject zero-valued resources before the machine is persisted. + // Without this, `machine create` succeeds and the failure only + // surfaces later at `machine start` (see QA BUG-44). + resources.validate()?; + validate_requested_network_backend( + &resources, + params.dns_filter_hosts.as_deref(), + params.port.len(), + )?; + if self.ssh_agent { + params.ssh_agent = true; + } + if self.cuda { + params.cuda = true; + } + if self.gpu { + params.gpu = true; + } + if self.rosetta { + params.rosetta = true; + } + // CLI --gpu-vram takes precedence over Smolfile gpu_vram. + if let Some(vram) = self.gpu_vram_mib { + params.gpu_vram_mib = Some(vram); + } + PortMapping::check_duplicates(¶ms.port) + .map_err(|e| smolvm::Error::config("validate ports", e))?; + vm_common::create_vm(params) + } + + /// Create a machine from a .smolmachine artifact. + fn run_from_smolmachine(&self, sidecar_path: &std::path::Path) -> smolvm::Result<()> { + use smolvm::data::resources::{DEFAULT_MICROVM_CPU_COUNT, DEFAULT_MICROVM_MEMORY_MIB}; + + if !sidecar_path.exists() { + return Err(smolvm::Error::config( + "create from .smolmachine", + format!("file not found: {}", sidecar_path.display()), + )); + } + + // Read manifest from the sidecar to get image metadata. + let manifest = smolvm_pack::packer::read_manifest_from_sidecar(sidecar_path) + .map_err(|e| smolvm::Error::agent("read .smolmachine", e.to_string()))?; + + // Reject a cross-architecture artifact up front: a packed VM/image carries + // native binaries and cannot boot under a different-arch guest kernel. Only + // the guest arch must match — the host OS does not (see the fn's docs). + smolvm::platform::ensure_artifact_arch_matches_host(&manifest.platform)?; + + // Read the footer now; the bundle is extracted into the machine's own + // data dir after `create_vm` succeeds (below), so a duplicate-name create + // cannot clobber an existing machine's layers. + let footer = smolvm_pack::packer::read_footer_from_sidecar(sidecar_path) + .map_err(|e| smolvm::Error::agent("read sidecar footer", e.to_string()))?; + + // A VM-mode pack (`--from-vm`) carries the source VM's overlay+storage + // DISKS (the real rootfs), not OCI layers. Capture the templates before + // `manifest` is moved into `params`; the disks are seeded from them after + // extraction below, or the machine boots the bare agent-rootfs with no + // /bin/sh (mirrors pack_run + the serve API create path). + let vm_seed: Option<(Option, Option, Option)> = + if manifest.mode == smolvm_pack::format::PackMode::Vm { + Some(( + manifest + .assets + .overlay_template + .as_ref() + .map(|t| t.path.clone()), + manifest + .assets + .storage_template + .as_ref() + .map(|t| t.path.clone()), + manifest.assets.overlay_logical_size, + )) + } else { + None + }; + + // Resolve the canonical path for storage in VmRecord. + let canonical_path = sidecar_path + .canonicalize() + .unwrap_or_else(|_| sidecar_path.to_path_buf()) + .to_string_lossy() + .into_owned(); + + let name = self + .name + .clone() + .unwrap_or_else(smolvm::util::generate_machine_name); + // `name` is moved into `params` below; keep a copy for the post-create + // extraction that targets this machine's own data dir. + let name_for_layers = name.clone(); + + // CLI flags override manifest defaults. + let cpus = if self.cpus != DEFAULT_MICROVM_CPU_COUNT { + self.cpus + } else { + manifest.cpus + }; + let mem = if self.mem != DEFAULT_MICROVM_MEMORY_MIB { + self.mem + } else { + manifest.mem + }; + + // A .smolmachine is an untrusted, portable artifact: validate its secret + // refs under the Untrusted scope, which rejects every source kind. A + // packed `from_env`/`from_file` ref would otherwise read THIS host's + // env/files at exec time — reject at create rather than carry an exfil + // primitive. Configure secrets locally via the CLI instead. + for (key, r) in &manifest.secret_refs { + smolvm::secrets::validate_ref(r, smolvm::secrets::ResolutionScope::Untrusted).map_err( + |e| { + smolvm::Error::config( + "create from .smolmachine", + format!("secret '{}': {} (packs may not carry secret refs)", key, e), + ) + }, + )?; + } + + let params = vm_common::CreateVmParams { + secret_refs: manifest.secret_refs, + name, + // A VM-mode pack is a VM, not a container: its synthetic `vm://` + // label would make exec/start/re-pack treat it as a pullable image + // (the /bin/sh-not-found bug). None routes every `image.is_some()` + // consumer to VM behavior; provenance is in `source_smolmachine`. + image: if vm_seed.is_some() { + None + } else { + Some(manifest.image) + }, + entrypoint: manifest.entrypoint, + cmd: manifest.cmd, + cpus, + mem, + volume: self.volume.clone(), + port: self.port.clone(), + net: self.net || manifest.network, + network_backend: self.net_backend, + dns: self.dns, + init: self.init.clone(), + env: { + let mut env = manifest.env; + env.extend(self.env.iter().cloned()); + env + }, + workdir: manifest.workdir, + storage_gb: self.storage, + overlay_gb: self.overlay, + allowed_cidrs: None, + restart_policy: None, + restart_max_retries: None, + restart_max_backoff_secs: None, + health_cmd: None, + health_interval_secs: None, + health_timeout_secs: None, + health_retries: None, + health_startup_grace_secs: None, + ssh_agent: self.ssh_agent, + cuda: self.cuda, + dns_filter_hosts: None, + gpu: manifest.gpu, + gpu_vram_mib: None, + rosetta: false, + source_smolmachine: Some(canonical_path), + }; + + let record = vm_common::build_vm_record(¶ms)?; + let reservation = vm_common::CreateVmReservation::reserve(&name_for_layers)?; + + // Create the machine data dir while the DB reservation is held, then + // extract before publishing the VM row. Other processes either see the + // reservation conflict or the finished VM, never a half-created record. + let create_result = (|| -> smolvm::Result<()> { + let manager = AgentManager::for_vm_with_sizes( + &name_for_layers, + params.storage_gb, + params.overlay_gb, + )?; + + let cache_dir = smolvm::agent::machine_layers_cache_dir(&name_for_layers); + smolvm_pack::extract::force_detach_layers_volume(&cache_dir); + match std::fs::remove_dir_all(&cache_dir) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => { + return Err(smolvm::Error::agent( + "clear packed layers cache", + e.to_string(), + )); + } + } + + println!("Extracting .smolmachine assets..."); + let result = smolvm_pack::extract::extract_sidecar( + sidecar_path, + &cache_dir, + &footer, + false, + false, + ) + .map_err(|e| smolvm::Error::agent("extract sidecar", e.to_string())); + // Detach unconditionally: extraction mounts the case-sensitive volume on + // macOS even when it later fails, so the detach must run on both success + // and failure paths to honor the "mounted iff running" invariant. + smolvm_pack::extract::force_detach_layers_volume(&cache_dir); + result?; + + // VM-mode pack: seed this machine's overlay+storage disks from the + // packed templates so a start boots the source VM's rootfs rather than + // the bare agent-rootfs. Shared with the serve API create path: it + // resizes the truncated templates into valid raw disks and removes the + // manager's default `.qcow2` overlays so the start resolves these. + // (Writing the resized copy onto `manager.overlay_path()` — the default + // `.qcow2` — handed the guest raw bytes named `.qcow2`, the /bin/sh-missing + // bug's disk counterpart.) + if let Some((overlay_template, storage_template, overlay_logical_size)) = &vm_seed { + let disk_dir = manager + .storage_path() + .parent() + .map(std::path::Path::to_path_buf) + .unwrap_or_else(|| smolvm::agent::vm_data_dir(&name_for_layers)); + smolvm::storage::seed_vm_mode_disks( + &disk_dir, + &cache_dir, + overlay_template.as_deref(), + storage_template.as_deref(), + *overlay_logical_size, + params.overlay_gb, + params.storage_gb, + ) + .map_err(|e| smolvm::Error::agent("seed VM-mode disks", e.to_string()))?; + } + + reservation.commit(&record)?; + Ok(()) + })(); + + if let Err(e) = create_result { + smolvm_pack::extract::force_detach_layers_volume( + &smolvm::agent::machine_layers_cache_dir(&name_for_layers), + ); + let data_dir = smolvm::agent::vm_data_dir(&name_for_layers); + if let Err(remove_err) = std::fs::remove_dir_all(&data_dir) { + if remove_err.kind() != std::io::ErrorKind::NotFound { + tracing::warn!( + machine = %name_for_layers, + dir = %data_dir.display(), + error = %remove_err, + "failed to remove machine data dir after create failure" + ); + } + } + return Err(e); + } + + vm_common::print_create_success(¶ms); + Ok(()) + } +} + +// ============================================================================ +// Start Command +// ============================================================================ + +/// Start a machine. +/// +/// Starts the VM process. If no name is given, starts the default VM. +#[derive(Args, Debug)] +pub struct StartCmd { + /// Machine to start (default: "default") + #[arg(short = 'n', long, value_name = "NAME")] + pub name: Option, + + /// Start as a fork base: back guest RAM with a memfd (CoW-cloneable) and + /// expose a control socket so the machine can be forked with `machine fork`. + #[arg(long)] + pub forkable: bool, + + #[command(flatten, next_help_heading = "Network")] + pub proxy_opts: crate::cli::proxy_opts::ProxyOpts, +} + +impl StartCmd { + pub fn run(self) -> smolvm::Result<()> { + let explicit_name = self.name.is_some(); + let name = self.name.unwrap_or_else(|| "default".to_string()); + let proxy = self.proxy_opts.proxy(); + let no_proxy = self.proxy_opts.no_proxy(); + // Forkable start: memfd-back guest RAM and register a control socket at a + // known path so `machine fork` can later freeze this machine as a CoW base. + let fork = if self.forkable { + vm_common::forkable_launch(&name) + } else { + vm_common::ForkLaunch::default() + }; + match vm_common::start_vm_named( + &name, proxy, no_proxy, /* from_snapshot */ false, fork, + ) { + Ok(()) => Ok(()), + Err(smolvm::Error::VmNotFound { .. }) if !explicit_name => { + // Only fall back to creating a default VM when no --name was given. + // With an explicit --name, VmNotFound is a real error. + vm_common::start_vm_default(proxy, no_proxy) + } + Err(e) => Err(e), + } + } +} + +// ============================================================================ +// Fork Command +// ============================================================================ + +/// Fork a running forkable machine into a new clone. +/// +/// Freezes the source (the "golden") via its control socket, copy-on-write +/// clones its disks, and boots the new machine from the golden's in-memory +/// snapshot instead of cold-booting — so the clone comes up already warm +/// (same processes, same filesystem state), in well under a second. +/// +/// The golden must have been started with `--forkable`. +#[derive(Args, Debug)] +pub struct ForkCmd { + /// The running, forkable source machine to clone from. + #[arg(long, value_name = "NAME")] + pub golden: String, + + /// Name for the new clone machine. + #[arg(short = 'n', long = "name", value_name = "NAME")] + pub clone: String, + + /// Make the clone itself forkable (memfd RAM + control socket), so it can + /// in turn be forked. + #[arg(long)] + pub forkable: bool, + + /// Pin the clone's inbound port forwards (repeatable). Without this, the + /// golden's forwards are remapped to freshly-allocated host ports. + #[arg(short = 'p', long = "port", value_parser = PortMapping::parse, value_name = "HOST:GUEST", help_heading = "Network")] + pub port: Vec, +} + +impl ForkCmd { + pub fn run(self) -> smolvm::Result<()> { + let ports: Vec<(u16, u16)> = self.port.iter().map(|p| (p.host, p.guest)).collect(); + vm_common::fork_vm(&self.golden, &self.clone, self.forkable, &ports) + } +} + +// ============================================================================ +// Stop Command +// ============================================================================ + +/// Stop a running machine. +/// +/// Gracefully stops the VM process. Running containers will be terminated. +#[derive(Args, Debug)] +pub struct StopCmd { + /// Machine to stop (default: "default") + #[arg(short = 'n', long, value_name = "NAME")] + pub name: Option, +} + +impl StopCmd { + pub fn run(self) -> smolvm::Result<()> { + let name = vm_common::resolve_vm_name(self.name)?; + match &name { + Some(name) => vm_common::stop_vm_named(name), + None => vm_common::stop_vm_default(), + } + } +} + +// ============================================================================ +// Delete Command +// ============================================================================ + +/// Delete a machine configuration. +/// +/// Removes the VM configuration. Does not delete container data. +#[derive(Args, Debug)] +pub struct DeleteCmd { + /// Machine to delete + #[arg(short = 'n', long, value_name = "NAME")] + pub name: String, + + /// Skip confirmation prompt + #[arg(short, long)] + pub force: bool, +} + +impl DeleteCmd { + pub fn run(&self) -> smolvm::Result<()> { + vm_common::delete_vm( + &self.name, + self.force, + DeleteVmOptions { + // Stop the VM before removing its config and data dir. + // Without this, deleting a running machine orphans the + // `_boot-vm` process (leaking host RAM) and removes the data + // dir out from under the live VM. The API delete handler and + // `delete_vm`'s own teardown already do this. + stop_if_running: true, + }, + ) + } +} + +// ============================================================================ +// Status Command +// ============================================================================ + +/// Show machine status. +/// +/// Displays whether the VM is running and its process ID. +#[derive(Args, Debug)] +pub struct StatusCmd { + /// Machine to check (default: "default") + #[arg(short = 'n', long, value_name = "NAME")] + pub name: Option, + + /// Output in JSON format + #[arg(long)] + pub json: bool, +} + +impl StatusCmd { + pub fn run(self) -> smolvm::Result<()> { + if self.json { + return vm_common::status_vm_json(&self.name); + } + vm_common::status_vm(&self.name, |_| {}) + } +} + +// ============================================================================ +// Ls Command +// ============================================================================ + +/// List all machines. +/// +/// Shows all configured VMs with their state, resources, and configuration. +#[derive(Args, Debug)] +pub struct LsCmd { + /// Show detailed configuration (mounts, ports, PID) + #[arg(short, long)] + pub verbose: bool, + + /// Output in JSON format + #[arg(long)] + pub json: bool, +} + +impl LsCmd { + pub fn run(&self) -> smolvm::Result<()> { + vm_common::list_vms(self.verbose, self.json) + } +} + +// ============================================================================ +// Resize Command +// ============================================================================ + +/// Resize a machine's disk resources. +/// +/// Expands the storage and/or overlay disk for a stopped machine. +/// The VM must be stopped before resizing. Disk expansion happens +/// immediately; filesystem resize occurs automatically on next boot. +/// +/// Examples: +/// smolvm machine resize --name my-vm --storage 50 +/// smolvm machine resize --name my-vm --overlay 20 +/// smolvm machine resize --name my-vm --storage 50 --overlay 20 +/// smolvm machine resize --storage 50 # default VM +#[derive(Args, Debug)] +#[command(group( + clap::ArgGroup::new("resize-target") + .required(true) + .args(["storage", "overlay"]) + .multiple(true) +))] +pub struct ResizeCmd { + /// Machine to resize (default: "default") + #[arg(short = 'n', long, value_name = "NAME")] + pub name: Option, + + /// Storage disk size in GiB (expand only) + #[arg(long, value_name = "GiB")] + pub storage: Option, + + /// Overlay disk size in GiB (expand only) + #[arg(long, value_name = "GiB")] + pub overlay: Option, +} + +impl ResizeCmd { + pub fn run(self) -> smolvm::Result<()> { + let name = vm_common::resolve_vm_name(self.name)?; + let name_str = name.as_deref().unwrap_or("default"); + + vm_common::resize_vm(name_str, self.storage, self.overlay).map_err(|e| { + if matches!(&e, smolvm::Error::InvalidState { .. }) { + smolvm::Error::agent( + "resize", + format!( + "VM '{}' is running. Stop it first with: smolvm machine stop --name {}", + name_str, name_str + ), + ) + } else { + e + } + }) + } +} + +// ============================================================================ +// Update Command +// ============================================================================ + +/// Modify settings on a stopped machine. +/// +/// Changes are applied to the DB record and take effect on the next +/// `machine start`. The machine must be stopped. +/// +/// Examples: +/// smolvm machine update --name myvm -v ./src:/app -p 8080:8080 +/// smolvm machine update --name myvm --cpus 4 --mem 4096 +/// smolvm machine update --name myvm --remove-volume ./src:/app +/// smolvm machine update --name myvm --net -e DEBUG=1 +#[derive(Args, Debug)] +pub struct UpdateCmd { + /// Machine to update + #[arg(short = 'n', long, value_name = "NAME")] + pub name: String, + + /// Add volume mount (HOST:GUEST[:ro]) + #[arg(short = 'v', long = "volume", value_name = "HOST:GUEST[:ro]")] + pub volume: Vec, + + /// Remove volume mount (HOST:GUEST) + #[arg(long, value_name = "HOST:GUEST")] + pub remove_volume: Vec, + + /// Add port mapping (HOST:GUEST) + #[arg(short = 'p', long = "port", value_parser = PortMapping::parse, value_name = "HOST:GUEST")] + pub port: Vec, + + /// Remove port mapping (HOST:GUEST) + #[arg(long, value_parser = PortMapping::parse, value_name = "HOST:GUEST")] + pub remove_port: Vec, + + /// Set vCPU count + #[arg(long, value_name = "N")] + pub cpus: Option, + + /// Set memory in MiB + #[arg(long, value_name = "MiB")] + pub mem: Option, + + /// Enable outbound network access + #[arg(long)] + pub net: bool, + + /// Disable outbound network access + #[arg(long, conflicts_with = "net")] + pub no_net: bool, + + /// Add/replace environment variable (KEY=VALUE) + #[arg(short = 'e', long = "env", value_name = "KEY=VALUE")] + pub env: Vec, + + /// Remove environment variable by key + #[arg(long, value_name = "KEY")] + pub remove_env: Vec, + + /// Set working directory + #[arg(short = 'w', long, value_name = "DIR")] + pub workdir: Option, + + /// Enable GPU acceleration + #[arg(long)] + pub gpu: bool, + + /// Disable GPU acceleration + #[arg(long, conflicts_with = "gpu")] + pub no_gpu: bool, + + /// Enable Rosetta 2 for x86_64 binary translation + #[arg(long)] + pub rosetta: bool, + + /// Disable Rosetta 2 + #[arg(long, conflicts_with = "rosetta")] + pub no_rosetta: bool, + + /// Storage disk size in GiB (expand only) + #[arg(long, value_name = "GiB")] + pub storage: Option, + + /// Overlay disk size in GiB (expand only) + #[arg(long, value_name = "GiB")] + pub overlay: Option, +} + +impl UpdateCmd { + pub fn run(self) -> smolvm::Result<()> { + use smolvm::config::RecordState; + use smolvm::data::storage::HostMount; + + let db = smolvm::db::SmolvmDb::open()?; + let record = db.get_vm(&self.name)?.ok_or_else(|| { + smolvm::Error::config("update", format!("machine '{}' not found", self.name)) + })?; + + // Must be stopped (same check as resize) + let state = record.actual_state(); + match state { + RecordState::Stopped | RecordState::Created => {} + _ => { + return Err(smolvm::Error::InvalidState { + expected: "stopped".into(), + actual: format!("{:?}", state), + }); + } + } + + // Validate proposed resource values using the same logic as machine start. + // Construct a temporary VmResources with the new values (falling back to + // the record's current values) and run validate() — single source of truth. + let proposed = smolvm::agent::VmResources { + cpus: self.cpus.unwrap_or(record.cpus), + memory_mib: self.mem.unwrap_or(record.mem), + ..record.vm_resources() + }; + proposed.validate()?; + + // Validate env specs have KEY=VALUE format with non-empty key + for spec in &self.env { + match spec.split_once('=') { + Some((key, _)) if !key.is_empty() => {} + _ => { + return Err(smolvm::Error::config( + "update", + format!("invalid env format '{}': expected KEY=VALUE", spec), + )); + } + } + } + + // Parse and validate new mounts (after state check so + // "machine is running" takes priority over "directory not found") + let new_mounts = HostMount::parse(&self.volume)?; + + // Validate no duplicate host ports after proposed changes + { + let mut final_ports: Vec = record + .ports + .iter() + .filter(|&&(h, g)| { + !self + .remove_port + .iter() + .any(|rm| rm.host == h && rm.guest == g) + }) + .map(|&(h, g)| PortMapping::new(h, g)) + .collect(); + for p in &self.port { + if !final_ports + .iter() + .any(|existing| existing.host == p.host && existing.guest == p.guest) + { + final_ports.push(*p); + } + } + PortMapping::check_duplicates(&final_ports) + .map_err(|e| smolvm::Error::config("update", e))?; + } + + // Expand physical disk files before the DB write. If expansion fails, + // no DB changes are made — the record stays consistent. + let mut changes: Vec = Vec::new(); + if self.storage.is_some() || self.overlay.is_some() { + let disk_changes = + vm_common::expand_disks(&self.name, &record, self.storage, self.overlay)?; + changes.extend(disk_changes); + } + + // Single DB transaction: all settings + disk sizes together. + db.update_vm(&self.name, |r| { + // Disk sizes (must match the physical expansion above) + if let Some(s) = self.storage { + r.storage_gb = Some(s); + } + if let Some(o) = self.overlay { + r.overlay_gb = Some(o); + } + // Volumes: add new, remove specified. + // Canonicalize the remove spec's source path so ./src matches + // the stored /absolute/path/to/src. + for rm in &self.remove_volume { + let canonical_rm = if let Some((rm_src, rm_tgt)) = rm.split_once(':') { + let resolved = std::fs::canonicalize(rm_src) + .unwrap_or_else(|_| std::path::PathBuf::from(rm_src)); + format!("{}:{}", resolved.display(), rm_tgt) + } else { + rm.clone() + }; + let before = r.mounts.len(); + r.mounts.retain(|(src, tgt, _)| { + let spec = format!("{}:{}", src, tgt); + spec != canonical_rm && spec != *rm + }); + if r.mounts.len() < before { + changes.push(format!(" removed volume: {}", rm)); + } + } + for m in &new_mounts { + let tuple = m.to_storage_tuple(); + if !r + .mounts + .iter() + .any(|(s, t, _)| *s == tuple.0 && *t == tuple.1) + { + changes.push(format!( + " added volume: {}:{}{}", + tuple.0, + tuple.1, + if tuple.2 { ":ro" } else { "" } + )); + r.mounts.push(tuple); + } + } + + // Ports: add new, remove specified + for rm in &self.remove_port { + let before = r.ports.len(); + r.ports.retain(|&(h, g)| h != rm.host || g != rm.guest); + if r.ports.len() < before { + changes.push(format!(" removed port: {}:{}", rm.host, rm.guest)); + } + } + for p in &self.port { + let tuple = p.to_tuple(); + if !r.ports.contains(&tuple) { + changes.push(format!(" added port: {}:{}", tuple.0, tuple.1)); + r.ports.push(tuple); + } + } + + // Resources + if let Some(cpus) = self.cpus { + changes.push(format!(" cpus: {} → {}", r.cpus, cpus)); + r.cpus = cpus; + } + if let Some(mem) = self.mem { + changes.push(format!(" memory: {} MiB → {} MiB", r.mem, mem)); + r.mem = mem; + } + + // Network + if self.net { + changes.push(" network: enabled".to_string()); + r.network = true; + } + if self.no_net { + changes.push(" network: disabled".to_string()); + r.network = false; + // Clear egress policy — allow_cidrs and dns_filter_hosts imply + // networking. Leaving them set would re-enable egress on start. + if r.allowed_cidrs.is_some() { + changes.push(" cleared allow_cidrs".to_string()); + r.allowed_cidrs = None; + } + if r.dns_filter_hosts.is_some() { + changes.push(" cleared dns_filter_hosts".to_string()); + r.dns_filter_hosts = None; + } + } + + // Env vars + for rm_key in &self.remove_env { + let before = r.env.len(); + r.env.retain(|(k, _)| k != rm_key); + if r.env.len() < before { + changes.push(format!(" removed env: {}", rm_key)); + } + } + for spec in &self.env { + if let Some((key, val)) = spec.split_once('=') { + r.env.retain(|(k, _)| k != key); + r.env.push((key.to_string(), val.to_string())); + changes.push(format!(" env: {}={}", key, val)); + } + } + + // Workdir + if let Some(ref wd) = self.workdir { + changes.push(format!(" workdir: {}", wd)); + r.workdir = Some(wd.clone()); + } + + // GPU + if self.gpu { + changes.push(" gpu: enabled".to_string()); + r.gpu = Some(true); + } + if self.no_gpu { + changes.push(" gpu: disabled".to_string()); + r.gpu = Some(false); + } + if self.rosetta { + changes.push(" rosetta: enabled".to_string()); + r.rosetta = Some(true); + } + if self.no_rosetta { + changes.push(" rosetta: disabled".to_string()); + r.rosetta = Some(false); + } + })?; + + if changes.is_empty() { + println!("No changes specified."); + } else { + println!("Updated machine '{}':", self.name); + for change in &changes { + println!("{}", change); + } + println!("\nStart with: smolvm machine start --name {}", self.name); + } + + Ok(()) + } +} + +// ============================================================================ +// Data Dir Command +// ============================================================================ + +/// Print the on-disk data directory for a named machine. +/// +/// Equivalent to calling `smolvm::agent::vm_data_dir(name)` — exposed as a +/// CLI command so shell scripts and external tooling have a single source +/// of truth for the path computation (which is hash-derived, not +/// name-derived). +#[derive(Args, Debug)] +pub struct DataDirCmd { + /// Machine name. + #[arg(short = 'n', long, value_name = "NAME")] + pub name: String, +} + +impl DataDirCmd { + pub fn run(self) -> smolvm::Result<()> { + // Error (exit 1) for a machine that does not exist, rather than + // printing a computed path for a name that was never created — + // consistent with `status`/`start`/`delete`. + let config = smolvm::config::SmolvmConfig::load()?; + if config.get_vm(&self.name).is_none() { + return Err(smolvm::Error::vm_not_found(&self.name)); + } + let dir = smolvm::agent::vm_data_dir(&self.name); + println!("{}", dir.display()); + Ok(()) + } +} + +// ============================================================================ +// Network Test Command +// ============================================================================ + +/// Test network connectivity directly from machine (debug TSI). +#[derive(Args, Debug)] +pub struct NetworkTestCmd { + /// Named machine to test (omit for default) + #[arg(long)] + pub name: Option, + + /// URL to test + #[arg(default_value = "http://1.1.1.1")] + pub url: String, +} + +impl NetworkTestCmd { + pub fn run(self) -> smolvm::Result<()> { + let manager = vm_common::get_vm_manager(&self.name)?; + let label = vm_common::vm_label(&self.name); + + // Ensure machine is running + let already_running = manager.try_connect_existing().is_some(); + if !already_running { + eprintln!("Starting machine '{}'...", label); + manager.ensure_running()?; + } + + // Connect and test + println!("Testing network from machine: {}", self.url); + let mut client = manager.connect()?; + let result = client.network_test(&self.url)?; + + println!( + "Result: {}", + serde_json::to_string_pretty(&result).unwrap_or_default() + ); + + // VM was already running — don't stop it when we're done + if already_running { + manager.detach(); + } + Ok(()) + } +} + +// ============================================================================ +// Images Command +// ============================================================================ + +/// List cached images and storage usage. +/// +/// Shows all OCI images cached in the machine's storage, along with their +/// sizes and layer counts. Also displays total storage usage. +/// +/// Examples: +/// smolvm machine images --name myvm +/// smolvm machine images --name myvm --json +#[derive(Args, Debug)] +pub struct ImagesCmd { + /// Machine to query + #[arg(long, required = true, value_name = "NAME")] + pub name: String, + + /// Output in JSON format + #[arg(long)] + pub json: bool, +} + +impl ImagesCmd { + pub fn run(self) -> smolvm::Result<()> { + // Validate VM exists before creating storage (for_vm creates dirs). + let db = smolvm::db::SmolvmDb::open()?; + let record = db.get_vm(&self.name)?.ok_or_else(|| { + smolvm::Error::config("images", format!("machine '{}' not found", self.name)) + })?; + + let manager = + AgentManager::for_vm_with_sizes(&self.name, record.storage_gb, record.overlay_gb)?; + + let started_for_query = if manager.try_connect_existing().is_some() { + manager.detach(); + false + } else { + eprintln!("Starting machine '{}' to query storage...", self.name); + manager.start()?; + true + }; + let mut client = AgentClient::connect_with_retry(manager.vsock_socket())?; + + let status = client.storage_status()?; + let images = client.list_images()?; + + if self.json { + let output = serde_json::json!({ + "storage": { + "total_bytes": status.total_bytes, + "used_bytes": status.used_bytes, + "layer_count": status.layer_count, + "image_count": status.image_count, + }, + "images": images, + }); + let json = serde_json::to_string_pretty(&output) + .map_err(|e| smolvm::Error::config("serialize json", e.to_string()))?; + println!("{}", json); + } else { + println!("Storage Usage:"); + println!(" Total: {}", format_bytes(status.total_bytes)); + println!(" Used: {}", format_bytes(status.used_bytes)); + println!(" Layers: {}", status.layer_count); + println!(); + + if images.is_empty() { + println!("No cached images."); + } else { + println!("Cached Images:"); + println!("{:<40} {:>10} {:>8}", "IMAGE", "SIZE", "LAYERS"); + println!("{}", "-".repeat(60)); + + for image in &images { + let name = if image.reference.len() > 38 { + format!("{}...", &image.reference[..35]) + } else { + image.reference.clone() + }; + println!( + "{:<40} {:>10} {:>8}", + name, + format_bytes(image.size), + image.layer_count + ); + } + + println!(); + println!("Total: {} images", images.len()); + } + } + + if started_for_query { + let _ = manager.stop(); + } + + Ok(()) + } +} + +// ============================================================================ +// Prune Command +// ============================================================================ + +/// Remove unused images and layers to free disk space. +/// +/// This removes layers that are not referenced by any cached image manifest. +/// Use --dry-run to see what would be removed without actually deleting. +/// +/// Examples: +/// smolvm machine prune --name myvm --dry-run +/// smolvm machine prune --name myvm +/// smolvm machine prune --name myvm --all +#[derive(Args, Debug)] +pub struct PruneCmd { + /// Machine to prune + #[arg(long, required = true, value_name = "NAME")] + pub name: String, + + /// Show what would be removed without actually removing + #[arg(long)] + pub dry_run: bool, + + /// Remove all cached images (not just unreferenced layers) + #[arg(long)] + pub all: bool, +} + +impl PruneCmd { + pub fn run(self) -> smolvm::Result<()> { + // Validate VM exists before creating storage (for_vm creates dirs). + let db = smolvm::db::SmolvmDb::open()?; + let record = db.get_vm(&self.name)?.ok_or_else(|| { + smolvm::Error::config("prune", format!("machine '{}' not found", self.name)) + })?; + + let manager = + AgentManager::for_vm_with_sizes(&self.name, record.storage_gb, record.overlay_gb)?; + + // Regular prune (unreferenced layers only) is safe on a running VM — + // referenced layers can't be collected. --all deletes manifests for + // layers that may be in active use, so it requires a stop first. + let already_running = manager.try_connect_existing().is_some(); + let started_for_prune; + + if already_running && self.all { + manager.detach(); + return Err(smolvm::Error::agent( + "prune", + format!("cannot prune --all while machine '{}' is running. Stop it first with: smolvm machine stop --name {}", self.name, self.name), + )); + } else if already_running { + started_for_prune = false; + manager.detach(); + } else { + eprintln!("Starting machine..."); + manager.start()?; + started_for_prune = true; + } + + let mut client = AgentClient::connect_with_retry(manager.vsock_socket())?; + + if self.all { + let images = client.list_images()?; + + if images.is_empty() { + println!("No cached images to remove."); + } else if record.image.is_some() { + // An image-backed machine needs its cached image to restart, so + // purging it would brick a *stopped* machine ("image not found" + // on the next start). Keep the cache and reclaim only + // unreferenced layers; to reclaim everything, delete the machine. + let total_size: u64 = images.iter().map(|i| i.size).sum(); + if self.dry_run { + let would_free = client.garbage_collect(true, false)?; + println!( + "Machine '{}' is image-backed: would keep {} cached image(s) ({}) it needs to restart, and free {} of unreferenced layers.", + self.name, + images.len(), + format_bytes(total_size), + format_bytes(would_free) + ); + } else { + let freed = client.garbage_collect(false, false)?; + println!( + "Kept {} cached image(s) in use by machine '{}'; freed {} of unreferenced layers.", + images.len(), + self.name, + format_bytes(freed) + ); + eprintln!( + "(--all keeps images a machine needs to restart; to reclaim everything: smolvm machine delete --name {})", + self.name + ); + } + } else { + // Bare VM: nothing depends on the image cache, so purge all. + let total_size: u64 = images.iter().map(|i| i.size).sum(); + if self.dry_run { + println!( + "Would remove {} images ({})", + images.len(), + format_bytes(total_size) + ); + for image in &images { + println!( + " - {} ({}, {} layers)", + image.reference, + format_bytes(image.size), + image.layer_count + ); + } + } else { + println!("Removing all cached images..."); + let freed = client.garbage_collect(false, true)?; + println!( + "Removed {} images, freed {}", + images.len(), + format_bytes(freed) + ); + } + } + } else if self.dry_run { + println!("Scanning for unreferenced layers..."); + let would_free = client.garbage_collect(true, false)?; + + if would_free > 0 { + println!( + "Would free {} of unreferenced layers", + format_bytes(would_free) + ); + } else { + println!("No unreferenced layers to remove."); + } + } else { + println!("Removing unreferenced layers..."); + let freed = client.garbage_collect(false, false)?; + + if freed > 0 { + println!("Freed {}", format_bytes(freed)); + } else { + println!("No unreferenced layers to remove."); + } + } + + // Only stop the VM if we started it for this prune operation. + // If the user's machine was already running, leave it running. + if started_for_prune { + let _ = manager.stop(); + } + + Ok(()) + } +} + +// ============================================================================ +// Cp (File Copy) Command +// ============================================================================ + +/// Copy files between host and a running machine. +/// +/// Uses `machine:path` syntax to specify the remote side. +/// +/// Examples: +/// smolvm machine cp ./script.py myvm:/workspace/script.py # upload +/// smolvm machine cp myvm:/workspace/output.json ./output.json # download +#[derive(Args, Debug)] +pub struct CpCmd { + /// Source path (local file or machine:path) + #[arg(value_name = "SRC")] + pub src: String, + + /// Destination path (local file or machine:path) + #[arg(value_name = "DST")] + pub dst: String, +} + +impl CpCmd { + pub fn run(self) -> smolvm::Result<()> { + // Parse src/dst to determine direction + let (machine_name, guest_path, local_path, is_upload) = + if let Some((name, path)) = self.src.split_once(':') { + // Download: machine:path -> local + (name.to_string(), path.to_string(), self.dst.clone(), false) + } else if let Some((name, path)) = self.dst.split_once(':') { + // Upload: local -> machine:path + (name.to_string(), path.to_string(), self.src.clone(), true) + } else { + return Err(smolvm::Error::config( + "cp", + "one of SRC or DST must use machine:path syntax (e.g., myvm:/workspace/file)", + )); + }; + + let (manager, mut client) = + vm_common::ensure_running_and_connect(&Some(machine_name.clone()))?; + // Detach so the VM keeps running after cp exits. + manager.detach(); + + // For image-based VMs, ensure the persistent container overlay is + // mounted so cp targets the container filesystem (not the VM rootfs). + // prepare_overlay is idempotent: reuses if mounted, remounts if upper + // exists, creates fresh otherwise. + if let Some(image) = smolvm::db::SmolvmDb::open() + .ok() + .and_then(|db| db.get_vm(&machine_name).ok().flatten()) + .and_then(|r| r.image.clone()) + { + let overlay_id = format!("persistent-{}", machine_name); + client.prepare_overlay(&image, &overlay_id)?; + } + + if is_upload { + // Stream from file — only one chunk (~1 MiB) in memory at a time. + let file = std::fs::File::open(&local_path).map_err(|e| { + smolvm::Error::agent("read local file", format!("{}: {}", local_path, e)) + })?; + let size = file.metadata().map(|m| m.len()).map_err(|e| { + smolvm::Error::agent("stat local file", format!("{}: {}", local_path, e)) + })?; + let mut bar = crate::cli::ProgressBar::new( + format!("Uploading {} -> {}", local_path, guest_path), + Some(size), + ); + client.write_file_from_reader_with_progress(&guest_path, file, size, None, |sent| { + bar.update(sent) + })?; + bar.finish(size); + } else { + // Stream to file — only one chunk (~16 MiB) in memory at a time. + let mut bar = crate::cli::ProgressBar::new( + format!("Downloading {} -> {}", guest_path, local_path), + None, + ); + let local = std::path::Path::new(&local_path); + let size = + client.read_file_to_path(&guest_path, local, |received| bar.update(received))?; + bar.finish(size); + } + + Ok(()) + } +} + +// ============================================================================ +// Monitor Command +// ============================================================================ + +/// Monitor a running machine with health checks and restart policy. +/// +/// Runs in the foreground, watching the machine and restarting on crash +/// or health check failure. Uses the restart policy from the machine's +/// config (set via Smolfile [restart] or --restart flag on create). +/// +/// Ctrl+C stops monitoring; the machine keeps running. +/// +/// Examples: +/// smolvm machine monitor --name myvm +/// smolvm machine monitor --name myvm --health-cmd "curl -f http://localhost:8080/health" +/// smolvm machine monitor --name myvm --restart always --interval 10 +#[derive(Args, Debug)] +pub struct MonitorCmd { + /// Machine to monitor (default: "default") + #[arg(short = 'n', long, value_name = "NAME")] + pub name: Option, + + /// Override restart policy (never, always, on-failure, unless-stopped) + #[arg(long, value_name = "POLICY")] + pub restart: Option, + + /// Health check command (run inside the VM via sh -c) + #[arg(long, value_name = "CMD")] + pub health_cmd: Option, + + /// Health check timeout in seconds + #[arg(long, default_value = "5", value_name = "SECS")] + pub health_timeout: u64, + + /// Check interval in seconds + #[arg(long, default_value = "5", value_name = "SECS")] + pub interval: u64, + + /// Health check failures before triggering restart + #[arg(long, default_value = "3", value_name = "N")] + pub health_retries: u32, +} + +impl MonitorCmd { + pub fn run(self) -> smolvm::Result<()> { + use smolvm::config::{RecordState, RestartPolicy}; + use smolvm::db::SmolvmDb; + use smolvm::Error; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + let name = self.name.unwrap_or_else(|| "default".to_string()); + + // Load machine config from DB + let db = SmolvmDb::open()?; + let record = db + .get_vm(&name)? + .ok_or_else(|| Error::vm_not_found(&name))?; + + // Build restart config: CLI override > VmRecord config + let mut restart = record.restart.clone(); + if let Some(ref policy_str) = self.restart { + restart.policy = policy_str + .parse::() + .map_err(|e| Error::config("--restart", e))?; + } + + // Resolve health check: CLI override > VmRecord config + let health_cmd = self + .health_cmd + .clone() + .map(|c| vec!["sh".into(), "-c".into(), c]) + .or_else(|| record.health_cmd.clone()); + let health_timeout = + Duration::from_secs(record.health_timeout_secs.unwrap_or(self.health_timeout)); + let health_retries = record.health_retries.unwrap_or(self.health_retries); + let interval = Duration::from_secs(record.health_interval_secs.unwrap_or(self.interval)); + let startup_grace = record + .health_startup_grace_secs + .map(Duration::from_secs) + .unwrap_or(Duration::ZERO); + + drop(db); + + // Ensure machine is running + let manager = AgentManager::for_vm(&name) + .map_err(|e| Error::agent("create agent manager", e.to_string()))?; + + if !manager.is_process_alive() { + println!("Machine '{}' is not running, starting...", name); + vm_common::start_vm_named( + &name, + None, + None, + /* from_snapshot */ false, + vm_common::ForkLaunch::default(), + )?; + } + + println!( + "Monitoring machine '{}' (policy: {}, interval: {}s)", + name, + restart.policy, + interval.as_secs() + ); + if health_cmd.is_some() { + println!( + " Health check: retries={}, timeout={}s", + health_retries, + health_timeout.as_secs() + ); + } + + // Ctrl+C handler via SIGINT + // + // SAFETY: `stop` is an Arc that lives until the end of this + // function. The cloned Arc below keeps a strong reference alive for the + // duration of the monitor loop, so the raw pointer stored in STOP_FLAG + // remains valid until after we break out of the loop and the function + // returns. The handler only does an atomic store, which is async-signal-safe. + let stop = Arc::new(AtomicBool::new(false)); + { + let stop = stop.clone(); + unsafe { + let _ = libc::signal(libc::SIGINT, { + static mut STOP_FLAG: *const AtomicBool = std::ptr::null(); + STOP_FLAG = Arc::as_ptr(&stop); + extern "C" fn handler(_: libc::c_int) { + unsafe { + if !STOP_FLAG.is_null() { + (*STOP_FLAG).store(true, Ordering::SeqCst); + } + } + } + handler as *const () as libc::sighandler_t + }); + } + } + + let mut consecutive_health_failures: u32 = 0; + let mut last_check = std::time::Instant::now(); + let mut last_start = std::time::Instant::now(); // tracks startup grace period + + loop { + std::thread::sleep(interval); + + if stop.load(Ordering::SeqCst) { + break; + } + + // Detect sleep/wake: if the elapsed wall time is much longer than + // the expected interval, the machine was likely suspended (laptop lid + // closed). Reset health failures and skip this cycle to give the VM + // time to recover network connections. + let elapsed = last_check.elapsed(); + last_check = std::time::Instant::now(); + if elapsed > interval * 3 { + let sleep_secs = elapsed.as_secs() - interval.as_secs(); + println!( + " detected suspend (~{}s) — skipping health check for recovery", + sleep_secs + ); + consecutive_health_failures = 0; + continue; + } + + // Refresh manager to pick up PID changes after restart + let manager = match AgentManager::for_vm(&name) { + Ok(m) => m, + Err(_) => continue, + }; + + if manager.is_process_alive() { + // Skip health checks during startup grace period + if !startup_grace.is_zero() && last_start.elapsed() < startup_grace { + continue; + } + + // Machine is alive — run health check if configured + if let Some(ref cmd) = health_cmd { + match AgentClient::connect_with_short_timeout(manager.vsock_socket()) { + Ok(mut client) => { + match client.vm_exec( + cmd.clone(), + vec![], + None, + Some(health_timeout), + None, + ) { + Ok((0, _, _)) => { + if consecutive_health_failures > 0 { + println!(" health check passed (recovered)"); + } + consecutive_health_failures = 0; + } + Ok((code, _, stderr)) => { + consecutive_health_failures += 1; + println!( + " health check failed (exit {}, {}/{}): {}", + code, + consecutive_health_failures, + health_retries, + String::from_utf8_lossy(&stderr).trim() + ); + } + Err(e) => { + consecutive_health_failures += 1; + println!( + " health check error ({}/{}): {}", + consecutive_health_failures, health_retries, e + ); + } + } + + if consecutive_health_failures >= health_retries { + println!(" unhealthy — stopping machine for restart"); + let _ = vm_common::stop_vm_named(&name); + continue; + } + } + Err(_) => { + consecutive_health_failures += 1; + println!( + " cannot connect to agent ({}/{})", + consecutive_health_failures, health_retries + ); + } + } + } + } else { + // Machine is dead + consecutive_health_failures = 0; + + let exit_code = manager.child_pid().and_then(smolvm::process::try_wait); + + println!( + " machine exited (exit code: {})", + exit_code + .map(|c| c.to_string()) + .unwrap_or_else(|| "unknown".into()) + ); + + // Update DB state + if let Ok(db) = SmolvmDb::open() { + let _ = db.update_vm(&name, |r| { + r.state = RecordState::Stopped; + r.pid = None; + r.last_exit_code = exit_code; + }); + } + + if restart.should_restart(exit_code) { + let backoff = restart.backoff_duration(); + restart.restart_count += 1; + + println!( + " restarting (attempt {}, backoff {}s)...", + restart.restart_count, + backoff.as_secs() + ); + + if let Ok(db) = SmolvmDb::open() { + let _ = db.update_vm(&name, |r| { + r.restart.restart_count = restart.restart_count; + }); + } + + std::thread::sleep(backoff); + + if stop.load(Ordering::SeqCst) { + break; + } + + match vm_common::start_vm_named( + &name, + None, + None, + /* from_snapshot */ false, + vm_common::ForkLaunch::default(), + ) { + Ok(()) => { + println!(" machine restarted"); + last_start = std::time::Instant::now(); + } + Err(e) => println!(" restart failed: {}", e), + } + } else { + println!( + " not restarting (policy: {}, count: {}/{})", + restart.policy, + restart.restart_count, + if restart.max_retries > 0 { + restart.max_retries.to_string() + } else { + "unlimited".into() + } + ); + break; + } + } + } + + // Mark user stopped + if let Ok(db) = SmolvmDb::open() { + let _ = db.update_vm(&name, |r| { + r.restart.user_stopped = true; + }); + } + + println!( + "\nStopped monitoring. Machine '{}' may still be running.", + name + ); + Ok(()) + } +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs new file mode 100644 index 0000000..3edceb5 --- /dev/null +++ b/src/cli/mod.rs @@ -0,0 +1,230 @@ +//! CLI command implementations. + +pub mod cleanup_ephemeral; +pub mod config; +pub mod internal_boot; +pub mod machine; +pub mod openapi; +pub mod pack; +pub mod pack_run; +pub mod parsers; +pub mod proxy_opts; +pub mod serve; +pub mod serve_tls; +pub mod smolfile; +pub mod vm_common; + +use std::io::Write; + +// ============================================================================ +// Display Helpers +// ============================================================================ + +/// Truncate a string to max length, adding "..." if needed. +/// +/// If the string fits within `max` characters, returns it unchanged. +/// Otherwise, truncates to `max - 3` characters and appends "...". +pub fn truncate(s: &str, max: usize) -> String { + if s.len() <= max { + s.to_string() + } else if max <= 3 { + "...".to_string() + } else { + format!("{}...", &s[..max - 3]) + } +} + +/// Format an optional PID as a suffix string. +/// +/// Returns " (PID: N)" if pid is Some, or empty string if None. +pub fn format_pid_suffix(pid: Option) -> String { + pid.map(|p| format!(" (PID: {})", p)).unwrap_or_default() +} + +/// Flush stdout and stderr, ignoring errors. +/// +/// Used to ensure output is visible before blocking operations. +pub fn flush_output() { + let _ = std::io::stdout().flush(); + let _ = std::io::stderr().flush(); +} + +/// Format bytes as human-readable string (e.g., "1.5 GB", "42.0 MB"). +pub fn format_bytes(bytes: u64) -> String { + const KB: u64 = 1024; + const MB: u64 = KB * 1024; + const GB: u64 = MB * 1024; + + if bytes >= GB { + format!("{:.1} GB", bytes as f64 / GB as f64) + } else if bytes >= MB { + format!("{:.1} MB", bytes as f64 / MB as f64) + } else if bytes >= KB { + format!("{:.1} KB", bytes as f64 / KB as f64) + } else { + format!("{} B", bytes) + } +} + +/// Throttled byte-transfer progress bar for streaming operations. +/// +/// Used by `machine cp` (upload + download). The producer calls +/// [`ProgressBar::update`] on every chunk; output is emitted at most +/// every `THROTTLE_MS` to avoid drowning the terminal on fast +/// transfers. [`ProgressBar::finish`] prints the final summary line +/// (with rate) and a newline. +/// +/// Output goes to **stderr** so a `cp ... > somefile` redirect +/// doesn't capture progress noise. +pub struct ProgressBar { + label: String, + /// Known total in bytes. `None` for streams of unknown length + /// (we still report bytes-so-far + rate). + total: Option, + started_at: std::time::Instant, + last_print: std::time::Instant, + /// Set to true when at least one progress line has been printed, + /// so `finish` knows to emit a leading `\r` to overwrite it. + printed: bool, +} + +impl ProgressBar { + const THROTTLE_MS: u128 = 250; + + /// `label` is shown at the start of every line ("Uploading", + /// "Downloading", etc.). `total` is `Some(bytes)` for known-size + /// transfers (uploads where we read the file first), `None` for + /// streaming-source cases. + pub fn new(label: impl Into, total: Option) -> Self { + let now = std::time::Instant::now(); + Self { + label: label.into(), + total, + started_at: now, + // Initialize so the first update() prints immediately — + // gives the user feedback that something started. + last_print: now - std::time::Duration::from_millis(THROTTLE_INITIAL_MS), + printed: false, + } + } + + /// Report the running total of bytes transferred. Throttled — + /// callers can invoke this on every chunk without flooding + /// stderr. + pub fn update(&mut self, bytes_so_far: u64) { + // Defer the terminal (100%) line to finish() so it is printed exactly + // once. Otherwise a small/single-chunk transfer prints "100%" here and + // again on completion — visible as two lines on non-TTY output where + // the overwriting `\r` has no effect. + if let Some(total) = self.total { + if total > 0 && bytes_so_far >= total { + return; + } + } + let now = std::time::Instant::now(); + if now.duration_since(self.last_print).as_millis() < Self::THROTTLE_MS { + return; + } + self.last_print = now; + self.printed = true; + let line = self.format_line(bytes_so_far); + eprint!("\r{}", line); + let _ = std::io::stderr().flush(); + } + + /// Print the final summary and a newline. Consumes self. + pub fn finish(self, bytes_total: u64) { + let line = self.format_line(bytes_total); + if self.printed { + // Overwrite the last throttled line. + eprintln!("\r{}", line); + } else { + eprintln!("{}", line); + } + } + + fn format_line(&self, bytes_so_far: u64) -> String { + let elapsed = self.started_at.elapsed().as_secs_f64().max(0.001); + let rate = bytes_so_far as f64 / elapsed; + let rate_str = format!("{}/s", format_bytes(rate as u64)); + match self.total { + Some(total) if total > 0 => { + let pct = (bytes_so_far as f64 * 100.0 / total as f64).min(100.0); + format!( + "{}: {} / {} ({:.1}%, {})", + self.label, + format_bytes(bytes_so_far), + format_bytes(total), + pct, + rate_str + ) + } + _ => format!( + "{}: {} ({})", + self.label, + format_bytes(bytes_so_far), + rate_str + ), + } + } +} + +/// Initial throttle offset — ensures the first `update()` call +/// always prints, so the user sees activity immediately. +const THROTTLE_INITIAL_MS: u64 = 1000; + +/// Pull an image with a CLI progress bar. +pub fn pull_with_progress( + client: &mut smolvm::agent::AgentClient, + image: &str, + oci_platform: Option<&str>, + proxy: Option<&str>, + no_proxy: Option<&str>, +) -> smolvm::Result { + eprint!("Pulling image {}...", image); + let _ = std::io::stderr().flush(); + + let mut last_percent = 0u8; + let mut syncing = false; + let result = client.pull_with_registry_config_and_progress( + image, + oci_platform, + proxy, + no_proxy, + |percent, _total, layer| { + if layer == "syncing" { + if !syncing { + eprint!( + "\rPulling image {}... [====================] 100% — syncing...", + image + ); + let _ = std::io::stderr().flush(); + syncing = true; + } + return; + } + let percent = percent as u8; + if percent != last_percent && percent <= 100 { + eprint!("\rPulling image {}... [", image); + let filled = (percent as usize) / 5; + for i in 0..20 { + if i < filled { + eprint!("="); + } else if i == filled { + eprint!(">"); + } else { + eprint!(" "); + } + } + eprint!("] {}%", percent); + let _ = std::io::stderr().flush(); + last_percent = percent; + } + }, + ); + eprintln!( + "\rPulling image {}... done. ", + image + ); + result +} diff --git a/src/cli/openapi.rs b/src/cli/openapi.rs new file mode 100644 index 0000000..eb02d69 --- /dev/null +++ b/src/cli/openapi.rs @@ -0,0 +1,57 @@ +//! OpenAPI export command. + +use clap::Args; +use std::fs; +use std::path::PathBuf; +use utoipa::OpenApi; + +use smolvm::ApiDoc; + +/// Export OpenAPI specification. +#[derive(Args, Debug)] +pub struct OpenapiCmd { + /// Output file path (defaults to stdout). + #[arg(short, long)] + output: Option, + + /// Output format. + #[arg(short, long, default_value = "json")] + format: OutputFormat, +} + +#[derive(Debug, Clone, Copy, Default, clap::ValueEnum)] +enum OutputFormat { + /// JSON format (OpenAPI 3.1) + #[default] + Json, + /// YAML format (OpenAPI 3.1) + Yaml, +} + +impl OpenapiCmd { + pub fn run(&self) -> Result<(), smolvm::Error> { + let spec = ApiDoc::openapi(); + + let output = match self.format { + OutputFormat::Json => spec + .to_pretty_json() + .map_err(|e| smolvm::Error::config("serialize openapi to json", e.to_string()))?, + OutputFormat::Yaml => serde_yaml::to_string(&spec) + .map_err(|e| smolvm::Error::config("serialize openapi to yaml", e.to_string()))?, + }; + + match &self.output { + Some(path) => { + fs::write(path, &output).map_err(|e| { + smolvm::Error::config(format!("write to {}", path.display()), e.to_string()) + })?; + eprintln!("OpenAPI spec written to {}", path.display()); + } + None => { + println!("{}", output); + } + } + + Ok(()) + } +} diff --git a/src/cli/pack.rs b/src/cli/pack.rs new file mode 100644 index 0000000..4222de5 --- /dev/null +++ b/src/cli/pack.rs @@ -0,0 +1,2285 @@ +//! Pack command for creating self-contained binaries. +//! +//! Creates a packed binary that contains: +//! - A stub executable +//! - Runtime libraries (libkrun, libkrunfw) +//! - Agent rootfs +//! - OCI image layers +//! - Configuration manifest + +use clap::{Args, Subcommand}; +use smolvm::agent::{resolve_disk_image, AgentClient, AgentManager, VmResources}; +use smolvm::data::disk::DiskFormat; +use smolvm::data::resources::DEFAULT_MICROVM_CPU_COUNT; +use smolvm::storage::{OVERLAY_DISK_FILENAME, STORAGE_DISK_FILENAME}; + +/// Default memory for packed VMs. Same as machine create — memory is elastic +/// via virtio balloon, so the host only commits what the guest actually uses. +pub(crate) const PACK_DEFAULT_MEMORY_MIB: u32 = 8192; +use sha2::{Digest, Sha256}; +use smolvm::config::{RecordState, SmolvmConfig}; +use smolvm::platform::{Arch, Os, Platform, VmExecutor}; +use smolvm::Error; +use smolvm_pack::assets::AssetCollector; +use smolvm_pack::format::{PackManifest, PackMode}; +use smolvm_pack::packer::Packer; +use smolvm_pack::signing::sign_with_hypervisor_entitlements; +use smolvm_protocol::AgentResponse; +use std::path::{Path, PathBuf}; +use tracing::{debug, info, warn}; + +/// Package and run self-contained VM executables. +#[derive(Subcommand, Debug)] +pub enum PackCmd { + /// Package an OCI image or VM snapshot into a self-contained executable + Create(PackCreateCmd), + + /// Run a VM from a packed .smolmachine sidecar file + Run(super::pack_run::PackRunCmd), + + /// Push a .smolmachine artifact to a registry + Push(PackPushCmd), + + /// Pull a .smolmachine artifact from a registry + Pull(PackPullCmd), + + /// Inspect a .smolmachine artifact in a registry (without downloading) + Inspect(PackInspectCmd), + + /// Clean up cached pack extractions to free disk space + Prune(PackPruneCmd), +} + +impl PackCmd { + pub fn run(self) -> smolvm::Result<()> { + match self { + PackCmd::Create(cmd) => cmd.run(), + PackCmd::Run(cmd) => cmd.run(), + PackCmd::Push(cmd) => cmd.run(), + PackCmd::Pull(cmd) => cmd.run(), + PackCmd::Inspect(cmd) => cmd.run(), + PackCmd::Prune(cmd) => cmd.run(), + } + } +} + +/// Package an OCI image or VM snapshot into a self-contained executable. +/// +/// Creates a single binary that can be distributed and run without smolvm installed. +/// The packed binary includes: +/// - Runtime libraries (libkrun, libkrunfw) +/// - Agent rootfs +/// - OCI image layers or VM overlay disk +/// - Default configuration +/// +/// Examples: +/// smolvm pack create alpine:latest -o my-alpine +/// smolvm pack create python:3.11-slim -o my-python --cpus 2 --mem 1024 +/// smolvm pack create myapp:latest -o myapp --entrypoint /app/run.sh +/// smolvm pack create --from-vm myvm -o my-devenv +#[derive(Args, Debug)] +pub struct PackCreateCmd { + /// Container image to pack (e.g., alpine:latest, python:3.11-slim) + #[arg( + long, + short = 'I', + value_name = "IMAGE", + required_unless_present_any = ["from_vm", "smolfile"], + conflicts_with = "from_vm" + )] + pub image: Option, + + /// Pack from a stopped VM snapshot instead of an OCI image + #[arg(long = "from-vm", value_name = "VM_NAME")] + pub from_vm: Option, + + /// For image-backed VMs created from a .smolmachine, rebuild lower layers from vm.image instead of preserving imported artifact layers. + #[arg(long = "rebase-from-image", requires = "from_vm")] + pub rebase_from_image: bool, + + /// Output file path for the packed binary + #[arg(short = 'o', long, value_name = "PATH")] + pub output: PathBuf, + + /// Default number of vCPUs for the packed VM + #[arg(long, default_value_t = DEFAULT_MICROVM_CPU_COUNT, value_name = "N")] + pub cpus: u8, + + /// Default memory in MiB for the packed VM + #[arg(long, default_value_t = PACK_DEFAULT_MEMORY_MIB, value_name = "MiB")] + pub mem: u32, + + /// Target OCI platform for multi-arch images (e.g., linux/arm64, linux/amd64) + /// + /// By default, uses the host architecture. Use this to override, for example + /// to pack x86_64 images for Rosetta on Apple Silicon. + #[arg(long = "oci-platform", value_name = "OS/ARCH")] + pub oci_platform: Option, + + /// Override the image entrypoint + #[arg(long, value_name = "CMD")] + pub entrypoint: Option, + + /// Skip code signing (macOS only) + #[arg(long)] + pub no_sign: bool, + + /// Pack as a single file (no sidecar) + /// + /// Creates one executable instead of binary + .smolmachine sidecar. + /// Simpler to distribute but may have issues with macOS notarization. + #[arg(long)] + pub single_file: bool, + + /// Path to stub executable (defaults to built-in) + #[arg(long, value_name = "PATH", hide = true)] + pub stub: Option, + + /// Path to library directory containing libkrun and libkrunfw + #[arg(long, value_name = "DIR", hide = true)] + pub lib_dir: Option, + + /// Path to agent rootfs directory + #[arg(long, value_name = "DIR", hide = true)] + pub rootfs_dir: Option, + + /// Load workload configuration from a Smolfile (TOML) + #[arg(long = "smolfile", visible_short_alias = 's', value_name = "PATH")] + pub smolfile: Option, + + /// Enable GPU acceleration (Vulkan via virtio-gpu) in the packed VM + /// + /// The packed binary will launch with a virtio-gpu device. The guest image + /// must include a compatible Vulkan ICD (e.g., Mesa Venus on Fedora via + /// the slp/mesa-libkrun-vulkan COPR, or standard Mesa on Linux hosts). + #[arg(long)] + pub gpu: bool, + + /// Directory under which to stage pack assets (pulled layers, the merged + /// layer, agent rootfs, and the ext4 template). Defaults to the smolvm cache + /// dir; point this at a roomy disk-backed path when the default filesystem is + /// small. Overrides the `SMOLVM_PACK_STAGING` env var. + #[arg(long = "staging-dir", value_name = "DIR")] + pub staging_dir: Option, + + #[command(flatten, next_help_heading = "Network")] + pub proxy_opts: crate::cli::proxy_opts::ProxyOpts, +} + +impl PackCreateCmd { + /// Resolve the directory under which the staging temp dir is created. + /// + /// Precedence: `--staging-dir` → `SMOLVM_PACK_STAGING` → the disk-backed + /// cache dir (`/smolvm`) → `$TMPDIR`. The default must NOT be + /// `$TMPDIR`: staging holds the whole image (the pulled layers, a merged + /// copy, the agent rootfs, and the ext4 template), and on most Linux distros + /// `$TMPDIR` is tmpfs (RAM-backed, small), so large images fail there with + /// ENOSPC/EDQUOT. + fn staging_root(&self) -> smolvm::Result { + let root = self + .staging_dir + .clone() + .or_else(|| std::env::var_os("SMOLVM_PACK_STAGING").map(PathBuf::from)) + .or_else(|| dirs::cache_dir().map(|c| c.join("smolvm"))) + .unwrap_or_else(std::env::temp_dir); + std::fs::create_dir_all(&root) + .map_err(|e| Error::agent("create staging root", e.to_string()))?; + Ok(root) + } + + pub fn run(self) -> smolvm::Result<()> { + if let Some(vm_name) = self.from_vm.clone() { + if self.oci_platform.is_some() { + warn!("--oci-platform is ignored with --from-vm (VM snapshot is arch-fixed)"); + } + info!(vm = %vm_name, output = %self.output.display(), "packing from VM"); + return self.pack_from_vm(vm_name); + } + + // Resolve config from Smolfile + CLI + let pack_config = crate::cli::smolfile::resolve_pack_config( + self.image.clone(), + self.entrypoint.clone(), + self.cpus, + self.mem, + self.oci_platform.clone(), + self.gpu, + self.smolfile.clone(), + )?; + + let image = pack_config.image.ok_or_else(|| { + Error::config( + "pack create", + "no image specified. Provide IMAGE argument or set 'image' in Smolfile", + ) + })?; + info!(image = %image, output = %self.output.display(), "packing image"); + + // Create temporary staging directory + let temp_dir = tempfile::Builder::new() + .prefix("pack-staging-") + .tempdir_in(self.staging_root()?) + .map_err(|e| Error::agent("create temp directory", e.to_string()))?; + let staging_dir = temp_dir.path().join("staging"); + + // Start a temporary agent VM with a unique identity so concurrent + // pack runs and the user's "default" VM don't collide. The prefix + // must start with an ascii-alphanumeric character to satisfy + // `validate_vm_name` when `AgentManager::for_vm` receives the name + // (see src/data/mod.rs). A leading underscore — the previous + // `__pack_` convention — was rejected outright and made every + // `smolvm pack create` invocation fail. + // Use PID + epoch nanos to avoid PID-reuse collisions with orphaned VMs. + let pack_vm_name = format!( + "pack-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ); + // Guard ensures VM is stopped on early error. Only removes temp data + // dir after confirmed stop — never deletes while VM may still be running. + let vm_data_dir = smolvm::agent::vm_data_dir(&pack_vm_name); + struct PackVmGuard { + manager: AgentManager, + data_dir: std::path::PathBuf, + finalized: bool, + } + impl PackVmGuard { + /// Stop VM and clean up temp dir. Propagates stop errors. + fn stop_and_cleanup(&mut self) -> smolvm::Result<()> { + self.manager.stop()?; + self.finalized = true; + if let Err(e) = std::fs::remove_dir_all(&self.data_dir) { + if e.kind() != std::io::ErrorKind::NotFound { + tracing::warn!( + error = %e, + dir = %self.data_dir.display(), + "failed to remove pack temp dir" + ); + } + } + Ok(()) + } + } + impl Drop for PackVmGuard { + fn drop(&mut self) { + if self.finalized { + return; + } + match self.manager.stop() { + Ok(()) => { + if let Err(e) = std::fs::remove_dir_all(&self.data_dir) { + if e.kind() != std::io::ErrorKind::NotFound { + tracing::warn!( + error = %e, + dir = %self.data_dir.display(), + "failed to remove pack temp dir" + ); + } + } + } + Err(e) => { + tracing::warn!( + error = %e, + "failed to stop pack VM; preserving temp data dir" + ); + } + } + } + } + + println!("Starting agent VM..."); + let manager = AgentManager::for_vm(&pack_vm_name)?; + manager.start_with_config( + Vec::new(), + VmResources { + cpus: 4, + memory_mib: 8192, + network: true, + network_backend: None, + dns: None, + gpu: false, + storage_gib: None, + overlay_gib: None, + gpu_vram_mib: None, + rosetta: false, + allowed_cidrs: None, + }, + )?; + let mut guard = PackVmGuard { + manager, + data_dir: vm_data_dir, + finalized: false, + }; + let mut client = guard.manager.connect()?; + + // Pull image + let image_info = crate::cli::pull_with_progress( + &mut client, + &image, + pack_config.oci_platform.as_deref(), + self.proxy_opts.proxy(), + self.proxy_opts.no_proxy(), + )?; + debug!(image_info = ?image_info, "image pulled"); + + println!( + "Image: {} ({} layers, {} bytes)", + image, image_info.layer_count, image_info.size + ); + + // Create asset collector and collect base assets + let mut collector = AssetCollector::new(staging_dir.clone()) + .map_err(|e| Error::agent("collect assets", e.to_string()))?; + self.collect_base_assets(&mut collector)?; + + // Export layers. For images with many layers (e.g., rocker/tidyverse + // has 20), we pre-merge all layers into a single directory in the VM + // so the packed binary has one lowerdir at runtime. Without this, the + // overlayfs multi-lowerdir mount fails on virtiofs-backed layers and + // falls back to a 15-second physical merge on every first exec. + if image_info.layer_count <= 1 { + // Single layer — export directly, no merge needed. + let layer_digest = &image_info.layers[0]; + let prefix = format!(" Layer 1/1: {}", &layer_digest[..19]); + print!("{}...", prefix); + let _ = std::io::Write::flush(&mut std::io::stdout()); + let layer_file = collector.layer_staging_path(layer_digest); + self.export_layer_to_file(&mut client, &image_info.digest, 0, &layer_file, &prefix)?; + collector + .register_layer(layer_digest) + .map_err(|e| Error::agent("collect layers", e.to_string()))?; + } else { + // Multiple layers — merge in the VM so runtime gets a single + // lowerdir that always mounts instantly. + println!( + "Merging {} layers in VM (one-time cost)...", + image_info.layer_count + ); + + // Build the merge command: extract each layer in order (bottom + // first), then tar the result. Layer order in image_info.layers + // is bottom-to-top, which is the correct copy order. + let layer_paths: Vec = image_info + .layers + .iter() + .map(|d| { + let id = d.strip_prefix("sha256:").unwrap_or(d); + format!("/storage/layers/{}", id) + }) + .collect(); + + // Copy layers bottom-up into /tmp/merged, then tar + let mut merge_script = String::from("set -e\nmkdir -p /tmp/merged\n"); + for (i, layer_path) in layer_paths.iter().enumerate() { + // cp -a preserves symlinks, permissions, ownership. + // Ignore exit code: cp may fail on device files or sockets + // that can't be copied, but the layer content is intact. + // Redirect stderr so warnings are visible in the output. + merge_script.push_str(&format!( + "echo 'Merging layer {}/{}...'\n\ + cp -a {}/. /tmp/merged/ || true\n", + i + 1, + image_info.layer_count, + layer_path + )); + } + // Verify disk space wasn't exhausted during merge + merge_script.push_str( + "if ! df /tmp/merged | awk 'NR==2{if($4<1024){exit 1}}'; then\n\ + echo 'MERGE_FAIL: disk full'; exit 1\nfi\n\ + echo 'Creating merged tar...'\n\ + tar cf /tmp/merged-layers.tar -C /tmp/merged .\n\ + echo 'MERGE_OK'\n", + ); + + let (exit_code, stdout, stderr) = client.vm_exec( + vec!["sh".to_string(), "-c".to_string(), merge_script], + vec![], + None, + None, + None, + )?; + + // stdout/stderr from vm_exec are now Vec; convert lossily + // for content checks and error messages (merge output is ASCII). + let stdout_str = String::from_utf8_lossy(&stdout); + let stderr_str = String::from_utf8_lossy(&stderr); + if exit_code != 0 || !stdout_str.contains("MERGE_OK") { + return Err(Error::agent( + "merge layers", + format!( + "layer merge failed (exit {}): {}", + exit_code, + if stderr_str.is_empty() { + &stdout_str + } else { + &stderr_str + } + ), + )); + } + + // Download the merged tar — streamed to disk (16 MB chunks, + // never holds the full tar in memory). + print!(" Exporting merged layer..."); + let _ = std::io::Write::flush(&mut std::io::stdout()); + let merged_hash = hex::encode(Sha256::digest( + format!("merged-{}", image_info.digest).as_bytes(), + )); + let merged_digest = format!("sha256:{}", merged_hash); + let merged_file = collector.layer_staging_path(&merged_digest); + + let total_bytes = client + .read_file_to_path("/tmp/merged-layers.tar", &merged_file, |_| {}) + .map_err(|e| Error::agent("export merged layer", e.to_string()))?; + println!(" {} MB done", total_bytes / (1024 * 1024)); + + collector + .register_layer(&merged_digest) + .map_err(|e| Error::agent("register merged layer", e.to_string()))?; + } + + // Stop agent and clean up temp VM data. Propagates stop errors + // so pack fails visibly if VM cannot be stopped. + guard.stop_and_cleanup()?; + + // Build manifest + let platform = format!("{}/{}", image_info.os, image_info.architecture); + let host_platform = Platform::current().host_oci_platform().to_string(); + let mut manifest = + PackManifest::new(image, image_info.digest.clone(), platform, host_platform); + manifest.image_size = image_info.size; + manifest.cpus = pack_config.cpus; + manifest.mem = pack_config.mem; + manifest.network = pack_config.net.unwrap_or(false); + manifest.gpu = pack_config.gpu; + + // Start with OCI image config as baseline + manifest.entrypoint = image_info.entrypoint.clone(); + manifest.cmd = image_info.cmd.clone(); + manifest.env = image_info.env.clone(); + manifest.workdir = image_info.workdir.clone(); + + // Layer Smolfile top-level env on top of image env + if !pack_config.env.is_empty() { + for e in &pack_config.env { + if let Some((key, _)) = e.split_once('=') { + // Remove any existing image env with the same key + manifest + .env + .retain(|existing| !existing.starts_with(&format!("{}=", key))); + } + manifest.env.push(e.clone()); + } + } + + // Carry Smolfile [secrets] refs into the manifest. Refs only — the + // plaintext is resolved on the run host at exec time, never packed. + manifest.secret_refs = pack_config.secret_refs.clone(); + + // Smolfile workdir overrides image workdir + if pack_config.workdir.is_some() { + manifest.workdir = pack_config.workdir; + } + + // Override entrypoint from Smolfile or CLI + if !pack_config.entrypoint.is_empty() { + manifest.entrypoint = pack_config.entrypoint; + } + + // Override cmd from Smolfile + if !pack_config.cmd.is_empty() { + manifest.cmd = pack_config.cmd; + } + + self.finalize_pack(manifest, collector, staging_dir) + } + + /// Pack from a stopped VM's overlay disk. + fn pack_from_vm(self, vm_name: String) -> smolvm::Result<()> { + // 1. Load config and verify VM exists and is stopped + let config = SmolvmConfig::load()?; + let vm = config + .vms + .get(&vm_name) + .ok_or_else(|| Error::agent("pack from VM", format!("VM '{}' not found", vm_name)))?; + + if vm.actual_state() == RecordState::Running { + return Err(Error::agent( + "pack from VM", + format!( + "VM '{}' is running. Stop it first with: smolvm machine stop --name {}", + vm_name, vm_name + ), + )); + } + + // 2. Locate the VM's disks. Default-size machines on Linux get qcow2 CoW + // overlays (overlay.qcow2 / storage.qcow2), not `.raw`, so resolve whichever + // format exists rather than hardcoding `.raw`. The overlay template is only + // consumed by VM-mode (bare) restores — container restores ignore it — so it + // is required only for non-image VMs. + let vm_dir = smolvm::agent::vm_data_dir(&vm_name); + let (overlay_disk, overlay_fmt) = resolve_disk_image(&vm_dir, OVERLAY_DISK_FILENAME); + let is_image_based = vm.image.is_some(); + let is_artifact_sourced_container = is_image_based && vm.source_smolmachine.is_some(); + if !is_image_based && !overlay_disk.exists() { + return Err(Error::agent( + "pack from VM", + format!( + "overlay disk not found at {}. The VM may not have been started yet.", + overlay_disk.display() + ), + )); + } + + println!("Packing VM '{}' snapshot...", vm_name); + + // 3. Create temporary staging directory + let temp_dir = tempfile::Builder::new() + .prefix("pack-staging-") + .tempdir_in(self.staging_root()?) + .map_err(|e| Error::agent("create temp directory", e.to_string()))?; + let staging_dir = temp_dir.path().join("staging"); + + let mut collector = AssetCollector::new(staging_dir.clone()) + .map_err(|e| Error::agent("collect assets", e.to_string()))?; + + // 4. For image-based VMs, preserve imported artifact layers by default + // and append the current container overlay. The explicit rebase path keeps + // the historical behavior of rebuilding lower layers from vm.image. + if is_artifact_sourced_container && !self.rebase_from_image { + let source_smolmachine = vm.source_smolmachine.clone().unwrap(); + let source_path = Path::new(&source_smolmachine); + let source_manifest = if source_path.exists() { + Some( + smolvm_pack::packer::read_manifest_from_sidecar(source_path) + .map_err(|e| Error::agent("read .smolmachine", e.to_string()))?, + ) + } else { + None + }; + + self.collect_base_assets(&mut collector)?; + let imported_layers = + match self.imported_layers_from_cache(&vm_name, source_manifest.as_ref())? { + Some(layers) => layers, + None => self.imported_layers_from_source_sidecar( + &vm_name, + &source_smolmachine, + temp_dir.path(), + )?, + }; + self.add_imported_layers(&mut collector, imported_layers)?; + self.export_container_overlay_from_vm(&mut collector, &vm_name, &vm_dir)?; + } else if is_image_based { + let image = vm.image.clone().unwrap(); + // A locally-sourced image (`--image -` / `--image file.tar` / a rootfs + // dir) is flattened on boot and has no registry manifest, so the + // layer-export path below — which pulls that manifest to enumerate base + // layers — cannot source it. Fail with a clear, actionable message + // instead of a confusing registry "UNAUTHORIZED" on `local:`. + if smolvm::data::image_source::is_local_ref(&image) { + return Err(Error::agent( + "pack from VM", + format!( + "VM '{vm_name}' was created from a local image ({image}). \ + `pack create --from-vm` can only snapshot VMs created from a \ + REGISTRY image — local archives and rootfs directories are \ + flattened on boot and have no registry manifest to re-pull. \ + Recreate the machine from a registry reference to pack it." + ), + )); + } + + self.collect_base_assets(&mut collector)?; + self.export_registry_image_layers_from_vm(&mut collector, &vm_name, &vm_dir, &image)?; + } else { + // Bare VM: just collect base assets, no layers needed. + self.collect_base_assets(&mut collector)?; + } + + // Add the overlay template (the VM's rootfs state). VM-mode restores boot + // from it; container restores ignore it entirely (every import path gates + // `overlay_template` on `PackMode::Vm`), so skip it for image-based VMs + // rather than flatten a qcow2 for nothing. A default-size overlay is a qcow2 + // CoW image, which must be flattened to a raw before it can be a template. + if !is_image_based { + let overlay_for_pack = match overlay_fmt { + DiskFormat::Raw => overlay_disk.clone(), + DiskFormat::Qcow2 => { + let flat = temp_dir.path().join("overlay-flat.raw"); + self.flatten_qcow2_to_raw(&overlay_disk, &flat)?; + flat + } + }; + println!("Copying overlay disk ({})...", overlay_for_pack.display()); + collector + .add_overlay_template(&overlay_for_pack) + .map_err(|e| Error::agent("collect overlay", e.to_string()))?; + } + + // 5. Resolve Smolfile overrides if provided + // Precedence: CLI > [artifact] > Smolfile top-level > VmRecord > default + let pack_config = crate::cli::smolfile::resolve_pack_config( + None, // no image for --from-vm + self.entrypoint.clone(), + self.cpus, + self.mem, + self.oci_platform.clone(), + self.gpu, + self.smolfile.clone(), + )?; + + // 6. Build manifest + let platform = format!("linux/{}", Arch::current().oci_arch()); + let host_platform = Platform::current().host_oci_platform().to_string(); + let mut manifest = PackManifest::new( + format!("vm://{}", vm_name), + "none".to_string(), + platform, + host_platform, + ); + if is_image_based { + manifest.mode = PackMode::Container; + manifest.image = vm.image.clone().unwrap_or_default(); + } else { + manifest.mode = PackMode::Vm; + } + manifest.cpus = pack_config.cpus; + manifest.mem = pack_config.mem; + // Smolfile > source VM record > default + manifest.network = pack_config.net.unwrap_or(vm.network); + // CLI --gpu > Smolfile gpu > source VM record gpu > false + manifest.gpu = pack_config.gpu || vm.gpu.unwrap_or(false); + + // Entrypoint baseline: VmRecord > /bin/sh default + manifest.entrypoint = if !vm.entrypoint.is_empty() { + vm.entrypoint.clone() + } else { + vec!["/bin/sh".to_string()] + }; + manifest.cmd = vm.cmd.clone(); + + // Start with VmRecord env/workdir as baseline + manifest.env = vm.env.iter().map(|(k, v)| format!("{}={}", k, v)).collect(); + manifest.workdir = vm.workdir.clone(); + + // Layer Smolfile env on top of VmRecord env + if !pack_config.env.is_empty() { + for e in &pack_config.env { + if let Some((key, _)) = e.split_once('=') { + manifest + .env + .retain(|existing| !existing.starts_with(&format!("{}=", key))); + } + manifest.env.push(e.clone()); + } + } + + // Baseline secret refs from the source VM record, then layer the + // Smolfile [secrets] on top (Smolfile wins on key collisions). Refs + // only — plaintext is resolved on the run host, never packed. + manifest.secret_refs = vm.secret_refs.clone(); + manifest.secret_refs.extend(pack_config.secret_refs.clone()); + + // Smolfile workdir overrides VmRecord workdir + if pack_config.workdir.is_some() { + manifest.workdir = pack_config.workdir; + } + + // Override entrypoint from Smolfile/[artifact] or CLI + if !pack_config.entrypoint.is_empty() { + manifest.entrypoint = pack_config.entrypoint; + } + + // Override cmd from Smolfile/[artifact] + if !pack_config.cmd.is_empty() { + manifest.cmd = pack_config.cmd; + } + + self.finalize_pack(manifest, collector, staging_dir) + } + + fn export_registry_image_layers_from_vm( + &self, + collector: &mut AssetCollector, + vm_name: &str, + vm_dir: &Path, + image: &str, + ) -> smolvm::Result<()> { + let (storage_disk, storage_fmt) = resolve_disk_image(vm_dir, STORAGE_DISK_FILENAME); + let pack_vm_name = format!( + "pack-fromvm-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ); + let vm_data = smolvm::agent::vm_data_dir(&pack_vm_name); + + println!("Starting agent VM to export layers..."); + let manager = AgentManager::for_vm(&pack_vm_name)?; + let features = smolvm::agent::LaunchFeatures { + extra_disks: vec![(storage_disk.clone(), false, storage_fmt)], + ..Default::default() + }; + manager.start_with_full_config( + Vec::new(), + Vec::new(), + VmResources { + cpus: 4, + memory_mib: 8192, + network: true, + network_backend: None, + dns: None, + gpu: false, + gpu_vram_mib: None, + rosetta: false, + storage_gib: None, + overlay_gib: None, + allowed_cidrs: None, + }, + features, + )?; + + let export_result: smolvm::Result<()> = (|| { + let mut client = manager.connect()?; + let (exit_code, _, stderr) = client.vm_exec( + vec![ + "sh".to_string(), + "-c".to_string(), + "mkdir -p /mnt/source-storage && mount /dev/vdc /mnt/source-storage" + .to_string(), + ], + vec![], + None, + None, + None, + )?; + if exit_code != 0 { + return Err(Error::agent( + "mount source storage in temp VM", + format!( + "mount failed (exit {}): {}", + exit_code, + String::from_utf8_lossy(&stderr) + ), + )); + } + + let image_info = crate::cli::pull_with_progress( + &mut client, + image, + None, + self.proxy_opts.proxy(), + self.proxy_opts.no_proxy(), + )?; + + println!("Exporting {} layers...", image_info.layer_count); + for (i, layer_digest) in image_info.layers.iter().enumerate() { + let prefix = format!( + " Layer {}/{}: {}", + i + 1, + image_info.layer_count, + &layer_digest[..std::cmp::min(19, layer_digest.len())] + ); + print!("{}...", prefix); + let _ = std::io::Write::flush(&mut std::io::stdout()); + let layer_file = collector.layer_staging_path(layer_digest); + self.export_layer_to_file( + &mut client, + &image_info.digest, + i, + &layer_file, + &prefix, + )?; + collector + .register_layer(layer_digest) + .map_err(|e| Error::agent("collect layers", e.to_string()))?; + } + + self.export_container_overlay_from_mounted_storage(collector, &mut client, vm_name)?; + + Ok(()) + })(); + + if let Err(e) = manager.stop() { + warn!(error = %e, "failed to stop pack temp VM"); + } + let _ = std::fs::remove_dir_all(&vm_data); + export_result + } + + fn export_container_overlay_from_mounted_storage( + &self, + collector: &mut AssetCollector, + client: &mut AgentClient, + vm_name: &str, + ) -> smolvm::Result<()> { + let overlay_dir = format!("/mnt/source-storage/overlays/persistent-{}/upper", vm_name); + println!("Exporting container overlay..."); + let (exit_code, _, stderr) = client.vm_exec( + vec![ + "sh".to_string(), + "-c".to_string(), + format!( + "if [ -d '{}' ] && [ \"$(ls -A '{}')\" ]; then \ + tar cf /tmp/overlay-export.tar -C '{}' . 2>/dev/null; \ + echo OVERLAY_OK; \ + else echo OVERLAY_EMPTY; fi", + overlay_dir, overlay_dir, overlay_dir + ), + ], + vec![], + None, + None, + None, + )?; + + if exit_code == 0 { + let tar_data = client.read_file("/tmp/overlay-export.tar")?; + if !tar_data.is_empty() { + let overlay_hash = hex::encode(Sha256::digest(&tar_data)); + let overlay_digest = format!("sha256:{}", overlay_hash); + let overlay_layer_file = collector.layer_staging_path(&overlay_digest); + std::fs::write(&overlay_layer_file, &tar_data) + .map_err(|e| Error::agent("write overlay layer", e.to_string()))?; + collector + .register_layer(&overlay_digest) + .map_err(|e| Error::agent("register overlay layer", e.to_string()))?; + println!(" Overlay layer: {} bytes", tar_data.len()); + } + } else { + tracing::debug!(stderr = %String::from_utf8_lossy(&stderr), "overlay export: no container changes found"); + } + + Ok(()) + } + + fn export_container_overlay_from_vm( + &self, + collector: &mut AssetCollector, + vm_name: &str, + vm_dir: &Path, + ) -> smolvm::Result<()> { + let (storage_disk, storage_fmt) = resolve_disk_image(vm_dir, STORAGE_DISK_FILENAME); + let pack_vm_name = format!( + "pack-fromvm-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ); + let vm_data = smolvm::agent::vm_data_dir(&pack_vm_name); + + println!("Starting agent VM to export container overlay..."); + let manager = AgentManager::for_vm(&pack_vm_name)?; + let features = smolvm::agent::LaunchFeatures { + extra_disks: vec![(storage_disk.clone(), false, storage_fmt)], + ..Default::default() + }; + manager.start_with_full_config( + Vec::new(), + Vec::new(), + VmResources { + cpus: 4, + memory_mib: 8192, + network: true, + network_backend: None, + dns: None, + gpu: false, + gpu_vram_mib: None, + rosetta: false, + storage_gib: None, + overlay_gib: None, + allowed_cidrs: None, + }, + features, + )?; + + let export_result: smolvm::Result<()> = (|| { + let mut client = manager.connect()?; + let (exit_code, _, stderr) = client.vm_exec( + vec![ + "sh".to_string(), + "-c".to_string(), + "mkdir -p /mnt/source-storage && mount /dev/vdc /mnt/source-storage" + .to_string(), + ], + vec![], + None, + None, + None, + )?; + if exit_code != 0 { + return Err(Error::agent( + "mount source storage in temp VM", + format!( + "mount failed (exit {}): {}", + exit_code, + String::from_utf8_lossy(&stderr) + ), + )); + } + + self.export_container_overlay_from_mounted_storage(collector, &mut client, vm_name)?; + + Ok(()) + })(); + + if let Err(e) = manager.stop() { + warn!(error = %e, "failed to stop pack temp VM"); + } + let _ = std::fs::remove_dir_all(&vm_data); + export_result + } + + fn imported_layers_from_cache( + &self, + vm_name: &str, + source_manifest: Option<&PackManifest>, + ) -> smolvm::Result>> { + let cache_dir = smolvm::agent::machine_layers_cache_dir(vm_name); + let pack_content_dir = + smolvm::agent::read_shared_pack_pointer(&cache_dir).unwrap_or(cache_dir); + let layers_dir = pack_content_dir.join("layers"); + + if let Some(manifest) = source_manifest { + let mut layers = Vec::new(); + for layer in &manifest.assets.layers { + let path = smolvm_pack::extract::resolve_cache_asset_path( + &pack_content_dir, + &layer.path, + "source layer", + ) + .map_err(|e| Error::agent("resolve source layer", e.to_string()))?; + if !path.exists() { + return Ok(None); + } + layers.push((layer.digest.clone(), path)); + } + return Ok(Some(layers)); + } + + let order_path = layers_dir.join("layer-order"); + if let Ok(contents) = std::fs::read_to_string(&order_path) { + let mut layers = Vec::new(); + for id in contents.lines().map(str::trim).filter(|id| !id.is_empty()) { + let path = layers_dir.join(format!("{id}.tar")); + if !path.exists() { + return Ok(None); + } + layers.push((format!("sha256:{id}"), path)); + } + return Ok(Some(layers)); + } + + let mut tar_layers = Vec::new(); + if let Ok(entries) = std::fs::read_dir(&layers_dir) { + for entry in entries { + let entry = entry.map_err(|e| Error::agent("read source layers", e.to_string()))?; + let path = entry.path(); + if path.extension().and_then(|s| s.to_str()) == Some("tar") { + tar_layers.push(path); + } + } + } + if tar_layers.len() == 1 { + let path = tar_layers.pop().unwrap(); + if let Some(id) = path.file_stem().and_then(|s| s.to_str()) { + return Ok(Some(vec![(format!("sha256:{id}"), path)])); + } + } + + Ok(None) + } + + fn imported_layers_from_source_sidecar( + &self, + vm_name: &str, + source_smolmachine: &str, + temp_dir: &Path, + ) -> smolvm::Result> { + let source_path = Path::new(source_smolmachine); + let fail = || { + Error::agent( + "pack from VM", + format!( + "VM '{vm_name}' was created from a .smolmachine artifact, but its imported layer closure could not be read from cache or source artifact: {source_smolmachine}" + ), + ) + }; + if !source_path.exists() { + return Err(fail()); + } + + let source_manifest = + smolvm_pack::packer::read_manifest_from_sidecar(source_path).map_err(|_| fail())?; + let footer = + smolvm_pack::packer::read_footer_from_sidecar(source_path).map_err(|_| fail())?; + let source_cache_dir = temp_dir.join("source-pack"); + std::fs::create_dir_all(&source_cache_dir).map_err(|_| fail())?; + smolvm_pack::extract::extract_sidecar(source_path, &source_cache_dir, &footer, true, false) + .map_err(|_| fail())?; + + let mut layers = Vec::new(); + for layer in &source_manifest.assets.layers { + let path = smolvm_pack::extract::resolve_cache_asset_path( + &source_cache_dir, + &layer.path, + "source layer", + ) + .map_err(|_| fail())?; + if !path.exists() { + return Err(fail()); + } + layers.push((layer.digest.clone(), path)); + } + Ok(layers) + } + + fn add_imported_layers( + &self, + collector: &mut AssetCollector, + layers: Vec<(String, PathBuf)>, + ) -> smolvm::Result<()> { + for (_, path) in &layers { + if !path.exists() { + return Err(Error::agent( + "collect source layers", + format!("source layer not found: {}", path.display()), + )); + } + } + + for (digest, path) in layers { + collector + .add_layer_from_file(&digest, &path) + .map_err(|e| Error::agent("collect source layers", e.to_string()))?; + } + Ok(()) + } + + /// Flatten a qcow2 CoW overlay into a standalone raw disk image. + /// + /// Default-size machines use a qcow2 overlay backed by the install's default + /// template, but a pack's overlay template must be a flat raw. There is no + /// host-side qcow2 reader (smolvm deliberately takes no qemu-img dependency), + /// so the conversion runs inside a throwaway agent VM: the source qcow2 is + /// attached read-only (libkrun resolves its backing chain) as `/dev/vdc` + /// alongside a fresh raw output as `/dev/vdd`, and the guest `dd`s one into the + /// other. `add_overlay_template` then strips trailing zeros so a mostly-empty + /// overlay still packs small, and extraction re-sparsifies it on import. + fn flatten_qcow2_to_raw(&self, qcow2_path: &Path, dest_raw: &Path) -> smolvm::Result<()> { + let virtual_size = read_qcow2_virtual_size(qcow2_path)?; + { + let f = std::fs::File::create(dest_raw) + .map_err(|e| Error::agent("create flatten target", e.to_string()))?; + f.set_len(virtual_size) + .map_err(|e| Error::agent("size flatten target", e.to_string()))?; + } + + let flatten_vm_name = format!( + "pack-flatten-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ); + let vm_data = smolvm::agent::vm_data_dir(&flatten_vm_name); + println!("Flattening qcow2 overlay to raw..."); + let manager = AgentManager::for_vm(&flatten_vm_name)?; + let features = smolvm::agent::LaunchFeatures { + // vdc = source qcow2 (read-only), vdd = fresh raw output. + extra_disks: vec![ + (qcow2_path.to_path_buf(), true, DiskFormat::Qcow2), + (dest_raw.to_path_buf(), false, DiskFormat::Raw), + ], + ..Default::default() + }; + manager.start_with_full_config( + Vec::new(), + Vec::new(), + VmResources { + cpus: 2, + memory_mib: 2048, + network: false, + network_backend: None, + dns: None, + gpu: false, + gpu_vram_mib: None, + rosetta: false, + storage_gib: None, + overlay_gib: None, + allowed_cidrs: None, + }, + features, + )?; + + let result: smolvm::Result<()> = (|| { + let mut client = manager.connect()?; + let (exit_code, _, stderr) = client.vm_exec( + vec![ + "sh".to_string(), + "-c".to_string(), + // busybox dd lacks GNU `conv=sparse`, so do a plain full copy + // and `sync`. The output is dense on the temp disk, but + // `add_overlay_template` strips trailing zeros so the pack stays + // small; the imported overlay is re-sparsified on extraction. + "dd if=/dev/vdc of=/dev/vdd bs=1M && sync".to_string(), + ], + vec![], + None, + None, + None, + )?; + if exit_code != 0 { + return Err(Error::agent( + "flatten qcow2 overlay", + format!( + "dd failed (exit {}): {}", + exit_code, + String::from_utf8_lossy(&stderr) + ), + )); + } + Ok(()) + })(); + + if let Err(e) = manager.stop() { + warn!(error = %e, "failed to stop pack flatten VM"); + } + let _ = std::fs::remove_dir_all(&vm_data); + result + } + + /// Collect base assets shared by both image and VM packing modes: + /// runtime libraries, agent rootfs, and a pre-formatted storage template. + fn collect_base_assets(&self, collector: &mut AssetCollector) -> smolvm::Result<()> { + println!("Collecting runtime libraries..."); + let lib_dir = self.find_lib_dir()?; + collector + .collect_libraries(&lib_dir) + .map_err(|e| Error::agent("collect libraries", e.to_string()))?; + + println!("Collecting agent rootfs..."); + let rootfs_dir = self.find_rootfs_dir()?; + collector + .collect_agent_rootfs(&rootfs_dir) + .map_err(|e| Error::agent("collect rootfs", e.to_string()))?; + + println!("Creating storage template..."); + collector + .create_storage_template() + .map_err(|e| Error::agent("create storage template", e.to_string()))?; + + Ok(()) + } + + /// Finalize pack: set inventory, assemble binary, print summary, and sign. + fn finalize_pack( + &self, + mut manifest: PackManifest, + collector: AssetCollector, + staging_dir: PathBuf, + ) -> smolvm::Result<()> { + let stub_path = self.find_smolvm_binary()?; + + manifest.assets = collector.into_inventory(); + + let collector = AssetCollector::new(staging_dir.clone()) + .map_err(|e| Error::agent("collect assets", e.to_string()))?; + + let packer = Packer::new(manifest) + .with_stub(&stub_path) + .with_asset_collector(collector); + + let label = if self.single_file { + "Assembling single-file packed binary" + } else { + "Assembling packed binary" + }; + let spinner = Spinner::start(label); + let info = if self.single_file { + packer + .pack_embedded(&self.output) + .map_err(|e| Error::agent("pack binary", e.to_string()))? + } else { + packer + .pack(&self.output) + .map_err(|e| Error::agent("pack binary", e.to_string()))? + }; + spinner.stop(); + + println!( + "Packed: {} (stub: {}KB, total: {}KB)", + self.output.display(), + info.stub_size / 1024, + info.total_size / 1024 + ); + if let Some(ref sidecar) = info.sidecar_path { + println!( + "Assets: {} ({}KB compressed)", + sidecar.display(), + info.assets_size / 1024 + ); + } else { + println!("Mode: single-file (no sidecar)"); + } + + // Sign on macOS + if Os::current().is_macos() && !self.no_sign { + println!("Signing binary with hypervisor entitlements..."); + if let Err(e) = sign_with_hypervisor_entitlements(&self.output) { + warn!(error = %e, "signing failed (binary may not run on fresh macOS)"); + eprintln!("Warning: Signing failed: {}", e); + eprintln!("The binary may require manual signing to use Hypervisor.framework"); + } else { + println!("Signed successfully"); + } + } + + // Embed libs in stub AFTER signing — SMOLLIBS footer must be at end of file + if !self.single_file { + smolvm_pack::packer::embed_libs_in_binary(&self.output, &staging_dir) + .map_err(|e| Error::agent("embed libraries", e.to_string()))?; + } + + println!("\nRun with: {}", self.output.display()); + if info.sidecar_path.is_some() { + println!("Note: Keep the .smolmachine file alongside the binary"); + } + println!("Options: --help for usage"); + + Ok(()) + } + + /// Find the library directory containing libkrun and libkrunfw. + fn find_lib_dir(&self) -> smolvm::Result { + if let Some(ref dir) = self.lib_dir { + return Ok(dir.clone()); + } + + // Use the same canonical resolver as the launcher and embedded runtime so + // `pack create` finds libkrun anywhere the rest of smolvm does: it honors + // the explicit `$SMOLVM_LIB_DIR` override (e.g. a distro libkrun, or a + // non-standard install dir) and the installed/exe-relative `lib/` layout. + // Without this, packing a build whose libs live outside the hardcoded + // candidates below required `--lib-dir`, even though the env var was set. + if let Some(dir) = smolvm::agent::find_lib_dir() { + debug!(lib_dir = %dir.display(), "found library directory via canonical resolver"); + return Ok(dir); + } + + // Next best: use the exact libkrun that this process has loaded, which + // guarantees the packed binary gets a library with all required symbols. + // (Rarely fires for `pack create`: the builder VM boots in a subprocess, + // so the packer process itself never dlopens libkrun.) + if let Some(dir) = Self::find_loaded_libkrun_dir() { + debug!(lib_dir = %dir.display(), "using libkrun from running process"); + return Ok(dir); + } + + // Fallback: a few well-known locations the canonical resolver does not + // check (Homebrew, /usr/local/lib, and the current working directory). + let platform_lib = format!("lib/linux-{}", std::env::consts::ARCH); + let candidates = [ + // Relative to executable + std::env::current_exe() + .ok() + .and_then(|p| p.parent().map(|d| d.join("lib"))), + std::env::current_exe() + .ok() + .and_then(|p| p.parent().and_then(|d| d.parent()).map(|d| d.join("lib"))), + // Source tree dev builds: /../../lib/linux-/ + std::env::current_exe().ok().and_then(|p| { + p.parent() + .and_then(|d| d.parent()) + .map(|d| d.join(&platform_lib)) + }), + // Source tree (CWD) + Some(PathBuf::from("lib")), + Some(PathBuf::from("./lib")), + Some(PathBuf::from(&platform_lib)), + // Homebrew + Some(PathBuf::from("/opt/homebrew/lib")), + Some(PathBuf::from("/usr/local/lib")), + ]; + + let lib_name = format!( + "libkrun.{}", + smolvm::platform::vm_executor().dylib_extension() + ); + + for candidate in candidates.into_iter().flatten() { + if candidate.join(&lib_name).exists() { + debug!(lib_dir = %candidate.display(), "found library directory"); + return Ok(candidate); + } + } + + Err(Error::agent( + "find libkrun", + "could not find libkrun library. Set SMOLVM_LIB_DIR or pass --lib-dir to specify the location.", + )) + } + + /// Find the directory of the libkrun that this process has already loaded. + /// + /// Uses `dlopen(RTLD_NOLOAD)` to get a handle to the already-loaded library + /// (without loading a new one), then `dladdr` to resolve the symbol back to + /// a filesystem path. This ensures the packer bundles the exact same library + /// that smolvm itself linked against — no version mismatches possible. + #[cfg(unix)] + fn find_loaded_libkrun_dir() -> Option { + use std::ffi::{CStr, CString}; + + unsafe { + let name = CString::new(smolvm::util::libkrun_filename()).ok()?; + let handle = libc::dlopen(name.as_ptr(), libc::RTLD_NOLOAD | libc::RTLD_LAZY); + if handle.is_null() { + return None; + } + + let sym_name = CString::new("krun_create_ctx").ok()?; + let sym = libc::dlsym(handle, sym_name.as_ptr()); + libc::dlclose(handle); + + if sym.is_null() { + return None; + } + + let mut info = std::mem::MaybeUninit::::uninit(); + if libc::dladdr(sym, info.as_mut_ptr()) != 0 { + let info = info.assume_init(); + if !info.dli_fname.is_null() { + let lib_path = CStr::from_ptr(info.dli_fname).to_string_lossy(); + return std::path::Path::new(lib_path.as_ref()) + .parent() + .map(|p| p.to_path_buf()); + } + } + } + + None + } + + /// Windows: the `dlopen(RTLD_NOLOAD)` + `dladdr` self-location trick has no + /// direct equivalent here; callers fall back to the configured/standard + /// library search paths. + #[cfg(not(unix))] + fn find_loaded_libkrun_dir() -> Option { + None + } + + /// Find the agent rootfs directory. + /// + /// Resolution order: + /// 1. Explicit `--rootfs-dir` flag + /// 2. `SMOLVM_AGENT_ROOTFS` env var + /// 3. Installed location (`~/.local/share/smolvm/agent-rootfs` on Linux, + /// `~/Library/Application Support/smolvm/agent-rootfs` on macOS) + /// + /// `target/agent-rootfs` is NOT checked — it can contain stale builds. + /// Use `--rootfs-dir` or `SMOLVM_AGENT_ROOTFS` env var to override. + fn find_rootfs_dir(&self) -> smolvm::Result { + if let Some(ref dir) = self.rootfs_dir { + return Ok(dir.clone()); + } + + let candidates = [ + // SMOLVM_AGENT_ROOTFS env var + std::env::var("SMOLVM_AGENT_ROOTFS").ok().map(PathBuf::from), + // Installed location (canonical) + dirs::data_dir().map(|d| d.join("smolvm/agent-rootfs")), + // Next to the executable (for distribution tarballs) + std::env::current_exe() + .ok() + .and_then(|p| p.parent().map(|d| d.join("agent-rootfs"))), + ]; + + for candidate in candidates.into_iter().flatten() { + // Use symlink_metadata instead of exists() because sbin/init + // is a symlink to a guest-only path (/usr/local/bin/smolvm-agent) + // that doesn't exist on the host. exists() follows symlinks and + // returns false for broken symlinks. + if std::fs::symlink_metadata(candidate.join("sbin/init")).is_ok() { + debug!(rootfs_dir = %candidate.display(), "found agent rootfs"); + return Ok(candidate); + } + } + + Err(Error::agent( + "find agent rootfs", + "could not find agent rootfs. Use --rootfs-dir to specify the location.", + )) + } + + /// Find the smolvm binary to embed as the packed runtime. + /// + /// The main smolvm binary auto-detects packed mode at startup, so it + /// serves as both the normal CLI and the packed binary runtime. + fn find_smolvm_binary(&self) -> smolvm::Result { + if let Some(ref path) = self.stub { + return Ok(path.clone()); + } + + let candidates = [ + // Build output + Some(PathBuf::from("target/release/smolvm")), + Some(PathBuf::from("target/debug/smolvm")), + // Distribution layout: smolvm-bin next to the wrapper script + std::env::current_exe() + .ok() + .and_then(|p| p.parent().map(|d| d.join("smolvm-bin"))), + // The running executable itself + std::env::current_exe().ok(), + // User data dir + dirs::data_dir().map(|d| d.join("smolvm/smolvm-bin")), + ]; + + for candidate in candidates.into_iter().flatten() { + if candidate.exists() { + debug!(stub = %candidate.display(), "found smolvm binary for packing"); + return Ok(candidate); + } + } + + Err(Error::agent( + "find smolvm binary", + "could not find smolvm binary. Build it with:\n \ + cargo build --release\n\ + Or use --stub to specify the path.", + )) + } + + /// Export a layer from the agent. + /// + /// The agent streams the layer as a sequence of `LayerData` chunks. + /// Export a layer from the agent, streaming chunks directly to a file on disk. + /// + /// No memory buffering — each 16MB chunk is written to disk as it arrives. + /// This supports layers of any size without hitting host memory limits. + fn export_layer_to_file( + &self, + client: &mut AgentClient, + image_digest: &str, + layer_index: usize, + dest: &std::path::Path, + progress_prefix: &str, + ) -> smolvm::Result<()> { + use smolvm_protocol::{AgentRequest, FILE_TRANSFER_MAX_TOTAL}; + use std::io::Write; + use std::time::{Duration, Instant}; + + const LAYER_EXPORT_TIMEOUT: Duration = Duration::from_secs(600); // 10 minutes + + let request = AgentRequest::ExportLayer { + image_digest: image_digest.to_string(), + layer_index, + }; + + let _timeout_guard = client.set_extended_read_timeout(LAYER_EXPORT_TIMEOUT)?; + client.send_raw(&request)?; + + let mut file = std::fs::File::create(dest).map_err(|e| { + Error::agent( + "export layer", + format!("failed to create {}: {}", dest.display(), e), + ) + })?; + + let start = Instant::now(); + let mut total_bytes = 0u64; + let mut last_progress = Instant::now(); + loop { + if start.elapsed() > LAYER_EXPORT_TIMEOUT { + return Err(Error::agent( + "export layer", + format!( + "layer export timed out after {}s (received {} bytes so far)", + LAYER_EXPORT_TIMEOUT.as_secs(), + total_bytes + ), + )); + } + + let response = client.recv_raw()?; + match response { + AgentResponse::DataChunk { data, done } => { + if !data.is_empty() { + // Cap the cumulative export size the same way the file-read + // paths do (client.rs): a compromised or buggy guest must not + // be able to exhaust the host disk with an endless chunk stream. + let next_total = total_bytes.saturating_add(data.len() as u64); + if next_total > FILE_TRANSFER_MAX_TOTAL { + let _ = std::fs::remove_file(dest); + return Err(Error::agent( + "export layer", + format!( + "guest streamed {} bytes, exceeding the {} byte cap", + next_total, FILE_TRANSFER_MAX_TOTAL + ), + )); + } + file.write_all(&data).map_err(|e| { + Error::agent("export layer", format!("write failed: {}", e)) + })?; + total_bytes = next_total; + + // Update progress every 500ms + if last_progress.elapsed() >= Duration::from_millis(500) { + print!("\r{}... {}", progress_prefix, fmt_bytes(total_bytes)); + let _ = std::io::stdout().flush(); + last_progress = Instant::now(); + } + } + if done { + file.flush().map_err(|e| { + Error::agent("export layer", format!("flush failed: {}", e)) + })?; + println!("\r{}... {} done", progress_prefix, fmt_bytes(total_bytes)); + return Ok(()); + } + } + AgentResponse::Error { message, .. } => { + return Err(Error::agent("export layer", message)); + } + _ => { + return Err(Error::agent("export layer", "unexpected response type")); + } + } + } + } +} + +// ============================================================================ +// Pack Prune Command +// ============================================================================ + +/// Clean up cached pack extractions to free disk space. +/// +/// Removes old extracted pack caches from ~/.cache/smolvm-pack/ and +/// ~/.cache/smolvm-libs/. By default keeps the 5 most recently used. +/// +/// Examples: +/// smolvm pack prune # keep 5 most recent +/// smolvm pack prune --keep 2 # keep 2 most recent +/// smolvm pack prune --all # remove everything +/// smolvm pack prune --dry-run # show what would be removed +#[derive(Args, Debug)] +pub struct PackPruneCmd { + /// Number of cached extractions to keep (default: 5) + #[arg(long, default_value = "5", value_name = "N")] + pub keep: usize, + + /// Remove all cached extractions + #[arg(long)] + pub all: bool, + + /// Show what would be removed without actually removing + #[arg(long)] + pub dry_run: bool, +} + +impl PackPruneCmd { + pub fn run(self) -> smolvm::Result<()> { + let keep = if self.all { 0 } else { self.keep }; + + let mut total_freed: u64 = 0; + let mut total_removed: usize = 0; + + // Clean pack sidecar cache + if let Some(base) = dirs::cache_dir() { + let pack_cache = base.join("smolvm-pack"); + let (freed, removed) = self.prune_cache_dir(&pack_cache, keep, "pack cache")?; + total_freed += freed; + total_removed += removed; + + // Clean libs cache + let libs_cache = base.join("smolvm-libs"); + let (freed, removed) = self.prune_cache_dir(&libs_cache, keep, "libs cache")?; + total_freed += freed; + total_removed += removed; + } + + if total_removed > 0 { + if self.dry_run { + println!( + "Would remove {} cached entries ({})", + total_removed, + crate::cli::format_bytes(total_freed) + ); + } else { + println!( + "Removed {} cached entries, freed {}", + total_removed, + crate::cli::format_bytes(total_freed) + ); + } + } else { + println!("No cached entries to remove."); + } + + Ok(()) + } + + fn prune_cache_dir( + &self, + base: &std::path::Path, + keep: usize, + label: &str, + ) -> smolvm::Result<(u64, usize)> { + if !base.exists() { + return Ok((0, 0)); + } + + // Collect entries with modification time (skip entries we can't stat) + let mut entries: Vec<(std::path::PathBuf, std::time::SystemTime, u64)> = vec![]; + let read_dir = match std::fs::read_dir(base) { + Ok(rd) => rd, + Err(e) => { + tracing::warn!(error = %e, path = %base.display(), "cannot read {}", label); + return Ok((0, 0)); + } + }; + for entry in read_dir { + let entry = match entry { + Ok(e) => e, + Err(_) => continue, + }; + let path = entry.path(); + if !path.is_dir() { + continue; + } + let metadata = match std::fs::metadata(&path) { + Ok(m) => m, + Err(e) => { + tracing::warn!(error = %e, path = %path.display(), "skipping unreadable entry in {}", label); + continue; + } + }; + let modified = metadata + .modified() + .unwrap_or(std::time::SystemTime::UNIX_EPOCH); + let size = dir_size(&path); + entries.push((path, modified, size)); + } + + // Sort by most recently modified (newest first) + entries.sort_by_key(|b| std::cmp::Reverse(b.1)); + + // Remove entries beyond keep count + let to_remove = if entries.len() > keep { + &entries[keep..] + } else { + return Ok((0, 0)); + }; + + let mut freed: u64 = 0; + let mut removed: usize = 0; + + for (path, _, size) in to_remove { + // Skip caches that have active leases (running VMs or daemons). + if smolvm_pack::extract::has_active_leases(path) { + println!(" skipping in-use cache: {} (active lease)", path.display()); + continue; + } + + if self.dry_run { + println!( + " would remove: {} ({})", + path.display(), + crate::cli::format_bytes(*size) + ); + } else { + // Detach any mounted case-sensitive volume before removing. + // Safe because we verified no active leases above. + smolvm_pack::extract::force_detach_layers_volume(path); + if let Err(e) = std::fs::remove_dir_all(path) { + tracing::warn!(error = %e, path = %path.display(), "failed to remove {}", label); + continue; + } + // Also remove lock file if present + let lock = path.with_extension("lock"); + let _ = std::fs::remove_file(&lock); + } + freed += size; + removed += 1; + } + + Ok((freed, removed)) + } +} + +// ============================================================================ +// Push / Pull — registry operations for .smolmachine artifacts +// ============================================================================ + +/// Push a .smolmachine artifact to a registry. +/// +/// Examples: +/// smolvm pack push myapp:v1 -f ./my-app.smolmachine +/// smolvm pack push registry.example.com/myapp:latest -f ./app.smolmachine +#[derive(Args, Debug)] +pub struct PackPushCmd { + /// Artifact reference (e.g., myapp:v1, registry.example.com/myapp:latest) + #[arg(value_name = "REFERENCE")] + pub reference: String, + + /// Path to the .smolmachine file to push + #[arg(short = 'f', long, value_name = "PATH")] + pub file: PathBuf, +} + +impl PackPushCmd { + pub fn run(self) -> smolvm::Result<()> { + if !self.file.exists() { + return Err(Error::agent( + "push", + format!("file not found: {}", self.file.display()), + )); + } + + let parsed = smolvm::registry::Reference::parse(&self.reference) + .map_err(|e| Error::agent("parse reference", e.to_string()))?; + let settings = smolvm::SmolSettings::load()?; + let client = build_registry_client(&parsed.registry, &settings.machines)?; + + let repo = parsed.repository(); + let tag = parsed.tag.as_deref().unwrap_or("latest"); + + eprintln!( + "Pushing {} to {}/{}:{}", + self.file.display(), + parsed.registry, + repo, + tag + ); + + let rt = tokio::runtime::Runtime::new() + .map_err(|e| Error::agent("create tokio runtime", e.to_string()))?; + + let result = rt + .block_on(smolvm_registry::push(&client, &repo, tag, &self.file)) + .map_err(|e| Error::agent("registry push", e.to_string()))?; + + eprintln!( + "Pushed successfully\n Layer: {} ({} bytes)\n Manifest: {}", + result.layer_digest, result.layer_size, result.manifest_digest, + ); + Ok(()) + } +} + +/// Pull a .smolmachine artifact from a registry. +/// +/// Examples: +/// smolvm pack pull myapp:v1 +/// smolvm pack pull myapp:v1 -o ./my-app.smolmachine +/// smolvm pack pull registry.example.com/myapp@sha256:abc123... +#[derive(Args, Debug)] +pub struct PackPullCmd { + /// Artifact reference (e.g., myapp:v1, registry.example.com/myapp:latest) + #[arg(value_name = "REFERENCE")] + pub reference: String, + + /// Output path for the downloaded .smolmachine file + #[arg(short = 'o', long, value_name = "PATH")] + pub output: Option, +} + +impl PackPullCmd { + pub fn run(self) -> smolvm::Result<()> { + let parsed = smolvm::registry::Reference::parse(&self.reference) + .map_err(|e| Error::agent("parse reference", e.to_string()))?; + let settings = smolvm::SmolSettings::load()?; + let client = build_registry_client(&parsed.registry, &settings.machines)?; + + let repo = parsed.repository(); + let tag_or_digest = parsed + .digest + .as_deref() + .or(parsed.tag.as_deref()) + .unwrap_or("latest"); + + eprintln!("Pulling {}/{}:{}", parsed.registry, repo, tag_or_digest); + + let rt = tokio::runtime::Runtime::new() + .map_err(|e| Error::agent("create tokio runtime", e.to_string()))?; + + let cache = smolvm_registry::BlobCache::open_default() + .map_err(|e| Error::agent("open blob cache", e.to_string()))?; + + // Optional brokered P2P peers for this one-shot CLI pull. Process-global + // env is fine here (single-shot command). Comma-separated node base URLs; + // empty/unset ⇒ registry-only. + let blob_peers: Vec = std::env::var("SMOLVM_BLOB_PEERS") + .ok() + .map(|v| { + v.split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect() + }) + .unwrap_or_default(); + + let result = rt + .block_on(smolvm_registry::pull( + &client, + &repo, + tag_or_digest, + self.output.as_deref(), + &cache, + &blob_peers, + )) + .map_err(|e| Error::agent("registry pull", e.to_string()))?; + + if result.cached { + eprintln!("Using cached blob ({})", result.digest); + } + + let dest = self.output.unwrap_or(result.path); + eprintln!( + "Pulled successfully -> {} ({} bytes)", + dest.display(), + result.size, + ); + + // Warn if the artifact targets a different host platform. + // This is not an error — the user may be inspecting or transferring + // the artifact — but it will not run on this host as-is. + if let Ok(manifest) = smolvm_pack::read_manifest_from_sidecar(&dest) { + let current = Platform::current().host_oci_platform(); + if manifest.host_platform != current { + eprintln!( + "Warning: this artifact was built for {} and will not run on {} (current platform).\ + \nTo run on {}, create a new pack:\ + \n\n smolvm pack create --image {} -o ", + manifest.host_platform, current, current, manifest.image, + ); + } + } + + Ok(()) + } +} + +/// Inspect a .smolmachine artifact in a registry without downloading the full artifact. +/// +/// Fetches only the OCI manifest and config blob (~1KB total) to display +/// metadata about the packed machine. +/// +/// Examples: +/// smolvm pack inspect myapp:v1 +/// smolvm pack inspect myapp:v1 --json +#[derive(Args, Debug)] +pub struct PackInspectCmd { + /// Artifact reference (e.g., myapp:v1, registry.example.com/myapp:latest) + #[arg(value_name = "REFERENCE")] + pub reference: String, + + /// Output as JSON + #[arg(long)] + pub json: bool, +} + +impl PackInspectCmd { + pub fn run(self) -> smolvm::Result<()> { + let parsed = smolvm::registry::Reference::parse(&self.reference) + .map_err(|e| Error::agent("parse reference", e.to_string()))?; + let settings = smolvm::SmolSettings::load()?; + let client = build_registry_client(&parsed.registry, &settings.machines)?; + + let repo = parsed.repository(); + let tag_or_digest = parsed + .digest + .as_deref() + .or(parsed.tag.as_deref()) + .unwrap_or("latest"); + + let rt = tokio::runtime::Runtime::new() + .map_err(|e| Error::agent("create tokio runtime", e.to_string()))?; + + rt.block_on(run_inspect( + &client, + &parsed, + &repo, + tag_or_digest, + self.json, + )) + } +} + +async fn run_inspect( + client: &smolvm_registry::RegistryClient, + parsed: &smolvm::registry::Reference, + repo: &str, + tag_or_digest: &str, + json_output: bool, +) -> smolvm::Result<()> { + // Fetch the OCI manifest (~200 bytes), resolving a multi-platform index to + // this machine's host-platform entry — same as `pull`, so inspect agrees with + // what pull would actually download instead of rejecting multi-arch tags. + let manifest_bytes = client + .get_manifest_resolved(repo, tag_or_digest) + .await + .map_err(|e| Error::agent("fetch manifest", e.to_string()))?; + + let oci_manifest: smolvm_registry::OciManifest = serde_json::from_slice(&manifest_bytes) + .map_err(|e| Error::agent("parse manifest", e.to_string()))?; + + // Extract layer size from OCI manifest. + let layer_size = oci_manifest.layers.first().map(|l| l.size).unwrap_or(0); + let layer_digest = oci_manifest + .layers + .first() + .map(|l| l.digest.as_str()) + .unwrap_or("unknown"); + + // Fetch config blob (~500 bytes) — contains the PackManifest. + let config_bytes = client + .pull_blob(repo, &oci_manifest.config.digest) + .await + .map_err(|e| Error::agent("fetch config", e.to_string()))?; + + let pack_manifest: smolvm_pack::PackManifest = serde_json::from_slice(&config_bytes) + .map_err(|e| Error::agent("parse config", e.to_string()))?; + + if json_output { + // Include layer size/digest in JSON output. + let mut json_val: serde_json::Value = serde_json::to_value(&pack_manifest) + .map_err(|e| Error::agent("serialize", e.to_string()))?; + if let Some(obj) = json_val.as_object_mut() { + obj.insert( + "layer_size".to_string(), + serde_json::Value::Number(layer_size.into()), + ); + obj.insert( + "layer_digest".to_string(), + serde_json::Value::String(layer_digest.to_string()), + ); + } + let json_str = serde_json::to_string_pretty(&json_val) + .map_err(|e| Error::agent("serialize inspect output", e.to_string()))?; + println!("{}", json_str); + } else { + let full_ref = format!("{}/{}:{}", parsed.registry, repo, tag_or_digest); + println!("Reference: {}", full_ref); + println!("Image: {}", pack_manifest.image); + println!("Platform: {}", pack_manifest.platform); + { + let current = Platform::current().host_oci_platform(); + if pack_manifest.host_platform == current { + println!("Host: {}", pack_manifest.host_platform); + } else { + println!( + "Host: {} [incompatible — current platform: {}]", + pack_manifest.host_platform, current + ); + } + } + println!("CPUs: {}", pack_manifest.cpus); + println!("Memory: {} MiB", pack_manifest.mem); + if !pack_manifest.entrypoint.is_empty() { + println!("Entrypoint: {}", pack_manifest.entrypoint.join(" ")); + } + if !pack_manifest.cmd.is_empty() { + println!("Cmd: {}", pack_manifest.cmd.join(" ")); + } + if let Some(ref wd) = pack_manifest.workdir { + println!("Workdir: {}", wd); + } + println!("Created: {}", pack_manifest.created); + println!("Version: {}", pack_manifest.smolvm_version); + println!("Size: {}", crate::cli::format_bytes(layer_size)); + println!("Digest: {}", layer_digest); + } + + Ok(()) +} + +/// Build a `RegistryClient` from a registry hostname, applying auth from config. +fn build_registry_client( + registry: &str, + config: &smolvm::registry::RegistryConfig, +) -> smolvm::Result { + let effective = config.get_mirror(registry).unwrap_or(registry); + + // Docker Hub: the user-facing name is "docker.io" but the Distribution API + // endpoint is "registry-1.docker.io". The config key stays "docker.io" so + // credential lookup is consistent; only the HTTP endpoint changes. + let api_host = match effective { + "docker.io" => "registry-1.docker.io", + h => h, + }; + + let base_url = if smolvm_registry::is_local_registry(api_host) { + format!("http://{}", api_host) + } else { + format!("https://{}", api_host) + }; + + let mut client = smolvm_registry::RegistryClient::new(base_url); + + if let Some(entry) = config.registries.get(registry) { + if let Some(identity_token) = &entry.identity_token { + // Upstream credential (e.g. Auth0 JWT): exchanged with the token service + // per-operation to obtain a short-lived OCI bearer token. + client = client.with_identity_token(identity_token.clone()); + } else if let Some(auth) = config.get_credentials(registry) { + if auth.username == "token" { + // Legacy direct-bearer convention: username="token" means the + // password value IS the bearer token, sent on every request. + client = client.with_token(auth.password); + } else { + // Standard Docker/OCI path: username+password are sent as Basic auth + // to the registry's token endpoint after a 401 Bearer challenge. + // Used for Docker Hub, GHCR, ECR, GCR, ACR, Harbor, and Quay. + client = client.with_basic_credentials(auth.username, auth.password); + } + } + } + + Ok(client) +} + +/// Format a byte count as a human-readable string (KB for < 1 MB, MB otherwise). +fn fmt_bytes(bytes: u64) -> String { + if bytes < 1024 * 1024 { + format!("{} KB", bytes / 1024) + } else { + format!("{} MB", bytes / (1024 * 1024)) + } +} + +/// Read a qcow2 image's virtual (guest-visible) size from its header — the +/// big-endian `u64` at byte offset 24, per the qcow2 spec. Lets the flatten path +/// size its raw output correctly without a qcow2 library. +fn read_qcow2_virtual_size(path: &Path) -> smolvm::Result { + use std::io::{Read, Seek, SeekFrom}; + let mut f = std::fs::File::open(path).map_err(|e| Error::agent("open qcow2", e.to_string()))?; + let mut magic = [0u8; 4]; + f.read_exact(&mut magic) + .map_err(|e| Error::agent("read qcow2 magic", e.to_string()))?; + if &magic != b"QFI\xfb" { + return Err(Error::agent( + "read qcow2", + format!("{} is not a qcow2 image (bad magic)", path.display()), + )); + } + f.seek(SeekFrom::Start(24)) + .map_err(|e| Error::agent("seek qcow2 size", e.to_string()))?; + let mut buf = [0u8; 8]; + f.read_exact(&mut buf) + .map_err(|e| Error::agent("read qcow2 size", e.to_string()))?; + Ok(u64::from_be_bytes(buf)) +} + +/// A simple terminal spinner that prints a rotating character every 200ms. +/// Stops automatically on drop (error paths) or via explicit `stop()`. +struct Spinner { + stop: std::sync::Arc, + handle: Option>, +} + +impl Spinner { + fn start(label: &str) -> Self { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + let stop = Arc::new(AtomicBool::new(false)); + let stop_clone = stop.clone(); + let label = label.to_string(); + + let handle = std::thread::spawn(move || { + let frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']; + let mut i = 0; + while !stop_clone.load(Ordering::Relaxed) { + print!("\r{} {}\x1b[K", frames[i % frames.len()], label); + let _ = std::io::Write::flush(&mut std::io::stdout()); + i += 1; + std::thread::sleep(std::time::Duration::from_millis(200)); + } + print!("\r\x1b[K"); + println!("{} done", label); + }); + + Spinner { + stop, + handle: Some(handle), + } + } + + fn stop(self) { + // Drop impl handles the actual shutdown + drop(self); + } +} + +impl Drop for Spinner { + fn drop(&mut self) { + self.stop.store(true, std::sync::atomic::Ordering::Relaxed); + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} + +/// Calculate the total size of a directory (recursive). +fn dir_size(path: &std::path::Path) -> u64 { + std::fs::read_dir(path) + .ok() + .map(|entries| { + entries + .filter_map(|e| e.ok()) + .map(|e| { + let meta = e.metadata().ok(); + if e.path().is_dir() { + dir_size(&e.path()) + } else { + meta.map(|m| m.len()).unwrap_or(0) + } + }) + .sum() + }) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Verify that dladdr-based libkrun discovery finds the loaded library. + /// + /// This test works because the test binary links against libkrun the same + /// way the smolvm binary does (via build.rs). If the library is loaded, + /// find_loaded_libkrun_dir() must return its directory. + #[test] + fn find_loaded_libkrun_dir_returns_valid_path() { + let dir = PackCreateCmd::find_loaded_libkrun_dir(); + + // On CI without libkrun, the function returns None — that's fine, + // the fallback search handles it. But when libkrun IS loaded + // (which it is for any machine that can run smolvm), it must return + // a valid directory containing the library. + if let Some(ref dir) = dir { + assert!(dir.exists(), "dladdr returned non-existent dir: {:?}", dir); + + let lib_name = smolvm::util::libkrun_filename(); + let lib_path = dir.join(lib_name); + assert!( + lib_path.exists(), + "dladdr dir {:?} does not contain {}", + dir, + lib_name + ); + } + // If None, libkrun wasn't loaded (e.g., weak link + library not found). + // This is expected in some CI environments and is not a failure. + } + + // ── build_registry_client auth path selection ──────────────────────────── + + #[test] + fn build_registry_client_uses_identity_token_when_set() { + let mut config = smolvm::registry::RegistryConfig::default(); + config.registries.insert( + "registry.smolmachines.com".to_string(), + smolvm::registry::RegistryEntry { + identity_token: Some("eyJ_upstream_jwt".to_string()), + ..Default::default() + }, + ); + + let client = build_registry_client("registry.smolmachines.com", &config).unwrap(); + assert_eq!( + client.identity_token(), + Some("eyJ_upstream_jwt"), + "identity_token must be passed to with_identity_token()" + ); + } + + #[test] + fn build_registry_client_standard_credentials_use_basic_auth() { + // A real username (not "token") triggers the Docker/OCI Basic challenge path. + let mut config = smolvm::registry::RegistryConfig::default(); + config.registries.insert( + "ghcr.io".to_string(), + smolvm::registry::RegistryEntry { + username: Some("github_user".to_string()), + password: Some("ghp_secret".to_string()), + ..Default::default() + }, + ); + + let client = build_registry_client("ghcr.io", &config).unwrap(); + assert_eq!(client.identity_token(), None); + assert_eq!( + client.basic_credentials(), + Some(("github_user", "ghp_secret")), + "standard username must route to with_basic_credentials()" + ); + } + + #[test] + fn build_registry_client_token_username_sends_direct_bearer() { + // username="token" is the legacy direct-bearer convention. + let mut config = smolvm::registry::RegistryConfig::default(); + config.registries.insert( + "custom.registry.io".to_string(), + smolvm::registry::RegistryEntry { + username: Some("token".to_string()), + password: Some("bearer_value".to_string()), + ..Default::default() + }, + ); + + let client = build_registry_client("custom.registry.io", &config).unwrap(); + assert_eq!(client.identity_token(), None); + assert_eq!(client.basic_credentials(), None); + } + + #[test] + fn build_registry_client_docker_hub_uses_api_endpoint() { + // docker.io must map to registry-1.docker.io for Distribution API calls. + let config = smolvm::registry::RegistryConfig::default(); + let client = build_registry_client("docker.io", &config).unwrap(); + assert_eq!( + client.base_url(), + "https://registry-1.docker.io", + "docker.io must map to registry-1.docker.io" + ); + } + + #[test] + fn build_registry_client_identity_token_wins_over_password() { + // When both are set (shouldn't happen in practice after set_credentials clears + // identity_token, but we verify the precedence rule is enforced). + let mut config = smolvm::registry::RegistryConfig::default(); + config.registries.insert( + "registry.smolmachines.com".to_string(), + smolvm::registry::RegistryEntry { + username: Some("user".to_string()), + password: Some("stale_password".to_string()), + identity_token: Some("eyJ_identity".to_string()), + ..Default::default() + }, + ); + + let client = build_registry_client("registry.smolmachines.com", &config).unwrap(); + assert_eq!( + client.identity_token(), + Some("eyJ_identity"), + "identity_token must take precedence over password" + ); + } +} diff --git a/src/cli/pack_run.rs b/src/cli/pack_run.rs new file mode 100644 index 0000000..8ba6667 --- /dev/null +++ b/src/cli/pack_run.rs @@ -0,0 +1,1990 @@ +//! `pack run` subcommand: run a VM from a `.smolmachine` sidecar file. +//! +//! This module provides two entry points: +//! +//! 1. **`PackRunCmd`** — the explicit `smolvm pack run` subcommand +//! 2. **`run_as_packed_binary()`** — auto-detected packed binary mode, +//! called from `main()` before clap parses the normal CLI +//! +//! Both paths converge on the same VM launch infrastructure. + +use crate::cli::parsers::{mounts_to_virtiofs_bindings, parse_env_spec}; +use clap::{Args, Parser, Subcommand}; +use smolvm::agent::launcher_dynamic::{ + launch_agent_vm_dynamic, KrunFunctions, PackedLaunchConfig, PackedMount, +}; +use smolvm::agent::{AgentClient, RunConfig, VmResources}; +use smolvm::data::network::PortMapping; +use smolvm::data::storage::HostMount; +use smolvm::network::{validate_requested_network_backend, NetworkBackend}; +use smolvm::platform::Platform; +use smolvm::Error; +use smolvm::DEFAULT_SHELL_CMD; +use smolvm_pack::detect::PackedMode; +use smolvm_pack::extract; +use smolvm_pack::format::PackMode; +use smolvm_pack::packer::{ + read_footer_from_sidecar, read_manifest_from_sidecar, verify_sidecar_checksum, +}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +/// Timeout waiting for the agent to become ready. +const AGENT_READY_TIMEOUT: Duration = Duration::from_secs(30); + +/// Resolve the lib directory containing libkrun/libkrunfw. +/// +/// Two-file mode: libs embedded in stub binary (SMOLLIBS footer). +/// Single-file mode: libs extracted from Mach-O section to cache_dir/lib/. +/// `smolvm pack run`: uses the host-installed libs. +fn resolve_lib_dir(cache_dir: &Path, debug: bool) -> smolvm::Result { + // Two-file mode: libs embedded in stub binary (SMOLLIBS footer) + if let Ok(exe_path) = std::env::current_exe() { + if let Ok(Some(lib_dir)) = extract::extract_libs_from_binary(&exe_path, debug) { + if debug { + eprintln!("debug: using libs from stub binary: {}", lib_dir.display()); + } + return Ok(lib_dir); + } + } + + // Single-file mode: libs extracted from Mach-O section alongside other assets + let cache_lib = cache_dir.join("lib"); + if cache_lib.exists() { + if debug { + eprintln!("debug: using libs from cache: {}", cache_lib.display()); + } + return Ok(cache_lib); + } + + // Host-installed libs (for `smolvm pack run .smolmachine`) + if let Some(host_lib) = smolvm::agent::find_lib_dir() { + if debug { + eprintln!( + "debug: using libs from host install: {}", + host_lib.display() + ); + } + return Ok(host_lib); + } + + Err(Error::agent( + "find libraries", + "libkrun/libkrunfw not found. The binary may be corrupted or the libraries are missing.", + )) +} + +/// Convert parsed mounts to PackedMount format for the VM launcher. +fn mounts_to_packed(mounts: &[smolvm::data::storage::HostMount]) -> Vec { + mounts + .iter() + .enumerate() + .map(|(i, m)| PackedMount { + tag: HostMount::mount_tag(i), + host_path: m.source.to_string_lossy().to_string(), + guest_path: m.target.to_string_lossy().to_string(), + read_only: m.read_only, + }) + .collect() +} + +/// Run a VM from a packed `.smolmachine` sidecar file. +/// +/// Extracts runtime assets (if not already cached), boots a VM using +/// dynamically loaded libkrun, and executes a command using the full +/// smolvm agent infrastructure. +/// +/// Examples: +/// smolvm pack run -- echo hello +/// smolvm pack run --sidecar my-app.smolmachine -it -- /bin/sh +/// smolvm pack run -p 8080:80 --net +#[derive(Args, Debug)] +pub struct PackRunCmd { + /// Path to the `.smolmachine` sidecar file. + /// + /// If not specified, looks for `.smolmachine` next to the + /// smolvm binary, or any `.smolmachine` file in the current directory. + #[arg(long, value_name = "PATH")] + pub sidecar: Option, + + /// Command and arguments to run (default: image entrypoint) + #[arg(trailing_var_arg = true, value_name = "COMMAND")] + pub command: Vec, + + /// Keep stdin open for interactive input + #[arg(short = 'i', long, help_heading = "Execution")] + pub interactive: bool, + + /// Allocate a pseudo-TTY (use with -i for interactive shells) + #[arg(short = 't', long, help_heading = "Execution")] + pub tty: bool, + + /// Kill command after duration (e.g., "30s", "5m") + #[arg( + long, + value_parser = crate::cli::parsers::parse_duration, + value_name = "DURATION", + help_heading = "Execution" + )] + pub timeout: Option, + + /// Set working directory inside container + #[arg(short = 'w', long, value_name = "DIR", help_heading = "Container")] + pub workdir: Option, + + /// Set environment variable (can be used multiple times) + #[arg( + short = 'e', + long = "env", + value_name = "KEY=VALUE", + help_heading = "Container" + )] + pub env: Vec, + + /// Mount host directory into container (can be used multiple times) + #[arg( + short = 'v', + long = "volume", + value_name = "HOST:CONTAINER[:ro]", + help_heading = "Container" + )] + pub volume: Vec, + + /// Expose port from container to host (can be used multiple times) + #[arg( + short = 'p', + long = "port", + value_parser = PortMapping::parse, + value_name = "HOST:GUEST", + help_heading = "Network" + )] + pub port: Vec, + + /// Enable outbound network access + #[arg(long, help_heading = "Network")] + pub net: bool, + + /// Select the networking backend. + #[arg(long = "net-backend", value_enum, help_heading = "Network")] + pub net_backend: Option, + + /// Number of virtual CPUs (overrides manifest default) + #[arg(long, value_name = "N", help_heading = "Resources")] + pub cpus: Option, + + /// Memory allocation in MiB (overrides manifest default) + #[arg(long, value_name = "MiB", help_heading = "Resources")] + pub mem: Option, + + /// Storage disk size in GiB (for OCI layers and container data) + #[arg(long, value_name = "GiB", help_heading = "Resources")] + pub storage: Option, + + /// Overlay disk size in GiB (for persistent rootfs changes) + #[arg(long, value_name = "GiB", help_heading = "Resources")] + pub overlay: Option, + + /// Re-extract assets even if already cached + #[arg(long)] + pub force_extract: bool, + + /// Show manifest info and exit + #[arg(long)] + pub info: bool, + + /// Enable debug output + #[arg(long)] + pub debug: bool, +} + +impl PackRunCmd { + /// Execute the pack run command. + pub fn run(self) -> smolvm::Result<()> { + // 1. Resolve sidecar path + let sidecar_path = resolve_sidecar_path(self.sidecar.as_deref())?; + + if self.debug { + eprintln!("debug: using sidecar: {}", sidecar_path.display()); + } + + // 2. Read footer and verify checksum before trusting any content + let footer = read_footer_from_sidecar(&sidecar_path) + .map_err(|e| Error::agent("read footer", e.to_string()))?; + + match verify_sidecar_checksum(&sidecar_path, &footer) { + Ok(true) => { + if self.debug { + eprintln!("debug: sidecar checksum verified ({:08x})", footer.checksum); + } + } + Ok(false) => { + return Err(Error::agent( + "verify sidecar", + format!( + "checksum mismatch for {}: sidecar may be corrupt or tampered with.\n\ + Try re-packing the image with `smolvm pack`.", + sidecar_path.display() + ), + )); + } + Err(e) => { + return Err(Error::agent( + "verify sidecar", + format!("failed to verify checksum: {}", e), + )); + } + } + + // 3. Read manifest (safe now that checksum is verified) + let manifest = read_manifest_from_sidecar(&sidecar_path) + .map_err(|e| Error::agent("read manifest", e.to_string()))?; + + // 4. Handle --info: show manifest and exit + if self.info { + print_manifest_info(&manifest, footer.checksum); + return Ok(()); + } + + // 5. Platform compatibility check — fail before extraction so the error + // is immediate and actionable rather than a cryptic dlopen failure. + { + let current = Platform::current().host_oci_platform(); + if manifest.host_platform != current { + return Err(Error::agent( + "platform mismatch", + format!( + "this artifact was built for {} but the current platform is {}\ + \n\nTo run on {}, create a new pack:\ + \n\n smolvm pack create --image -o \ + \n\nOr push platform-specific artifacts under separate tags:\ + \n\n smolvm pack push myrepo:v1-darwin-arm64 -f myapp-darwin.smolmachine\ + \n smolvm pack push myrepo:v1-linux-amd64 -f myapp-linux.smolmachine", + manifest.host_platform, current, current, + ), + )); + } + } + + // 6. Extract assets to cache (locked to prevent concurrent extraction races) + let cache_dir = extract::get_cache_dir(footer.checksum) + .map_err(|e| Error::agent("get cache dir", e.to_string()))?; + + extract::extract_sidecar( + &sidecar_path, + &cache_dir, + &footer, + self.force_extract, + self.debug, + ) + .map_err(|e| Error::agent("extract assets", e.to_string()))?; + + // 7. Set up paths — use a unique runtime directory per invocation so + // concurrent runs of the same checksum don't conflict on + // storage.ext4 / agent.sock. tempdir_in gives us a truly unique + // directory that survives PID reuse and abrupt termination. + let rootfs_path = cache_dir.join("agent-rootfs"); + let lib_dir = resolve_lib_dir(&cache_dir, self.debug)?; + let layers_lease = extract::acquire_layers_lease(&cache_dir, self.debug) + .map_err(|e| Error::agent("acquire layers lease", e.to_string()))?; + let layers_dir = &layers_lease.path; + let runtime_parent = cache_dir.join("runtime"); + std::fs::create_dir_all(&runtime_parent) + .map_err(|e| Error::agent("create runtime parent", e.to_string()))?; + let runtime_dir = tempfile::tempdir_in(&runtime_parent) + .map_err(|e| Error::agent("create runtime dir", e.to_string()))?; + + let storage_path = runtime_dir.path().join("storage.ext4"); + let vsock_path = runtime_dir.path().join("agent.sock"); + + // Compute auto-sized storage before creating the disk so both the + // disk file and VmResources use the same value. + let storage_gib = storage_gib_for_manifest(self.storage, &manifest); + + // Create storage disk (each invocation gets its own copy) + let template = manifest + .assets + .storage_template + .as_ref() + .map(|t| t.path.as_str()); + extract::create_or_copy_storage_disk(&cache_dir, template, &storage_path, storage_gib) + .map_err(|e| Error::agent("create storage disk", e.to_string()))?; + + let overlay_runtime_path = setup_vm_overlay( + &manifest, + &cache_dir, + &runtime_dir.path().join("overlay.raw"), + self.overlay, + )?; + + // 8. Parse CLI args + let mounts = HostMount::parse(&self.volume)?; + let port_mappings = PortMapping::to_tuples(&self.port); + + let resources = VmResources { + cpus: self.cpus.unwrap_or(manifest.cpus), + memory_mib: self.mem.unwrap_or(manifest.mem), + network: self.net || manifest.network || !self.port.is_empty(), + network_backend: self.net_backend, + dns: None, + gpu: manifest.gpu, + storage_gib, + overlay_gib: self.overlay, + gpu_vram_mib: None, + rosetta: false, + allowed_cidrs: None, + }; + validate_requested_network_backend(&resources, None, self.port.len())?; + + // Build packed mounts for the launcher + let packed_mounts = mounts_to_packed(&mounts); + + if self.debug { + eprintln!("debug: rootfs={}", rootfs_path.display()); + eprintln!("debug: lib_dir={}", lib_dir.display()); + eprintln!("debug: storage={}", storage_path.display()); + eprintln!("debug: vsock={}", vsock_path.display()); + eprintln!( + "debug: resources cpus={} mem={} net={} gpu={}", + resources.cpus, resources.memory_mib, resources.network, resources.gpu + ); + } + + // 9. Launch VM with dynamically loaded libkrun. + // + // On Unix we fork a session-leader child that dlopen's libkrun directly. + // On Windows there is no usable fork, so we mirror the non-packed + // launcher's subprocess model: serialize a `BootConfig` and re-spawn this + // executable as `current_exe _boot-vm `, which loads libkrun + // from `SMOLVM_LIB_DIR` and boots the VM. See `src/agent/manager.rs`. + smolvm::process::install_sigchld_handler(); + + let console_log_path = runtime_dir.path().join("console.log"); + + #[cfg(unix)] + let child_pid = { + let vsock_path_clone = vsock_path.clone(); + smolvm::process::fork_session_leader(move || { + // Child process: load libkrun via dlopen and launch VM + let krun = match unsafe { KrunFunctions::load(&lib_dir) } { + Ok(k) => k, + Err(e) => { + eprintln!("failed to load libkrun: {}", e); + smolvm::process::exit_child(1); + } + }; + + let config = PackedLaunchConfig { + rootfs_path: &rootfs_path, + storage_path: &storage_path, + vsock_socket: &vsock_path_clone, + layers_dir, + mounts: &packed_mounts, + port_mappings: &port_mappings, + resources, + overlay_path: overlay_runtime_path.as_deref(), + debug: self.debug, + console_log: console_log_path, + }; + + // Detach from parent's terminal so libkrun doesn't + // steal keystrokes or corrupt terminal state. + smolvm::process::detach_stdio(); + + if let Err(e) = launch_agent_vm_dynamic(&krun, &config) { + let msg = format!("launch_agent_vm_dynamic failed: {}\n", e); + let _ = std::fs::write(&config.console_log, &msg); + } + + smolvm::process::exit_child(1); + }) + .map_err(|e| Error::agent("fork VM process", e.to_string()))? + }; + + #[cfg(not(unix))] + let child_pid = { + // These bindings are consumed by the Unix fork closure above; on + // Windows the BootConfig carries the same data, so silence the + // unused-variable warnings rather than reshape the surrounding code. + let _ = (&packed_mounts, &port_mappings); + + // The overlay file is always created on disk before this point + // (`setup_vm_overlay`); fall back to the well-known runtime path when + // the helper returned None so the boot subprocess attaches it. + let overlay_disk_path = overlay_runtime_path + .clone() + .unwrap_or_else(|| runtime_dir.path().join("overlay.raw")); + + let boot_config = smolvm::agent::boot_config::BootConfig { + rootfs_path: rootfs_path.clone(), + storage_disk_path: storage_path.clone(), + overlay_disk_path, + vsock_socket: vsock_path.clone(), + console_log: Some(console_log_path.clone()), + startup_error_log: runtime_dir.path().join("startup-error.log"), + storage_size_gb: storage_gib.unwrap_or(smolvm::storage::DEFAULT_STORAGE_SIZE_GIB), + overlay_size_gb: self + .overlay + .unwrap_or(smolvm::storage::DEFAULT_OVERLAY_SIZE_GIB), + mounts: mounts.clone(), + ports: self.port.clone(), + resources: resources.clone(), + ssh_agent_socket: None, + cuda: false, + dns_filter_hosts: None, + packed_layers_dir: Some(layers_dir.to_path_buf()), + pack_idmap_source: None, + extra_disks: vec![], + }; + + let config_path = runtime_dir.path().join("boot-config.json"); + let config_json = serde_json::to_vec(&boot_config) + .map_err(|e| Error::agent("serialize boot config", e.to_string()))?; + std::fs::write(&config_path, &config_json) + .map_err(|e| Error::agent("write boot config", e.to_string()))?; + + let exe = std::env::current_exe() + .map_err(|e| Error::agent("find smolvm binary", e.to_string()))?; + let mut cmd = std::process::Command::new(&exe); + cmd.args(["_boot-vm", &config_path.to_string_lossy()]) + // The boot subprocess loads libkrun/libkrunfw from here. + .env("SMOLVM_LIB_DIR", &lib_dir) + // The CLI detaches the VM; don't arm the parent-death watchdog. + .env("SMOLVM_BOOT_WATCH_PARENT", "0") + // Per-VM readiness marker name so the agent writes its own marker. + .env( + smolvm_protocol::guest_env::READY_MARKER, + smolvm_protocol::AGENT_READY_MARKER, + ); + if self.debug { + cmd.stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()); + } else { + cmd.stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + } + // Detach from the launching console and give the VM its own process + // group so a Ctrl-C in the launcher isn't forwarded — mirrors the + // `_boot-vm` spawn in `src/agent/manager.rs`. + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + const DETACHED_PROCESS: u32 = 0x0000_0008; + const CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200; + cmd.creation_flags(DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP); + } + let child = cmd + .spawn() + .map_err(|e| Error::agent("spawn boot subprocess", e.to_string()))?; + // `Pid` is `i32` on Windows while `Child::id` returns `u32`; the + // downstream liveness/start-time/kill helpers take `Pid`. + child.id() as smolvm::process::Pid + }; + + // Capture the child's start time so we can verify PID identity + // later (guards against PID reuse). The proc info may not be + // available on the very first try if the kernel hasn't finished + // setting up the child, so retry briefly. + let child_start_time = { + let mut st = smolvm::process::process_start_time(child_pid); + if st.is_none() && smolvm::process::is_alive(child_pid) { + for _ in 0..5 { + std::thread::sleep(Duration::from_millis(1)); + st = smolvm::process::process_start_time(child_pid); + if st.is_some() { + break; + } + } + } + // If the child is alive but we still can't get its start + // time, we have no way to safely verify PID identity later. + // Kill it now (we KNOW it's our child — we just forked it) + // rather than risk either an orphan or a misidentified kill. + if st.is_none() && smolvm::process::is_alive(child_pid) { + let _ = smolvm::process::stop_process_fast(child_pid, Duration::from_secs(5), true); + // Clean up runtime dir ourselves since the guard won't + // be created. + if let Err(e) = std::fs::remove_dir_all(runtime_dir.path()) { + tracing::debug!(error = %e, "cleanup: remove runtime dir after failed child start"); + } + return Err(Error::agent( + "verify child process", + "unable to capture child start time for safe lifecycle management", + )); + } + st + }; + + if self.debug { + eprintln!("debug: forked VM process with PID {}", child_pid); + } + + // Guard ensures the VM child is terminated and runtime dir is + // cleaned up on every exit path — including ? propagation, + // panics, AND the explicit process::exit() on the success path + // (which skips Rust destructors, so we must drop manually). + // start_time is guaranteed Some when the child is alive (enforced + // above), so is_our_process_strict always has data to verify. + let child_guard = ChildGuard { + pid: child_pid, + start_time: child_start_time, + runtime_dir, + }; + + // 10. Parent: wait for agent, connect, execute command + let mut client = wait_for_agent(&vsock_path, self.debug)?; + + let exit_code = execute_command(&mut client, &manifest, &self, &mounts)?; + + // std::process::exit skips destructors, so drop explicitly first. + drop(child_guard); + drop(layers_lease); // releases layers volume lease (detaches if last) + std::process::exit(exit_code); + } +} + +/// RAII guard that terminates the VM child process and cleans up the +/// per-invocation runtime directory on drop. +struct ChildGuard { + pid: smolvm::process::Pid, + start_time: Option, + runtime_dir: tempfile::TempDir, +} + +impl Drop for ChildGuard { + fn drop(&mut self) { + // Only signal the child if we can verify it's still ours via + // start_time. When start_time is None (child exited before we + // could query it), we skip signaling entirely — the PID may have + // been recycled and we must not target an unrelated process. + if smolvm::process::is_our_process_strict(self.pid, self.start_time) { + let _ = smolvm::process::stop_process_fast(self.pid, Duration::from_secs(5), true); + } + // TempDir removes itself on drop, but we also do an explicit + // remove to handle partially-cleaned states. + if let Err(e) = std::fs::remove_dir_all(self.runtime_dir.path()) { + tracing::debug!(error = %e, "cleanup: remove runtime dir on drop"); + } + } +} + +/// Resolve the path to the `.smolmachine` sidecar file. +fn resolve_sidecar_path(explicit: Option<&Path>) -> smolvm::Result { + // Explicit path + if let Some(path) = explicit { + if path.exists() { + return Ok(path.to_path_buf()); + } + return Err(Error::agent( + "find sidecar", + format!( + "sidecar file not found: {}\nSpecify with --sidecar PATH", + path.display() + ), + )); + } + + // Try next to the executable: .smolmachine + if let Ok(exe) = std::env::current_exe() { + let sidecar = smolvm_pack::sidecar_path_for(&exe); + if sidecar.exists() { + return Ok(sidecar); + } + } + + // Try any .smolmachine file in the current directory + if let Ok(cwd) = std::env::current_dir() { + let entries: Vec<_> = std::fs::read_dir(&cwd) + .into_iter() + .flatten() + .filter_map(|e| e.ok()) + .filter(|e| e.path().extension().is_some_and(|ext| ext == "smolmachine")) + .collect(); + + if entries.len() == 1 { + return Ok(entries[0].path()); + } + if entries.len() > 1 { + return Err(Error::agent( + "find sidecar", + "multiple .smolmachine files in current directory. Specify with --sidecar PATH" + .to_string(), + )); + } + } + + Err(Error::agent( + "find sidecar", + "no .smolmachine sidecar file found.\n\ + Specify with: smolvm pack run --sidecar PATH", + )) +} + +/// Set up the overlay disk for VM mode. +/// +/// If the manifest specifies VM mode, copies the overlay template from +/// `cache_dir` to `dest`. Returns the overlay path (for `PackedLaunchConfig`) +/// or `None` for container mode. +/// +/// Fails hard if the manifest is VM mode but the overlay template is missing. +fn setup_vm_overlay( + manifest: &smolvm_pack::PackManifest, + cache_dir: &Path, + dest: &Path, + overlay_gb: Option, +) -> smolvm::Result> { + if manifest.mode == PackMode::Vm { + // VM mode: use the overlay template from the pack + let overlay_template = manifest + .assets + .overlay_template + .as_ref() + .map(|t| t.path.as_str()); + + extract::copy_overlay_template( + cache_dir, + overlay_template, + dest, + overlay_gb, + manifest.assets.overlay_logical_size, + ) + .map_err(|e| { + Error::agent( + "setup overlay", + format!( + "VM mode overlay template is missing or corrupt: {}. \ + Try re-packing with `smolvm pack --from-vm`.", + e + ), + ) + })?; + + return Ok(Some(dest.to_path_buf())); + } + + // OCI image mode: create a fresh overlay disk so the guest has a + // writable root (needed for crun to mkdir /dev, mount proc, etc.) + if !dest.exists() { + let size_gb = overlay_gb.unwrap_or(smolvm::storage::DEFAULT_OVERLAY_SIZE_GIB); + smolvm::storage::OverlayDisk::open_or_create_at(dest, size_gb) + .map_err(|e| Error::agent("create overlay disk", e.to_string()))?; + } + + Ok(Some(dest.to_path_buf())) +} + +/// Wait for the agent to become ready on the vsock socket. +fn wait_for_agent(vsock_path: &Path, debug: bool) -> smolvm::Result { + use std::thread; + use std::time::Instant; + + let start = Instant::now(); + let poll_interval = Duration::from_millis(100); + + loop { + if start.elapsed() > AGENT_READY_TIMEOUT { + return Err(Error::agent( + "wait for agent", + format!( + "agent did not become ready within {} seconds", + AGENT_READY_TIMEOUT.as_secs() + ), + )); + } + + if vsock_path.exists() { + // Connect opens the Unix socket to the muxer, but the guest agent + // may not be listening on vsock port 6000 yet. We must ping to + // verify end-to-end connectivity before declaring the agent ready. + match AgentClient::connect(vsock_path) { + Ok(mut client) => match client.ping() { + Ok(_) => { + if debug { + eprintln!( + "debug: agent ready after {:.1}s", + start.elapsed().as_secs_f64() + ); + } + return Ok(client); + } + Err(_) => { + // Muxer accepted but guest agent not ready yet + } + }, + Err(_) => { + // Socket exists but not connectable yet + } + } + } + + thread::sleep(poll_interval); + } +} + +/// Compute storage size for a packed VM. +/// +/// If the user passed `--storage`, use that. Otherwise, auto-size based on the +/// image's extracted on-disk size (recorded in manifest during `pack create`). +/// Formula: `max(20, image_gib * 3 + 5)` — the 3x covers overlay copy-up, +/// container setup, and ext4 metadata; the +5 gives headroom for user data. +fn storage_gib_for_manifest( + explicit: Option, + manifest: &smolvm_pack::PackManifest, +) -> Option { + if explicit.is_some() { + return explicit; + } + if manifest.image_size == 0 { + // Legacy manifest without image_size — use default. + return None; + } + let image_gib = manifest.image_size / (1024 * 1024 * 1024); + // Layers are extracted to /storage, then overlayfs copies writable parts, + // and crun sets up the container rootfs. 3x the image size + 10 GiB + // headroom covers the worst case. Floor at 20 GiB for small images. + let needed = std::cmp::max(image_gib * 3 + 10, 20); + Some(needed) +} + +/// Build the command to execute from manifest defaults and CLI overrides. +fn build_command(manifest: &smolvm_pack::PackManifest, cli_command: &[String]) -> Vec { + if !cli_command.is_empty() { + return cli_command.to_vec(); + } + + // Use manifest entrypoint + cmd + let mut cmd = manifest.entrypoint.clone(); + cmd.extend(manifest.cmd.clone()); + + if cmd.is_empty() { + vec![DEFAULT_SHELL_CMD.to_string()] + } else { + cmd + } +} + +/// Build environment variables from manifest defaults and CLI overrides. +fn build_env( + manifest: &smolvm_pack::PackManifest, + cli_env: &[String], +) -> smolvm::Result> { + let mut env: Vec<(String, String)> = manifest + .env + .iter() + .filter_map(|e| parse_env_spec(e)) + .collect(); + + // A .smolmachine is a portable artifact that may have been authored on + // another host or by another party. Validate every packed ref under the + // Untrusted scope FIRST — which now rejects every source kind — so a packed + // `from_env`/`from_file` ref cannot be resolved against the running host's + // env or files (an exfil primitive). Resolution does not enforce scope on + // its own, so this gate is what makes the path fail closed. In practice a + // pack carries no resolvable secret; configure secrets locally instead. + for (key, r) in &manifest.secret_refs { + smolvm::secrets::validate_ref(r, smolvm::secrets::ResolutionScope::Untrusted).map_err( + |e| { + smolvm::Error::config( + "run packed machine", + format!("secret '{}': {} (packs may not carry secret refs)", key, e), + ) + }, + )?; + } + env.extend(smolvm::secrets::expose_into_env( + smolvm::secrets::resolve_refs_to_env( + &manifest.secret_refs, + smolvm::secrets::ResolutionScope::Untrusted, + )?, + )); + + // CLI env overrides manifest env and resolved secrets + for spec in cli_env { + if let Some((key, value)) = parse_env_spec(spec) { + // Remove existing key if present + env.retain(|(k, _)| k != &key); + env.push((key, value)); + } + } + + Ok(env) +} + +/// Execute the command in the VM using the existing AgentClient. +/// +/// In Container mode, runs via `client.run_non_interactive()` / `client.run_interactive()` (crun container). +/// In VM mode, runs via `client.vm_exec()` / `client.vm_exec_interactive()` (direct in rootfs). +fn execute_command( + client: &mut AgentClient, + manifest: &smolvm_pack::PackManifest, + args: &PackRunCmd, + mounts: &[smolvm::data::storage::HostMount], +) -> smolvm::Result { + let command = build_command(manifest, &args.command); + let env = build_env(manifest, &args.env)?; + let workdir = args.workdir.clone().or_else(|| manifest.workdir.clone()); + + let params = ExecParams { + command, + env, + workdir, + interactive: args.interactive, + tty: args.tty, + timeout: args.timeout, + }; + execute_packed_command(client, manifest, params, mounts, None) +} + +/// Resolved execution parameters for a packed command. +struct ExecParams { + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + interactive: bool, + tty: bool, + timeout: Option, +} + +/// Execute a command in the VM — shared by both `PackRunCmd` and packed binary paths. +#[allow(clippy::too_many_arguments)] +fn execute_packed_command( + client: &mut AgentClient, + manifest: &smolvm_pack::PackManifest, + params: ExecParams, + mounts: &[smolvm::data::storage::HostMount], + persistent_overlay_id: Option, +) -> smolvm::Result { + let ExecParams { + command, + env, + workdir, + interactive, + tty, + timeout, + } = params; + match manifest.mode { + PackMode::Vm => { + // VM mode: execute directly in the VM rootfs + if interactive || tty { + client.vm_exec_interactive(command, env, workdir, timeout, tty) + } else { + let (exit_code, stdout, stderr) = + client.vm_exec(command, env, workdir, timeout, None)?; + + if !stdout.is_empty() { + let _ = std::io::stdout().write_all(&stdout); + } + if !stderr.is_empty() { + let _ = std::io::stderr().write_all(&stderr); + } + crate::cli::flush_output(); + Ok(exit_code) + } + } + PackMode::Container => { + // Container mode: run inside crun container + let mount_bindings = mounts_to_virtiofs_bindings(mounts); + + if interactive || tty { + let config = RunConfig::new(&manifest.image, command) + .with_env(env) + .with_workdir(workdir) + .with_mounts(mount_bindings) + .with_timeout(timeout) + .with_tty(tty) + .with_persistent_overlay(persistent_overlay_id.clone()); + client.run_interactive(config) + } else { + let config = RunConfig::new(&manifest.image, command) + .with_env(env) + .with_workdir(workdir) + .with_mounts(mount_bindings) + .with_timeout(timeout) + .with_persistent_overlay(persistent_overlay_id); + let (exit_code, stdout, stderr) = client.run_non_interactive(config)?; + + if !stdout.is_empty() { + let _ = std::io::stdout().write_all(&stdout); + } + if !stderr.is_empty() { + let _ = std::io::stderr().write_all(&stderr); + } + crate::cli::flush_output(); + Ok(exit_code) + } + } + } +} + +// =========================================================================== +// Packed binary auto-detection entry point +// =========================================================================== + +// CLI for packed binary mode. Parsed in `run_as_packed_binary()` before +// the normal smolvm CLI. Uses subcommands matching the smolvm pattern. +#[derive(Parser, Debug)] +#[command(about = "a smol machine")] +#[command( + long_about = "A portable, self-contained virtual machine.\n\nRun with no arguments to execute the packaged entrypoint.\nUse subcommands for more control." +)] +#[command(version)] +#[command(subcommand_required = false)] +struct PackedCli { + /// Subcommand to execute (defaults to `run` if omitted) + #[command(subcommand)] + command: Option, + + /// Force re-extraction of assets + #[arg(long, global = true)] + force_extract: bool, + + /// Print debug information + #[arg(long, global = true)] + debug: bool, +} + +#[derive(Subcommand, Debug)] +enum PackedCmd { + /// Run a command in an ephemeral VM (cleaned up after exit) + Run(PackedRunArgs), + /// Start a persistent VM + Start(PackedStartArgs), + /// Execute a command in a running VM + Exec(PackedExecArgs), + /// Open an interactive shell in a running VM + #[command(visible_alias = "sh")] + Shell, + /// Stop the VM + Stop, + /// Show VM status + Status, + /// Show packed binary info + Info, +} + +/// Arguments for the `run` subcommand (ephemeral execution). +#[derive(Args, Debug, Default)] +struct PackedRunArgs { + /// Command to run (overrides image entrypoint/cmd) + #[arg(trailing_var_arg = true, value_name = "COMMAND")] + command: Vec, + + /// Keep stdin open for interactive input + #[arg(short = 'i', long)] + interactive: bool, + + /// Allocate a pseudo-TTY (use with -i for interactive shells) + #[arg(short = 't', long)] + tty: bool, + + /// Kill command after duration (e.g., "30s", "5m") + #[arg(long, value_parser = crate::cli::parsers::parse_duration, value_name = "DURATION")] + timeout: Option, + + /// Working directory inside the container + #[arg(short = 'w', long = "workdir", value_name = "PATH")] + workdir: Option, + + /// Set environment variable (KEY=VALUE) + #[arg(short = 'e', long = "env", value_name = "KEY=VALUE")] + env: Vec, + + /// Mount a volume (HOST:GUEST[:ro]) + #[arg(short = 'v', long = "volume", value_name = "HOST:GUEST[:ro]")] + volume: Vec, + + /// Expose port from container to host + #[arg(short = 'p', long = "port", value_parser = PortMapping::parse, value_name = "HOST:GUEST")] + port: Vec, + + /// Enable outbound network access + #[arg(long)] + net: bool, + + /// Select the networking backend. + #[arg(long = "net-backend", value_enum)] + net_backend: Option, + + /// Number of vCPUs (overrides default) + #[arg(long, value_name = "N")] + cpus: Option, + + /// Memory in MiB (overrides default) + #[arg(long, value_name = "MiB")] + mem: Option, + + /// Storage disk size in GiB + #[arg(long, value_name = "GiB")] + storage: Option, + + /// Overlay disk size in GiB + #[arg(long, value_name = "GiB")] + overlay: Option, +} + +/// Arguments for the `start` subcommand (persistent daemon). +#[derive(Args, Debug)] +struct PackedStartArgs { + /// Number of vCPUs (overrides default) + #[arg(long, value_name = "N")] + cpus: Option, + + /// Memory in MiB (overrides default) + #[arg(long, value_name = "MiB")] + mem: Option, + + /// Storage disk size in GiB + #[arg(long, value_name = "GiB")] + storage: Option, + + /// Overlay disk size in GiB + #[arg(long, value_name = "GiB")] + overlay: Option, + + /// Mount a volume (HOST:GUEST[:ro]) + #[arg(short = 'v', long = "volume", value_name = "HOST:GUEST[:ro]")] + volume: Vec, + + /// Expose port from container to host + #[arg(short = 'p', long = "port", value_parser = PortMapping::parse, value_name = "HOST:GUEST")] + port: Vec, + + /// Enable outbound network access + #[arg(long)] + net: bool, + + /// Select the networking backend. + #[arg(long = "net-backend", value_enum)] + net_backend: Option, +} + +/// Arguments for the `exec` subcommand (run in existing VM). +#[derive(Args, Debug, Default)] +struct PackedExecArgs { + /// Command to run + #[arg(trailing_var_arg = true, value_name = "COMMAND")] + command: Vec, + + /// Keep stdin open for interactive input + #[arg(short = 'i', long)] + interactive: bool, + + /// Allocate a pseudo-TTY (use with -i for interactive shells) + #[arg(short = 't', long)] + tty: bool, + + /// Kill command after duration (e.g., "30s", "5m") + #[arg(long, value_parser = crate::cli::parsers::parse_duration, value_name = "DURATION")] + timeout: Option, + + /// Working directory inside the container + #[arg(short = 'w', long = "workdir", value_name = "PATH")] + workdir: Option, + + /// Set environment variable (KEY=VALUE) + #[arg(short = 'e', long = "env", value_name = "KEY=VALUE")] + env: Vec, + // Note: -v/--volume is intentionally absent from exec. Virtiofs devices + // are fixed at VM boot time, so mounts must be specified on `start`. +} + +/// Entry point when auto-detection determines we are a packed binary. +/// +/// Called from `main()` before clap parses the normal CLI. +/// Parses its own `PackedCli` args and executes accordingly. +/// Never returns — calls `std::process::exit()`. +pub fn run_as_packed_binary(mode: PackedMode) -> ! { + // Use argv[0] as the binary name so --help and --version display + // "my-app 0.5.1" instead of "smolvm 0.5.1". + let bin_name = std::env::args() + .next() + .and_then(|a| { + std::path::Path::new(&a) + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + }) + .unwrap_or_else(|| "packed-binary".to_string()); + + let args: Vec = std::iter::once(bin_name.clone()) + .chain(std::env::args().skip(1)) + .collect(); + // Leak is safe: this function is `-> !` so the process exits after parsing. + let name_static: &'static str = Box::leak(bin_name.into_boxed_str()); + let matches = ::command() + .name(name_static) + .get_matches_from(args); + let cli = ::from_arg_matches(&matches) + .unwrap_or_else(|e| e.exit()); + + let result = pack_run_inner(mode, cli); + match result { + Ok(()) => std::process::exit(0), + Err(e) => { + eprintln!("error: {}", e); + std::process::exit(1); + } + } +} + +fn pack_run_inner(mode: PackedMode, cli: PackedCli) -> smolvm::Result<()> { + let debug = cli.debug; + let force_extract = cli.force_extract; + let command = cli + .command + .unwrap_or(PackedCmd::Run(PackedRunArgs::default())); + + match command { + PackedCmd::Run(args) => run_ephemeral(mode, args, debug, force_extract), + PackedCmd::Start(args) => daemon_start(&mode, args, debug, force_extract), + PackedCmd::Exec(args) => { + let checksum = mode_checksum(&mode); + let manifest = read_manifest_for_mode(&mode)?; + daemon_exec(checksum, args, debug, &manifest) + } + PackedCmd::Shell => { + let checksum = mode_checksum(&mode); + let manifest = read_manifest_for_mode(&mode)?; + daemon_exec( + checksum, + PackedExecArgs { + command: vec!["/bin/sh".to_string()], + interactive: true, + tty: true, + ..Default::default() + }, + debug, + &manifest, + ) + } + PackedCmd::Stop => { + let checksum = mode_checksum(&mode); + daemon_stop(checksum, debug) + } + PackedCmd::Status => { + let checksum = mode_checksum(&mode); + daemon_status(checksum) + } + PackedCmd::Info => { + let checksum = mode_checksum(&mode); + let manifest = read_manifest_for_mode(&mode)?; + print_manifest_info(&manifest, checksum); + Ok(()) + } + } +} + +/// Run an ephemeral VM from the packed binary (cleaned up after exit). +fn run_ephemeral( + mode: PackedMode, + args: PackedRunArgs, + debug: bool, + force_extract: bool, +) -> smolvm::Result<()> { + match mode { + PackedMode::Sidecar { + sidecar_path, + footer: _, + } => { + // Construct PackRunCmd from PackedRunArgs and delegate to existing path + let cmd = PackRunCmd { + sidecar: Some(sidecar_path), + command: args.command, + interactive: args.interactive, + tty: args.tty, + timeout: args.timeout, + workdir: args.workdir, + env: args.env, + volume: args.volume, + port: args.port, + net: args.net, + net_backend: args.net_backend, + cpus: args.cpus, + mem: args.mem, + storage: args.storage, + overlay: args.overlay, + force_extract, + info: false, + debug, + }; + cmd.run() + } + + #[cfg(target_os = "macos")] + PackedMode::Section { + manifest, + checksum, + assets_ptr, + assets_size, + } => run_section_mode( + *manifest, + checksum, + assets_ptr, + assets_size, + args, + debug, + force_extract, + ), + + PackedMode::Embedded { exe_path, footer } => { + run_embedded_mode(exe_path, footer, args, debug, force_extract) + } + } +} + +/// Run from Mach-O section-embedded assets. +#[cfg(target_os = "macos")] +fn run_section_mode( + manifest: smolvm_pack::PackManifest, + checksum: u32, + assets_ptr: *const u8, + assets_size: usize, + args: PackedRunArgs, + debug: bool, + force_extract: bool, +) -> smolvm::Result<()> { + let cache_dir = extract::get_cache_dir(checksum) + .map_err(|e| Error::agent("get cache dir", e.to_string()))?; + + let needs_extract = force_extract || !extract::is_extracted(&cache_dir); + if needs_extract { + unsafe { + extract::extract_from_section(&cache_dir, assets_ptr, assets_size, debug) + .map_err(|e| Error::agent("extract section assets", e.to_string()))?; + } + } + + run_from_cache(&cache_dir, &manifest, args, debug) +} + +/// Run from binary-appended assets. +fn run_embedded_mode( + exe_path: PathBuf, + footer: smolvm_pack::PackFooter, + args: PackedRunArgs, + debug: bool, + force_extract: bool, +) -> smolvm::Result<()> { + // Read manifest from the binary + let manifest = smolvm_pack::read_manifest(&exe_path) + .map_err(|e| Error::agent("read manifest", e.to_string()))?; + + let cache_dir = extract::get_cache_dir(footer.checksum) + .map_err(|e| Error::agent("get cache dir", e.to_string()))?; + + let needs_extract = force_extract || !extract::is_extracted(&cache_dir); + if needs_extract { + extract::extract_from_binary(&exe_path, &cache_dir, &footer, debug) + .map_err(|e| Error::agent("extract embedded assets", e.to_string()))?; + } + + run_from_cache(&cache_dir, &manifest, args, debug) +} + +/// Shared launch path for section and embedded modes. +/// +/// Assets are already extracted to `cache_dir`. Boot VM and run the command. +fn run_from_cache( + cache_dir: &Path, + manifest: &smolvm_pack::PackManifest, + args: PackedRunArgs, + debug: bool, +) -> smolvm::Result<()> { + let rootfs_path = cache_dir.join("agent-rootfs"); + let lib_dir = resolve_lib_dir(cache_dir, debug)?; + let layers_lease = extract::acquire_layers_lease(cache_dir, debug) + .map_err(|e| Error::agent("acquire layers lease", e.to_string()))?; + let layers_dir = &layers_lease.path; + let runtime_parent = cache_dir.join("runtime"); + std::fs::create_dir_all(&runtime_parent) + .map_err(|e| Error::agent("create runtime parent", e.to_string()))?; + let runtime_dir = tempfile::tempdir_in(&runtime_parent) + .map_err(|e| Error::agent("create runtime dir", e.to_string()))?; + + let storage_path = runtime_dir.path().join("storage.ext4"); + let vsock_path = runtime_dir.path().join("agent.sock"); + + let storage_gib = storage_gib_for_manifest(args.storage, manifest); + + let template = manifest + .assets + .storage_template + .as_ref() + .map(|t| t.path.as_str()); + extract::create_or_copy_storage_disk(cache_dir, template, &storage_path, storage_gib) + .map_err(|e| Error::agent("create storage disk", e.to_string()))?; + + let overlay_runtime_path = setup_vm_overlay( + manifest, + cache_dir, + &runtime_dir.path().join("overlay.raw"), + args.overlay, + )?; + + let mounts = HostMount::parse(&args.volume)?; + let port_mappings = PortMapping::to_tuples(&args.port); + let resources = VmResources { + cpus: args.cpus.unwrap_or(manifest.cpus), + memory_mib: args.mem.unwrap_or(manifest.mem), + network: args.net || manifest.network || !args.port.is_empty(), + network_backend: args.net_backend, + dns: None, + gpu: manifest.gpu, + storage_gib, + overlay_gib: args.overlay, + gpu_vram_mib: None, + rosetta: false, + allowed_cidrs: None, + }; + validate_requested_network_backend(&resources, None, args.port.len())?; + + let packed_mounts = mounts_to_packed(&mounts); + + smolvm::process::install_sigchld_handler(); + + let console_log_path = runtime_dir.path().join("console.log"); + let vsock_path_clone = vsock_path.clone(); + #[cfg(unix)] + let child_pid = smolvm::process::fork_session_leader(move || { + let krun = match unsafe { KrunFunctions::load(&lib_dir) } { + Ok(k) => k, + Err(e) => { + eprintln!("failed to load libkrun: {}", e); + smolvm::process::exit_child(1); + } + }; + + let config = PackedLaunchConfig { + rootfs_path: &rootfs_path, + storage_path: &storage_path, + vsock_socket: &vsock_path_clone, + layers_dir, + mounts: &packed_mounts, + port_mappings: &port_mappings, + resources, + overlay_path: overlay_runtime_path.as_deref(), + debug, + console_log: console_log_path, + }; + + // Detach from parent's terminal so libkrun doesn't + // steal keystrokes or corrupt terminal state. + smolvm::process::detach_stdio(); + + if let Err(e) = launch_agent_vm_dynamic(&krun, &config) { + let msg = format!("launch_agent_vm_dynamic failed: {}\n", e); + let _ = std::fs::write(&config.console_log, &msg); + } + smolvm::process::exit_child(1); + }) + .map_err(|e| Error::agent("fork VM process", e.to_string()))?; + + // Windows has no fork(): re-spawn this executable as `_boot-vm `, + // mirroring the non-packed launcher and the `--sidecar` path. BootConfig + // carries the same packed launch data (rootfs/disks/layers/ports). + #[cfg(not(unix))] + let child_pid = { + let _ = (&packed_mounts, &port_mappings, &vsock_path_clone); + let overlay_disk_path = overlay_runtime_path + .clone() + .unwrap_or_else(|| runtime_dir.path().join("overlay.raw")); + let boot_config = smolvm::agent::boot_config::BootConfig { + rootfs_path: rootfs_path.clone(), + storage_disk_path: storage_path.clone(), + overlay_disk_path, + vsock_socket: vsock_path.clone(), + console_log: Some(console_log_path.clone()), + startup_error_log: runtime_dir.path().join("startup-error.log"), + storage_size_gb: storage_gib.unwrap_or(smolvm::storage::DEFAULT_STORAGE_SIZE_GIB), + overlay_size_gb: args + .overlay + .unwrap_or(smolvm::storage::DEFAULT_OVERLAY_SIZE_GIB), + mounts: mounts.clone(), + ports: args.port.clone(), + resources: resources.clone(), + ssh_agent_socket: None, + cuda: false, + dns_filter_hosts: None, + packed_layers_dir: Some(layers_dir.to_path_buf()), + pack_idmap_source: None, + extra_disks: vec![], + }; + let config_path = runtime_dir.path().join("boot-config.json"); + let config_json = serde_json::to_vec(&boot_config) + .map_err(|e| Error::agent("serialize boot config", e.to_string()))?; + std::fs::write(&config_path, &config_json) + .map_err(|e| Error::agent("write boot config", e.to_string()))?; + let exe = std::env::current_exe() + .map_err(|e| Error::agent("find smolvm binary", e.to_string()))?; + let mut cmd = std::process::Command::new(&exe); + cmd.args(["_boot-vm", &config_path.to_string_lossy()]) + .env("SMOLVM_LIB_DIR", &lib_dir) + .env("SMOLVM_BOOT_WATCH_PARENT", "0") + .env( + smolvm_protocol::guest_env::READY_MARKER, + smolvm_protocol::AGENT_READY_MARKER, + ); + if debug { + cmd.stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::inherit()) + .stderr(std::process::Stdio::inherit()); + } else { + cmd.stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + } + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + const DETACHED_PROCESS: u32 = 0x0000_0008; + const CREATE_NEW_PROCESS_GROUP: u32 = 0x0000_0200; + cmd.creation_flags(DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP); + } + let child = cmd + .spawn() + .map_err(|e| Error::agent("spawn boot subprocess", e.to_string()))?; + child.id() as smolvm::process::Pid + }; + + let child_start_time = { + let mut st = smolvm::process::process_start_time(child_pid); + if st.is_none() && smolvm::process::is_alive(child_pid) { + for _ in 0..5 { + std::thread::sleep(Duration::from_millis(1)); + st = smolvm::process::process_start_time(child_pid); + if st.is_some() { + break; + } + } + } + if st.is_none() && smolvm::process::is_alive(child_pid) { + let _ = smolvm::process::stop_process_fast(child_pid, Duration::from_secs(5), true); + if let Err(e) = std::fs::remove_dir_all(runtime_dir.path()) { + tracing::debug!(error = %e, "cleanup: remove runtime dir after failed daemon child start"); + } + return Err(Error::agent( + "verify child process", + "unable to capture child start time for safe lifecycle management", + )); + } + st + }; + + let child_guard = ChildGuard { + pid: child_pid, + start_time: child_start_time, + runtime_dir, + }; + + let mut client = wait_for_agent(&vsock_path, debug)?; + + let params = ExecParams { + command: build_command(manifest, &args.command), + env: build_env(manifest, &args.env)?, + workdir: args.workdir.or_else(|| manifest.workdir.clone()), + interactive: args.interactive, + tty: args.tty, + timeout: args.timeout, + }; + let exit_code = execute_packed_command(&mut client, manifest, params, &mounts, None)?; + + drop(child_guard); + drop(layers_lease); + std::process::exit(exit_code); +} + +fn print_manifest_info(manifest: &smolvm_pack::PackManifest, checksum: u32) { + let mode_str = match manifest.mode { + PackMode::Container => "container", + PackMode::Vm => "vm", + }; + println!("Mode: {}", mode_str); + println!("Image: {}", manifest.image); + println!("Digest: {}", manifest.digest); + println!("Platform: {}", manifest.platform); + println!("CPUs: {}", manifest.cpus); + println!("Memory: {} MiB", manifest.mem); + if manifest.network { + println!("Network: enabled"); + } + if !manifest.entrypoint.is_empty() { + println!("Entrypoint: {}", manifest.entrypoint.join(" ")); + } + if !manifest.cmd.is_empty() { + println!("Cmd: {}", manifest.cmd.join(" ")); + } + if let Some(ref wd) = manifest.workdir { + println!("Workdir: {}", wd); + } + if !manifest.env.is_empty() { + println!("Env:"); + for e in &manifest.env { + println!(" {}", e); + } + } + println!("Checksum: {:08x}", checksum); +} + +// =========================================================================== +// Daemon mode helpers and implementation +// =========================================================================== + +/// Extract the checksum from any PackedMode variant. +fn mode_checksum(mode: &PackedMode) -> u32 { + match mode { + #[cfg(target_os = "macos")] + PackedMode::Section { checksum, .. } => *checksum, + PackedMode::Embedded { footer, .. } => footer.checksum, + PackedMode::Sidecar { footer, .. } => footer.checksum, + } +} + +/// Get the daemon state directory for a given checksum. +/// +/// Returns `~/.cache/smolvm-pack/{checksum:08x}/daemon/`. +fn daemon_dir(checksum: u32) -> smolvm::Result { + let cache_dir = extract::get_cache_dir(checksum) + .map_err(|e| Error::agent("get cache dir", e.to_string()))?; + Ok(cache_dir.join("daemon")) +} + +/// Read PID and start time from the daemon PID file. +/// +/// The PID file format is: `{pid}\n{start_time}`. +/// Returns `None` if the file doesn't exist or is malformed. +fn read_daemon_pid(checksum: u32) -> Option<(smolvm::process::Pid, Option)> { + let dir = daemon_dir(checksum).ok()?; + let pid_path = dir.join("agent.pid"); + let contents = std::fs::read_to_string(&pid_path).ok()?; + let mut lines = contents.lines(); + let pid: smolvm::process::Pid = lines.next()?.parse().ok()?; + let start_time: Option = lines.next().and_then(|s| s.parse().ok()); + Some((pid, start_time)) +} + +/// Write PID and start time to the daemon PID file. +fn write_daemon_pid( + checksum: u32, + pid: smolvm::process::Pid, + start_time: Option, +) -> smolvm::Result<()> { + let dir = daemon_dir(checksum)?; + let pid_path = dir.join("agent.pid"); + let contents = match start_time { + Some(st) => format!("{}\n{}", pid, st), + None => format!("{}", pid), + }; + std::fs::write(&pid_path, contents).map_err(|e| Error::agent("write PID file", e.to_string())) +} + +/// Read the manifest for any PackedMode variant. +fn read_manifest_for_mode(mode: &PackedMode) -> smolvm::Result { + match mode { + #[cfg(target_os = "macos")] + PackedMode::Section { manifest, .. } => Ok((**manifest).clone()), + PackedMode::Embedded { exe_path, .. } => smolvm_pack::read_manifest(exe_path) + .map_err(|e| Error::agent("read manifest", e.to_string())), + PackedMode::Sidecar { sidecar_path, .. } => { + smolvm_pack::read_manifest_from_sidecar(sidecar_path) + .map_err(|e| Error::agent("read manifest", e.to_string())) + } + } +} + +/// Ensure assets are extracted to the cache directory for the given mode. +fn ensure_extracted(mode: &PackedMode, force: bool, debug: bool) -> smolvm::Result { + let checksum = mode_checksum(mode); + let cache_dir = extract::get_cache_dir(checksum) + .map_err(|e| Error::agent("get cache dir", e.to_string()))?; + + let needs_extract = force || !extract::is_extracted(&cache_dir); + if needs_extract { + match mode { + #[cfg(target_os = "macos")] + PackedMode::Section { + assets_ptr, + assets_size, + .. + } => unsafe { + extract::extract_from_section(&cache_dir, *assets_ptr, *assets_size, debug) + .map_err(|e| Error::agent("extract section assets", e.to_string()))?; + }, + PackedMode::Embedded { + exe_path, footer, .. + } => { + extract::extract_from_binary(exe_path, &cache_dir, footer, debug) + .map_err(|e| Error::agent("extract embedded assets", e.to_string()))?; + } + PackedMode::Sidecar { + sidecar_path, + footer, + .. + } => { + extract::extract_sidecar(sidecar_path, &cache_dir, footer, force, debug) + .map_err(|e| Error::agent("extract sidecar assets", e.to_string()))?; + } + } + } + + Ok(cache_dir) +} + +/// Check if the daemon is currently running and connectable. +fn is_daemon_running(checksum: u32) -> bool { + let Some((pid, start_time)) = read_daemon_pid(checksum) else { + return false; + }; + + // Check PID identity (guards against PID reuse) + if !smolvm::process::is_our_process_strict(pid, start_time) { + return false; + } + + // Try to actually connect and ping + let dir = match daemon_dir(checksum) { + Ok(d) => d, + Err(_) => return false, + }; + let sock_path = dir.join("agent.sock"); + if !sock_path.exists() { + return false; + } + + AgentClient::connect(&sock_path) + .and_then(|mut c| c.ping()) + .is_ok() +} + +/// Start the daemon VM. +/// +/// Extracts assets if needed, creates the daemon directory, forks a child +/// process that runs the VM, writes a PID file, and waits for the agent +/// to become ready. +fn daemon_start( + mode: &PackedMode, + args: PackedStartArgs, + debug: bool, + force_extract: bool, +) -> smolvm::Result<()> { + let checksum = mode_checksum(mode); + let manifest = read_manifest_for_mode(mode)?; + + // Extract assets to cache + let cache_dir = ensure_extracted(mode, force_extract, debug)?; + + // Create daemon directory + let daemon = cache_dir.join("daemon"); + std::fs::create_dir_all(&daemon) + .map_err(|e| Error::agent("create daemon dir", e.to_string()))?; + + // Check if already running + if is_daemon_running(checksum) { + let (pid, _) = read_daemon_pid(checksum).unwrap(); + println!("Daemon already running (PID: {})", pid); + return Ok(()); + } + + // Clean up stale PID/socket files from previous runs + if let Err(e) = std::fs::remove_file(daemon.join("agent.pid")) { + tracing::debug!(error = %e, "cleanup: remove stale daemon PID file"); + } + if let Err(e) = std::fs::remove_file(daemon.join("agent.sock")) { + tracing::debug!(error = %e, "cleanup: remove stale daemon socket"); + } + + // Create storage disk if not exists (preserves existing disk on restart) + let storage_gib = storage_gib_for_manifest(args.storage, &manifest); + let storage_path = daemon.join("storage.ext4"); + if !storage_path.exists() { + let template = manifest + .assets + .storage_template + .as_ref() + .map(|t| t.path.as_str()); + extract::create_or_copy_storage_disk(&cache_dir, template, &storage_path, storage_gib) + .map_err(|e| Error::agent("create storage disk", e.to_string()))?; + } + + // Create overlay disk (preserves existing disk on restart) + let overlay_daemon_path = { + let overlay_path = daemon.join("overlay.raw"); + if !overlay_path.exists() { + setup_vm_overlay(&manifest, &cache_dir, &overlay_path, args.overlay)?; + } + Some(overlay_path) + }; + + let vsock_path = daemon.join("agent.sock"); + + // Parse CLI args + let mounts = HostMount::parse(&args.volume)?; + let port_mappings = PortMapping::to_tuples(&args.port); + let resources = VmResources { + cpus: args.cpus.unwrap_or(manifest.cpus), + memory_mib: args.mem.unwrap_or(manifest.mem), + network: args.net || manifest.network || !args.port.is_empty(), + network_backend: args.net_backend, + dns: None, + gpu: manifest.gpu, + storage_gib, + overlay_gib: args.overlay, + gpu_vram_mib: None, + rosetta: false, + allowed_cidrs: None, + }; + validate_requested_network_backend(&resources, None, args.port.len())?; + + let packed_mounts = mounts_to_packed(&mounts); + + let rootfs_path = cache_dir.join("agent-rootfs"); + let lib_dir = resolve_lib_dir(&cache_dir, debug)?; + // Use a temporary RAII lease to mount the volume for the launch config. + // The persistent daemon lease is created after fork with the real child + // PID. If fork fails, the RAII lease cleans up normally. If fork + // succeeds, the daemon lease keeps the volume mounted after the RAII + // lease drops. + let layers_lease = extract::acquire_layers_lease(&cache_dir, debug) + .map_err(|e| Error::agent("acquire layers lease", e.to_string()))?; + let layers_dir = layers_lease.path.clone(); + + if debug { + eprintln!("debug: daemon dir={}", daemon.display()); + eprintln!("debug: rootfs={}", rootfs_path.display()); + eprintln!("debug: lib_dir={}", lib_dir.display()); + eprintln!("debug: storage={}", storage_path.display()); + eprintln!("debug: vsock={}", vsock_path.display()); + eprintln!( + "debug: resources cpus={} mem={} net={}", + resources.cpus, resources.memory_mib, resources.network + ); + } + + // Fork child → launch VM + smolvm::process::install_sigchld_handler(); + + let console_log_path = daemon.join("console.log"); + let vsock_path_clone = vsock_path.clone(); + let child_pid = smolvm::process::fork_session_leader(move || { + let krun = match unsafe { KrunFunctions::load(&lib_dir) } { + Ok(k) => k, + Err(e) => { + eprintln!("failed to load libkrun: {}", e); + smolvm::process::exit_child(1); + } + }; + + let config = PackedLaunchConfig { + rootfs_path: &rootfs_path, + storage_path: &storage_path, + vsock_socket: &vsock_path_clone, + layers_dir: &layers_dir, + mounts: &packed_mounts, + port_mappings: &port_mappings, + resources, + overlay_path: overlay_daemon_path.as_deref(), + debug, + console_log: console_log_path, + }; + + // Detach from parent's terminal before launching the VM. + // Without this, libkrun's threads inherit stdin and steal + // keystrokes from the user's shell. + smolvm::process::detach_stdio(); + + if let Err(e) = launch_agent_vm_dynamic(&krun, &config) { + let msg = format!("launch_agent_vm_dynamic failed: {}\n", e); + let _ = std::fs::write(&config.console_log, &msg); + } + + smolvm::process::exit_child(1); + }) + .map_err(|e| Error::agent("fork VM process", e.to_string()))?; + + // Capture child start time for PID identity verification + let child_start_time = { + let mut st = smolvm::process::process_start_time(child_pid); + if st.is_none() && smolvm::process::is_alive(child_pid) { + for _ in 0..5 { + std::thread::sleep(Duration::from_millis(1)); + st = smolvm::process::process_start_time(child_pid); + if st.is_some() { + break; + } + } + } + if st.is_none() && smolvm::process::is_alive(child_pid) { + let _ = smolvm::process::stop_process_fast(child_pid, Duration::from_secs(5), true); + return Err(Error::agent( + "verify child process", + "unable to capture child start time for safe lifecycle management", + )); + } + st + }; + + // Write PID file + write_daemon_pid(checksum, child_pid, child_start_time)?; + + // Create the persistent daemon lease with the real child PID. + // This must happen before the RAII layers_lease drops — otherwise the + // volume would be detached between Drop and this call. + extract::acquire_daemon_lease(&cache_dir, child_pid, debug) + .map_err(|e| Error::agent("acquire daemon lease", e.to_string()))?; + + if debug { + eprintln!("debug: forked VM process with PID {}", child_pid); + } + + // Wait for agent to become ready + println!("Starting daemon..."); + let _client = wait_for_agent(&vsock_path, debug)?; + + println!("Daemon started (PID: {})", child_pid); + Ok(()) +} + +/// Execute a command in the running daemon VM. +fn daemon_exec( + checksum: u32, + args: PackedExecArgs, + debug: bool, + manifest: &smolvm_pack::PackManifest, +) -> smolvm::Result<()> { + let dir = daemon_dir(checksum)?; + let sock_path = dir.join("agent.sock"); + + // Check daemon is running + if !is_daemon_running(checksum) { + return Err(Error::agent( + "daemon exec", + "daemon is not running. Start it with: start", + )); + } + + if debug { + eprintln!("debug: connecting to daemon at {}", sock_path.display()); + } + + // Connect to agent + let mut client = AgentClient::connect(&sock_path)?; + + // Build command from args or manifest defaults + // Virtiofs devices are fixed at boot — exec cannot add new host mounts. + let mounts: Vec = Vec::new(); + let params = ExecParams { + command: build_command(manifest, &args.command), + env: build_env(manifest, &args.env)?, + workdir: args.workdir.or_else(|| manifest.workdir.clone()), + interactive: args.interactive, + tty: args.tty, + timeout: args.timeout, + }; + + // Daemon exec uses a persistent overlay so filesystem changes (package + // installs, config writes) survive across exec calls, matching the + // behavior of `machine exec` on persistent machines. + let overlay_id = Some("daemon".to_string()); + let exit_code = execute_packed_command(&mut client, manifest, params, &mounts, overlay_id)?; + + std::process::exit(exit_code); +} + +/// Stop the daemon VM. +fn daemon_stop(checksum: u32, debug: bool) -> smolvm::Result<()> { + let Some((pid, start_time)) = read_daemon_pid(checksum) else { + println!("Daemon not running"); + return Ok(()); + }; + + let dir = daemon_dir(checksum)?; + let sock_path = dir.join("agent.sock"); + + // Try graceful shutdown via agent protocol. + // If the agent responds, this also confirms the PID belongs to our VM. + let mut vsock_confirmed = false; + if sock_path.exists() { + if let Ok(mut client) = AgentClient::connect(&sock_path) { + vsock_confirmed = true; + if debug { + eprintln!("debug: sending shutdown to agent"); + } + let _ = client.shutdown(); + } + } + + // Identity check: vsock acknowledgement OR strict PID start-time match. + // We intentionally do NOT use the lenient is_our_process() here because + // it treats any alive PID as "ours" when start_time is None — which risks + // killing an unrelated process if the OS reused the PID. + let identity_ok = vsock_confirmed || smolvm::process::is_our_process_strict(pid, start_time); + if identity_ok { + if debug { + eprintln!( + "debug: stopping process {} (start_time: {:?})", + pid, start_time + ); + } + let _ = smolvm::process::stop_vm_process( + pid, + Duration::from_secs(5), + smolvm::process::VM_SIGKILL_TIMEOUT, + ); + } + + // Clean up PID and socket files (keep storage.ext4 for persistence) + if let Err(e) = std::fs::remove_file(dir.join("agent.pid")) { + tracing::debug!(error = %e, "cleanup: remove daemon PID file"); + } + if let Err(e) = std::fs::remove_file(dir.join("agent.sock")) { + tracing::debug!(error = %e, "cleanup: remove daemon socket"); + } + + // Release the persistent daemon lease. Detaches the case-sensitive + // volume if no other leases remain. + let cache_dir = extract::get_cache_dir(checksum) + .map_err(|e| Error::agent("get cache dir", e.to_string()))?; + extract::release_daemon_lease(&cache_dir); + + println!("Daemon stopped"); + Ok(()) +} + +/// Check daemon status. +fn daemon_status(checksum: u32) -> smolvm::Result<()> { + let Some((pid, start_time)) = read_daemon_pid(checksum) else { + println!("Status: not running"); + return Ok(()); + }; + + // Check if PID is still our process + if !smolvm::process::is_our_process_strict(pid, start_time) { + println!("Status: not running (stale PID file)"); + // Clean up stale files + if let Ok(dir) = daemon_dir(checksum) { + if let Err(e) = std::fs::remove_file(dir.join("agent.pid")) { + tracing::debug!(error = %e, "cleanup: remove stale status PID file"); + } + if let Err(e) = std::fs::remove_file(dir.join("agent.sock")) { + tracing::debug!(error = %e, "cleanup: remove stale status socket"); + } + } + return Ok(()); + } + + // Try to connect and ping + let dir = daemon_dir(checksum)?; + let sock_path = dir.join("agent.sock"); + + if sock_path.exists() { + if let Ok(mut client) = AgentClient::connect(&sock_path) { + if client.ping().is_ok() { + println!("Status: running (PID: {})", pid); + return Ok(()); + } + } + } + + println!("Status: running (PID: {}, agent not responding)", pid); + Ok(()) +} diff --git a/src/cli/parsers.rs b/src/cli/parsers.rs new file mode 100644 index 0000000..effbac7 --- /dev/null +++ b/src/cli/parsers.rs @@ -0,0 +1,294 @@ +//! Shared CLI argument parsers. +//! +//! This module consolidates parser functions used across multiple CLI commands +//! to eliminate code duplication and ensure consistent validation. + +use smolvm::data::storage::HostMount; +use std::time::Duration; + +/// Parse a duration string (e.g., "30s", "5m", "1h"). +pub fn parse_duration(s: &str) -> Result { + let d = humantime::parse_duration(s).map_err(|e| e.to_string())?; + if d.is_zero() { + return Err("timeout must be greater than 0 (e.g., 5sec, 30s)".to_string()); + } + Ok(d) +} + +/// Parse and validate `--gpu-vram `. Rejects 0 at CLI parse time +/// so the user gets a clear error instead of an opaque libkrun +/// allocation failure later. See +/// `smolvm::data::resources::validate_gpu_vram_mib`. +pub fn parse_gpu_vram_mib(s: &str) -> Result { + let v: u32 = s + .parse() + .map_err(|_| format!("'{}' is not a valid MiB value", s))?; + smolvm::data::resources::validate_gpu_vram_mib(Some(v)).map_err(|e| e.to_string())?; + Ok(v) +} + +/// Parse a byte size like `16GiB`, `16G`, `512M`, or a plain byte count +/// (`17179869184`). Suffixes are binary (1K = 1024); `K/M/G/T`, optionally with +/// a trailing `i` and/or `B`, are accepted case-insensitively. Used for +/// `--max-image-size`. +pub fn parse_size_bytes(s: &str) -> Result { + let t = s.trim(); + if t.is_empty() { + return Err("size must not be empty".to_string()); + } + let split = t + .find(|c: char| !c.is_ascii_digit() && c != '.') + .unwrap_or(t.len()); + let (num, unit) = t.split_at(split); + let value: f64 = num + .parse() + .map_err(|_| format!("'{s}' is not a valid size (expected a number, optional K/M/G/T)"))?; + let mult: u64 = match unit.trim().to_ascii_lowercase().as_str() { + "" | "b" => 1, + "k" | "ki" | "kib" | "kb" => 1 << 10, + "m" | "mi" | "mib" | "mb" => 1 << 20, + "g" | "gi" | "gib" | "gb" => 1 << 30, + "t" | "ti" | "tib" | "tb" => 1u64 << 40, + other => { + return Err(format!( + "invalid size unit '{other}' in '{s}' (use K, M, G, or T)" + )) + } + }; + let bytes = (value * mult as f64) as u64; + if bytes == 0 { + return Err(format!("size '{s}' must be greater than zero")); + } + Ok(bytes) +} + +/// Parse and validate an image reference. Rejects empty/whitespace-only +/// values at CLI parse time so `--image ""` errors immediately instead of +/// creating an unusable machine that only fails once the VM starts. +pub fn parse_image(s: &str) -> Result { + if s.trim().is_empty() { + return Err("image reference must not be empty".to_string()); + } + Ok(s.to_string()) +} + +// Env parsing delegated to the library. +pub use smolvm::util::{parse_env_list, parse_env_spec}; + +/// Assign positional virtiofs tags to a list of (guest_target, read_only) +/// pairs and return the agent's `(tag, target, read_only)` binding form. +/// +/// The tag format is `smolvm{index}` and matches the virtiofs device +/// libkrun exposed at VM start in the same order. Order in equals order +/// out — caller must preserve the original ordering of mounts between +/// VM start and any subsequent agent request that references the tags. +/// +/// Generic over any iterator of `(String, bool)` so both +/// [`HostMount`]-shaped inputs and `VmRecord`-shaped tuples (host, +/// target, ro) can adapt at the boundary without a parallel helper. Two +/// thin wrappers below ([`mounts_to_virtiofs_bindings`] and +/// [`record_mounts_to_runconfig_bindings`]) preserve the caller-side +/// ergonomics. +fn assign_virtiofs_tags(items: I) -> Vec<(String, String, bool)> +where + I: IntoIterator, +{ + items + .into_iter() + .enumerate() + .map(|(i, (target, ro))| (HostMount::mount_tag(i), target, ro)) + .collect() +} + +/// Convert parsed [`HostMount`] list to virtiofs binding format for agent. +/// +/// Used by `machine run` paths that already hold the validated, parsed +/// mount type. See [`assign_virtiofs_tags`] for the tag rule. +pub fn mounts_to_virtiofs_bindings(mounts: &[HostMount]) -> Vec<(String, String, bool)> { + assign_virtiofs_tags( + mounts + .iter() + .map(|m| (m.target.to_string_lossy().into_owned(), m.read_only)), + ) +} + +/// Convert a `VmRecord`-style mount list to virtiofs binding format. +/// +/// `VmRecord` stores mounts as `(host_source, guest_target, read_only)` +/// already-validated triples (see `src/data/storage.rs::HostMount::to_storage_tuple`). +/// The host source is dropped — the agent only needs the guest-facing +/// target and the tag. See [`assign_virtiofs_tags`] for the tag rule. +pub fn record_mounts_to_runconfig_bindings( + mounts: &[(String, String, bool)], +) -> Vec<(String, String, bool)> { + assign_virtiofs_tags( + mounts + .iter() + .map(|(_host, target, ro)| (target.clone(), *ro)), + ) +} + +// Network helpers delegated to the library. +pub use smolvm::smolfile::{parse_cidr, resolve_host_to_cidrs}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_gpu_vram_mib_rejects_zero() { + let err = parse_gpu_vram_mib("0").unwrap_err(); + assert!( + err.contains("positive") || err.contains("> 0") || err.contains("omit"), + "expected actionable message, got: {}", + err + ); + } + + #[test] + fn parse_gpu_vram_mib_rejects_nonnumeric() { + assert!(parse_gpu_vram_mib("abc").is_err()); + assert!(parse_gpu_vram_mib("").is_err()); + assert!(parse_gpu_vram_mib("-1").is_err()); + assert!(parse_gpu_vram_mib("2.5").is_err()); + } + + #[test] + fn parse_gpu_vram_mib_accepts_positive() { + assert_eq!(parse_gpu_vram_mib("1").unwrap(), 1); + assert_eq!(parse_gpu_vram_mib("4096").unwrap(), 4096); + } + + #[test] + fn resolve_host_bare_ip() { + let cidrs = resolve_host_to_cidrs("1.2.3.4").unwrap(); + assert_eq!(cidrs, vec!["1.2.3.4/32"]); + } + + #[test] + fn resolve_host_rejects_port_suffix() { + let err = resolve_host_to_cidrs("example.com:443").unwrap_err(); + assert!(err.contains("port suffixes are not supported"), "{}", err); + } + + #[test] + fn resolve_host_rejects_ipv6_with_port() { + let err = resolve_host_to_cidrs("[::1]:80").unwrap_err(); + assert!(err.contains("port suffixes are not supported"), "{}", err); + } + + #[test] + fn resolve_host_real_hostname() { + // Resolve a well-known hostname — should return at least one IP + let cidrs = resolve_host_to_cidrs("one.one.one.one").unwrap(); + assert!(!cidrs.is_empty()); + // IPv4 results should be /32 CIDRs; IPv6 results should be /128 CIDRs. + for cidr in &cidrs { + assert!( + cidr.ends_with("/32") || cidr.ends_with("/128"), + "expected host CIDR, got {}", + cidr + ); + } + } + + #[test] + fn resolve_host_nonexistent_domain() { + let err = + resolve_host_to_cidrs("this-domain-does-not-exist-smolvm-test.invalid").unwrap_err(); + assert!(err.contains("failed to resolve"), "{}", err); + } + + #[test] + fn parse_cidr_valid() { + assert_eq!(parse_cidr("10.0.0.0/8").unwrap(), "10.0.0.0/8"); + assert_eq!(parse_cidr("1.1.1.1").unwrap(), "1.1.1.1/32"); + } + + #[test] + fn parse_cidr_invalid() { + assert!(parse_cidr("not-a-cidr").is_err()); + } + + #[test] + fn record_mounts_to_runconfig_bindings_assigns_positional_tags() { + // Tags are positional so they line up with the virtiofs devices + // libkrun exposed at VM start. Two mounts → "smolvm0", "smolvm1", + // preserving the read-only flag and the guest target verbatim. + let mounts = vec![ + ("/host/src".to_string(), "/app".to_string(), false), + ("/host/data".to_string(), "/data".to_string(), true), + ]; + let bindings = record_mounts_to_runconfig_bindings(&mounts); + assert_eq!( + bindings, + vec![ + ("smolvm0".to_string(), "/app".to_string(), false), + ("smolvm1".to_string(), "/data".to_string(), true), + ] + ); + } + + #[test] + fn record_mounts_to_runconfig_bindings_empty_input() { + // No mounts → no bindings. Init code calls this unconditionally; + // empty must round-trip cleanly without panicking on enumerate. + assert!(record_mounts_to_runconfig_bindings(&[]).is_empty()); + } + + #[test] + fn assign_virtiofs_tags_keeps_order_and_assigns_zero_based_index() { + // The shared core. The two public wrappers are thin adapters + // around this — pin the indexing rule here so neither wrapper + // can drift away from the canonical "smolvm{i}" naming or from + // preserving caller order. + let out = assign_virtiofs_tags(vec![ + ("/a".to_string(), false), + ("/b".to_string(), true), + ("/c".to_string(), false), + ]); + assert_eq!( + out, + vec![ + ("smolvm0".to_string(), "/a".to_string(), false), + ("smolvm1".to_string(), "/b".to_string(), true), + ("smolvm2".to_string(), "/c".to_string(), false), + ] + ); + } + + #[test] + fn mounts_to_virtiofs_bindings_matches_record_form_for_same_inputs() { + // The two public wrappers must agree on identical inputs — they + // route to the same core. If a future refactor adds a wrapper + // that *doesn't* go through `assign_virtiofs_tags`, this test + // will catch the divergence. + use std::path::PathBuf; + let host_mount = HostMount { + source: PathBuf::from("/tmp"), // any existing dir; not validated by the converter + target: PathBuf::from("/app"), + read_only: true, + }; + let from_parsed = mounts_to_virtiofs_bindings(&[host_mount]); + let from_record = + record_mounts_to_runconfig_bindings(&[("/tmp".to_string(), "/app".to_string(), true)]); + assert_eq!(from_parsed, from_record); + } + + #[test] + fn parse_size_bytes_units_and_errors() { + assert_eq!(parse_size_bytes("1024"), Ok(1024)); + assert_eq!(parse_size_bytes("16GiB"), Ok(16 * 1024 * 1024 * 1024)); + assert_eq!(parse_size_bytes("16G"), Ok(16 * 1024 * 1024 * 1024)); + assert_eq!(parse_size_bytes("512M"), Ok(512 * 1024 * 1024)); + assert_eq!(parse_size_bytes("2k"), Ok(2048)); + assert_eq!( + parse_size_bytes("1.5G"), + Ok(1024 * 1024 * 1024 + 512 * 1024 * 1024) + ); + assert!(parse_size_bytes("").is_err()); + assert!(parse_size_bytes("0").is_err()); + assert!(parse_size_bytes("abc").is_err()); + assert!(parse_size_bytes("10X").is_err()); + } +} diff --git a/src/cli/proxy_opts.rs b/src/cli/proxy_opts.rs new file mode 100644 index 0000000..cf6f438 --- /dev/null +++ b/src/cli/proxy_opts.rs @@ -0,0 +1,30 @@ +//! Shared `--proxy` / `--no-proxy` flags for subcommands that pull images. +//! +//! Flatten this struct into a subcommand's `Args` derive to expose the flags +//! consistently. The values flow into `AgentRequest::Pull` and are set on +//! the `crane` subprocess as `HTTP_PROXY` / `HTTPS_PROXY` / `NO_PROXY`. + +use clap::Args; + +#[derive(Args, Debug, Clone, Default)] +pub struct ProxyOpts { + /// Proxy URL used for the in-VM image pull (sets HTTP_PROXY and HTTPS_PROXY + /// on the registry client). Example: `http://192.168.127.254:3128`. + #[arg(long, value_name = "URL", global = false)] + pub proxy: Option, + + /// Comma-separated NO_PROXY list of hosts/CIDRs that bypass the proxy + /// during image pull. Example: `127.0.0.1,localhost,.internal`. + #[arg(long, value_name = "LIST", global = false)] + pub no_proxy: Option, +} + +impl ProxyOpts { + pub fn proxy(&self) -> Option<&str> { + self.proxy.as_deref() + } + + pub fn no_proxy(&self) -> Option<&str> { + self.no_proxy.as_deref() + } +} diff --git a/src/cli/serve.rs b/src/cli/serve.rs new file mode 100644 index 0000000..e15baf8 --- /dev/null +++ b/src/cli/serve.rs @@ -0,0 +1,643 @@ +//! HTTP API server command. + +use axum::Router; +use clap::Parser; +use std::net::SocketAddr; +#[cfg(unix)] +use std::path::PathBuf; +use std::sync::Arc; + +use smolvm::api::state::ApiState; +use smolvm::Result; + +use super::openapi::OpenapiCmd; + +/// Start the HTTP API server for programmatic control. +#[derive(Parser, Debug)] +#[command(about = "Start the HTTP API server for programmatic machine management")] +pub enum ServeCmd { + /// Start the HTTP API server + #[command(after_long_help = "\ +Machines persist independently of the server - they continue running even if the server stops. + +API ENDPOINTS: + GET /health Health check + POST /api/v1/machines Create machine + GET /api/v1/machines List machines + GET /api/v1/machines/:id Get machine status + POST /api/v1/machines/:id/start Start machine + POST /api/v1/machines/:id/stop Stop machine + POST /api/v1/machines/:id/exec Execute command + DELETE /api/v1/machines/:id Delete machine + +EXAMPLES: + smolvm serve start Listen on the default Unix socket (unix:///$XDG_RUNTIME_DIR/smolvm.sock) + smolvm serve start -l 0.0.0.0:9000 Listen on all interfaces, port 9000 + smolvm serve start -l unix:///tmp/smol.sock Listen on a Unix domain socket + smolvm serve start -v Enable verbose logging")] + Start(ServeStartCmd), + + /// Export OpenAPI specification for SDK generation + Openapi(OpenapiCmd), +} + +impl ServeCmd { + pub fn run(self) -> Result<()> { + match self { + ServeCmd::Start(cmd) => cmd.run(), + ServeCmd::Openapi(cmd) => cmd.run(), + } + } +} + +#[derive(Parser, Debug)] +pub struct ServeStartCmd { + /// Address and port or Unix socket path to listen on + #[arg( + short, + long, + default_value_t = default_listen_value(), + value_name = "ADDR:PORT|PATH" + )] + listen: String, + + /// Enable debug logging (or set RUST_LOG=debug) + #[arg(short, long)] + verbose: bool, + + /// CORS allowed origins (repeatable). Defaults to localhost:8080 and localhost:3000. + #[arg(long = "cors-origin", value_name = "ORIGIN")] + cors_origins: Vec, + + /// Output logs as structured JSON (for log aggregators) + #[arg(long)] + json_logs: bool, + + /// Seccomp syscall-allowlist mode for VM boot subprocesses (untrusted-guest + /// hardening): `enforce` kills the VMM on a disallowed syscall, `audit` logs + /// only, `off` disables. x86_64-Linux only; ignored elsewhere. A pre-set + /// SMOLVM_SECCOMP env var takes precedence. + #[arg(long, value_name = "MODE", default_value = "enforce")] + seccomp: String, + + /// Landlock filesystem-confinement mode for VM boot subprocesses: `enforce` + /// restricts each VMM to its own rootfs/disks/devices (denying the rest of + /// the host fs), `off` disables. Linux-only; ignored elsewhere. A pre-set + /// SMOLVM_LANDLOCK env var takes precedence. + #[arg(long, value_name = "MODE", default_value = "enforce")] + landlock: String, +} + +impl ServeStartCmd { + /// Run the serve command. + pub fn run(self) -> Result<()> { + // Set JSON log format for the logging initializer to pick up + if self.json_logs { + std::env::set_var("SMOLVM_LOG_FORMAT", "json"); + } + + // Data root. Per-VM uid isolation needs every smolvm path traversable by + // the dropped uids; XDG-under-a-700-home isn't, a system data root is. + // serve additionally auto-defaults to /var/lib/smolvm when privileged + // (allow_auto = true). An explicit SMOLVM_DATA_DIR was already applied for + // every command in main(); calling again is idempotent. Single-threaded + // before the tokio runtime, so set_var is safe. + smolvm::process::apply_system_data_root(/* allow_auto */ true); + + // Lock the state dirs holding machine records / credentials / config down + // to 0700 so a Landlock-exempt fork clone (which runs as its golden's uid) + // can't read other tenants' data through the now world-traversable data + // root. These sit OUTSIDE the traversable VM-data/rootfs chains, so this + // doesn't affect VM boots. + #[cfg(target_os = "linux")] + if smolvm::process::vm_uid_drop_active() { + use std::os::unix::fs::PermissionsExt; + let mut sensitive: Vec = Vec::new(); + if let Some(d) = dirs::data_local_dir().or_else(dirs::data_dir) { + sensitive.push(d.join("smolvm").join("server")); + sensitive.push(d.join("smolvm").join("node-credentials")); + } + if let Some(h) = dirs::home_dir() { + sensitive.push(h.join(".config").join("smolvm")); + } + for dir in sensitive { + let _ = std::fs::create_dir_all(&dir); + let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)); + } + } + + let listen_target = ListenTarget::parse(&self.listen)?; + + // Set up verbose logging if requested + if self.verbose { + // Re-initialize logging at debug level + // Note: This won't work if logging is already initialized, + // but the RUST_LOG env var can be used instead + tracing::info!("verbose logging enabled"); + } + + // Per-VM resource isolation + lossless-restart placement. Two paths: + // + // - systemd host: adopt each VM into its OWN `smolvm-vm-.scope` after + // fork (a sibling unit owned by PID1), so a `serve` restart doesn't kill + // or orphan it — the VM isn't in the service cgroup, so systemd won't hit + // `219/CGROUP` recreating the unit. Caps become scope properties. We do + // NOT set SMOLVM_CGROUP_ROOT here so the VM boot subprocess skips + // self-placement; the parent adopts it instead. See + // docs/lossless-serve-restart.md. + // - non-systemd (dev/containers): fall back to a delegated cgroup root + // advertised via SMOLVM_CGROUP_ROOT so every VM boot subprocess places + // itself in a per-VM cgroup. No lossless restart there, which is fine. + // + // Done here — single-threaded, before the tokio runtime — so set_var is + // safe. See docs/runtime-isolation-hardening.md. + #[cfg(target_os = "linux")] + if smolvm::systemd_scope::is_available() { + std::env::set_var("SMOLVM_VM_USE_SCOPE", "1"); + tracing::info!("per-VM systemd transient scopes enabled (lossless serve restart)"); + } else if let Some(root) = smolvm::process::setup_cgroup_delegation_root() { + tracing::info!(cgroup_root = %root.display(), "per-VM cgroup resource caps enabled"); + std::env::set_var("SMOLVM_CGROUP_ROOT", &root); + } + + // Default-on: enable the seccomp syscall allowlist on every VM boot + // subprocess. `--seccomp` selects enforce|audit|off (default enforce); a + // pre-set SMOLVM_SECCOMP env wins for ad-hoc overrides. Inherited by the + // spawned `_boot-vm`. See docs/runtime-isolation-hardening.md. + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + if std::env::var_os("SMOLVM_SECCOMP").is_none() { + std::env::set_var("SMOLVM_SECCOMP", &self.seccomp); + if self.seccomp != "off" { + tracing::info!(mode = %self.seccomp, "VM seccomp syscall filtering enabled"); + } + } + + // Default-on: confine each VM boot subprocess's filesystem view via + // Landlock. `--landlock` selects enforce|off (default enforce); a pre-set + // SMOLVM_LANDLOCK env wins. Inherited by the spawned `_boot-vm`. + #[cfg(target_os = "linux")] + if std::env::var_os("SMOLVM_LANDLOCK").is_none() { + std::env::set_var("SMOLVM_LANDLOCK", &self.landlock); + if self.landlock != "off" { + tracing::info!(mode = %self.landlock, "VM filesystem confinement (Landlock) enabled"); + } + } + + // Default-on, fail-closed: a `serve` node hosts untrusted tenant guests, + // so force the STRICT egress floor (blocks cloud metadata, host LAN, the + // control plane, loopback, and co-resident tenants) rather than inferring + // it from `SMOLVM_PUBLISH_ADDR`. A dropped publish-addr must NOT silently + // downgrade the floor to metadata-only and expose the host loopback door + // to a guest. Single-tenant/self-host operators can opt down with + // `SMOLVM_EGRESS_FLOOR=metadata|off`. Inherited by the spawned `_boot-vm`. + if std::env::var_os("SMOLVM_EGRESS_FLOOR").is_none() { + std::env::set_var("SMOLVM_EGRESS_FLOOR", "strict"); + tracing::info!("egress floor set to strict (multi-tenant serve default)"); + } + + // Per-VM uid isolation preflight. When serve is privileged each VMM drops + // to its own unprivileged uid (process::vm_drop_ids), containing a + // guest→VMM escape to one VM. That only works if the data root is + // traversable (others-execute) by the drop uid — an XDG-under-a-700-home + // layout is not, and the VMM would die with a cryptic readiness timeout. + // Warn loudly with the fix instead. Opt out with SMOLVM_VM_UID_DROP=off. + #[cfg(target_os = "linux")] + if smolvm::process::vm_uid_drop_active() { + let cache_root = smolvm::agent::vm_cache_root(); + match smolvm::process::first_nontraversable_ancestor(&cache_root) { + Some(blocker) => tracing::warn!( + blocker = %blocker.display(), + "per-VM uid isolation is active but {b} is not traversable (o+x) by \ + unprivileged uids — VMMs will fail to start. Use a world-traversable data \ + root (e.g. run serve with HOME=/var/lib/smolvm) or `chmod o+x {b}`, or \ + disable with SMOLVM_VM_UID_DROP=off", + b = blocker.display(), + ), + None => tracing::info!( + "per-VM uid isolation active (each VMM drops to its own unprivileged uid)" + ), + } + } + + // Create the runtime with signal handling enabled + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .map_err(smolvm::error::Error::Io)?; + + runtime.block_on(async move { self.run_server(listen_target).await }) + } + + async fn run_server(self, listen_target: ListenTarget) -> Result<()> { + // On Windows `ListenTarget` has only the `Tcp` variant (Unix-socket + // listening is unix-gated), making this match irrefutable there. + #[cfg_attr(not(unix), allow(irrefutable_let_patterns))] + if let ListenTarget::Tcp(addr) = &listen_target { + if addr.ip().is_unspecified() { + eprintln!( + "WARNING: Server is listening on all interfaces ({}).", + addr.ip() + ); + eprintln!(" The API has no authentication - any network client can control this host."); + eprintln!(" Consider using the default Unix socket or --listen 127.0.0.1:8080 for local-only access."); + } + } + + // VM boot subprocesses are detached and would zombie on exit; they are + // reaped SELECTIVELY (per registered PID) by the supervisor tick via + // smolvm::process::reap_vm_children(). We deliberately do NOT install the + // global waitpid(-1) SIGCHLD handler here: serve's concurrent boots run + // busctl/mkfs `.output()` subprocesses that a global reaper would steal, + // causing ECHILD ("No child processes") and failed scope adoption. + + // Install Prometheus metrics recorder and mark start time + if let Some(handle) = smolvm::api::install_metrics_recorder() { + let _ = smolvm::api::METRICS_HANDLE.set(handle); + } + smolvm::api::handlers::health::mark_server_start(); + + // Create shared state and load persisted machines + let state = Arc::new(ApiState::new().map_err(|e| { + smolvm::error::Error::config("initialize api state", format!("{:?}", e)) + })?); + let loaded = state.load_persisted_machines(); + if !loaded.is_empty() { + println!( + "Reconnected to {} existing machine(es): {}", + loaded.len(), + loaded.join(", ") + ); + } + // GC VM data dirs no machine record references (legacy/orphan disk leaks). + // Server-only: this owns the node's cache and runs before serving requests. + let reclaimed = state.reclaim_dangling_vm_dirs(); + if reclaimed > 0 { + println!("Reclaimed {reclaimed} dangling VM data dir(es)"); + } + + // Create shutdown channel for supervisor + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + + // Spawn supervisor task + let supervisor_state = state.clone(); + let supervisor_shutdown = shutdown_rx.clone(); + let supervisor_handle = tokio::spawn(async move { + let supervisor = + smolvm::api::supervisor::Supervisor::new(supervisor_state, supervisor_shutdown); + supervisor.run().await; + }); + + // Create router + let drain_state = state.clone(); + let app = smolvm::api::create_router(state, self.cors_origins.clone()); + + // Resolve the serve API's TLS posture before binding. In fleet mode this + // is fail-closed: a missing/partial mTLS config aborts startup rather + // than silently serving plain HTTP (control↔node mTLS, increment 3). + let tls = super::serve_tls::resolve_tls().map_err(|e| smolvm::error::Error::Config { + operation: "serve tls".to_string(), + reason: e.to_string(), + })?; + + // Listen server on TCP or Unix socket + match listen_target { + ListenTarget::Tcp(addr) => self.serve_tcp(addr, app, tls).await?, + #[cfg(unix)] + ListenTarget::Unix(path) => self.serve_unix(path, app).await?, + } + + // The HTTP server has stopped accepting (graceful shutdown on SIGTERM). + // VMs survive a normal `serve` restart (reconnect on next start), so this + // is opt-in: on a host teardown (autoscaler scale-in) set + // SMOLVM_DRAIN_ON_SHUTDOWN to stop running VMs cleanly — flushing disk + // state — instead of letting the host hard-kill them. + let drain = std::env::var("SMOLVM_DRAIN_ON_SHUTDOWN") + .map(|v| v != "0" && !v.eq_ignore_ascii_case("false")) + .unwrap_or(false); + if drain { + smolvm::api::handlers::machines::drain_machines(&drain_state).await; + } else { + // Non-draining shutdown (a binary-upgrade restart): VMs must survive + // for the next `serve` process to reconnect to. Skipping drain isn't + // enough — `AgentManager::drop` stops any VM it owns, so tearing down + // `ApiState` would kill every running VM. Disarm each manager's Drop + // first, mirroring the CLI's detach-before-exit. + drain_state.detach_all(); + } + + // Signal all background tasks to stop + let _ = shutdown_tx.send(true); + + // Wait for supervisor to finish (with timeout) + match tokio::time::timeout(std::time::Duration::from_secs(5), supervisor_handle).await { + Ok(_) => tracing::debug!("supervisor shut down cleanly"), + Err(_) => tracing::warn!("supervisor did not shut down within 5 seconds"), + } + + Ok(()) + } + + async fn serve_tcp( + &self, + addr: SocketAddr, + app: Router, + tls: Option>, + ) -> Result<()> { + if let Some(tls_config) = tls { + return Self::serve_tcp_tls(addr, app, tls_config).await; + } + + let listener = tokio::net::TcpListener::bind(addr) + .await + .map_err(smolvm::error::Error::Io)?; + + tracing::info!(address = %addr, "starting HTTP API server"); + println!("smolvm API server listening on http://{}", addr); + + axum::serve(listener, app) + .with_graceful_shutdown(shutdown_signal()) + .await + .map_err(smolvm::error::Error::Io) + } + + /// HTTPS variant with mutual TLS (fleet mode). `axum-server`'s rustls + /// acceptor performs the handshake + client-cert verification configured in + /// `tls_config`; graceful shutdown is driven through its `Handle` (the + /// `axum::serve` graceful-shutdown future doesn't apply here). + /// + /// Because mTLS locks the whole network port to CA-signed clients, we ALSO + /// bind a plain-HTTP listener on loopback (see `serve_tls::local_plain_addr`) + /// so the node's own agent can keep polling `/capacity` locally — it is not + /// an mTLS client and is unreachable from the network anyway. + async fn serve_tcp_tls( + addr: SocketAddr, + app: Router, + tls_config: std::sync::Arc, + ) -> Result<()> { + // Loopback plain-HTTP door for the local node-agent. + if let Some(local_addr) = super::serve_tls::local_plain_addr(addr) { + if !local_addr.ip().is_loopback() { + return Err(smolvm::error::Error::config( + "serve local addr", + format!("SMOLVM_SERVE_LOCAL_ADDR {local_addr} must be loopback"), + )); + } + // Bind synchronously, then hand the std listener to a DEDICATED + // single-thread runtime on its own OS thread. The loopback door's + // whole job is liveness (`/capacity`), and it must keep answering + // even when the main multi-thread runtime's reactor stalls under + // load — which is exactly when the node-agent most needs a truthful + // answer. Sharing the main runtime lets a stall silently wedge the + // accept loop (a TCP timeout the agent can't distinguish from a dead + // node); an isolated reactor turns that into a fast `503` driven by + // the runtime-liveness heartbeat (see `ApiState::runtime_stalled`). + let std_listener = + std::net::TcpListener::bind(local_addr).map_err(smolvm::error::Error::Io)?; + std_listener + .set_nonblocking(true) + .map_err(smolvm::error::Error::Io)?; + let local_app = app.clone(); + tracing::info!(address = %local_addr, "starting loopback HTTP door (local node-agent, isolated runtime)"); + println!( + "smolvm local API (loopback, plain) on http://{}", + local_addr + ); + std::thread::Builder::new() + .name("smolvm-loopback-api".to_string()) + .spawn(move || { + let rt = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(rt) => rt, + Err(e) => { + tracing::error!(error = %e, "loopback door runtime failed to build"); + return; + } + }; + rt.block_on(async move { + // Register the listener with THIS runtime's reactor. + let listener = match tokio::net::TcpListener::from_std(std_listener) { + Ok(l) => l, + Err(e) => { + tracing::error!(error = %e, "loopback door listener registration failed"); + return; + } + }; + let _ = axum::serve(listener, local_app) + .with_graceful_shutdown(shutdown_signal()) + .await; + }); + }) + .map_err(smolvm::error::Error::Io)?; + } + + let rustls_config = axum_server::tls_rustls::RustlsConfig::from_config(tls_config); + let handle = axum_server::Handle::new(); + + // Trip graceful shutdown on the same signal the plain path observes. + let shutdown_handle = handle.clone(); + tokio::spawn(async move { + shutdown_signal().await; + shutdown_handle.graceful_shutdown(Some(std::time::Duration::from_secs(5))); + }); + + tracing::info!(address = %addr, "starting HTTPS API server (mTLS, client cert required)"); + println!("smolvm API server listening on https://{} (mTLS)", addr); + + axum_server::bind_rustls(addr, rustls_config) + .handle(handle) + .serve(app.into_make_service()) + .await + .map_err(smolvm::error::Error::Io) + } + + #[cfg(unix)] + async fn serve_unix(&self, path: PathBuf, app: Router) -> Result<()> { + let socket_guard = UnixSocketGuard::bind(&path)?; + let listener = + tokio::net::UnixListener::bind(&socket_guard.path).map_err(smolvm::error::Error::Io)?; + + tracing::info!(path = %socket_guard.path.display(), "starting HTTP API server"); + println!( + "smolvm API server listening on unix://{}", + socket_guard.path.display() + ); + + axum::serve(listener, app) + .with_graceful_shutdown(shutdown_signal()) + .await + .map_err(smolvm::error::Error::Io) + } +} + +#[derive(Debug, Clone)] +enum ListenTarget { + Tcp(SocketAddr), + #[cfg(unix)] + Unix(PathBuf), +} + +impl ListenTarget { + fn parse(value: &str) -> Result { + if let Ok(addr) = value.parse::() { + return Ok(Self::Tcp(addr)); + } + + #[cfg(unix)] + { + // If the value looks like an intended IP:PORT (contains ':' + // but failed SocketAddr parsing), report the parse failure + // rather than silently treating it as a Unix socket path. + if !value.starts_with("unix://") && !value.starts_with('/') && value.contains(':') { + return Err(smolvm::error::Error::config( + "parse listen address", + format!( + "invalid address '{}': expected a valid ADDR:PORT or a unix:// path", + value + ), + )); + } + let path = value.strip_prefix("unix://").unwrap_or(value); + Ok(Self::Unix(PathBuf::from(path))) + } + + #[cfg(not(unix))] + { + Err(smolvm::error::Error::config( + "parse listen address", + format!("invalid address '{}': expected ADDR:PORT", value), + )) + } + } +} + +fn default_listen_value() -> String { + #[cfg(unix)] + { + let path = dirs::runtime_dir() + .unwrap_or_else(|| PathBuf::from("/tmp")) + .join("smolvm.sock") + .display() + .to_string(); + format!("unix://{path}") + } + + #[cfg(not(unix))] + { + String::from("127.0.0.1:8080") + } +} + +#[cfg(unix)] +#[derive(Debug)] +struct UnixSocketGuard { + path: PathBuf, +} + +#[cfg(unix)] +impl UnixSocketGuard { + fn bind(path: &std::path::Path) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(smolvm::error::Error::Io)?; + } + + match std::fs::remove_file(path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(smolvm::error::Error::Io(e)), + } + + Ok(Self { + path: path.to_path_buf(), + }) + } +} + +#[cfg(unix)] +impl Drop for UnixSocketGuard { + fn drop(&mut self) { + if let Err(e) = std::fs::remove_file(&self.path) { + if e.kind() != std::io::ErrorKind::NotFound { + tracing::warn!(path = %self.path.display(), error = %e, "failed to remove unix socket"); + } + } + } +} + +/// Wait for shutdown signal. +/// Note: VMs run independently and survive a normal shutdown/restart; they are +/// only stopped when SMOLVM_DRAIN_ON_SHUTDOWN is set (see run_server). +/// Use DELETE /api/v1/machines/:id to stop specific VMs. +async fn shutdown_signal() { + let ctrl_c = async { + if let Err(e) = tokio::signal::ctrl_c().await { + tracing::error!(error = %e, "failed to listen for Ctrl+C"); + std::future::pending::<()>().await; + } + }; + + #[cfg(unix)] + let terminate = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut signal) => { + signal.recv().await; + } + Err(e) => { + tracing::error!(error = %e, "failed to install SIGTERM handler"); + std::future::pending::<()>().await; + } + } + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => {}, + _ = terminate => {}, + } + + tracing::info!("shutdown signal received"); + eprintln!("\nShutting down server (VMs continue running)..."); +} + +#[cfg(test)] +mod tests { + use super::ListenTarget; + + #[test] + fn parse_tcp_listen_target() { + let target = ListenTarget::parse("127.0.0.1:8080").expect("tcp target should parse"); + match target { + ListenTarget::Tcp(addr) => assert_eq!(addr.to_string(), "127.0.0.1:8080"), + #[cfg(unix)] + ListenTarget::Unix(path) => panic!("expected tcp, got unix path {}", path.display()), + } + } + + #[cfg(unix)] + #[test] + fn parse_unix_listen_target() { + let target = ListenTarget::parse("/tmp/smol.sock").expect("unix target should parse"); + match target { + ListenTarget::Unix(path) => { + assert_eq!(path, std::path::PathBuf::from("/tmp/smol.sock")) + } + ListenTarget::Tcp(addr) => panic!("expected unix, got tcp address {addr}"), + } + } + + #[cfg(unix)] + #[test] + fn parse_unix_listen_target_with_prefix() { + let target = + ListenTarget::parse("unix:///tmp/smol.sock").expect("unix target should parse"); + match target { + ListenTarget::Unix(path) => { + assert_eq!(path, std::path::PathBuf::from("/tmp/smol.sock")) + } + ListenTarget::Tcp(addr) => panic!("expected unix, got tcp address {addr}"), + } + } +} diff --git a/src/cli/serve_tls.rs b/src/cli/serve_tls.rs new file mode 100644 index 0000000..940fc50 --- /dev/null +++ b/src/cli/serve_tls.rs @@ -0,0 +1,229 @@ +//! mTLS for the fleet serve API (control↔node, increment 3 of the mTLS plan). +//! +//! In a fleet, the worker runs `smolvm serve` and is driven by the control +//! plane. That channel must be mutually authenticated: the node presents a +//! CA-signed **server** cert, and only a client presenting a CA-signed **client** +//! cert (the control plane) may connect. This module builds the rustls +//! `ServerConfig` that enforces `require_and_verify_client_cert`. +//! +//! **Fail-closed:** when `SMOLVM_SERVE_REQUIRE_MTLS=1` the serve API refuses to +//! start without TLS configured — it must never fall back to plain HTTP / the +//! interim bearer token when the deploy declared it should be mTLS-protected. +//! This is a DEDICATED opt-in, deliberately NOT keyed off `SMOLVM_PUBLISH_ADDR` +//! (which every worker sets for the published-port datapath — overloading it +//! would make plain-HTTP workers refuse to boot). + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use rustls::server::WebPkiClientVerifier; +use rustls::{RootCertStore, ServerConfig}; +use rustls_pki_types::{pem::PemObject, CertificateDer, PrivateKeyDer}; + +/// Env var holding the node's PEM **server** cert (signed by the node-CA). +const ENV_CERT: &str = "SMOLVM_SERVE_TLS_CERT"; +/// Env var holding the node's PEM private key. +const ENV_KEY: &str = "SMOLVM_SERVE_TLS_KEY"; +/// Env var holding the PEM node-CA cert used to verify the control's client cert. +const ENV_CLIENT_CA: &str = "SMOLVM_SERVE_TLS_CLIENT_CA"; +/// Dedicated opt-in: the deploy declares this serve API MUST run mTLS. When set, +/// a missing/partial cert config is fatal (fail-closed) rather than a silent +/// fall-back to plain HTTP. NOT keyed off `SMOLVM_PUBLISH_ADDR` (see module doc). +const ENV_REQUIRE_MTLS: &str = "SMOLVM_SERVE_REQUIRE_MTLS"; + +/// True when the deploy has declared this serve API must be mTLS-protected. +pub fn require_mtls() -> bool { + matches!( + std::env::var(ENV_REQUIRE_MTLS).ok().as_deref(), + Some("1") | Some("true") + ) +} + +/// Loopback plain-HTTP address for the **local** node-agent when the main port +/// runs mTLS. mTLS locks the whole network port to CA-signed clients, but the +/// node's own agent polls `/capacity` locally over plain HTTP — so we open a +/// second door bound to loopback only (unreachable from the network). Defaults +/// to `127.0.0.1:`; override with `SMOLVM_SERVE_LOCAL_ADDR`. +/// Returns `None` only if an override is set but unparseable. +pub fn local_plain_addr(main: std::net::SocketAddr) -> Option { + match std::env::var("SMOLVM_SERVE_LOCAL_ADDR") + .ok() + .filter(|v| !v.is_empty()) + { + Some(v) => v.parse().ok(), + None => Some(std::net::SocketAddr::from(( + std::net::Ipv4Addr::LOCALHOST, + main.port().wrapping_add(1), + ))), + } +} + +fn env_path(name: &str) -> Option { + std::env::var_os(name) + .filter(|v| !v.is_empty()) + .map(PathBuf::from) +} + +/// Resolve the serve API's TLS posture from the environment. +/// +/// - All three TLS env vars set ⇒ `Ok(Some(config))` (mTLS, client cert required). +/// - None set, mTLS not required ⇒ `Ok(None)` (plain HTTP, local/dev/non-mTLS worker). +/// - **`SMOLVM_SERVE_REQUIRE_MTLS` set but TLS not fully configured ⇒ `Err` (fail-closed).** +/// - A partial config (some but not all vars) ⇒ `Err` (misconfiguration). +pub fn resolve_tls() -> Result>, String> { + let cert = env_path(ENV_CERT); + let key = env_path(ENV_KEY); + let client_ca = env_path(ENV_CLIENT_CA); + + match (cert, key, client_ca) { + (Some(cert), Some(key), Some(client_ca)) => { + Ok(Some(build_server_config(&cert, &key, &client_ca)?)) + } + (None, None, None) => { + if require_mtls() { + Err(format!( + "{ENV_REQUIRE_MTLS} is set but {ENV_CERT}/{ENV_KEY}/{ENV_CLIENT_CA} are unset — \ + refusing to start without client-cert verification (fail-closed)" + )) + } else { + Ok(None) + } + } + _ => Err(format!( + "incomplete serve TLS config: set all of {ENV_CERT}, {ENV_KEY}, {ENV_CLIENT_CA} (or none)" + )), + } +} + +/// Build a rustls server config that requires + verifies a client cert chained +/// to the node-CA. +fn build_server_config( + cert_path: &Path, + key_path: &Path, + client_ca_path: &Path, +) -> Result, String> { + // Pin the ring provider explicitly rather than relying on a process-global + // install — avoids ordering hazards if anything else touches rustls. + let provider = Arc::new(rustls::crypto::ring::default_provider()); + + // Client-cert trust anchor: the node-CA. Only the control plane holds a + // client cert signed by it. + let mut roots = RootCertStore::empty(); + for ca in CertificateDer::pem_file_iter(client_ca_path) + .map_err(|e| format!("read client CA {}: {e}", client_ca_path.display()))? + { + let ca = ca.map_err(|e| format!("parse client CA cert: {e}"))?; + roots + .add(ca) + .map_err(|e| format!("add client CA to root store: {e}"))?; + } + if roots.is_empty() { + return Err(format!( + "client CA {} contained no certificates", + client_ca_path.display() + )); + } + let verifier = WebPkiClientVerifier::builder_with_provider(Arc::new(roots), provider.clone()) + .build() + .map_err(|e| format!("build client-cert verifier: {e}"))?; + + // Our server identity (node server cert + key). + let certs: Vec> = CertificateDer::pem_file_iter(cert_path) + .map_err(|e| format!("read server cert {}: {e}", cert_path.display()))? + .collect::>() + .map_err(|e| format!("parse server cert chain: {e}"))?; + if certs.is_empty() { + return Err(format!( + "server cert {} contained no certificates", + cert_path.display() + )); + } + let key = PrivateKeyDer::from_pem_file(key_path) + .map_err(|e| format!("read server key {}: {e}", key_path.display()))?; + + let mut config = ServerConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|e| format!("rustls protocol versions: {e}"))? + .with_client_cert_verifier(verifier) + .with_single_cert(certs, key) + .map_err(|e| format!("install server cert/key: {e}"))?; + // axum-server speaks h2 + http/1.1; advertise both via ALPN. + config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()]; + + Ok(Arc::new(config)) +} + +#[cfg(test)] +mod tests { + use super::*; + + // resolve_tls reads process env; these run serially via a shared guard to + // avoid cross-test interference on the shared env vars. + fn lock() -> std::sync::MutexGuard<'static, ()> { + static L: std::sync::Mutex<()> = std::sync::Mutex::new(()); + L.lock().unwrap_or_else(|e| e.into_inner()) + } + + fn clear() { + for v in [ENV_CERT, ENV_KEY, ENV_CLIENT_CA, ENV_REQUIRE_MTLS] { + std::env::remove_var(v); + } + } + + #[test] + fn no_env_not_required_is_plain_http() { + let _g = lock(); + clear(); + assert!(resolve_tls().unwrap().is_none()); + } + + #[test] + fn require_mtls_without_tls_fails_closed() { + let _g = lock(); + clear(); + std::env::set_var(ENV_REQUIRE_MTLS, "1"); + let err = resolve_tls().unwrap_err(); + assert!(err.contains("fail-closed"), "{err}"); + clear(); + } + + #[test] + fn local_plain_addr_defaults_to_loopback_port_plus_one() { + let _g = lock(); + std::env::remove_var("SMOLVM_SERVE_LOCAL_ADDR"); + let main: std::net::SocketAddr = "0.0.0.0:8080".parse().unwrap(); + let local = local_plain_addr(main).unwrap(); + assert!(local.ip().is_loopback()); + assert_eq!(local.port(), 8081); + } + + #[test] + fn local_plain_addr_honors_override() { + let _g = lock(); + std::env::set_var("SMOLVM_SERVE_LOCAL_ADDR", "127.0.0.1:9999"); + let local = local_plain_addr("0.0.0.0:8080".parse().unwrap()).unwrap(); + assert_eq!(local.port(), 9999); + std::env::remove_var("SMOLVM_SERVE_LOCAL_ADDR"); + } + + #[test] + fn publish_addr_alone_does_not_force_mtls() { + // A worker sets SMOLVM_PUBLISH_ADDR for the datapath; that must NOT make + // a plain-HTTP serve refuse to start. + let _g = lock(); + clear(); + std::env::set_var("SMOLVM_PUBLISH_ADDR", "0.0.0.0"); + assert!(resolve_tls().unwrap().is_none()); + std::env::remove_var("SMOLVM_PUBLISH_ADDR"); + } + + #[test] + fn partial_tls_config_is_rejected() { + let _g = lock(); + clear(); + std::env::set_var(ENV_CERT, "/tmp/x.crt"); + let err = resolve_tls().unwrap_err(); + assert!(err.contains("incomplete"), "{err}"); + clear(); + } +} diff --git a/src/cli/smolfile.rs b/src/cli/smolfile.rs new file mode 100644 index 0000000..487bb17 --- /dev/null +++ b/src/cli/smolfile.rs @@ -0,0 +1,434 @@ +//! Smolfile CLI integration — merges Smolfile config with CLI flags. +//! +//! Types and parsing live in [`smolvm::smolfile`]. This module provides +//! the merge logic that combines Smolfile values with CLI arguments +//! to produce [`CreateVmParams`] and [`PackConfig`]. + +use crate::cli::parsers::parse_cidr; +use crate::cli::vm_common::CreateVmParams; +use smolvm::data::network::PortMapping; +use smolvm::data::resources::{DEFAULT_MICROVM_CPU_COUNT, DEFAULT_MICROVM_MEMORY_MIB}; +use smolvm::network::NetworkBackend; +use std::path::PathBuf; + +// Re-export from the library +pub use smolvm::smolfile::{parse_duration_secs, Smolfile}; + +/// Load and parse a Smolfile from the given path. +pub fn load(path: &std::path::Path) -> smolvm::Result { + smolvm::smolfile::load(path) +} + +/// Build `CreateVmParams` by merging CLI flags with an optional Smolfile. +/// +/// CLI flags override Smolfile values. For Vec fields, CLI values are appended +/// to Smolfile values. For scalar fields, non-default CLI values take priority. +/// +/// Merge precedence: +/// image: CLI > Smolfile > None (bare Alpine) +/// entrypoint: CLI override > Smolfile > image metadata +/// cmd: CLI trailing args > Smolfile cmd (full replacement) +/// env: Smolfile + CLI extends +/// init: Smolfile + CLI extends +#[allow(clippy::too_many_arguments)] +pub fn build_create_params( + name: String, + cli_image: Option, + cli_entrypoint: Option, + cli_cmd: Vec, + cli_cpus: u8, + cli_mem: u32, + cli_volume: Vec, + cli_port: Vec, + cli_net: bool, + cli_network_backend: Option, + cli_dns: Option, + cli_init: Vec, + cli_env: Vec, + cli_workdir: Option, + smolfile_path: Option, + cli_storage_gb: Option, + cli_overlay_gb: Option, + cli_allow_cidr: Vec, +) -> smolvm::Result { + let cidrs_to_option = |v: Vec| if v.is_empty() { None } else { Some(v) }; + + let sf = match smolfile_path { + Some(path) => load(&path)?, + None => { + let net = cli_net || !cli_allow_cidr.is_empty() || cli_dns.is_some(); + return Ok(CreateVmParams { + secret_refs: Default::default(), + name, + image: cli_image, + entrypoint: cli_entrypoint.map(|e| vec![e]).unwrap_or_default(), + cmd: cli_cmd, + cpus: cli_cpus, + mem: cli_mem, + volume: cli_volume, + port: cli_port, + net, + network_backend: cli_network_backend, + dns: cli_dns, + init: cli_init, + env: cli_env, + workdir: cli_workdir, + storage_gb: cli_storage_gb, + overlay_gb: cli_overlay_gb, + allowed_cidrs: cidrs_to_option(cli_allow_cidr), + restart_policy: None, + restart_max_retries: None, + restart_max_backoff_secs: None, + health_cmd: None, + health_interval_secs: None, + health_timeout_secs: None, + health_retries: None, + health_startup_grace_secs: None, + ssh_agent: false, + cuda: false, + gpu: false, + gpu_vram_mib: None, + rosetta: false, + dns_filter_hosts: None, + source_smolmachine: None, + }); + } + }; + + // Image: CLI > Smolfile > None + let image = cli_image.or(sf.image); + + // Entrypoint: CLI > Smolfile + let entrypoint = if let Some(ep) = cli_entrypoint { + vec![ep] + } else { + sf.entrypoint + }; + + // Cmd: CLI > Smolfile (full replacement, not append) + let cmd = if cli_cmd.is_empty() { sf.cmd } else { cli_cmd }; + + // Resolve [dev] fields, falling back to top-level + let dev = sf.dev.unwrap_or_default(); + + // Ports: [dev].ports > top-level ports, then CLI extends + let sf_ports = if !dev.ports.is_empty() { + dev.ports + } else { + sf.ports + }; + let mut ports: Vec = sf_ports + .iter() + .map(|s| PortMapping::parse(s)) + .collect::, _>>() + .map_err(|e| smolvm::Error::config("smolfile ports", e))?; + ports.extend(cli_port); + + // Volumes: [dev].volumes > top-level volumes, then CLI extends + let sf_volumes = if !dev.volumes.is_empty() { + dev.volumes + } else { + sf.volumes + }; + let mut volumes = sf_volumes; + volumes.extend(cli_volume); + + // Env: top-level env + [dev].env + CLI extends (whitespace trimmed) + let mut env: Vec = sf.env.into_iter().map(|e| e.trim().to_string()).collect(); + env.extend(dev.env.into_iter().map(|e| e.trim().to_string())); + env.extend(cli_env.into_iter().map(|e| e.trim().to_string())); + + // Init: [dev].init > top-level init, then CLI extends + let sf_init = if !dev.init.is_empty() { + dev.init + } else { + sf.init + }; + let mut init = sf_init; + init.extend(cli_init); + + // Workdir: CLI > [dev].workdir > top-level workdir + let dev_workdir = dev.workdir; + + // Scalars: CLI non-default overrides Smolfile + let default_cpus = DEFAULT_MICROVM_CPU_COUNT; + let default_mem = DEFAULT_MICROVM_MEMORY_MIB; + + let cpus = if cli_cpus != default_cpus { + cli_cpus + } else { + sf.cpus.unwrap_or(cli_cpus) + }; + + let mem = if cli_mem != default_mem { + cli_mem + } else { + sf.memory.unwrap_or(cli_mem) + }; + + let net = if cli_net { + true + } else { + sf.net.unwrap_or(false) + }; + + let gpu = sf.gpu.unwrap_or(false); + let rosetta = sf.rosetta.unwrap_or(false); + + let workdir = cli_workdir.or(dev_workdir).or(sf.workdir); + + // Scalars: CLI overrides Smolfile + let storage_gb = cli_storage_gb.or(sf.storage); + let overlay_gb = cli_overlay_gb.or(sf.overlay); + + // Merge network policy: [network] section, then CLI extends + let network = sf.network.unwrap_or_default(); + + // Preserve original hostnames for DNS filtering. + // Do NOT resolve these to CIDRs here — CDN-backed hosts rotate IPs and the + // resolved addresses would be stale by the time the machine is started. + // Re-resolution happens at `machine start` time (see start_vm_named). + let sf_allow_hosts = network.allow_hosts; + + // Parse [network].allow_cidrs — these are explicit stable CIDRs, stored as-is. + let mut allowed_cidrs_vec: Vec = Vec::new(); + let sf_cidrs: Vec = network + .allow_cidrs + .iter() + .map(|s| parse_cidr(s)) + .collect::, _>>() + .map_err(|e| smolvm::Error::config("smolfile [network] allow_cidrs", e))?; + allowed_cidrs_vec.extend(sf_cidrs); + + // CLI extends + allowed_cidrs_vec.extend(cli_allow_cidr); + + // --allow-cidr / --allow-host / [network] / --dns implies --net + let net = if !allowed_cidrs_vec.is_empty() || !sf_allow_hosts.is_empty() || cli_dns.is_some() { + true + } else { + net + }; + let allowed_cidrs = cidrs_to_option(allowed_cidrs_vec); + + // Restart policy from [restart] section + let restart_policy = sf + .restart + .as_ref() + .and_then(|r| r.policy.as_deref()) + .map(|p| { + p.parse::() + .map_err(|e| smolvm::Error::config("smolfile [restart] policy", e)) + }) + .transpose()?; + let restart_max_retries = sf.restart.as_ref().and_then(|r| r.max_retries); + let restart_max_backoff_secs = sf + .restart + .as_ref() + .and_then(|r| r.max_backoff.as_ref()) + .and_then(|s| parse_duration_secs(s)); + + // Health check from [health] section + let health_cmd = sf + .health + .as_ref() + .filter(|h| !h.exec.is_empty()) + .map(|h| h.exec.clone()); + let health_interval_secs = sf + .health + .as_ref() + .and_then(|h| h.interval.as_ref()) + .and_then(|s| parse_duration_secs(s)); + let health_timeout_secs = sf + .health + .as_ref() + .and_then(|h| h.timeout.as_ref()) + .and_then(|s| parse_duration_secs(s)); + let health_retries = sf.health.as_ref().and_then(|h| h.retries); + let health_startup_grace_secs = sf + .health + .as_ref() + .and_then(|h| h.startup_grace.as_ref()) + .and_then(|s| parse_duration_secs(s)); + + Ok(CreateVmParams { + secret_refs: sf.secrets, + name, + image, + entrypoint, + cmd, + cpus, + mem, + volume: volumes, + port: ports, + net, + network_backend: cli_network_backend, + dns: cli_dns, + init, + env, + workdir, + storage_gb, + overlay_gb, + allowed_cidrs, + restart_policy, + restart_max_retries, + restart_max_backoff_secs, + health_cmd, + health_interval_secs, + health_timeout_secs, + health_retries, + health_startup_grace_secs, + ssh_agent: sf.auth.as_ref().and_then(|a| a.ssh_agent).unwrap_or(false), + cuda: sf.cuda.unwrap_or(false), + gpu, + gpu_vram_mib: sf.gpu_vram, + rosetta, + dns_filter_hosts: if sf_allow_hosts.is_empty() { + None + } else { + Some(sf_allow_hosts) + }, + source_smolmachine: None, + }) +} + +/// Resolved pack configuration from Smolfile + CLI args. +pub struct PackConfig { + /// Resolved image. + pub image: Option, + /// Resolved entrypoint. + pub entrypoint: Vec, + /// Resolved cmd. + pub cmd: Vec, + /// Resolved vCPU count. + pub cpus: u8, + /// Resolved memory in MiB. + pub mem: u32, + /// Target OCI platform. + pub oci_platform: Option, + /// Resolved environment variables. + pub env: Vec, + /// Resolved working directory. + pub workdir: Option, + /// Whether outbound networking is enabled. + /// `None` = unspecified (caller decides default), `Some(true)` = explicitly + /// enabled, `Some(false)` = explicitly disabled. This tri-state is needed + /// so `--from-vm` can distinguish "Smolfile says net = false" from "no + /// Smolfile, fall back to source VM's setting". + pub net: Option, + /// Whether GPU acceleration is enabled in the packed VM. + pub gpu: bool, + /// Secrets carried into the pack manifest as references. They are resolved + /// to plaintext on the run host at exec time and are never packed as + /// values. + pub secret_refs: std::collections::BTreeMap, +} + +/// Resolve pack configuration by merging CLI flags with an optional Smolfile. +/// +/// Merge precedence: +/// image: CLI --image > Smolfile image > None +/// entrypoint: CLI --entrypoint > [artifact].entrypoint > Smolfile entrypoint > image metadata +/// cmd: [artifact].cmd > Smolfile cmd > image metadata +/// cpus: CLI --cpus (non-default) > [artifact].cpus > Smolfile cpus > default +/// memory: CLI --mem (non-default) > [artifact].memory > Smolfile memory > default +/// oci_platform: CLI --oci-platform > [artifact].oci_platform > None +/// env: Smolfile top-level env (trimmed) +/// workdir: Smolfile top-level workdir +/// gpu: CLI --gpu (true overrides) > Smolfile gpu > false +pub fn resolve_pack_config( + cli_image: Option, + cli_entrypoint: Option, + cli_cpus: u8, + cli_mem: u32, + cli_oci_platform: Option, + cli_gpu: bool, + smolfile_path: Option, +) -> smolvm::Result { + let default_cpus = DEFAULT_MICROVM_CPU_COUNT; + let default_mem = crate::cli::pack::PACK_DEFAULT_MEMORY_MIB; + let sf = match smolfile_path { + Some(path) => load(&path)?, + None => { + return Ok(PackConfig { + image: cli_image, + entrypoint: cli_entrypoint.map(|e| vec![e]).unwrap_or_default(), + cmd: vec![], + cpus: cli_cpus, + mem: cli_mem, + oci_platform: cli_oci_platform, + env: vec![], + workdir: None, + net: None, + gpu: cli_gpu, + secret_refs: Default::default(), + }); + } + }; + + // Resolve [artifact] (preferred) or [pack] (alias) + let artifact = sf.artifact.or(sf.pack).unwrap_or_default(); + + // Image: CLI > Smolfile top-level + let image = cli_image.or(sf.image); + + // Entrypoint: CLI > [artifact] > top-level + let entrypoint = if let Some(ep) = cli_entrypoint { + vec![ep] + } else if !artifact.entrypoint.is_empty() { + artifact.entrypoint + } else { + sf.entrypoint + }; + + // Cmd: [artifact] > top-level + let cmd = if !artifact.cmd.is_empty() { + artifact.cmd + } else { + sf.cmd + }; + + // Scalars: CLI non-default > [artifact] > top-level > default + let cpus = if cli_cpus != default_cpus { + cli_cpus + } else { + artifact.cpus.or(sf.cpus).unwrap_or(cli_cpus) + }; + + let mem = if cli_mem != default_mem { + cli_mem + } else { + artifact.memory.or(sf.memory).unwrap_or(cli_mem) + }; + + // oci_platform: CLI > [artifact] + let oci_platform = cli_oci_platform.or(artifact.oci_platform); + + Ok(PackConfig { + image, + entrypoint, + cmd, + cpus, + mem, + oci_platform, + env: sf.env.into_iter().map(|e| e.trim().to_string()).collect(), + workdir: sf.workdir, + // [network].allow_hosts / allow_cidrs implies net = true, + // matching the same logic in build_create_params(). + // Preserve the tri-state: None = unspecified, Some = explicit. + net: { + let network_section_implies_net = sf + .network + .as_ref() + .is_some_and(|n| !n.allow_hosts.is_empty() || !n.allow_cidrs.is_empty()); + if network_section_implies_net { + Some(true) + } else { + sf.net // None if key absent, Some(true/false) if explicit + } + }, + // CLI --gpu wins; Smolfile gpu = true also enables it. + gpu: cli_gpu || sf.gpu.unwrap_or(false), + secret_refs: sf.secrets, + }) +} diff --git a/src/cli/vm_common.rs b/src/cli/vm_common.rs new file mode 100644 index 0000000..76d64b4 --- /dev/null +++ b/src/cli/vm_common.rs @@ -0,0 +1,2528 @@ +//! Shared helpers for machine CLI commands. +//! +//! The `machine` subcommand exposes lifecycle commands +//! (create, start, stop, delete, ls). This module provides the common +//! implementations used by those commands. + +use crate::cli::{format_pid_suffix, truncate}; +use smolvm::agent::{vm_data_dir, AgentManager}; +use smolvm::config::{RecordState, SmolvmConfig, VmRecord}; +use smolvm::data::network::PortMapping; +use smolvm::data::resources::{DEFAULT_MICROVM_CPU_COUNT, DEFAULT_MICROVM_MEMORY_MIB}; +use smolvm::data::storage::HostMount; +use smolvm::data::validate_vm_name; +use smolvm::db::SmolvmDb; +use smolvm::network::NetworkBackend; +use smolvm::secrets::SecretRef; +use smolvm::storage::{DEFAULT_OVERLAY_SIZE_GIB, DEFAULT_STORAGE_SIZE_GIB}; +use smolvm_protocol::ImageInfo; +use std::collections::BTreeMap; +use std::io::Write; + +// ============================================================================ +// Shared helpers +// ============================================================================ + +/// Resolve an optional VM name: if no name is given and a VM named "default" +/// exists in the config database, return `Some("default")` so callers route +/// through the named-VM code path (which loads config, init commands, network +/// settings, etc.). Otherwise returns the input unchanged. +pub fn resolve_vm_name(name: Option) -> smolvm::Result> { + if name.is_some() { + return Ok(name); + } + // Use direct DB lookup instead of SmolvmConfig::load() to avoid + // loading all config + all VMs just to check if "default" exists. + let db = SmolvmDb::open()?; + if db.get_vm("default")?.is_some() { + Ok(Some("default".to_string())) + } else { + Ok(None) + } +} + +/// Get the agent manager for an optional name (default if `None`). +/// +/// When no name is given, uses `AgentManager::new_default()` which is +/// canonicalized to `for_vm("default")` — same socket/PID/storage paths +/// regardless of whether the caller specifies a name or not. +pub fn get_vm_manager(name: &Option) -> smolvm::Result { + if let Some(name) = name { + AgentManager::for_vm(name) + } else { + AgentManager::new_default() + } +} + +/// Return the display label for an optional VM name. +pub fn vm_label(name: &Option) -> String { + name.as_deref().unwrap_or("default").to_string() +} + +/// Ensure a VM is running and return a connected client. +/// +/// This is the common pattern used by exec commands in the machine subcommand. +/// It resolves the VM manager, checks connectivity, and establishes a client connection. +pub fn ensure_running_and_connect( + name: &Option, +) -> smolvm::Result<(AgentManager, smolvm::agent::AgentClient)> { + let manager = get_vm_manager(name)?; + let label = vm_label(name); + let start_hint = match name { + Some(name) => format!("smolvm machine start --name {}", name), + None => "smolvm machine start".to_string(), + }; + + if manager.try_connect_existing().is_none() { + // Distinguish "machine not found" from "machine stopped" + // so a typo in the name gives a clear error rather than the + // generic "not running / use start" message. + if let Some(ref n) = name { + let exists = SmolvmDb::open() + .ok() + .and_then(|db| db.get_vm(n).ok().flatten()) + .is_some(); + if !exists { + return Err(smolvm::Error::vm_not_found(n)); + } + } + + // Best-effort reconcile: if we can't connect to the agent + // but the libkrun PID is alive, we're in the bug 2 zombie + // state — record says "running" but agent is dead. Mark + // the record `Unreachable` so subsequent `machine list` + // calls reflect truth without re-pinging. DB write is + // best-effort: we ignore failures so a bad DB doesn't mask + // the real "can't connect" error the user is about to see. + mark_unreachable_if_zombie(&label); + + // Distinguish "machine does not exist" from "machine exists but is + // stopped". Without this a typo in --name reports "is not running" + // and points at `start`, which then fails differently (QA BUG-45). + let exists = SmolvmDb::open() + .ok() + .and_then(|db| db.get_vm(&label).ok().flatten()) + .is_some(); + if !exists { + return Err(smolvm::Error::agent( + "connect", + format!("machine '{}' not found", label), + )); + } + + return Err(smolvm::Error::agent( + "connect", + format!( + "machine '{}' is not running. Use '{}' first.", + label, start_hint + ), + )); + } + + let client = smolvm::agent::AgentClient::connect_with_retry(manager.vsock_socket())?; + Ok((manager, client)) +} + +/// CLI wrapper around `state_probe::recover_if_unreachable` that +/// prints a one-line notice when recovery actually runs. The shared +/// helper is silent (the HTTP API doesn't have a stdout to write +/// to); CLI callers want the operator to see the zombie teardown. +fn cli_recover_if_unreachable(name: &str) { + // Peek at the record before recovery so we can show the PID in + // the notice. Losing the PID after recovery is fine — the DB + // gets cleared — but we want the operator to know *which* + // process got killed. + let pid_for_notice = SmolvmDb::open() + .ok() + .and_then(|db| db.get_vm(name).ok().flatten()) + .and_then(|r| r.pid); + + if smolvm::agent::state_probe::recover_if_unreachable(name) { + println!( + "Machine '{}' is unreachable (PID {} alive but agent unresponsive); \ + cleaning up.", + name, + pid_for_notice.unwrap_or(0) + ); + } +} + +/// If the VM record says `Running` and the libkrun PID is alive but +/// the agent isn't responding, transition the record to +/// `Unreachable`. Caller invokes this on `ensure_running_and_connect` +/// failure so the next `machine list` is honest. +/// +/// All errors are swallowed (logged at debug level) — this is a +/// best-effort cleanup, not a critical path. +fn mark_unreachable_if_zombie(name: &str) { + let Ok(mut config) = SmolvmConfig::load() else { + return; + }; + let Some(record) = config.get_vm(name) else { + return; + }; + // Only transition Running → Unreachable. Stopped/Created/Failed + // are already accurate. + if record.state != RecordState::Running { + return; + } + if !record.is_process_alive() { + // PID dead → next list will see Stopped without our help. + return; + } + // PID alive + ensure_running_and_connect failed → zombie. Persist + // the new state. Update via the closure-based helper if available; + // fall back to nothing on failure (best-effort). + let _ = config.update_vm(name, |r| { + r.state = RecordState::Unreachable; + }); + tracing::debug!( + machine = %name, + "marked machine Unreachable: PID alive but agent not responding" + ); +} + +/// Print command output and exit with the given code. +/// +/// Prints stdout to stdout, stderr to stderr, detaches the manager +/// (keeping the VM running), and exits the process. +pub fn print_output_and_exit( + manager: &AgentManager, + exit_code: i32, + stdout: &[u8], + stderr: &[u8], +) -> ! { + // write_all on raw bytes preserves binary output (image bytes, tarballs, + // etc.) that print!("{}", ...) would corrupt or refuse to write. + if !stdout.is_empty() { + let _ = std::io::stdout().write_all(stdout); + } + if !stderr.is_empty() { + let _ = std::io::stderr().write_all(stderr); + } + crate::cli::flush_output(); + manager.detach(); + std::process::exit(exit_code); +} + +// ============================================================================ +// Init runner +// ============================================================================ + +/// Run a machine's `init` commands list against the agent. +/// +/// Branches on `image`: +/// +/// - `Some(img)`: each command runs *inside* the container's rootfs via +/// `client.run_non_interactive`. `record_mounts` are bind-mounted into +/// the container (so `[dev].volumes` like `.:/app` are visible to init, +/// not just to later `machine exec` calls). `overlay_id` ensures +/// filesystem changes (e.g. `pacman -Syu` package installs) persist +/// across this init invocation and into future `machine exec` calls — +/// matches the convention `machine exec` already uses +/// (`src/cli/machine.rs:750`). +/// +/// - `None`: each command runs in the agent's bare VM filesystem via +/// `client.vm_exec`. There's no container, so `record_mounts` and +/// `overlay_id` are unused on this branch. +/// +pub(crate) struct InitRunContext<'a> { + pub(crate) image: Option<&'a str>, + pub(crate) image_info: Option<&'a ImageInfo>, + pub(crate) env: &'a [(String, String)], + pub(crate) workdir: Option<&'a str>, + pub(crate) record_mounts: &'a [(String, String, bool)], + pub(crate) overlay_id: &'a str, +} + +/// On the first non-zero exit, returns an error containing the command +/// index, exit code, and any stdout/stderr the command produced. +/// **Both** streams are surfaced because package managers often write +/// the actual failure reason to stdout (`pacman`'s "target not found", +/// `apt`'s resolver diagnostics) — surfacing only stderr would leave +/// the operator with an exit code and no explanation. The caller is +/// responsible for stopping the VM if appropriate. +pub(crate) fn run_init_commands( + client: &mut smolvm::agent::AgentClient, + init: &[String], + context: InitRunContext<'_>, +) -> smolvm::Result<()> { + if init.is_empty() { + return Ok(()); + } + println!("Running {} init command(s)...", init.len()); + for (i, cmd) in init.iter().enumerate() { + if let Some(image) = context.image { + let defaults = + resolve_image_runtime_defaults(context.image_info, context.env, context.workdir); + let config = build_init_run_config( + image, + cmd, + &defaults, + context.record_mounts, + context.overlay_id, + ) + .with_tty(true); + // Interactive run streams the command's output live so the user sees + // progress (a slow compile, apt-get, etc.). The output isn't buffered + // here, so on a non-zero exit we report just the code — the detail was + // already printed above. Not checking the exit code would silently + // ignore a failed init command (e.g. a broken `apt install`). + let exit_code = client.run_interactive(config)?; + if exit_code != 0 { + return Err(smolvm::Error::agent( + "init", + format_init_failure(i, exit_code, "", ""), + )); + } + } else { + let (exit_code, stdout, stderr) = client.vm_exec( + init_argv(cmd), + context.env.to_vec(), + context.workdir.map(|s| s.to_string()), + None, + None, + )?; + if exit_code != 0 { + // Init output is generally text — lossy conversion is fine for + // error messages. Binary init output isn't a real use case. + return Err(smolvm::Error::agent( + "init", + format_init_failure( + i, + exit_code, + &String::from_utf8_lossy(&stdout), + &String::from_utf8_lossy(&stderr), + ), + )); + } + }; + } + Ok(()) +} + +/// Wrap a single init command line in `sh -c` argv form. Init commands +/// are user-supplied shell snippets (e.g. `"pacman -Sy && pacman -S git"`) +/// — we intentionally route them through `sh` so operators can use shell +/// features (`&&`, `|`, env expansion) without quoting gymnastics. +fn init_argv(cmd: &str) -> Vec { + vec!["sh".into(), "-c".into(), cmd.to_string()] +} + +/// Resolve effective env/workdir for image-backed execution. +/// +/// Image metadata provides the baseline defaults. Explicit values from the +/// CLI/Smolfile/persisted machine config override those defaults by key, while +/// workdir uses the explicit value when present and otherwise falls back to the +/// image's `WorkingDir`. Image `User` flows through unchanged because there is +/// no CLI or persisted override for it today. +pub(crate) struct ImageRuntimeDefaults { + pub(crate) env: Vec<(String, String)>, + pub(crate) workdir: Option, + pub(crate) user: Option, +} + +pub(crate) fn resolve_image_runtime_defaults( + image_info: Option<&ImageInfo>, + env: &[(String, String)], + explicit_workdir: Option<&str>, +) -> ImageRuntimeDefaults { + let mut resolved_env = Vec::new(); + + if let Some(image_info) = image_info { + for spec in &image_info.env { + if let Some((key, value)) = smolvm::util::parse_env_spec(spec) { + apply_env_override(&mut resolved_env, key, value); + } + } + } + + resolved_env = merge_env_overrides(&resolved_env, env); + + let workdir = explicit_workdir + .map(str::to_string) + .or_else(|| image_info.and_then(|info| info.workdir.clone())); + + let user = image_info.and_then(|info| info.user.clone()); + + ImageRuntimeDefaults { + env: resolved_env, + workdir, + user, + } +} + +pub(crate) fn merge_env_overrides( + base_env: &[(String, String)], + overrides: &[(String, String)], +) -> Vec<(String, String)> { + let mut env = base_env.to_vec(); + for (key, value) in overrides { + apply_env_override(&mut env, key.clone(), value.clone()); + } + env +} + +fn apply_env_override(env: &mut Vec<(String, String)>, key: String, value: String) { + env.retain(|(existing, _)| existing != &key); + env.push((key, value)); +} + +/// Build the `RunConfig` an image-based init command runs under. +/// +/// Pure function so the *shape* of the request (overlay ID, mount tags, +/// env, workdir, the `sh -c` wrap) can be unit-tested without mocking +/// `AgentClient`. Any of these silently regressing — e.g. mounts not +/// flowing through, or overlay ID drifting from the machine name — +/// would leave init working but `machine exec` no longer seeing init's +/// effects, exactly the class of bug that would lurk for months. +fn build_init_run_config( + image: &str, + cmd: &str, + defaults: &ImageRuntimeDefaults, + record_mounts: &[(String, String, bool)], + overlay_id: &str, +) -> smolvm::agent::RunConfig { + let mounts = crate::cli::parsers::record_mounts_to_runconfig_bindings(record_mounts); + smolvm::agent::RunConfig::new(image, init_argv(cmd)) + .with_env(defaults.env.clone()) + .with_workdir(defaults.workdir.clone()) + .with_user(defaults.user.clone()) + .with_mounts(mounts) + .with_persistent_overlay(Some(overlay_id.to_string())) +} + +/// Compose the user-facing init-failure message. Pure function — split +/// out for testability and so the formatting choice (which stream goes +/// in which order, separators, trimming) is in one place. +fn format_init_failure(index: usize, exit_code: i32, stdout: &str, stderr: &str) -> String { + let so = stdout.trim(); + let se = stderr.trim(); + let suffix = match (so, se) { + ("", "") => String::new(), + (so, "") => format!(": {}", so), + ("", se) => format!(": {}", se), + // Both populated: keep stderr first (canonical error channel) + // but include stdout because `pacman`/`apt`/`dnf` often put the + // real reason there. Single line, semicolon-separated, so the + // message stays grep-friendly. + (so, se) => format!(": {}; stdout: {}", se, so), + }; + format!("init[{}] failed (exit {}){}", index, exit_code, suffix) +} + +// ============================================================================ +// Create +// ============================================================================ + +/// Parameters for [`create_vm`]. +pub struct CreateVmParams { + pub name: String, + pub image: Option, + pub entrypoint: Vec, + pub cmd: Vec, + pub cpus: u8, + pub mem: u32, + pub volume: Vec, + pub port: Vec, + pub net: bool, + pub network_backend: Option, + pub dns: Option, + pub init: Vec, + pub env: Vec, + pub workdir: Option, + pub storage_gb: Option, + pub overlay_gb: Option, + pub allowed_cidrs: Option>, + pub restart_policy: Option, + pub restart_max_retries: Option, + pub restart_max_backoff_secs: Option, + pub health_cmd: Option>, + pub health_interval_secs: Option, + pub health_timeout_secs: Option, + pub health_retries: Option, + pub health_startup_grace_secs: Option, + pub ssh_agent: bool, + /// Enable CUDA-over-vsock (remote guest CUDA Driver-API to the host GPU). + pub cuda: bool, + /// Enable GPU acceleration (virtio-gpu with Venus/Vulkan). + pub gpu: bool, + /// GPU VRAM size in MiB (None = default). Ignored when gpu is false. + pub gpu_vram_mib: Option, + /// Enable Rosetta 2 for x86_64 binary translation on Apple Silicon. + pub rosetta: bool, + /// Hostnames for DNS filtering (from --allow-host / [network].allow_hosts). + pub dns_filter_hosts: Option>, + /// Absolute path to .smolmachine sidecar (for machines created with --from). + pub source_smolmachine: Option, + /// Secret refs from Smolfile `[secrets]`. The refs themselves are + /// persisted to the VM record (they are not sensitive); resolved + /// plaintext values are produced per-launch and never touch the DB. + pub secret_refs: BTreeMap, +} + +/// Resolve refs supplied by a trusted-local caller (CLI with a Smolfile passed +/// by the host user) into `(name, value)` env pairs. Empty vec if no refs, so +/// callers can unconditionally `.extend()`. Resolved values live in a +/// `Zeroizing` inside `resolve_refs_to_env` and are scrubbed as soon as +/// it returns — the caller must not log them. +pub fn resolve_secret_refs_for_env( + refs: &BTreeMap, +) -> smolvm::Result> { + smolvm::secrets::resolve_refs_to_env(refs, smolvm::secrets::ResolutionScope::TrustedLocal) + .map(smolvm::secrets::expose_into_env) +} + +/// Build the exec-time env for a persistent VM: the record's own `env` +/// (`KEY=VALUE` strings) plus freshly resolved `secret_refs`, formatted the +/// same way. Called at `machine start` (init + entrypoint) and `machine exec`. +/// Scope is `RecordReplay` — the refs were written by a trusted-local actor at +/// create time. The resolved plaintext stays in the returned vector and never +/// touches the record or the DB. +pub fn record_env_with_secrets(record: &VmRecord) -> smolvm::Result> { + let mut env = record.env.clone(); + env.extend(smolvm::secrets::expose_into_env( + smolvm::secrets::resolve_refs_to_env( + &record.secret_refs, + smolvm::secrets::ResolutionScope::RecordReplay, + )?, + )); + Ok(env) +} + +/// Create a named machine configuration (does not start it). +pub fn create_vm(params: CreateVmParams) -> smolvm::Result<()> { + let record = build_vm_record(¶ms)?; + let reservation = CreateVmReservation::reserve(¶ms.name)?; + reservation.commit(&record)?; + print_create_success(¶ms); + Ok(()) +} + +/// Cross-process reservation held while a create operation prepares any +/// name-derived on-disk state. Dropping it releases the DB reservation unless +/// it was committed. +pub(crate) struct CreateVmReservation { + db: SmolvmDb, + name: String, + token: String, + completed: bool, +} + +impl CreateVmReservation { + pub(crate) fn reserve(name: &str) -> smolvm::Result { + let db = SmolvmDb::open()?; + let token = SmolvmDb::create_reservation_token(); + if !db.reserve_vm_create(name, &token)? { + return Err(smolvm::Error::config( + "create machine", + format!("machine '{}' already exists or is being created", name), + )); + } + Ok(Self { + db, + name: name.to_string(), + token, + completed: false, + }) + } + + pub(crate) fn commit(mut self, record: &VmRecord) -> smolvm::Result<()> { + if !self + .db + .commit_reserved_vm(&self.name, &self.token, record)? + { + return Err(smolvm::Error::config( + "create machine", + format!( + "machine '{}' already exists or is no longer reserved", + self.name + ), + )); + } + self.completed = true; + Ok(()) + } +} + +impl Drop for CreateVmReservation { + fn drop(&mut self) { + if !self.completed { + if let Err(e) = self + .db + .release_vm_create_reservation(&self.name, &self.token) + { + tracing::warn!( + machine = %self.name, + error = %e, + "failed to release DB create reservation" + ); + } + } + } +} + +pub(crate) fn build_vm_record(params: &CreateVmParams) -> smolvm::Result { + // Validate name before touching the database. The on-disk layout uses + // a hash-derived directory (see `vm_data_dir`), so the name itself has + // no impact on socket path length — only character sanity + a generous + // length cap are needed here. + validate_vm_name(¶ms.name, "machine name") + .map_err(|reason| smolvm::Error::config("create machine", reason))?; + + // Parse and validate volume mounts + let mounts = HostMount::parse(¶ms.volume)? + .into_iter() + .map(|m| m.to_storage_tuple()) + .collect(); + + // Convert port mappings to tuple format for storage + let ports = PortMapping::to_tuples(¶ms.port); + + // Parse environment variables for init + let env = smolvm::util::parse_env_list(¶ms.env); + + // Validate every ref against the CLI trust scope before persisting. + // The CLI caller is `TrustedLocal`, so all source kinds are allowed, + // but structural rules (exactly one source, absolute from_file paths) + // still apply and catch Smolfile typos before they reach the DB. + for (name, r) in ¶ms.secret_refs { + smolvm::secrets::validate_ref(r, smolvm::secrets::ResolutionScope::TrustedLocal).map_err( + |e| smolvm::Error::config("create machine", format!("secret '{}': {}", name, e)), + )?; + } + + // Create record with restart policy if configured + let restart = smolvm::config::RestartConfig { + policy: params + .restart_policy + .clone() + .unwrap_or(smolvm::config::RestartPolicy::Never), + max_retries: params.restart_max_retries.unwrap_or(0), + max_backoff_secs: params.restart_max_backoff_secs.unwrap_or(0), + ..Default::default() + }; + let mut record = VmRecord::new_with_restart( + params.name.clone(), + params.cpus, + params.mem, + mounts, + ports, + params.net, + restart, + ); + record.init = params.init.clone(); + record.env = env; + record.secret_refs = params.secret_refs.clone(); + record.workdir = params.workdir.clone(); + record.storage_gb = params.storage_gb; + record.overlay_gb = params.overlay_gb; + record.allowed_cidrs = params.allowed_cidrs.clone(); + record.network_backend = params.network_backend; + record.dns = params.dns; + record.gpu = if params.gpu { Some(true) } else { None }; + record.rosetta = if params.rosetta { Some(true) } else { None }; + // Same invariant the CLI enforces, applied again here because + // Smolfile values arrive through `params.gpu_vram_mib` without + // passing through the clap value_parser. + record.gpu_vram_mib = smolvm::data::resources::validate_gpu_vram_mib(params.gpu_vram_mib) + .map_err(|e| smolvm::Error::config("create machine", format!("gpu_vram: {}", e)))?; + record.image = params.image.clone(); + record.entrypoint = params.entrypoint.clone(); + record.cmd = params.cmd.clone(); + record.health_cmd = params.health_cmd.clone(); + record.health_interval_secs = params.health_interval_secs; + record.health_timeout_secs = params.health_timeout_secs; + record.health_retries = params.health_retries; + record.health_startup_grace_secs = params.health_startup_grace_secs; + record.ssh_agent = params.ssh_agent; + record.cuda = params.cuda; + record.dns_filter_hosts = params.dns_filter_hosts.clone(); + record.source_smolmachine = params.source_smolmachine.clone(); + + Ok(record) +} + +pub(crate) fn print_create_success(params: &CreateVmParams) { + println!("Created machine: {}", params.name); + println!(" CPUs: {}, Memory: {} MiB", params.cpus, params.mem); + if !params.volume.is_empty() { + println!(" Mounts: {}", params.volume.len()); + } + if !params.port.is_empty() { + println!(" Ports: {}", params.port.len()); + } + if !params.init.is_empty() { + println!(" Init commands: {}", params.init.len()); + } + println!( + "\nUse 'smolvm machine start --name {}' to start the machine", + params.name + ); + println!( + "Then use 'smolvm machine exec --name {} -- ' to run commands", + params.name + ); +} + +// ============================================================================ +// Fork +// ============================================================================ + +/// Per-launch fork parameters, threaded into `start_vm_named` instead of mutating +/// process-global env vars. The launcher (in the spawned `_boot-vm`) reads these +/// from its own env, which the manager now sets explicitly on the child — so the +/// long-lived `serve` process never does a racy `std::env::set_var`. +#[derive(Debug, Default, Clone)] +pub struct ForkLaunch { + /// Start as a fork base: memfd-back guest RAM and expose `control_socket`. + pub forkable: bool, + /// Boot as a fork clone, restoring from the golden's snapshot at this dir. + pub snapshot_dir: Option, + /// Control socket path (set together with `forkable`). + pub control_socket: Option, +} + +/// Fork parameters for starting `name` as a forkable base (memfd RAM + a control +/// socket at the machine's known path), so `machine fork` can later freeze it. +pub fn forkable_launch(name: &str) -> ForkLaunch { + ForkLaunch { + forkable: true, + control_socket: Some(smolvm::agent::fork::control_socket_path(name)), + snapshot_dir: None, + } +} + +/// Fork a running, forkable `golden` machine into a new `clone`. +/// +/// Freezes the golden (it stays paused as the shared copy-on-write base — its +/// guest RAM is mapped `MAP_PRIVATE` by clones, so it must not run again while +/// clones exist), copy-on-write clones its disks, and boots the clone from the +/// golden's in-memory snapshot. +pub fn fork_vm( + golden: &str, + clone: &str, + clone_forkable: bool, + pinned_ports: &[(u16, u16)], +) -> smolvm::Result<()> { + let db = SmolvmDb::open()?; + + // Freeze + snapshot the golden, register the clone (CoW disks + DB record). + // The launch-agnostic mechanics live in the lib (`agent::fork`) so the CLI + // and the serve API share one implementation. + eprintln!("Freezing golden '{golden}' as fork base..."); + let prep = smolvm::agent::fork::prepare_fork(&db, golden, clone, pinned_ports, clone_forkable)?; + for (golden_host, guest, clone_host) in &prep.port_remaps { + if pinned_ports.is_empty() { + eprintln!( + " port {golden_host}->{guest} (golden) remapped to {clone_host}->{guest} (clone)" + ); + } else { + eprintln!(" port {clone_host}->{guest} (pinned)"); + } + } + + // Boot the clone from the golden's snapshot instead of cold-booting. + eprintln!("Booting clone '{clone}' from snapshot..."); + let result = start_vm_named( + clone, + None, + None, + /* from_snapshot */ true, + ForkLaunch { + snapshot_dir: Some(prep.snapshot_dir.clone()), + ..Default::default() + }, + ); + if result.is_ok() { + // Fresh on-disk identity (hostname, machine-id, SSH host keys, RNG). + // FAIL-CLOSED: if the reset can't be confirmed, stop the booted clone and + // roll it back rather than leave it live with the golden's secrets. + smolvm::agent::fork::fail_closed_on_rejuvenation( + smolvm::agent::fork::rejuvenate_clone(clone), + || { + if let Ok(manager) = AgentManager::for_vm(clone) { + manager.kill(); + manager.cleanup_data_dir(); + } + let _ = db.remove_vm(clone); + let _ = std::fs::remove_dir_all(vm_data_dir(clone)); + }, + )?; + eprintln!( + "Forked '{golden}' -> '{clone}'. Golden stays frozen as the fork base \ + (do not start it again while clones exist)." + ); + } else { + let _ = db.remove_vm(clone); + let _ = std::fs::remove_dir_all(vm_data_dir(clone)); + } + result +} + +// ============================================================================ +// Start +// ============================================================================ + +/// Resolve a machine's persistent-workload command, docker-style: use the +/// machine's own `(entrypoint, cmd)` if it set either of them, otherwise fall +/// back to the image's OCI `(entrypoint, cmd)`. Returns the `(entrypoint, cmd)` +/// to persist and run. An explicit machine command always wins over the image's. +pub(crate) fn default_workload_to_image( + record_entrypoint: Vec, + record_cmd: Vec, + image_entrypoint: &[String], + image_cmd: &[String], +) -> (Vec, Vec) { + if !record_entrypoint.is_empty() || !record_cmd.is_empty() { + (record_entrypoint, record_cmd) + } else { + (image_entrypoint.to_vec(), image_cmd.to_vec()) + } +} + +/// Start a named machine that has a config record. +/// +/// Uses direct DB operations instead of SmolvmConfig::load() to avoid +/// loading all config settings and all VM records. Only reads the single +/// named record (1 DB cycle) and updates it after start (1 DB cycle). +pub fn start_vm_named( + name: &str, + proxy: Option<&str>, + no_proxy: Option<&str>, + from_snapshot: bool, + fork: ForkLaunch, +) -> smolvm::Result<()> { + use smolvm::Error; + + // Direct DB lookup — 1 read cycle instead of loading everything + let db = SmolvmDb::open()?; + let mut record = db.get_vm(name)?.ok_or_else(|| Error::vm_not_found(name))?; + + // Resolve via the shared probe (PID + vsock ping). The plain + // `actual_state()` is PID-only and would treat a zombie VMM + // (alive process, dead agent) as Running — exactly the bug 2 + // case where `start` later said "already running" but every + // `exec` failed. + match smolvm::agent::state_probe::resolve_state(name, &record) { + RecordState::Running => { + let pid_suffix = format_pid_suffix(record.pid); + println!("Machine '{}' already running{}", name, pid_suffix); + return Ok(()); + } + RecordState::Unreachable => { + // Zombie VMM: kill it, clear the record, fall through to + // a clean fresh start. + cli_recover_if_unreachable(name); + } + RecordState::Frozen => { + // Snapshot-frozen fork base: relaunching it writable would + // corrupt the clones whose disks CoW-back onto it. Refuse — + // delete the clones first to reuse the name. + let clones = db.dependent_clones(name).unwrap_or_default(); + return Err(Error::agent( + "start", + format!( + "'{name}' is the fork base for {} live clone(s) ({}); their disks are \ + copy-on-write overlays backed by its disks, so it cannot be re-launched \ + while they exist — delete the clones first", + clones.len(), + clones.join(", ") + ), + )); + } + RecordState::Stopped | RecordState::Created | RecordState::Failed => { + // Normal start path. Kill any orphaned _boot-vm process left by + // a previous failed start — if one is holding ports/sockets, this + // fresh start would hit the same error without this cleanup. + kill_orphaned_boot_process(name); + } + } + + let mounts = record.host_mounts(); + let ports = record.port_mappings(); + let mut resources = record.vm_resources(); + + // Re-resolve allow_hosts to fresh CIDRs at start time. + // Hostnames for CDN-backed services (e.g. dl-cdn.alpinelinux.org) rotate + // IPs — storing resolved CIDRs at `machine create` time means the egress + // policy goes stale. We re-resolve here so the policy always reflects + // current DNS, then merge with any explicit allow_cidrs stored in the DB. + // + // IMPORTANT: always initialize allowed_cidrs (even to an empty Vec) when + // dns_filter_hosts is set. This ensures launcher.rs always calls + // krun_set_egress_policy, even when every hostname fails to resolve. + // Without this, a DNS outage at start time causes the VM to boot with no + // egress restriction at all (fail-open). With it, the policy starts as + // deny-all and launcher.rs's ensure_dns_in_cidrs adds 1.1.1.1/32 as the + // minimum (fail-closed: only DNS reachable until resolution succeeds). + if let Some(ref hosts) = record.dns_filter_hosts { + if !hosts.is_empty() { + let existing = resources.allowed_cidrs.get_or_insert_with(Vec::new); + for host in hosts { + match crate::cli::parsers::resolve_host_to_cidrs(host) { + Ok(cidrs) => existing.extend(cidrs), + Err(e) => eprintln!( + "Warning: could not resolve '{}' for egress policy: {}", + host, e + ), + } + } + } + } + + // Check for host port conflicts with other running VMs. + if !ports.is_empty() { + check_port_conflicts(name, &ports, &db)?; + } + + // Start agent VM + let manager = AgentManager::for_vm_with_sizes(name, record.storage_gb, record.overlay_gb) + .map_err(|e| Error::agent("create agent manager", e.to_string()))?; + + let mount_info = if !mounts.is_empty() { + format!(" with {} mount(s)", mounts.len()) + } else { + String::new() + }; + let port_info = if !ports.is_empty() { + format!(" and {} port mapping(s)", ports.len()) + } else { + String::new() + }; + eprintln!("Starting machine '{}'{}{}...", name, mount_info, port_info); + + // Resolve SSH agent socket path if enabled + let ssh_agent_socket = if record.ssh_agent { + match std::env::var("SSH_AUTH_SOCK") { + Ok(path) => Some(std::path::PathBuf::from(path)), + Err(_) => { + return Err(Error::config( + "ssh-agent", + "SSH_AUTH_SOCK is not set. Start an SSH agent with: eval $(ssh-agent) && ssh-add", + )); + } + } + } else { + None + }; + + // If the machine was created from a .smolmachine, this acquires a lease on + // the machine's own pre-extracted layers (extracted at create time) and sets + // packed_layers_dir so the launcher mounts them via virtiofs — the guest uses + // the pre-extracted layers instead of pulling, with no dependency on the + // original bundle file. Shared with the API and embedded start paths. + let mut features = smolvm::agent::LaunchFeatures { + ssh_agent_socket, + cuda: record.cuda, + dns_filter_hosts: record.dns_filter_hosts.clone(), + ..Default::default() + } + .with_packed_layers( + &smolvm::agent::machine_layers_cache_dir(name), + record.source_smolmachine.as_deref(), + )?; + // Fork params (forkable base / clone-from-snapshot) — carried per-launch into + // the boot subprocess's env by the manager, not via process-global env vars. + features.forkable = fork.forkable; + features.snapshot_dir = fork.snapshot_dir; + features.control_socket = fork.control_socket; + // A machine created from a local image archive/dir persists a `local:…` + // reference; re-derive its virtiofs mount dir so the guest assembles the + // rootfs from it instead of pulling. + if features.packed_layers_dir.is_none() { + if let Some(dir) = record + .image + .as_deref() + .and_then(smolvm::data::image_source::packed_layers_dir_for_ref) + { + features.packed_layers_dir = Some(dir); + } + } + + // First boot pulls the base image in-guest, subject to the egress filter — + // fold the image's registry into the enforced policy so a hostname scope + // doesn't block its own pull. Subsequent starts skip the pull, so they keep + // the user's scope unwidened. + if !record.init_completed { + features.allow_image_pull_egress( + record.image.as_deref(), + features.packed_layers_dir.is_some(), + ); + } + + let _ = manager + .ensure_running_with_full_config(mounts, ports, resources, features) + .map_err(|e| Error::agent("start machine", e.to_string()))?; + + // Get PID immediately (cheap) and print output before DB write + let pid = manager.child_pid(); + + // Install SIGINT guard so Ctrl+C during init/pull kills the VM process + // instead of orphaning it. Disarmed before detach. + let _sigint_guard = pid.map(smolvm::process::SigintGuard::new); + + // Pull image first (if configured), then run init. Init can + // target the container's rootfs (via `run_non_interactive`) when + // an image is set, so the container layers must be in place + // before init runs — otherwise any init command referencing the + // image's filesystem (package managers, distro-specific paths) + // would hit the bare Alpine agent and fail with "not found". + let mut client = smolvm::agent::AgentClient::connect_with_retry(manager.vsock_socket())?; + + // Resolve secret refs to plaintext on the host and inject them only into + // the env handed to guest commands (init + workload entrypoint). Only the + // refs persist on the record/DB; the resolved plaintext lives in `exec_env` + // for the duration of this call and never leaves the host or reaches the DB. + let exec_env = record_env_with_secrets(&record)?; + + // On first boot, pull the image and run init commands. On subsequent + // starts, skip both — image manifests/layers persist on the storage disk + // and the container overlay is remounted (not recreated). + if !record.init_completed { + let uses_packed_layers = record.source_smolmachine.is_some() + || record + .image + .as_deref() + .is_some_and(smolvm::data::image_source::is_local_ref); + let image_info = if uses_packed_layers { + // Layers already mounted via virtiofs — no pull needed. + None + } else if let Some(ref image) = record.image { + eprintln!("Pulling {}...", image); + Some(crate::cli::pull_with_progress( + &mut client, + image, + None, + proxy, + no_proxy, + )?) + } else { + None + }; + + if let Err(e) = run_init_commands( + &mut client, + &record.init, + InitRunContext { + image: record.image.as_deref(), + image_info: image_info.as_ref(), + env: &exec_env, + workdir: record.workdir.as_deref(), + record_mounts: &record.mounts, + overlay_id: name, + }, + ) { + if let Err(stop_err) = manager.stop() { + tracing::warn!(error = %stop_err, "failed to stop machine after init failure"); + } + return Err(e); + } + + // Docker-like default: if the machine has no command of its own, adopt + // the image's OCI entrypoint+cmd as its persistent workload so an image + // VM runs its image's program on start (and forked clones inherit it). + // Persisted here on first boot (where the image config is available) so + // later starts reuse it without re-pulling. + if let Some(info) = image_info.as_ref() { + let (ep, cmd) = default_workload_to_image( + record.entrypoint.clone(), + record.cmd.clone(), + &info.entrypoint, + &info.cmd, + ); + if ep != record.entrypoint || cmd != record.cmd { + record.entrypoint = ep.clone(); + record.cmd = cmd.clone(); + let _ = db.update_vm(name, |r| { + r.entrypoint = ep.clone(); + r.cmd = cmd.clone(); + }); + } + } + + // Mark init as completed so subsequent starts skip pull + init. + // Done before workload start so a CMD failure doesn't re-trigger init. + if !record.init.is_empty() || record.image.is_some() { + let _ = db.update_vm(name, |r| { + r.init_completed = true; + }); + } + } else if !record.init.is_empty() { + println!( + "Init already completed, skipping {} command(s)", + record.init.len() + ); + } + + if let Some(ref img) = record.image { + // Image-based machine: launch the workload container in the background. + // An empty command → the agent resolves the image's own ENTRYPOINT+CMD, + // so service-style images start as their authors intended. But a clone + // booted from a fork snapshot already has the golden's workload running + // in its restored memory; relaunching here would double-manage the crun + // container and hang every later `machine exec`, so skip the (re)launch + // entirely when restoring from a snapshot — the forked container is + // inherited as-is. + let mut cmd = record.entrypoint.clone(); + cmd.extend(record.cmd.clone()); + if !from_snapshot { + let mount_bindings = + crate::cli::parsers::record_mounts_to_runconfig_bindings(&record.mounts); + let bg_config = smolvm::agent::RunConfig::new(img, cmd) + .with_env(exec_env.clone()) + .with_workdir(record.workdir.clone()) + .with_user(record.user.clone()) + .with_mounts(mount_bindings) + .with_persistent_overlay(Some(name.to_string())); + if let Err(e) = client.run_container_detached(bg_config) { + if let Err(stop_err) = manager.stop() { + tracing::warn!(error = %stop_err, "failed to stop machine after CMD launch failure"); + } + return Err(smolvm::Error::agent("start background CMD", format!("{e}"))); + } + } else { + tracing::info!( + "clone booted from snapshot: workload container inherited from fork, skipping relaunch" + ); + } + println!("Machine '{}' running (PID: {})", name, pid.unwrap_or(0)); + } else { + // No image — bare VM mode. Run entrypoint+cmd if configured. As with the + // image branch, a snapshot-restored clone already ran this on the golden + // (its effects are in the restored memory/disk), so don't re-run it. + let mut bare_cmd = record.entrypoint.clone(); + bare_cmd.extend(record.cmd.clone()); + if !bare_cmd.is_empty() && !from_snapshot { + // Reuse the secrets already resolved into `exec_env` above — avoids + // a second store load + decrypt and a duplicate audit-log record. + // The plaintext stays in this vector and never touches the record/DB. + let (exit_code, stdout, stderr) = + client.vm_exec(bare_cmd, exec_env, record.workdir.clone(), None, None)?; + if !stdout.is_empty() { + let _ = std::io::stdout().write_all(&stdout); + } + if !stderr.is_empty() { + let _ = std::io::stderr().write_all(&stderr); + } + if exit_code != 0 { + eprintln!("workload exited with code {}", exit_code); + } + } + println!("Machine '{}' running (PID: {})", name, pid.unwrap_or(0)); + } + + // Persist running state. The 15s busy_timeout handles SQLite contention + // from concurrent starts — no application-level retry needed. + let pid_start_time = pid.and_then(smolvm::process::process_start_time); + if let Err(e) = db.update_vm(name, |r| { + r.state = RecordState::Running; + r.pid = pid; + r.pid_start_time = pid_start_time; + }) { + tracing::warn!(error = %e, vm = %name, "failed to persist running state"); + } + + // Keep VM running (persistent) + manager.detach(); + Ok(()) +} + +/// Persist a named VM as running in the database. +/// +/// Creates the record if it doesn't exist, then updates state to Running +/// with the current PID and optional config overrides (cpus, mem, etc.). +pub fn persist_named_running( + config: &mut SmolvmConfig, + name: &str, + pid: Option, + overrides: Option, +) -> smolvm::Result<()> { + if config.get_vm(name).is_none() { + let record = VmRecord::new( + name.to_string(), + DEFAULT_MICROVM_CPU_COUNT, + DEFAULT_MICROVM_MEMORY_MIB, + vec![], + vec![], + false, + ); + config.insert_vm(name.to_string(), record)?; + } + let pid_start_time = pid.and_then(smolvm::process::process_start_time); + config + .update_vm(name, |r| { + r.state = RecordState::Running; + r.pid = pid; + r.pid_start_time = pid_start_time; + if let Some(ref o) = overrides { + r.cpus = o.cpus; + r.mem = o.mem; + r.mounts = o.mounts.clone(); + r.ports = o.ports.clone(); + r.network = o.network; + r.network_backend = o.network_backend; + r.dns = o.dns; + r.storage_gb = o.storage_gb; + r.overlay_gb = o.overlay_gb; + r.allowed_cidrs = o.allowed_cidrs.clone(); + r.init = o.init.clone(); + r.init_completed = false; + r.env = o.env.clone(); + r.secret_refs = o.secret_refs.clone(); + r.workdir = o.workdir.clone(); + r.user = o.user.clone(); + r.image = o.image.clone(); + r.entrypoint = o.entrypoint.clone(); + r.cmd = o.cmd.clone(); + r.ssh_agent = o.ssh_agent; + r.cuda = o.cuda; + r.dns_filter_hosts = o.dns_filter_hosts.clone(); + r.gpu = if o.gpu { Some(true) } else { None }; + r.gpu_vram_mib = o.gpu_vram_mib; + r.rosetta = if o.rosetta { Some(true) } else { None }; + } + }) + .ok_or_else(|| smolvm::Error::config( + "persist machine record", + format!("VM record for '{}' missing after insert", name), + ))? + // Flatten: Option> → the ok_or_else above handles None, + // now propagate the inner Result (DB write failure). + ?; + Ok(()) +} + +/// Config overrides for a VM record. +pub struct DefaultVmOverrides { + pub cpus: u8, + pub mem: u32, + pub mounts: Vec<(String, String, bool)>, + pub ports: Vec<(u16, u16)>, + pub network: bool, + pub network_backend: Option, + pub dns: Option, + pub storage_gb: Option, + pub overlay_gb: Option, + pub allowed_cidrs: Option>, + pub init: Vec, + pub env: Vec<(String, String)>, + pub secret_refs: BTreeMap, + pub workdir: Option, + pub user: Option, + pub image: Option, + pub entrypoint: Vec, + pub cmd: Vec, + pub ssh_agent: bool, + pub cuda: bool, + pub dns_filter_hosts: Option>, + pub gpu: bool, + pub gpu_vram_mib: Option, + pub rosetta: bool, +} + +/// Check if any running VM already binds to the same host ports. +/// +/// Iterates all VM records, skipping the current VM (`self_name`), and checks +/// for host port overlaps with running VMs. This prevents silent port binding +/// failures where two VMs claim the same host port but only one succeeds. +fn check_port_conflicts( + self_name: &str, + ports: &[PortMapping], + db: &SmolvmDb, +) -> smolvm::Result<()> { + let host_ports: std::collections::HashSet = ports.iter().map(|p| p.host).collect(); + if host_ports.is_empty() { + return Ok(()); + } + + let all_vms = db.list_vms()?; + for (name, record) in &all_vms { + if name == self_name { + continue; + } + // Only check running VMs (PID-based quick check). + if record.actual_state() != smolvm::config::RecordState::Running { + continue; + } + for (host, _guest) in &record.ports { + if host_ports.contains(host) { + return Err(smolvm::Error::config( + "start machine", + format!( + "host port {} is already in use by running machine '{}'", + host, name + ), + )); + } + } + } + Ok(()) +} + +/// Start the default machine. +pub fn start_vm_default(proxy: Option<&str>, no_proxy: Option<&str>) -> smolvm::Result<()> { + let manager = AgentManager::new_default()?; + + if manager.try_connect_existing().is_some() { + let pid_suffix = format_pid_suffix(manager.child_pid()); + println!("Machine 'default' already running{}", pid_suffix); + manager.detach(); + return Ok(()); + } + + // try_connect_existing failed — could be "really stopped" or + // "zombie VMM with dead agent". Recover the zombie case before + // starting fresh; no-op otherwise. + cli_recover_if_unreachable("default"); + + eprintln!("Starting machine 'default'..."); + manager.ensure_running()?; + + let mut config = SmolvmConfig::load()?; + persist_named_running(&mut config, "default", manager.child_pid(), None)?; + + // Pull image (if persisted via `machine run -d -s`) before running + // init, then run init through the shared runner — same fix as + // `start_vm_named`. Both paths must agree so an init that works on + // a named machine also works on the default one. + let record = config.get_vm("default").cloned(); + + if let Some(record) = record { + let needs_pull = record.image.is_some(); + let needs_init = !record.init.is_empty(); + + if needs_pull || needs_init { + let mut client = + smolvm::agent::AgentClient::connect_with_retry(manager.vsock_socket())?; + + // Resolve secret refs to plaintext on the host for init only; refs + // (not values) persist on the record, the plaintext stays here. + let exec_env = record_env_with_secrets(&record)?; + + let image_info = if let Some(ref image) = record.image { + eprintln!("Pulling {}...", image); + Some(crate::cli::pull_with_progress( + &mut client, + image, + None, + proxy, + no_proxy, + )?) + } else { + None + }; + + if let Err(e) = run_init_commands( + &mut client, + &record.init, + InitRunContext { + image: record.image.as_deref(), + image_info: image_info.as_ref(), + env: &exec_env, + workdir: record.workdir.as_deref(), + record_mounts: &record.mounts, + overlay_id: "default", + }, + ) { + if let Err(stop_err) = manager.stop() { + tracing::warn!(error = %stop_err, "failed to stop machine after init failure"); + } + return Err(e); + } + } + } + + println!( + "Machine 'default' running (PID: {})", + manager.child_pid().unwrap_or(0) + ); + + manager.detach(); + Ok(()) +} + +// ============================================================================ +// Stop +// ============================================================================ + +/// Stop a named machine that has a config record (or fall back to +/// agent-only stop if the name is not in config). +pub fn stop_vm_named(name: &str) -> smolvm::Result<()> { + let mut config = SmolvmConfig::load()?; + + // Check config for the named VM + let record = match config.get_vm(name) { + Some(r) => r.clone(), + None => { + // Not in config — try to stop a running VM with this name directly + let manager = AgentManager::for_vm(name)?; + if manager.try_connect_existing().is_some() { + println!("Stopping machine '{}'...", name); + manager.stop()?; + println!("Machine '{}' stopped", name); + return Ok(()); + } + // Not in config and no running VM with this name — genuinely + // not found. Exit non-zero, consistent with status/start/delete. + return Err(smolvm::Error::vm_not_found(name)); + } + }; + + // Resolve via the shared probe so an `Unreachable` VM (live PID, + // dead agent) is correctly stopped instead of skipped with a + // misleading "not running" message. `cli_recover_if_unreachable` + // handles that case by killing the zombie VMM; after it runs the + // record is `Stopped` and `manager.stop()` becomes a no-op. + let resolved = smolvm::agent::state_probe::resolve_state(name, &record); + match resolved { + RecordState::Unreachable => { + cli_recover_if_unreachable(name); + // Process is gone — detach the layers volume so a non-running + // machine never holds a mount (invariant: mounted iff running). + // macOS hdiutil detach; a no-op on Linux. + if record.source_smolmachine.is_some() { + smolvm_pack::extract::force_detach_layers_volume( + &smolvm::agent::machine_layers_cache_dir(name), + ); + } + println!("Stopped machine: {}", name); + return Ok(()); + } + RecordState::Running => { + // fall through to the normal stop path + } + RecordState::Frozen => { + // Snapshot-frozen fork base: it must outlive its clones (their + // disks CoW-back onto its disks). Refuse instead of tearing it + // down — mirrors `delete`'s guard. + let clones = SmolvmDb::open()?.dependent_clones(name).unwrap_or_default(); + return Err(smolvm::Error::agent( + "stop", + format!( + "machine '{name}' is the fork base for {} live clone(s) ({}); \ + stop or delete the clones first", + clones.len(), + clones.join(", ") + ), + )); + } + other => { + // Not running. If a prior start mounted the layers volume but the + // VM failed to boot, it could still be mounted — detach it. Safe: + // resolve_state() probed liveness, so the process is confirmed dead. + // macOS hdiutil detach; a no-op on Linux. + if record.source_smolmachine.is_some() { + smolvm_pack::extract::force_detach_layers_volume( + &smolvm::agent::machine_layers_cache_dir(name), + ); + } + // Defense-in-depth: a failed `machine start` may have left an + // orphaned _boot-vm process that the DB doesn't know about (PID + // never persisted). Check the on-disk PID file and, if the + // process is alive and its argv matches this VM's boot config + // path, kill it so the user doesn't have to hunt it manually. + kill_orphaned_boot_process(name); + println!("Machine '{}' is not running (state: {})", name, other); + return Ok(()); + } + } + + println!("Stopping machine '{}'...", name); + + let manager = AgentManager::for_vm(name) + .map_err(|e| smolvm::Error::agent("create agent manager", e.to_string()))?; + manager.stop()?; + + // Detach the machine's case-sensitive layers volume now that its process is + // gone (macOS hdiutil mount; no-op on Linux). The volume is owned 1:1 by this + // machine, so the detach is safe and re-acquired on the next start. + if record.source_smolmachine.is_some() { + smolvm_pack::extract::force_detach_layers_volume(&smolvm::agent::machine_layers_cache_dir( + name, + )); + } + + config.update_vm(name, |r| { + r.state = RecordState::Stopped; + r.pid = None; + r.pid_start_time = None; + }); + + println!("Stopped machine: {}", name); + Ok(()) +} + +/// Kill an orphaned `_boot-vm` process for a machine that the DB thinks is +/// not running. +/// +/// When `machine start` fails after spawning the `_boot-vm` child (e.g., +/// EADDRINUSE in the virtio-net runtime), `finalize_launch` kills the child +/// with SIGTERM + SIGKILL. But if that cleanup itself fails (or a pre-fix +/// binary left an orphan), this fallback catches it: read the on-disk PID +/// file, verify the process argv matches this VM's boot-config path, and +/// kill it. Best-effort — never fails the caller. +fn kill_orphaned_boot_process(name: &str) { + let data_dir = smolvm::agent::vm_data_dir(name); + let pid_file = data_dir.join("agent.pid"); + let boot_config = data_dir.join("boot-config.json"); + + let content = match std::fs::read_to_string(&pid_file) { + Ok(c) => c, + Err(_) => return, + }; + let pid: i32 = match content.lines().next().and_then(|l| l.trim().parse().ok()) { + Some(p) => p, + None => return, + }; + + if !smolvm::process::is_alive(pid) { + // Stale PID file — clean it up. + let _ = std::fs::remove_file(&pid_file); + return; + } + + // Verify the process is actually our _boot-vm for this VM (not a + // recycled PID belonging to something else). + if !smolvm::process::cmdline_contains(pid, &boot_config.to_string_lossy()) { + return; + } + + eprintln!( + "Killing orphaned _boot-vm process (PID {}) for machine '{}'", + pid, name + ); + if let Err(e) = smolvm::process::stop_vm_process( + pid, + std::time::Duration::from_secs(2), + smolvm::process::VM_SIGKILL_TIMEOUT, + ) { + tracing::warn!( + pid, error = %e, + "failed to kill orphaned _boot-vm process" + ); + } + let _ = std::fs::remove_file(&pid_file); +} + +/// Stop the default machine. +pub fn stop_vm_default() -> smolvm::Result<()> { + let manager = AgentManager::new_default()?; + + // try_connect_existing sets internal state if agent is reachable; + // stop() handles both responsive agents and orphans via PID file. + manager.try_connect_existing(); + println!("Stopping machine 'default'..."); + manager.stop()?; + + // Update database record if it exists + if let Ok(mut config) = SmolvmConfig::load() { + // Detach the per-machine layers volume now that the process is gone, so a + // bundle-sourced default machine never holds a mount while stopped (macOS + // hdiutil; no-op on Linux). Gated on the record so non-bundle machines are + // untouched; the volume is owned 1:1 by "default" and re-acquired on start. + let is_bundle = config + .get_vm("default") + .map(|r| r.source_smolmachine.is_some()) + .unwrap_or(false); + if is_bundle { + smolvm_pack::extract::force_detach_layers_volume( + &smolvm::agent::machine_layers_cache_dir("default"), + ); + } + config.update_vm("default", |r| { + r.state = RecordState::Stopped; + r.pid = None; + r.pid_start_time = None; + }); + } + + println!("Machine 'default' stopped"); + + Ok(()) +} + +// ============================================================================ +// Delete +// ============================================================================ + +/// Options for machine delete behavior. +pub struct DeleteVmOptions { + /// If true, stop the VM before deleting when it is running. + pub stop_if_running: bool, +} + +/// Delete a named machine configuration. +pub fn delete_vm(name: &str, force: bool, options: DeleteVmOptions) -> smolvm::Result<()> { + let mut config = SmolvmConfig::load()?; + + // Check if exists + let record = config + .get_vm(name) + .ok_or_else(|| smolvm::Error::vm_not_found(name))? + .clone(); + + // A golden's disks are the copy-on-write backing for its clones' overlays, + // so it must outlive them. Refuse to delete a golden while clones depend on + // it (unless forced, in which case the clones' overlays are left dangling). + let dependent_clones = SmolvmDb::open()?.dependent_clones(name)?; + if !dependent_clones.is_empty() { + if !force { + return Err(smolvm::Error::agent( + "delete", + format!( + "machine '{name}' is the fork base for {} clone(s) ({}); \ + delete the clones first, or use --force to break them", + dependent_clones.len(), + dependent_clones.join(", ") + ), + )); + } + tracing::warn!( + golden = name, + clones = %dependent_clones.join(", "), + "force-deleting a golden; dependent clones' disk overlays will dangle" + ); + } + + // Stop if running (machine run does this). Use the shared + // resolver so an `Unreachable` VM (live PID, dead agent) is also + // torn down — otherwise the record gets deleted while the zombie + // libkrun process keeps running, orphaned forever. + if options.stop_if_running { + match smolvm::agent::state_probe::resolve_state(name, &record) { + RecordState::Running => { + if let Ok(manager) = AgentManager::for_vm(name) { + println!("Stopping machine '{}'...", name); + if let Err(e) = manager.stop() { + tracing::warn!(error = %e, "failed to stop machine"); + } + } + } + RecordState::Unreachable => { + // Reap unconditionally: we're past the dependent-clones + // guard above, so either this isn't a fork base or --force + // was given. The guarded `cli_recover_if_unreachable` would + // skip a frozen fork base, orphaning its VMM after we remove + // the record below. + smolvm::agent::state_probe::recover_unreachable_machine(&record); + } + RecordState::Frozen => { + // A frozen fork base only reaches here under --force (the + // dependent-clones guard above blocks the non-force path). + // Reap its paused VMM so it isn't orphaned once the record + // is removed; the clones' overlays are left dangling, as + // the force-delete warning already states. + smolvm::agent::state_probe::recover_unreachable_machine(&record); + } + _ => {} + } + } + + // Confirm deletion unless --force + if !force { + eprint!("Delete machine '{}'? [y/N] ", name); + let mut input = String::new(); + if std::io::stdin().read_line(&mut input).is_ok() { + let input = input.trim().to_lowercase(); + if input != "y" && input != "yes" { + println!("Cancelled"); + return Ok(()); + } + } else { + println!("Cancelled"); + return Ok(()); + } + } + + // Remove from config (persists immediately to database) + config.remove_vm(name); + + // If the machine was created from a .smolmachine, detach its case-sensitive + // layers volume (macOS hdiutil mount; no-op on Linux) before removing the + // data dir below — otherwise the `rm -rf` fails with "Resource busy". The + // lease was intentionally leaked with `std::mem::forget` at start time so the + // volume stayed mounted while the VM ran. The volume lives under this + // machine's own data dir and is owned 1:1 by it, so the detach is + // unconditional and cannot affect any other machine. + if record.source_smolmachine.is_some() { + smolvm_pack::extract::force_detach_layers_volume(&smolvm::agent::machine_layers_cache_dir( + name, + )); + } + + let data_dir = vm_data_dir(name); + if data_dir.exists() { + println!("Cleaning up data directory for vm: {}", name); + // Release this VM's per-VM uid (if any) before the dir holding its + // `.vm-uid` record is removed. See process::free_vm_uid. + smolvm::process::free_vm_uid(&smolvm::agent::vm_uid_registry_dir(), &data_dir); + if let Err(e) = std::fs::remove_dir_all(&data_dir) { + tracing::warn!(error = %e, "Failed to remove VM data directory: {}", data_dir.display()); + } + } + + // The VM's readiness marker lives in the *shared* agent rootfs, not its data + // dir, so the removal above doesn't take it. Sweep it (and any other markers + // orphaned by a crash/kill) now that this VM's data dir is gone, so the + // rootfs doesn't accumulate stale markers (which also broke `pack create`). + smolvm::agent::prune_orphaned_ready_markers(); + + println!("Deleted machine: {}", name); + Ok(()) +} + +// ============================================================================ +// Status +// ============================================================================ + +/// Show status of a named or default machine. +/// +/// The `extra` callback is invoked when the VM is running, allowing callers +/// to display additional information (e.g., machine lists containers). +pub fn status_vm(name: &Option, extra: F) -> smolvm::Result<()> +where + F: FnOnce(&AgentManager), +{ + let label = vm_label(name); + + // A frozen fork base's paused agent never answers; connecting to it + // would block for the full vsock timeout (the `machine status` hang). + // Detect it cheaply from the record (no probe) and report Frozen + // without attempting a connection. + if let Some(n) = name { + if let Some(record) = SmolvmDb::open() + .ok() + .and_then(|db| db.get_vm(n).ok().flatten()) + { + if smolvm::agent::state_probe::is_frozen_fork_base(n, &record) { + println!("Machine '{}': {}", label, RecordState::Frozen); + return Ok(()); + } + } + } + + let manager = get_vm_manager(name)?; + + if manager.try_connect_existing().is_some() { + let pid_suffix = crate::cli::format_pid_suffix(manager.child_pid()); + println!("Machine '{}': running{}", label, pid_suffix); + extra(&manager); + manager.detach(); + } else if let Some(ref n) = name { + // Agent not reachable. Report the precise state from the registry + // (stopped / failed / created / unreachable), consistent with + // `machine list`, instead of a flat "not running". A missing record + // with an explicit name is "not found". + match SmolvmDb::open() + .ok() + .and_then(|db| db.get_vm(n).ok().flatten()) + { + Some(record) => { + let state = smolvm::agent::state_probe::resolve_state(n, &record); + println!("Machine '{}': {}", label, state); + } + None => return Err(smolvm::Error::vm_not_found(n)), + } + } else { + // Default/unnamed VM: no record to resolve. + println!("Machine '{}': not running", label); + } + + Ok(()) +} + +/// Build the per-machine JSON object shared by `machine list --json` and +/// `machine status --json` so the two outputs never drift apart. +fn machine_status_json(name: &str, record: &VmRecord) -> serde_json::Value { + // Resolve via vsock probe so the JSON reflects truth (Unreachable vs + // Running) instead of trusting the PID-only check. + let actual_state = smolvm::agent::state_probe::resolve_state(name, record); + // Expose the persisted health command as a single shell-friendly string + // when it was stored as `["sh", "-c", ""]`; otherwise a space-joined + // argv so the field is always a string. + let health_cmd_str = record.health_cmd.as_ref().map(|argv| { + if argv.len() == 3 && argv[0] == "sh" && argv[1] == "-c" { + argv[2].clone() + } else { + argv.join(" ") + } + }); + + let mut obj = serde_json::json!({ + "name": name, + "state": actual_state.to_string(), + "cpus": record.cpus, + "memory_mib": record.mem, + "pid": record.pid, + "mounts": record.mounts.len(), + "ports": record.ports.len(), + "created_at": record.created_at, + "storage_gb": record.storage_gb, + "overlay_gb": record.overlay_gb, + "image": record.image, + "entrypoint": record.entrypoint, + "cmd": record.cmd, + "ephemeral": record.ephemeral, + "gpu": record.gpu.unwrap_or(false), + "gpu_vram_mib": record.gpu_vram_mib, + "restart_policy": record.restart.policy.to_string(), + "restart_max_retries": record.restart.max_retries, + "restart_count": record.restart.restart_count, + "health_cmd": health_cmd_str, + "health_interval_secs": record.health_interval_secs, + "health_timeout_secs": record.health_timeout_secs, + "health_retries": record.health_retries, + "health_startup_grace_secs": record.health_startup_grace_secs, + }); + obj.as_object_mut() + .unwrap() + .insert("network".into(), serde_json::json!(record.network)); + obj +} + +/// Emit a single machine's status as JSON — the same object shape as +/// `machine list --json`. Errors if the machine does not exist. +pub fn status_vm_json(name: &Option) -> smolvm::Result<()> { + let label = vm_label(name); + let config = SmolvmConfig::load()?; + // Build the owned JSON value inside the match so the borrow of `config` + // ends before `config` is dropped. + let obj = match config.list_vms().find(|(n, _)| *n == &label) { + Some((_, record)) => machine_status_json(&label, record), + None => { + return Err(smolvm::Error::config( + "machine status", + format!("machine '{}' not found", label), + )) + } + }; + let json = serde_json::to_string_pretty(&obj) + .map_err(|e| smolvm::Error::config("serialize json", e.to_string()))?; + println!("{}", json); + Ok(()) +} + +// ============================================================================ +// List +// ============================================================================ + +/// List all machines. +pub fn list_vms(verbose: bool, json: bool) -> smolvm::Result<()> { + let config = SmolvmConfig::load()?; + let vms: Vec<_> = config.list_vms().collect(); + + let empty_label = "No machines found"; + + if vms.is_empty() { + if !json { + println!("{}", empty_label); + } else { + println!("[]"); + } + return Ok(()); + } + + if json { + let json_vms: Vec<_> = vms + .iter() + .map(|(name, record)| machine_status_json(name, record)) + .collect(); + let json = serde_json::to_string_pretty(&json_vms) + .map_err(|e| smolvm::Error::config("serialize json", e.to_string()))?; + println!("{}", json); + } else { + println!( + "{:<20} {:<12} {:>5} {:>10} {:>7} {:>7} {:>8} {:>8}", + "NAME", "STATE", "CPUS", "MEMORY", "MOUNTS", "PORTS", "STORAGE", "OVERLAY" + ); + println!("{}", "-".repeat(88)); + + for (name, record) in vms { + let actual_state = smolvm::agent::state_probe::resolve_state(name, record); + let state_display = if record.ephemeral { + format!("{} (eph)", actual_state) + } else { + actual_state.to_string() + }; + let storage_gb = record.storage_gb.unwrap_or(DEFAULT_STORAGE_SIZE_GIB); + let overlay_gb = record.overlay_gb.unwrap_or(DEFAULT_OVERLAY_SIZE_GIB); + println!( + "{:<20} {:<12} {:>5} {:>10} {:>7} {:>7} {:>8} {:>8}", + truncate(name, 18), + state_display, + record.cpus, + format!("{} MiB", record.mem), + record.mounts.len(), + record.ports.len(), + format!("{} GiB", storage_gb), + format!("{} GiB", overlay_gb), + ); + + if verbose { + if let Some(pid) = record.pid { + println!(" PID: {}", pid); + } + for (host, guest, ro) in &record.mounts { + let ro_str = if *ro { " (ro)" } else { "" }; + println!(" Mount: {} -> {}{}", host, guest, ro_str); + } + for (host, guest) in &record.ports { + println!(" Port: {} -> {}", host, guest); + } + if record.network { + println!(" Network: enabled"); + } + if record.gpu.unwrap_or(false) { + match record.gpu_vram_mib { + Some(vram) => println!(" GPU: enabled ({} MiB VRAM)", vram), + None => println!(" GPU: enabled"), + } + } + for cmd in &record.init { + println!(" Init: {}", cmd); + } + for (k, v) in &record.env { + println!(" Env: {}={}", k, v); + } + if let Some(wd) = &record.workdir { + println!(" Workdir: {}", wd); + } + let created = + std::time::UNIX_EPOCH + std::time::Duration::from_secs(record.created_at); + println!(" Created: {}", humantime::format_rfc3339_seconds(created)); + println!(); + } + } + } + + Ok(()) +} + +// ============================================================================ +// Resize +// ============================================================================ + +/// Resize a microVM's disk resources. +/// +/// The VM must be stopped before resizing. Only expansion is supported +/// (no shrinking to prevent data loss). +/// Expand physical disk files for a VM. Does NOT update the DB record — +/// the caller is responsible for persisting the new sizes. +/// +/// Returns a list of human-readable change descriptions for display. +/// Validates no-shrink and performs the physical I/O. +pub fn expand_disks( + name: &str, + record: &smolvm::config::VmRecord, + new_storage_gb: Option, + new_overlay_gb: Option, +) -> smolvm::Result> { + use smolvm::data::disk::{Overlay, Storage}; + use smolvm::storage::{expand_disk, DEFAULT_OVERLAY_SIZE_GIB, DEFAULT_STORAGE_SIZE_GIB}; + + let current_storage_gb = record.storage_gb.unwrap_or(DEFAULT_STORAGE_SIZE_GIB); + let current_overlay_gb = record.overlay_gb.unwrap_or(DEFAULT_OVERLAY_SIZE_GIB); + + // Validate no shrinking + if let Some(s) = new_storage_gb { + if s < current_storage_gb { + return Err(smolvm::Error::config( + "resize", + format!( + "storage disk cannot be shrunk from {} GiB to {} GiB. Only expanding is supported to prevent data loss.", + current_storage_gb, s + ), + )); + } + } + if let Some(o) = new_overlay_gb { + if o < current_overlay_gb { + return Err(smolvm::Error::config( + "resize", + format!( + "overlay disk cannot be shrunk from {} GiB to {} GiB. Only expanding is supported to prevent data loss.", + current_overlay_gb, o + ), + )); + } + } + + let manager = AgentManager::for_vm(name) + .map_err(|e| smolvm::Error::agent("get agent manager", e.to_string()))?; + + let mut changes = Vec::new(); + + if let Some(storage_gb) = new_storage_gb { + if storage_gb > current_storage_gb { + let storage_path = manager.storage_path(); + expand_disk::(storage_path, storage_gb) + .map_err(|e| smolvm::Error::storage("expand storage disk", e.to_string()))?; + changes.push(format!( + " storage: {} GiB → {} GiB", + current_storage_gb, storage_gb + )); + } + } + + if let Some(overlay_gb) = new_overlay_gb { + if overlay_gb > current_overlay_gb { + let overlay_path = manager.overlay_path(); + expand_disk::(overlay_path, overlay_gb) + .map_err(|e| smolvm::Error::storage("expand overlay disk", e.to_string()))?; + changes.push(format!( + " overlay: {} GiB → {} GiB", + current_overlay_gb, overlay_gb + )); + } + } + + Ok(changes) +} + +/// Legacy wrapper: expand disks AND update the DB in one call. +/// Used by the hidden `machine resize` backward-compat command. +pub fn resize_vm( + name: &str, + new_storage_gb: Option, + new_overlay_gb: Option, +) -> smolvm::Result<()> { + use smolvm::config::RecordState; + use smolvm::db::SmolvmDb; + + let db = SmolvmDb::open()?; + let record = db + .get_vm(name)? + .ok_or_else(|| smolvm::Error::vm_not_found(name))? + .clone(); + + let actual_state = record.actual_state(); + match actual_state { + RecordState::Stopped | RecordState::Created => {} + _ => { + return Err(smolvm::Error::InvalidState { + expected: "stopped".into(), + actual: format!("{:?}", actual_state), + }); + } + } + + let changes = expand_disks(name, &record, new_storage_gb, new_overlay_gb)?; + + db.update_vm(name, |r| { + if let Some(s) = new_storage_gb { + r.storage_gb = Some(s); + } + if let Some(o) = new_overlay_gb { + r.overlay_gb = Some(o); + } + })?; + + if changes.is_empty() { + println!("No disk changes needed."); + } else { + println!("Resized machine '{}':", name); + for c in &changes { + println!("{}", c); + } + println!("Filesystem will expand on next boot."); + } + + Ok(()) +} + +// ============================================================================ +// Ephemeral VM Tracking +// ============================================================================ + +/// Register an ephemeral VM in the database for tracking. +/// +/// Called by `machine run` after the VM is forked. The record is removed +/// on clean exit. Stale records from crashes are cleaned up by +/// `cleanup_orphaned_ephemeral_vms()`. +pub fn register_ephemeral_vm( + name: &str, + pid: Option, + cpus: u8, + mem: u32, + network: bool, + image: Option, +) { + let mut record = VmRecord::new(name.to_string(), cpus, mem, vec![], vec![], network); + record.ephemeral = true; + record.state = RecordState::Running; + record.pid = pid; + record.image = image; + + if let Ok(db) = SmolvmDb::open() { + if let Err(e) = db.insert_vm(name, &record) { + tracing::debug!(error = %e, name, "failed to register ephemeral VM"); + } + } +} + +/// Remove an ephemeral VM record from the database. +pub fn deregister_ephemeral_vm(name: &str) { + if let Ok(db) = SmolvmDb::open() { + if let Err(e) = db.remove_vm(name) { + tracing::debug!(error = %e, name, "failed to deregister ephemeral VM"); + } + } +} + +/// Names of ephemeral VMs that are orphans (dead or PID-less), capped at `limit`. +/// +/// Pure (no I/O) so the reaping policy is unit-testable: given the VM list and a +/// liveness probe, it returns at most `limit` orphan names in list order. A +/// non-ephemeral or still-alive record is never returned. +fn orphaned_ephemeral_names( + vms: &[(String, VmRecord)], + is_alive: impl Fn(i32) -> bool, + limit: usize, +) -> Vec<&str> { + let mut out = Vec::new(); + for (name, record) in vms { + if out.len() >= limit { + break; + } + if !record.ephemeral { + continue; + } + let is_orphan = match record.pid { + Some(pid) => !is_alive(pid), + None => true, // No PID recorded — stale + }; + if is_orphan { + out.push(name.as_str()); + } + } + out +} + +/// Clean up ALL orphaned ephemeral VM records. +/// +/// Called at the start of every machine command EXCEPT `machine run` (which uses +/// the bounded variant on its hot path). Fast path: if no ephemeral records +/// exist, this is a single DB read (~0.2ms). +pub fn cleanup_orphaned_ephemeral_vms() { + cleanup_orphaned_ephemeral_vms_bounded(usize::MAX); +} + +/// Clean up orphaned ephemeral VM records, removing at most `limit` of them. +/// +/// `machine run` is the one hot-path caller and passes a small cap: a workflow +/// that ONLY ever calls `machine run` (e.g. a wrapper that spawns a fresh +/// ephemeral VM per task) never reaches the unbounded sweep above, so any run +/// whose detached `_cleanup-ephemeral` helper didn't finish (Ctrl-C / SIGKILL / +/// host sleep mid-run) would leak its data dir forever. The cap lets such a +/// backlog self-heal over a few runs instead of stalling one boot on a large +/// `remove_dir_all` storm. Removal order follows the DB list order. Fast path: +/// no ephemeral records → a single DB read. +pub fn cleanup_orphaned_ephemeral_vms_bounded(limit: usize) { + if limit == 0 { + return; + } + let db = match SmolvmDb::open() { + Ok(db) => db, + Err(_) => return, + }; + let vms = match db.list_vms() { + Ok(vms) => vms, + Err(_) => return, + }; + for name in orphaned_ephemeral_names(&vms, smolvm::process::is_alive, limit) { + tracing::debug!(name = %name, "cleaning up orphaned ephemeral VM"); + let _ = db.remove_vm(name); + let dir = smolvm::agent::vm_data_dir(name); + if dir.exists() { + let _ = std::fs::remove_dir_all(&dir); + } + } +} + +#[cfg(test)] +mod init_runner_tests { + use super::*; + + // The ephemeral-reap policy: only ephemeral + (dead or PID-less) records, in + // list order, capped at `limit`. Persistent and still-alive VMs are never + // returned — this is what keeps the bounded `machine run` sweep from touching + // a concurrent run's live VM. + #[test] + fn orphaned_ephemeral_names_filters_and_caps() { + use smolvm::config::VmRecord; + let mk = |name: &str, ephemeral: bool, pid: Option| { + let mut r = VmRecord::new(name.to_string(), 1, 256, vec![], vec![], false); + r.ephemeral = ephemeral; + r.pid = pid; + (name.to_string(), r) + }; + let vms = vec![ + mk("persistent", false, Some(10)), // not ephemeral -> skip + mk("alive", true, Some(20)), // ephemeral but alive -> skip + mk("dead1", true, Some(30)), // ephemeral + dead -> orphan + mk("nopid", true, None), // ephemeral + no PID -> orphan + mk("dead2", true, Some(40)), // ephemeral + dead -> orphan + ]; + let alive = |pid: i32| pid == 20; // only the live VM's PID is alive + + assert_eq!( + orphaned_ephemeral_names(&vms, alive, usize::MAX), + vec!["dead1", "nopid", "dead2"] + ); + assert_eq!( + orphaned_ephemeral_names(&vms, alive, 2), + vec!["dead1", "nopid"], + "cap limits how many are reaped per call" + ); + assert!(orphaned_ephemeral_names(&vms, alive, 0).is_empty()); + } + + fn sample_image_info(env: Vec<&str>, workdir: Option<&str>, user: Option<&str>) -> ImageInfo { + ImageInfo { + reference: "alpine:latest".to_string(), + digest: "sha256:test".to_string(), + size: 0, + created: None, + architecture: "x86_64".to_string(), + os: "linux".to_string(), + layer_count: 0, + layers: Vec::new(), + entrypoint: Vec::new(), + cmd: Vec::new(), + env: env.into_iter().map(str::to_string).collect(), + workdir: workdir.map(str::to_string), + user: user.map(str::to_string), + } + } + + #[test] + fn default_workload_to_image_falls_back_to_image_when_machine_has_none() { + let (ep, cmd) = default_workload_to_image( + Vec::new(), + Vec::new(), + &["/entry".to_string()], + &["arg".to_string()], + ); + assert_eq!(ep, ["/entry"]); + assert_eq!(cmd, ["arg"]); + } + + #[test] + fn default_workload_to_image_prefers_explicit_machine_command() { + // An explicit machine command always wins over the image's defaults, + // even if only one of entrypoint/cmd is set. + let (ep, cmd) = default_workload_to_image( + Vec::new(), + vec!["own-cmd".to_string()], + &["/image-entry".to_string()], + &["image-arg".to_string()], + ); + assert!(ep.is_empty()); + assert_eq!(cmd, ["own-cmd"]); + } + + #[test] + fn default_workload_to_image_empty_when_neither_has_a_command() { + let (ep, cmd) = default_workload_to_image(Vec::new(), Vec::new(), &[], &[]); + assert!(ep.is_empty()); + assert!(cmd.is_empty()); + } + + #[test] + fn format_init_failure_includes_stderr_only() { + // Single stream → no "stdout:" / "stderr:" labels needed; the + // colon-prefixed form keeps the message compact for the common + // case of a tool writing only to stderr. + let msg = format_init_failure(0, 1, "", "command not found"); + assert_eq!(msg, "init[0] failed (exit 1): command not found"); + } + + #[test] + fn format_init_failure_includes_stdout_only() { + // Some tools emit their failure reason on stdout instead of + // stderr (curl with -s, certain pacman/apt failure modes). + // Dropping stdout would leave the operator with just an exit + // code and no explanation. + let msg = format_init_failure(2, 127, "could not resolve mirror", ""); + assert_eq!(msg, "init[2] failed (exit 127): could not resolve mirror"); + } + + #[test] + fn format_init_failure_combines_both_streams() { + // Both populated: stderr leads (canonical error channel) but + // stdout follows so package-manager errors that put the real + // reason on stdout are still visible. Single-line for greppability. + let msg = format_init_failure(0, 2, "saw 3 errors", "fatal: aborting"); + assert_eq!( + msg, + "init[0] failed (exit 2): fatal: aborting; stdout: saw 3 errors" + ); + } + + #[test] + fn format_init_failure_handles_empty_streams() { + // Some commands exit non-zero with no output (e.g. `false`). + // The error must still be informative — the index + exit code + // alone tell the user which command failed and how. + let msg = format_init_failure(5, 1, "", ""); + assert_eq!(msg, "init[5] failed (exit 1)"); + } + + #[test] + fn format_init_failure_trims_whitespace() { + // Subprocess output usually ends in a trailing newline; the + // formatter trims so the assembled message doesn't have weird + // mid-line breaks. + let msg = format_init_failure(0, 1, " ", " bad thing happened \n"); + assert_eq!(msg, "init[0] failed (exit 1): bad thing happened"); + } + + #[test] + fn init_argv_routes_through_sh_dash_c() { + // Shell wrapping is load-bearing: user init strings commonly + // chain commands with `&&`, pipe through tools, rely on env + // expansion. If a future refactor "simplifies" by passing the + // command argv directly to exec, those features break silently. + assert_eq!( + init_argv("pacman -Sy && pacman -S git"), + vec![ + "sh".to_string(), + "-c".to_string(), + "pacman -Sy && pacman -S git".to_string(), + ] + ); + } + + #[test] + fn build_init_run_config_overlay_matches_machine_name() { + // The overlay ID is what makes init's filesystem changes visible + // to subsequent `machine exec`. If this drifts (e.g. someone + // hardcodes "init-overlay"), `pacman -S git` during init would + // succeed but `git --version` post-start would fail with "not + // found" — exactly the user-confusing regression we're guarding + // against. + let config = build_init_run_config( + "alpine", + "true", + &ImageRuntimeDefaults { + env: vec![], + workdir: None, + user: None, + }, + &[], + "my-vm", + ); + assert_eq!(config.persistent_overlay_id.as_deref(), Some("my-vm")); + } + + #[test] + fn build_init_run_config_threads_env_workdir_image() { + // Each input must reach the agent untouched. The runner passes + // record values verbatim; if a `with_*` call gets dropped in a + // refactor, the user's `[dev].env` or `[dev].workdir` would + // silently stop applying to init. + let env = vec![("HTTP_PROXY".to_string(), "http://proxy:3128".to_string())]; + let config = build_init_run_config( + "debian:slim", + "apt update", + &ImageRuntimeDefaults { + env: env.clone(), + workdir: Some("/work".to_string()), + user: Some("steam".to_string()), + }, + &[], + "vm", + ); + // Image is canonicalized by `RunConfig::new` (normalize_image_ref): + // bare `debian:slim` → fully-qualified `docker.io/library/debian:slim`. + assert_eq!(config.image, "docker.io/library/debian:slim"); + assert_eq!(config.env, env); + assert_eq!(config.workdir.as_deref(), Some("/work")); + assert_eq!(config.user.as_deref(), Some("steam")); + // Command is sh-wrapped; assert the wrapped form arrives. + assert_eq!( + config.command, + vec!["sh".to_string(), "-c".to_string(), "apt update".to_string(),] + ); + } + + #[test] + fn build_init_run_config_assigns_virtiofs_tags_to_mounts() { + // Mount tags are positional and must align with the virtiofs + // devices libkrun set up at VM start. If the converter were + // skipped (or renamed and not rewired), init would still run + // but mounted volumes wouldn't be visible inside the container. + let mounts = vec![ + ("/host/src".to_string(), "/app".to_string(), false), + ("/host/data".to_string(), "/data".to_string(), true), + ]; + let config = build_init_run_config( + "alpine", + "true", + &ImageRuntimeDefaults { + env: vec![], + workdir: None, + user: None, + }, + &mounts, + "vm", + ); + assert_eq!( + config.mounts, + vec![ + ("smolvm0".to_string(), "/app".to_string(), false), + ("smolvm1".to_string(), "/data".to_string(), true), + ] + ); + } + + #[test] + fn build_init_run_config_no_mounts_no_workdir() { + // The image path is also the bare-minimum path: image + cmd is + // a valid init invocation. No mounts, no workdir, no env — must + // still produce a usable RunConfig (vs. e.g. panicking on + // `unwrap` somewhere in the builder). + let config = build_init_run_config( + "alpine", + "echo hi", + &ImageRuntimeDefaults { + env: vec![], + workdir: None, + user: None, + }, + &[], + "vm", + ); + assert!(config.mounts.is_empty()); + assert!(config.workdir.is_none()); + assert!(config.env.is_empty()); + assert!(config.user.is_none()); + assert_eq!(config.persistent_overlay_id.as_deref(), Some("vm")); + } + + #[test] + fn resolve_image_runtime_defaults_uses_image_env_workdir_and_user() { + let image_info = sample_image_info( + vec!["FOO=from-image", "BAR=from-image"], + Some("/image-workdir"), + Some("steam"), + ); + + let defaults = resolve_image_runtime_defaults(Some(&image_info), &[], None); + + assert_eq!( + defaults.env, + vec![ + ("FOO".to_string(), "from-image".to_string()), + ("BAR".to_string(), "from-image".to_string()), + ] + ); + assert_eq!(defaults.workdir.as_deref(), Some("/image-workdir")); + assert_eq!(defaults.user.as_deref(), Some("steam")); + } + + #[test] + fn image_workdir_flows_into_init_run_config_when_no_explicit_workdir_is_set() { + let image_info = sample_image_info( + vec!["FOO=from-image"], + Some("/image-workdir"), + Some("steam"), + ); + let defaults = resolve_image_runtime_defaults(Some(&image_info), &[], None); + let config = build_init_run_config("alpine:latest", "pwd", &defaults, &[], "vm"); + + assert_eq!(config.workdir.as_deref(), Some("/image-workdir")); + assert_eq!(config.user.as_deref(), Some("steam")); + assert_eq!( + config.env, + vec![("FOO".to_string(), "from-image".to_string())] + ); + } + + #[test] + fn resolve_image_runtime_defaults_explicit_values_override_image_defaults() { + let image_info = sample_image_info( + vec!["FOO=from-image", "BAR=from-image"], + Some("/image-workdir"), + Some("steam"), + ); + let env = vec![ + ("BAR".to_string(), "from-cli".to_string()), + ("BAZ".to_string(), "from-cli".to_string()), + ]; + + let defaults = + resolve_image_runtime_defaults(Some(&image_info), &env, Some("/explicit-workdir")); + + assert_eq!( + defaults.env, + vec![ + ("FOO".to_string(), "from-image".to_string()), + ("BAR".to_string(), "from-cli".to_string()), + ("BAZ".to_string(), "from-cli".to_string()), + ] + ); + assert_eq!(defaults.workdir.as_deref(), Some("/explicit-workdir")); + assert_eq!(defaults.user.as_deref(), Some("steam")); + } + + #[test] + fn resolve_image_runtime_defaults_ignores_invalid_image_env_and_last_value_wins() { + let image_info = sample_image_info( + vec!["BROKEN", "=empty-key", "FOO=from-image", "FOO=last-image"], + Some("/image-workdir"), + Some("1000:1000"), + ); + let env = vec![ + ("BAR".to_string(), "from-cli".to_string()), + ("BAR".to_string(), "last-cli".to_string()), + ]; + + let defaults = resolve_image_runtime_defaults(Some(&image_info), &env, None); + + assert_eq!( + defaults.env, + vec![ + ("FOO".to_string(), "last-image".to_string()), + ("BAR".to_string(), "last-cli".to_string()), + ] + ); + assert_eq!(defaults.workdir.as_deref(), Some("/image-workdir")); + assert_eq!(defaults.user.as_deref(), Some("1000:1000")); + } + + #[test] + fn resolve_image_runtime_defaults_falls_back_to_explicit_values_without_image_info() { + let env = vec![("FOO".to_string(), "from-explicit".to_string())]; + + let defaults = resolve_image_runtime_defaults(None, &env, Some("/explicit-workdir")); + + assert_eq!(defaults.env, env); + assert_eq!(defaults.workdir.as_deref(), Some("/explicit-workdir")); + assert!(defaults.user.is_none()); + } + + #[test] + fn merge_env_overrides_last_value_wins_by_key() { + let base_env = vec![ + ("FOO".to_string(), "from-record".to_string()), + ("BAR".to_string(), "from-record".to_string()), + ]; + let overrides = vec![ + ("BAR".to_string(), "from-cli".to_string()), + ("BAZ".to_string(), "from-cli".to_string()), + ]; + + let merged = merge_env_overrides(&base_env, &overrides); + + assert_eq!( + merged, + vec![ + ("FOO".to_string(), "from-record".to_string()), + ("BAR".to_string(), "from-cli".to_string()), + ("BAZ".to_string(), "from-cli".to_string()), + ] + ); + } +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..95a7f02 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,1120 @@ +//! Global smolvm configuration. +//! +//! This module handles persistent configuration storage for smolvm, +//! including default settings and VM registry. +//! +//! State is persisted to a SQLite database at `~/.local/share/smolvm/server/smolvm.db`. +//! For backward compatibility, `SmolvmConfig` maintains an in-memory cache of VMs +//! and provides the same API as the old confy-based implementation. + +use crate::data::network; +use crate::data::resources::{DEFAULT_MICROVM_CPU_COUNT, DEFAULT_MICROVM_MEMORY_MIB}; +use crate::db::SmolvmDb; +use crate::error::Result; +use crate::network::NetworkBackend; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// VM lifecycle state. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum RecordState { + /// Container exists, VM not started. + #[default] + Created, + /// VM process is running. + Running, + /// VM exited cleanly. + Stopped, + /// VM crashed or error. + Failed, + /// libkrun VMM process is alive but the guest agent is not + /// responding to vsock pings. Typical cause: the agent crashed + /// (OOM, panic, kernel issue) while the VMM stayed up — common + /// aftermath of a workload that exhausted guest resources. + /// `machine list` shows this so operators see the truth instead + /// of a misleading "running"; `machine start` recovers by + /// killing the zombie VMM and starting fresh. + Unreachable, + /// libkrun VMM process is alive but deliberately frozen as a fork + /// base: `machine fork` snapshotted it and its clones' disk overlays + /// are copy-on-write backed by its disks. Its guest agent is paused + /// and never answers a vsock ping — so it is reported *without* a + /// liveness probe (it would otherwise look identical to an + /// `Unreachable` zombie) and is never reaped or auto-restarted: it + /// must outlive its clones. Resolved on the fly when a record has + /// dependent clones; not persisted. + Frozen, +} + +impl std::fmt::Display for RecordState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RecordState::Created => write!(f, "created"), + RecordState::Running => write!(f, "running"), + RecordState::Stopped => write!(f, "stopped"), + RecordState::Failed => write!(f, "failed"), + RecordState::Unreachable => write!(f, "unreachable"), + RecordState::Frozen => write!(f, "frozen"), + } + } +} + +/// Restart policy for a machine. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "kebab-case")] +pub enum RestartPolicy { + /// Never restart the machine automatically. + #[default] + Never, + /// Always restart the machine when it exits. + Always, + /// Restart only if the machine exited with a non-zero exit code. + OnFailure, + /// Restart unless the user explicitly stopped the machine. + UnlessStopped, +} + +impl std::fmt::Display for RestartPolicy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RestartPolicy::Never => write!(f, "never"), + RestartPolicy::Always => write!(f, "always"), + RestartPolicy::OnFailure => write!(f, "on-failure"), + RestartPolicy::UnlessStopped => write!(f, "unless-stopped"), + } + } +} + +impl std::str::FromStr for RestartPolicy { + type Err = String; + + fn from_str(s: &str) -> std::result::Result { + match s.to_lowercase().as_str() { + "never" => Ok(RestartPolicy::Never), + "always" => Ok(RestartPolicy::Always), + "on-failure" | "onfailure" => Ok(RestartPolicy::OnFailure), + "unless-stopped" | "unlessstopped" => Ok(RestartPolicy::UnlessStopped), + _ => Err(format!("invalid restart policy: {}", s)), + } + } +} + +/// Restart configuration for a machine. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RestartConfig { + /// The restart policy. + #[serde(default)] + pub policy: RestartPolicy, + /// Maximum number of restart attempts (0 = unlimited). + #[serde(default)] + pub max_retries: u32, + /// Maximum backoff duration in seconds (0 = use default 300s). + #[serde(default)] + pub max_backoff_secs: u64, + /// Current restart count. + #[serde(default)] + pub restart_count: u32, + /// Whether the user explicitly stopped this machine. + #[serde(default)] + pub user_stopped: bool, +} + +impl RestartConfig { + /// Determine whether the machine should be restarted based on the policy, + /// exit code, and current restart count. + pub fn should_restart(&self, exit_code: Option) -> bool { + // Check max retries limit (0 = unlimited) + if self.max_retries > 0 && self.restart_count >= self.max_retries { + return false; + } + match self.policy { + RestartPolicy::Never => false, + RestartPolicy::Always => true, + RestartPolicy::OnFailure => exit_code != Some(0), + RestartPolicy::UnlessStopped => !self.user_stopped, + } + } + + /// Default maximum backoff duration in seconds (5 minutes). + const DEFAULT_MAX_BACKOFF_SECS: u64 = 300; + + /// Maximum exponent for backoff calculation (2^8 = 256s). + const MAX_BACKOFF_EXPONENT: u32 = 8; + + /// Calculate exponential backoff duration for the current restart count. + /// + /// Formula: 2^n seconds, capped at max_backoff_secs (default 300s). + pub fn backoff_duration(&self) -> std::time::Duration { + let max_secs = if self.max_backoff_secs > 0 { + self.max_backoff_secs + } else { + Self::DEFAULT_MAX_BACKOFF_SECS + }; + let exponent = self.restart_count.min(Self::MAX_BACKOFF_EXPONENT); + std::time::Duration::from_secs(2u64.pow(exponent).min(max_secs)) + } +} + +/// Global smolvm configuration with database-backed persistence. +/// +/// This struct provides backward-compatible access to VM records while +/// using SQLite for ACID-compliant storage. The `vms` field is an in-memory +/// cache that is kept in sync with the database. +#[derive(Debug, Clone)] +pub struct SmolvmConfig { + /// Database handle for persistence. + db: SmolvmDb, + /// Configuration format version. + pub version: u8, + /// Default number of vCPUs for new VMs. + pub default_cpus: u8, + /// Default memory in MiB for new VMs. + pub default_mem: u32, + /// Default DNS server for VMs with network egress. + pub default_dns: String, + /// Storage volume path (macOS only, for case-sensitive filesystem). + #[cfg(target_os = "macos")] + pub storage_volume: String, + /// Registry of known VMs (by name) - in-memory cache. + pub vms: HashMap, +} + +impl SmolvmConfig { + /// Create a new configuration with default values. + /// + /// This is the fallible version of `Default::default()`. Use this when + /// you need to handle database initialization errors. + pub fn try_default() -> Result { + Ok(Self { + db: SmolvmDb::open()?, + version: 1, + default_cpus: DEFAULT_MICROVM_CPU_COUNT, + default_mem: DEFAULT_MICROVM_MEMORY_MIB, + default_dns: network::default_dns(), + #[cfg(target_os = "macos")] + storage_volume: String::new(), + vms: HashMap::new(), + }) + } +} + +impl SmolvmConfig { + /// Load configuration from the database. + /// + /// Opens the database and loads all config settings and VM records in a + /// single database transaction (1 open/close cycle instead of 6). + pub fn load() -> Result { + let db = SmolvmDb::open()?; + + // Load all config + all VMs in one DB round-trip + let (config_map, vms) = db.load_all()?; + + let version = config_map + .get("version") + .and_then(|s| s.parse().ok()) + .unwrap_or(1); + let default_cpus = config_map + .get("default_cpus") + .and_then(|s| s.parse().ok()) + .unwrap_or(DEFAULT_MICROVM_CPU_COUNT); + let default_mem = config_map + .get("default_mem") + .and_then(|s| s.parse().ok()) + .unwrap_or(DEFAULT_MICROVM_MEMORY_MIB); + let default_dns = config_map + .get("default_dns") + .cloned() + .unwrap_or_else(network::default_dns); + + #[cfg(target_os = "macos")] + let storage_volume = config_map + .get("storage_volume") + .cloned() + .unwrap_or_default(); + + Ok(Self { + db, + version, + default_cpus, + default_mem, + default_dns, + #[cfg(target_os = "macos")] + storage_volume, + vms, + }) + } + + /// Save global configuration to the database. + /// + /// Persists all global config settings in a single DB transaction + /// (1 open/close cycle instead of 4). VM records are not saved here + /// since writes are immediate via `update_vm()` and `insert_vm()`. + pub fn save(&self) -> Result<()> { + let version_str = self.version.to_string(); + let cpus_str = self.default_cpus.to_string(); + let mem_str = self.default_mem.to_string(); + + #[cfg(not(target_os = "macos"))] + let settings: Vec<(&str, &str)> = vec![ + ("version", version_str.as_str()), + ("default_cpus", cpus_str.as_str()), + ("default_mem", mem_str.as_str()), + ("default_dns", self.default_dns.as_str()), + ]; + + #[cfg(target_os = "macos")] + let settings: Vec<(&str, &str)> = { + let mut s = vec![ + ("version", version_str.as_str()), + ("default_cpus", cpus_str.as_str()), + ("default_mem", mem_str.as_str()), + ("default_dns", self.default_dns.as_str()), + ]; + if !self.storage_volume.is_empty() { + s.push(("storage_volume", self.storage_volume.as_str())); + } + s + }; + + self.db.save_config(&settings) + } + + /// Insert a VM record (persists immediately to database). + pub fn insert_vm(&mut self, name: String, record: VmRecord) -> Result<()> { + self.db.insert_vm(&name, &record)?; + self.vms.insert(name, record); + Ok(()) + } + + /// Remove a VM from the registry. + pub fn remove_vm(&mut self, id: &str) -> Option { + // Remove from database (ignore errors, just log) + if let Err(e) = self.db.remove_vm(id) { + tracing::warn!(error = %e, vm = %id, "failed to remove VM from database"); + } + self.vms.remove(id) + } + + /// Get a VM record by ID. + pub fn get_vm(&self, id: &str) -> Option<&VmRecord> { + self.vms.get(id) + } + + /// List all VM records. + pub fn list_vms(&self) -> impl Iterator { + self.vms.iter() + } + + /// Update a VM record in place (persists immediately to database). + /// + /// Returns `None` if the record doesn't exist, `Some(Err)` if the DB write + /// fails, `Some(Ok)` on success. Callers that need fail-closed semantics + /// should check both. + pub fn update_vm(&mut self, id: &str, f: F) -> Option> + where + F: FnOnce(&mut VmRecord), + { + if let Some(record) = self.vms.get_mut(id) { + f(record); + // Persist to database + Some(self.db.insert_vm(id, record)) + } else { + None + } + } + + /// Get the underlying database handle. + pub fn db(&self) -> &SmolvmDb { + &self.db + } +} + +/// Record of a VM in the registry. +/// +/// This stores machine configuration only. Container configuration +/// is managed separately via the container commands. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VmRecord { + /// VM name/ID. + pub name: String, + + /// Creation timestamp (seconds since Unix epoch). + #[serde(deserialize_with = "deserialize_timestamp", default)] + pub created_at: u64, + + /// VM lifecycle state. + #[serde(default)] + pub state: RecordState, + + /// Process ID when running. + #[serde(default)] + pub pid: Option, + + /// Process start time (seconds since epoch) for PID verification. + /// Used alongside PID to detect PID reuse by the OS. + #[serde(default)] + pub pid_start_time: Option, + + /// Number of vCPUs. + #[serde(default = "default_cpus")] + pub cpus: u8, + + /// Memory in MiB. + #[serde(default = "default_mem")] + pub mem: u32, + + /// Volume mounts (host_path, guest_path, read_only). + #[serde(default)] + pub mounts: Vec<(String, String, bool)>, + + /// Port mappings (host_port, guest_port). + #[serde(default)] + pub ports: Vec<(u16, u16)>, + + /// Enable outbound network access (TSI). + #[serde(default)] + pub network: bool, + + /// Enable GPU acceleration (virtio-gpu with Venus/Vulkan). + #[serde(default)] + pub gpu: Option, + + /// GPU shared-memory region size in MiB. `None` → default + /// (`DEFAULT_GPU_VRAM_MIB`). Ignored unless `gpu` is true. + #[serde(default)] + pub gpu_vram_mib: Option, + + /// Enable Rosetta 2 for x86_64 binary translation on Apple Silicon. + #[serde(default)] + pub rosetta: Option, + + /// Restart configuration. + #[serde(default)] + pub restart: RestartConfig, + + /// Last exit code from the VM process. + #[serde(default)] + pub last_exit_code: Option, + + /// Commands to run on first VM start (via `sh -c`). + #[serde(default)] + pub init: Vec, + + /// Whether init commands have already completed successfully. + /// Set to true after first successful run; reset when init commands change. + #[serde(default)] + pub init_completed: bool, + + /// Environment variables for init commands. + #[serde(default)] + pub env: Vec<(String, String)>, + + /// Secret references declared by a Smolfile `[secrets]` section, keyed + /// by the guest-side env var name. Resolved to plaintext at each VM + /// start (and for `machine exec`) and appended to the env vector — the + /// plaintext values never touch this record or the DB. + #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")] + pub secret_refs: std::collections::BTreeMap, + + /// Working directory for the container workload. + #[serde(default)] + pub workdir: Option, + + /// Container user (UID or username). Resolved from image metadata at first + /// launch and stored so restart can replay the same identity. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// Storage disk size in GiB (None = default 20 GiB). + #[serde(default)] + pub storage_gb: Option, + + /// Overlay disk size in GiB (None = default 10 GiB). + #[serde(default)] + pub overlay_gb: Option, + + /// Allowed egress CIDR ranges. None = unrestricted, Some([]) = deny all. + #[serde(default)] + pub allowed_cidrs: Option>, + + /// Preferred network backend override for machine launch. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network_backend: Option, + + /// Custom DNS resolver for the guest. None = backend default. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dns: Option, + + /// OCI image for auto-container creation on start. + #[serde(default)] + pub image: Option, + + /// Entrypoint for the container. + #[serde(default)] + pub entrypoint: Vec, + + /// Default command for the container. + #[serde(default)] + pub cmd: Vec, + + /// Health check command (run inside VM to verify workload is healthy). + #[serde(default)] + pub health_cmd: Option>, + + /// Health check interval in seconds. + #[serde(default)] + pub health_interval_secs: Option, + + /// Health check timeout in seconds. + #[serde(default)] + pub health_timeout_secs: Option, + + /// Health check failure threshold before marking unhealthy. + #[serde(default)] + pub health_retries: Option, + + /// Grace period in seconds before health checks start after boot. + #[serde(default)] + pub health_startup_grace_secs: Option, + + /// Enable SSH agent forwarding into the VM. + #[serde(default)] + pub ssh_agent: bool, + + /// Enable CUDA-over-vsock: smolvm starts a host CUDA server and remotes the + /// guest's CUDA Driver-API calls to the host NVIDIA GPU. + #[serde(default)] + pub cuda: bool, + + /// Hostnames for DNS filtering. When set, the guest DNS proxy filters + /// queries against this allowlist. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dns_filter_hosts: Option>, + + /// True for `machine run` VMs. Auto-deleted on exit or cleanup sweep. + #[serde(default)] + pub ephemeral: bool, + + /// Absolute path to the .smolmachine sidecar this machine was created from. + /// When set, `machine start` extracts layers from the sidecar and mounts + /// them via virtiofs instead of pulling the image from a registry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_smolmachine: Option, + + /// Name of the golden VM this machine was forked from, if any. A clone's + /// block disks are copy-on-write overlays backed by the golden's disks, so + /// the golden must outlive its clones. The disk *format* is not recorded + /// here — it is derived from the on-disk file (`.qcow2` vs `.raw`), which is + /// the single source of truth (see `agent::resolve_disk_image`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub golden: Option, +} + +/// Deserialize `created_at` from either a legacy JSON string `"1705312345"` or +/// the current integer `1705312345`. Old DB records stored it as a string. +fn deserialize_timestamp<'de, D>(deserializer: D) -> std::result::Result +where + D: serde::Deserializer<'de>, +{ + use serde::Deserialize; + #[derive(Deserialize)] + #[serde(untagged)] + enum StrOrU64 { + Str(String), + U64(u64), + } + match StrOrU64::deserialize(deserializer)? { + StrOrU64::U64(n) => Ok(n), + StrOrU64::Str(s) => s.parse::().map_err(serde::de::Error::custom), + } +} + +fn default_cpus() -> u8 { + 1 +} + +fn default_mem() -> u32 { + 512 +} + +impl VmRecord { + /// Create a new VM record. + pub fn new( + name: String, + cpus: u8, + mem: u32, + mounts: Vec<(String, String, bool)>, + ports: Vec<(u16, u16)>, + network: bool, + ) -> Self { + Self { + name, + created_at: crate::util::current_timestamp(), + state: RecordState::Created, + pid: None, + pid_start_time: None, + cpus, + mem, + mounts, + ports, + network, + gpu: None, + gpu_vram_mib: None, + rosetta: None, + restart: RestartConfig::default(), + last_exit_code: None, + init: Vec::new(), + init_completed: false, + env: Vec::new(), + secret_refs: std::collections::BTreeMap::new(), + workdir: None, + user: None, + storage_gb: None, + overlay_gb: None, + allowed_cidrs: None, + network_backend: None, + dns: None, + image: None, + entrypoint: Vec::new(), + cmd: Vec::new(), + health_cmd: None, + health_interval_secs: None, + health_timeout_secs: None, + health_retries: None, + health_startup_grace_secs: None, + ssh_agent: false, + cuda: false, + dns_filter_hosts: None, + ephemeral: false, + source_smolmachine: None, + golden: None, + } + } + + /// Create a new VM record with restart configuration. + pub fn new_with_restart( + name: String, + cpus: u8, + mem: u32, + mounts: Vec<(String, String, bool)>, + ports: Vec<(u16, u16)>, + network: bool, + restart: RestartConfig, + ) -> Self { + Self { + name, + created_at: crate::util::current_timestamp(), + state: RecordState::Created, + pid: None, + pid_start_time: None, + cpus, + mem, + mounts, + ports, + network, + gpu: None, + gpu_vram_mib: None, + rosetta: None, + restart, + last_exit_code: None, + init: Vec::new(), + init_completed: false, + env: Vec::new(), + secret_refs: std::collections::BTreeMap::new(), + workdir: None, + user: None, + storage_gb: None, + overlay_gb: None, + allowed_cidrs: None, + network_backend: None, + dns: None, + image: None, + entrypoint: Vec::new(), + cmd: Vec::new(), + health_cmd: None, + health_interval_secs: None, + health_timeout_secs: None, + health_retries: None, + health_startup_grace_secs: None, + ssh_agent: false, + cuda: false, + dns_filter_hosts: None, + ephemeral: false, + source_smolmachine: None, + golden: None, + } + } + + /// Check if the VM process is still alive. + /// + /// Uses start time verification to detect PID reuse by the OS. + /// Falls back to PID-only check for legacy records without start time. + pub fn is_process_alive(&self) -> bool { + if let Some(pid) = self.pid { + crate::process::is_our_process(pid, self.pid_start_time) + } else { + false + } + } + + /// Get the actual state, checking if running process is still alive. + pub fn actual_state(&self) -> RecordState { + if self.state == RecordState::Running { + if self.is_process_alive() { + RecordState::Running + } else { + RecordState::Stopped // Process died + } + } else { + self.state.clone() + } + } + + /// Convert stored mounts to HostMount format. + pub fn host_mounts(&self) -> Vec { + self.mounts + .iter() + .map(|(host, guest, ro)| crate::data::storage::HostMount { + source: std::path::PathBuf::from(host), + target: std::path::PathBuf::from(guest), + read_only: *ro, + }) + .collect() + } + + /// Convert stored ports to PortMapping format. + pub fn port_mappings(&self) -> Vec { + self.ports + .iter() + .map(|(host, guest)| crate::data::network::PortMapping::new(*host, *guest)) + .collect() + } + + /// Convert record fields to VmResources. + pub fn vm_resources(&self) -> crate::agent::VmResources { + crate::agent::VmResources { + cpus: self.cpus, + memory_mib: self.mem, + network: self.network, + network_backend: self.network_backend, + gpu: self.gpu.unwrap_or(false), + gpu_vram_mib: self.gpu_vram_mib, + rosetta: self.rosetta.unwrap_or(false), + storage_gib: self.storage_gb, + overlay_gib: self.overlay_gb, + allowed_cidrs: self.allowed_cidrs.clone(), + dns: self.dns, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_vm_record_serialization() { + let record = VmRecord::new( + "test".to_string(), + 2, + 512, + vec![("/host".to_string(), "/guest".to_string(), false)], + vec![(8080, 80)], + false, + ); + + let json = serde_json::to_string(&record).unwrap(); + let deserialized: VmRecord = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.name, record.name); + assert_eq!(deserialized.mounts, record.mounts); + } + + #[test] + fn test_vm_record_secret_refs_roundtrip() { + use crate::secrets::SecretRef; + let mut record = VmRecord::new("r".into(), 1, 256, vec![], vec![], false); + record.secret_refs.insert( + "TLS_KEY".into(), + SecretRef { + from_env: None, + from_file: Some("/run/secrets/tls.key".into()), + }, + ); + record.secret_refs.insert( + "DB_URL".into(), + SecretRef { + from_env: Some("PROD_DB".into()), + from_file: None, + }, + ); + + let json = serde_json::to_string(&record).unwrap(); + // Ref metadata — not sensitive — roundtrips through serde_json. + assert!(json.contains("TLS_KEY")); + assert!(json.contains("PROD_DB")); + + let back: VmRecord = serde_json::from_str(&json).unwrap(); + assert_eq!(back.secret_refs.len(), 2); + assert_eq!( + back.secret_refs["TLS_KEY"] + .from_file + .as_ref() + .map(|p| p.to_string_lossy().into_owned()), + Some("/run/secrets/tls.key".to_string()) + ); + assert_eq!( + back.secret_refs["DB_URL"].from_env.as_deref(), + Some("PROD_DB") + ); + } + + #[test] + fn test_vm_record_with_restart() { + let restart = RestartConfig { + policy: RestartPolicy::Always, + max_retries: 5, + ..Default::default() + }; + let record = + VmRecord::new_with_restart("test".to_string(), 2, 512, vec![], vec![], false, restart); + + let json = serde_json::to_string(&record).unwrap(); + let deserialized: VmRecord = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.restart.policy, RestartPolicy::Always); + assert_eq!(deserialized.restart.max_retries, 5); + } + + #[test] + fn test_record_state_display() { + assert_eq!(RecordState::Created.to_string(), "created"); + assert_eq!(RecordState::Running.to_string(), "running"); + assert_eq!(RecordState::Stopped.to_string(), "stopped"); + assert_eq!(RecordState::Failed.to_string(), "failed"); + } + + #[test] + fn test_restart_policy_display_and_parse() { + assert_eq!(RestartPolicy::Never.to_string(), "never"); + assert_eq!(RestartPolicy::Always.to_string(), "always"); + assert_eq!(RestartPolicy::OnFailure.to_string(), "on-failure"); + assert_eq!(RestartPolicy::UnlessStopped.to_string(), "unless-stopped"); + + assert_eq!( + "never".parse::().unwrap(), + RestartPolicy::Never + ); + assert_eq!( + "always".parse::().unwrap(), + RestartPolicy::Always + ); + assert_eq!( + "on-failure".parse::().unwrap(), + RestartPolicy::OnFailure + ); + assert_eq!( + "unless-stopped".parse::().unwrap(), + RestartPolicy::UnlessStopped + ); + } + + #[test] + fn test_restart_policy_serialization() { + let policy = RestartPolicy::OnFailure; + let json = serde_json::to_string(&policy).unwrap(); + assert_eq!(json, "\"on-failure\""); + + let deserialized: RestartPolicy = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized, RestartPolicy::OnFailure); + } + + #[test] + fn test_restart_config_default() { + let config = RestartConfig::default(); + assert_eq!(config.policy, RestartPolicy::Never); + assert_eq!(config.max_retries, 0); + assert_eq!(config.restart_count, 0); + assert!(!config.user_stopped); + } + + #[test] + fn test_should_restart() { + // (policy, max_retries, restart_count, user_stopped, last_exit_code, expected, desc) + let cases = [ + ( + RestartPolicy::Never, + 0, + 0, + false, + None, + false, + "never policy", + ), + ( + RestartPolicy::Always, + 0, + 5, + false, + None, + true, + "always policy", + ), + ( + RestartPolicy::Always, + 3, + 3, + false, + None, + false, + "max retries reached", + ), + ( + RestartPolicy::Always, + 3, + 2, + false, + None, + true, + "under max retries", + ), + ( + RestartPolicy::OnFailure, + 0, + 0, + false, + Some(1), + true, + "on-failure non-zero exit", + ), + ( + RestartPolicy::OnFailure, + 0, + 0, + false, + Some(0), + false, + "on-failure clean exit", + ), + ( + RestartPolicy::OnFailure, + 0, + 0, + false, + None, + true, + "on-failure unknown exit", + ), + ( + RestartPolicy::UnlessStopped, + 0, + 0, + false, + None, + true, + "unless-stopped running", + ), + ( + RestartPolicy::UnlessStopped, + 0, + 0, + true, + None, + false, + "unless-stopped user stopped", + ), + ]; + + for (policy, max_retries, restart_count, user_stopped, last_exit_code, expected, desc) in + cases + { + let config = RestartConfig { + policy, + max_retries, + restart_count, + user_stopped, + ..Default::default() + }; + assert_eq!(config.should_restart(last_exit_code), expected, "{}", desc); + } + } + + #[test] + fn test_backoff_duration() { + use std::time::Duration; + let make = |count| RestartConfig { + restart_count: count, + ..Default::default() + }; + assert_eq!(make(0).backoff_duration(), Duration::from_secs(1)); + assert_eq!(make(1).backoff_duration(), Duration::from_secs(2)); + assert_eq!(make(2).backoff_duration(), Duration::from_secs(4)); + assert_eq!(make(3).backoff_duration(), Duration::from_secs(8)); + assert_eq!(make(8).backoff_duration(), Duration::from_secs(256)); + // Exponent capped at 8 → 256s for any count >= 8 + assert_eq!(make(9).backoff_duration(), Duration::from_secs(256)); + assert_eq!(make(100).backoff_duration(), Duration::from_secs(256)); + } + + #[test] + fn test_backoff_duration_respects_max_backoff() { + use std::time::Duration; + let config = RestartConfig { + restart_count: 8, + max_backoff_secs: 30, + ..Default::default() + }; + // 2^8 = 256, but capped at 30s + assert_eq!(config.backoff_duration(), Duration::from_secs(30)); + } + + // ======================================================================== + // Resize-related tests + // ======================================================================== + + #[test] + fn test_vm_record_storage_overlay_fields() { + // Test that storage_gb and overlay_gb fields work correctly + let mut record = VmRecord::new("test-vm".to_string(), 1, 512, vec![], vec![], false); + + // Initially None (uses defaults) + assert!(record.storage_gb.is_none()); + assert!(record.overlay_gb.is_none()); + + // Set storage_gb + record.storage_gb = Some(50); + assert_eq!(record.storage_gb, Some(50)); + + // Set overlay_gb + record.overlay_gb = Some(20); + assert_eq!(record.overlay_gb, Some(20)); + } + + #[test] + fn test_vm_record_partial_update() { + // Test that we can update only some fields (partial update pattern) + let mut record = VmRecord::new("test-vm".to_string(), 1, 512, vec![], vec![], false); + record.storage_gb = Some(20); + record.overlay_gb = Some(10); + + // Simulate partial update - only storage changes + let new_storage_gb: Option = Some(50); + let new_overlay_gb: Option = None; + + if let Some(s) = new_storage_gb { + record.storage_gb = Some(s); + } + if let Some(o) = new_overlay_gb { + record.overlay_gb = Some(o); + } + + assert_eq!(record.storage_gb, Some(50)); + assert_eq!(record.overlay_gb, Some(10)); // Unchanged + } + + #[test] + fn test_vm_record_vm_resources_includes_storage() { + // Test that vm_resources() includes storage_gb and overlay_gb + let mut record = VmRecord::new("test-vm".to_string(), 2, 1024, vec![], vec![], false); + record.storage_gb = Some(50); + record.overlay_gb = Some(20); + + let resources = record.vm_resources(); + assert_eq!(resources.cpus, 2); + assert_eq!(resources.memory_mib, 1024); + assert_eq!(resources.storage_gib, Some(50)); + assert_eq!(resources.overlay_gib, Some(20)); + } + + #[test] + fn test_vm_record_serialization_with_storage_overlay() { + // Test that storage_gb and overlay_gb serialize/deserialize correctly + let mut record = VmRecord::new("test-vm".to_string(), 1, 512, vec![], vec![], false); + record.storage_gb = Some(50); + record.overlay_gb = Some(20); + + let json = serde_json::to_string(&record).unwrap(); + + // Verify fields are in JSON + assert!(json.contains("storage_gb")); + assert!(json.contains("overlay_gb")); + assert!(json.contains("50")); + assert!(json.contains("20")); + + let deserialized: VmRecord = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.storage_gb, Some(50)); + assert_eq!(deserialized.overlay_gb, Some(20)); + } + + #[test] + fn test_vm_record_gpu_field() { + // GPU defaults to None (not set) + let record = VmRecord::new("test".to_string(), 2, 1024, vec![], vec![], false); + assert_eq!(record.gpu, None); + assert!(!record.vm_resources().gpu); + + // GPU set to true + let mut record = VmRecord::new("test".to_string(), 2, 1024, vec![], vec![], false); + record.gpu = Some(true); + assert!(record.vm_resources().gpu); + + // GPU serializes/deserializes + let json = serde_json::to_string(&record).unwrap(); + let deserialized: VmRecord = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.gpu, Some(true)); + + // New records default to gpu = None → vm_resources().gpu = false + let default_record = VmRecord::new("default".to_string(), 1, 512, vec![], vec![], false); + assert_eq!(default_record.gpu, None); + assert!(!default_record.vm_resources().gpu); + } + + #[test] + fn vm_record_gpu_vram_mib_flows_through_full_persistence_cycle() { + // End-to-end plumbing test: + // CreateVmParams-like assignment → VmRecord → serde_json (DB) + // → deserialized VmRecord → vm_resources() → effective + // + // This is the chain that runs every time a user creates a + // machine with `--gpu-vram N`, stops it, and starts it again. + // A silent break anywhere in the chain (e.g., someone drops + // the assignment, adds a new field and forgets to copy it, + // changes the Option shape) fires this test. + + use crate::agent::VmResources; + + // 1. Start with a record, set the field the way `create_vm` does. + let mut record = VmRecord::new("vramtest".into(), 2, 1024, vec![], vec![], false); + record.gpu = Some(true); + record.gpu_vram_mib = Some(1024); + + // 2. Roundtrip through JSON (the redb value format). + let json = serde_json::to_vec(&record).unwrap(); + let back: VmRecord = serde_json::from_slice(&json).unwrap(); + assert_eq!( + back.gpu_vram_mib, + Some(1024), + "gpu_vram_mib must survive DB roundtrip" + ); + + // 3. Convert to VmResources the way `start_vm_named` does. + let res: VmResources = back.vm_resources(); + assert_eq!(res.gpu_vram_mib, Some(1024)); + assert_eq!( + res.effective_gpu_vram_mib(), + 1024, + "launcher will pass 1024 MiB to krun_set_gpu_options2" + ); + + // 4. And the unset path: default in, default out. + let mut record = VmRecord::new("vramdefault".into(), 1, 512, vec![], vec![], false); + record.gpu = Some(true); + // gpu_vram_mib left as None + let json = serde_json::to_vec(&record).unwrap(); + let back: VmRecord = serde_json::from_slice(&json).unwrap(); + assert_eq!(back.gpu_vram_mib, None); + assert_eq!( + back.vm_resources().effective_gpu_vram_mib(), + crate::data::resources::DEFAULT_GPU_VRAM_MIB, + ); + } +} diff --git a/src/cuda_host.rs b/src/cuda_host.rs new file mode 100644 index 0000000..f135462 --- /dev/null +++ b/src/cuda_host.rs @@ -0,0 +1,94 @@ +//! Host-side CUDA-over-vsock server (smolvm-owned lifecycle). +//! +//! When a machine is launched with CUDA enabled, smolvm starts this listener on +//! a per-VM AF_UNIX socket and points the launcher's `cuda_socket` at it (the +//! vsock port is registered `listen=false`, so libkrun connects *to* this +//! socket when the guest opens the CUDA port). Each guest connection is served +//! by [`smolvm_cuda::host::serve`] against a freshly-created backend. +//! +//! Backend selection is automatic per connection: the real driver +//! ([`GpuBackend`]) when `nvcuda.dll` / `libcuda.so.1` loads, otherwise the CPU +//! emulation backend so the transport still works (and is testable) on a host +//! with no NVIDIA GPU. + +use crate::platform::uds::UdsListener; +use smolvm_cuda::host::{serve, Backend, CpuBackend, GpuBackend}; +use std::path::Path; +use std::sync::OnceLock; +use std::thread; + +/// Provider for the guest-RAM regions, each `(gpa_start, host_va, len)`, +/// installed by the launcher (via `krun_get_guest_ram`) before the VM starts. +/// Each connection's backend queries it to enable guest-RAM zero-copy +/// `memcpy_gpa_*`. `None` outside a microVM or on a libkrun without the API. +/// Guest RAM is usually split into a low and a high region around the 4 GiB +/// PCI hole. +type GuestRamProvider = Box Option> + Send + Sync>; +static GUEST_RAM: OnceLock = OnceLock::new(); + +/// Install the guest-RAM provider. Called once by the launcher before +/// `krun_start_enter`; the closure resolves lazily once the VM is up. +pub fn set_guest_ram_provider(f: GuestRamProvider) { + let _ = GUEST_RAM.set(f); +} + +/// Start the CUDA host server on `socket_path` in a background thread. +/// +/// The caller passes the same path to `LaunchConfig::cuda_socket`. Returns once +/// the listener is bound; serving continues until the process exits. +pub fn start(socket_path: &Path) -> std::io::Result<()> { + // Clean up any stale socket from a previous run. + let _ = std::fs::remove_file(socket_path); + + let listener = UdsListener::bind(socket_path)?; + let path_display = socket_path.display().to_string(); + + thread::Builder::new() + .name("cuda-host".into()) + .spawn(move || { + tracing::info!(path = path_display, "CUDA host server listening"); + for stream in listener.incoming() { + match stream { + Ok(stream) => { + thread::Builder::new() + .name("cuda-host-conn".into()) + .spawn(move || { + let mut backend = make_backend(); + if let Err(e) = serve(stream, backend.as_mut()) { + tracing::debug!(error = %e, "CUDA host connection ended"); + } + }) + .ok(); + } + Err(e) => { + tracing::debug!(error = %e, "CUDA host accept error"); + } + } + } + })?; + + Ok(()) +} + +/// Pick the best available backend for one connection: the real GPU driver if it +/// loads, else CPU emulation. Logged once per connection so the mode is visible. +fn make_backend() -> Box { + match GpuBackend::load() { + Ok(mut gpu) => { + tracing::info!("cuda-host: GPU driver backend ready"); + // Enable guest-RAM zero-copy if the launcher published the regions. + if let Some(regions) = GUEST_RAM.get().and_then(|f| f()) { + tracing::info!( + count = regions.len(), + "cuda-host: guest-RAM zero-copy enabled" + ); + gpu.set_guest_ram(regions); + } + Box::new(gpu) + } + Err(e) => { + tracing::info!("cuda-host: no GPU driver ({e}) — CPU emulation backend"); + Box::new(CpuBackend::default()) + } + } +} diff --git a/src/data/consts.rs b/src/data/consts.rs new file mode 100644 index 0000000..8361bf6 --- /dev/null +++ b/src/data/consts.rs @@ -0,0 +1,18 @@ +/// Bytes per mebibyte +pub const BYTES_PER_MIB: u64 = 1024 * 1024; + +/// Bytes per gibibyte (GiB). +pub const BYTES_PER_GIB: u64 = 1024 * 1024 * 1024; + +/// Name of the environment variable that overrides the directory used to +/// locate bundled native libraries for smolvm. +/// +/// If set, smolvm checks this directory before falling back to paths relative +/// to the current executable. This is primarily used by embedded runtimes. +pub const ENV_SMOLVM_LIB_DIR: &str = "SMOLVM_LIB_DIR"; + +/// Name of the environment variable that controls libkrun's log level. +/// +/// Accepted values are integer levels understood by libkrun +/// (`0 = off`, `1 = error`, `2 = warn`, `3 = info`, `4 = debug`). +pub const ENV_SMOLVM_KRUN_LOG_LEVEL: &str = "SMOLVM_KRUN_LOG_LEVEL"; diff --git a/src/data/disk.rs b/src/data/disk.rs new file mode 100644 index 0000000..3cea024 --- /dev/null +++ b/src/data/disk.rs @@ -0,0 +1,106 @@ +//! Canonical shared disk type metadata. + +use serde::{Deserialize, Serialize}; + +use crate::data::storage::{ + DEFAULT_OVERLAY_SIZE_GIB, DEFAULT_STORAGE_SIZE_GIB, OVERLAY_DISK_FILENAME, + STORAGE_DISK_FILENAME, +}; + +/// On-disk image format for a VM's block disks. Fork clones attach a `Qcow2` +/// copy-on-write overlay backed by the golden's `Raw` disk; every other VM uses +/// `Raw` directly. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum DiskFormat { + /// A flat raw disk image. + #[default] + Raw, + /// A qcow2 image, used as a copy-on-write overlay over a backing disk. + Qcow2, +} + +impl DiskFormat { + /// File extension used for disks of this format. + pub fn extension(self) -> &'static str { + match self { + DiskFormat::Raw => "raw", + DiskFormat::Qcow2 => "qcow2", + } + } + + /// The disk-format integer libkrun's `krun_add_disk2` expects. + pub fn to_krun_u32(self) -> u32 { + match self { + DiskFormat::Raw => 0, + DiskFormat::Qcow2 => 1, + } + } +} + +/// Marker type for the persistent rootfs overlay disk. +#[derive(Debug, Clone, Copy)] +pub enum Overlay {} + +/// Marker type for the shared storage disk. +#[derive(Debug, Clone, Copy)] +pub enum Storage {} + +/// Compile-time metadata for a typed VM disk. +pub trait DiskType { + /// Human-readable disk type name used in logs and errors. + const NAME: &'static str; + /// Default filename for this disk type. + const DEFAULT_FILENAME: &'static str; + /// Default size for this disk type, in GiB. + const DEFAULT_SIZE_GIB: u64; + /// Preformatted template filename for this disk type. + const TEMPLATE_FILENAME: &'static str; + /// ext4 volume label used when formatting this disk type. + const VOLUME_LABEL: &'static str; +} + +impl DiskType for Overlay { + const NAME: &'static str = "overlay"; + const DEFAULT_FILENAME: &'static str = OVERLAY_DISK_FILENAME; + const DEFAULT_SIZE_GIB: u64 = DEFAULT_OVERLAY_SIZE_GIB; + const TEMPLATE_FILENAME: &'static str = "overlay-template.ext4"; + const VOLUME_LABEL: &'static str = "smolvm-overlay"; +} + +impl DiskType for Storage { + const NAME: &'static str = "storage"; + const DEFAULT_FILENAME: &'static str = STORAGE_DISK_FILENAME; + const DEFAULT_SIZE_GIB: u64 = DEFAULT_STORAGE_SIZE_GIB; + const TEMPLATE_FILENAME: &'static str = "storage-template.ext4"; + const VOLUME_LABEL: &'static str = "smolvm"; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn disk_format_defaults_to_raw() { + assert_eq!(DiskFormat::default(), DiskFormat::Raw); + } + + #[test] + fn disk_format_extension_and_krun_value() { + assert_eq!(DiskFormat::Raw.extension(), "raw"); + assert_eq!(DiskFormat::Qcow2.extension(), "qcow2"); + // Must match libkrun's ImageType: Raw=0, Qcow2=1. + assert_eq!(DiskFormat::Raw.to_krun_u32(), 0); + assert_eq!(DiskFormat::Qcow2.to_krun_u32(), 1); + } + + #[test] + fn disk_format_serde_roundtrip_lowercase() { + assert_eq!( + serde_json::to_string(&DiskFormat::Qcow2).unwrap(), + "\"qcow2\"" + ); + let parsed: DiskFormat = serde_json::from_str("\"raw\"").unwrap(); + assert_eq!(parsed, DiskFormat::Raw); + } +} diff --git a/src/data/errors.rs b/src/data/errors.rs new file mode 100644 index 0000000..50953f0 --- /dev/null +++ b/src/data/errors.rs @@ -0,0 +1,662 @@ +//! Error types for smolvm. +//! +//! # Error Message Style Guide +//! +//! All error messages follow a consistent format for clarity and actionability: +//! +//! - **Format**: `" failed: "` or `" not found: "` +//! - **Case**: All lowercase (Rust convention for error messages) +//! - **Context**: Include relevant identifiers (VM name, path, key) when available +//! - **Actionability**: Messages should help users understand what went wrong and how to fix it +//! +//! ## Preferred Patterns +//! +//! ```text +//! // Operation failures (use "failed" consistently) +//! "vm creation failed: insufficient memory" +//! "database write failed: disk full" +//! +//! // Not found errors (use structured variants) +//! "vm not found: my-vm" +//! "mount source not found: /path/to/dir" +//! +//! // Invalid state/input errors +//! "invalid vm state: expected running, got stopped" +//! "invalid mount path: source must be absolute" +//! ``` +//! +//! ## Anti-patterns to Avoid +//! +//! ```text +//! // Don't use "error" as a suffix - redundant +//! "storage error: ..." // Bad +//! "storage operation failed: ..." // Good +//! +//! // Don't omit context when available +//! "failed to open database" // Bad - which database? +//! "database open failed: /path/to/db" // Good +//! ``` + +use std::path::PathBuf; +use thiserror::Error; + +/// Classification for agent errors, used to map to HTTP status codes +/// without fragile string matching on error messages. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum AgentErrorKind { + /// Resource not found (maps to 404). + NotFound, + /// Conflict / resource already exists (maps to 409). + Conflict, + /// General error (maps to 500). + #[default] + Other, +} + +/// Result type alias using smolvm's Error type. +pub type Result = std::result::Result; + +/// Errors that can occur in smolvm operations. +/// +/// Error messages follow a consistent format. See module documentation for style guide. +#[derive(Error, Debug)] +pub enum Error { + // ======================================================================== + // VM Lifecycle Errors + // ======================================================================== + /// Failed to create a VM. + #[error("vm creation failed: {0}")] + VmCreation(String), + + /// VM not found by name. + #[error("vm not found: {name}")] + VmNotFound { + /// Name of the VM that was not found. + name: String, + }, + + /// Hypervisor is not available on this system. + #[error("hypervisor not available: {0}")] + HypervisorUnavailable(String), + + /// VM is in an invalid state for the requested operation. + #[error("invalid vm state: expected {expected}, got {actual}")] + InvalidState { + /// Expected state. + expected: String, + /// Actual state. + actual: String, + }, + + // ======================================================================== + // Rootfs Errors + // ======================================================================== + /// Rootfs operation failed. + #[error("rootfs operation failed: {0}")] + Rootfs(String), + + /// Rootfs path does not exist. + #[error("rootfs not found: {}", path.display())] + RootfsNotFound { + /// Path that was not found. + path: PathBuf, + }, + + // ======================================================================== + // Storage Errors + // ======================================================================== + /// Storage operation failed. + #[error("storage operation failed: {operation}: {reason}")] + Storage { + /// The operation that failed (e.g., "create directory", "copy template"). + operation: String, + /// The reason for the failure. + reason: String, + }, + + /// Disk not found at expected path. + #[error("disk not found: {}", path.display())] + DiskNotFound { + /// Path to the disk. + path: PathBuf, + }, + + // ======================================================================== + // Mount Errors + // ======================================================================== + /// Mount operation failed. + #[error("mount operation failed: {operation}: {reason}")] + Mount { + /// The operation that failed (e.g., "validate source", "canonicalize path"). + operation: String, + /// The reason for the failure. + reason: String, + }, + + /// Invalid mount path specification. + #[error("invalid mount path: {reason}")] + InvalidMountPath { + /// Explanation of why the path is invalid. + reason: String, + }, + + /// Mount source path does not exist. + #[error("mount source not found: {}", path.display())] + MountSourceNotFound { + /// Path that was not found. + path: PathBuf, + }, + + // ======================================================================== + // Configuration Errors + // ======================================================================== + /// Configuration operation failed. + #[error("config operation failed: {operation}: {reason}")] + Config { + /// The operation that failed (e.g., "load", "save", "parse"). + operation: String, + /// The reason for the failure. + reason: String, + }, + + // ======================================================================== + // Database Errors + // ======================================================================== + /// Database operation failed. + #[error("database operation failed: {operation}: {reason}")] + Database { + /// The operation that failed (e.g., "open", "read", "write"). + operation: String, + /// The reason for the failure. + reason: String, + }, + + /// Database is not available (closed or not initialized). + #[error("database not available: {0}")] + DatabaseUnavailable(String), + + // ======================================================================== + // Command Execution Errors + // ======================================================================== + /// External command failed. + #[error("command '{command}' failed: {reason}")] + CommandFailed { + /// The command that failed. + command: String, + /// Error message or reason for failure. + reason: String, + }, + + // ======================================================================== + // Agent VM Errors + // ======================================================================== + /// Agent operation failed (structured). + #[error("agent operation failed: {operation}: {reason}")] + Agent { + /// The operation that failed (e.g., "start", "connect", "pull image"). + operation: String, + /// The reason for the failure. + reason: String, + /// Classification for HTTP status mapping. + kind: AgentErrorKind, + }, + + // ======================================================================== + // KVM Errors (Linux) + // ======================================================================== + /// KVM is not available (module not loaded). + #[error("kvm not available: {0}")] + KvmUnavailable(String), + + /// KVM permission denied (user not in kvm group). + #[error("kvm permission denied: {0}")] + KvmPermission(String), + + // ======================================================================== + // IO Errors + // ======================================================================== + /// IO error wrapper. + #[error("io operation failed: {0}")] + Io(#[from] std::io::Error), +} + +impl Error { + // ======================================================================== + // VM Error Constructors + // ======================================================================== + + /// Create a VM creation error. + pub fn vm_creation(reason: impl Into) -> Self { + Self::VmCreation(reason.into()) + } + + /// Create a VM not found error. + pub fn vm_not_found(name: impl Into) -> Self { + Self::VmNotFound { name: name.into() } + } + + // ======================================================================== + // Rootfs Error Constructors + // ======================================================================== + + /// Create a rootfs operation error. + pub fn rootfs(reason: impl Into) -> Self { + Self::Rootfs(reason.into()) + } + + // ======================================================================== + // Storage Error Constructors + // ======================================================================== + + /// Create a storage operation error. + pub fn storage(operation: impl Into, reason: impl Into) -> Self { + Self::Storage { + operation: operation.into(), + reason: reason.into(), + } + } + + // ======================================================================== + // Mount Error Constructors + // ======================================================================== + + /// Create a mount operation error. + pub fn mount(operation: impl Into, reason: impl Into) -> Self { + Self::Mount { + operation: operation.into(), + reason: reason.into(), + } + } + + /// Create an invalid mount path error. + pub fn invalid_mount_path(reason: impl Into) -> Self { + Self::InvalidMountPath { + reason: reason.into(), + } + } + + // ======================================================================== + // Config Error Constructors + // ======================================================================== + + /// Create a config operation error. + pub fn config(operation: impl Into, reason: impl Into) -> Self { + Self::Config { + operation: operation.into(), + reason: reason.into(), + } + } + + // ======================================================================== + // Database Error Constructors + // ======================================================================== + + /// Create a database operation error. + pub fn database(operation: impl Into, reason: impl Into) -> Self { + Self::Database { + operation: operation.into(), + reason: reason.into(), + } + } + + /// Create a database unavailable error. + pub fn database_unavailable(reason: impl Into) -> Self { + Self::DatabaseUnavailable(reason.into()) + } + + // ======================================================================== + // Command Error Constructors + // ======================================================================== + + /// Create a command failed error. + pub fn command_failed(command: impl Into, reason: impl Into) -> Self { + Self::CommandFailed { + command: command.into(), + reason: reason.into(), + } + } + + // ======================================================================== + // Agent Error Constructors + // ======================================================================== + + /// Create an agent operation error. + pub fn agent(operation: impl Into, reason: impl Into) -> Self { + Self::Agent { + operation: operation.into(), + reason: reason.into(), + kind: AgentErrorKind::Other, + } + } + + /// Create an agent "not found" error (maps to 404). + pub fn agent_not_found(operation: impl Into, reason: impl Into) -> Self { + Self::Agent { + operation: operation.into(), + reason: reason.into(), + kind: AgentErrorKind::NotFound, + } + } + + /// Create an agent "conflict" error (maps to 409). + pub fn agent_conflict(operation: impl Into, reason: impl Into) -> Self { + Self::Agent { + operation: operation.into(), + reason: reason.into(), + kind: AgentErrorKind::Conflict, + } + } + + // ======================================================================== + // KVM Error Constructors + // ======================================================================== + + /// Create a KVM unavailable error. + pub fn kvm_unavailable(reason: impl Into) -> Self { + Self::KvmUnavailable(reason.into()) + } + + /// Create a KVM permission error. + pub fn kvm_permission(reason: impl Into) -> Self { + Self::KvmPermission(reason.into()) + } + + /// Returns true if this is an `Io` variant. + pub fn is_io(&self) -> bool { + matches!(self, Self::Io(_)) + } + + /// If this is an `Io` variant, return the inner `ErrorKind`. + pub fn source_io_error_kind(&self) -> Option { + match self { + Self::Io(e) => Some(e.kind()), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Error messages should include context that helps users fix the problem. + /// These tests verify that error messages contain actionable information. + + // ======================================================================== + // VM Error Tests + // ======================================================================== + + #[test] + fn test_vm_not_found_includes_name() { + let err = Error::vm_not_found("my-test-vm"); + let msg = err.to_string(); + assert!(msg.contains("my-test-vm"), "Error should include VM name"); + assert!(msg.contains("not found"), "Error should indicate not found"); + } + + #[test] + fn test_vm_creation_includes_reason() { + let err = Error::vm_creation("insufficient memory"); + let msg = err.to_string(); + assert!( + msg.contains("creation failed"), + "Error should indicate creation failed" + ); + assert!( + msg.contains("insufficient memory"), + "Error should include reason" + ); + } + + #[test] + fn test_invalid_state_includes_both_states() { + let err = Error::InvalidState { + expected: "running".to_string(), + actual: "stopped".to_string(), + }; + let msg = err.to_string(); + assert!( + msg.contains("running"), + "Error should include expected state" + ); + assert!(msg.contains("stopped"), "Error should include actual state"); + } + + // ======================================================================== + // Storage Error Tests + // ======================================================================== + + #[test] + fn test_storage_error_includes_operation_and_reason() { + let err = Error::storage("create directory", "permission denied"); + let msg = err.to_string(); + assert!( + msg.contains("create directory"), + "Error should include operation" + ); + assert!( + msg.contains("permission denied"), + "Error should include reason" + ); + assert!( + msg.contains("operation failed"), + "Error should indicate failure" + ); + } + + // ======================================================================== + // Mount Error Tests + // ======================================================================== + + #[test] + fn test_mount_source_not_found_includes_path() { + let err = Error::MountSourceNotFound { + path: PathBuf::from("/nonexistent/path"), + }; + let msg = err.to_string(); + assert!( + msg.contains("/nonexistent/path"), + "Error should include the path" + ); + assert!(msg.contains("not found"), "Error should indicate not found"); + } + + #[test] + fn test_mount_error_includes_operation_and_reason() { + let err = Error::mount("validate source", "path does not exist"); + let msg = err.to_string(); + assert!( + msg.contains("validate source"), + "Error should include operation" + ); + assert!( + msg.contains("path does not exist"), + "Error should include reason" + ); + } + + #[test] + fn test_invalid_mount_path_includes_reason() { + let err = Error::invalid_mount_path("source must be absolute"); + let msg = err.to_string(); + assert!( + msg.contains("absolute"), + "Error should explain what's wrong" + ); + assert!( + msg.contains("invalid mount path"), + "Error should indicate invalid path" + ); + } + + // ======================================================================== + // Rootfs Error Tests + // ======================================================================== + + #[test] + fn test_rootfs_not_found_includes_path() { + let err = Error::RootfsNotFound { + path: PathBuf::from("/my/rootfs"), + }; + let msg = err.to_string(); + assert!( + msg.contains("/my/rootfs"), + "Error should include rootfs path" + ); + } + + // ======================================================================== + // Database Error Tests + // ======================================================================== + + #[test] + fn test_database_error_includes_operation_and_reason() { + let err = Error::database("open", "file locked by another process"); + let msg = err.to_string(); + assert!(msg.contains("open"), "Error should include operation"); + assert!(msg.contains("file locked"), "Error should include reason"); + assert!( + msg.contains("operation failed"), + "Error should indicate failure" + ); + } + + #[test] + fn test_database_unavailable_includes_reason() { + let err = Error::database_unavailable("database is closed"); + let msg = err.to_string(); + assert!(msg.contains("closed"), "Error should include reason"); + assert!( + msg.contains("not available"), + "Error should indicate unavailable" + ); + } + + // ======================================================================== + // Command Error Tests + // ======================================================================== + + #[test] + fn test_command_failed_includes_command_and_reason() { + let err = Error::command_failed("crane", "image not found"); + let msg = err.to_string(); + assert!(msg.contains("crane"), "Error should include command name"); + assert!( + msg.contains("image not found"), + "Error should include reason" + ); + assert!(msg.contains("failed"), "Error should indicate failure"); + } + + // ======================================================================== + // Agent Error Tests + // ======================================================================== + + #[test] + fn test_agent_error_includes_operation_and_reason() { + let err = Error::agent("pull image", "registry unavailable"); + let msg = err.to_string(); + assert!(msg.contains("pull image"), "Error should include operation"); + assert!( + msg.contains("registry unavailable"), + "Error should include reason" + ); + assert!( + msg.contains("operation failed"), + "Error should indicate failure" + ); + } + + // ======================================================================== + // Config Error Tests + // ======================================================================== + + #[test] + fn test_config_error_includes_operation_and_reason() { + let err = Error::config("load", "file not found"); + let msg = err.to_string(); + assert!(msg.contains("load"), "Error should include operation"); + assert!( + msg.contains("file not found"), + "Error should include reason" + ); + } + + // ======================================================================== + // Error Message Format Consistency Tests + // ======================================================================== + + #[test] + fn test_all_errors_are_lowercase() { + // Verify error messages don't start with capital letters (Rust convention) + let errors: Vec = vec![ + Error::vm_creation("test"), + Error::vm_not_found("test"), + Error::rootfs("test"), + Error::storage("op", "reason"), + Error::mount("op", "reason"), + Error::invalid_mount_path("reason"), + Error::config("op", "reason"), + Error::database("op", "reason"), + Error::database_unavailable("reason"), + Error::command_failed("cmd", "reason"), + Error::agent("op", "reason"), + Error::kvm_unavailable("reason"), + Error::kvm_permission("reason"), + ]; + + for err in errors { + let msg = err.to_string(); + let first_char = msg.chars().next().unwrap(); + assert!( + first_char.is_lowercase(), + "Error message should start lowercase: {}", + msg + ); + } + } + + #[test] + fn test_error_messages_contain_failed_or_not_found() { + // Verify error messages follow our convention of "failed" or "not found" + let operation_errors: Vec = vec![ + Error::vm_creation("test"), + Error::storage("op", "reason"), + Error::mount("op", "reason"), + Error::database("op", "reason"), + Error::command_failed("cmd", "reason"), + Error::agent("op", "reason"), + ]; + + for err in operation_errors { + let msg = err.to_string(); + assert!( + msg.contains("failed"), + "Operation error should contain 'failed': {}", + msg + ); + } + + let not_found_errors: Vec = vec![ + Error::vm_not_found("test"), + Error::MountSourceNotFound { + path: PathBuf::from("/test"), + }, + Error::RootfsNotFound { + path: PathBuf::from("/test"), + }, + Error::DiskNotFound { + path: PathBuf::from("/test"), + }, + ]; + + for err in not_found_errors { + let msg = err.to_string(); + assert!( + msg.contains("not found"), + "Not found error should contain 'not found': {}", + msg + ); + } + } +} diff --git a/src/data/image_source.rs b/src/data/image_source.rs new file mode 100644 index 0000000..ee9c687 --- /dev/null +++ b/src/data/image_source.rs @@ -0,0 +1,498 @@ +//! Classifying the `--image` value into an image *source*. +//! +//! smolvm is a microVM runtime, not a container runtime: it does not parse OCI +//! manifests, extract layers, or apply whiteouts itself. It delegates producing +//! a root filesystem to container-specific tooling (the guest's bundled `crane` +//! for registries and `docker save` archives, or the user for an already- +//! unpacked directory). This module's only job is to decide *which* kind of +//! source the user gave, so the right delegation runs. +//! +//! Detection is intentionally conservative so a bare registry reference is never +//! mistaken for a local path: +//! - `-` → an archive streamed on stdin (`docker save img | smolvm … --image -`). +//! - an explicit path (`/…`, `./…`, `../…`) or an archive-extensioned name +//! (`*.tar`, `*.tar.gz`, `*.tgz`) → a local source. A directory is used as a +//! ready-made rootfs; anything else is treated as a `docker save` archive. +//! - everything else (`alpine`, `repo:tag`, `ghcr.io/owner/repo@sha256:…`) → a +//! registry reference, even if a same-named file happens to sit in the cwd. + +use crate::{Error, Result}; +use sha2::{Digest, Sha256}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +/// Where a machine's root filesystem comes from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ImageSource { + /// An OCI registry reference (`alpine`, `repo:tag`, `repo@sha256:…`). + /// Resolved by the guest agent pulling with `crane` — the existing path. + Registry(String), + /// A `docker save` / `podman save` tar archive (optionally gzipped), read + /// from a file or stdin. Flattened into a rootfs by `crane export`. + Archive(ArchiveInput), + /// An already-unpacked root filesystem directory (apptainer-style). Used + /// as-is — no extraction. + Directory(PathBuf), +} + +/// Where an [`ImageSource::Archive`]'s bytes come from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ArchiveInput { + /// Read the archive from this file path. + File(PathBuf), + /// Read the archive from stdin (`--image -`). + Stdin, +} + +/// Known archive filename suffixes that mark a bare name as a local archive +/// even without an explicit path prefix (e.g. `image.tar` in the cwd). +const ARCHIVE_SUFFIXES: [&str; 3] = [".tar", ".tar.gz", ".tgz"]; + +/// Classify a `--image` value into its [`ImageSource`]. +/// +/// This decides *intent* only; whether the path actually exists (and is a valid +/// archive) is validated when the source is resolved, so a missing `./foo.tar` +/// produces a clear "no such file" rather than a confusing registry error. +pub fn classify(image: &str) -> ImageSource { + if image == "-" { + return ImageSource::Archive(ArchiveInput::Stdin); + } + if looks_local(image) { + let path = Path::new(image); + // Only an existing directory is treated as a ready-made rootfs; a + // missing path or a regular file is treated as an archive (resolve + // surfaces a clear error if it's absent). + if path.is_dir() { + return ImageSource::Directory(path.to_path_buf()); + } + return ImageSource::Archive(ArchiveInput::File(path.to_path_buf())); + } + ImageSource::Registry(image.to_string()) +} + +/// Whether a value should be treated as a local path rather than a registry +/// reference: an explicit path prefix or a known archive suffix. +fn looks_local(image: &str) -> bool { + image.starts_with('/') + || image.starts_with("./") + || image.starts_with("../") + || ARCHIVE_SUFFIXES + .iter() + .any(|suffix| image.ends_with(suffix)) +} + +/// Whether a local path is a Dockerfile rather than an image archive — so we can +/// reject it with a build-first hint instead of a confusing flatten failure. +/// Detected by the conventional name (`Dockerfile`/`Containerfile`, or a +/// `*.Dockerfile`/`*.Containerfile` suffix) or a first meaningful line of `FROM`. +fn looks_like_dockerfile(path: &Path) -> bool { + if let Some(name) = path.file_name().and_then(|n| n.to_str()) { + let lower = name.to_ascii_lowercase(); + if lower == "dockerfile" + || lower == "containerfile" + || lower.ends_with(".dockerfile") + || lower.ends_with(".containerfile") + { + return true; + } + } + // Content sniff over the first few KB only (never slurp a multi-GB archive): + // skip blank/comment lines; a Dockerfile opens with FROM (or an ARG before + // it). A tar archive's header is binary, so the UTF-8 read simply fails. + let mut head = [0u8; 4096]; + let Ok(mut file) = std::fs::File::open(path) else { + return false; + }; + let Ok(n) = std::io::Read::read(&mut file, &mut head) else { + return false; + }; + let Ok(text) = std::str::from_utf8(&head[..n]) else { + return false; + }; + for line in text.lines().take(50) { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let word = line.split_whitespace().next().unwrap_or(""); + return word.eq_ignore_ascii_case("FROM") || word.eq_ignore_ascii_case("ARG"); + } + false +} + +/// A classified source resolved into something launchable. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ResolvedImage { + /// Pulled from a registry by the guest agent — the existing path. + Registry(String), + /// A local source materialized on the host. `reference` is the stable ref + /// persisted on the machine record; `packed_layers_dir` is mounted into the + /// guest via virtiofs (the same path `.smolmachine` layers use). + Local { + /// Stable `local:` / `local-dir:` reference to persist on + /// the machine record so later starts re-resolve to the same source. + reference: String, + /// Host directory mounted into the guest via virtiofs — the staged + /// archive's cache dir, or the rootfs directory itself. + packed_layers_dir: PathBuf, + }, +} + +/// Largest archive accepted. The staged copy plus the guest's flattened rootfs +/// both consume disk, so the default guards against runaway/hostile inputs while +/// still covering very large dev images. +/// +/// Override with `SMOLVM_MAX_IMAGE_BYTES` (in bytes), or per-invocation with the +/// `--max-image-size` flag on `machine run`/`machine create` (which sets that +/// env var). A legitimate large image is a valid reason to raise it; the cap +/// only exists to bound disk use for untrusted inputs. +pub fn max_archive_bytes() -> u64 { + const DEFAULT: u64 = 8 * 1024 * 1024 * 1024; + std::env::var("SMOLVM_MAX_IMAGE_BYTES") + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|&n| n > 0) + .unwrap_or(DEFAULT) +} +/// Streaming hash/copy buffer. +const COPY_CHUNK: usize = 1 << 20; +/// Filename the staged archive is stored under inside its cache dir. +const ARCHIVE_FILE: &str = "archive.tar"; + +/// Resolve a classified [`ImageSource`] into a [`ResolvedImage`]. +/// +/// Registry refs pass straight through (the guest pulls them). Archives are +/// content-hashed and staged into a shared cache, so identical inputs dedupe +/// and re-runs skip the staging; directories are validated and used in place. +/// Both local kinds yield a `packed_layers_dir` to mount and a `local:…` +/// reference to persist on the machine record. +pub fn resolve(source: ImageSource) -> Result { + match source { + ImageSource::Registry(reference) => Ok(ResolvedImage::Registry(reference)), + ImageSource::Directory(path) => resolve_directory(&path), + ImageSource::Archive(input) => resolve_archive(input), + } +} + +fn resolve_directory(path: &Path) -> Result { + let canonical = path.canonicalize().map_err(|e| { + Error::config( + "--image", + format!("cannot use rootfs directory {}: {}", path.display(), e), + ) + })?; + if !canonical.is_dir() { + return Err(Error::config( + "--image", + format!("{} is not a directory", canonical.display()), + )); + } + Ok(ResolvedImage::Local { + reference: format!("{LOCAL_DIR_PREFIX}{}", canonical.display()), + packed_layers_dir: canonical, + }) +} + +fn resolve_archive(input: ArchiveInput) -> Result { + let cache_base = archive_cache_base()?; + std::fs::create_dir_all(&cache_base)?; + let hash = match input { + ArchiveInput::File(path) => stage_from_file(&path, &cache_base)?, + ArchiveInput::Stdin => stage_from_stdin(&cache_base)?, + }; + Ok(ResolvedImage::Local { + reference: format!("{LOCAL_ARCHIVE_PREFIX}{hash}"), + packed_layers_dir: cache_base.join(hash), + }) +} + +/// Hash an on-disk archive and hardlink (or copy) it into `cache//`. +fn stage_from_file(path: &Path, cache_base: &Path) -> Result { + let meta = std::fs::metadata(path).map_err(|e| { + Error::config( + "--image", + format!("cannot read archive {}: {}", path.display(), e), + ) + })?; + if !meta.is_file() { + return Err(Error::config( + "--image", + format!("{} is not a file", path.display()), + )); + } + if looks_like_dockerfile(path) { + return Err(Error::config( + "--image", + format!( + "{} looks like a Dockerfile, not an image.\n\ + smolvm boots images, it does not build them — build first, then \ + pass the result:\n \ + docker build -t myapp . && docker save myapp | smolvm machine run --image - -- ...", + path.display() + ), + )); + } + if meta.len() > max_archive_bytes() { + return Err(too_large(meta.len())); + } + let hash = hash_file(path)?; + let archive_path = cache_base.join(&hash).join(ARCHIVE_FILE); + if !archive_path.exists() { + std::fs::create_dir_all(archive_path.parent().expect("hash dir has a parent"))?; + link_or_copy(path, &archive_path)?; + } + Ok(hash) +} + +/// Stream stdin to a temp file (hashing as we go), then place it at +/// `cache//`. Single-pass, so one write is unavoidable. +fn stage_from_stdin(cache_base: &Path) -> Result { + let mut tmp = tempfile::NamedTempFile::new_in(cache_base)?; + let mut hasher = Sha256::new(); + let mut stdin = std::io::stdin().lock(); + let mut buf = vec![0u8; COPY_CHUNK]; + let mut total: u64 = 0; + loop { + let n = stdin.read(&mut buf)?; + if n == 0 { + break; + } + total += n as u64; + if total > max_archive_bytes() { + return Err(too_large(total)); + } + hasher.update(&buf[..n]); + tmp.write_all(&buf[..n])?; + } + tmp.flush()?; + let hash = hex::encode(hasher.finalize()); + let archive_path = cache_base.join(&hash).join(ARCHIVE_FILE); + if archive_path.exists() { + return Ok(hash); // already staged; the temp file is dropped/removed + } + std::fs::create_dir_all(archive_path.parent().expect("hash dir has a parent"))?; + tmp.persist(&archive_path) + .map_err(|e| Error::storage("stage stdin archive", e.to_string()))?; + Ok(hash) +} + +fn hash_file(path: &Path) -> Result { + let mut file = std::fs::File::open(path)?; + let mut hasher = Sha256::new(); + let mut buf = vec![0u8; COPY_CHUNK]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + } + Ok(hex::encode(hasher.finalize())) +} + +/// Hardlink `src` into the cache, copying as a fallback across filesystems. +fn link_or_copy(src: &Path, dst: &Path) -> Result<()> { + if std::fs::hard_link(src, dst).is_ok() { + return Ok(()); + } + std::fs::copy(src, dst)?; + Ok(()) +} + +/// Reference prefixes produced by [`resolve`] for the two local-source kinds. +const LOCAL_ARCHIVE_PREFIX: &str = "local:"; +const LOCAL_DIR_PREFIX: &str = "local-dir:"; + +/// Whether a persisted image reference points at a local source (produced by +/// [`resolve`]) rather than a registry. +pub fn is_local_ref(reference: &str) -> bool { + reference.starts_with(LOCAL_ARCHIVE_PREFIX) || reference.starts_with(LOCAL_DIR_PREFIX) +} + +/// Map a persisted `local:…`/`local-dir:…` reference back to the host directory +/// to mount into the guest via virtiofs, so a later `start` re-resolves to the +/// same source without re-staging. Returns `None` for a registry reference. The +/// directory may be gone if the archive cache was pruned — `start` then fails +/// with a clear "no such directory" rather than silently pulling. +pub fn packed_layers_dir_for_ref(reference: &str) -> Option { + if let Some(hash) = reference.strip_prefix(LOCAL_ARCHIVE_PREFIX) { + return archive_cache_base().ok().map(|base| base.join(hash)); + } + reference.strip_prefix(LOCAL_DIR_PREFIX).map(PathBuf::from) +} + +fn archive_cache_base() -> Result { + let base = dirs::cache_dir() + .ok_or_else(|| Error::storage("image archive cache", "no cache directory available"))?; + Ok(base.join("smolvm-image-archives")) +} + +fn too_large(bytes: u64) -> Error { + let limit = max_archive_bytes(); + Error::config( + "--image", + format!( + "image archive is {bytes} bytes, over the {limit}-byte limit. \ + Raise it with --max-image-size (e.g. --max-image-size 16GiB) or the \ + SMOLVM_MAX_IMAGE_BYTES env var if this is a legitimate large image." + ), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registry_refs_are_not_treated_as_local() { + for r in [ + "alpine", + "alpine:3.20", + "ghcr.io/owner/repo", + "ghcr.io/owner/repo:v1", + "repo@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "localhost:5000/myimage:dev", + ] { + assert_eq!(classify(r), ImageSource::Registry(r.to_string()), "{r}"); + } + } + + #[test] + fn stdin_dash_is_an_archive() { + assert_eq!(classify("-"), ImageSource::Archive(ArchiveInput::Stdin)); + } + + #[test] + fn archive_extensions_and_paths_are_local_archives() { + // Suffix-based (bare name in cwd) — file need not exist to classify. + for a in ["image.tar", "image.tar.gz", "image.tgz"] { + assert_eq!( + classify(a), + ImageSource::Archive(ArchiveInput::File(PathBuf::from(a))), + "{a}" + ); + } + // Explicit path prefixes, even without an archive extension. + for a in ["./img", "/abs/img.tar", "../up/img"] { + assert_eq!( + classify(a), + ImageSource::Archive(ArchiveInput::File(PathBuf::from(a))), + "{a}" + ); + } + } + + #[test] + fn dockerfiles_are_rejected_with_a_build_hint() { + let dir = tempfile::tempdir().unwrap(); + + // By name: `Dockerfile` (any case) and `*.Dockerfile`. + for name in [ + "Dockerfile", + "dockerfile", + "Containerfile", + "app.Dockerfile", + ] { + let p = dir.path().join(name); + std::fs::write(&p, b"unreadable as a name test").unwrap(); + assert!(looks_like_dockerfile(&p), "{name}"); + } + + // By content: a file not named Dockerfile but opening with FROM. + let by_content = dir.path().join("build-recipe"); + std::fs::write( + &by_content, + b"# build\nFROM alpine:3.20\nRUN apk add curl\n", + ) + .unwrap(); + assert!(looks_like_dockerfile(&by_content)); + + // A binary archive must not be mistaken for a Dockerfile. + let archive = dir.path().join("image.bin"); + std::fs::write(&archive, [0u8, 1, 2, 0xff, b'F', b'R', b'O', b'M']).unwrap(); + assert!(!looks_like_dockerfile(&archive)); + + // And resolve surfaces the build-first hint, not a flatten failure. + let err = stage_from_file(&dir.path().join("Dockerfile"), dir.path()).unwrap_err(); + assert!(err.to_string().contains("build"), "{err}"); + } + + #[test] + fn existing_directory_is_a_directory_source() { + let dir = tempfile::tempdir().unwrap(); + // Reference it with an explicit path prefix so it reads as local. + let as_path = format!("{}/.", dir.path().display()); + match classify(&as_path) { + ImageSource::Directory(p) => assert!(p.is_dir()), + other => panic!("expected Directory, got {other:?}"), + } + } + + #[test] + fn bare_name_matching_a_cwd_file_still_resolves_to_registry() { + // A registry ref like `alpine` must not be hijacked by a same-named + // file: it isn't an explicit path and has no archive suffix. + assert_eq!( + classify("alpine"), + ImageSource::Registry("alpine".to_string()) + ); + } + + #[test] + fn resolve_registry_passes_through() { + assert_eq!( + resolve(ImageSource::Registry("alpine".into())).unwrap(), + ResolvedImage::Registry("alpine".into()) + ); + } + + #[test] + fn stage_from_file_hashes_and_dedupes() { + let cache = tempfile::tempdir().unwrap(); + let src = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(src.path(), b"fake docker save archive").unwrap(); + + let h1 = stage_from_file(src.path(), cache.path()).unwrap(); + let staged = cache.path().join(&h1).join(ARCHIVE_FILE); + assert!(staged.exists(), "archive staged at {}", staged.display()); + assert_eq!(std::fs::read(&staged).unwrap(), b"fake docker save archive"); + + // Identical content → identical hash, idempotent (no error, reused). + let h2 = stage_from_file(src.path(), cache.path()).unwrap(); + assert_eq!(h1, h2); + } + + #[test] + fn local_refs_round_trip_to_a_packed_layers_dir() { + assert!(is_local_ref("local:abc123")); + assert!(is_local_ref("local-dir:/srv/rootfs")); + assert!(!is_local_ref("alpine")); + assert!(!is_local_ref("ghcr.io/o/r:v1")); + + // A dir ref maps straight back to the directory. + assert_eq!( + packed_layers_dir_for_ref("local-dir:/srv/rootfs"), + Some(PathBuf::from("/srv/rootfs")) + ); + // An archive ref maps under the shared cache base, keyed by hash. + let dir = packed_layers_dir_for_ref("local:deadbeef").unwrap(); + assert!(dir.ends_with("smolvm-image-archives/deadbeef")); + // Registry refs have no packed-layers dir. + assert_eq!(packed_layers_dir_for_ref("alpine"), None); + } + + #[test] + fn resolve_directory_yields_local_with_the_dir() { + let dir = tempfile::tempdir().unwrap(); + match resolve_directory(dir.path()).unwrap() { + ResolvedImage::Local { + reference, + packed_layers_dir, + } => { + assert!(reference.starts_with("local-dir:")); + assert_eq!(packed_layers_dir, dir.path().canonicalize().unwrap()); + } + other => panic!("expected Local, got {other:?}"), + } + } +} diff --git a/src/data/mod.rs b/src/data/mod.rs new file mode 100644 index 0000000..5c64bc4 --- /dev/null +++ b/src/data/mod.rs @@ -0,0 +1,168 @@ +//! Canonical shared data models and constants for smolvm. + +/// Shared constants used across the runtime and adapters. +pub mod consts; +/// Canonical disk type metadata shared by storage helpers. +pub mod disk; +/// Canonical error types used by the shared data layer. +#[path = "errors.rs"] +pub mod error; +/// Classifying `--image` into a registry ref, local archive, or rootfs dir. +pub mod image_source; +/// Canonical network-related data models. +pub mod network; +/// Canonical resource configuration data models. +pub mod resources; +/// Canonical storage and mount data models. +pub mod storage; + +/// Target VM identifier used by shared operations. +pub enum VmTarget { + /// The default micro vm + Default, + /// A specifically named micro vm + Named(String), +} + +impl VmTarget { + /// Return the stable VM name for this target. + pub fn name(&self) -> &str { + match self { + Self::Default => "default", + Self::Named(name) => name.as_str(), + } + } +} + +impl From<&str> for VmTarget { + fn from(name: &str) -> VmTarget { + if name == "default" { + VmTarget::Default + } else { + VmTarget::Named(String::from(name)) + } + } +} + +/// Sanity upper bound on VM name length. +/// +/// The on-disk layout uses a fixed-length hash of the name as the directory +/// name (see [`crate::agent::vm_data_dir`]), so name length doesn't affect +/// the socket path or any other filesystem budget. This constant is purely +/// a UX cap to reject obviously-absurd input (500+ char names). +pub const MAX_VM_NAME_LENGTH: usize = 128; + +/// Validate a persisted VM name. +pub fn validate_vm_name(name: &str, label: &str) -> Result<(), String> { + let first_char = name + .chars() + .next() + .ok_or_else(|| format!("{label} cannot be empty"))?; + + if name.len() > MAX_VM_NAME_LENGTH { + return Err(format!( + "{label} too long: {} characters (max {})", + name.len(), + MAX_VM_NAME_LENGTH + )); + } + + if !first_char.is_ascii_alphanumeric() { + return Err(format!( + "{label} must start with a letter or digit (got: {name:?})" + )); + } + + if name.ends_with('-') { + return Err(format!("{label} cannot end with a hyphen (got: {name:?})")); + } + + let mut prev_was_hyphen = false; + for c in name.chars() { + if c == '-' { + if prev_was_hyphen { + return Err(format!("{label} cannot contain consecutive hyphens")); + } + prev_was_hyphen = true; + } else { + prev_was_hyphen = false; + } + + if !c.is_ascii_alphanumeric() && c != '-' && c != '_' { + if c == '/' || c == '\\' { + return Err(format!("{label} cannot contain path separators")); + } + + return Err(format!("{label} contains invalid character: '{c}'")); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn validate_vm_name_accepts_valid_names() { + let valid = [ + "test", + "my-resource", + "my_resource", + "test123", + "123test", + "a", + "test-resource-123", + "TEST_RESOURCE", + ]; + + for name in valid { + assert!( + validate_vm_name(name, "machine name").is_ok(), + "expected '{name}' to be valid" + ); + } + } + + #[test] + fn validate_vm_name_enforces_max_length() { + assert!(validate_vm_name(&"a".repeat(MAX_VM_NAME_LENGTH), "machine name").is_ok()); + assert!(validate_vm_name(&"a".repeat(MAX_VM_NAME_LENGTH + 1), "machine name").is_err()); + } + + #[test] + fn validate_vm_name_rejects_invalid_names() { + let invalid = [ + ("", "empty"), + ("-test", "starts with hyphen"), + ("_test", "starts with underscore"), + (".test", "starts with dot"), + ("test-", "ends with hyphen"), + ("test--name", "consecutive hyphens"), + ("test/name", "forward slash"), + ("test\\name", "backslash"), + ("test name", "space"), + ("test@name", "at sign"), + ("../test", "path traversal"), + ("test.name", "dot"), + ("test:name", "colon"), + ("test#name", "hash"), + ]; + + for (name, desc) in invalid { + assert!( + validate_vm_name(name, "machine name").is_err(), + "expected '{name}' ({desc}) to be invalid" + ); + } + } + + #[test] + fn validate_vm_name_formats_with_label() { + assert_eq!( + validate_vm_name("test/name", "machine name").unwrap_err(), + "machine name cannot contain path separators" + ); + } +} diff --git a/src/data/network.rs b/src/data/network.rs new file mode 100644 index 0000000..45f639c --- /dev/null +++ b/src/data/network.rs @@ -0,0 +1,268 @@ +use ipnet::IpNet; +use std::net::{IpAddr, Ipv4Addr}; + +/// Fallback DNS server (Cloudflare) used when the host's resolver cannot be detected. +pub const FALLBACK_DNS: &str = "1.1.1.1"; + +/// Fallback DNS server as `IpAddr`. +pub const FALLBACK_DNS_ADDR: IpAddr = IpAddr::V4(Ipv4Addr::new(1, 1, 1, 1)); + +/// Detect the host's primary DNS server from /etc/resolv.conf. +/// Falls back to Cloudflare (1.1.1.1) if detection fails. +pub fn host_dns() -> IpAddr { + host_dns_from_resolv("/etc/resolv.conf").unwrap_or(FALLBACK_DNS_ADDR) +} + +/// Parse the first nameserver from a resolv.conf file. +fn host_dns_from_resolv(path: &str) -> Option { + let contents = std::fs::read_to_string(path).ok()?; + for line in contents.lines() { + let line = line.trim(); + if let Some(addr_str) = line.strip_prefix("nameserver") { + let addr_str = addr_str.trim(); + // Skip loopback — it's typically a local resolver (systemd-resolved, + // dnsmasq) that isn't reachable from inside the VM. + if let Ok(ip) = addr_str.parse::() { + if !ip.is_loopback() { + return Some(ip); + } + } + } + } + None +} + +/// Default DNS as string — prefers host's resolver, falls back to 1.1.1.1. +pub fn default_dns() -> String { + host_dns().to_string() +} + +/// Default DNS as `IpAddr`. +pub fn default_dns_addr() -> IpAddr { + host_dns() +} + +/// TCP port mapping from host to guest. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct PortMapping { + /// Port on the host. + pub host: u16, + /// Port inside the guest. + pub guest: u16, +} + +/// Check if any CIDR in the list covers the given IP address. +pub fn cidrs_contain_ip(cidrs: &[String], ip: &str) -> bool { + let ip: IpAddr = match ip.parse() { + Ok(ip) => ip, + Err(_) => return false, + }; + cidrs.iter().any(|cidr| { + cidr.parse::() + .or_else(|_| cidr.parse::().map(IpNet::from)) + .is_ok_and(|net| net.contains(&ip)) + }) +} + +/// Returns true if every CIDR in the list falls entirely within loopback ranges +/// (127.0.0.0/8 for IPv4, ::1/128 for IPv6). +/// +/// An empty slice returns false — no CIDRs means no policy at all, which is +/// distinct from an explicitly loopback-only policy. +pub fn cidrs_all_loopback(cidrs: &[String]) -> bool { + if cidrs.is_empty() { + return false; + } + cidrs.iter().all(|cidr| { + cidr.parse::() + .or_else(|_| cidr.parse::().map(IpNet::from)) + .is_ok_and(|net| net.network().is_loopback()) + }) +} + +/// Ensure the default DNS server is reachable in a CIDR allowlist. +/// +/// If none of the existing CIDRs cover the DNS IP, appends it as /32. +/// +/// Skipped when all CIDRs are loopback ranges — a loopback-only policy +/// intentionally blocks all external traffic, so auto-adding the DNS server +/// would violate the user's intent (e.g. `--outbound-localhost-only`). +pub fn ensure_dns_in_cidrs(cidrs: &mut Vec) { + if cidrs_all_loopback(cidrs) { + return; + } + let dns = host_dns(); + if !cidrs_contain_ip(cidrs, &dns.to_string()) { + cidrs.push(IpNet::from(dns).to_string()); + } +} + +impl PortMapping { + /// Create a new port mapping. + pub fn new(host: u16, guest: u16) -> Self { + Self { host, guest } + } + + /// Create a port mapping where host and guest ports are the same. + pub fn same(port: u16) -> Self { + Self { + host: port, + guest: port, + } + } + + /// Convert to `(host, guest)` tuple for storage. + pub fn to_tuple(&self) -> (u16, u16) { + (self.host, self.guest) + } + + /// Batch convert port mappings to tuple format. + pub fn to_tuples(ports: &[Self]) -> Vec<(u16, u16)> { + ports.iter().map(|p| p.to_tuple()).collect() + } + + /// Check for duplicate host ports in a list of mappings. + pub fn check_duplicates(ports: &[Self]) -> Result<(), String> { + let mut seen = std::collections::HashSet::new(); + for p in ports { + if !seen.insert(p.host) { + return Err(format!( + "duplicate host port {}: each host port can only be mapped once", + p.host + )); + } + } + Ok(()) + } + + /// Parse a port mapping specification (`HOST:GUEST` or `PORT`). + pub fn parse(spec: &str) -> Result { + if let Some((host, guest)) = spec.split_once(':') { + let host: u16 = host + .parse() + .map_err(|_| format!("invalid host port: {}", host))?; + if host == 0 { + return Err("host port 0 is not valid for VM port forwarding".to_string()); + } + let guest: u16 = guest + .parse() + .map_err(|_| format!("invalid guest port: {}", guest))?; + if guest == 0 { + return Err("guest port 0 is not valid for VM port forwarding".to_string()); + } + Ok(Self::new(host, guest)) + } else { + let port: u16 = spec + .parse() + .map_err(|_| format!("invalid port: {}", spec))?; + if port == 0 { + return Err("port 0 is not valid for VM port forwarding".to_string()); + } + Ok(Self::same(port)) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cidrs_contain_ip() { + assert!(cidrs_contain_ip(&["1.1.1.1".into()], "1.1.1.1")); + assert!(cidrs_contain_ip(&["1.1.1.1/32".into()], "1.1.1.1")); + assert!(cidrs_contain_ip(&["0.0.0.0/0".into()], "8.8.8.8")); + assert!(cidrs_contain_ip(&["10.0.0.0/8".into()], "10.5.3.1")); + assert!(!cidrs_contain_ip(&["10.0.0.0/8".into()], "1.1.1.1")); + assert!(cidrs_contain_ip( + &["192.168.1.0/24".into()], + "192.168.1.100" + )); + assert!(!cidrs_contain_ip(&["192.168.1.0/24".into()], "192.168.2.1")); + assert!(cidrs_contain_ip( + &["10.0.0.0/8".into(), "1.1.1.1/32".into()], + "1.1.1.1" + )); + assert!(!cidrs_contain_ip(&[], "1.1.1.1")); + assert!(!cidrs_contain_ip(&["not-a-cidr".into()], "1.1.1.1")); + } + + #[test] + fn test_ensure_dns_adds_when_missing() { + let dns_cidr = IpNet::from(host_dns()).to_string(); + let mut cidrs = vec!["10.0.0.0/8".to_string()]; + ensure_dns_in_cidrs(&mut cidrs); + assert_eq!(cidrs.len(), 2); + assert!(cidrs.contains(&dns_cidr)); + } + + #[test] + fn test_ensure_dns_skips_when_covered_by_subnet() { + // Build a subnet that actually covers the detected DNS server. + let dns = host_dns(); + let covering_cidr = match dns { + IpAddr::V4(v4) => format!("{}.0.0.0/8", v4.octets()[0]), + IpAddr::V6(v6) => { + // Use a /16 covering the detected IPv6 address. + let segs = v6.segments(); + format!("{:x}::/16", segs[0]) + } + }; + let mut cidrs = vec![covering_cidr]; + ensure_dns_in_cidrs(&mut cidrs); + assert_eq!(cidrs.len(), 1); + } + + #[test] + fn test_ensure_dns_skips_when_exact_match() { + let dns_cidr = IpNet::from(host_dns()).to_string(); + let mut cidrs = vec!["10.0.0.0/8".to_string(), dns_cidr]; + ensure_dns_in_cidrs(&mut cidrs); + assert_eq!(cidrs.len(), 2); + } + + #[test] + fn test_ensure_dns_skips_for_loopback_only_policy() { + let mut cidrs = vec!["127.0.0.0/8".to_string(), "::1/128".to_string()]; + ensure_dns_in_cidrs(&mut cidrs); + assert_eq!( + cidrs.len(), + 2, + "DNS must not be added for loopback-only policy" + ); + } + + #[test] + fn test_ensure_dns_adds_when_non_loopback_cidr_present() { + let dns_cidr = IpNet::from(host_dns()).to_string(); + let mut cidrs = vec!["127.0.0.0/8".to_string(), "10.0.0.0/8".to_string()]; + ensure_dns_in_cidrs(&mut cidrs); + assert_eq!(cidrs.len(), 3); + assert!(cidrs.contains(&dns_cidr)); + } + + #[test] + fn test_port_mapping_rejects_zero() { + assert!(PortMapping::parse("0:80").is_err()); + assert!(PortMapping::parse("80:0").is_err()); + assert!(PortMapping::parse("0").is_err()); + assert!(PortMapping::parse("1:1").is_ok()); + assert!(PortMapping::parse("8080:80").is_ok()); + } + + #[test] + fn test_cidrs_all_loopback() { + assert!(cidrs_all_loopback(&[ + "127.0.0.0/8".into(), + "::1/128".into() + ])); + assert!(cidrs_all_loopback(&["127.0.0.1/32".into()])); + assert!(!cidrs_all_loopback(&[])); + assert!(!cidrs_all_loopback(&["10.0.0.0/8".into()])); + assert!(!cidrs_all_loopback(&[ + "127.0.0.0/8".into(), + "10.0.0.0/8".into() + ])); + assert!(!cidrs_all_loopback(&["0.0.0.0/0".into()])); + } +} diff --git a/src/data/resources.rs b/src/data/resources.rs new file mode 100644 index 0000000..97e422c --- /dev/null +++ b/src/data/resources.rs @@ -0,0 +1,216 @@ +/// Default agent VM virtual CPU count. +/// vCPU threads sleep in the hypervisor when idle, so over-provisioning +/// is low-cost — the host OS time-slices them like any other threads. +pub const DEFAULT_MICROVM_CPU_COUNT: u8 = 4; +/// Default agent VM memory in MiB. +/// Virtio balloon with free page reporting means this is a ceiling, not a +/// reservation — the host only consumes what the guest actually uses. +pub const DEFAULT_MICROVM_MEMORY_MIB: u32 = 8192; + +/// Default VRAM (host shared memory region) exposed to the guest +/// virtio-gpu device, in MiB. +/// +/// This sizes the shared buffer libkrun reserves for GPU transfers +/// (textures, staging, Venus/Vulkan descriptor pools). 4 GiB is large +/// enough for typical Vulkan workloads including headless browsers and +/// modest compute, without being so big that low-memory hosts commit +/// most of their RAM when GPU is enabled. Override with +/// `--gpu-vram ` or Smolfile `gpu_vram = `. +pub const DEFAULT_GPU_VRAM_MIB: u32 = 4096; + +use crate::network::NetworkBackend; + +/// Resources available to a micro vm. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct VmResources { + /// Number of vCPUs. + pub cpus: u8, + /// Memory in MiB. + pub memory_mib: u32, + /// Enable outbound network access (TSI). + pub network: bool, + /// Preferred network backend override. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub network_backend: Option, + /// Enable GPU acceleration (virtio-gpu with Venus/Vulkan). + #[serde(default)] + pub gpu: bool, + /// GPU shared-memory region size in MiB (ignored if `gpu` is false). + /// `None` → use `DEFAULT_GPU_VRAM_MIB`. + #[serde(default)] + pub gpu_vram_mib: Option, + /// Enable Rosetta 2 for x86_64 binary translation on Apple Silicon. + #[serde(default)] + pub rosetta: bool, + /// Storage disk size in GiB (None = default 20 GiB). + pub storage_gib: Option, + /// Overlay disk size in GiB (None = default 10 GiB). + pub overlay_gib: Option, + /// Allowed egress CIDR ranges. None = unrestricted, Some([]) = deny all. + #[serde(default)] + pub allowed_cidrs: Option>, + /// Custom DNS resolver for the guest. None = backend default. + /// + /// Under TSI this becomes the guest's `/etc/resolv.conf` nameserver (the + /// guest's datagrams are proxied to it directly). Under virtio-net it + /// becomes the gateway's upstream resolver. Lets a VM on a network that + /// blocks the default resolver (e.g. 8.8.8.8) still resolve names. + #[serde(default)] + pub dns: Option, +} + +/// Minimum memory required for the VM to boot (kernel + agent). +const MIN_MEMORY_MIB: u32 = 64; + +/// Maximum vCPUs supported by the hypervisor (Hypervisor.framework on macOS). +/// Requests above this are silently capped by libkrun. +const MAX_EFFECTIVE_CPUS: u8 = 16; + +impl VmResources { + /// Validate resource values before starting a VM. Returns an error with + /// a clear message for values that would cause an opaque hypervisor failure. + pub fn validate(&self) -> Result<(), crate::Error> { + if self.cpus == 0 { + return Err(crate::Error::config( + "validate resources", + "CPU count must be at least 1", + )); + } + if self.cpus > MAX_EFFECTIVE_CPUS { + eprintln!( + "warning: requested {} vCPUs but the hypervisor supports at most {}; \ + the VM will run with {} vCPUs", + self.cpus, MAX_EFFECTIVE_CPUS, MAX_EFFECTIVE_CPUS + ); + } + if self.memory_mib < MIN_MEMORY_MIB { + return Err(crate::Error::config( + "validate resources", + format!( + "memory must be at least {} MiB (got {} MiB)", + MIN_MEMORY_MIB, self.memory_mib + ), + )); + } + // `None` means "use the default size"; only an explicit 0 is invalid. + // Caught here so `machine create` rejects it up front instead of + // persisting a machine that can never start. + if self.storage_gib == Some(0) { + return Err(crate::Error::config( + "validate resources", + "storage disk size must be greater than 0 GiB", + )); + } + if self.overlay_gib == Some(0) { + return Err(crate::Error::config( + "validate resources", + "overlay disk size must be greater than 0 GiB", + )); + } + Ok(()) + } +} + +impl Default for VmResources { + fn default() -> Self { + Self { + cpus: DEFAULT_MICROVM_CPU_COUNT, + memory_mib: DEFAULT_MICROVM_MEMORY_MIB, + network: false, + network_backend: None, + gpu: false, + gpu_vram_mib: None, + rosetta: false, + storage_gib: None, + overlay_gib: None, + allowed_cidrs: None, + dns: None, + } + } +} + +impl VmResources { + /// Effective GPU VRAM in MiB, applying the default when unset. + /// + /// Invariant: the returned value is always `>= 1`. `gpu_vram_mib` + /// is validated at ingress (`validate_gpu_vram_mib`) so it cannot + /// contain `Some(0)`; the default is 4 GiB. A `None` here means + /// "caller didn't specify," and we fill it with the default. + pub fn effective_gpu_vram_mib(&self) -> u32 { + self.gpu_vram_mib.unwrap_or(DEFAULT_GPU_VRAM_MIB) + } +} + +/// Validate a user-supplied GPU VRAM value. `Some(0)` is nonsensical +/// — virtio-gpu's shared memory region can't be zero-sized without +/// the guest driver crashing on first use. We reject at ingress +/// (CLI, Smolfile, and direct DB writes) rather than passing 0 down +/// to libkrun and hoping for a clean failure. +/// +/// `None` is fine (means "use default"). Positive values are +/// accepted without an upper bound; libkrun errors at allocation +/// time if the host can't back the region. +/// +/// Returns the value unchanged on success so callers can chain: +/// `params.gpu_vram_mib = validate_gpu_vram_mib(user_input)?;` +pub fn validate_gpu_vram_mib(v: Option) -> Result, &'static str> { + match v { + Some(0) => Err("--gpu-vram must be a positive number of MiB; \ + use a value >= 1 or omit the flag to get the default"), + other => Ok(other), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn effective_gpu_vram_defaults_when_unset() { + let r = VmResources::default(); + assert_eq!(r.effective_gpu_vram_mib(), DEFAULT_GPU_VRAM_MIB); + } + + #[test] + fn effective_gpu_vram_honors_override() { + let r = VmResources { + gpu_vram_mib: Some(8192), + ..Default::default() + }; + assert_eq!(r.effective_gpu_vram_mib(), 8192); + } + + #[test] + fn vm_resources_serde_roundtrip_preserves_vram() { + let r = VmResources { + gpu: true, + gpu_vram_mib: Some(2048), + ..Default::default() + }; + let json = serde_json::to_string(&r).unwrap(); + let back: VmResources = serde_json::from_str(&json).unwrap(); + assert_eq!(back.gpu_vram_mib, Some(2048)); + } + + #[test] + fn validate_gpu_vram_mib_rejects_zero() { + assert!(validate_gpu_vram_mib(Some(0)).is_err()); + } + + #[test] + fn validate_gpu_vram_mib_accepts_positive() { + assert_eq!(validate_gpu_vram_mib(Some(1)).unwrap(), Some(1)); + assert_eq!(validate_gpu_vram_mib(Some(4096)).unwrap(), Some(4096)); + // No upper bound — libkrun errors at allocation time if the + // host can't back the region. + assert_eq!( + validate_gpu_vram_mib(Some(u32::MAX)).unwrap(), + Some(u32::MAX) + ); + } + + #[test] + fn validate_gpu_vram_mib_accepts_none() { + assert_eq!(validate_gpu_vram_mib(None).unwrap(), None); + } +} diff --git a/src/data/storage.rs b/src/data/storage.rs new file mode 100644 index 0000000..b969fc3 --- /dev/null +++ b/src/data/storage.rs @@ -0,0 +1,394 @@ +use crate::data::error::{Error, Result}; +use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; + +/// Default size for the rootfs overlay disk (10 GiB sparse). +/// +/// This is a sparse file — only actually-written data consumes host disk space. +/// 10 GiB provides headroom for package installation (`apk add`, `pip install`, etc.) +/// without hitting "No space left on device" during typical development workflows. +pub const DEFAULT_OVERLAY_SIZE_GIB: u64 = 10; + +/// Default size for the shared storage disk (20 GiB sparse). +pub const DEFAULT_STORAGE_SIZE_GIB: u64 = 20; + +/// Overlay disk filename. +pub const OVERLAY_DISK_FILENAME: &str = "overlay.raw"; + +/// Storage disk filename. +pub const STORAGE_DISK_FILENAME: &str = "storage.raw"; + +/// Host directory mount. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct HostMount { + /// Path on the host. + pub source: PathBuf, + + /// Path inside the guest. + pub target: PathBuf, + + /// Read-only mount (default: true per DESIGN.md). + pub read_only: bool, +} + +impl HostMount { + /// Protected host paths that must never be mounted into the guest. + #[cfg(target_os = "macos")] + const ILLEGAL_SOURCE_MOUNT_PATH: &[(&str, bool)] = &[ + ("/", false), + ("/private/var", false), + ("/private/var/run", true), + ("/private/var/log", true), + ("/private/etc", true), + ("/usr", true), + ("/bin", true), + ("/sbin", true), + ("/lib", true), + ("/System", true), + ("/Library", true), + ]; + + /// Protected host paths that must never be mounted into the guest. + #[cfg(target_os = "linux")] + const ILLEGAL_SOURCE_MOUNT_PATH: &[(&str, bool)] = &[ + ("/", false), + ("/var", false), + ("/var/run", true), + ("/var/log", true), + ("/etc", true), + ("/usr", true), + ("/bin", true), + ("/sbin", true), + ("/lib", true), + ("/proc", true), + ("/sys", true), + ("/dev", true), + ("/run", true), + ]; + + /// Protected host paths that must never be mounted into the guest (Windows). + #[cfg(target_os = "windows")] + const ILLEGAL_SOURCE_MOUNT_PATH: &[(&str, bool)] = &[ + ("C:\\", false), + ("C:\\Windows", true), + ("C:\\Program Files", true), + ("C:\\Program Files (x86)", true), + ("C:\\ProgramData", true), + ]; + + /// Create a host mount with an explicit read-only flag. + pub fn new( + source: impl Into, + target: impl Into, + read_only: bool, + ) -> Result { + let mut mount = Self { + source: source.into(), + target: target.into(), + read_only, + }; + + if !mount.source.exists() { + return Err(Error::MountSourceNotFound { + path: mount.source.clone(), + }); + } + + if !mount.source.is_dir() { + return Err(Error::mount( + "validate host path", + format!( + "source path on host must be a directory (virtiofs limitation): {}", + mount.source.display() + ), + )); + } + + mount.source = mount.source.canonicalize().map_err(|e| { + Error::mount( + "canonicalize host path", + format!("'{}': {}", mount.source.display(), e), + ) + })?; + Self::validate(&mount)?; + Ok(mount) + } + + /// Parse a mount specification (`host_path:guest_path[:ro|:rw]`). + /// + /// If no mode is provided, the mount defaults to writable. + /// The source path is validated, required to be a directory, and canonicalized. + fn _parse(spec: &str) -> Result { + // Parse from the right so a Windows host path keeps its drive-letter + // colon (e.g. `C:\data:/data:ro`). The guest path is a Unix path with no + // colon, and the optional trailing mode is `ro`/`rw`; everything before + // the guest path is the host source. + let (rest, read_only) = match spec.rsplit_once(':') { + Some((head, "ro")) => (head, true), + Some((head, "rw")) => (head, false), + _ => (spec, false), + }; + match rest.rsplit_once(':') { + Some((source, target)) if !source.is_empty() && !target.is_empty() => { + Self::new(source, target, read_only) + } + _ => Err(Error::invalid_mount_path(format!( + "invalid format '{}' (expected host:guest[:ro|:rw])", + spec + ))), + } + } + + /// Parse multiple mount specifications. + pub fn parse(specs: &[String]) -> Result> { + let mounts: Vec = specs + .iter() + .map(|spec| Self::_parse(spec)) + .collect::>()?; + + // Reject duplicate guest targets. Silently letting a later `-v` + // shadow an earlier one (second-wins) hides a likely user mistake + // and makes the effective mount ambiguous. + let mut seen = std::collections::HashSet::new(); + for m in &mounts { + if !seen.insert(&m.target) { + return Err(Error::mount( + "validate mounts", + format!( + "duplicate mount target: {} is specified more than once", + m.target.display() + ), + )); + } + } + + Ok(mounts) + } + + fn validate(mount: &Self) -> Result<()> { + for (illegal_path, block_subtree) in Self::ILLEGAL_SOURCE_MOUNT_PATH { + let illegal_path = Path::new(illegal_path); + if mount.source == illegal_path + || (*block_subtree && mount.source.starts_with(illegal_path)) + { + return Err(Error::mount( + "validate host path", + format!( + "source path on host is a protected system path and cannot be mounted: {}", + mount.source.display() + ), + )); + } + } + + // Guest paths are always Linux paths; check Linux-absolute (leading '/') + // rather than `Path::is_absolute()`, which is host-platform-specific and + // wrongly rejects "/data" on Windows (where absolute means `C:\...`). + let target_is_absolute = mount.target.to_str().is_some_and(|t| t.starts_with('/')); + if !target_is_absolute { + return Err(Error::mount( + "validate guest path", + format!( + "target path on guest should be an absolute directory: {}", + mount.target.display() + ), + )); + } + + Ok(()) + } + + /// Generate a virtiofs mount tag for a given index. + /// + /// Mount tags follow the format "smolvm0", "smolvm1", etc. and are used + /// consistently across the host launcher, API handlers, and guest agent. + pub fn mount_tag(index: usize) -> String { + format!("smolvm{}", index) + } + + /// Create without validation (for loading from database). + /// + /// Use this only when loading persisted mounts that were previously validated. + pub fn from_storage_tuple(source: String, target: String, read_only: bool) -> Self { + Self { + source: PathBuf::from(source), + target: PathBuf::from(target), + read_only, + } + } + + /// Convert this mount to tuple format for persistence. + pub fn to_storage_tuple(&self) -> (String, String, bool) { + ( + self.source.to_string_lossy().to_string(), + self.target.to_string_lossy().to_string(), + self.read_only, + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn parse_rejects_duplicate_target() { + // Sources must exist (HostMount::_parse validates source presence), + // so use real temp directories. + let base = std::env::temp_dir(); + let a = base.join("smolvm_dup_a"); + let b = base.join("smolvm_dup_b"); + std::fs::create_dir_all(&a).unwrap(); + std::fs::create_dir_all(&b).unwrap(); + let (a, b) = (a.to_string_lossy(), b.to_string_lossy()); + + let err = HostMount::parse(&[format!("{a}:/app"), format!("{b}:/app")]).unwrap_err(); + assert!( + err.to_string().contains("duplicate mount target"), + "expected duplicate-target error, got: {err}" + ); + // Distinct targets are fine. + assert!(HostMount::parse(&[format!("{a}:/app"), format!("{b}:/data")]).is_ok()); + } + + fn parse_one(spec: &str) -> HostMount { + HostMount::parse(&[spec.to_string()]).unwrap().remove(0) + } + + #[test] + fn test_new_mount_rejects_illegal_source_mount_path() { + for path in ["/", "/etc", "/var", "/var/run", "/var/log"] { + let result = HostMount::new(path, "/guest/path", true); + assert!(result.is_err(), "{} should be rejected", path); + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("protected system path"), + "Error should explain why {} is blocked, got: {}", + path, + err_msg + ); + assert!( + err_msg.contains("cannot be mounted"), + "Error should explain that {} cannot be mounted, got: {}", + path, + err_msg + ); + } + } + + #[test] + fn test_new_mount_allows_safe_source_under_var() { + let temp_dir = std::env::temp_dir().join("smolvm-safe-var-path"); + std::fs::create_dir_all(&temp_dir).unwrap(); + + let result = HostMount::new(&temp_dir, "/guest/path", true); + + let _ = std::fs::remove_dir_all(&temp_dir); + + assert!( + result.is_ok(), + "safe paths under the OS temp directory should remain mountable" + ); + } + + #[test] + fn test_parse_mount_spec_basic() { + let mount = parse_one("/tmp:/guest/path"); + assert_eq!(mount.source, PathBuf::from("/tmp").canonicalize().unwrap()); + assert_eq!(mount.target, PathBuf::from("/guest/path")); + assert!(!mount.read_only); + } + + #[test] + fn test_parse_mount_spec_read_only() { + let mount = parse_one("/tmp:/guest/path:ro"); + assert!(mount.read_only); + } + + #[test] + fn test_parse_mount_spec_explicit_rw() { + let mount = parse_one("/tmp:/guest/path:rw"); + assert!(!mount.read_only); + } + + #[test] + fn test_parse_mount_spec_invalid() { + assert!(HostMount::parse(&["/only/one/path".to_string()]).is_err()); + assert!(HostMount::parse(&["".to_string()]).is_err()); + assert!(HostMount::parse(&["/a:/b:/c:/d".to_string()]).is_err()); + } + + #[test] + fn test_parse_mount_spec_paths_with_spaces() { + let temp_dir = std::env::current_dir() + .unwrap() + .join("target") + .join("smolvm mount with spaces"); + std::fs::create_dir_all(&temp_dir).unwrap(); + + let spec = format!("{}:/guest/path", temp_dir.display()); + let mount = parse_one(&spec); + + assert_eq!(mount.source, temp_dir.canonicalize().unwrap()); + assert_eq!(mount.target, PathBuf::from("/guest/path")); + + let _ = std::fs::remove_dir_all(&temp_dir); + } + + #[test] + fn test_parse_mount_spec_invalid_mode() { + let result = HostMount::parse(&["/host:/guest:invalid".to_string()]); + assert!(result.is_err()); + } + + #[test] + fn test_parse_mount_spec_too_many_colons() { + let result = HostMount::parse(&["/a:/b:ro:extra".to_string()]); + assert!(result.is_err()); + } + + #[test] + fn test_new_mount_nonexistent_source() { + let result = HostMount::new("/nonexistent/path/12345abcde", "/guest/path", true); + assert!(matches!(result, Err(Error::MountSourceNotFound { .. }))); + } + + #[test] + fn test_new_mount_disallows_relative_target() { + let result = HostMount::new("/tmp", "relative/path", true); + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("absolute"), + "Error should explain that guest target paths must be absolute" + ); + } + + #[test] + fn test_new_mount_existing_source() { + let result = HostMount::new("/tmp", "/guest/tmp", true); + assert!(result.is_ok()); + } + + #[test] + fn test_new_mount_rejects_single_file() { + let temp_file = std::env::temp_dir().join("smolvm_test_file.txt"); + std::fs::write(&temp_file, "test").unwrap(); + + let result = HostMount::new(temp_file.to_str().unwrap(), "/guest/file.txt", true); + + let _ = std::fs::remove_file(&temp_file); + + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("directory"), + "Error should suggest directory mount" + ); + assert!( + err_msg.contains("virtiofs limitation"), + "Error should explain the virtiofs directory requirement" + ); + } +} diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 0000000..8147617 --- /dev/null +++ b/src/db.rs @@ -0,0 +1,944 @@ +//! Database module for persistent state storage. +//! +//! Provides ACID-compliant storage using SQLite for VM state persistence +//! with atomic transactions and concurrent access safety. +//! +//! The connection handle is cached for the lifetime of the `SmolvmDb` +//! instance, amortising connection open cost across all operations. +//! +//! SQLite is configured in WAL mode with a 5s busy_timeout, so concurrent +//! CLI invocations share the database file without manual retry logic. + +use crate::config::VmRecord; +use crate::error::{Error, Result}; +use parking_lot::{Condvar, Mutex}; +use rusqlite::{params, Connection, OptionalExtension}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::Duration; + +/// SQLite busy_timeout: how long a blocked writer waits for the write lock +/// before returning SQLITE_BUSY. Set high enough to survive burst contention +/// from concurrent CLI processes (e.g., 10-20 VMs starting simultaneously). +const BUSY_TIMEOUT: Duration = Duration::from_secs(15); + +/// Long enough for a legitimate slow create/extract, short enough that a +/// crashed creator does not reserve a name forever. +const CREATE_RESERVATION_TTL_SECS: u64 = 60 * 60; + +/// Max SQLite connections held open by the pool. WAL allows these to read +/// concurrently (writes still serialize at the SQLite layer, gated by +/// `busy_timeout`), so a slow write can no longer block reads — the prior +/// single-`Mutex` design serialized EVERY db call, which let a +/// stalled write park the async reactor and wedge the liveness probes +/// (see `tests/reactor_wedge.rs`). Sized to comfortably cover the API server's +/// concurrent handlers without holding a large fan of file descriptors. +const POOL_MAX_CONNS: usize = 8; + +/// A small fixed-capacity pool of SQLite connections to the same database file. +/// +/// Each connection is opened with the WAL pragmas + `busy_timeout`, so multiple +/// readers proceed in parallel and a writer only blocks other *writers*. A +/// connection is checked out for the duration of one `with_conn` closure and +/// returned on drop (discarded if the closure panicked, so a half-applied +/// statement can't be handed to the next caller). Checkout blocks only when all +/// `POOL_MAX_CONNS` are in use — never behind an unrelated read. +struct ConnPool { + path: PathBuf, + inner: Mutex, + available: Condvar, +} + +struct PoolInner { + /// Connections opened and not currently checked out. + idle: Vec, + /// Total connections in existence (idle + checked out). + open: usize, +} + +impl ConnPool { + fn new(path: PathBuf) -> Self { + Self { + path, + inner: Mutex::new(PoolInner { + idle: Vec::new(), + open: 0, + }), + available: Condvar::new(), + } + } + + /// Take a connection, opening a new one (up to `POOL_MAX_CONNS`) or waiting + /// for one to be returned. Opening happens outside the lock so a slow open + /// never blocks other checkouts/checkins. + fn checkout(&self) -> Result { + let mut inner = self.inner.lock(); + loop { + if let Some(conn) = inner.idle.pop() { + return Ok(conn); + } + if inner.open < POOL_MAX_CONNS { + inner.open += 1; + drop(inner); + match SmolvmDb::open_connection(&self.path) { + Ok(conn) => return Ok(conn), + Err(e) => { + // Roll back the reservation and let a waiter retry. + self.inner.lock().open -= 1; + self.available.notify_one(); + return Err(e); + } + } + } + // Pool saturated: wait for a checkin. + self.available.wait(&mut inner); + } + } + + fn checkin(&self, conn: Connection) { + self.inner.lock().idle.push(conn); + self.available.notify_one(); + } + + /// Drop a connection without returning it (used when its closure panicked, + /// so its possibly-dirty state is not reused). Frees a slot for a new open. + fn discard(&self) { + self.inner.lock().open -= 1; + self.available.notify_one(); + } +} + +/// RAII guard returning a checked-out connection to the pool on drop. +struct PooledConn<'a> { + pool: &'a ConnPool, + conn: Option, +} + +impl Drop for PooledConn<'_> { + fn drop(&mut self) { + if let Some(conn) = self.conn.take() { + if std::thread::panicking() { + self.pool.discard(); + } else { + self.pool.checkin(conn); + } + } + } +} + +/// Extension trait to convert errors into `Error::database`. +trait DbResultExt { + fn db_err(self, operation: impl Into) -> Result; +} + +impl DbResultExt for std::result::Result { + fn db_err(self, operation: impl Into) -> Result { + self.map_err(|e| Error::database(operation, e.to_string())) + } +} + +/// Thread-safe database handle for smolvm state persistence. +/// +/// Uses the standard WAL split: a single dedicated WRITER connection (behind a +/// mutex) serializes all mutations in-process — so they never contend at the +/// SQLite write-lock layer (no `SQLITE_BUSY` spinning) — while READS go through a +/// small pool of separate connections that run concurrently under WAL. A reader +/// therefore never waits on the writer, so a stalled write can no longer park the +/// async reactor that serves the liveness probes (the single-`Mutex` +/// failure mode; see `tests/reactor_wedge.rs`). Connections open lazily. +/// Cross-process concurrency is still handled by WAL + busy_timeout. +#[derive(Clone)] +pub struct SmolvmDb { + path: PathBuf, + /// Single connection serializing writes (and the rare read that must observe + /// its own just-committed write on the same connection). Opened on first use. + writer: Arc>>, + /// Pool of connections for concurrent reads. Never used for writes. + readers: Arc, +} + +impl std::fmt::Debug for SmolvmDb { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SmolvmDb") + .field("path", &self.path) + .field("writer_open", &self.writer.lock().is_some()) + .field("reader_conns", &self.readers.inner.lock().open) + .finish() + } +} + +impl SmolvmDb { + /// Run a closure with the single writer connection, opening it on first use. + /// Serializes all writers in-process so they never collide at the SQLite + /// write lock. Use for every mutation (and any read that must see a write it + /// just made on this connection). + fn with_conn(&self, f: F) -> Result + where + F: FnOnce(&mut Connection) -> Result, + { + let mut guard = self.writer.lock(); + if guard.is_none() { + *guard = Some(Self::open_connection(&self.path)?); + } + f(guard.as_mut().expect("writer connection present")) + } + + /// Run a closure with a pooled READ connection. Concurrent reads use + /// different connections (up to `POOL_MAX_CONNS`) and, under WAL, never block + /// on the writer — so a stalled write can't serialize or wedge reads. MUST + /// NOT be used for writes (that would reintroduce SQLite write contention). + fn with_read_conn(&self, f: F) -> Result + where + F: FnOnce(&mut Connection) -> Result, + { + let conn = self.readers.checkout()?; + let mut guard = PooledConn { + pool: &self.readers, + conn: Some(conn), + }; + f(guard.conn.as_mut().expect("reader connection present")) + } + + /// Open the SQLite connection, configure pragmas, and ensure tables exist. + fn open_connection(path: &Path) -> Result { + let conn = Connection::open(path) + .map_err(|e| Error::database_unavailable(format!("open database: {}", e)))?; + + // WAL lets readers and writers overlap across processes; synchronous=NORMAL + // is safe under WAL and significantly faster than the default FULL. + conn.execute_batch("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;") + .db_err("configure pragmas")?; + conn.busy_timeout(BUSY_TIMEOUT).db_err("set busy_timeout")?; + + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS vms ( + name TEXT PRIMARY KEY NOT NULL, + data BLOB NOT NULL + ); + CREATE TABLE IF NOT EXISTS vm_create_reservations ( + name TEXT PRIMARY KEY NOT NULL, + owner_token TEXT NOT NULL, + owner_pid INTEGER NOT NULL, + created_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS config ( + key TEXT PRIMARY KEY NOT NULL, + value TEXT NOT NULL + );", + ) + .db_err("create tables")?; + + Ok(conn) + } + + /// Open the database at the default location. + /// + /// Default path: `~/Library/Application Support/smolvm/server/smolvm.db` (macOS) + /// or `~/.local/share/smolvm/server/smolvm.db` (Linux) + /// + /// If the database doesn't exist, it will be created. + pub fn open() -> Result { + let path = Self::default_path()?; + Self::open_at(&path) + } + + /// Open the database at a specific path. Parent directories are created + /// if missing; the connection itself is opened lazily on first use. + pub fn open_at(path: &Path) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).db_err("create directory")?; + } + + Ok(Self { + path: path.to_path_buf(), + writer: Arc::new(Mutex::new(None)), + readers: Arc::new(ConnPool::new(path.to_path_buf())), + }) + } + + /// Get the default database path. + pub fn default_path() -> Result { + let data_dir = dirs::data_local_dir().ok_or_else(|| { + Error::database_unavailable("could not determine local data directory") + })?; + Ok(data_dir.join("smolvm").join("server").join("smolvm.db")) + } + + /// Initialize database tables. + /// + /// Tables are created automatically when the connection opens, so this + /// just forces the connection open. Retained for API compatibility. + pub fn init_tables(&self) -> Result<()> { + self.with_conn(|_| Ok(())) + } + + // ======================================================================== + // VM Operations + // ======================================================================== + + /// Insert or update a VM record. + pub fn insert_vm(&self, name: &str, record: &VmRecord) -> Result<()> { + let json = serde_json::to_vec(record).db_err("serialize vm record")?; + self.with_conn(|conn| { + conn.execute( + "INSERT INTO vms (name, data) VALUES (?1, ?2) + ON CONFLICT(name) DO UPDATE SET data = excluded.data", + params![name, json], + ) + .db_err(format!("insert vm '{}'", name))?; + Ok(()) + }) + } + + /// Insert a VM record only if it doesn't already exist. + /// + /// Returns `Ok(true)` if inserted, `Ok(false)` if the name already exists. + /// Atomicity is provided by SQLite's `INSERT OR IGNORE`. A name with an + /// active create reservation is treated as already taken so older callers + /// that have not been threaded through the reservation API cannot clobber a + /// machine whose per-machine data directory is being prepared. + pub fn insert_vm_if_not_exists(&self, name: &str, record: &VmRecord) -> Result { + let json = serde_json::to_vec(record).db_err("serialize vm record")?; + self.with_conn(|conn| { + let changed = conn + .execute( + "INSERT OR IGNORE INTO vms (name, data) + SELECT ?1, ?2 + WHERE NOT EXISTS ( + SELECT 1 FROM vm_create_reservations WHERE name = ?1 + )", + params![name, json], + ) + .db_err(format!("insert vm '{}'", name))?; + Ok(changed == 1) + }) + } + + /// Generate an opaque token identifying this process's create reservation. + pub fn create_reservation_token() -> String { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + format!( + "{}-{}-{}", + std::process::id(), + crate::util::current_timestamp(), + nanos + ) + } + + /// Reserve a VM name across processes before touching its data directory. + /// + /// Returns `Ok(false)` when the VM already exists or another live creator + /// owns the reservation. Dead/stale reservations are reaped before the + /// insert attempt so a crashed creator does not permanently wedge a name. + pub fn reserve_vm_create(&self, name: &str, owner_token: &str) -> Result { + let owner_pid = i64::from(std::process::id()); + let now = crate::util::current_timestamp(); + let stale_before = now.saturating_sub(CREATE_RESERVATION_TTL_SECS); + + self.with_conn(|conn| { + let tx = conn.transaction().db_err("begin create reservation")?; + + if let Some((existing_pid, created_at)) = tx + .query_row( + "SELECT owner_pid, created_at + FROM vm_create_reservations + WHERE name = ?1", + params![name], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, u64>(1)?)), + ) + .optional() + .db_err(format!("read create reservation '{}'", name))? + { + let pid_alive = existing_pid > 0 + && crate::process::is_alive(existing_pid as crate::process::Pid); + if !pid_alive || created_at <= stale_before { + tx.execute( + "DELETE FROM vm_create_reservations WHERE name = ?1", + params![name], + ) + .db_err(format!("remove stale create reservation '{}'", name))?; + } + } + + let exists: bool = tx + .query_row( + "SELECT EXISTS(SELECT 1 FROM vms WHERE name = ?1)", + params![name], + |row| row.get(0), + ) + .db_err(format!("check vm '{}'", name))?; + if exists { + tx.commit().db_err("commit create reservation check")?; + return Ok(false); + } + + let changed = tx + .execute( + "INSERT OR IGNORE INTO vm_create_reservations + (name, owner_token, owner_pid, created_at) + VALUES (?1, ?2, ?3, ?4)", + params![name, owner_token, owner_pid, now], + ) + .db_err(format!("reserve vm '{}'", name))?; + + tx.commit().db_err("commit create reservation")?; + Ok(changed == 1) + }) + } + + /// Persist a VM record and release the matching create reservation atomically. + /// + /// Returns `Ok(false)` if the caller does not own the reservation or if the + /// VM row already exists. + pub fn commit_reserved_vm( + &self, + name: &str, + owner_token: &str, + record: &VmRecord, + ) -> Result { + let json = serde_json::to_vec(record).db_err("serialize vm record")?; + self.with_conn(|conn| { + let tx = conn.transaction().db_err("begin reserved vm commit")?; + + let owns_reservation: bool = tx + .query_row( + "SELECT EXISTS( + SELECT 1 FROM vm_create_reservations + WHERE name = ?1 AND owner_token = ?2 + )", + params![name, owner_token], + |row| row.get(0), + ) + .db_err(format!("check create reservation '{}'", name))?; + if !owns_reservation { + tx.commit().db_err("commit reservation ownership check")?; + return Ok(false); + } + + let changed = tx + .execute( + "INSERT OR IGNORE INTO vms (name, data) VALUES (?1, ?2)", + params![name, json], + ) + .db_err(format!("insert reserved vm '{}'", name))?; + + tx.execute( + "DELETE FROM vm_create_reservations + WHERE name = ?1 AND owner_token = ?2", + params![name, owner_token], + ) + .db_err(format!("release create reservation '{}'", name))?; + + tx.commit().db_err("commit reserved vm")?; + Ok(changed == 1) + }) + } + + /// Release a create reservation if it is still owned by `owner_token`. + pub fn release_vm_create_reservation(&self, name: &str, owner_token: &str) -> Result<()> { + self.with_conn(|conn| { + conn.execute( + "DELETE FROM vm_create_reservations + WHERE name = ?1 AND owner_token = ?2", + params![name, owner_token], + ) + .db_err(format!("release create reservation '{}'", name))?; + Ok(()) + }) + } + + /// Get a VM record by name. + pub fn get_vm(&self, name: &str) -> Result> { + self.with_read_conn(|conn| { + let data: Option> = conn + .query_row( + "SELECT data FROM vms WHERE name = ?1", + params![name], + |row| row.get(0), + ) + .optional() + .db_err(format!("get vm '{}'", name))?; + + match data { + Some(bytes) => { + let record: VmRecord = serde_json::from_slice(&bytes) + .db_err(format!("deserialize vm record '{}'", name))?; + Ok(Some(record)) + } + None => Ok(None), + } + }) + } + + /// Remove a VM record by name, returning the removed record if it existed. + /// + /// Read + delete happen in a single transaction to prevent TOCTOU races. + pub fn remove_vm(&self, name: &str) -> Result> { + self.with_conn(|conn| { + let tx = conn.transaction().db_err("begin transaction")?; + + let data: Option> = tx + .query_row( + "SELECT data FROM vms WHERE name = ?1", + params![name], + |row| row.get(0), + ) + .optional() + .db_err(format!("get vm '{}'", name))?; + + let record = match data { + Some(bytes) => { + let r: VmRecord = serde_json::from_slice(&bytes) + .db_err(format!("deserialize vm record '{}'", name))?; + tx.execute("DELETE FROM vms WHERE name = ?1", params![name]) + .db_err(format!("remove vm '{}'", name))?; + Some(r) + } + None => None, + }; + + tx.commit().db_err("commit vm removal")?; + Ok(record) + }) + } + + /// List all VM records. + pub fn list_vms(&self) -> Result> { + self.with_read_conn(|conn| { + let mut stmt = conn + .prepare_cached("SELECT name, data FROM vms") + .db_err("prepare list_vms")?; + let rows = stmt + .query_map([], |row| { + let name: String = row.get(0)?; + let data: Vec = row.get(1)?; + Ok((name, data)) + }) + .db_err("query vms")?; + + let mut vms = Vec::new(); + for row in rows { + let (name, data) = row.db_err("read vms row")?; + let record: VmRecord = serde_json::from_slice(&data) + .db_err(format!("deserialize vm record '{}'", name))?; + vms.push((name, record)); + } + Ok(vms) + }) + } + + /// Names of VMs forked from `golden`. Their block disks are copy-on-write + /// overlays backed by the golden's disks, so the golden must outlive them + /// and must not be re-run with writable disks while they exist. + pub fn dependent_clones(&self, golden: &str) -> Result> { + Ok(self + .list_vms()? + .into_iter() + .filter(|(_, r)| r.golden.as_deref() == Some(golden)) + .map(|(name, _)| name) + .collect()) + } + + /// Update a VM record in place using a closure. + /// + /// Returns the updated record if found, `None` if not found. Read + + /// write happen in a single transaction to prevent lost updates. + pub fn update_vm(&self, name: &str, f: F) -> Result> + where + F: FnOnce(&mut VmRecord), + { + self.with_conn(|conn| { + let tx = conn.transaction().db_err("begin transaction")?; + + let data: Option> = tx + .query_row( + "SELECT data FROM vms WHERE name = ?1", + params![name], + |row| row.get(0), + ) + .optional() + .db_err(format!("get vm '{}'", name))?; + + let updated = match data { + Some(bytes) => { + let mut record: VmRecord = serde_json::from_slice(&bytes) + .db_err(format!("deserialize vm record '{}'", name))?; + f(&mut record); + let new_data = serde_json::to_vec(&record).db_err("serialize vm record")?; + tx.execute( + "UPDATE vms SET data = ?2 WHERE name = ?1", + params![name, new_data], + ) + .db_err(format!("update vm '{}'", name))?; + Some(record) + } + None => None, + }; + + tx.commit().db_err("commit vm update")?; + Ok(updated) + }) + } + + /// Load all VMs into an in-memory HashMap (for compatibility layer). + pub fn load_all_vms(&self) -> Result> { + let vms = self.list_vms()?; + Ok(vms.into_iter().collect()) + } + + /// Load all config settings and VM records in a single transaction. + pub fn load_all(&self) -> Result<(HashMap, HashMap)> { + self.with_conn(|conn| { + let tx = conn.transaction().db_err("begin read transaction")?; + + let mut config = HashMap::new(); + { + let mut stmt = tx + .prepare_cached("SELECT key, value FROM config") + .db_err("prepare list config")?; + let rows = stmt + .query_map([], |row| { + let k: String = row.get(0)?; + let v: String = row.get(1)?; + Ok((k, v)) + }) + .db_err("query config")?; + for row in rows { + let (k, v) = row.db_err("read config row")?; + config.insert(k, v); + } + } + + let mut vms = HashMap::new(); + { + let mut stmt = tx + .prepare_cached("SELECT name, data FROM vms") + .db_err("prepare list vms")?; + let rows = stmt + .query_map([], |row| { + let name: String = row.get(0)?; + let data: Vec = row.get(1)?; + Ok((name, data)) + }) + .db_err("query vms")?; + for row in rows { + let (name, data) = row.db_err("read vms row")?; + let record: VmRecord = serde_json::from_slice(&data) + .db_err(format!("deserialize vm record '{}'", name))?; + vms.insert(name, record); + } + } + + tx.commit().db_err("commit read transaction")?; + Ok((config, vms)) + }) + } + + /// Save multiple config key-value pairs in a single transaction. + pub fn save_config(&self, settings: &[(&str, &str)]) -> Result<()> { + self.with_conn(|conn| { + let tx = conn.transaction().db_err("begin transaction")?; + { + let mut stmt = tx + .prepare_cached( + "INSERT INTO config (key, value) VALUES (?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ) + .db_err("prepare set config")?; + for (k, v) in settings { + stmt.execute(params![k, v]) + .db_err(format!("set config '{}'", k))?; + } + } + tx.commit().db_err("commit config save")?; + Ok(()) + }) + } + + // ======================================================================== + // Global Config Operations + // ======================================================================== + + /// Get a global configuration value. + pub fn get_config(&self, key: &str) -> Result> { + self.with_conn(|conn| { + conn.query_row( + "SELECT value FROM config WHERE key = ?1", + params![key], + |row| row.get::<_, String>(0), + ) + .optional() + .db_err(format!("get config '{}'", key)) + }) + } + + /// Set a global configuration value. + pub fn set_config(&self, key: &str, value: &str) -> Result<()> { + self.with_conn(|conn| { + conn.execute( + "INSERT INTO config (key, value) VALUES (?1, ?2) + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + params![key, value], + ) + .db_err(format!("set config '{}'", key))?; + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::RecordState; + use tempfile::TempDir; + + fn temp_db() -> (TempDir, SmolvmDb) { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("test.db"); + let db = SmolvmDb::open_at(&path).unwrap(); + (dir, db) + } + + #[test] + fn test_db_crud_operations() { + let (_dir, db) = temp_db(); + + let record = VmRecord::new( + "test-vm".to_string(), + 2, + 1024, + vec![("/host".to_string(), "/guest".to_string(), false)], + vec![(8080, 80)], + false, + ); + + db.insert_vm("test-vm", &record).unwrap(); + + let retrieved = db.get_vm("test-vm").unwrap().unwrap(); + assert_eq!(retrieved.name, "test-vm"); + assert_eq!(retrieved.cpus, 2); + assert_eq!(retrieved.mem, 1024); + + let updated = db + .update_vm("test-vm", |r| { + r.state = RecordState::Running; + r.pid = Some(12345); + }) + .unwrap() + .unwrap(); + assert_eq!(updated.state, RecordState::Running); + assert_eq!(updated.pid, Some(12345)); + + let vms = db.list_vms().unwrap(); + assert_eq!(vms.len(), 1); + assert_eq!(vms[0].0, "test-vm"); + + let removed = db.remove_vm("test-vm").unwrap().unwrap(); + assert_eq!(removed.name, "test-vm"); + + assert!(db.get_vm("test-vm").unwrap().is_none()); + } + + /// A read must not block behind a stalled write. Before the connection pool, + /// every db call shared one `Mutex`, so a write stalled on + /// SQLite's `busy_timeout` held that mutex and serialized ALL reads behind + /// it — the serialization that let a stalled write park the async reactor and + /// wedge the liveness probes in production (see `tests/reactor_wedge.rs`). + /// With the pool the read uses a different WAL connection and returns at once. + /// Pre-pool this asserts in ~15s (busy_timeout) and fails; post-pool ~ms. + #[test] + fn read_does_not_block_behind_a_stalled_write() { + let (dir, db) = temp_db(); + let path = dir.path().join("test.db"); + db.insert_vm( + "m0", + &VmRecord::new("m0".to_string(), 1, 256, vec![], vec![], false), + ) + .unwrap(); + + // Warm the pool to 2 idle connections so the read below reuses one rather + // than opening a fresh connection while the external write lock is held. + std::thread::scope(|s| { + for _ in 0..2 { + s.spawn(|| { + let _ = db.get_vm("m0"); + std::thread::sleep(Duration::from_millis(60)); + }); + } + }); + + // A second connection to the same file holds the SQLite write lock — + // exactly what concurrent cross-process create-reservations do under churn. + let blocker = Connection::open(&path).unwrap(); + blocker.busy_timeout(Duration::from_secs(30)).unwrap(); + blocker.execute_batch("BEGIN IMMEDIATE").unwrap(); + + // A SmolvmDb write now stalls on busy_timeout, holding one pooled connection. + let db_w = db.clone(); + let writer = std::thread::spawn(move || { + let _ = db_w.insert_vm( + "m1", + &VmRecord::new("m1".to_string(), 1, 256, vec![], vec![], false), + ); + }); + std::thread::sleep(Duration::from_millis(300)); // let the write grab a conn + stall + + // Concurrent read on a DIFFERENT pooled connection (WAL): must be immediate. + let start = std::time::Instant::now(); + let got = db.get_vm("m0").unwrap(); + let elapsed = start.elapsed(); + assert!(got.is_some(), "read returned no record"); + assert!( + elapsed < Duration::from_secs(2), + "read blocked {elapsed:?} behind a stalled write — the pool is not isolating reads from writes" + ); + + blocker.execute_batch("COMMIT").ok(); + let _ = writer.join(); + } + + #[test] + fn test_db_concurrent_access() { + let (_dir, db) = temp_db(); + + let handles: Vec<_> = (0..10) + .map(|i| { + let db = db.clone(); + std::thread::spawn(move || { + let name = format!("vm-{}", i); + let record = VmRecord::new(name.clone(), 1, 512, vec![], vec![], false); + db.insert_vm(&name, &record).unwrap(); + }) + }) + .collect(); + + for handle in handles { + handle.join().unwrap(); + } + + let vms = db.list_vms().unwrap(); + assert_eq!(vms.len(), 10); + } + + #[test] + fn test_config_settings() { + let (_dir, db) = temp_db(); + + db.set_config("test_key", "test_value").unwrap(); + + let value = db.get_config("test_key").unwrap().unwrap(); + assert_eq!(value, "test_value"); + + assert!(db.get_config("nonexistent").unwrap().is_none()); + } + + #[test] + fn test_update_nonexistent_vm() { + let (_dir, db) = temp_db(); + + let result = db.update_vm("nonexistent", |_| {}).unwrap(); + assert!(result.is_none()); + } + + #[test] + fn test_remove_nonexistent_vm() { + let (_dir, db) = temp_db(); + + let result = db.remove_vm("nonexistent").unwrap(); + assert!(result.is_none()); + } + + #[test] + fn test_insert_vm_if_not_exists() { + let (_dir, db) = temp_db(); + + let record = VmRecord::new("test-vm".to_string(), 1, 512, vec![], vec![], false); + + let inserted = db.insert_vm_if_not_exists("test-vm", &record).unwrap(); + assert!(inserted, "first insert should succeed"); + + let inserted = db.insert_vm_if_not_exists("test-vm", &record).unwrap(); + assert!(!inserted, "second insert should fail (already exists)"); + + let vms = db.list_vms().unwrap(); + assert_eq!(vms.len(), 1); + + let record2 = VmRecord::new("test-vm2".to_string(), 2, 1024, vec![], vec![], false); + let inserted = db.insert_vm_if_not_exists("test-vm2", &record2).unwrap(); + assert!(inserted, "different name should succeed"); + + let vms = db.list_vms().unwrap(); + assert_eq!(vms.len(), 2); + } + + #[test] + fn test_create_reservation_blocks_unreserved_insert() { + let (_dir, db) = temp_db(); + let token = SmolvmDb::create_reservation_token(); + let record = VmRecord::new("reserved-vm".to_string(), 1, 512, vec![], vec![], false); + + assert!(db.reserve_vm_create("reserved-vm", &token).unwrap()); + assert!( + !db.insert_vm_if_not_exists("reserved-vm", &record).unwrap(), + "legacy unreserved insert must not publish a reserved name" + ); + assert!( + db.get_vm("reserved-vm").unwrap().is_none(), + "reservation must not create a visible VM row" + ); + + assert!(db + .commit_reserved_vm("reserved-vm", &token, &record) + .unwrap()); + assert!(db.get_vm("reserved-vm").unwrap().is_some()); + } + + #[test] + fn test_create_reservation_is_exclusive_and_releasable() { + let (_dir, db) = temp_db(); + let first = SmolvmDb::create_reservation_token(); + let second = SmolvmDb::create_reservation_token(); + + assert!(db.reserve_vm_create("contested", &first).unwrap()); + assert!( + !db.reserve_vm_create("contested", &second).unwrap(), + "second live creator must not reserve the same name" + ); + + db.release_vm_create_reservation("contested", &first) + .unwrap(); + assert!( + db.reserve_vm_create("contested", &second).unwrap(), + "released reservation should make the name available" + ); + } + + #[test] + fn test_insert_vm_if_not_exists_concurrent() { + let (_dir, db) = temp_db(); + + let handles: Vec<_> = (0..10) + .map(|_| { + let db = db.clone(); + std::thread::spawn(move || { + let record = + VmRecord::new("contested-name".to_string(), 1, 512, vec![], vec![], false); + db.insert_vm_if_not_exists("contested-name", &record) + .unwrap() + }) + }) + .collect(); + + let results: Vec = handles.into_iter().map(|h| h.join().unwrap()).collect(); + + let success_count = results.iter().filter(|&&r| r).count(); + assert_eq!(success_count, 1, "exactly one insert should succeed"); + + let vms = db.list_vms().unwrap(); + assert_eq!(vms.len(), 1); + } +} diff --git a/src/disk_utils.rs b/src/disk_utils.rs new file mode 100644 index 0000000..7cc0921 --- /dev/null +++ b/src/disk_utils.rs @@ -0,0 +1,673 @@ +//! Disk image helpers: sparse creation, template/copy-on-write cloning, and +//! resizing of the raw VM disk images. + +use crate::data::consts::BYTES_PER_GIB; +use crate::data::disk::DiskType; +use crate::error::{Error, Result}; +use crate::platform::Os; +use std::io::{Seek, SeekFrom, Write}; +use std::path::Path; + +/// Common search paths for e2fsprogs tools (mkfs.ext4, e2fsck, resize2fs). +const E2FSPROGS_PATH_PREFIXES: &[&str] = &[ + "/opt/homebrew/opt/e2fsprogs/sbin", // macOS ARM (Homebrew) + "/usr/local/opt/e2fsprogs/sbin", // macOS Intel (Homebrew) + "/opt/homebrew/sbin", // macOS ARM (Homebrew alt) + "/usr/local/sbin", // macOS Intel (Homebrew alt) + "/sbin", // Linux + "/usr/sbin", // Linux alt +]; + +/// Find an e2fsprogs tool by name (e.g., "mkfs.ext4", "e2fsck", "resize2fs"). +/// +/// Searches common installation paths, then falls back to PATH lookup. +pub(crate) fn find_e2fsprogs_tool(name: &str) -> Option { + for prefix in E2FSPROGS_PATH_PREFIXES { + let path = format!("{}/{}", prefix, name); + if Path::new(&path).exists() { + return Some(path); + } + } + + if std::process::Command::new(name) + // Every e2fsprogs tool we care about supports `--version`; this is + // the cheapest non-destructive probe that confirms the binary is + // present on PATH and executable before we try to use it for real. + .arg("--version") + .output() + .is_ok() + { + return Some(name.to_string()); + } + + None +} + +/// Create a sparse disk image file. +pub(crate) fn create_sparse_disk(path: &Path, size_bytes: u64) -> Result<()> { + use std::fs::OpenOptions; + + tracing::info!( + path = %path.display(), + disk_type = D::NAME, + size_gb = size_bytes / BYTES_PER_GIB, + "creating sparse {} disk", + D::NAME, + ); + + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|e| Error::storage("create sparse disk", e.to_string()))?; + write_last_byte( + &mut file, + size_bytes, + "seek to create disk", + "write disk tail", + ) +} + +/// Copy a disk from a pre-formatted template, resizing to target size. +/// +/// On macOS, uses `clonefile()` for instant APFS copy-on-write cloning. +/// On Linux, falls back to `fs::copy` (which uses `copy_file_range` for +/// sparse-aware copying on supported filesystems). +pub(crate) fn copy_disk_from_template( + disk_path: &Path, + size_bytes: u64, + template_path: &Path, +) -> Result<()> { + use std::fs::OpenOptions; + + tracing::info!( + template = %template_path.display(), + target = %disk_path.display(), + disk_type = D::NAME, + "copying {} from template", + D::NAME, + ); + + if let Some(parent) = disk_path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| Error::storage("create directory", e.to_string()))?; + } + + clone_or_copy_file(template_path, disk_path)?; + + // Extend the sparse file to the target size. The ext4 filesystem inside + // is still at the template's original size — the guest agent expands it + // at boot via resize2fs in a parallel thread (~10ms, non-blocking). + // We do NOT run resize2fs on the host: it takes ~880ms to expand the + // filesystem metadata synchronously, blocking the entire boot path. + let current_size = std::fs::metadata(disk_path) + .map_err(|e| Error::storage("read copied disk metadata", e.to_string()))? + .len(); + if current_size < size_bytes { + let mut file = OpenOptions::new() + .write(true) + .open(disk_path) + .map_err(|e| Error::storage("open for resize", e.to_string()))?; + write_last_byte(&mut file, size_bytes, "seek for resize", "extend disk")?; + } + + tracing::info!( + path = %disk_path.display(), + disk_type = D::NAME, + "{} copied from template", + D::NAME + ); + Ok(()) +} + +/// Expand a sparse disk image file to a new size. +pub(crate) fn expand_sparse_disk(path: &Path, new_size_gb: u64) -> Result<()> { + use std::fs::OpenOptions; + + // Use checked_mul so an absurd request (e.g. 2^34 GiB) returns a clean + // error instead of overflowing u64 — which would debug-panic or, in release, + // wrap to a tiny/zero byte count and silently truncate the sparse file. + let new_size_bytes = new_size_gb.checked_mul(BYTES_PER_GIB).ok_or_else(|| { + Error::storage( + "expand disk", + format!( + "requested size {} GiB is too large (overflows the maximum addressable byte count)", + new_size_gb + ), + ) + })?; + let current_size = std::fs::metadata(path) + .map_err(|e| Error::storage("get disk metadata", e.to_string()))? + .len(); + + if new_size_bytes == current_size { + // Already at target size — idempotent. This handles retries after a + // partial failure where the disk was expanded but the DB wasn't updated. + return Ok(()); + } + if new_size_bytes < current_size { + return Err(Error::storage( + "expand disk", + format!( + "new size ({} GiB) must be larger than current size ({} GiB). Shrinking is not supported.", + new_size_gb, + current_size / BYTES_PER_GIB + ), + )); + } + + tracing::info!( + path = %path.display(), + disk_type = D::NAME, + current_gb = current_size / BYTES_PER_GIB, + new_gb = new_size_gb, + "expanding {} disk", + D::NAME + ); + + let mut file = OpenOptions::new() + .write(true) + .open(path) + .map_err(|e| Error::storage("open disk for expansion", e.to_string()))?; + write_last_byte( + &mut file, + new_size_bytes, + "seek to expand", + "write to expand", + )?; + file.sync_all() + .map_err(|e| Error::storage("sync after expand", e.to_string()))?; + + tracing::info!( + path = %path.display(), + disk_type = D::NAME, + new_gb = new_size_gb, + "{} disk expanded successfully", + D::NAME + ); + Ok(()) +} + +/// Format a disk with mkfs.ext4 (requires e2fsprogs). +pub(crate) fn format_disk_with_mkfs(disk_path: &Path) -> Result<()> { + tracing::info!( + path = %disk_path.display(), + disk_type = D::NAME, + "formatting {} disk with mkfs.ext4", + D::NAME + ); + + let mkfs_path = find_e2fsprogs_tool("mkfs.ext4").ok_or_else(|| { + let hint = if Os::current().is_macos() { + "On macOS, install with: brew install e2fsprogs" + } else { + "On Linux, install with: apt install e2fsprogs (or equivalent)" + }; + Error::storage( + "find mkfs.ext4", + format!( + "mkfs.ext4 not found - required for {} disk formatting.\n {}", + D::NAME, + hint + ), + ) + })?; + + let path_str = disk_path + .to_str() + .ok_or_else(|| Error::storage("validate path", "disk path contains invalid characters"))?; + + let output = std::process::Command::new(mkfs_path) + .args([ + // Force creation on a regular file rather than requiring a block + // device. Our "disk" here is a raw sparse image file on the host. + "-F", + // Quiet mode keeps stderr/stdout small on success. We only need the + // detailed mkfs output when the command fails. + "-q", + // Set the reserved-blocks percentage flag. + "-m", + // Reserve 0% of blocks for root. This is a VM-owned data disk, not + // a host root filesystem, so holding capacity back for root is just + // wasted space. + "0", + // Specify ext4 feature flags explicitly. + "-O", + // Disable the journal. These disks are scratch/data images managed + // by a VM, and dropping the journal reduces write amplification and + // space overhead for our use case. + "^has_journal", + // Set the filesystem label. + "-L", + // The label lets the guest-side tooling distinguish storage and + // overlay disks in logs and inspection output. + D::VOLUME_LABEL, + // The target sparse image file to format. + path_str, + ]) + .output() + .map_err(|e| Error::storage("run mkfs.ext4", e.to_string()))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(Error::storage("format with mkfs.ext4", stderr.to_string())); + } + + tracing::info!( + path = %disk_path.display(), + disk_type = D::NAME, + "{} disk formatted successfully", + D::NAME + ); + Ok(()) +} + +pub(crate) fn write_last_byte( + file: &mut std::fs::File, + size_bytes: u64, + seek_context: &str, + write_context: &str, +) -> Result<()> { + assert!(size_bytes > 0, "disk size must be greater than 0"); + + // On Windows/NTFS a file is dense by default: seeking past the end and + // writing a tail byte allocates every block in between (a 20 GiB disk would + // need 20 GiB free, failing with ERROR_DISK_FULL). Mark the file sparse + // first so only written extents consume host space — matching the implicit + // sparse behavior of Unix filesystems. + #[cfg(windows)] + mark_file_sparse(file).map_err(|e| Error::storage("mark disk sparse", e.to_string()))?; + + file.seek(SeekFrom::Start(size_bytes - 1)) + .map_err(|e| Error::storage(seek_context, e.to_string()))?; + file.write_all(&[0]) + .map_err(|e| Error::storage(write_context, e.to_string()))?; + Ok(()) +} + +/// Mark an open file as sparse (Windows/NTFS) so writing a tail byte at a large +/// offset doesn't allocate every intermediate block. No-op semantics elsewhere +/// (Unix filesystems are sparse by default and never call this). +#[cfg(windows)] +fn mark_file_sparse(file: &std::fs::File) -> std::io::Result<()> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::IO::DeviceIoControl; + // FSCTL_SET_SPARSE control code (winioctl.h). + const FSCTL_SET_SPARSE: u32 = 0x000900C4; + let mut returned: u32 = 0; + // SAFETY: `file` is a valid open handle; FSCTL_SET_SPARSE uses no in/out buffers. + let ok = unsafe { + DeviceIoControl( + file.as_raw_handle(), + FSCTL_SET_SPARSE, + std::ptr::null(), + 0, + std::ptr::null_mut(), + 0, + &mut returned, + std::ptr::null_mut(), + ) + }; + if ok == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +/// Clone a file using the platform-optimal copy method. +/// +/// - macOS: `clonefile()` for instant APFS copy-on-write +/// - Linux: sparse copy via `SEEK_HOLE`/`SEEK_DATA` (copies only data regions) +/// - Fallback: `fs::copy` +/// +/// Disk templates are sparse files (~500KB actual data in a 512MB logical file). +/// `std::fs::copy()` copies all bytes including zero regions. The sparse copy +/// path skips holes, reducing copy time from ~400ms to ~5ms. +pub fn clone_or_copy_file(src: &Path, dst: &Path) -> Result<()> { + #[cfg(target_os = "macos")] + { + use std::ffi::CString; + + if dst.exists() { + let _ = std::fs::remove_file(dst); + } + + let src_c = CString::new(src.to_string_lossy().as_bytes()) + .map_err(|e| Error::storage("clonefile src path", e.to_string()))?; + let dst_c = CString::new(dst.to_string_lossy().as_bytes()) + .map_err(|e| Error::storage("clonefile dst path", e.to_string()))?; + + let ret = unsafe { libc::clonefile(src_c.as_ptr(), dst_c.as_ptr(), 0) }; + if ret == 0 { + tracing::debug!(src = %src.display(), dst = %dst.display(), "clonefile succeeded"); + return Ok(()); + } + + tracing::debug!( + src = %src.display(), + errno = std::io::Error::last_os_error().raw_os_error().unwrap_or(0), + "clonefile failed, falling back to fs::copy" + ); + } + + #[cfg(target_os = "linux")] + { + // TODO(reflink): try a FICLONE ioctl before sparse_copy for true + // copy-on-write on btrfs/XFS hosts. (Fork clones already get + // filesystem-independent block CoW via qcow2 overlays; this would only + // speed up the remaining full-copy callers on reflink-capable hosts.) + match sparse_copy(src, dst) { + Ok(bytes) => { + tracing::debug!( + src = %src.display(), dst = %dst.display(), + bytes_copied = bytes, "sparse copy succeeded" + ); + return Ok(()); + } + Err(e) => { + tracing::debug!(src = %src.display(), error = %e, "sparse copy failed, falling back"); + } + } + } + + #[cfg(windows)] + { + // std::fs::copy materialises holes on NTFS, so a large-virtual sparse + // template (e.g. a 10 GiB overlay with ~50 MB of real data) balloons to + // its full logical size. Copy sparsely instead, mirroring the Linux path. + match sparse_copy_windows(src, dst) { + Ok(bytes) => { + tracing::debug!( + src = %src.display(), dst = %dst.display(), + bytes_copied = bytes, "windows sparse copy succeeded" + ); + return Ok(()); + } + Err(e) => { + tracing::debug!(src = %src.display(), error = %e, "windows sparse copy failed, falling back to fs::copy"); + } + } + } + + std::fs::copy(src, dst).map_err(|e| Error::storage("copy file", e.to_string()))?; + Ok(()) +} + +/// Copy a sparse file on Windows/NTFS by replicating only its allocated regions, +/// leaving holes as holes — the Win32 analogue of the Linux `SEEK_HOLE`/`SEEK_DATA` +/// path. A scan-and-skip-zeros copy would have to read the whole logical size +/// (20+ GiB for a disk image) and, worse, allocate any non-zero region it finds; +/// `FSCTL_QUERY_ALLOCATED_RANGES` reports exactly the regions NTFS has backed, so +/// a 20 GiB image holding a few MB copies a few MB. +#[cfg(windows)] +fn sparse_copy_windows(src: &Path, dst: &Path) -> std::io::Result { + use std::io::{Read, Seek, SeekFrom, Write}; + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::IO::DeviceIoControl; + + // FSCTL_QUERY_ALLOCATED_RANGES (winioctl.h) + its in/out record. LARGE_INTEGER + // fields are signed 64-bit byte counts. + const FSCTL_QUERY_ALLOCATED_RANGES: u32 = 0x0009_40CF; + const ERROR_MORE_DATA: i32 = 234; + #[repr(C)] + #[derive(Clone, Copy)] + struct AllocatedRange { + file_offset: i64, + length: i64, + } + + let mut src_file = std::fs::File::open(src)?; + let src_len = src_file.metadata()?.len(); + if dst.exists() { + std::fs::remove_file(dst)?; + } + let mut dst_file = std::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(dst)?; + if src_len == 0 { + return Ok(0); + } + // Mark sparse BEFORE extending so the holes are never allocated. + mark_file_sparse(&dst_file)?; + dst_file.set_len(src_len)?; + + let mut total: u64 = 0; + let mut buf = vec![0u8; 1024 * 1024]; + let mut out: Vec = vec![ + AllocatedRange { + file_offset: 0, + length: 0 + }; + 1024 + ]; + // Query allocated ranges starting at 0; the FSCTL fills as many as fit, and + // signals ERROR_MORE_DATA when there are more (resume past the last range). + let mut query = AllocatedRange { + file_offset: 0, + length: src_len as i64, + }; + loop { + let mut returned: u32 = 0; + // SAFETY: src handle is live; `query` is a valid input record; `out` is a + // valid, correctly-sized output array; `returned` is a writable out-param. + let ok = unsafe { + DeviceIoControl( + src_file.as_raw_handle(), + FSCTL_QUERY_ALLOCATED_RANGES, + &query as *const _ as *const core::ffi::c_void, + std::mem::size_of::() as u32, + out.as_mut_ptr() as *mut core::ffi::c_void, + (out.len() * std::mem::size_of::()) as u32, + &mut returned, + std::ptr::null_mut(), + ) + }; + let more = + ok == 0 && std::io::Error::last_os_error().raw_os_error() == Some(ERROR_MORE_DATA); + if ok == 0 && !more { + return Err(std::io::Error::last_os_error()); + } + let count = returned as usize / std::mem::size_of::(); + if count == 0 { + break; + } + let mut resume = query.file_offset; + for r in &out[..count] { + let mut pos = r.file_offset as u64; + let end = (r.file_offset + r.length) as u64; + while pos < end { + let want = std::cmp::min((end - pos) as usize, buf.len()); + src_file.seek(SeekFrom::Start(pos))?; + src_file.read_exact(&mut buf[..want])?; + dst_file.seek(SeekFrom::Start(pos))?; + dst_file.write_all(&buf[..want])?; + pos += want as u64; + total += want as u64; + } + resume = r.file_offset + r.length; + } + if !more { + break; + } + query.file_offset = resume; + query.length = src_len as i64 - resume; + if query.length <= 0 { + break; + } + } + tracing::debug!(total, "sparse_copy_windows: allocated-range copy done"); + Ok(total) +} + +/// Copy only data regions of a sparse file via SEEK_HOLE/SEEK_DATA. +#[cfg(target_os = "linux")] +fn sparse_copy(src: &Path, dst: &Path) -> std::io::Result { + use std::os::unix::io::AsRawFd; + + let src_file = std::fs::File::open(src)?; + let src_len = src_file.metadata()?.len(); + if src_len == 0 { + std::fs::File::create(dst)?; + return Ok(0); + } + + let dst_file = std::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(dst)?; + dst_file.set_len(src_len)?; + + let src_fd = src_file.as_raw_fd(); + let dst_fd = dst_file.as_raw_fd(); + let mut total: u64 = 0; + let mut offset: i64 = 0; + let mut buf = vec![0u8; 256 * 1024]; + + loop { + let data_start = unsafe { libc::lseek(src_fd, offset, libc::SEEK_DATA) }; + if data_start < 0 { + if std::io::Error::last_os_error().raw_os_error() == Some(libc::ENXIO) { + break; + } + return Err(std::io::Error::last_os_error()); + } + let hole_start = unsafe { libc::lseek(src_fd, data_start, libc::SEEK_HOLE) }; + let data_end = if hole_start < 0 { + src_len as i64 + } else { + hole_start + }; + + let mut pos = data_start; + while pos < data_end { + let to_read = std::cmp::min((data_end - pos) as usize, buf.len()); + let n = + unsafe { libc::pread(src_fd, buf.as_mut_ptr() as *mut libc::c_void, to_read, pos) }; + if n <= 0 { + break; + } + let written = unsafe { + libc::pwrite(dst_fd, buf.as_ptr() as *const libc::c_void, n as usize, pos) + }; + if written <= 0 { + return Err(std::io::Error::last_os_error()); + } + pos += n as i64; + total += n as u64; + } + offset = data_end; + } + Ok(total) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Seek, SeekFrom, Write}; + + /// `clone_or_copy_file` must reproduce a sparse file byte-for-byte on every + /// platform — including the Windows sparse-copy path, which skips zero runs. + /// We build a file with a head data region, a large hole, and a tail data + /// region, then assert the clone's length and full contents match. + #[test] + fn clone_or_copy_preserves_sparse_contents() { + let dir = tempfile::tempdir().unwrap(); + let src = dir.path().join("src.img"); + let dst = dir.path().join("dst.img"); + + let logical_len: u64 = 8 * 1024 * 1024; // 8 MiB logical + let head = b"HEAD-DATA-REGION"; + let tail = b"TAIL-DATA-REGION"; + { + let mut f = std::fs::File::create(&src).unwrap(); + f.write_all(head).unwrap(); + f.set_len(logical_len).unwrap(); + f.seek(SeekFrom::Start(logical_len - tail.len() as u64)) + .unwrap(); + f.write_all(tail).unwrap(); + } + + clone_or_copy_file(&src, &dst).expect("clone_or_copy_file"); + + let src_bytes = std::fs::read(&src).unwrap(); + let dst_bytes = std::fs::read(&dst).unwrap(); + assert_eq!( + dst_bytes.len() as u64, + logical_len, + "clone must preserve logical length" + ); + assert_eq!( + src_bytes, dst_bytes, + "clone must be byte-identical to source" + ); + assert_eq!(&dst_bytes[..head.len()], head); + assert_eq!(&dst_bytes[dst_bytes.len() - tail.len()..], tail); + + // On Windows the whole point is that the clone stays sparse — the 8 MiB + // hole must NOT be allocated on disk. GetCompressedFileSizeW reports the + // real on-disk allocation; assert it's a tiny fraction of the logical size. + #[cfg(windows)] + { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::GetCompressedFileSizeW; + let wide: Vec = dst + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let mut high: u32 = 0; + // SAFETY: `wide` is a valid NUL-terminated path; `high` is a writable out-param. + let low = unsafe { GetCompressedFileSizeW(wide.as_ptr(), &mut high) }; + let allocated = ((high as u64) << 32) | (low as u64); + assert!( + allocated < logical_len / 2, + "sparse clone must not allocate the full {logical_len} bytes (got {allocated} allocated)" + ); + } + } + + /// Regression: `expand_sparse_disk` must reject a size whose byte count + /// overflows u64 with a clean `Err`, never a panic (debug) or a wrapped / + /// zero-length sparse file (release). `2^34 GiB * 2^30 bytes/GiB = 2^64`, + /// which is exactly the u64 overflow boundary. + #[test] + fn expand_rejects_overflowing_size_gb() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("overflow.img"); + // The path need not exist: the overflow check runs before any file I/O, + // so this must fail on the multiply, not on a missing-file error. + let overflow_gb: u64 = 1u64 << 34; // * BYTES_PER_GIB (2^30) == 2^64 + let err = expand_sparse_disk::(&path, overflow_gb) + .expect_err("overflowing size_gb must return Err, not panic or truncate"); + let msg = format!("{}", err); + assert!( + msg.contains("too large") && msg.contains("overflow"), + "expected an overflow error, got: {msg}" + ); + assert!( + !path.exists(), + "no disk file should be created when the size overflows" + ); + } + + /// A merely-large-but-valid size must not be rejected by the overflow guard + /// (it will fail later on the missing file, proving the multiply succeeded). + #[test] + fn expand_allows_large_but_valid_size_gb() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("missing.img"); + // 1 GiB below the overflow boundary — the multiply is fine; the call + // then fails reading metadata of the nonexistent path. + let big_gb: u64 = (1u64 << 34) - 1; + let err = expand_sparse_disk::(&path, big_gb) + .expect_err("nonexistent path should fail after a successful multiply"); + let msg = format!("{}", err); + assert!( + !msg.contains("too large"), + "a valid (non-overflowing) size must not trip the overflow guard: {msg}" + ); + } +} diff --git a/src/dns_filter.rs b/src/dns_filter.rs new file mode 100644 index 0000000..25bea61 --- /dev/null +++ b/src/dns_filter.rs @@ -0,0 +1,372 @@ +//! Host-side DNS filtering proxy. +//! +//! Receives raw DNS query packets from the guest agent over vsock, +//! checks the queried domain against an allowlist, and either resolves +//! upstream (allowed) or returns NXDOMAIN (blocked). +//! +//! No third-party DNS library — the parser extracts just the domain name +//! from the DNS question section, which is sufficient for filtering. + +use std::io::{self, Read, Write}; +use std::net::UdpSocket; + +/// DNS filter configuration. +#[derive(Debug, Clone)] +pub struct DnsFilter { + /// Allowed domains (exact match + wildcard subdomains). + /// e.g., "api.stripe.com" allows "api.stripe.com" and "foo.api.stripe.com". + allowed: Vec, + /// Upstream DNS resolver address. + upstream: String, +} + +impl DnsFilter { + /// Create a new DNS filter from a list of allowed hostnames. + pub fn new(allowed_hosts: Vec, upstream: String) -> Self { + Self { + allowed: allowed_hosts, + upstream, + } + } + + /// Check if a domain is allowed by the filter. + pub fn is_allowed(&self, domain: &str) -> bool { + let domain = domain.trim_end_matches('.'); + self.allowed.iter().any(|pattern| { + let pattern = pattern.trim_end_matches('.'); + domain.eq_ignore_ascii_case(pattern) + || domain + .to_ascii_lowercase() + .ends_with(&format!(".{}", pattern.to_ascii_lowercase())) + }) + } + + /// Handle a raw DNS query: filter and resolve or return NXDOMAIN. + pub fn handle_query(&self, raw_query: &[u8]) -> Vec { + let domain = match extract_domain_from_query(raw_query) { + Some(d) => d, + None => { + tracing::debug!("could not parse DNS query, returning SERVFAIL"); + return build_servfail(raw_query); + } + }; + + if self.is_allowed(&domain) { + tracing::debug!(domain, "DNS query allowed, resolving upstream"); + match self.resolve_upstream(raw_query) { + Ok(resp) => resp, + Err(e) => { + tracing::warn!(domain, error = %e, "upstream DNS resolution failed"); + build_servfail(raw_query) + } + } + } else { + tracing::info!(domain, "DNS query blocked by policy"); + build_nxdomain(raw_query) + } + } + + /// Forward a DNS query to the upstream resolver and return the response. + /// + /// Tries UDP first. If the upstream sets the TC (truncated) bit in the + /// response, retries via TCP to retrieve the full answer. + fn resolve_upstream(&self, raw_query: &[u8]) -> io::Result> { + let resp = self.resolve_upstream_udp(raw_query)?; + // TC bit is flags byte 2 bit 1. When set the response was truncated + // and the caller should retry over TCP (RFC 1035 §4.2.1). + if resp.len() >= 3 && resp[2] & 0x02 != 0 { + tracing::debug!("upstream UDP response truncated (TC=1), retrying via TCP"); + return self.resolve_upstream_tcp(raw_query); + } + Ok(resp) + } + + fn resolve_upstream_udp(&self, raw_query: &[u8]) -> io::Result> { + let socket = UdpSocket::bind("0.0.0.0:0")?; + socket.set_read_timeout(Some(std::time::Duration::from_secs(5)))?; + socket.send_to(raw_query, format!("{}:53", self.upstream))?; + // 4096 bytes covers standard EDNS0 payloads; responses larger than + // this will have TC=1 set by the upstream, triggering TCP retry. + let mut buf = [0u8; 4096]; + let (len, _) = socket.recv_from(&mut buf)?; + Ok(buf[..len].to_vec()) + } + + fn resolve_upstream_tcp(&self, raw_query: &[u8]) -> io::Result> { + use std::net::{SocketAddr, TcpStream}; + let addr: SocketAddr = format!("{}:53", self.upstream) + .parse() + .map_err(|e| io::Error::new(io::ErrorKind::InvalidInput, e))?; + let mut stream = TcpStream::connect_timeout(&addr, std::time::Duration::from_secs(5))?; + stream.set_read_timeout(Some(std::time::Duration::from_secs(5)))?; + // TCP DNS framing: 2-byte BE length prefix (RFC 1035 §4.2.2). + let len = raw_query.len() as u16; + stream.write_all(&len.to_be_bytes())?; + stream.write_all(raw_query)?; + stream.flush()?; + let mut len_buf = [0u8; 2]; + stream.read_exact(&mut len_buf)?; + let resp_len = u16::from_be_bytes(len_buf) as usize; + let mut resp = vec![0u8; resp_len]; + stream.read_exact(&mut resp)?; + Ok(resp) + } +} + +/// Handle a single vsock connection from the guest DNS proxy. +/// +/// Reads a length-prefixed DNS query, filters it, and sends back +/// the response with the same framing. +pub fn handle_connection(filter: &DnsFilter, stream: &mut (impl Read + Write)) -> io::Result<()> { + // Read: [2-byte BE length] [raw DNS query] + let mut len_buf = [0u8; 2]; + stream.read_exact(&mut len_buf)?; + let query_len = u16::from_be_bytes(len_buf) as usize; + + if query_len == 0 || query_len > 1024 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid DNS query length: {query_len}"), + )); + } + + let mut query = vec![0u8; query_len]; + stream.read_exact(&mut query)?; + + // Filter and resolve + let response = filter.handle_query(&query); + + // Write: [2-byte BE length] [raw DNS response] + let resp_len = response.len() as u16; + stream.write_all(&resp_len.to_be_bytes())?; + stream.write_all(&response)?; + stream.flush()?; + + Ok(()) +} + +// ============================================================================ +// Minimal DNS packet parsing (no external dependency) +// ============================================================================ + +/// Extract the queried domain name from a raw DNS query packet. +/// +/// Returns None if the packet is malformed or has no question section. +fn extract_domain_from_query(packet: &[u8]) -> Option { + // DNS header is 12 bytes + if packet.len() < 12 { + return None; + } + + // QDCOUNT (question count) at bytes 4-5 + let qdcount = u16::from_be_bytes([packet[4], packet[5]]); + if qdcount == 0 { + return None; + } + + // Question section starts at byte 12 + let mut pos = 12; + let mut labels: Vec = Vec::new(); + + loop { + if pos >= packet.len() { + return None; + } + + let label_len = packet[pos] as usize; + if label_len == 0 { + break; // Root label — end of name + } + + // Pointer (compression) — shouldn't appear in queries, but handle it + if label_len & 0xC0 == 0xC0 { + return None; // Don't follow pointers in queries + } + + pos += 1; + if pos + label_len > packet.len() { + return None; + } + + let label = std::str::from_utf8(&packet[pos..pos + label_len]).ok()?; + labels.push(label.to_string()); + pos += label_len; + } + + if labels.is_empty() { + return None; + } + + Some(labels.join(".")) +} + +/// Build an NXDOMAIN response from a query. +fn build_nxdomain(query: &[u8]) -> Vec { + if query.len() < 12 { + return vec![]; + } + let mut resp = query.to_vec(); + resp[2] = 0x80 | (resp[2] & 0x78); // QR=1, preserve opcode + resp[3] = (resp[3] & 0xF0) | 0x03; // RCODE=NXDOMAIN (3) + // Set RA (recursion available) + resp[3] |= 0x80; + // Zero answer/authority/additional counts + if resp.len() >= 12 { + resp[6..12].fill(0); + } + resp +} + +/// Build a SERVFAIL response from a query. +fn build_servfail(query: &[u8]) -> Vec { + if query.len() < 12 { + return vec![]; + } + let mut resp = query.to_vec(); + resp[2] = 0x80 | (resp[2] & 0x78); // QR=1, preserve opcode + resp[3] = (resp[3] & 0xF0) | 0x02; // RCODE=SERVFAIL (2) + resp[3] |= 0x80; // RA + if resp.len() >= 12 { + resp[6..12].fill(0); + } + resp +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a minimal DNS query packet for a domain. + fn build_query(domain: &str) -> Vec { + let mut packet = vec![ + 0x12, 0x34, // ID + 0x01, 0x00, // Flags: standard query, RD=1 + 0x00, 0x01, // QDCOUNT: 1 + 0x00, 0x00, // ANCOUNT: 0 + 0x00, 0x00, // NSCOUNT: 0 + 0x00, 0x00, // ARCOUNT: 0 + ]; + for label in domain.split('.') { + packet.push(label.len() as u8); + packet.extend_from_slice(label.as_bytes()); + } + packet.push(0x00); // Root label + packet.extend_from_slice(&[0x00, 0x01]); // QTYPE: A + packet.extend_from_slice(&[0x00, 0x01]); // QCLASS: IN + packet + } + + #[test] + fn test_extract_domain() { + let query = build_query("api.stripe.com"); + assert_eq!( + extract_domain_from_query(&query), + Some("api.stripe.com".to_string()) + ); + } + + #[test] + fn test_extract_domain_single_label() { + let query = build_query("localhost"); + assert_eq!( + extract_domain_from_query(&query), + Some("localhost".to_string()) + ); + } + + #[test] + fn test_extract_domain_short_packet() { + assert_eq!(extract_domain_from_query(&[0x00; 5]), None); + } + + #[test] + fn test_filter_allowed_exact() { + let filter = DnsFilter::new(vec!["api.stripe.com".into()], "1.1.1.1".into()); + assert!(filter.is_allowed("api.stripe.com")); + assert!(filter.is_allowed("api.stripe.com.")); + } + + #[test] + fn test_filter_allowed_subdomain() { + let filter = DnsFilter::new(vec!["stripe.com".into()], "1.1.1.1".into()); + assert!(filter.is_allowed("api.stripe.com")); + assert!(filter.is_allowed("dashboard.stripe.com")); + assert!(!filter.is_allowed("notstripe.com")); + } + + #[test] + fn test_filter_blocked() { + let filter = DnsFilter::new(vec!["api.stripe.com".into()], "1.1.1.1".into()); + assert!(!filter.is_allowed("attacker.com")); + assert!(!filter.is_allowed("stripe.com")); // not a subdomain of api.stripe.com + } + + #[test] + fn test_filter_case_insensitive() { + let filter = DnsFilter::new(vec!["API.Stripe.COM".into()], "1.1.1.1".into()); + assert!(filter.is_allowed("api.stripe.com")); + } + + #[test] + fn test_nxdomain_response() { + let query = build_query("attacker.com"); + let resp = build_nxdomain(&query); + assert_eq!(resp[0], 0x12); // ID preserved + assert_eq!(resp[1], 0x34); + assert_eq!(resp[2] & 0x80, 0x80); // QR=1 + assert_eq!(resp[3] & 0x0F, 0x03); // RCODE=NXDOMAIN + } + + #[test] + fn test_handle_query_allowed() { + // This test actually resolves DNS, so it needs network. + // Using a domain that's guaranteed to exist. + let filter = DnsFilter::new(vec!["one.one.one.one".into()], "1.1.1.1".into()); + let query = build_query("one.one.one.one"); + let response = filter.handle_query(&query); + // Should be a valid DNS response (not NXDOMAIN) + assert!(response.len() >= 12); + assert_eq!(response[2] & 0x80, 0x80); // QR=1 + assert_ne!(response[3] & 0x0F, 0x03); // NOT NXDOMAIN + } + + #[test] + fn test_handle_query_blocked() { + let filter = DnsFilter::new(vec!["api.stripe.com".into()], "1.1.1.1".into()); + let query = build_query("attacker.com"); + let response = filter.handle_query(&query); + assert!(response.len() >= 12); + assert_eq!(response[3] & 0x0F, 0x03); // RCODE=NXDOMAIN + } + + #[test] + fn test_tc_bit_detection() { + // Verify the TC (truncated) bit position: flags byte 2, bit 1 (0x02). + // A UDP response with TC=1 should trigger TCP retry in resolve_upstream. + let mut resp = build_query("example.com"); + resp[2] = 0x80; // QR=1, TC=0 + assert_eq!(resp[2] & 0x02, 0x00); // TC clear + + resp[2] |= 0x02; // Set TC=1 + assert_eq!(resp[2] & 0x02, 0x02); // TC set — would trigger TCP retry + + // Sanity: NXDOMAIN and SERVFAIL responses have TC=0 (we build them). + let nxdomain = build_nxdomain(&build_query("example.com")); + assert_eq!(nxdomain[2] & 0x02, 0x00); + let servfail = build_servfail(&build_query("example.com")); + assert_eq!(servfail[2] & 0x02, 0x00); + } + + #[test] + fn test_handle_connection_blocked() { + let filter = DnsFilter::new(vec!["api.stripe.com".into()], "1.1.1.1".into()); + let query = build_query("attacker.com"); + + // Test the filter logic directly — connection framing is trivial + // (2-byte length prefix) and tested implicitly via integration tests. + let response = filter.handle_query(&query); + + assert_eq!(response[0], 0x12); // ID preserved + assert_eq!(response[3] & 0x0F, 0x03); // RCODE=NXDOMAIN + } +} diff --git a/src/dns_filter_listener.rs b/src/dns_filter_listener.rs new file mode 100644 index 0000000..b5b7d0e --- /dev/null +++ b/src/dns_filter_listener.rs @@ -0,0 +1,53 @@ +//! Host-side DNS filter listener. +//! +//! Starts a Unix socket listener that accepts connections from the guest +//! DNS proxy and filters DNS queries against a hostname allowlist. + +use crate::dns_filter::{handle_connection, DnsFilter}; +use crate::platform::uds::UdsListener; +use std::path::Path; +use std::sync::Arc; +use std::thread; + +/// Start the DNS filter listener on a Unix socket. +/// +/// The listener runs in a background thread and handles DNS queries from +/// the guest agent's DNS proxy. +/// +/// The caller should pass the socket path to `LaunchConfig::dns_filter_socket` +/// so libkrun maps it to vsock port 6002. +pub fn start(socket_path: &Path, allowed_hosts: Vec) -> std::io::Result<()> { + // Clean up stale socket + let _ = std::fs::remove_file(socket_path); + + let listener = UdsListener::bind(socket_path)?; + + let filter = Arc::new(DnsFilter::new( + allowed_hosts, + crate::data::network::default_dns(), + )); + + let path_display = socket_path.display().to_string(); + + thread::Builder::new() + .name("dns-filter-host".into()) + .spawn(move || { + tracing::info!(path = path_display, "DNS filter listener started"); + + for stream in listener.incoming() { + match stream { + Ok(mut stream) => { + let filter = Arc::clone(&filter); + if let Err(e) = handle_connection(&filter, &mut stream) { + tracing::debug!(error = %e, "DNS filter connection error"); + } + } + Err(e) => { + tracing::debug!(error = %e, "DNS filter accept error"); + } + } + } + })?; + + Ok(()) +} diff --git a/src/embedded/control.rs b/src/embedded/control.rs new file mode 100644 index 0000000..db2711a --- /dev/null +++ b/src/embedded/control.rs @@ -0,0 +1,315 @@ +//! DB-backed VM lifecycle helpers for embedded SDK backends. + +use crate::agent::{AgentClient, AgentManager, HostMount, LaunchFeatures, VmResources}; +use crate::config::{RecordState, VmRecord}; +use crate::data::network::PortMapping; +use crate::data::validate_vm_name; +use crate::db::SmolvmDb; +use crate::embedded::handle::VmHandle; +use crate::{Error, Result}; + +/// Runtime configuration supplied by an embedded SDK constructor. +#[derive(Debug, Clone)] +pub struct MachineSpec { + /// Unique machine name. + pub name: String, + /// Host directory mounts to expose in the guest. + pub mounts: Vec, + /// Host-to-guest port mappings. + pub ports: Vec, + /// VM resources for this machine. + pub resources: VmResources, + /// Whether the machine should persist across stop/start. + pub persistent: bool, +} + +impl MachineSpec { + /// Convert the embedded-machine spec into the canonical DB record. + pub fn to_record(&self) -> VmRecord { + let mut record = VmRecord::new( + self.name.clone(), + self.resources.cpus, + self.resources.memory_mib, + self.mounts + .iter() + .map(HostMount::to_storage_tuple) + .collect(), + self.ports.iter().map(PortMapping::to_tuple).collect(), + self.resources.network, + ); + record.storage_gb = self.resources.storage_gib; + record.overlay_gb = self.resources.overlay_gib; + record.allowed_cidrs = self.resources.allowed_cidrs.clone(); + record.gpu = Some(self.resources.gpu); + record.gpu_vram_mib = self.resources.gpu_vram_mib; + record.ephemeral = !self.persistent; + record + } +} + +/// Create a DB record for a new SDK machine. +pub fn create_vm(db: &SmolvmDb, spec: &MachineSpec) -> Result<()> { + validate_vm_name(&spec.name, "name") + .map_err(|reason| Error::config("validate machine name", reason))?; + let record = spec.to_record(); + if db.insert_vm_if_not_exists(&spec.name, &record)? { + Ok(()) + } else { + Err(Error::agent_conflict( + "create machine", + format!("machine '{}' already exists", spec.name), + )) + } +} + +/// Load a persisted VM record. +pub fn get_record(db: &SmolvmDb, name: &str) -> Result { + db.get_vm(name)?.ok_or_else(|| Error::vm_not_found(name)) +} + +/// Start a persisted VM and update its DB state. +pub fn start_vm(db: &SmolvmDb, name: &str) -> Result { + let record = get_record(db, name)?; + let handle = start_vm_from_record(&record)?; + mark_running(db, name, handle.child_pid())?; + Ok(handle) +} + +fn start_vm_from_record(record: &VmRecord) -> Result { + launch_from_record(record, LaunchFeatures::default()) +} + +/// Boot `record` with the given launch features and return a handle. Shared by +/// the plain, forkable-golden, and fork-clone start paths so they can't drift. +fn launch_from_record(record: &VmRecord, features: LaunchFeatures) -> Result { + let manager = + AgentManager::for_vm_with_sizes(&record.name, record.storage_gb, record.overlay_gb) + .map_err(|e| Error::agent("create agent manager", e.to_string()))?; + + manager + .ensure_running_with_full_config( + record.host_mounts(), + record.port_mappings(), + record.vm_resources(), + features, + ) + .map_err(|e| Error::agent("start machine", e.to_string()))?; + + Ok(VmHandle::new(manager, None)) +} + +/// Start a persisted VM as a FORKABLE fork base: its guest RAM is backed by a +/// memfd (copy-on-write cloneable) and a control socket is exposed so the machine +/// can later be forked with [`fork_vm`]. Same mechanics as the CLI's +/// `machine start --forkable`, surfaced for the embedded SDK. +pub fn start_forkable_vm(db: &SmolvmDb, name: &str) -> Result { + let record = get_record(db, name)?; + let features = LaunchFeatures { + forkable: true, + control_socket: Some(crate::agent::fork::control_socket_path(name)), + ..LaunchFeatures::default() + }; + let handle = launch_from_record(&record, features)?; + mark_running(db, name, handle.child_pid())?; + Ok(handle) +} + +/// Fork a running, forkable `golden` into a new `clone` via copy-on-write guest +/// RAM + disks (same host). Freezes the golden (it stays paused as the shared +/// base — clones map its RAM `MAP_PRIVATE`, so it must not run again while clones +/// exist), boots the clone from the golden's snapshot, and returns the clone's +/// handle. `pinned_ports` are `(host, guest)` inbound forwards for the clone; +/// empty means the golden's forwards are remapped to freshly-allocated host +/// ports. Shares `agent::fork` with the CLI/serve fork paths. +pub fn fork_vm( + db: &SmolvmDb, + golden: &str, + clone: &str, + pinned_ports: &[(u16, u16)], +) -> Result { + // Freeze + snapshot the golden, register the clone (CoW disks + DB record). + // `clone_forkable = false`: a clone can't itself be re-forked (nested fork). + let prep = crate::agent::fork::prepare_fork(db, golden, clone, pinned_ports, false)?; + + // Boot the clone from the golden's in-memory snapshot instead of cold-booting. + let features = LaunchFeatures { + snapshot_dir: Some(prep.snapshot_dir.clone()), + ..LaunchFeatures::default() + }; + match launch_from_record(&prep.clone_record, features) { + Ok(mut handle) => { + // Fresh on-disk identity (hostname, machine-id, SSH host keys, RNG). + // FAIL-CLOSED: if the reset can't be confirmed, stop the booted clone + // and roll it back rather than leave it live with the golden's + // per-machine secrets. + crate::agent::fork::fail_closed_on_rejuvenation( + crate::agent::fork::rejuvenate_clone(clone), + || { + let _ = handle.stop(); + let _ = db.remove_vm(clone); + let _ = std::fs::remove_dir_all(crate::agent::vm_data_dir(clone)); + }, + )?; + mark_running(db, clone, handle.child_pid())?; + Ok(handle) + } + Err(e) => { + // prepare_fork already registered the clone; roll it back on boot failure. + let _ = db.remove_vm(clone); + let _ = std::fs::remove_dir_all(crate::agent::vm_data_dir(clone)); + Err(e) + } + } +} + +/// Connect to an already-running VM and return a cached handle. +pub fn connect_vm(db: &SmolvmDb, name: &str) -> Result { + let record = get_record(db, name)?; + let manager = AgentManager::for_vm_with_sizes(name, record.storage_gb, record.overlay_gb) + .map_err(|e| Error::agent("create agent manager", e.to_string()))?; + + if manager.try_connect_existing().is_none() { + return Err(Error::agent_not_found( + "connect machine", + format!("machine '{}' is not running", name), + )); + } + + let client = AgentClient::connect_with_retry(manager.vsock_socket())?; + Ok(VmHandle::new(manager, Some(client))) +} + +/// Stop a persisted VM and update its DB state. +pub fn stop_vm(db: &SmolvmDb, name: &str) -> Result<()> { + let record = get_record(db, name)?; + let manager = AgentManager::for_vm_with_sizes(name, record.storage_gb, record.overlay_gb) + .map_err(|e| Error::agent("create agent manager", e.to_string()))?; + manager.try_connect_existing(); + manager.stop()?; + // Detach the per-machine layers volume if a (possibly cross-tool) bundle start + // left it mounted. Unconditional on purpose: the embedded record may carry no + // source_smolmachine even when a CLI/API `machine create ` extracted and + // mounted this name's volume in the shared DB, so we cannot gate on it. + // force_detach is infallible and no-ops when unmounted; macOS hdiutil detach, a + // compile-time no-op on Linux. + smolvm_pack::extract::force_detach_layers_volume(&crate::agent::machine_layers_cache_dir(name)); + mark_stopped(db, name) +} + +/// Remove a VM record and its storage directory. +pub fn delete_vm(db: &SmolvmDb, name: &str) -> Result<()> { + let removed = db.remove_vm(name)?; + if removed.is_none() { + return Err(Error::vm_not_found(name)); + } + + let data_dir = crate::agent::vm_data_dir(name); + // Detach the per-machine layers volume before removing the data dir, else on + // macOS the live mountpoint under it makes remove_dir_all fail with "Resource + // busy", stranding both the mount and the data dir. Unconditional on purpose: + // the embedded record may carry no source_smolmachine even when a CLI/API create + // mounted this name's volume. hdiutil detach; a no-op on Linux and when nothing + // is mounted. + smolvm_pack::extract::force_detach_layers_volume(&crate::agent::machine_layers_cache_dir(name)); + if data_dir.exists() { + std::fs::remove_dir_all(&data_dir).map_err(|e| { + Error::storage( + "delete machine data", + format!("{}: {}", data_dir.display(), e), + ) + })?; + } + + Ok(()) +} + +/// Mark a machine record as running. +pub fn mark_running(db: &SmolvmDb, name: &str, pid: Option) -> Result<()> { + let pid_start_time = pid.and_then(crate::process::process_start_time); + db.update_vm(name, |record| { + record.state = RecordState::Running; + record.pid = pid; + record.pid_start_time = pid_start_time; + })? + .ok_or_else(|| Error::vm_not_found(name))?; + Ok(()) +} + +/// Mark a machine record as stopped. +pub fn mark_stopped(db: &SmolvmDb, name: &str) -> Result<()> { + db.update_vm(name, |record| { + record.state = RecordState::Stopped; + record.pid = None; + record.pid_start_time = None; + })? + .ok_or_else(|| Error::vm_not_found(name))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_db() -> SmolvmDb { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "smolvm-embedded-control-{}-{}.db", + std::process::id(), + unique + )); + SmolvmDb::open_at(&path).unwrap() + } + + fn test_spec(name: &str, persistent: bool) -> MachineSpec { + MachineSpec { + name: name.to_string(), + mounts: Vec::new(), + ports: Vec::new(), + resources: VmResources::default(), + persistent, + } + } + + #[test] + fn record_ephemeral_follows_persistent_flag() { + assert!(test_spec("ephemeral", false).to_record().ephemeral); + assert!(!test_spec("persistent", true).to_record().ephemeral); + } + + #[test] + fn record_carries_gpu_resources() { + // GPU must survive MachineSpec -> VmRecord (the `_boot-vm` config), + // otherwise the SDK's `resources.gpu` is silently dropped before launch. + let mut spec = test_spec("gpu", false); + spec.resources.gpu = true; + spec.resources.gpu_vram_mib = Some(512); + let record = spec.to_record(); + assert_eq!(record.gpu, Some(true)); + assert_eq!(record.gpu_vram_mib, Some(512)); + assert!(record.vm_resources().gpu); + + // Default (no GPU) records leave gpu off. + let plain = test_spec("plain", false).to_record(); + assert_eq!(plain.gpu, Some(false)); + assert!(!plain.vm_resources().gpu); + } + + #[test] + fn create_vm_rejects_duplicates() { + let db = test_db(); + let spec = test_spec("duplicate", false); + create_vm(&db, &spec).unwrap(); + + let err = create_vm(&db, &spec).unwrap_err(); + assert!(matches!( + err, + Error::Agent { + kind: crate::error::AgentErrorKind::Conflict, + .. + } + )); + } +} diff --git a/src/embedded/handle.rs b/src/embedded/handle.rs new file mode 100644 index 0000000..1838258 --- /dev/null +++ b/src/embedded/handle.rs @@ -0,0 +1,135 @@ +//! Runtime VM handle for embedded SDK backends. + +use std::time::Duration; + +use crate::agent::{AgentClient, AgentManager, ExecEvent, RunConfig}; +use crate::Result; +use smolvm_protocol::ImageInfo; + +/// Handle to a running VM process. +pub struct VmHandle { + manager: AgentManager, + client: Option, +} + +// SAFETY: The embedded runtime stores `VmHandle` behind a mutex and only moves +// it into blocking worker threads. `AgentManager` guards its mutable state +// internally, and `AgentClient` owns a Unix stream that is safe to move between +// threads when access is serialized by the handle mutex. +unsafe impl Send for VmHandle {} + +impl VmHandle { + /// Construct a handle from an already-created process manager. + pub fn new(manager: AgentManager, client: Option) -> Self { + Self { manager, client } + } + + /// Get the child PID if known. + pub fn child_pid(&self) -> Option { + self.manager.child_pid() + } + + /// Check whether the VM process is alive. + pub fn is_process_alive(&self) -> bool { + self.manager.is_process_alive() + } + + /// Return the agent manager state as a string. + pub fn state(&self) -> String { + self.manager.state().to_string() + } + + fn client_mut(&mut self) -> Result<&mut AgentClient> { + if self.client.is_none() { + self.client = Some(self.manager.connect()?); + } + Ok(self.client.as_mut().expect("client initialized")) + } + + /// Execute a command directly in the VM. + pub fn exec( + &mut self, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + ) -> Result<(i32, Vec, Vec)> { + self.client_mut()? + .vm_exec(command, env, workdir, timeout, None) + } + + /// Pull an OCI image and run a command inside it. + /// + /// Returns `(exit_code, stdout_bytes, stderr_bytes)`. Bytes are raw + /// to preserve binary output. + pub fn run( + &mut self, + image: &str, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + ) -> Result<(i32, Vec, Vec)> { + let client = self.client_mut()?; + client.pull_with_registry_config(image)?; + let config = RunConfig::new(image, command) + .with_env(env) + .with_workdir(workdir) + .with_timeout(timeout); + client.run_non_interactive(config) + } + + /// Pull an OCI image into the VM storage. + pub fn pull_image(&mut self, image: &str) -> Result { + self.client_mut()?.pull_with_registry_config(image) + } + + /// List cached OCI images in the VM storage. + pub fn list_images(&mut self) -> Result> { + self.client_mut()?.list_images() + } + + /// Write a file into the VM. + pub fn write_file(&mut self, path: &str, data: &[u8], mode: Option) -> Result<()> { + self.client_mut()?.write_file(path, data, mode) + } + + /// Read a file from the VM. + pub fn read_file(&mut self, path: &str) -> Result> { + self.client_mut()?.read_file(path) + } + + /// Execute a command, delivering stdout/stderr/exit events LIVE via the + /// callback as each frame arrives (no buffering). SDKs bridge this to a + /// language-native iterator (Python generator / JS async iterator). + pub fn exec_streaming_with( + &mut self, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + on_event: F, + ) -> Result<()> { + self.client_mut()? + .vm_exec_streaming_with(command, env, workdir, timeout, on_event) + } + + /// Stream a command's output LIVE while running it inside the image's + /// container overlay (the `RunConfig` carries the persistent-overlay id). + /// The streaming counterpart of `run` — used for image machines so streamed + /// execs share the same container filesystem + persistence as non-streaming + /// execs, instead of running in the bare agent rootfs. + pub fn run_streaming_with( + &mut self, + config: RunConfig, + on_event: F, + ) -> Result<()> { + self.client_mut()?.run_streaming_with(config, on_event) + } + + /// Stop the VM and drop the cached agent client. + pub fn stop(&mut self) -> Result<()> { + self.client = None; + self.manager.stop() + } +} diff --git a/src/embedded/mod.rs b/src/embedded/mod.rs new file mode 100644 index 0000000..5746811 --- /dev/null +++ b/src/embedded/mod.rs @@ -0,0 +1,8 @@ +//! Language-neutral embedded runtime support for SDK bindings. + +mod control; +mod handle; +mod runtime; + +pub use control::MachineSpec; +pub use runtime::{runtime, EmbeddedRuntime}; diff --git a/src/embedded/runtime.rs b/src/embedded/runtime.rs new file mode 100644 index 0000000..09dbf08 --- /dev/null +++ b/src/embedded/runtime.rs @@ -0,0 +1,514 @@ +//! Process-local runtime registry for embedded machines. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, MutexGuard, OnceLock, RwLock}; +use std::time::Duration; + +use crate::agent::{ExecEvent, RunConfig}; +use crate::config::RecordState; +use crate::db::SmolvmDb; +use crate::embedded::control::{self, MachineSpec}; +use crate::embedded::handle::VmHandle; +use crate::{Error, Result}; +use smolvm_protocol::ImageInfo; + +type SharedHandle = Arc>; + +/// Stateful runtime shared by all embedded machine objects in this process. +pub struct EmbeddedRuntime { + db: SmolvmDb, + registry: RwLock>, + name_locks: RwLock>>>, +} + +impl EmbeddedRuntime { + /// Create a runtime backed by the default smolvm database. + pub fn new() -> Result { + Ok(Self::with_db(SmolvmDb::open()?)) + } + + /// Create a runtime backed by an explicit database handle. + pub fn with_db(db: SmolvmDb) -> Self { + Self { + db, + registry: RwLock::new(HashMap::new()), + name_locks: RwLock::new(HashMap::new()), + } + } + + /// Create a persisted machine record. + pub fn create_machine(&self, spec: MachineSpec) -> Result<()> { + self.with_name_lock(&spec.name, || control::create_vm(&self.db, &spec)) + } + + /// Start or reconnect to a persisted machine and cache its handle. + pub fn start_machine(&self, name: &str) -> Result<()> { + self.with_name_lock(name, || { + if let Some(handle) = self.cached_handle(name)? { + let alive = lock_handle(&handle)?.is_process_alive(); + if alive { + return Ok(()); + } + self.remove_cached_handle(name)?; + } + + let handle = control::start_vm(&self.db, name)?; + self.insert_handle(name, handle)?; + Ok(()) + }) + } + + /// Start (or reconnect to) a persisted machine as a FORKABLE fork base and + /// cache its handle. Like [`start_machine`](Self::start_machine) but boots + /// with memfd-backed guest RAM + a control socket so it can later be forked + /// with [`fork_machine`](Self::fork_machine) — the basis for local RL rollout + /// branching. + pub fn start_forkable_machine(&self, name: &str) -> Result<()> { + self.with_name_lock(name, || { + if let Some(handle) = self.cached_handle(name)? { + if lock_handle(&handle)?.is_process_alive() { + return Ok(()); + } + self.remove_cached_handle(name)?; + } + + let handle = control::start_forkable_vm(&self.db, name)?; + self.insert_handle(name, handle)?; + Ok(()) + }) + } + + /// Fork a running, forkable `golden` into a new `clone` (copy-on-write guest + /// RAM + disks, same host) and cache the clone's handle so it can be exec'd + /// by name. `ports` are `(host, guest)` inbound forwards for the clone; empty + /// remaps the golden's forwards to fresh host ports. The golden freezes as the + /// shared base and must not be started again while clones exist. + pub fn fork_machine(&self, golden: &str, clone: &str, ports: &[(u16, u16)]) -> Result<()> { + self.with_name_lock(clone, || { + let handle = control::fork_vm(&self.db, golden, clone, ports)?; + self.insert_handle(clone, handle)?; + Ok(()) + }) + } + + /// Connect to an already-running machine and cache its handle. + pub fn connect_machine(&self, name: &str) -> Result<()> { + self.with_name_lock(name, || { + if let Some(handle) = self.cached_handle(name)? { + if lock_handle(&handle)?.is_process_alive() { + return Ok(()); + } + self.remove_cached_handle(name)?; + } + + let handle = control::connect_vm(&self.db, name)?; + self.insert_handle(name, handle)?; + Ok(()) + }) + } + + /// Stop a machine and persist stopped state. + pub fn stop_machine(&self, name: &str) -> Result<()> { + self.with_name_lock(name, || { + if let Some(handle) = self.remove_cached_handle(name)? { + lock_handle(&handle)?.stop()?; + control::mark_stopped(&self.db, name)?; + return Ok(()); + } + + control::stop_vm(&self.db, name) + }) + } + + /// Stop best-effort, remove from the registry and DB, and delete storage. + pub fn delete_machine(&self, name: &str) -> Result<()> { + self.with_name_lock(name, || { + if let Some(handle) = self.remove_cached_handle(name)? { + let _ = lock_handle(&handle)?.stop(); + } else { + let _ = control::stop_vm(&self.db, name); + } + + // Idempotent: deleting an already-deleted machine is a no-op success + // (the desired end state — gone — already holds). Lets SDK callers + // call delete() more than once without an error. + match control::delete_vm(&self.db, name) { + Ok(()) | Err(crate::Error::VmNotFound { .. }) => {} + Err(e) => return Err(e), + } + self.remove_name_lock(name)?; + Ok(()) + }) + } + + /// Execute a command directly in the VM. + pub fn exec( + &self, + name: &str, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + ) -> Result<(i32, Vec, Vec)> { + let handle = self.started_handle(name)?; + let mut handle = lock_handle(&handle)?; + handle.exec(command, env, workdir, timeout) + } + + /// Pull an OCI image and run a command inside it. + pub fn run( + &self, + name: &str, + image: &str, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + ) -> Result<(i32, Vec, Vec)> { + let handle = self.started_handle(name)?; + let mut handle = lock_handle(&handle)?; + handle.run(image, command, env, workdir, timeout) + } + + /// Pull an OCI image into the machine's storage. + pub fn pull_image(&self, name: &str, image: &str) -> Result { + let handle = self.started_handle(name)?; + let mut handle = lock_handle(&handle)?; + handle.pull_image(image) + } + + /// List cached OCI images in the machine's storage. + pub fn list_images(&self, name: &str) -> Result> { + let handle = self.started_handle(name)?; + let mut handle = lock_handle(&handle)?; + handle.list_images() + } + + /// Write a file into the machine. + pub fn write_file( + &self, + name: &str, + path: &str, + data: Vec, + mode: Option, + ) -> Result<()> { + let handle = self.started_handle(name)?; + let mut handle = lock_handle(&handle)?; + handle.write_file(path, &data, mode) + } + + /// Read a file from the machine. + pub fn read_file(&self, name: &str, path: &str) -> Result> { + let handle = self.started_handle(name)?; + let mut handle = lock_handle(&handle)?; + handle.read_file(path) + } + + /// The machine's image, if it is an image (container-workload) machine. + /// Streamed execs on such a machine must run inside its persistent container + /// overlay so their writes survive — matching non-streaming exec. + fn image_of(&self, name: &str) -> Option { + self.db.get_vm(name).ok().flatten().and_then(|r| r.image) + } + + /// Execute a command and collect streaming output events. + pub fn exec_streaming( + &self, + name: &str, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + ) -> Result> { + let mut events = Vec::new(); + self.exec_streaming_with(name, command, env, workdir, timeout, |e| events.push(e))?; + Ok(events) + } + + /// Execute a command, delivering streaming output events LIVE via the + /// callback as they arrive (no buffering). The live counterpart of + /// `exec_streaming`; SDKs bridge the callback to a native iterator. + pub fn exec_streaming_with( + &self, + name: &str, + command: Vec, + env: Vec<(String, String)>, + workdir: Option, + timeout: Option, + on_event: F, + ) -> Result<()> { + let image = self.image_of(name); + let handle = self.started_handle(name)?; + let mut handle = lock_handle(&handle)?; + match image { + // Image machine: stream inside the machine's persistent container + // overlay so streamed installs/writes persist across execs and + // restarts, like non-streaming exec. Keyed by machine name. + Some(image) => { + let config = RunConfig::new(image, command) + .with_env(env) + .with_workdir(workdir) + .with_timeout(timeout) + .with_persistent_overlay(Some(name.to_string())); + handle.run_streaming_with(config, on_event) + } + // Bare VM: stream directly against the guest. + None => handle.exec_streaming_with(command, env, workdir, timeout, on_event), + } + } + + /// Get the child PID if the machine is running. + pub fn pid(&self, name: &str) -> Option { + if let Ok(Some(handle)) = self.cached_handle(name) { + if let Ok(handle) = handle.lock() { + if let Some(pid) = handle.child_pid() { + return Some(pid); + } + } + } + + self.db + .get_vm(name) + .ok() + .flatten() + .and_then(|record| record.pid) + } + + /// Return whether the machine process is currently running. + pub fn is_running(&self, name: &str) -> bool { + if let Ok(Some(handle)) = self.cached_handle(name) { + if let Ok(handle) = handle.lock() { + return handle.is_process_alive(); + } + } + + self.db + .get_vm(name) + .ok() + .flatten() + .is_some_and(|record| record.actual_state() == RecordState::Running) + } + + /// Get the current machine state as a string. + pub fn state(&self, name: &str) -> String { + if let Ok(Some(handle)) = self.cached_handle(name) { + if let Ok(handle) = handle.lock() { + return handle.state(); + } + } + + match self.db.get_vm(name).ok().flatten() { + Some(record) if record.actual_state() == RecordState::Running => "running".into(), + Some(record) if record.actual_state() == RecordState::Failed => "failed".into(), + _ => "stopped".into(), + } + } + + fn started_handle(&self, name: &str) -> Result { + self.cached_handle(name)? + .ok_or_else(|| Error::InvalidState { + expected: "started".into(), + actual: "not started".into(), + }) + } + + fn cached_handle(&self, name: &str) -> Result> { + let registry = self + .registry + .read() + .map_err(|e| Error::agent("runtime registry", e.to_string()))?; + Ok(registry.get(name).cloned()) + } + + fn insert_handle(&self, name: &str, handle: VmHandle) -> Result<()> { + let mut registry = self + .registry + .write() + .map_err(|e| Error::agent("runtime registry", e.to_string()))?; + registry.insert(name.to_string(), Arc::new(Mutex::new(handle))); + Ok(()) + } + + fn remove_cached_handle(&self, name: &str) -> Result> { + let mut registry = self + .registry + .write() + .map_err(|e| Error::agent("runtime registry", e.to_string()))?; + Ok(registry.remove(name)) + } + + fn with_name_lock(&self, name: &str, op: F) -> Result + where + F: FnOnce() -> Result, + { + let lock = self.lock_for_name(name)?; + let _guard = lock_name(&lock)?; + op() + } + + fn lock_for_name(&self, name: &str) -> Result>> { + if let Some(lock) = self + .name_locks + .read() + .map_err(|e| Error::agent("runtime name locks", e.to_string()))? + .get(name) + .cloned() + { + return Ok(lock); + } + + let mut locks = self + .name_locks + .write() + .map_err(|e| Error::agent("runtime name locks", e.to_string()))?; + Ok(locks + .entry(name.to_string()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone()) + } + + fn remove_name_lock(&self, name: &str) -> Result<()> { + let mut locks = self + .name_locks + .write() + .map_err(|e| Error::agent("runtime name locks", e.to_string()))?; + locks.remove(name); + Ok(()) + } +} + +fn lock_name(lock: &Arc>) -> Result> { + lock.lock() + .map_err(|e| Error::agent("runtime name lock", e.to_string())) +} + +fn lock_handle(handle: &SharedHandle) -> Result> { + handle + .lock() + .map_err(|e| Error::agent("runtime handle", e.to_string())) +} + +/// Return the process-local embedded runtime singleton. +pub fn runtime() -> Result> { + static RUNTIME: OnceLock> = OnceLock::new(); + + if let Some(runtime) = RUNTIME.get() { + return Ok(runtime.clone()); + } + + let runtime = Arc::new(EmbeddedRuntime::new()?); + match RUNTIME.set(runtime.clone()) { + Ok(()) => Ok(runtime), + Err(_) => Ok(RUNTIME + .get() + .expect("runtime initialized by competing thread") + .clone()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_db() -> SmolvmDb { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "smolvm-embedded-runtime-{}-{}.db", + std::process::id(), + unique + )); + SmolvmDb::open_at(&path).unwrap() + } + + fn test_spec(name: &str, persistent: bool) -> MachineSpec { + MachineSpec { + name: name.to_string(), + mounts: Vec::new(), + ports: Vec::new(), + resources: crate::agent::VmResources::default(), + persistent, + } + } + + #[test] + fn remove_name_lock_removes_entry() { + let runtime = EmbeddedRuntime::with_db(test_db()); + runtime.lock_for_name("runtime-remove-lock").unwrap(); + + runtime.remove_name_lock("runtime-remove-lock").unwrap(); + + assert!(runtime + .name_locks + .read() + .expect("name locks should not be poisoned") + .is_empty()); + } + + #[test] + fn remove_name_lock_ignores_missing_entry() { + let runtime = EmbeddedRuntime::with_db(test_db()); + runtime.remove_name_lock("runtime-missing-lock").unwrap(); + + assert!(runtime + .name_locks + .read() + .expect("name locks should not be poisoned") + .is_empty()); + } + + #[test] + fn runtime_rejects_duplicate_create() { + let runtime = EmbeddedRuntime::with_db(test_db()); + runtime + .create_machine(test_spec("runtime-duplicate", false)) + .unwrap(); + + let err = runtime + .create_machine(test_spec("runtime-duplicate", false)) + .unwrap_err(); + assert!(matches!( + err, + Error::Agent { + kind: crate::error::AgentErrorKind::Conflict, + .. + } + )); + } + + #[test] + fn runtime_state_defaults_to_stopped_for_created_record() { + let runtime = EmbeddedRuntime::with_db(test_db()); + runtime + .create_machine(test_spec("runtime-state", true)) + .unwrap(); + + assert_eq!(runtime.state("runtime-state"), "stopped"); + assert!(!runtime.is_running("runtime-state")); + assert_eq!(runtime.pid("runtime-state"), None); + } + + #[test] + fn delete_machine_removes_name_lock_entry() { + let runtime = EmbeddedRuntime::with_db(test_db()); + runtime + .create_machine(test_spec("runtime-delete-lock", true)) + .unwrap(); + + assert!(runtime + .name_locks + .read() + .expect("name locks should not be poisoned") + .contains_key("runtime-delete-lock")); + + runtime.delete_machine("runtime-delete-lock").unwrap(); + + assert!(!runtime + .name_locks + .read() + .expect("name locks should not be poisoned") + .contains_key("runtime-delete-lock")); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..b3752b6 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,124 @@ +//! smolvm - OCI-native microVM runtime +//! +//! smolvm is a library and CLI for running microVMs with strong isolation +//! and OCI container compatibility. +//! +//! # Architecture +//! +//! ```text +//! ┌─────────────────────────────────────────────────┐ +//! │ smolvm CLI / Library │ +//! ├─────────────────────────────────────────────────┤ +//! │ VM abstraction (VmBackend, VmHandle) │ +//! ├─────────────────────────────────────────────────┤ +//! │ libkrun (Hypervisor.framework / KVM) │ +//! ├─────────────────────────────────────────────────┤ +//! │ libkrunfw (embedded Linux kernel) │ +//! └─────────────────────────────────────────────────┘ +//! ``` +//! +//! # Example +//! +//! ```no_run +//! use smolvm::{VmConfig, RootfsSource, default_backend}; +//! +//! // Create a VM configuration +//! let config = VmConfig::builder(RootfsSource::path("/path/to/rootfs")) +//! .memory(1024) // 1 GB +//! .cpus(2) +//! .command(vec!["/bin/sh".into()]) +//! .build(); +//! +//! // Get the default backend for this platform +//! let backend = default_backend().unwrap(); +//! +//! // Create and run the VM +//! let mut vm = backend.create(config).unwrap(); +//! let exit = vm.wait().unwrap(); +//! +//! println!("VM exited with: {}", exit); +//! ``` +//! +//! # Features +//! +//! - VM creation and lifecycle management +//! - Rootfs from path or OCI images +//! - Host directory mounts via virtiofs +//! - Network egress via NAT +//! - vsock control channel +//! - Persistent overlay disks +//! - `exec` into running VMs +//! +//! # Platform Support +//! +//! | Platform | Backend | Status | +//! |----------|---------|--------| +//! | macOS (Apple Silicon) | libkrun + Hypervisor.framework | ✅ | +//! | macOS (Intel) | libkrun + Hypervisor.framework | ✅ | +//! | Linux (arm64) | libkrun + KVM | ✅ | +//! | Linux (x86_64) | libkrun + KVM | ✅ | + +#![warn(missing_docs)] +#![warn(clippy::all)] + +pub mod agent; +pub mod api; +pub mod config; +pub mod cuda_host; +/// Canonical shared data models and constants used across adapters. +pub mod data; +pub mod db; +pub mod disk_utils; +pub mod dns_filter; +pub mod dns_filter_listener; +/// Language-neutral embedded runtime support shared by SDK adapters. +pub mod embedded; +pub mod log_rotation; +pub mod network; +pub mod platform; +pub mod process; +pub mod registry; +pub mod secrets; +pub mod settings; +pub mod smolfile; +pub mod storage; +pub mod systemd_scope; +pub mod util; +pub mod vm; + +/// Compatibility re-exports for smolvm error types. +/// +/// The canonical error model lives under [`crate::data::error`]. This module +/// remains as a stable facade for existing `crate::error` and `smolvm::error` +/// imports. +pub mod error { + pub use crate::data::error::{AgentErrorKind, Error, Result}; +} + +// ============================================================================ +// Default Command Constants +// ============================================================================ + +/// Default interactive command — spawns a shell. +pub const DEFAULT_SHELL_CMD: &str = "/bin/sh"; + +/// Default idle command — keeps a container alive without doing work. +pub const DEFAULT_IDLE_CMD: &[&str] = &["sleep", "infinity"]; + +// Re-export main types for convenience +pub use agent::{AgentClient, AgentManager}; +pub use api::ApiDoc; +pub use config::{RecordState, RestartConfig, RestartPolicy, SmolvmConfig, VmRecord}; +pub use data::resources::VmResources; +pub use data::storage::HostMount; +pub use db::SmolvmDb; +pub use error::{Error, Result}; +pub use process::ChildProcess; +pub use registry::{RegistryAuth, RegistryConfig}; +pub use settings::SmolSettings; +pub use vm::config::{NetworkPolicy, RootfsSource, Timeouts, VmConfig, VmId}; +pub use vm::state::{ExitReason, VmState}; +pub use vm::{default_backend, VmBackend, VmHandle}; + +/// Library version. +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/src/log_rotation.rs b/src/log_rotation.rs new file mode 100644 index 0000000..ee22d85 --- /dev/null +++ b/src/log_rotation.rs @@ -0,0 +1,320 @@ +//! Log rotation utilities for machine console logs. +//! +//! Provides automatic log rotation when log files exceed a size threshold. +//! Rotated logs follow the pattern: `filename.1`, `filename.2`, etc. + +use crate::data::consts::BYTES_PER_MIB; +use std::fs; +use std::io; +use std::path::Path; + +/// Maximum log file size before rotation (10 MB). +const MAX_LOG_SIZE: u64 = 10 * BYTES_PER_MIB; + +/// Maximum number of rotated log files to keep. +const MAX_LOG_FILES: usize = 3; + +/// Rotate a log file if it exceeds the size limit. +/// +/// If the log file is larger than `MAX_LOG_SIZE`, it will be rotated: +/// - Current log -> `log.1` +/// - `log.1` -> `log.2` +/// - `log.2` -> `log.3` +/// - `log.3` -> deleted +/// +/// Returns `Ok(true)` if rotation occurred, `Ok(false)` if no rotation needed. +pub fn rotate_if_needed(log_path: &Path) -> io::Result { + let metadata = match fs::metadata(log_path) { + Ok(m) => m, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(false), + Err(e) => return Err(e), + }; + + if metadata.len() < MAX_LOG_SIZE { + return Ok(false); + } + + rotate(log_path)?; + Ok(true) +} + +/// Force rotate a log file regardless of size. +/// +/// Rotates the log file following the same pattern as `rotate_if_needed`. +/// +/// The active (live) log is rotated with **copy-truncate** semantics rather +/// than a rename. The live console log is written by libkrun through an +/// intentionally-leaked append fd bound to the file's inode for the VM's whole +/// lifetime (see `KrunFunctions::console_output_to_file`). A `rename` would +/// leave that leaked fd pointing at the renamed inode (`.1`), so libkrun would +/// keep appending to `.1` — which then rotates onward and gets deleted, losing +/// all live output, while `tail agent-console.log` sees a frozen file. +/// +/// Instead we copy the live file's contents to `.1` and then truncate the +/// original in place. Because the leaked fd is `O_APPEND`, its next write lands +/// at offset 0 of the now-empty inode, so the writer stays valid and continues +/// into the same file. Already-rotated `.1`/`.2`/`.3` files are nobody's live +/// fd, so they are still shifted with plain renames. +/// +/// Tradeoff: like `logrotate`'s own `copytruncate`, any bytes written between +/// the copy and the truncate are lost. We minimise that window by truncating +/// immediately after the copy; under active writing a tiny loss is the accepted +/// copy-truncate semantics. +pub fn rotate(log_path: &Path) -> io::Result<()> { + let log_str = log_path.display().to_string(); + + // Delete the oldest rotated file if it exists + let oldest = format!("{}.{}", log_str, MAX_LOG_FILES); + if Path::new(&oldest).exists() { + fs::remove_file(&oldest)?; + } + + // Rotate existing files: .2 -> .3, .1 -> .2 (plain renames — nothing holds + // an open fd to these already-rotated files). + for i in (1..MAX_LOG_FILES).rev() { + let from = format!("{}.{}", log_str, i); + let to = format!("{}.{}", log_str, i + 1); + if Path::new(&from).exists() { + fs::rename(&from, &to)?; + } + } + + // Copy-truncate the live log to .1, preserving the leaked writer fd's inode. + let first_rotated = format!("{}.1", log_str); + fs::copy(log_path, &first_rotated)?; + truncate_in_place(log_path)?; + + Ok(()) +} + +/// Truncate a file to zero length in place, keeping its inode (and therefore any +/// already-open fds) valid. Uses `OpenOptions::write(true).truncate(true)` which +/// maps to `open(..., O_TRUNC)` — it does not create a new inode. +fn truncate_in_place(log_path: &Path) -> io::Result<()> { + fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(log_path)?; + Ok(()) +} + +/// Get the total size of all log files (current + rotated). +pub fn total_log_size(log_path: &Path) -> io::Result { + let mut total = 0u64; + let log_str = log_path.display().to_string(); + + // Current log + if let Ok(metadata) = fs::metadata(log_path) { + total += metadata.len(); + } + + // Rotated logs + for i in 1..=MAX_LOG_FILES { + let rotated = format!("{}.{}", log_str, i); + if let Ok(metadata) = fs::metadata(&rotated) { + total += metadata.len(); + } + } + + Ok(total) +} + +/// Clean up all log files (current + rotated). +pub fn cleanup_logs(log_path: &Path) -> io::Result<()> { + let log_str = log_path.display().to_string(); + + // Remove current log + if log_path.exists() { + fs::remove_file(log_path)?; + } + + // Remove rotated logs + for i in 1..=MAX_LOG_FILES { + let rotated = format!("{}.{}", log_str, i); + if Path::new(&rotated).exists() { + fs::remove_file(&rotated)?; + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::TempDir; + + fn create_test_log(dir: &TempDir, content_size: usize) -> std::path::PathBuf { + let log_path = dir.path().join("test.log"); + let mut file = fs::File::create(&log_path).unwrap(); + let content = vec![b'x'; content_size]; + file.write_all(&content).unwrap(); + log_path + } + + #[test] + fn test_no_rotation_needed() { + let dir = TempDir::new().unwrap(); + let log_path = create_test_log(&dir, 1000); // 1KB, below threshold + + assert!(!rotate_if_needed(&log_path).unwrap()); + assert!(log_path.exists()); + } + + #[test] + fn test_rotation_when_size_exceeded() { + let dir = TempDir::new().unwrap(); + // Create a log that's over the threshold + // Use smaller size for test to avoid memory issues + let log_path = dir.path().join("test.log"); + { + let mut file = fs::File::create(&log_path).unwrap(); + // Write enough to trigger rotation (we'll temporarily reduce threshold) + file.write_all(&vec![b'x'; 1000]).unwrap(); + } + + // Force rotate to test the rotation logic + rotate(&log_path).unwrap(); + + // Copy-truncate keeps the live file (now empty) and copies data to .1. + assert!(log_path.exists()); + assert_eq!(fs::metadata(&log_path).unwrap().len(), 0); + let rotated = dir.path().join("test.log.1"); + assert!(rotated.exists()); + } + + #[test] + fn test_rotation_chain() { + let dir = TempDir::new().unwrap(); + let log_path = dir.path().join("test.log"); + + // Create .1 and .2 files + fs::write(dir.path().join("test.log.1"), b"old1").unwrap(); + fs::write(dir.path().join("test.log.2"), b"old2").unwrap(); + fs::write(&log_path, b"current").unwrap(); + + rotate(&log_path).unwrap(); + + // Check rotation happened correctly. Copy-truncate leaves the live file + // present but empty; its data is copied into .1. + assert!(log_path.exists()); + assert_eq!(fs::metadata(&log_path).unwrap().len(), 0); + assert_eq!( + fs::read_to_string(dir.path().join("test.log.1")).unwrap(), + "current" + ); + assert_eq!( + fs::read_to_string(dir.path().join("test.log.2")).unwrap(), + "old1" + ); + assert_eq!( + fs::read_to_string(dir.path().join("test.log.3")).unwrap(), + "old2" + ); + } + + #[test] + fn test_oldest_file_deleted() { + let dir = TempDir::new().unwrap(); + let log_path = dir.path().join("test.log"); + + // Create all rotated files + fs::write(&log_path, b"current").unwrap(); + fs::write(dir.path().join("test.log.1"), b"old1").unwrap(); + fs::write(dir.path().join("test.log.2"), b"old2").unwrap(); + fs::write(dir.path().join("test.log.3"), b"old3").unwrap(); + + rotate(&log_path).unwrap(); + + // .3 should have been deleted, then recreated from .2 + assert_eq!( + fs::read_to_string(dir.path().join("test.log.3")).unwrap(), + "old2" + ); + } + + #[test] + fn test_total_log_size() { + let dir = TempDir::new().unwrap(); + let log_path = dir.path().join("test.log"); + + fs::write(&log_path, b"12345").unwrap(); // 5 bytes + fs::write(dir.path().join("test.log.1"), b"123").unwrap(); // 3 bytes + fs::write(dir.path().join("test.log.2"), b"12").unwrap(); // 2 bytes + + assert_eq!(total_log_size(&log_path).unwrap(), 10); + } + + #[test] + fn test_cleanup_logs() { + let dir = TempDir::new().unwrap(); + let log_path = dir.path().join("test.log"); + + fs::write(&log_path, b"current").unwrap(); + fs::write(dir.path().join("test.log.1"), b"old1").unwrap(); + fs::write(dir.path().join("test.log.2"), b"old2").unwrap(); + + cleanup_logs(&log_path).unwrap(); + + assert!(!log_path.exists()); + assert!(!dir.path().join("test.log.1").exists()); + assert!(!dir.path().join("test.log.2").exists()); + } + + // Regression test for C2: rotation must not rename the live file out from + // under a writer that holds a leaked append fd (as libkrun does for the VM + // console). Copy-truncate must keep that fd valid and pointing at the same + // inode, so pre-rotation data lands in .1 and post-rotation writes via the + // held fd land in the (now truncated) live file — no data lost. + #[test] + fn test_copytruncate_preserves_open_append_fd() { + use std::fs::OpenOptions; + + let dir = TempDir::new().unwrap(); + let log_path = dir.path().join("agent-console.log"); + + // Open an append fd and keep it open across the rotation, mirroring the + // leaked libkrun console fd. + let mut writer = OpenOptions::new() + .create(true) + .append(true) + .open(&log_path) + .unwrap(); + + writer.write_all(b"before-rotation\n").unwrap(); + writer.flush().unwrap(); + + rotate(&log_path).unwrap(); + + // The pre-rotation data was copied into .1. + assert_eq!( + fs::read_to_string(dir.path().join("agent-console.log.1")).unwrap(), + "before-rotation\n" + ); + // The live file still exists (same inode) and was truncated to empty. + assert!(log_path.exists()); + assert_eq!(fs::metadata(&log_path).unwrap().len(), 0); + + // The still-open append fd keeps writing to the same inode; O_APPEND + // means the next write lands at offset 0 of the truncated file. + writer.write_all(b"after-rotation\n").unwrap(); + writer.flush().unwrap(); + + // Post-rotation data is in the live file, not leaked into .1. + assert_eq!(fs::read_to_string(&log_path).unwrap(), "after-rotation\n"); + assert_eq!( + fs::read_to_string(dir.path().join("agent-console.log.1")).unwrap(), + "before-rotation\n" + ); + } + + #[test] + fn test_rotate_nonexistent_file() { + let dir = TempDir::new().unwrap(); + let log_path = dir.path().join("nonexistent.log"); + + // Should not error, just return false + assert!(!rotate_if_needed(&log_path).unwrap()); + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..a0e8eaa --- /dev/null +++ b/src/main.rs @@ -0,0 +1,151 @@ +//! smolvm CLI entry point. + +use clap::{Parser, Subcommand}; +use tracing_subscriber::EnvFilter; + +mod cli; + +/// smolvm - build and run portable, self-contained virtual machines +#[derive(Parser, Debug)] +#[command(name = "smolvm")] +#[command( + about = "Build and run portable, self-contained virtual machines", + after_help = "Agents: run `smolvm --help` for full documentation including CLI reference and Smolfile schema" +)] +#[command( + long_about = include_str!("../AGENTS.md") +)] +#[command(version)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand, Debug)] +enum Commands { + /// Manage machines (create, start, stop, exec) + #[command(subcommand, visible_alias = "vm")] + Machine(cli::machine::MachineCmd), + + /// Start the HTTP API server for programmatic control + #[command(subcommand)] + Serve(cli::serve::ServeCmd), + + /// Package and run self-contained VM executables + #[command(subcommand)] + Pack(cli::pack::PackCmd), + + /// Manage smolvm configuration (registries, defaults) + #[command(subcommand)] + Config(cli::config::ConfigCmd), + + /// Internal: boot a VM subprocess (not for direct use) + #[command(name = "_boot-vm", hide = true)] + BootVm { + /// Path to boot config JSON file + config: std::path::PathBuf, + }, + + /// Internal: clean up an ephemeral VM after its command exits (not for direct use) + #[command(name = "_cleanup-ephemeral", hide = true)] + CleanupEphemeral { + /// Ephemeral VM name (its data dir + DB record are both keyed by this) + vm_name: String, + /// VM process PID + pid: i32, + /// Process start time for PID-reuse verification (0 = unknown, skips strict check) + start_time: u64, + }, +} + +fn main() { + // Honor an explicit SMOLVM_DATA_DIR for EVERY command (so the CLI and serve + // agree on where smolvm state lives) before anything computes a path. The + // auto /var/lib/smolvm default is serve-only (applied in its run()). Done + // first, single-threaded, so the set_var is safe. + smolvm::process::apply_system_data_root(/* allow_auto */ false); + + // Auto-detect packed binary mode BEFORE parsing the normal CLI. + // If this executable has a `.smolmachine` sidecar, appended assets, + // or a Mach-O section with packed data, run as a packed binary instead. + // + // EXCEPTION: skip auto-detection when re-invoked as `_boot-vm`. On Windows + // (and any non-fork path) pack rehydrate boots the VM by re-spawning this + // same packed executable as `current_exe _boot-vm `. That child + // still carries the packed footer/sidecar, so without this guard it would + // re-trigger packed-mode detection and rehydrate again instead of booting + // the VM from the config it was handed. + let is_boot_vm = + std::env::args_os().nth(1).as_deref() == Some(std::ffi::OsStr::new("_boot-vm")); + if !is_boot_vm { + if let Some(mode) = smolvm_pack::detect_packed_mode() { + cli::pack_run::run_as_packed_binary(mode); + } + } + + let cli = Cli::parse(); + + // Initialize logging based on RUST_LOG or default to warn + init_logging(); + + tracing::debug!(version = smolvm::VERSION, "starting smolvm"); + + // Execute command + // Note: orphan cleanup is handled per-command (skipped for ephemeral `machine run`). + let result = match cli.command { + Commands::Machine(cmd) => cmd.run(), + Commands::Serve(cmd) => cmd.run(), + Commands::Pack(cmd) => cmd.run(), + Commands::Config(cmd) => cmd.run(), + Commands::BootVm { config } => cli::internal_boot::run(config), + Commands::CleanupEphemeral { + vm_name, + pid, + start_time, + } => { + cli::cleanup_ephemeral::run(&vm_name, pid, start_time); + Ok(()) + } + }; + + // Handle errors + if let Err(e) = result { + tracing::debug!(error = %e, "command failed"); + eprintln!("Error: {}", e); + std::process::exit(1); + } +} + +/// Initialize the tracing subscriber. +/// +/// JSON mode is enabled via `SMOLVM_LOG_FORMAT=json` env var or when +/// running as `smolvm serve --json-logs`. Default is human-readable. +fn init_logging() { + let json = std::env::var("SMOLVM_LOG_FORMAT") + .map(|v| v == "json") + .unwrap_or(false); + + // Skip EnvFilter::try_from_default_env() when RUST_LOG is not set — + // avoids parsing an env var that doesn't exist. + let filter = match std::env::var_os("RUST_LOG") { + Some(_) => { + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("smolvm=warn")) + } + None => EnvFilter::new("smolvm=warn"), + }; + + if json { + tracing_subscriber::fmt() + .json() + .with_writer(std::io::stderr) + .with_env_filter(filter) + .with_current_span(true) + .init(); + } else { + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter(filter) + .with_target(false) + .init(); + } +} diff --git a/src/network/backend.rs b/src/network/backend.rs new file mode 100644 index 0000000..2a43cf3 --- /dev/null +++ b/src/network/backend.rs @@ -0,0 +1,73 @@ +use clap::ValueEnum; + +/// Feature set advertised to libkrun for unixstream-backed virtio-net. +/// +/// The current smoltcp-backed MVP expects ordinary packets from the guest. +/// Leave checksum and segmentation offloads disabled until the host path +/// explicitly handles those packet shapes. +pub const COMPAT_NET_FEATURES: u32 = 0; +/// TSI feature bit that enables INET socket hijacking. +pub const TSI_FEATURE_HIJACK_INET: u32 = 1 << 0; + +/// Network backend override for machine launch. +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, + ValueEnum, + utoipa::ToSchema, +)] +#[serde(rename_all = "kebab-case")] +pub enum NetworkBackend { + /// Use libkrun TSI networking. + #[value(name = "tsi")] + Tsi, + /// Use virtio-net with the host-side smolvm network stack. + #[value(name = "virtio-net")] + VirtioNet, +} + +impl NetworkBackend { + /// Stable CLI/storage label for the backend. + pub const fn as_str(self) -> &'static str { + match self { + Self::Tsi => "tsi", + Self::VirtioNet => "virtio-net", + } + } + + /// Human-readable backend label for logs. + pub const fn log_label(self) -> &'static str { + match self { + Self::Tsi => "tsi", + Self::VirtioNet => "virtio-net", + } + } +} + +impl std::fmt::Display for NetworkBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +#[cfg(test)] +mod tests { + use super::NetworkBackend; + + #[test] + fn virtio_net_serializes_to_canonical_name() { + let value = serde_json::to_string(&NetworkBackend::VirtioNet).unwrap(); + assert_eq!(value, "\"virtio-net\""); + } + + #[test] + fn legacy_virtio_name_is_rejected() { + let value = serde_json::from_str::("\"virtio\""); + assert!(value.is_err()); + } +} diff --git a/src/network/launch.rs b/src/network/launch.rs new file mode 100644 index 0000000..3693a5b --- /dev/null +++ b/src/network/launch.rs @@ -0,0 +1,321 @@ +use crate::data::resources::VmResources; +use crate::network::backend::NetworkBackend; + +/// Effective backend selected for a launch. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EffectiveNetworkBackend { + /// No network device. + None, + /// TSI networking. + Tsi, + /// Virtio-net networking. + VirtioNet, +} + +/// Network launch decision for a VM. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct LaunchNetworkPlan { + /// Selected backend. + pub backend: EffectiveNetworkBackend, +} + +impl LaunchNetworkPlan { + /// Whether the launch should attach any network backend at all. + pub const fn has_network(self) -> bool { + !matches!(self.backend, EffectiveNetworkBackend::None) + } +} + +/// Compute the effective launch backend from user intent. +/// +/// virtio-net now enforces the full egress policy (CIDR + allow-host DNS +/// filtering) and serves inbound published ports, so an explicit virtio-net +/// request is always honored; nothing downgrades to TSI. +pub fn plan_launch_network( + resources: &VmResources, + dns_filter_hosts: Option<&[String]>, + port_count: usize, +) -> LaunchNetworkPlan { + let has_ports = port_count > 0; + let has_cidr_policy = resources + .allowed_cidrs + .as_ref() + .is_some_and(|cidrs| !cidrs.is_empty()); + let has_dns_filter = dns_filter_hosts.is_some_and(|hosts| !hosts.is_empty()); + let wants_network = resources.network || has_ports || has_cidr_policy || has_dns_filter; + + if !wants_network { + return LaunchNetworkPlan { + backend: EffectiveNetworkBackend::None, + }; + } + + // Published ports need the inbound path, which only virtio-net provides. + // When the caller didn't pick a backend explicitly, default to virtio-net + // IFF there are ports (TSI otherwise — it's lighter for outbound-only VMs). + // + // Fleet/multi-tenant mode (SMOLVM_PUBLISH_ADDR set) additionally routes ALL + // machines through virtio-net, so the egress hard-floor (the smolvm-network + // `EgressPolicy` that denies metadata/internal/loopback) applies to no-ports + // machines too. TSI's egress filter lives in libkrun and may not carry the + // floor, so a default outbound-only tenant VM must not silently fall to TSI + // on a fleet node. Outside fleet mode, no-ports VMs keep the lighter default. + // + // An explicit egress policy (--allow-cidr/--allow-host/--outbound-localhost-only) + // must also use virtio-net: its egress allow-list is enforced reliably by the + // host-side smolvm-network stack, whereas TSI's in-libkrun filter does not + // enforce it (verified: a non-allowed IP stays reachable under TSI). Falling + // to TSI here would make the egress flags a silent no-op — a security hole — + // so a policy forces virtio-net unless the caller explicitly picked a backend. + let fleet_mode = std::env::var_os("SMOLVM_PUBLISH_ADDR").is_some(); + let backend = resources.network_backend.unwrap_or( + if has_ports || fleet_mode || has_cidr_policy || has_dns_filter { + NetworkBackend::VirtioNet + } else { + NetworkBackend::Tsi + }, + ); + // virtio-net works on Windows too: the host-side smolvm-network gateway is + // cross-platform and libkrun's net device bridges over an AF_UNIX path + // (Windows 10 1809+ has native AF_UNIX). See launcher's VirtioNet arm. + match backend { + NetworkBackend::Tsi => LaunchNetworkPlan { + backend: EffectiveNetworkBackend::Tsi, + }, + NetworkBackend::VirtioNet => LaunchNetworkPlan { + backend: EffectiveNetworkBackend::VirtioNet, + }, + } +} + +/// Validate the requested networking against what each backend can do. +/// +/// - Published ports need virtio-net: TSI is outbound-only, so a port request on +/// TSI (the default) would silently never accept connections — reject instead. +/// - `--net-backend virtio-net` with no networking intent at all is rejected. +pub fn validate_requested_network_backend( + resources: &VmResources, + dns_filter_hosts: Option<&[String]>, + port_count: usize, +) -> crate::Result<()> { + // An egress policy is only enforced under virtio-net; TSI would silently let + // it through. + let has_egress_policy = resources + .allowed_cidrs + .as_ref() + .is_some_and(|c| !c.is_empty()) + || dns_filter_hosts.is_some_and(|h| !h.is_empty()); + + // Mirror plan_launch_network's default: unset backend + (ports OR egress + // policy) ⇒ virtio-net. So only an EXPLICIT `--net-backend tsi` alongside + // ports/egress is a misconfig. + let backend = resources + .network_backend + .unwrap_or(if port_count > 0 || has_egress_policy { + NetworkBackend::VirtioNet + } else { + NetworkBackend::Tsi + }); + + // Published ports require the inbound path that only virtio-net has. With + // the default above this only fires when the caller EXPLICITLY forced TSI. + if port_count > 0 && backend != NetworkBackend::VirtioNet { + return Err(crate::Error::config( + "ports", + "published ports require the virtio-net backend (TSI is outbound-only); \ + remove --net-backend tsi or set it to virtio-net", + )); + } + + // Same for an egress policy — fires only on EXPLICIT TSI + policy. + if has_egress_policy && backend != NetworkBackend::VirtioNet { + return Err(crate::Error::config( + "egress", + "egress policy (--allow-cidr/--allow-host/--outbound-localhost-only) requires the \ + virtio-net backend; TSI does not enforce it. Remove --net-backend tsi or set it to virtio-net", + )); + } + + if resources.network_backend != Some(NetworkBackend::VirtioNet) { + return Ok(()); + } + + let has_cidr_policy = resources + .allowed_cidrs + .as_ref() + .is_some_and(|cidrs| !cidrs.is_empty()); + let has_dns_filter = dns_filter_hosts.is_some_and(|hosts| !hosts.is_empty()); + let wants_network = resources.network || port_count > 0 || has_cidr_policy || has_dns_filter; + + if !wants_network { + return Err(crate::Error::config( + "--net-backend", + "--net-backend virtio-net requires --net", + )); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn resources() -> VmResources { + VmResources::default() + } + + #[test] + fn test_no_network_plan() { + let plan = plan_launch_network(&resources(), None, 0); + assert_eq!(plan.backend, EffectiveNetworkBackend::None); + } + + #[test] + fn test_default_network_uses_tsi() { + let mut resources = resources(); + resources.network = true; + let plan = plan_launch_network(&resources, None, 0); + assert_eq!(plan.backend, EffectiveNetworkBackend::Tsi); + } + + #[test] + fn test_plain_virtio_selects_virtio_backend() { + let mut resources = resources(); + resources.network = true; + resources.network_backend = Some(NetworkBackend::VirtioNet); + let plan = plan_launch_network(&resources, None, 0); + assert_eq!(plan.backend, EffectiveNetworkBackend::VirtioNet); + } + + #[test] + fn test_ports_work_with_virtio() { + let mut resources = resources(); + resources.network = true; + resources.network_backend = Some(NetworkBackend::VirtioNet); + let plan = plan_launch_network(&resources, None, 1); + assert_eq!(plan.backend, EffectiveNetworkBackend::VirtioNet); + } + + #[test] + fn test_cidr_policy_stays_virtio() { + // CIDR egress policy is enforced by the virtio-net gateway, so it no + // longer downgrades an explicit virtio-net request. + let mut resources = resources(); + resources.network = true; + resources.network_backend = Some(NetworkBackend::VirtioNet); + resources.allowed_cidrs = Some(vec!["1.1.1.1/32".into()]); + let plan = plan_launch_network(&resources, None, 0); + assert_eq!(plan.backend, EffectiveNetworkBackend::VirtioNet); + } + + #[test] + fn test_dns_filter_stays_virtio() { + // allow-host filtering is now enforced by the virtio-net gateway, so an + // explicit virtio-net request is honored rather than downgraded. + let mut resources = resources(); + resources.network = true; + resources.network_backend = Some(NetworkBackend::VirtioNet); + let hosts = ["example.com".to_string()]; + let plan = plan_launch_network(&resources, Some(&hosts), 0); + assert_eq!(plan.backend, EffectiveNetworkBackend::VirtioNet); + } + + #[test] + fn test_cidr_policy_default_selects_virtio() { + // No explicit backend + a CIDR egress policy must default to virtio-net + // (TSI doesn't enforce egress, so defaulting there is a silent no-op). + let mut resources = resources(); + resources.network = true; + resources.allowed_cidrs = Some(vec!["1.1.1.1/32".into()]); + let plan = plan_launch_network(&resources, None, 0); + assert_eq!(plan.backend, EffectiveNetworkBackend::VirtioNet); + } + + #[test] + fn test_dns_filter_default_selects_virtio() { + let mut resources = resources(); + resources.network = true; + let hosts = ["example.com".to_string()]; + let plan = plan_launch_network(&resources, Some(&hosts), 0); + assert_eq!(plan.backend, EffectiveNetworkBackend::VirtioNet); + } + + #[test] + fn test_validate_egress_explicit_tsi_rejected() { + let mut resources = resources(); + resources.network = true; + resources.network_backend = Some(NetworkBackend::Tsi); + resources.allowed_cidrs = Some(vec!["1.1.1.1/32".into()]); + assert!(validate_requested_network_backend(&resources, None, 0).is_err()); + } + + #[test] + fn test_validate_plain_virtio_allowed() { + let mut resources = resources(); + resources.network = true; + resources.network_backend = Some(NetworkBackend::VirtioNet); + validate_requested_network_backend(&resources, None, 0).unwrap(); + } + + #[test] + fn test_validate_ports_allowed_for_virtio() { + let mut resources = resources(); + resources.network = true; + resources.network_backend = Some(NetworkBackend::VirtioNet); + validate_requested_network_backend(&resources, None, 1).unwrap(); + } + + #[test] + fn test_ports_default_to_virtio() { + // Ports with NO explicit backend now auto-select virtio-net (the inbound + // path) — both in the plan and in validation, so it "just works". + let mut resources = resources(); + resources.network = true; + let plan = plan_launch_network(&resources, None, 1); + assert_eq!(plan.backend, EffectiveNetworkBackend::VirtioNet); + validate_requested_network_backend(&resources, None, 1).unwrap(); + } + + #[test] + fn test_validate_ports_explicit_tsi_rejected() { + // Only an EXPLICIT TSI choice alongside ports is a misconfig (TSI has no + // inbound path); the auto-default case above is allowed. + let mut resources = resources(); + resources.network = true; + resources.network_backend = Some(NetworkBackend::Tsi); + let err = validate_requested_network_backend(&resources, None, 1).unwrap_err(); + assert!(err.to_string().contains("require the virtio-net backend")); + } + + #[test] + fn test_no_ports_defaults_to_tsi() { + // Outbound-only VMs keep the lighter TSI default — no virtio overhead. + let mut resources = resources(); + resources.network = true; + assert_eq!( + plan_launch_network(&resources, None, 0).backend, + EffectiveNetworkBackend::Tsi + ); + } + + #[test] + fn test_validate_cidr_allowed_for_virtio() { + // CIDR egress policy is now honored on virtio-net, so validation passes. + let mut resources = resources(); + resources.network = true; + resources.network_backend = Some(NetworkBackend::VirtioNet); + resources.allowed_cidrs = Some(vec!["1.1.1.1/32".into()]); + validate_requested_network_backend(&resources, None, 0).unwrap(); + } + + #[test] + fn test_validate_dns_filter_allowed_for_virtio() { + // allow-host is now honored on virtio-net, so validation passes. + let mut resources = resources(); + resources.network = true; + resources.network_backend = Some(NetworkBackend::VirtioNet); + let hosts = ["example.com".to_string()]; + validate_requested_network_backend(&resources, Some(&hosts), 0).unwrap(); + } +} diff --git a/src/network/mod.rs b/src/network/mod.rs new file mode 100644 index 0000000..2b6c576 --- /dev/null +++ b/src/network/mod.rs @@ -0,0 +1,14 @@ +//! Network configuration and backend selection. + +/// Backend selection and serialization helpers. +pub mod backend; +/// Launch-time backend planning and request validation rules. +pub mod launch; +pub mod policy; + +pub use backend::NetworkBackend; +pub use launch::{ + plan_launch_network, validate_requested_network_backend, EffectiveNetworkBackend, + LaunchNetworkPlan, +}; +pub use policy::get_dns_server; diff --git a/src/network/policy.rs b/src/network/policy.rs new file mode 100644 index 0000000..5252acc --- /dev/null +++ b/src/network/policy.rs @@ -0,0 +1,38 @@ +//! Network policy helpers. + +use crate::data::network; +use crate::vm::config::NetworkPolicy; +use std::net::IpAddr; + +/// Get the DNS server for a network policy. +pub fn get_dns_server(policy: &NetworkPolicy) -> Option { + match policy { + NetworkPolicy::None => None, + NetworkPolicy::Egress { dns, .. } => Some(dns.unwrap_or_else(network::default_dns_addr)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_dns_server() { + assert!(get_dns_server(&NetworkPolicy::None).is_none()); + + let dns = get_dns_server(&NetworkPolicy::Egress { + dns: None, + allowed_cidrs: None, + }) + .unwrap(); + assert_eq!(dns, crate::data::network::default_dns_addr()); + + let custom: IpAddr = "8.8.8.8".parse().unwrap(); + let dns = get_dns_server(&NetworkPolicy::Egress { + dns: Some(custom), + allowed_cidrs: None, + }) + .unwrap(); + assert_eq!(dns.to_string(), "8.8.8.8"); + } +} diff --git a/src/platform/linux.rs b/src/platform/linux.rs new file mode 100644 index 0000000..ee23bcf --- /dev/null +++ b/src/platform/linux.rs @@ -0,0 +1,221 @@ +//! Linux-specific platform implementations. +//! +//! This module provides Linux implementations for VM execution. +//! On Linux, virtiofs devices are auto-mounted by the kernel, so no +//! wrapper script is needed. + +use crate::error::{Error, Result}; +use crate::platform::traits::{RosettaSupport, VmExecutor}; +use std::ffi::CString; +use std::fs; +use std::path::Path; + +/// Check if KVM is available and accessible on this system. +/// +/// This checks: +/// 1. `/dev/kvm` exists (KVM module is loaded) +/// 2. Current user has read/write access to `/dev/kvm` +/// +/// # Errors +/// +/// Returns `Error::KvmUnavailable` if KVM module is not loaded. +/// Returns `Error::KvmPermission` if the user lacks access to `/dev/kvm`. +pub fn check_kvm_available() -> Result<()> { + use std::os::unix::fs::MetadataExt; + + let kvm_path = Path::new("/dev/kvm"); + + // Check if /dev/kvm exists + if !kvm_path.exists() { + return Err(Error::KvmUnavailable( + "KVM not available. Ensure KVM kernel module is loaded.\n\ + Try: sudo modprobe kvm && sudo modprobe kvm_intel # (or kvm_amd)" + .to_string(), + )); + } + + // Check read/write access by trying to open the file + match fs::OpenOptions::new().read(true).write(true).open(kvm_path) { + Ok(_) => { + tracing::debug!("KVM access verified"); + } + Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => { + // Get the group that owns /dev/kvm for the error message + let group_hint = match fs::metadata(kvm_path) { + Ok(meta) => { + let _gid = meta.gid(); + // Common default on most Linux distros + "kvm".to_string() + } + Err(_) => "kvm".to_string(), + }; + + return Err(Error::KvmPermission(format!( + "Cannot access /dev/kvm (permission denied).\n\ + Add your user to the '{}' group:\n\ + \n\ + sudo usermod -aG {} $USER\n\ + \n\ + Then log out and log back in for the change to take effect.", + group_hint, group_hint + ))); + } + Err(e) => { + return Err(Error::KvmUnavailable(format!( + "Failed to access /dev/kvm: {}", + e + ))); + } + } + + Ok(()) +} + +/// Linux VM executor implementation. +/// +/// On Linux, virtiofs devices are automatically mounted by the kernel, +/// so this executor doesn't need to wrap commands with a mount script. +pub struct LinuxExecutor; + +impl VmExecutor for LinuxExecutor { + fn requires_mount_wrapper(&self) -> bool { + false + } + + fn build_exec_command( + &self, + command: &Option>, + _mounts: &[(String, String)], // Ignored on Linux - kernel handles virtiofs + _rootfs: &Path, + _rosetta: bool, // Ignored on Linux - Rosetta is macOS-only + ) -> Result<(CString, Vec<*const libc::c_char>, Vec)> { + // Linux doesn't need mount wrapper; execute command directly + let default_cmd = vec!["/bin/sh".to_string()]; + let cmd = command.as_ref().unwrap_or(&default_cmd); + + if cmd.is_empty() { + return Err(Error::vm_creation("command cannot be empty")); + } + + let exec_path = CString::new(cmd[0].as_str()) + .map_err(|_| Error::vm_creation("invalid command path"))?; + + // Skip argv[0] - libkrun/init.krun handles it via KRUN_INIT + let cstrings: Vec = cmd + .iter() + .skip(1) + .map(|s| CString::new(s.as_str())) + .collect::, _>>() + .map_err(|_| Error::vm_creation("invalid command argument"))?; + + let mut argv: Vec<*const libc::c_char> = cstrings.iter().map(|s| s.as_ptr()).collect(); + argv.push(std::ptr::null()); + + Ok((exec_path, argv, cstrings)) + } + + fn tool_search_paths(&self) -> &'static [&'static str] { + &["/sbin", "/usr/sbin", "/usr/local/sbin"] + } + + fn dylib_extension(&self) -> &'static str { + "so" + } + + fn library_search_paths(&self) -> &'static [&'static str] { + &[ + "/usr/lib", + "/usr/local/lib", + "/usr/lib64", + "/usr/local/lib64", + "/usr/lib/x86_64-linux-gnu", + "/usr/lib/aarch64-linux-gnu", + ] + } +} + +/// Linux Rosetta support (stub - always unavailable). +/// +/// Rosetta 2 is a macOS-only feature, so this implementation +/// always returns false for availability checks. +pub struct LinuxRosetta; + +impl RosettaSupport for LinuxRosetta { + fn is_available(&self) -> bool { + false + } + + fn runtime_path(&self) -> Option<&'static str> { + None + } +} + +/// Get the Rosetta support instance for Linux. +pub fn rosetta_support() -> LinuxRosetta { + LinuxRosetta +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_linux_executor_no_mount_wrapper() { + let executor = LinuxExecutor; + assert!(!executor.requires_mount_wrapper()); + } + + #[test] + fn test_linux_dylib_extension() { + let executor = LinuxExecutor; + assert_eq!(executor.dylib_extension(), "so"); + } + + #[test] + fn test_linux_rosetta_unavailable() { + let rosetta = LinuxRosetta; + assert!(!rosetta.is_available()); + assert!(rosetta.runtime_path().is_none()); + } + + #[test] + fn test_rosetta_needs_rosetta() { + let rosetta = LinuxRosetta; + // The needs_rosetta logic is in the trait default impl + assert!(rosetta.needs_rosetta("linux/amd64")); + assert!(!rosetta.needs_rosetta("linux/arm64")); + } + + #[test] + fn test_check_kvm_available_does_not_panic() { + // This test just ensures the function doesn't panic + // The actual result depends on the system configuration + let _ = super::check_kvm_available(); + } + + #[test] + fn test_linux_ignores_mounts_in_exec_command() { + let executor = LinuxExecutor; + let tmp = TempDir::new().unwrap(); + let cmd = Some(vec!["/bin/echo".to_string(), "hello".to_string()]); + // Linux should ignore mounts - kernel handles virtiofs automatically + let mounts = vec![("smolvm0".to_string(), "/data".to_string())]; + + let (exec_path, _argv, cstrings) = executor + .build_exec_command(&cmd, &mounts, tmp.path(), false) + .unwrap(); + + // Should return direct command, ignoring mounts + assert_eq!(exec_path.to_str().unwrap(), "/bin/echo"); + assert_eq!(cstrings.len(), 1); + assert_eq!(cstrings[0].to_str().unwrap(), "hello"); + + // No mount script should be created + let script_path = tmp.path().join("tmp/smolvm-mount.sh"); + assert!( + !script_path.exists(), + "Linux should not create mount script" + ); + } +} diff --git a/src/platform/macos.rs b/src/platform/macos.rs new file mode 100644 index 0000000..8a92995 --- /dev/null +++ b/src/platform/macos.rs @@ -0,0 +1,431 @@ +//! macOS-specific platform implementations. +//! +//! This module provides macOS implementations for VM execution and Rosetta support. +//! The key difference from Linux is that macOS requires explicit virtiofs mounting +//! via a wrapper script, as the kernel doesn't auto-mount virtiofs devices. + +use crate::error::{Error, Result}; +use crate::platform::traits::{RosettaSupport, VmExecutor}; +use std::ffi::CString; +use std::fs::{self, File}; +use std::io::Write; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; + +/// macOS VM executor implementation. +/// +/// On macOS, virtiofs devices need to be explicitly mounted by the guest. +/// This executor wraps user commands with a mount script when virtiofs +/// mounts are present. +pub struct MacOsExecutor; + +impl VmExecutor for MacOsExecutor { + fn requires_mount_wrapper(&self) -> bool { + true + } + + fn build_exec_command( + &self, + command: &Option>, + mounts: &[(String, String)], + rootfs: &Path, + rosetta: bool, + ) -> Result<(CString, Vec<*const libc::c_char>, Vec)> { + // Need mount wrapper if we have mounts OR if rosetta is enabled + if mounts.is_empty() && !rosetta { + return build_exec_args_direct(command); + } + + // Write mount script and use it as the command + let script_path = write_mount_script(rootfs, mounts, rosetta)?; + + let default_cmd = vec!["/bin/sh".to_string()]; + let user_cmd = command.as_ref().unwrap_or(&default_cmd); + + // exec_path is the mount script + let exec_path = CString::new(script_path.as_str()) + .map_err(|_| Error::vm_creation("invalid script path"))?; + + // argv is the user's command and arguments (passed to the script via $@) + // Note: For the mount wrapper, we pass ALL args including argv[0] + // because the script uses exec "$@" which expects the full command + let cstrings: Vec = user_cmd + .iter() + .map(|s| CString::new(s.as_str())) + .collect::, _>>() + .map_err(|_| Error::vm_creation("invalid command argument"))?; + + let mut argv: Vec<*const libc::c_char> = cstrings.iter().map(|s| s.as_ptr()).collect(); + argv.push(std::ptr::null()); + + tracing::debug!("using mount wrapper with {} virtiofs mounts", mounts.len()); + Ok((exec_path, argv, cstrings)) + } + + fn tool_search_paths(&self) -> &'static [&'static str] { + &[ + "/opt/homebrew/opt/e2fsprogs/sbin", + "/usr/local/opt/e2fsprogs/sbin", + "/opt/homebrew/sbin", + "/usr/local/sbin", + "/sbin", + "/usr/sbin", + ] + } + + fn dylib_extension(&self) -> &'static str { + "dylib" + } + + fn library_search_paths(&self) -> &'static [&'static str] { + &[ + "/opt/homebrew/lib", + "/usr/local/lib", + "/opt/homebrew/opt/libkrun/lib", + "/usr/local/opt/libkrun/lib", + ] + } +} + +/// macOS Rosetta 2 support implementation. +/// +/// Rosetta 2 allows running x86_64 binaries on Apple Silicon Macs. +/// This is only available on ARM64 macOS systems with Rosetta installed. +pub struct MacOsRosetta; + +/// Directory holding the Linux Rosetta runtime on macOS. This is the virtiofs +/// mount *source*: the guest mounts it at [`ROSETTA_GUEST_PATH`] so that the +/// translator appears at `/rosetta`. It is a subdirectory of +/// `/Library/Apple/usr/libexec/oah` — NOT that parent directory, whose `rosetta` +/// entry does not exist (the parent holds `libRosettaRuntime` + `RosettaLinux/`). +const ROSETTA_RUNTIME_PATH: &str = "/Library/Apple/usr/libexec/oah/RosettaLinux"; + +impl RosettaSupport for MacOsRosetta { + #[cfg(target_arch = "aarch64")] + fn is_available(&self) -> bool { + // Require the Linux translator itself, not just the runtime dir: it is + // what the guest's ptrace wrapper execs, and its absence is exactly the + // failure that makes an attached mount useless. + Path::new(ROSETTA_RUNTIME_PATH).join("rosetta").exists() + } + + #[cfg(not(target_arch = "aarch64"))] + fn is_available(&self) -> bool { + false // Rosetta only available on ARM64 + } + + #[cfg(target_arch = "aarch64")] + fn runtime_path(&self) -> Option<&'static str> { + if self.is_available() { + Some(ROSETTA_RUNTIME_PATH) + } else { + None + } + } + + #[cfg(not(target_arch = "aarch64"))] + fn runtime_path(&self) -> Option<&'static str> { + None + } +} + +/// Get the Rosetta support instance for macOS. +pub fn rosetta_support() -> MacOsRosetta { + MacOsRosetta +} + +/// Write a mount helper script to the rootfs. +/// +/// This script mounts all virtiofs volumes before executing the user's command. +/// It's written to /tmp inside the rootfs to avoid mutating the container image. +/// If rosetta is enabled, it also mounts the Rosetta runtime and registers binfmt_misc. +fn write_mount_script(rootfs: &Path, mounts: &[(String, String)], rosetta: bool) -> Result { + let tmp_dir = rootfs.join("tmp"); + if !tmp_dir.exists() { + fs::create_dir_all(&tmp_dir) + .map_err(|e| Error::vm_creation(format!("failed to create /tmp in rootfs: {}", e)))?; + } + + let host_path = tmp_dir.join("smolvm-mount.sh"); + let guest_path = "/tmp/smolvm-mount.sh"; + + let mut file = File::create(&host_path) + .map_err(|e| Error::vm_creation(format!("failed to create mount script: {}", e)))?; + + // Helper to write lines with proper error handling + let write_line = |file: &mut File, line: &str| -> Result<()> { + writeln!(file, "{}", line) + .map_err(|e| Error::vm_creation(format!("failed to write mount script: {}", e))) + }; + + write_line(&mut file, "#!/bin/sh")?; + write_line(&mut file, "set -e")?; + + // Mount each virtiofs volume. + // Both tag and guest_mount are single-quoted to prevent shell injection. + // Any embedded single quotes are escaped as `'\''` (end quote, literal + // quote via backslash, restart quote). + for (tag, guest_mount) in mounts { + write_line( + &mut file, + &format!("mkdir -p '{}'", shell_escape(guest_mount)), + )?; + write_line( + &mut file, + &format!( + "mount -t virtiofs '{}' '{}'", + shell_escape(tag), + shell_escape(guest_mount) + ), + )?; + } + + // If Rosetta is enabled, mount the runtime and register binfmt_misc + if rosetta { + write_line(&mut file, "# Mount Rosetta runtime")?; + write_line( + &mut file, + &format!( + "mkdir -p '{}'", + shell_escape(crate::vm::rosetta::ROSETTA_GUEST_PATH) + ), + )?; + write_line( + &mut file, + &format!( + "mount -t virtiofs '{}' '{}'", + shell_escape(crate::vm::rosetta::ROSETTA_TAG), + shell_escape(crate::vm::rosetta::ROSETTA_GUEST_PATH) + ), + )?; + + // Register Rosetta with binfmt_misc for x86_64 ELF binaries + write_line(&mut file, "# Register Rosetta with binfmt_misc")?; + write_line(&mut file, crate::vm::rosetta::BINFMT_REGISTER_CMD)?; + } + + // Clean up the mount script after execution + write_line(&mut file, "rm -f \"$0\"")?; + + // Execute the user's command + write_line(&mut file, "exec \"$@\"")?; + + // Make executable + let perms = fs::Permissions::from_mode(0o755); + fs::set_permissions(&host_path, perms).map_err(|e| { + Error::vm_creation(format!("failed to set mount script permissions: {}", e)) + })?; + + tracing::debug!( + "wrote mount script to {:?} (rosetta={})", + host_path, + rosetta + ); + Ok(guest_path.to_string()) +} + +/// Escape a string for safe inclusion inside single quotes in a shell script. +/// +/// Single-quoting in POSIX shell prevents all interpretation except for the +/// single-quote character itself. To include a literal `'`, we end the +/// current quoted segment, insert an escaped quote (`\'`), and restart +/// quoting: `'foo'\''bar'` produces the literal string `foo'bar`. +fn shell_escape(s: &str) -> String { + s.replace('\'', "'\\''") +} + +/// Build exec arguments without mount wrapper (for no-mounts case). +/// +/// Note: libkrun expects argv to NOT include argv[0] - only arguments. +/// The command path is passed separately via exec_path/KRUN_INIT. +fn build_exec_args_direct( + command: &Option>, +) -> Result<(CString, Vec<*const libc::c_char>, Vec)> { + let default_cmd = vec!["/bin/sh".to_string()]; + let cmd = command.as_ref().unwrap_or(&default_cmd); + + if cmd.is_empty() { + return Err(Error::vm_creation("command cannot be empty")); + } + + let exec_path = + CString::new(cmd[0].as_str()).map_err(|_| Error::vm_creation("invalid command path"))?; + + // Skip argv[0] - libkrun/init.krun handles it via KRUN_INIT + let cstrings: Vec = cmd + .iter() + .skip(1) + .map(|s| CString::new(s.as_str())) + .collect::, _>>() + .map_err(|_| Error::vm_creation("invalid command argument"))?; + + let mut argv: Vec<*const libc::c_char> = cstrings.iter().map(|s| s.as_ptr()).collect(); + argv.push(std::ptr::null()); + + Ok((exec_path, argv, cstrings)) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_macos_executor_requires_mount_wrapper() { + let executor = MacOsExecutor; + assert!(executor.requires_mount_wrapper()); + } + + #[test] + fn test_macos_dylib_extension() { + let executor = MacOsExecutor; + assert_eq!(executor.dylib_extension(), "dylib"); + } + + #[test] + fn test_rosetta_needs_rosetta() { + let rosetta = MacOsRosetta; + assert!(rosetta.needs_rosetta("linux/amd64")); + assert!(rosetta.needs_rosetta("linux/x86_64")); + assert!(!rosetta.needs_rosetta("linux/arm64")); + } + + #[test] + fn test_build_exec_no_mounts_returns_direct_command() { + let executor = MacOsExecutor; + let tmp = TempDir::new().unwrap(); + let cmd = Some(vec!["/bin/echo".to_string(), "hello".to_string()]); + + let (exec_path, argv, cstrings) = executor + .build_exec_command(&cmd, &[], tmp.path(), false) + .unwrap(); + + // With no mounts, should return direct command + assert_eq!(exec_path.to_str().unwrap(), "/bin/echo"); + // argv should have 1 arg + null terminator (argv[0] skipped per libkrun convention) + assert_eq!(cstrings.len(), 1); + assert_eq!(cstrings[0].to_str().unwrap(), "hello"); + assert_eq!(argv.len(), 2); // ["hello", null] + } + + #[test] + fn test_build_exec_with_mounts_creates_script() { + let executor = MacOsExecutor; + let tmp = TempDir::new().unwrap(); + let cmd = Some(vec!["/bin/cat".to_string(), "/data/file.txt".to_string()]); + let mounts = vec![("smolvm0".to_string(), "/data".to_string())]; + + let (exec_path, _argv, _cstrings) = executor + .build_exec_command(&cmd, &mounts, tmp.path(), false) + .unwrap(); + + // With mounts, should return mount script path + assert_eq!(exec_path.to_str().unwrap(), "/tmp/smolvm-mount.sh"); + + // Verify script was created with correct content + let script_path = tmp.path().join("tmp/smolvm-mount.sh"); + assert!(script_path.exists(), "mount script should be created"); + + let content = std::fs::read_to_string(&script_path).unwrap(); + assert!(content.contains("#!/bin/sh"), "script should have shebang"); + assert!( + content.contains("mkdir -p '/data'"), + "script should create mount point" + ); + assert!( + content.contains("mount -t virtiofs 'smolvm0' '/data'"), + "script should mount virtiofs" + ); + assert!( + content.contains("exec \"$@\""), + "script should exec user command" + ); + } + + #[test] + fn test_build_exec_default_command() { + let executor = MacOsExecutor; + let tmp = TempDir::new().unwrap(); + + let (exec_path, _argv, _cstrings) = executor + .build_exec_command(&None, &[], tmp.path(), false) + .unwrap(); + + // Default command should be /bin/sh + assert_eq!(exec_path.to_str().unwrap(), "/bin/sh"); + } + + #[test] + fn test_build_exec_with_rosetta_creates_script() { + let executor = MacOsExecutor; + let tmp = TempDir::new().unwrap(); + let cmd = Some(vec!["/bin/sh".to_string()]); + + // With rosetta=true but no mounts, should still create wrapper script + let (exec_path, _argv, _cstrings) = executor + .build_exec_command(&cmd, &[], tmp.path(), true) + .unwrap(); + + assert_eq!(exec_path.to_str().unwrap(), "/tmp/smolvm-mount.sh"); + + // Verify script was created with Rosetta setup + let script_path = tmp.path().join("tmp/smolvm-mount.sh"); + assert!(script_path.exists(), "mount script should be created"); + + let content = std::fs::read_to_string(&script_path).unwrap(); + assert!( + content.contains("/mnt/rosetta"), + "script should mount Rosetta" + ); + assert!( + content.contains("binfmt_misc"), + "script should register binfmt_misc" + ); + } + + #[test] + fn test_shell_escape_no_special_chars() { + assert_eq!(shell_escape("smolvm0"), "smolvm0"); + assert_eq!(shell_escape("/data/dir"), "/data/dir"); + } + + #[test] + fn test_shell_escape_single_quotes() { + assert_eq!(shell_escape("it's"), "it'\\''s"); + } + + #[test] + fn test_mount_script_escapes_malicious_input() { + let tmp = TempDir::new().unwrap(); + let mounts = vec![( + "smolvm0".to_string(), + "/data'; rm -rf /; echo '".to_string(), + )]; + let cmd = Some(vec!["/bin/sh".to_string()]); + + let executor = MacOsExecutor; + let _ = executor + .build_exec_command(&cmd, &mounts, tmp.path(), false) + .unwrap(); + + let content = std::fs::read_to_string(tmp.path().join("tmp/smolvm-mount.sh")).unwrap(); + + // The single quotes in the malicious input must be escaped as '\'' + // so the shell never sees an unquoted semicolon. + assert!(content.contains("'\\''"), "single quotes should be escaped"); + // The mount command must NOT contain unescaped single-quote boundaries + // that would let the injected semicolon run as a separate command. + // In the escaped form, the sequence is: '/data'\''...' which is safe. + // Verify the mkdir and mount both use the escaped path: + assert!( + content.contains("mkdir -p '/data'\\''"), + "mkdir should use escaped path, got: {}", + content + ); + assert!( + content.contains("mount -t virtiofs 'smolvm0' '/data'\\''"), + "mount should use escaped path, got: {}", + content + ); + } +} diff --git a/src/platform/mod.rs b/src/platform/mod.rs new file mode 100644 index 0000000..d0a9a3e --- /dev/null +++ b/src/platform/mod.rs @@ -0,0 +1,422 @@ +//! Platform abstraction for smolvm. +//! +//! This module provides compile-time and runtime platform detection, +//! along with traits for platform-specific behaviors. +//! +//! # Architecture +//! +//! The platform module centralizes all platform-specific logic that was +//! previously scattered across multiple files. It provides: +//! +//! - **Enums**: `Os`, `Arch`, and `Platform` for type-safe platform identification +//! - **Traits**: `VmExecutor` and `RosettaSupport` for platform-specific behaviors +//! - **Implementations**: macOS and Linux implementations of these traits +//! +//! # Example +//! +//! ``` +//! use smolvm::platform::{Os, Arch, Platform, native_platform}; +//! +//! // Get current platform info +//! let os = Os::current(); +//! let arch = Arch::current(); +//! let platform = Platform::current(); +//! +//! // Get OCI platform string for container images +//! let oci = native_platform(); // e.g., "linux/arm64" +//! ``` +//! +//! # Platform Support +//! +//! | OS | Arch | Supported | +//! |----|------|-----------| +//! | macOS | ARM64 (Apple Silicon) | ✅ | +//! | macOS | x86_64 (Intel) | ✅ | +//! | Linux | ARM64 | ✅ | +//! | Linux | x86_64 | ✅ | + +mod traits; + +pub mod uds; + +#[cfg(target_os = "macos")] +pub mod macos; + +#[cfg(target_os = "linux")] +pub mod linux; + +#[cfg(target_os = "windows")] +pub mod windows; + +pub use traits::{RosettaSupport, VmExecutor}; + +/// Host operating system. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Os { + /// macOS (Darwin) + MacOs, + /// Linux + Linux, + /// Windows + Windows, +} + +impl Os { + /// Get the current OS at compile time. + #[cfg(target_os = "macos")] + pub const fn current() -> Self { + Os::MacOs + } + + /// Get the current OS at compile time. + #[cfg(target_os = "linux")] + pub const fn current() -> Self { + Os::Linux + } + + /// Get the current OS at compile time. + #[cfg(target_os = "windows")] + pub const fn current() -> Self { + Os::Windows + } + + /// Returns true if running on macOS. + pub const fn is_macos(&self) -> bool { + matches!(self, Os::MacOs) + } + + /// Returns true if running on Linux. + pub const fn is_linux(&self) -> bool { + matches!(self, Os::Linux) + } + + /// Returns true if running on Windows. + pub const fn is_windows(&self) -> bool { + matches!(self, Os::Windows) + } +} + +impl std::fmt::Display for Os { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Os::MacOs => write!(f, "macos"), + Os::Linux => write!(f, "linux"), + Os::Windows => write!(f, "windows"), + } + } +} + +/// Host CPU architecture. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Arch { + /// ARM 64-bit (aarch64) + Arm64, + /// x86 64-bit (amd64) + X86_64, +} + +impl Arch { + /// Get the current architecture at compile time. + #[cfg(target_arch = "aarch64")] + pub const fn current() -> Self { + Arch::Arm64 + } + + /// Get the current architecture at compile time. + #[cfg(target_arch = "x86_64")] + pub const fn current() -> Self { + Arch::X86_64 + } + + /// Convert to OCI platform architecture string. + /// + /// Returns "arm64" or "amd64" as used in OCI image manifests. + pub const fn oci_arch(&self) -> &'static str { + match self { + Arch::Arm64 => "arm64", + Arch::X86_64 => "amd64", + } + } + + /// Returns true if running on ARM64. + pub const fn is_arm64(&self) -> bool { + matches!(self, Arch::Arm64) + } + + /// Returns true if running on x86_64. + pub const fn is_x86_64(&self) -> bool { + matches!(self, Arch::X86_64) + } +} + +impl std::fmt::Display for Arch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Arch::Arm64 => write!(f, "arm64"), + Arch::X86_64 => write!(f, "x86_64"), + } + } +} + +/// Combined platform identifier (OS + architecture). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Platform { + /// Operating system + pub os: Os, + /// CPU architecture + pub arch: Arch, +} + +impl Platform { + /// Get the current platform at compile time. + pub const fn current() -> Self { + Self { + os: Os::current(), + arch: Arch::current(), + } + } + + /// Convert to OCI platform string (e.g., "linux/arm64"). + /// + /// Note: The OS is always "linux" for container images, + /// regardless of the host OS, since VMs run Linux guests. + pub const fn oci_platform(&self) -> &'static str { + match self.arch { + Arch::Arm64 => "linux/arm64", + Arch::X86_64 => "linux/amd64", + } + } + + /// OCI platform string for the **host** (e.g., "darwin/arm64"). + /// + /// Used for registry Image Index resolution — tells the registry which + /// host OS+arch this `.smolmachine` runs on. Distinct from `oci_platform()` + /// which always returns `linux/*` (the guest). + pub const fn host_oci_platform(&self) -> &'static str { + match (self.os, self.arch) { + (Os::MacOs, Arch::Arm64) => "darwin/arm64", + (Os::MacOs, Arch::X86_64) => "darwin/amd64", + (Os::Linux, Arch::Arm64) => "linux/arm64", + (Os::Linux, Arch::X86_64) => "linux/amd64", + (Os::Windows, Arch::Arm64) => "windows/arm64", + (Os::Windows, Arch::X86_64) => "windows/amd64", + } + } + + /// Check if this platform supports Rosetta 2. + /// + /// Rosetta 2 is only available on Apple Silicon Macs. + pub const fn supports_rosetta(&self) -> bool { + matches!((self.os, self.arch), (Os::MacOs, Arch::Arm64)) + } +} + +impl std::fmt::Display for Platform { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}/{}", self.os, self.arch) + } +} + +/// Current platform constant (zero-cost at runtime). +pub const CURRENT_PLATFORM: Platform = Platform::current(); + +/// Get native platform string for OCI images. +/// +/// Returns "linux/arm64" or "linux/amd64" based on host architecture. +/// This is the platform string to use when pulling container images. +pub fn native_platform() -> &'static str { + CURRENT_PLATFORM.oci_platform() +} + +/// Reject a packed artifact whose guest CPU architecture differs from this host's. +/// +/// A `.smolmachine` carries architecture-specific native binaries — a VM-mode pack +/// holds a compiled guest rootfs, an image-mode pack holds per-arch OCI layers — so +/// an `arm64` artifact cannot run under an `amd64` guest kernel, or vice versa. Used +/// by `machine create --from` and the serve create handler before they extract or +/// boot, so the failure is an immediate, actionable message rather than a cryptic +/// exec-format crash mid-boot. +/// +/// Unlike the single-file `pack run` path — which additionally requires a matching +/// host OS because it dlopens the libs bundled *into* the executable — a sidecar +/// rehydrated via `create --from` boots through the HOST's own libkrun, so only the +/// guest ARCHITECTURE must match; the host OS (macOS vs Linux) is irrelevant. That +/// is what lets a VM packed on a Mac rehydrate on a same-arch Linux node. +/// +/// `artifact_platform` is the manifest's guest platform (e.g. `"linux/arm64"`). A +/// blank or unrecognized arch is allowed through rather than risk falsely rejecting +/// an otherwise-valid artifact. +pub fn ensure_artifact_arch_matches_host(artifact_platform: &str) -> crate::Result<()> { + let host_arch = Arch::current().oci_arch(); + let artifact_arch = artifact_platform.rsplit('/').next().unwrap_or("").trim(); + if matches!(artifact_arch, "amd64" | "arm64") && artifact_arch != host_arch { + return Err(crate::Error::agent( + "platform mismatch", + format!( + "this artifact is built for architecture '{artifact_arch}' (platform \ + '{artifact_platform}'), but this host is '{host_arch}'. A packed VM or image \ + carries native binaries and cannot run on a different CPU architecture — re-pack \ + on an '{artifact_arch}' host, or use an '{host_arch}' artifact." + ), + )); + } + Ok(()) +} + +/// Get the platform-specific VM executor. +/// +/// Returns an executor that handles platform differences in VM execution, +/// particularly around virtiofs mount handling. +#[cfg(target_os = "macos")] +pub fn vm_executor() -> macos::MacOsExecutor { + macos::MacOsExecutor +} + +/// Get the platform-specific VM executor. +#[cfg(target_os = "linux")] +pub fn vm_executor() -> linux::LinuxExecutor { + linux::LinuxExecutor +} + +/// Get the platform-specific Rosetta support handler. +#[cfg(target_os = "macos")] +pub fn rosetta() -> macos::MacOsRosetta { + macos::rosetta_support() +} + +/// Get the platform-specific Rosetta support handler. +#[cfg(target_os = "linux")] +pub fn rosetta() -> linux::LinuxRosetta { + linux::rosetta_support() +} + +/// Get the platform-specific VM executor. +#[cfg(target_os = "windows")] +pub fn vm_executor() -> windows::WindowsExecutor { + windows::WindowsExecutor +} + +/// Get the platform-specific Rosetta support handler. +#[cfg(target_os = "windows")] +pub fn rosetta() -> windows::WindowsRosetta { + windows::rosetta_support() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_artifact_arch_guard() { + let host = Arch::current().oci_arch(); + let other = if host == "amd64" { "arm64" } else { "amd64" }; + + // Matching arch (with the linux/ prefix) is accepted. + assert!(ensure_artifact_arch_matches_host(&format!("linux/{host}")).is_ok()); + // Bare arch string also accepted when it matches. + assert!(ensure_artifact_arch_matches_host(host).is_ok()); + // The opposite arch is rejected. + assert!(ensure_artifact_arch_matches_host(&format!("linux/{other}")).is_err()); + // Host OS is irrelevant — only the arch matters (Mac-built pack on Linux). + assert!(ensure_artifact_arch_matches_host(&format!("darwin/{host}")).is_ok()); + // Blank / unrecognized arch is allowed through rather than false-rejected. + assert!(ensure_artifact_arch_matches_host("").is_ok()); + assert!(ensure_artifact_arch_matches_host("linux/riscv64").is_ok()); + } + + #[test] + fn test_current_platform_is_valid() { + let platform = Platform::current(); + // Should not panic and should have valid values + let _ = platform.oci_platform(); + let _ = platform.os.to_string(); + let _ = platform.arch.to_string(); + } + + #[test] + fn test_oci_platform_format() { + let platform_str = native_platform(); + assert!(platform_str.starts_with("linux/")); + assert!( + platform_str == "linux/arm64" || platform_str == "linux/amd64", + "unexpected platform: {}", + platform_str + ); + } + + #[test] + fn test_arch_oci_strings() { + assert_eq!(Arch::Arm64.oci_arch(), "arm64"); + assert_eq!(Arch::X86_64.oci_arch(), "amd64"); + } + + #[test] + fn test_platform_supports_rosetta() { + let macos_arm = Platform { + os: Os::MacOs, + arch: Arch::Arm64, + }; + let macos_intel = Platform { + os: Os::MacOs, + arch: Arch::X86_64, + }; + let linux_arm = Platform { + os: Os::Linux, + arch: Arch::Arm64, + }; + let linux_x86 = Platform { + os: Os::Linux, + arch: Arch::X86_64, + }; + + assert!(macos_arm.supports_rosetta()); + assert!(!macos_intel.supports_rosetta()); + assert!(!linux_arm.supports_rosetta()); + assert!(!linux_x86.supports_rosetta()); + } + + #[test] + fn test_os_helpers() { + assert!(Os::MacOs.is_macos()); + assert!(!Os::MacOs.is_linux()); + assert!(Os::Linux.is_linux()); + assert!(!Os::Linux.is_macos()); + } + + #[test] + fn test_arch_helpers() { + assert!(Arch::Arm64.is_arm64()); + assert!(!Arch::Arm64.is_x86_64()); + assert!(Arch::X86_64.is_x86_64()); + assert!(!Arch::X86_64.is_arm64()); + } + + #[test] + fn test_platform_display() { + let platform = Platform { + os: Os::MacOs, + arch: Arch::Arm64, + }; + assert_eq!(platform.to_string(), "macos/arm64"); + } + + #[test] + fn test_host_oci_platform_all_combos() { + let cases = [ + (Os::MacOs, Arch::Arm64, "darwin/arm64"), + (Os::MacOs, Arch::X86_64, "darwin/amd64"), + (Os::Linux, Arch::Arm64, "linux/arm64"), + (Os::Linux, Arch::X86_64, "linux/amd64"), + ]; + for (os, arch, expected) in cases { + let p = Platform { os, arch }; + assert_eq!( + p.host_oci_platform(), + expected, + "failed for {:?}/{:?}", + os, + arch + ); + } + } +} diff --git a/src/platform/traits.rs b/src/platform/traits.rs new file mode 100644 index 0000000..313cbba --- /dev/null +++ b/src/platform/traits.rs @@ -0,0 +1,91 @@ +//! Platform behavior traits. +//! +//! These traits define platform-specific behaviors that differ between +//! macOS and Linux, particularly around VM execution and Rosetta support. + +use crate::error::Result; +use std::ffi::CString; +use std::path::Path; + +/// Trait for platform-specific VM execution behaviors. +/// +/// This abstracts over differences in how VMs are executed on different +/// platforms, particularly around virtiofs mount handling. +/// +/// On macOS, virtiofs devices must be explicitly mounted by the guest +/// using a wrapper script. On Linux, the kernel auto-mounts them. +pub trait VmExecutor: Send + Sync { + /// Whether this platform requires explicit virtiofs mount commands. + /// + /// Returns `true` on macOS where virtiofs devices need manual mounting, + /// `false` on Linux where the kernel handles it automatically. + fn requires_mount_wrapper(&self) -> bool; + + /// Build the execution command, optionally wrapping with mount script. + /// + /// # Arguments + /// + /// * `command` - The user's command to execute (None defaults to /bin/sh) + /// * `mounts` - List of (virtiofs_tag, guest_path) tuples for virtiofs mounts + /// * `rootfs` - Path to the rootfs directory on the host + /// * `rosetta` - Whether Rosetta is enabled for x86_64 binary support + /// + /// # Returns + /// + /// Tuple of (exec_path, argv_pointers, cstrings_to_keep_alive). + /// The cstrings must be kept alive for the duration of the argv usage. + /// + /// Note: libkrun expects argv to contain only arguments (not `argv[0]`), + /// as it passes exec_path via KRUN_INIT environment variable. + fn build_exec_command( + &self, + command: &Option>, + mounts: &[(String, String)], + rootfs: &Path, + rosetta: bool, + ) -> Result<(CString, Vec<*const libc::c_char>, Vec)>; + + /// Get platform-specific paths for finding system tools. + /// + /// Returns paths where tools like `mkfs.ext4` might be found. + /// On macOS this includes Homebrew paths, on Linux standard system paths. + fn tool_search_paths(&self) -> &'static [&'static str]; + + /// Get the dynamic library extension for this platform. + /// + /// Returns "dylib" on macOS, "so" on Linux. + fn dylib_extension(&self) -> &'static str; + + /// Get common library search paths for this platform. + /// + /// Used for finding libraries like libkrun at runtime. + fn library_search_paths(&self) -> &'static [&'static str]; +} + +/// Trait for Rosetta 2 support detection and configuration. +/// +/// Rosetta 2 allows running x86_64 binaries on Apple Silicon Macs. +/// This trait provides a uniform interface that returns "not available" +/// on platforms where Rosetta doesn't exist. +pub trait RosettaSupport { + /// Check if Rosetta 2 is available on this system. + /// + /// Returns `true` only on Apple Silicon Macs with Rosetta installed. + fn is_available(&self) -> bool; + + /// Get the path to the Rosetta runtime directory. + /// + /// Returns `Some("/Library/Apple/usr/libexec/oah")` if Rosetta is available, + /// `None` otherwise. + fn runtime_path(&self) -> Option<&'static str>; + + /// Check if the given platform string requires Rosetta. + /// + /// Returns `true` for x86_64/amd64 platforms when running on ARM. + fn needs_rosetta(&self, platform: &str) -> bool { + let platform_lower = platform.to_lowercase(); + platform_lower.contains("amd64") + || platform_lower.contains("x86_64") + || platform_lower.contains("x86-64") + } +} diff --git a/src/platform/uds.rs b/src/platform/uds.rs new file mode 100644 index 0000000..a868be6 --- /dev/null +++ b/src/platform/uds.rs @@ -0,0 +1,197 @@ +//! Cross-platform AF_UNIX (Unix-domain socket) support. +//! +//! The agent control channel, the vsock-port bridges, and the fork control +//! socket all talk over `AF_UNIX` stream sockets. `std` exposes +//! `std::os::unix::net::{UnixStream, UnixListener}` only on Unix, but AF_UNIX +//! is available on Windows 10 1809+ as well. `socket2` provides a portable +//! `Domain::UNIX` socket on every supported platform, so this module wraps it +//! into a small stream/listener API the rest of the host code shares. +//! +//! [`UdsStream`] implements [`std::io::Read`]/[`Write`] (via `socket2::Socket`) +//! and the timeout/clone/shutdown helpers the agent client relies on. On Unix +//! the raw fd is exposed for the interactive-terminal `poll()` loop; that loop +//! is itself Unix-only. + +use socket2::{Domain, SockAddr, Socket, Type}; +use std::io::{self, Read, Write}; +use std::net::Shutdown; +use std::path::Path; +use std::time::Duration; + +/// A connected AF_UNIX stream socket. +#[derive(Debug)] +pub struct UdsStream { + inner: Socket, +} + +impl UdsStream { + /// Connect to the AF_UNIX socket at `path`. + pub fn connect(path: impl AsRef) -> io::Result { + let addr = SockAddr::unix(path.as_ref())?; + let sock = Socket::new(Domain::UNIX, Type::STREAM, None)?; + sock.connect(&addr)?; + Ok(Self { inner: sock }) + } + + /// Wrap an already-connected `socket2::Socket` (e.g. one returned by + /// [`UdsListener::accept`]). + pub fn from_socket(inner: Socket) -> Self { + Self { inner } + } + + /// Create an unnamed, connected pair of AF_UNIX stream sockets. + /// + /// Backed by `socketpair(2)`; Unix-only (Windows has no socketpair for + /// AF_UNIX). Used by the agent-client unit tests. + #[cfg(unix)] + pub fn pair() -> io::Result<(Self, Self)> { + let (a, b) = Socket::pair(Domain::UNIX, Type::STREAM, None)?; + Ok((Self { inner: a }, Self { inner: b })) + } + + /// Create a connected pair of stream sockets (Windows test helper). + /// + /// Windows has no AF_UNIX `socketpair`, so this returns a loopback TCP pair. + /// The agent-client unit tests only push bytes through a connected bidi + /// stream, so the transport is interchangeable here. + #[cfg(windows)] + pub fn pair() -> io::Result<(Self, Self)> { + use std::net::{Ipv4Addr, SocketAddrV4, TcpListener, TcpStream}; + let listener = TcpListener::bind(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0))?; + let addr = listener.local_addr()?; + let a = TcpStream::connect(addr)?; + let (b, _) = listener.accept()?; + Ok(( + Self { + inner: Socket::from(a), + }, + Self { + inner: Socket::from(b), + }, + )) + } + + /// Set the read timeout. `None` disables the timeout (blocking). + pub fn set_read_timeout(&self, dur: Option) -> io::Result<()> { + self.inner.set_read_timeout(dur) + } + + /// Get the current read timeout, if any. + pub fn read_timeout(&self) -> io::Result> { + self.inner.read_timeout() + } + + /// Set the write timeout. `None` disables the timeout (blocking). + pub fn set_write_timeout(&self, dur: Option) -> io::Result<()> { + self.inner.set_write_timeout(dur) + } + + /// Clone the underlying socket handle (a new fd/handle referring to the + /// same connection), mirroring `UnixStream::try_clone`. + pub fn try_clone(&self) -> io::Result { + Ok(Self { + inner: self.inner.try_clone()?, + }) + } + + /// Shut down the read, write, or both halves of the connection. + pub fn shutdown(&self, how: Shutdown) -> io::Result<()> { + self.inner.shutdown(how) + } + + /// Borrow the underlying `socket2::Socket`. + pub fn as_socket(&self) -> &Socket { + &self.inner + } + + /// Raw Windows `SOCKET` handle for the interactive-terminal poll loop. + /// + /// Returned as a `u64` (the width of a `SOCKET`) so callers can hand it to + /// the WinSock APIs the poll loop uses. + #[cfg(windows)] + pub fn raw_socket(&self) -> u64 { + use std::os::windows::io::AsRawSocket; + self.inner.as_raw_socket() + } +} + +#[cfg(unix)] +impl std::os::unix::io::AsRawFd for UdsStream { + fn as_raw_fd(&self) -> std::os::unix::io::RawFd { + self.inner.as_raw_fd() + } +} + +impl Read for UdsStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + (&self.inner).read(buf) + } +} + +impl Read for &UdsStream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + (&self.inner).read(buf) + } +} + +impl Write for UdsStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + (&self.inner).write(buf) + } + fn flush(&mut self) -> io::Result<()> { + (&self.inner).flush() + } +} + +impl Write for &UdsStream { + fn write(&mut self, buf: &[u8]) -> io::Result { + (&self.inner).write(buf) + } + fn flush(&mut self) -> io::Result<()> { + (&self.inner).flush() + } +} + +/// A listening AF_UNIX socket. +#[derive(Debug)] +pub struct UdsListener { + inner: Socket, +} + +impl UdsListener { + /// Bind a listening AF_UNIX socket at `path`. + /// + /// The caller is responsible for removing any stale socket file first + /// (matching `UnixListener::bind` semantics, which also fail on EADDRINUSE). + pub fn bind(path: impl AsRef) -> io::Result { + let addr = SockAddr::unix(path.as_ref())?; + let sock = Socket::new(Domain::UNIX, Type::STREAM, None)?; + sock.bind(&addr)?; + sock.listen(128)?; + Ok(Self { inner: sock }) + } + + /// Accept a single incoming connection. + pub fn accept(&self) -> io::Result { + let (sock, _addr) = self.inner.accept()?; + Ok(UdsStream::from_socket(sock)) + } + + /// Iterate over incoming connections, mirroring `UnixListener::incoming`. + pub fn incoming(&self) -> Incoming<'_> { + Incoming { listener: self } + } +} + +/// Iterator over incoming connections to a [`UdsListener`]. +pub struct Incoming<'a> { + listener: &'a UdsListener, +} + +impl Iterator for Incoming<'_> { + type Item = io::Result; + + fn next(&mut self) -> Option { + Some(self.listener.accept()) + } +} diff --git a/src/platform/windows.rs b/src/platform/windows.rs new file mode 100644 index 0000000..6f3a560 --- /dev/null +++ b/src/platform/windows.rs @@ -0,0 +1,83 @@ +//! Windows-specific platform implementations. +//! +//! Windows support compiles the host control plane; the VM-execution code +//! paths mirror the Linux executor (libkrun on Windows auto-mounts the guest's +//! virtiofs devices, so no mount-wrapper script is needed). Rosetta is a +//! macOS-only feature and is unavailable here. + +use crate::error::{Error, Result}; +use crate::platform::traits::{RosettaSupport, VmExecutor}; +use std::ffi::CString; +use std::path::Path; + +/// Windows VM executor implementation. +/// +/// Like the Linux executor, this executes the user command directly without a +/// virtiofs mount-wrapper script. +pub struct WindowsExecutor; + +impl VmExecutor for WindowsExecutor { + fn requires_mount_wrapper(&self) -> bool { + false + } + + fn build_exec_command( + &self, + command: &Option>, + _mounts: &[(String, String)], + _rootfs: &Path, + _rosetta: bool, + ) -> Result<(CString, Vec<*const libc::c_char>, Vec)> { + let default_cmd = vec!["/bin/sh".to_string()]; + let cmd = command.as_ref().unwrap_or(&default_cmd); + + if cmd.is_empty() { + return Err(Error::vm_creation("command cannot be empty")); + } + + let exec_path = CString::new(cmd[0].as_str()) + .map_err(|_| Error::vm_creation("invalid command path"))?; + + let cstrings: Vec = cmd + .iter() + .skip(1) + .map(|s| CString::new(s.as_str())) + .collect::, _>>() + .map_err(|_| Error::vm_creation("invalid command argument"))?; + + let mut argv: Vec<*const libc::c_char> = cstrings.iter().map(|s| s.as_ptr()).collect(); + argv.push(std::ptr::null()); + + Ok((exec_path, argv, cstrings)) + } + + fn tool_search_paths(&self) -> &'static [&'static str] { + &[] + } + + fn dylib_extension(&self) -> &'static str { + "dll" + } + + fn library_search_paths(&self) -> &'static [&'static str] { + &[] + } +} + +/// Windows Rosetta support (stub - always unavailable). +pub struct WindowsRosetta; + +impl RosettaSupport for WindowsRosetta { + fn is_available(&self) -> bool { + false + } + + fn runtime_path(&self) -> Option<&'static str> { + None + } +} + +/// Get the Rosetta support instance for Windows. +pub fn rosetta_support() -> WindowsRosetta { + WindowsRosetta +} diff --git a/src/process.rs b/src/process.rs new file mode 100644 index 0000000..1eeb7a3 --- /dev/null +++ b/src/process.rs @@ -0,0 +1,2881 @@ +//! Process management utilities. +//! +//! This module provides utilities for managing child processes, +//! including signal handling and graceful shutdown. + +#[cfg(unix)] +use std::os::fd::IntoRawFd; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +use crate::error::{Error, Result}; + +/// Portable process-id type. On Unix this is `libc::pid_t`; on Windows there is +/// no POSIX `pid_t`, so we use `i32` (process IDs there are `u32` but `i32` +/// matches the existing signatures and the sentinel `-1` values used here). +#[cfg(unix)] +pub type Pid = libc::pid_t; +/// Portable process-id type (Windows): there is no POSIX `pid_t`, so `i32` is +/// used (matching the existing signatures and `-1` sentinels). +#[cfg(not(unix))] +pub type Pid = i32; + +/// Windows process-control helpers backing the cross-platform liveness/kill/wait +/// API. Mirror the semantics of the Unix `kill`/`waitpid` paths using the Win32 +/// process APIs. +#[cfg(windows)] +mod win { + use windows_sys::Win32::Foundation::{CloseHandle, HANDLE, STILL_ACTIVE, WAIT_OBJECT_0}; + use windows_sys::Win32::System::Threading::{ + GetExitCodeProcess, OpenProcess, TerminateProcess, WaitForSingleObject, INFINITE, + PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_TERMINATE, + }; + + // `GetExitCodeProcess` writes `STILL_ACTIVE` (= STATUS_PENDING, 0x103) while + // the process is running. + const STILL_RUNNING: u32 = STILL_ACTIVE as u32; + + /// Open a process handle, returning `None` if it cannot be opened. + fn open(pid: u32, access: u32) -> Option { + // SAFETY: OpenProcess is a simple FFI call; we validate the handle. + let handle = unsafe { OpenProcess(access, 0, pid) }; + if handle.is_null() { + None + } else { + Some(handle) + } + } + + fn exit_code(handle: HANDLE) -> Option { + let mut code: u32 = 0; + // SAFETY: handle is a live process handle for the duration of the call. + let ok = unsafe { GetExitCodeProcess(handle, &mut code) }; + if ok == 0 { + None + } else { + Some(code) + } + } + + pub fn process_is_alive(pid: u32) -> bool { + match open(pid, PROCESS_QUERY_LIMITED_INFORMATION) { + Some(handle) => { + let alive = exit_code(handle) + .map(|c| c == STILL_RUNNING) + .unwrap_or(false); + unsafe { CloseHandle(handle) }; + alive + } + None => false, + } + } + + pub fn process_try_wait(pid: u32) -> Option { + let handle = open(pid, PROCESS_QUERY_LIMITED_INFORMATION)?; + let result = match exit_code(handle) { + Some(code) if code == STILL_RUNNING => None, + Some(code) => Some(code as i32), + None => None, + }; + unsafe { CloseHandle(handle) }; + result + } + + pub fn process_wait(pid: u32) -> i32 { + let Some(handle) = open(pid, PROCESS_QUERY_LIMITED_INFORMATION) else { + return -1; + }; + // SAFETY: handle is valid; INFINITE blocks until the process exits. + let waited = unsafe { WaitForSingleObject(handle, INFINITE) }; + let code = if waited == WAIT_OBJECT_0 { + exit_code(handle).map(|c| c as i32).unwrap_or(-1) + } else { + -1 + }; + unsafe { CloseHandle(handle) }; + code + } + + pub fn process_kill(pid: u32) -> bool { + match open(pid, PROCESS_TERMINATE) { + Some(handle) => { + // SAFETY: handle has PROCESS_TERMINATE rights. + let ok = unsafe { TerminateProcess(handle, 1) } != 0; + unsafe { CloseHandle(handle) }; + ok + } + None => false, + } + } + + /// Process creation time as a u64 (Windows FILETIME: 100 ns ticks since + /// 1601). Stable for the life of the process, so it pins PID identity + /// against reuse — the same role the start-time-from-/proc plays on Linux. + pub fn process_start_time(pid: u32) -> Option { + use windows_sys::Win32::Foundation::FILETIME; + use windows_sys::Win32::System::Threading::GetProcessTimes; + let handle = open(pid, PROCESS_QUERY_LIMITED_INFORMATION)?; + let mut creation = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let mut exit = creation; + let mut kernel = creation; + let mut user = creation; + // SAFETY: handle is a live process handle for the duration of the call; + // all four FILETIME out-params are valid, writable locals. + let ok = + unsafe { GetProcessTimes(handle, &mut creation, &mut exit, &mut kernel, &mut user) }; + unsafe { CloseHandle(handle) }; + if ok == 0 { + None + } else { + Some(filetime_to_u64(&creation)) + } + } + + /// Combine a FILETIME's high/low halves into a single u64 (100 ns ticks). + fn filetime_to_u64(ft: &windows_sys::Win32::Foundation::FILETIME) -> u64 { + ((ft.dwHighDateTime as u64) << 32) | (ft.dwLowDateTime as u64) + } + + /// Sample cumulative CPU time (ns) and resident set size (bytes) for a + /// process. Mirrors the Linux `/proc//{stat,statm}` and macOS + /// `proc_pidinfo` sampling so `machine monitor` reports live CPU/RSS on + /// Windows too. Returns `None` if the process is gone or stats are + /// unavailable. + pub fn process_stats(pid: u32) -> Option<(u64, u64)> { + use windows_sys::Win32::Foundation::FILETIME; + use windows_sys::Win32::System::ProcessStatus::{ + GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS, + }; + use windows_sys::Win32::System::Threading::GetProcessTimes; + + let handle = open(pid, PROCESS_QUERY_LIMITED_INFORMATION)?; + + let zero = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let (mut creation, mut exit, mut kernel, mut user) = (zero, zero, zero, zero); + // SAFETY: handle is live for the call; all four FILETIME out-params are + // valid, writable locals. + let times_ok = + unsafe { GetProcessTimes(handle, &mut creation, &mut exit, &mut kernel, &mut user) }; + + let mut pmc: PROCESS_MEMORY_COUNTERS = unsafe { std::mem::zeroed() }; + let pmc_size = std::mem::size_of::() as u32; + pmc.cb = pmc_size; + // SAFETY: handle is live; pmc is a valid, correctly-sized counters buffer. + let mem_ok = unsafe { GetProcessMemoryInfo(handle, &mut pmc, pmc_size) }; + + unsafe { CloseHandle(handle) }; + if times_ok == 0 || mem_ok == 0 { + return None; + } + + // GetProcessTimes' kernel/user are 100 ns ticks; sum and scale to ns. + let cpu_time_ns = filetime_to_u64(&kernel) + .saturating_add(filetime_to_u64(&user)) + .saturating_mul(100); + Some((cpu_time_ns, pmc.WorkingSetSize as u64)) + } +} + +/// Flag indicating whether SIGCHLD handler has been installed. Only the Unix +/// SIGCHLD-reaping path uses this; Windows has no zombie-reaping model. +#[cfg_attr(not(unix), allow(dead_code))] +static SIGCHLD_HANDLER_INSTALLED: AtomicBool = AtomicBool::new(false); + +/// Default timeout for graceful shutdown before SIGKILL. +pub const DEFAULT_STOP_TIMEOUT: Duration = Duration::from_secs(10); + +/// Default timeout for SIGKILL to take effect. +pub const SIGKILL_WAIT: Duration = Duration::from_millis(50); + +/// Aggressive poll interval for fast process shutdown and agent readiness. +pub const FAST_POLL_INTERVAL: Duration = Duration::from_millis(10); + +/// Number of aggressive polls before backing off to slower intervals. +pub const FAST_POLL_COUNT: u32 = 10; + +/// Exit code returned when the actual exit status cannot be determined. +/// This happens when a process is confirmed dead but waitpid() fails to +/// retrieve the exit status (e.g., process was reaped by another handler). +pub const UNKNOWN_EXIT_CODE: i32 = -1; + +/// Close inherited file descriptors starting at `min_fd`. +/// +/// This is used in freshly spawned/forked VM launcher children to avoid holding +/// parent database locks, sockets, and other resources. On Linux it uses +/// `close_range` when available. On other platforms it enumerates `/dev/fd` and +/// closes only descriptors that are actually open, avoiding an expensive loop to +/// very large `getdtablesize()` values on macOS. +#[cfg(unix)] +pub fn close_inherited_fds_from(min_fd: i32) { + if min_fd < 0 { + return; + } + + #[cfg(target_os = "linux")] + unsafe { + let ret = libc::syscall(libc::SYS_close_range, min_fd as u32, u32::MAX, 0u32); + if ret == 0 { + return; + } + } + + if close_fds_from_dev_fd(min_fd) { + return; + } + + close_fds_by_range(min_fd); +} + +/// Windows: file descriptors are not inherited across `CreateProcess` the way +/// POSIX fds are (handle inheritance is opt-in per-handle), so there is nothing +/// to bulk-close here. +#[cfg(not(unix))] +pub fn close_inherited_fds_from(_min_fd: i32) {} + +#[cfg(unix)] +fn close_fds_from_dev_fd(min_fd: i32) -> bool { + let entries = match std::fs::read_dir("/dev/fd") { + Ok(entries) => entries, + Err(_) => return false, + }; + + let mut fds: Vec = entries + .filter_map(|entry| entry.ok()) + .filter_map(|entry| entry.file_name().to_string_lossy().parse::().ok()) + .filter(|fd| *fd >= min_fd) + .collect(); + + // Drop the read_dir handle before closing; its fd may have appeared in the + // collected list, and closing an already-closed fd is harmless. + fds.sort_unstable(); + fds.dedup(); + + for fd in fds { + unsafe { + libc::close(fd); + } + } + + true +} + +#[cfg(unix)] +fn close_fds_by_range(min_fd: i32) { + let max_fd = unsafe { libc::getdtablesize() }; + for fd in min_fd..max_fd { + unsafe { + libc::close(fd); + } + } +} + +/// Apply best-effort, low-risk hardening to *this* process before it becomes the +/// VMM host for an untrusted guest — the info-leak / escalation baseline that the +/// heavier work (seccomp, cgroup caps, privilege drop) builds on top of. +/// +/// Effects (Linux; a near-no-op on macOS dev, which is single-tenant): +/// - `PR_SET_NO_NEW_PRIVS`: a guest→VMM escape cannot regain privileges via a +/// setuid binary. Affects nothing the VMM legitimately does. +/// - `PR_SET_DUMPABLE = 0`: not ptrace-able, and `/proc//{mem,maps,…}` +/// become root-owned, so a compromised VMM can't read a neighbor VM's guest +/// RAM. Self-access to `/proc/self` is unaffected, so libkrun boots normally +/// (verified on Linux/KVM across bare/network/GPU VMs). +/// - `RLIMIT_CORE = 0`: never write a core dump, which would contain guest RAM +/// (the main on-crash memory-disclosure vector). POSIX — both OSes. +/// +/// All calls are best-effort: failures are ignored (hardening is additive and +/// must never block boot). +pub fn harden_self() { + #[cfg(target_os = "linux")] + unsafe { + // Block setuid privilege escalation after a guest→VMM escape. + libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + // Non-dumpable: not ptrace-able, and /proc//{mem,maps,…} become + // root-owned so another same-uid VMM can't read this VM's guest RAM + // (cross-tenant memory leak). Self-access to /proc/self stays permitted, + // so libkrun still reads its own maps and boots normally (verified on + // Linux/KVM across bare/network/GPU VMs). + // + // EXCEPTION: a forkable golden must stay dumpable. Its CoW clones map the + // golden's guest RAM by opening /proc//fd/, which + // PR_SET_DUMPABLE=0 would deny (EACCES → clone can't boot). The + // RLIMIT_CORE=0 below still prevents a core dump from leaking the + // golden's RAM, so only same-uid /proc access + ptrace are relaxed — + // acceptable for a fork pool whose clones legitimately share its memory. + if std::env::var_os("SMOLVM_FORKABLE").is_none() { + libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0); + } + } + // RLIMIT_CORE=0: never write a core dump (which would contain guest RAM). + // POSIX-only; Windows has no rlimit and no core-dump-to-file model here. + #[cfg(unix)] + unsafe { + let lim = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + libc::setrlimit(libc::RLIMIT_CORE, &lim); + } +} + +/// (Re)assert the process's `PR_SET_DUMPABLE` flag. +/// +/// `setuid()` clears dumpable, after which `ptrace_may_access` denies even a +/// same-uid reader of `/proc//{mem,fd}` without `CAP_SYS_PTRACE`. A forkable +/// golden's clones map its guest-RAM memfd via `/proc//fd`, so the +/// forkable boot re-asserts dumpable *after* its per-VM uid drop — without this, +/// fork breaks under uid isolation. Only the golden's own uid (under per-VM uids, +/// exactly its clones) can reach it. Linux-only; no-op elsewhere. +#[cfg(target_os = "linux")] +pub fn set_dumpable(dumpable: bool) { + unsafe { libc::prctl(libc::PR_SET_DUMPABLE, i32::from(dumpable), 0, 0, 0) }; +} + +/// No-op where `prctl` isn't applicable (macOS dev). +#[cfg(not(target_os = "linux"))] +pub fn set_dumpable(_dumpable: bool) {} + +// ============================================================================ +// Per-VM cgroup v2 resource caps (noisy-neighbor / host-DoS containment) +// +// Each VMM subprocess is placed in its own cgroup v2 leaf with cpu/pids/memory +// limits so an untrusted guest cannot peg host CPU, fork-bomb the host, or +// balloon VMM memory to harm neighbors. See docs/runtime-isolation-hardening.md. +// +// Two pieces, because of the cgroup v2 "no internal processes" rule (a cgroup +// may hold processes OR delegate controllers to children, not both): +// 1. `setup_cgroup_delegation_root` (supervisor): vacate our own process into +// a leaf so the parent can become a delegated root that distributes +// cpu/memory/pids to per-VM children. +// 2. `place_in_cgroup` (VMM subprocess): create a capped `vm-` leaf under +// that root and join it. +// Both are best-effort and Linux-only — a near-no-op on macOS dev (single +// tenant), and on Linux they degrade to "uncapped but still booting" whenever +// cgroup v2 isn't delegated to us, since hardening must never block boot. +// ============================================================================ + +/// CPU accounting period for `cpu.max` (cgroup v2 default, 100 ms). +#[cfg(target_os = "linux")] +const CGROUP_CPU_PERIOD_US: u64 = 100_000; + +/// Cap on host tasks (threads/processes) a single VMM may spawn. Guest processes +/// run *inside* the VM and are not host tasks, so the VMM itself needs only a few +/// dozen (one per vCPU plus virtio/vsock/gpu workers). 1024 is generous headroom +/// that still severs a host-side fork bomb after a guest→VMM escape. +#[cfg(target_os = "linux")] +const CGROUP_PIDS_MAX: u32 = 1024; + +/// Extra MiB granted on top of guest RAM for VMM/virtio/gpu overhead when sizing +/// `memory.max`. This is defense-in-depth atop the guest RAM bound already +/// enforced by `krun_set_vm_config`, not the primary memory control. +#[cfg(target_os = "linux")] +const CGROUP_MEM_OVERHEAD_MIB: u64 = 768; + +/// Resolve this process's cgroup v2 directory from `/proc/self/cgroup`. +/// +/// Returns `None` on a cgroup v1 / hybrid host (no unified `0::` line) or if the +/// path can't be read — callers then skip cgroup caps. +#[cfg(target_os = "linux")] +fn cgroup_v2_self_dir() -> Option { + let content = std::fs::read_to_string("/proc/self/cgroup").ok()?; + // The unified-hierarchy entry is the line beginning with "0::". + let rel = content.lines().find_map(|l| l.strip_prefix("0::"))?.trim(); + if rel.is_empty() { + return None; + } + Some(std::path::Path::new("/sys/fs/cgroup").join(rel.trim_start_matches('/'))) +} + +/// Write a single cgroup control file (no trailing newline needed by the kernel). +#[cfg(target_os = "linux")] +fn write_cgroup(dir: &std::path::Path, file: &str, val: &str) -> std::io::Result<()> { + use std::io::Write; + let mut f = std::fs::OpenOptions::new() + .write(true) + .open(dir.join(file))?; + f.write_all(val.as_bytes()) +} + +/// Supervisor-side: turn our current cgroup into a delegated root under which +/// per-VM subprocesses can be capped. +/// +/// Because of the "no internal processes" rule, we first move *our own* process +/// into a `supervisor` leaf, then enable `cpu`/`memory`/`pids` distribution on +/// the now-empty parent. That parent is returned as the root to pass to +/// [`place_in_cgroup`]. Pass the returned path to each VMM subprocess (e.g. via +/// the `SMOLVM_CGROUP_ROOT` env var) so it can self-place. +/// +/// Returns `None` (and changes nothing observable) when cgroup v2 isn't +/// delegated to us — e.g. the process isn't under a systemd unit with +/// `Delegate=yes`, or we lack write access. The caller proceeds without caps. +#[cfg(target_os = "linux")] +pub fn setup_cgroup_delegation_root() -> Option { + let root = cgroup_v2_self_dir()?; + // Vacate our process into a leaf so `root` may distribute controllers. + let supervisor = root.join("supervisor"); + let _ = std::fs::create_dir(&supervisor); + let pid = unsafe { libc::getpid() }; + if write_cgroup(&supervisor, "cgroup.procs", &pid.to_string()).is_err() { + // Not delegated / no permission — no per-VM caps available. + return None; + } + // Now that `root` holds no processes, enable the controllers we cap on. + if write_cgroup(&root, "cgroup.subtree_control", "+cpu +memory +pids").is_err() { + return None; + } + Some(root) +} + +/// VMM-subprocess-side: place THIS process into a capped `vm-` leaf under +/// `root`, deriving limits from the VM's resources. +/// +/// Caps applied (best-effort, each independently): +/// - `cpu.max` = `vcpus * 100ms / 100ms` → bounds CPU to ~`vcpus` cores. +/// - `pids.max` = [`CGROUP_PIDS_MAX`] → caps host tasks (fork-bomb containment). +/// - `memory.max` = guest RAM + [`CGROUP_MEM_OVERHEAD_MIB`] → defense-in-depth. +/// +/// `root` must be a delegated root with `cpu`/`memory`/`pids` enabled in its +/// `subtree_control` (see [`setup_cgroup_delegation_root`]); otherwise the limit +/// files won't exist and the caps silently degrade while the process still runs. +/// Never blocks boot. +#[cfg(target_os = "linux")] +pub fn place_in_cgroup(root: &std::path::Path, vcpus: u8, memory_mib: u32) { + let pid = unsafe { libc::getpid() }; + let vm = root.join(format!("vm-{pid}")); + if let Err(e) = std::fs::create_dir(&vm) { + if e.kind() != std::io::ErrorKind::AlreadyExists { + tracing::debug!(error = %e, dir = %vm.display(), + "could not create per-VM cgroup; VMM running uncapped"); + return; + } + } + + // Set limits on the leaf *before* joining it. + let quota_us = (vcpus.max(1) as u64) * CGROUP_CPU_PERIOD_US; + let _ = write_cgroup( + &vm, + "cpu.max", + &format!("{quota_us} {CGROUP_CPU_PERIOD_US}"), + ); + let _ = write_cgroup(&vm, "pids.max", &CGROUP_PIDS_MAX.to_string()); + let mem_bytes = (memory_mib as u64 + CGROUP_MEM_OVERHEAD_MIB) * 1024 * 1024; + let _ = write_cgroup(&vm, "memory.max", &mem_bytes.to_string()); + + // Join the leaf last; from here the caps above govern this process tree. + if let Err(e) = write_cgroup(&vm, "cgroup.procs", &pid.to_string()) { + tracing::debug!(error = %e, "failed to join per-VM cgroup; VMM running uncapped"); + let _ = std::fs::remove_dir(&vm); // leave no empty cgroup behind + return; + } + tracing::info!( + vcpus, memory_mib, cgroup = %vm.display(), + "placed VMM subprocess in per-VM cgroup with cpu/pids/memory caps" + ); +} + +// ============================================================================ +// Seccomp-BPF syscall allowlist for the VM boot subprocess +// +// libkrun (unlike Firecracker) installs no seccomp filter, so a guest→VMM escape +// would inherit the host's full syscall surface. This installs a Firecracker-style +// allowlist (via the `seccompiler` crate) on the boot subprocess before it enters +// the guest run loop, confining a compromised VMM to the ~dozens of syscalls a +// running microVM legitimately needs — and denying the escape-amplifying ones +// (ptrace, kexec_load, *_module, mount, bpf, perf_event_open, process_vm_*, +// setns, unshare, …), none of which appear in a real VM's syscall trace. +// +// The allowlist was derived empirically by stracing a full VM lifecycle +// (boot + exec + stop) on a Linux/KVM host; see docs/runtime-isolation-hardening.md. +// x86_64-Linux only for now (the production target); a no-op stub elsewhere. +// Gated by SMOLVM_SECCOMP=audit|enforce so rollout is opt-in. +// ============================================================================ + +/// Install the seccomp allowlist on the calling thread (and, by inheritance, on +/// every thread it later spawns — the vCPU/worker threads libkrun creates). Must +/// be called while still single-threaded, before `krun_start_enter`. +/// +/// `enforce = true` → a non-allowlisted syscall kills the process (KillProcess). +/// `enforce = false` → audit mode: non-allowlisted syscalls are logged but allowed +/// (SECCOMP Log), so a run surfaces any missing syscalls without breaking the VM. +#[cfg(all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") +))] +pub fn install_seccomp_filter(enforce: bool) -> std::result::Result<(), String> { + // Build the BPF program (allocates) and apply it (a single seccomp syscall, + // allocation-free). Split out so tests can build in the parent and apply in a + // forked child without allocating post-fork. + let program = build_seccomp_program(enforce)?; + seccompiler::apply_filter(&program).map_err(|e| e.to_string())?; + Ok(()) +} + +/// Compile the syscall allowlist into a seccomp BPF program. See +/// [`install_seccomp_filter`] for the policy. +#[cfg(all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") +))] +fn build_seccomp_program(enforce: bool) -> std::result::Result { + use seccompiler::{BpfProgram, SeccompAction, SeccompFilter, TargetArch}; + use std::collections::BTreeMap; + + // Syscalls observed across a full VM lifecycle (boot + exec + stop), plus a + // small margin of common runtime syscalls that vDSO/timing can hide from a + // single trace. An empty rule vec allows the syscall unconditionally. + // + // Kept as a hand-grouped table (rustfmt would explode it one-per-line): + // a security allowlist is far easier to audit when related syscalls sit + // together under a category comment. + #[rustfmt::skip] + let mut allowed: Vec = vec![ + // file & block I/O (storage/overlay disks, virtio-blk, layer files) + libc::SYS_read, libc::SYS_write, libc::SYS_pread64, libc::SYS_pwrite64, + libc::SYS_preadv, libc::SYS_pwritev, libc::SYS_openat, libc::SYS_close, + libc::SYS_close_range, libc::SYS_lseek, libc::SYS_fsync, libc::SYS_fallocate, + libc::SYS_ftruncate, libc::SYS_fstat, libc::SYS_newfstatat, libc::SYS_statx, + libc::SYS_fstatfs, libc::SYS_statfs, libc::SYS_fcntl, libc::SYS_flock, + libc::SYS_dup, libc::SYS_dup3, libc::SYS_getdents64, + libc::SYS_readlinkat, libc::SYS_faccessat, libc::SYS_faccessat2, libc::SYS_umask, + libc::SYS_fgetxattr, libc::SYS_flistxattr, libc::SYS_pipe2, + // memory (guest RAM, dlopen of libkrun) + libc::SYS_mmap, libc::SYS_munmap, libc::SYS_mremap, libc::SYS_mprotect, + libc::SYS_madvise, libc::SYS_brk, + // memfd_create: a forkable machine (`machine start --forkable`) backs its + // guest RAM with a memfd so clones can MAP_PRIVATE it copy-on-write. Used + // only on the fork base, but harmless (anonymous in-memory file, no host + // fs reach) to allow for every VM. Without it a forkable boot is SIGSYS- + // killed under `seccomp=enforce`. + libc::SYS_memfd_create, + // KVM + device ioctls, eventfd plumbing + libc::SYS_ioctl, libc::SYS_eventfd2, + // epoll / poll event loops (virtio, vsock/TSI) + libc::SYS_epoll_create1, libc::SYS_epoll_ctl, + libc::SYS_epoll_pwait, libc::SYS_ppoll, + // timerfd — the virtio-net host network stack's event loop arms timers + // (TCP retransmit/poll) via timerfd; without these the VMM is SIGSYS-killed + // at start as soon as a published port enables virtio-net. + libc::SYS_timerfd_create, libc::SYS_timerfd_settime, libc::SYS_timerfd_gettime, + // sockets (vsock/TSI data path, host networking) + libc::SYS_socket, libc::SYS_socketpair, libc::SYS_connect, libc::SYS_bind, + libc::SYS_listen, libc::SYS_accept, libc::SYS_sendto, libc::SYS_recvfrom, + libc::SYS_sendmsg, libc::SYS_sendmmsg, libc::SYS_recvmsg, libc::SYS_setsockopt, + libc::SYS_getsockopt, libc::SYS_shutdown, + // threads & synchronization (vCPU/worker threads, render-thread priority) + libc::SYS_clone, libc::SYS_clone3, libc::SYS_futex, libc::SYS_set_robust_list, + libc::SYS_set_tid_address, libc::SYS_rseq, libc::SYS_sched_yield, + libc::SYS_sched_getaffinity, libc::SYS_sched_setaffinity, libc::SYS_sched_getparam, + libc::SYS_sched_setscheduler, libc::SYS_setpriority, libc::SYS_membarrier, + libc::SYS_gettid, libc::SYS_getpid, libc::SYS_getppid, + // signals + libc::SYS_rt_sigaction, libc::SYS_rt_sigprocmask, libc::SYS_rt_sigreturn, + libc::SYS_sigaltstack, libc::SYS_signalfd4, libc::SYS_kill, libc::SYS_tgkill, + libc::SYS_tkill, + // time + libc::SYS_clock_nanosleep, libc::SYS_nanosleep, libc::SYS_clock_gettime, + libc::SYS_gettimeofday, + // process lifecycle & misc identity/limits + libc::SYS_wait4, libc::SYS_exit, libc::SYS_exit_group, libc::SYS_restart_syscall, + libc::SYS_prctl, libc::SYS_prlimit64, libc::SYS_getrusage, + libc::SYS_sysinfo, libc::SYS_uname, libc::SYS_getrandom, + libc::SYS_getuid, libc::SYS_geteuid, libc::SYS_getgid, libc::SYS_getegid, + // capget/capset: virtiofs drops CAP_FSETID around a passthrough write/ + // create (`drop_effective_cap`, via the `caps` crate) so the kernel + // strips setuid/setgid bits on non-owner writes — same root-VMM / + // non-root-guest passthrough path as setres{u,g}id below. A cap-dropped + // VMM can't raise caps outside its bounding set, so this grants nothing. + libc::SYS_capget, libc::SYS_capset, libc::SYS_setpgid, + // accept4 + sock{name,peername}: the published-port listener threads + // (smolvm-tcp-*) and virtio-net path. Rust's TcpListener uses accept4, + // not accept — the audit run logged these (288/51/52 on x86_64) because + // they post-date the original capture. + libc::SYS_accept4, libc::SYS_getsockname, libc::SYS_getpeername, + // virtiofs passthrough executes the guest's FS mutations on the HOST, so + // the host VMM issues the full *at family + fd/symlink xattrs. Derived + // from fs/linux/passthrough.rs (NOT just the audit trace, which only + // exercised mkdirat/symlinkat) so a guest that mknod/link/setxattr/chmod + // doesn't trip `enforce`. SYS_* exist on both arches; BTreeMap dedups any + // overlap with the arch-gated lists below. + libc::SYS_mkdirat, libc::SYS_mknodat, libc::SYS_symlinkat, libc::SYS_linkat, + libc::SYS_unlinkat, libc::SYS_renameat2, libc::SYS_fchmodat, libc::SYS_fchownat, + libc::SYS_fchmod, libc::SYS_fdatasync, libc::SYS_utimensat, libc::SYS_copy_file_range, + libc::SYS_fsetxattr, libc::SYS_fremovexattr, + libc::SYS_lgetxattr, libc::SYS_lsetxattr, libc::SYS_llistxattr, libc::SYS_lremovexattr, + // virtiofs scopes each request to the guest process's uid/gid before + // touching the host fs (so DAC checks run as the guest user, not as a + // root VMM) via per-thread setres{u,g}id — passthrough.rs `scoped_cred!` + // / `set_creds`. Reached only when the VMM keeps CAP_SETUID (root, no + // per-VM uid drop) AND a NON-root guest process does fs I/O: a path a + // single-uid (all-root) trace never exercises, which is why a diverse- + // workload review caught it and the original audit didn't. Without these + // an unprivileged guest writing through virtiofs SIGSYS-kills the VMM + // under `enforce`. Safe to allow: a capability-dropped (uid-isolated) + // VMM still can't escalate — the kernel enforces CAP_SETUID regardless, + // so the syscall just EPERMs. Matches virtiofsd's own allowlist. + libc::SYS_setresuid, libc::SYS_setresgid, + ]; + + // Legacy syscalls present only on x86_64; aarch64 exposes only the *at/p + // variants (already in the common list above) plus a few of its own. These + // libc::SYS_* constants don't exist on the other arch, so they must be + // arch-gated. The arm64 set is a starting point — refine from an audit run. + #[cfg(target_arch = "x86_64")] + allowed.extend_from_slice(&[ + libc::SYS_dup2, + libc::SYS_readlink, + libc::SYS_unlink, + libc::SYS_rename, + libc::SYS_mkdir, + libc::SYS_access, + libc::SYS_epoll_wait, + libc::SYS_poll, + libc::SYS_arch_prctl, + ]); + #[cfg(target_arch = "aarch64")] + allowed.extend_from_slice(&[libc::SYS_unlinkat, libc::SYS_renameat2, libc::SYS_mkdirat]); + + let rules: BTreeMap> = + allowed.iter().map(|&nr| (nr, Vec::new())).collect(); + + let mismatch_action = if enforce { + SeccompAction::KillProcess + } else { + SeccompAction::Log + }; + let target_arch = if cfg!(target_arch = "aarch64") { + TargetArch::aarch64 + } else { + TargetArch::x86_64 + }; + let filter = SeccompFilter::new(rules, mismatch_action, SeccompAction::Allow, target_arch) + .map_err(|e| e.to_string())?; + let program: BpfProgram = match filter.try_into() { + Ok(prog) => prog, + Err(e) => return Err(e.to_string()), + }; + Ok(program) +} + +/// No-op stub where seccomp isn't applicable (macOS dev, Linux arches other +/// than x86_64/aarch64). +#[cfg(not(all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") +)))] +pub fn install_seccomp_filter(_enforce: bool) -> std::result::Result<(), String> { + Ok(()) +} + +// ============================================================================ +// Landlock filesystem restriction for the VM boot subprocess +// +// Confines the VMM's filesystem view so a guest→VMM escape cannot read or write +// host files outside what this specific VM needs: read+exec on the rootfs / libs +// / system dirs, read-write on this VM's own data dir + the device nodes a VMM +// uses. Other tenants' VM data, host secrets, and the rest of the filesystem +// become inaccessible. Best-effort and Linux-only (Landlock LSM); a no-op stub +// elsewhere. Paths are derived per-VM from the boot config by the caller. +// +// MUST be installed AFTER cgroup placement (which writes /sys/fs/cgroup) and +// BEFORE the seccomp filter (whose allowlist omits the landlock_* syscalls). +// ============================================================================ + +/// Restrict this process and its descendants to the given filesystem paths via +/// Landlock: `read_exec` paths get read+execute, `read_write` paths get full +/// access; everything else on the host becomes inaccessible. Missing paths are +/// silently skipped. Returns `Err` only if the ruleset can't be created/applied +/// at all (e.g. kernel without Landlock) — the caller decides fail-open/closed. +#[cfg(target_os = "linux")] +pub fn restrict_filesystem( + read_exec: &[std::path::PathBuf], + read_write: &[std::path::PathBuf], +) -> std::result::Result<(), String> { + use landlock::{ + Access, AccessFs, PathBeneath, PathFd, Ruleset, RulesetAttr, RulesetCreatedAttr, ABI, + }; + + // ABI V1 is supported on every Landlock-capable kernel; it governs + // read/write/execute and directory mutation — enough to confine file access. + let abi = ABI::V1; + let ro = AccessFs::from_read(abi); + let rw = AccessFs::from_all(abi); + + let mut ruleset = Ruleset::default() + .handle_access(AccessFs::from_all(abi)) + .map_err(|e| e.to_string())? + .create() + .map_err(|e| e.to_string())?; + + for p in read_exec { + if let Ok(fd) = PathFd::new(p) { + ruleset = ruleset + .add_rule(PathBeneath::new(fd, ro)) + .map_err(|e| e.to_string())?; + } + } + for p in read_write { + if let Ok(fd) = PathFd::new(p) { + ruleset = ruleset + .add_rule(PathBeneath::new(fd, rw)) + .map_err(|e| e.to_string())?; + } + } + + ruleset.restrict_self().map_err(|e| e.to_string())?; + Ok(()) +} + +/// No-op stub where Landlock isn't applicable (macOS dev). +#[cfg(not(target_os = "linux"))] +pub fn restrict_filesystem( + _read_exec: &[std::path::PathBuf], + _read_write: &[std::path::PathBuf], +) -> std::result::Result<(), String> { + Ok(()) +} + +// ============================================================================ +// Drop the VMM to an unprivileged uid (bounds escape blast radius) +// +// Run each VMM as a powerless, ideally per-VM uid so a guest→VMM escape can't +// signal/ptrace the serve process or other tenants' VMs, nor touch root-owned +// host files. Requires the spawning supervisor to be privileged +// (CAP_SETUID/SETGID — i.e. serve as root) AND this VM's data dir + disks + +// socket dir to be owned by `uid` (the VMM opens them after the drop). The +// `kvm` group is kept (supplementary) so /dev/kvm stays openable. +// +// Order in the boot path: AFTER cgroup placement (which needs privilege to write +// cgroup.procs) and BEFORE Landlock/seccomp (which work unprivileged once +// no_new_privs is set). See docs/runtime-isolation-hardening.md. +// ============================================================================ + +/// Look up the `kvm` group's gid, so it can be kept as a supplementary group +/// across the privilege drop (the VMM needs `/dev/kvm`). +#[cfg(target_os = "linux")] +fn kvm_group_gid() -> Option { + let grp = unsafe { libc::getgrnam(c"kvm".as_ptr()) }; + if grp.is_null() { + None + } else { + Some(unsafe { (*grp).gr_gid }) + } +} + +/// Irreversibly drop this process to (`uid`, `gid`), keeping only the `kvm` +/// supplementary group. Returns `Err` if any step fails so the caller can fail +/// closed — the VMM must never run with more privilege than requested. +/// +/// Uses `setgroups` → `setgid` → `setuid` (gid before uid, since `setgid` needs +/// privilege the `setuid` would shed), then verifies uid 0 cannot be regained. +#[cfg(target_os = "linux")] +pub fn drop_privileges(uid: u32, gid: u32) -> std::result::Result<(), String> { + unsafe { + // Keep only kvm as a supplementary group (drop all others). + let groups: Vec = kvm_group_gid().into_iter().collect(); + if libc::setgroups(groups.len(), groups.as_ptr()) != 0 { + return Err(format!("setgroups: {}", std::io::Error::last_os_error())); + } + if libc::setgid(gid as libc::gid_t) != 0 { + return Err(format!( + "setgid({gid}): {}", + std::io::Error::last_os_error() + )); + } + if libc::setuid(uid as libc::uid_t) != 0 { + return Err(format!( + "setuid({uid}): {}", + std::io::Error::last_os_error() + )); + } + // Defense-in-depth: a complete drop (real+effective+saved) means uid 0 + // can no longer be regained. If it can, the drop was partial — fail. + if uid != 0 && libc::setuid(0) == 0 { + return Err("privilege drop incomplete: regained uid 0".into()); + } + } + Ok(()) +} + +/// No-op stub where privilege dropping isn't applicable (macOS dev). +#[cfg(not(target_os = "linux"))] +pub fn drop_privileges(_uid: u32, _gid: u32) -> std::result::Result<(), String> { + Ok(()) +} + +// ============================================================================ +// Per-VM uid allocation (defense-in-depth for a guest→VMM escape) +// +// When the launcher runs privileged (root `serve`), each VMM is dropped to its +// own dedicated unprivileged uid so a guest→VMM escape is contained to that one +// VM: it can't ptrace/read other tenants' VMMs (different uid), reach their +// disks (different ownership), or touch root/host files. The uid is derived +// deterministically from the VM's data dir (stable across restarts, no +// allocation state), mapped into a high range well clear of system, regular, +// `nobody`, and systemd-DynamicUser uids. A fork clone uses its GOLDEN's uid so +// it can map the golden's guest-RAM memfd via /proc//fd (same uid). +// ============================================================================ + +/// Base of the reserved per-VM uid range. Above normal system (<1000), user +/// (1000–60000), `nobody` (65534), and systemd DynamicUser (61184–65519) uids. +#[cfg(target_os = "linux")] +const VM_UID_BASE: u32 = 2_000_000; +/// Span of the per-VM uid range: the allocator hands out the lowest free uid in +/// `[BASE, BASE+SPAN)`. 100M is far more than any node hosts at once. +#[cfg(target_os = "linux")] +const VM_UID_SPAN: u32 = 100_000_000; + +/// Whether per-VM uid isolation applies here: the launcher is privileged (so it +/// can chown + setuid) and the operator hasn't opted out with +/// `SMOLVM_VM_UID_DROP=off`. +#[cfg(target_os = "linux")] +pub fn vm_uid_drop_active() -> bool { + let is_root = unsafe { libc::geteuid() } == 0; + let opted_out = + std::env::var_os("SMOLVM_VM_UID_DROP").as_deref() == Some(std::ffi::OsStr::new("off")); + is_root && !opted_out +} + +/// No-op where the uid drop isn't applicable (macOS dev). +#[cfg(not(target_os = "linux"))] +pub fn vm_uid_drop_active() -> bool { + false +} + +/// Relocate all smolvm state under a single system data root by pointing `HOME` +/// at it before any path is computed — every `dirs::`-derived path (VM dirs, +/// agent rootfs, templates, server DB) follows. This is what lets per-VM uid +/// isolation's dropped uids traverse to their data (an XDG-under-a-700-home +/// layout can't). `SMOLVM_DATA_DIR` is honored for **every** command (so the CLI +/// and serve agree); `allow_auto` additionally defaults to `/var/lib/smolvm` when +/// privileged with the uid drop active and no XDG override (serve only — a +/// one-off root CLI invocation shouldn't silently switch roots). Must be called +/// single-threaded, before the tokio runtime, so `set_var` is safe. +#[cfg(target_os = "linux")] +pub fn apply_system_data_root(allow_auto: bool) { + let root = if let Some(explicit) = std::env::var_os("SMOLVM_DATA_DIR") { + std::path::PathBuf::from(explicit) + } else if allow_auto + && vm_uid_drop_active() + && std::env::var_os("XDG_CACHE_HOME").is_none() + && std::env::var_os("XDG_DATA_HOME").is_none() + { + std::path::PathBuf::from("/var/lib/smolvm") + } else { + return; + }; + match std::fs::create_dir_all(&root) { + Ok(()) => { + use std::os::unix::fs::PermissionsExt; + // 0755 so dropped VMM uids can traverse to their data. + let _ = std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o755)); + } + Err(e) => { + tracing::warn!(root = %root.display(), error = %e, "failed to create smolvm data root") + } + } + // Registry auth (crane/docker) falls back to `$HOME/.docker`, which we're about + // to move off the operator's real home — pin DOCKER_CONFIG to the ORIGINAL + // `~/.docker` (if it exists and the operator hasn't set DOCKER_CONFIG) so + // private image pulls keep finding their credentials after the relocation. + if std::env::var_os("DOCKER_CONFIG").is_none() { + if let Some(dir) = dirs::home_dir() + .map(|h| h.join(".docker")) + .filter(|d| d.is_dir()) + { + std::env::set_var("DOCKER_CONFIG", dir); + } + } + std::env::set_var("HOME", &root); + std::env::remove_var("XDG_CACHE_HOME"); + std::env::remove_var("XDG_DATA_HOME"); + std::env::remove_var("XDG_CONFIG_HOME"); + tracing::info!(data_root = %root.display(), "smolvm state rooted at a system data dir"); +} + +/// No-op where the data root isn't applicable (macOS dev). +#[cfg(not(target_os = "linux"))] +pub fn apply_system_data_root(_allow_auto: bool) {} + +/// Per-VM uid isolation needs every ancestor of the data root to be traversable +/// (others-execute) by the drop uid, or the dropped VMM can't reach its own +/// files (it fails with a cryptic readiness timeout). Returns the first ancestor +/// of `path` (walking up) that a non-owner can't traverse, or `None` if the whole +/// chain is fine. `serve` uses it to warn the operator up front. +#[cfg(target_os = "linux")] +pub fn first_nontraversable_ancestor(path: &std::path::Path) -> Option { + use std::os::unix::fs::PermissionsExt; + let mut cur = Some(path); + while let Some(dir) = cur { + if let Ok(meta) = std::fs::metadata(dir) { + if meta.is_dir() && meta.permissions().mode() & 0o001 == 0 { + return Some(dir.to_path_buf()); + } + } + cur = dir.parent(); + } + None +} + +/// No-op where the uid drop isn't applicable (macOS dev). +#[cfg(not(target_os = "linux"))] +pub fn first_nontraversable_ancestor(_path: &std::path::Path) -> Option { + None +} + +/// The VM key recorded in registry marker `registry_dir/`, if present. +#[cfg(target_os = "linux")] +fn uid_marker_key(registry_dir: &std::path::Path, uid: u32) -> Option { + std::fs::read_to_string(registry_dir.join(uid.to_string())) + .ok() + .map(|s| s.trim().to_string()) +} + +/// The uid already registered to `vm_key` (scan), or `None` if unallocated. +#[cfg(target_os = "linux")] +fn registered_uid(registry_dir: &std::path::Path, vm_key: &str) -> Option { + for e in std::fs::read_dir(registry_dir).ok()?.flatten() { + let uid = e.file_name().to_str().and_then(|s| s.parse::().ok()); + if let Some(uid) = uid { + if uid_marker_key(registry_dir, uid).as_deref() == Some(vm_key) { + return Some(uid); + } + } + } + None +} + +/// Allocate a stable, **collision-free** unprivileged uid for the VM identified +/// by `vm_key` (its data-dir hash), recording the assignment in `registry_dir` +/// so no two live VMs ever share a uid. Idempotent — the same key returns the +/// same uid until [`free_vm_uid`] releases it; the result is cached in +/// `key_dir/.vm-uid` to skip the scan on restart. Claims the lowest free uid in +/// the reserved range atomically (`O_EXCL`) so concurrent boots can't collide. +#[cfg(target_os = "linux")] +pub fn allocate_vm_uid( + registry_dir: &std::path::Path, + key_dir: &std::path::Path, + vm_key: &str, +) -> std::io::Result { + std::fs::create_dir_all(registry_dir)?; + let cache = key_dir.join(".vm-uid"); + // Fast path: a cached uid whose marker still belongs to us. + if let Some(uid) = std::fs::read_to_string(&cache) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + if uid_marker_key(registry_dir, uid).as_deref() == Some(vm_key) { + return Ok(uid); + } + } + // Registered under our key already (cache lost)? + if let Some(uid) = registered_uid(registry_dir, vm_key) { + let _ = std::fs::write(&cache, uid.to_string()); + return Ok(uid); + } + // Claim the lowest free uid atomically. A uid whose marker points at a VM + // whose data dir is gone is STALE — a delete path that didn't free it, or a + // crash — so reclaim it. This makes the registry self-healing across every + // delete path (no leak even if some path forgets to call `free_vm_uid`). The + // VM data dirs are the registry's sibling (`<…>/smolvm/uids` ⇄ `…/vms`); a + // marker only exists after its VM's data dir was created, so "marker present + // but data dir absent" reliably means deleted, never mid-boot. + let vms_dir = registry_dir.parent().map(|p| p.join("vms")); + for uid in VM_UID_BASE..VM_UID_BASE.saturating_add(VM_UID_SPAN) { + let marker = registry_dir.join(uid.to_string()); + if let Some(key) = uid_marker_key(registry_dir, uid) { + let live = vms_dir + .as_ref() + .map(|v| v.join(&key).exists()) + .unwrap_or(true); + if live { + continue; // held by a live VM + } + let _ = std::fs::remove_file(&marker); // stale → reclaim below + } + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&marker) + { + Ok(mut f) => { + use std::io::Write; + f.write_all(vm_key.as_bytes())?; + let _ = std::fs::write(&cache, uid.to_string()); + return Ok(uid); + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, // raced + Err(e) => return Err(e), + } + } + Err(std::io::Error::other("per-VM uid range exhausted")) +} + +/// Release the uid registered to the VM at `key_dir` (on VM delete) so it can be +/// reused. No-op if the VM had no uid (drop inactive, or a fork clone — which +/// shares its golden's uid and never claims its own). Linux-only. +#[cfg(target_os = "linux")] +pub fn free_vm_uid(registry_dir: &std::path::Path, key_dir: &std::path::Path) { + if let Some(uid) = std::fs::read_to_string(key_dir.join(".vm-uid")) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + let _ = std::fs::remove_file(registry_dir.join(uid.to_string())); + } +} + +/// No-op where the uid drop isn't applicable (macOS dev). +#[cfg(not(target_os = "linux"))] +pub fn free_vm_uid(_registry_dir: &std::path::Path, _key_dir: &std::path::Path) {} + +/// The `(uid, gid)` a VM's VMM should drop to, allocated **collision-free** from +/// `registry_dir`. Returns: +/// - `None` — the drop doesn't apply (unprivileged launcher, or +/// `SMOLVM_VM_UID_DROP=off`); boot proceeds without a drop. +/// - `Some(Err(_))` — the drop is **active but allocation failed**; the caller +/// MUST refuse to boot (fail closed — never silently run the VMM over- +/// privileged, the same contract as `drop_privileges`). +/// - `Some(Ok((uid, gid)))` — drop to this id. +/// +/// A fork clone (`snapshot_dir` set, laid out as +/// `/fork-snapshots/`) resolves to the GOLDEN's uid so it can +/// map the golden's memfd. gid mirrors uid (a per-VM group). +#[cfg(target_os = "linux")] +pub fn vm_drop_ids( + registry_dir: &std::path::Path, + data_dir: &std::path::Path, + snapshot_dir: Option<&std::path::Path>, +) -> Option> { + if !vm_uid_drop_active() { + return None; + } + let key_dir = match snapshot_dir { + Some(snap) => snap.parent().and_then(|p| p.parent()).unwrap_or(data_dir), + None => data_dir, + }; + let Some(vm_key) = key_dir.file_name().and_then(|n| n.to_str()) else { + return Some(Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "VM data dir name is not valid UTF-8", + ))); + }; + Some(allocate_vm_uid(registry_dir, key_dir, vm_key).map(|uid| (uid, uid))) +} + +/// No-op where the uid drop isn't applicable (macOS dev). +#[cfg(not(target_os = "linux"))] +pub fn vm_drop_ids( + _registry_dir: &std::path::Path, + _data_dir: &std::path::Path, + _snapshot_dir: Option<&std::path::Path>, +) -> Option> { + None +} + +/// Add others-execute to `dir` and every ancestor that lacks it, so a dropped +/// VMM uid can traverse to it. Execute-only (no read): traversal works but the +/// dirs can't be listed and file contents stay governed by their own perms. +/// Used under uid isolation for the data/rootfs/template path chains. Idempotent, +/// best-effort. Linux-only. +#[cfg(target_os = "linux")] +pub fn ensure_traversable(dir: &std::path::Path) { + use std::os::unix::fs::PermissionsExt; + let mut cur = Some(dir); + while let Some(d) = cur { + if d.as_os_str().is_empty() { + break; + } + if let Ok(meta) = std::fs::metadata(d) { + let mode = meta.permissions().mode(); + if meta.is_dir() && mode & 0o001 == 0 { + let _ = std::fs::set_permissions(d, std::fs::Permissions::from_mode(mode | 0o001)); + } + } + cur = d.parent(); + } +} + +/// No-op where the uid drop isn't applicable (macOS dev). +#[cfg(not(target_os = "linux"))] +pub fn ensure_traversable(_dir: &std::path::Path) {} + +/// Recursively `lchown` `path` to `(uid, gid)` (symlinks not followed). Used by +/// the privileged launcher to hand a VM's data dir + disks + sockets to the uid +/// its VMM will drop to. Linux-only; the caller is root. +#[cfg(target_os = "linux")] +pub fn chown_tree(path: &std::path::Path, uid: u32, gid: u32) -> std::io::Result<()> { + use std::os::unix::ffi::OsStrExt; + let c = std::ffi::CString::new(path.as_os_str().as_bytes()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + if unsafe { libc::lchown(c.as_ptr(), uid, gid) } != 0 { + return Err(std::io::Error::last_os_error()); + } + // Recurse into real directories only (not symlinked ones). + let meta = std::fs::symlink_metadata(path)?; + if meta.file_type().is_dir() { + for entry in std::fs::read_dir(path)? { + chown_tree(&entry?.path(), uid, gid)?; + } + } + Ok(()) +} + +/// No-op where chown isn't applicable (macOS dev). +#[cfg(not(target_os = "linux"))] +pub fn chown_tree(_path: &std::path::Path, _uid: u32, _gid: u32) -> std::io::Result<()> { + Ok(()) +} + +/// Build a user namespace whose single-entry uid/gid maps make on-disk id 0 +/// appear as `(uid, gid)` *through an idmapped mount*, and return an fd to its +/// `/proc//ns/user` (the open fd keeps the namespace alive after the helper +/// process exits). +/// +/// Mechanics (validated against kernels 6.17/6.18): fork a helper that +/// `unshare(CLONE_NEWUSER)`s and blocks; the parent (still privileged) writes the +/// child's `uid_map`/`gid_map`. A map line is ` `, and an +/// idmapped mount treats the ON-DISK id as an nsid and maps it DOWN to a host +/// kuid — so to surface on-disk 0 as host `uid` we write `0 1` (NOT +/// ` 0 1`, which surfaces the overflow uid and denies every read). The +/// parent MUST wait until the child is inside the new userns before writing the +/// maps (a two-pipe handshake), or the write races the `unshare` and fails EPERM. +/// `setgroups` is denied before `gid_map` (required to write a gid map). +#[cfg(target_os = "linux")] +fn make_idmap_userns(uid: u32, gid: u32) -> std::io::Result { + let errno = || std::io::Error::last_os_error(); + // ready: child -> parent (userns created); go: parent -> child (maps written). + let mut ready = [-1i32; 2]; + let mut go = [-1i32; 2]; + if unsafe { libc::pipe(ready.as_mut_ptr()) } != 0 { + return Err(errno()); + } + if unsafe { libc::pipe(go.as_mut_ptr()) } != 0 { + let e = errno(); + unsafe { + libc::close(ready[0]); + libc::close(ready[1]); + } + return Err(e); + } + let pid = unsafe { libc::fork() }; + if pid < 0 { + let e = errno(); + unsafe { + libc::close(ready[0]); + libc::close(ready[1]); + libc::close(go[0]); + libc::close(go[1]); + } + return Err(e); + } + if pid == 0 { + // CHILD: async-signal-safe calls only (post-fork in a possibly threaded + // process). Become a fresh userns, signal the parent, block until the maps + // are written, then exit — the parent's open ns fd keeps the userns alive. + if unsafe { libc::unshare(libc::CLONE_NEWUSER) } != 0 { + unsafe { libc::_exit(1) }; + } + let sig: [u8; 1] = *b"r"; + unsafe { + libc::write(ready[1], sig.as_ptr() as *const libc::c_void, 1); + } + let mut got = 0u8; + unsafe { + libc::read(go[0], &mut got as *mut u8 as *mut libc::c_void, 1); + libc::_exit(0); + } + } + + // PARENT. Close the ends we don't use, then wait for the child's "ready". + unsafe { + libc::close(ready[1]); + libc::close(go[0]); + } + let mut got = 0u8; + let _ = unsafe { libc::read(ready[0], &mut got as *mut u8 as *mut libc::c_void, 1) }; + + // Write the maps from the privileged parent. A single entry mapping host id + // `uid`/`gid` is permitted via the ns-creator's CAP_SETUID/CAP_SETGID path. + let finish = |result: std::io::Result| -> std::io::Result { + let go_w: [u8; 1] = *b"x"; + unsafe { + libc::write(go[1], go_w.as_ptr() as *const libc::c_void, 1); + libc::close(go[1]); + libc::close(ready[0]); + let mut status = 0; + libc::waitpid(pid, &mut status, 0); + } + result + }; + if let Err(e) = std::fs::write(format!("/proc/{pid}/uid_map"), format!("0 {uid} 1\n")) { + return finish(Err(e)); + } + // Must deny setgroups(2) before a gid_map may be written in a userns. + let _ = std::fs::write(format!("/proc/{pid}/setgroups"), "deny"); + if let Err(e) = std::fs::write(format!("/proc/{pid}/gid_map"), format!("0 {gid} 1\n")) { + return finish(Err(e)); + } + let ns_path = std::ffi::CString::new(format!("/proc/{pid}/ns/user")).unwrap(); + let nsfd = unsafe { libc::open(ns_path.as_ptr(), libc::O_RDONLY | libc::O_CLOEXEC) }; + if nsfd < 0 { + return finish(Err(errno())); + } + finish(Ok(nsfd)) +} + +/// Present the root-owned shared pack at `shared` onto the per-VM mountpoint +/// `target` via an idmapped bind mount that maps on-disk uid/gid 0 -> `(uid, gid)` +/// — so the VMM (about to drop to that uid) reads every file as its owner, while +/// the underlying shared copy stays root-only on disk (a sibling VM's uid can't +/// read it directly). This is what lets one root-owned shared copy replace the +/// per-machine extract + chown while preserving the per-VM uid isolation (#456). +/// +/// The mount is made in a fresh **private** mount namespace, so it is visible only +/// to this VMM process (and the libkrun threads it later spawns) and is torn down +/// automatically when the process exits — no teardown plumbing, no propagation to +/// the host or sibling VMs. MUST run while still privileged (CAP_SYS_ADMIN) and +/// BEFORE Landlock/seccomp/`drop_privileges`. Linux ≥ 5.12. +#[cfg(target_os = "linux")] +pub fn setup_pack_idmap_mount( + shared: &std::path::Path, + target: &std::path::Path, + uid: u32, + gid: u32, +) -> std::io::Result<()> { + use std::os::unix::ffi::OsStrExt; + let errno = || std::io::Error::last_os_error(); + + // Our own mount namespace: the idmap mount lives and dies with this process. + if unsafe { libc::unshare(libc::CLONE_NEWNS) } != 0 { + return Err(errno()); + } + // Don't propagate mounts back into the host's mount namespace. + if unsafe { + libc::mount( + std::ptr::null(), + c"/".as_ptr(), + std::ptr::null(), + libc::MS_REC | libc::MS_PRIVATE, + std::ptr::null(), + ) + } != 0 + { + return Err(errno()); + } + + let userns_fd = make_idmap_userns(uid, gid)?; + let close_userns = || unsafe { + libc::close(userns_fd); + }; + + let shared_c = std::ffi::CString::new(shared.as_os_str().as_bytes()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + let target_c = std::ffi::CString::new(target.as_os_str().as_bytes()) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + + // Clone the shared subtree into a detached mount object. mount_setattr's idmap + // requires a freshly-cloned mount with no other users, which OPEN_TREE_CLONE + // provides; AT_RECURSIVE covers any submounts. + let open_flags: libc::c_uint = libc::OPEN_TREE_CLONE + | libc::AT_RECURSIVE as libc::c_uint + | libc::O_CLOEXEC as libc::c_uint; + let tree = unsafe { + libc::syscall( + libc::SYS_open_tree, + libc::AT_FDCWD, + shared_c.as_ptr(), + open_flags, + ) + } as libc::c_int; + if tree < 0 { + let e = errno(); + close_userns(); + return Err(e); + } + let close_tree = || unsafe { + libc::close(tree); + }; + + // Attach the idmap (recursively) to the cloned tree. + let mut attr: libc::mount_attr = unsafe { std::mem::zeroed() }; + attr.attr_set = libc::MOUNT_ATTR_IDMAP; + attr.userns_fd = userns_fd as u64; + let rc = unsafe { + libc::syscall( + libc::SYS_mount_setattr, + tree, + c"".as_ptr(), + (libc::AT_EMPTY_PATH | libc::AT_RECURSIVE) as libc::c_uint, + &attr as *const libc::mount_attr, + std::mem::size_of::() as libc::size_t, + ) + }; + if rc != 0 { + let e = errno(); + close_tree(); + close_userns(); + return Err(e); + } + + // Move the now-idmapped tree onto the per-VM mountpoint. + let rc = unsafe { + libc::syscall( + libc::SYS_move_mount, + tree, + c"".as_ptr(), + libc::AT_FDCWD, + target_c.as_ptr(), + libc::MOVE_MOUNT_F_EMPTY_PATH as libc::c_uint, + ) + }; + let result = if rc != 0 { Err(errno()) } else { Ok(()) }; + // The kernel holds its own references now; our fds are no longer needed. + close_tree(); + close_userns(); + result +} + +/// PIDs of detached VM boot subprocesses to reap. Registered by +/// [`register_vm_child`] right after spawn (the boot subprocess is intentionally +/// detached — own process group, never `wait()`ed — so it becomes a zombie on +/// exit). Swept by [`reap_vm_children`] from the supervisor tick. +/// +/// This is the SELECTIVE reaper used by `serve`, replacing a global +/// `waitpid(-1)` SIGCHLD handler. An unscoped `waitpid(-1)` steals the exit +/// status from ANY exited child — including the `busctl` / `mkfs` / `resize2fs` +/// subprocesses that `.output()`/`.wait()` callers (e.g. systemd-scope adoption) +/// are actively waiting on — producing `ECHILD` ("No child processes") races +/// under concurrent VM boots. Reaping only registered VM PIDs leaves those +/// transient subprocesses to their own waits. Mirrors the guest agent's +/// `BG_CHILDREN` pattern (`crates/smolvm-agent/src/main.rs`). +static VM_CHILDREN: OnceLock>> = OnceLock::new(); + +fn vm_children() -> &'static Mutex> { + VM_CHILDREN.get_or_init(|| Mutex::new(Vec::new())) +} + +/// Track a detached VM boot subprocess PID so a later [`reap_vm_children`] sweep +/// reaps it. The Rust `Child` handle's `Drop` does not `wait()`, so the caller +/// can let it drop after recording the PID. +pub fn register_vm_child(pid: i32) { + vm_children().lock().unwrap().push(pid); +} + +/// Reap any exited registered VM children (non-blocking, per-PID). Called from +/// the serve supervisor tick. Scoped to registered PIDs so it never steals an +/// exit status from a sibling `.output()`/`.wait()` (the `ECHILD` fix). +#[cfg(target_os = "linux")] +pub fn reap_vm_children() { + let mut guard = vm_children().lock().unwrap(); + guard.retain(|&pid| { + let ret = unsafe { libc::waitpid(pid, std::ptr::null_mut(), libc::WNOHANG) }; + match ret { + // >0 = reaped; drop from tracking. + r if r > 0 => false, + // 0 = still running; keep for the next sweep. + 0 => true, + // <0 = error (typically ECHILD — already gone). Drop either way. + _ => false, + } + }); +} + +/// No-op on non-Linux: VM scope adoption + the serve supervisor reaper are +/// Linux-only; nothing registers VM children here. +#[cfg(not(target_os = "linux"))] +pub fn reap_vm_children() {} + +/// Install a GLOBAL SIGCHLD handler that reaps every terminated child via +/// `waitpid(-1, WNOHANG)`. Used only by the `pack_run` fork-pool paths (single +/// process, no concurrent `.output()` subprocesses racing it). +/// +/// **Do NOT use this in `serve`** — its concurrent VM boots run `busctl`/`mkfs` +/// `.output()` calls that this handler would reap out from under, causing +/// `ECHILD`. `serve` uses the selective [`register_vm_child`]/[`reap_vm_children`] +/// pair instead. +/// +/// The handler is only installed once; subsequent calls are no-ops. +/// +/// # Safety +/// +/// This function installs a signal handler which must be async-signal-safe. +/// The handler only calls waitpid() which is safe. +#[cfg(unix)] +pub fn install_sigchld_handler() { + // Only install once + if SIGCHLD_HANDLER_INSTALLED.swap(true, Ordering::SeqCst) { + return; + } + + unsafe { + let mut sa: libc::sigaction = std::mem::zeroed(); + sa.sa_sigaction = sigchld_handler as *const () as usize; + sa.sa_flags = libc::SA_RESTART | libc::SA_NOCLDSTOP; + libc::sigemptyset(&mut sa.sa_mask); + + if libc::sigaction(libc::SIGCHLD, &sa, std::ptr::null_mut()) != 0 { + // Failed to install handler, reset flag + SIGCHLD_HANDLER_INSTALLED.store(false, Ordering::SeqCst); + tracing::warn!("failed to install SIGCHLD handler"); + } else { + tracing::debug!("installed SIGCHLD handler for zombie reaping"); + } + } +} + +/// Windows has no SIGCHLD / zombie-reaping model; child cleanup is handled by +/// the OS when the last handle closes. This is a no-op. +#[cfg(not(unix))] +pub fn install_sigchld_handler() {} + +/// SIGCHLD signal handler that reaps zombie children. +/// +/// This handler is async-signal-safe as it only calls waitpid(). +#[cfg(unix)] +extern "C" fn sigchld_handler(_sig: libc::c_int) { + // Reap all terminated children (non-blocking) + // Loop until no more children to reap + loop { + let result = unsafe { libc::waitpid(-1, std::ptr::null_mut(), libc::WNOHANG) }; + if result <= 0 { + // No more children to reap (0) or error (-1) + break; + } + // Successfully reaped a child, continue to check for more + } +} + +/// Check if a process is alive. +/// +/// Returns true if the process exists and is running. +#[cfg(unix)] +pub fn is_alive(pid: Pid) -> bool { + unsafe { libc::kill(pid, 0) == 0 } +} + +/// Check if a process is alive (Windows). +/// +/// Opens the process with minimal rights and checks its exit code; a process +/// that is still running reports `STILL_ACTIVE`. Falls back to `false` if the +/// handle cannot be opened (process gone or access denied). +#[cfg(windows)] +pub fn is_alive(pid: Pid) -> bool { + win::process_is_alive(pid as u32) +} + +/// Wait for a process to exit (non-blocking check). +/// +/// Returns `Some(exit_code)` if the process has exited, `None` if still running. +/// Handles EINTR by retrying the waitpid call. +#[cfg(unix)] +pub fn try_wait(pid: Pid) -> Option { + loop { + let mut status: libc::c_int = 0; + let result = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + + if result == pid { + // Process exited + let exit_code = if libc::WIFEXITED(status) { + libc::WEXITSTATUS(status) + } else if libc::WIFSIGNALED(status) { + 128 + libc::WTERMSIG(status) + } else { + -1 + }; + return Some(exit_code); + } else if result < 0 { + let err = std::io::Error::last_os_error(); + if err.kind() == std::io::ErrorKind::Interrupted { + // EINTR - interrupted by signal, retry + continue; + } + // ECHILD: not our child (e.g., session-leader reparented to init). + // Return None so callers fall back to is_alive() (kill -0) polling. + return None; + } else { + // Still running + return None; + } + } +} + +/// Wait for a process to exit (non-blocking check) — Windows. +#[cfg(windows)] +pub fn try_wait(pid: Pid) -> Option { + win::process_try_wait(pid as u32) +} + +/// Wait for a process to exit (blocking). +/// +/// Returns the exit code. Handles EINTR by retrying the waitpid call. +#[cfg(unix)] +pub fn wait(pid: Pid) -> i32 { + loop { + let mut status: libc::c_int = 0; + let result = unsafe { libc::waitpid(pid, &mut status, 0) }; + + if result < 0 { + let err = std::io::Error::last_os_error(); + if err.kind() == std::io::ErrorKind::Interrupted { + // EINTR - interrupted by signal, retry + continue; + } + return -1; + } + + return if libc::WIFEXITED(status) { + libc::WEXITSTATUS(status) + } else if libc::WIFSIGNALED(status) { + 128 + libc::WTERMSIG(status) + } else { + -1 + }; + } +} + +/// Wait for a process to exit (blocking) — Windows. +#[cfg(windows)] +pub fn wait(pid: Pid) -> i32 { + win::process_wait(pid as u32) +} + +/// Send SIGTERM to a process. +/// +/// Returns true if the signal was sent successfully. +#[cfg(unix)] +pub fn terminate(pid: Pid) -> bool { + unsafe { libc::kill(pid, libc::SIGTERM) == 0 } +} + +/// Request termination of a process (Windows). +/// +/// There is no graceful SIGTERM equivalent; this calls `TerminateProcess`. +#[cfg(windows)] +pub fn terminate(pid: Pid) -> bool { + win::process_kill(pid as u32) +} + +/// Send SIGKILL to a process. +/// +/// Returns true if the signal was sent successfully. +#[cfg(unix)] +pub fn kill(pid: Pid) -> bool { + unsafe { libc::kill(pid, libc::SIGKILL) == 0 } +} + +/// Forcibly terminate a process (Windows) via `TerminateProcess`. +#[cfg(windows)] +pub fn kill(pid: Pid) -> bool { + win::process_kill(pid as u32) +} + +/// Get the start time of a process (seconds since epoch). +/// +/// Used alongside PID to create a stable process identity that survives +/// PID reuse. If the process at a given PID has a different start time +/// than expected, it's a different process (PID was recycled). +#[cfg(target_os = "macos")] +pub fn process_start_time(pid: Pid) -> Option { + // Use proc_pidinfo(PROC_PIDTBSDINFO) — the modern macOS API for + // process information, which has stable struct layouts. + extern "C" { + fn proc_pidinfo( + pid: libc::c_int, + flavor: libc::c_int, + arg: u64, + buffer: *mut libc::c_void, + buffersize: libc::c_int, + ) -> libc::c_int; + } + + const PROC_PIDTBSDINFO: libc::c_int = 3; + + /// Subset of `struct proc_bsdinfo` from . + #[repr(C)] + struct ProcBsdInfo { + pbi_flags: u32, + pbi_status: u32, + pbi_xstatus: u32, + pbi_pid: u32, + pbi_ppid: u32, + pbi_uid: u32, + pbi_gid: u32, + pbi_ruid: u32, + pbi_rgid: u32, + pbi_svuid: u32, + pbi_svgid: u32, + _rfu_1: u32, + pbi_comm: [u8; 16], // MAXCOMLEN + pbi_name: [u8; 32], // 2 * MAXCOMLEN + pbi_nfiles: u32, + pbi_pgid: u32, + pbi_pjobc: u32, + e_tdev: u32, + e_tpgid: u32, + pbi_nice: i32, + pbi_start_tvsec: u64, + pbi_start_tvusec: u64, + } + + let mut info: ProcBsdInfo = unsafe { std::mem::zeroed() }; + let ret = unsafe { + proc_pidinfo( + pid, + PROC_PIDTBSDINFO, + 0, + &mut info as *mut _ as *mut libc::c_void, + std::mem::size_of::() as libc::c_int, + ) + }; + if ret > 0 { + let usec = info.pbi_start_tvsec * 1_000_000 + info.pbi_start_tvusec; + // proc_pidinfo can return a zeroed struct for session-leader children + // (e.g., VM processes that called setsid()). Treat 0 as unavailable. + if usec > 0 { + Some(usec) + } else { + None + } + } else { + None + } +} + +/// Get the start time of a process (clock ticks since boot from /proc/pid/stat field 22). +#[cfg(target_os = "linux")] +pub fn process_start_time(pid: Pid) -> Option { + let stat = std::fs::read_to_string(format!("/proc/{}/stat", pid)).ok()?; + // Format: pid (comm) state ppid ... starttime ... + // comm can contain spaces and parentheses, so find the last ')' first. + let after_comm = stat.rfind(')')? + 2; + let fields: Vec<&str> = stat.get(after_comm..)?.split_whitespace().collect(); + // After ") ", fields are: state(0) ppid(1) ... starttime(19) + fields.get(19)?.parse::().ok() +} + +/// Get the start time of a process (Windows process creation FILETIME). +#[cfg(windows)] +pub fn process_start_time(pid: Pid) -> Option { + win::process_start_time(pid as u32) +} + +/// Get the start time of a process (stub for unsupported platforms). +#[cfg(not(any(target_os = "macos", target_os = "linux", windows)))] +pub fn process_start_time(_pid: Pid) -> Option { + None +} + +/// Sampled stats for one process. +#[derive(Debug, Clone, Copy)] +pub struct ProcessStats { + /// Cumulative CPU time (user + system) in nanoseconds since process start. + pub cpu_time_ns: u64, + /// Resident set size in bytes (physical memory currently held by the process). + pub rss_bytes: u64, +} + +/// Sample CPU time and RSS for a single process. Returns None if the PID is +/// dead or stats cannot be read. Both values are cumulative — caller must +/// compute deltas across samples to derive a rate (e.g., fractional CPUs). +#[cfg(target_os = "macos")] +pub fn process_stats(pid: Pid) -> Option { + extern "C" { + fn proc_pidinfo( + pid: libc::c_int, + flavor: libc::c_int, + arg: u64, + buffer: *mut libc::c_void, + buffersize: libc::c_int, + ) -> libc::c_int; + } + + const PROC_PIDTASKINFO: libc::c_int = 4; + + /// Subset of `struct proc_taskinfo` from . CPU times are in + /// mach_absolute_time units, which on Apple Silicon equal 1 nanosecond. + /// (For full portability we'd convert via mach_timebase_info; on arm64 + /// macOS the ratio is 1:1, and smolvm targets Apple Silicon.) + #[repr(C)] + struct ProcTaskInfo { + pti_virtual_size: u64, + pti_resident_size: u64, + pti_total_user: u64, + pti_total_system: u64, + pti_threads_user: u64, + pti_threads_system: u64, + pti_policy: i32, + pti_faults: i32, + pti_pageins: i32, + pti_cow_faults: i32, + pti_messages_sent: i32, + pti_messages_received: i32, + pti_syscalls_mach: i32, + pti_syscalls_unix: i32, + pti_csw: i32, + pti_threadnum: i32, + pti_numrunning: i32, + pti_priority: i32, + } + + let mut info: ProcTaskInfo = unsafe { std::mem::zeroed() }; + let ret = unsafe { + proc_pidinfo( + pid, + PROC_PIDTASKINFO, + 0, + &mut info as *mut _ as *mut libc::c_void, + std::mem::size_of::() as libc::c_int, + ) + }; + if ret <= 0 { + return None; + } + Some(ProcessStats { + cpu_time_ns: info.pti_total_user.saturating_add(info.pti_total_system), + rss_bytes: info.pti_resident_size, + }) +} + +/// Sample CPU time and RSS for a single process on Linux via /proc//{stat,statm}. +#[cfg(target_os = "linux")] +pub fn process_stats(pid: Pid) -> Option { + let stat = std::fs::read_to_string(format!("/proc/{}/stat", pid)).ok()?; + // Field 14 (utime) and 15 (stime) — both in clock ticks since process start. + let after_comm = stat.rfind(')')? + 2; + let fields: Vec<&str> = stat[after_comm..].split_whitespace().collect(); + let utime: u64 = fields.get(11)?.parse().ok()?; + let stime: u64 = fields.get(12)?.parse().ok()?; + let clock_ticks_per_sec = unsafe { libc::sysconf(libc::_SC_CLK_TCK) } as u64; + if clock_ticks_per_sec == 0 { + return None; + } + let cpu_time_ns = (utime + stime).saturating_mul(1_000_000_000) / clock_ticks_per_sec; + + let statm = std::fs::read_to_string(format!("/proc/{}/statm", pid)).ok()?; + let rss_pages: u64 = statm.split_whitespace().nth(1)?.parse().ok()?; + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as u64; + Some(ProcessStats { + cpu_time_ns, + rss_bytes: rss_pages.saturating_mul(page_size), + }) +} + +/// Sample CPU time and RSS for a process on Windows via the Win32 process APIs +/// (`GetProcessTimes` + `GetProcessMemoryInfo`). +#[cfg(windows)] +pub fn process_stats(pid: Pid) -> Option { + let (cpu_time_ns, rss_bytes) = win::process_stats(pid as u32)?; + Some(ProcessStats { + cpu_time_ns, + rss_bytes, + }) +} + +/// Sample CPU time and RSS for a process (stub on platforms without a +/// supported implementation). Always returns `None`. +#[cfg(not(any(target_os = "macos", target_os = "linux", windows)))] +pub fn process_stats(_pid: Pid) -> Option { + None +} + +/// Backward-compatible start time comparison. +/// +/// Handles the transition from seconds to microseconds on macOS: old records +/// stored seconds (~10^9), new code returns microseconds (~10^15). Values below +/// 10^12 are treated as seconds and compared at second granularity. +fn start_time_matches(actual: u64, expected: u64) -> bool { + if actual == expected { + return true; + } + // Backward compat: old macOS records stored seconds, new code uses microseconds. + // Seconds-epoch values are < 10^12; microsecond values are > 10^15. + if expected < 1_000_000_000_000 && actual >= 1_000_000_000_000 { + return actual / 1_000_000 == expected; + } + false +} + +/// Check if a PID belongs to our process by verifying start time. +/// +/// If `expected_start_time` is None (legacy records), falls back to PID-only check. +/// Use [`is_our_process_strict`] for signal/kill paths where false positives are dangerous. +pub fn is_our_process(pid: Pid, expected_start_time: Option) -> bool { + if !is_alive(pid) { + return false; + } + if let Some(expected) = expected_start_time { + match process_start_time(pid) { + Some(actual) => start_time_matches(actual, expected), + None => false, + } + } else { + // Legacy record without start time — assume ours for status checks + true + } +} + +/// Strict version of [`is_our_process`] for signal/kill paths. +/// +/// Returns `false` when start time is missing (legacy records) rather than +/// assuming the PID is ours. Prevents accidentally signaling an unrelated +/// process that reused the same PID. +pub fn is_our_process_strict(pid: Pid, expected_start_time: Option) -> bool { + if !is_alive(pid) { + return false; + } + match expected_start_time { + Some(expected) => match process_start_time(pid) { + Some(actual) => start_time_matches(actual, expected), + None => false, + }, + None => { + tracing::warn!( + pid, + "refusing to verify process without start time (legacy record)" + ); + false + } + } +} + +/// Send SIGTERM only if the PID still belongs to our process. +/// +/// Uses strict verification — refuses to signal without start time. +pub fn terminate_verified(pid: Pid, start_time: Option) -> bool { + if is_our_process_strict(pid, start_time) { + terminate(pid) + } else { + false + } +} + +/// Send SIGKILL only if the PID still belongs to our process. +/// +/// Uses strict verification — refuses to signal without start time. +pub fn kill_verified(pid: Pid, start_time: Option) -> bool { + if is_our_process_strict(pid, start_time) { + kill(pid) + } else { + false + } +} + +/// Identity fallback for our own VM subprocess when start-time can't be verified. +/// +/// A live VM is `smolvm-bin _boot-vm /boot-config.json`; that config +/// path is unique per VM, so an alive PID whose argv contains it is unambiguously +/// our VM — a recycled PID belonging to an unrelated process could not carry that +/// exact argument. Lets a teardown confidently kill a VM whose agent vsock is +/// wedged (so no shutdown ack) and whose start-time record is missing, instead of +/// leaking it as an untracked orphan. Returns false if the PID is dead or its +/// argv can't be read / doesn't match. +pub fn cmdline_contains(pid: Pid, needle: &str) -> bool { + if needle.is_empty() || !is_alive(pid) { + return false; + } + match read_cmdline(pid) { + Some(cmd) => cmd.contains(needle), + None => false, + } +} + +/// Read a process's argv as a single space-joined string (best-effort). +#[cfg(target_os = "linux")] +fn read_cmdline(pid: Pid) -> Option { + // /proc//cmdline is NUL-separated argv; join with spaces so a + // `contains` on a path substring works regardless of arg boundaries. + let raw = std::fs::read(format!("/proc/{pid}/cmdline")).ok()?; + if raw.is_empty() { + return None; + } + Some( + raw.split(|b| *b == 0) + .map(|s| String::from_utf8_lossy(s)) + .collect::>() + .join(" "), + ) +} + +/// Read a process's argv (macOS: via `ps -o command=`). Best-effort; the orphan +/// leak this guards is Linux-prod, so macOS just needs to not misfire. +#[cfg(not(target_os = "linux"))] +fn read_cmdline(pid: Pid) -> Option { + let out = std::process::Command::new("ps") + .args(["-p", &pid.to_string(), "-o", "command="]) + .output() + .ok()?; + if !out.status.success() { + return None; + } + let s = String::from_utf8_lossy(&out.stdout).trim().to_string(); + if s.is_empty() { + None + } else { + Some(s) + } +} + +/// Gracefully stop a process. +/// +/// 1. Sends SIGTERM +/// 2. Waits up to `timeout` for graceful exit +/// 3. If still running and `force` is true, sends SIGKILL +/// +/// Returns `Ok(exit_code)` on success, `Err` if timeout without force. +pub fn stop_process(pid: Pid, timeout: Duration, force: bool) -> Result { + // Check if already dead + if !is_alive(pid) { + // Try to reap zombie + if let Some(code) = try_wait(pid) { + return Ok(code); + } + return Ok(0); + } + + // Send SIGTERM + if !terminate(pid) { + // Process already dead - signal couldn't be sent. + // Try to get exit code; if unavailable (e.g., already reaped), use unknown. + return Ok(try_wait(pid).unwrap_or(UNKNOWN_EXIT_CODE)); + } + + // Wait for graceful exit + let start = Instant::now(); + let poll_interval = Duration::from_millis(100); + + while start.elapsed() < timeout { + if let Some(code) = try_wait(pid) { + return Ok(code); + } + + if !is_alive(pid) { + // Process died during wait - get exit code or return unknown. + return Ok(try_wait(pid).unwrap_or(UNKNOWN_EXIT_CODE)); + } + + std::thread::sleep(poll_interval); + } + + // Timeout reached + if force { + tracing::debug!(pid = pid, "SIGTERM timeout, sending SIGKILL"); + kill(pid); + + // Wait for SIGKILL to take effect + std::thread::sleep(SIGKILL_WAIT); + + // Reap the process + Ok(wait(pid)) + } else { + Err(Error::agent( + "stop process", + format!("timeout waiting for process {} to stop", pid), + )) + } +} + +/// Optimized process stop with aggressive polling for fast response. +/// +/// This version uses a two-phase polling strategy: +/// 1. Aggressive polling (10ms intervals) for the first 100ms +/// 2. Backs off to 100ms intervals for the remainder +/// +/// This minimizes latency for processes that exit quickly while +/// still being efficient for slower shutdowns. +pub fn stop_process_fast(pid: Pid, timeout: Duration, force: bool) -> Result { + // Check if already dead + if !is_alive(pid) { + if let Some(code) = try_wait(pid) { + return Ok(code); + } + return Ok(0); + } + + // Send SIGTERM + if !terminate(pid) { + return Ok(try_wait(pid).unwrap_or(UNKNOWN_EXIT_CODE)); + } + + // Two-phase polling: aggressive first, then back off + let start = Instant::now(); + let mut poll_count: u32 = 0; + + while start.elapsed() < timeout { + // Check immediately, then poll + if let Some(code) = try_wait(pid) { + return Ok(code); + } + + if !is_alive(pid) { + return Ok(try_wait(pid).unwrap_or(UNKNOWN_EXIT_CODE)); + } + + // Aggressive polling for first ~100ms, then back off + let poll_interval = if poll_count < FAST_POLL_COUNT { + FAST_POLL_INTERVAL // 10ms + } else { + Duration::from_millis(100) + }; + poll_count += 1; + + std::thread::sleep(poll_interval); + } + + // Timeout reached + if force { + tracing::debug!(pid = pid, "SIGTERM timeout, sending SIGKILL"); + kill(pid); + + // Brief wait then poll for exit + std::thread::sleep(SIGKILL_WAIT); + Ok(try_wait(pid).unwrap_or_else(|| wait(pid))) + } else { + Err(Error::agent( + "stop process", + format!("timeout waiting for process {} to stop", pid), + )) + } +} + +/// Default SIGTERM timeout for VM processes (3 seconds). +/// +/// Generous to accommodate guest shutdown + Hypervisor.framework teardown. +pub const VM_SIGTERM_TIMEOUT: Duration = Duration::from_secs(3); + +/// Default SIGKILL timeout for VM processes (3 seconds). +/// +/// On macOS, Hypervisor.framework VMs can be in uninterruptible kernel state +/// (`hv_vcpu_run`). Even SIGKILL may take 1-3 seconds while the kernel tears +/// down VM resources. This timeout must be long enough for that cleanup. +pub const VM_SIGKILL_TIMEOUT: Duration = Duration::from_secs(3); + +/// Stop a VM process with Hypervisor-aware timeouts. +/// +/// Two-phase shutdown: +/// 1. SIGTERM → poll up to `sigterm_timeout` with early exit +/// 2. If still alive: SIGKILL → poll up to `sigkill_timeout` with early exit +/// +/// Callers must verify process identity BEFORE calling this function. +/// +/// Returns `Ok(exit_code)` if the process exited, `Err` if still alive. +pub fn stop_vm_process( + pid: Pid, + sigterm_timeout: Duration, + sigkill_timeout: Duration, +) -> Result { + if !is_alive(pid) { + return Ok(try_wait(pid).unwrap_or(UNKNOWN_EXIT_CODE)); + } + + // Phase 1: SIGTERM + poll + if !terminate(pid) { + return Ok(try_wait(pid).unwrap_or(UNKNOWN_EXIT_CODE)); + } + + if let Some(code) = poll_for_exit(pid, sigterm_timeout) { + return Ok(code); + } + + // Phase 2: SIGKILL + poll + tracing::debug!(pid, "SIGTERM timeout, sending SIGKILL"); + kill(pid); + + if let Some(code) = poll_for_exit(pid, sigkill_timeout) { + return Ok(code); + } + + Err(Error::agent( + "stop vm process", + format!("process {} still alive after SIGTERM+SIGKILL", pid), + )) +} + +/// Poll for process exit with aggressive-then-backoff strategy. +/// +/// Returns `Some(exit_code)` if the process exits within the timeout. +fn poll_for_exit(pid: Pid, timeout: Duration) -> Option { + let start = Instant::now(); + let mut poll_count: u32 = 0; + + while start.elapsed() < timeout { + if let Some(code) = try_wait(pid) { + return Some(code); + } + if !is_alive(pid) { + return Some(try_wait(pid).unwrap_or(UNKNOWN_EXIT_CODE)); + } + + let interval = if poll_count < FAST_POLL_COUNT { + FAST_POLL_INTERVAL + } else { + Duration::from_millis(100) + }; + poll_count += 1; + std::thread::sleep(interval); + } + None +} + +/// Result of a fork operation. +#[derive(Debug)] +pub enum ForkResult { + /// This is the parent process. Contains the child's PID. + Parent(Pid), + /// This is the child process. + Child, +} + +/// Fork a child process that becomes a session leader. +/// +/// This function provides a safe interface to fork a child process and +/// have it call `setsid()` to become a session leader. This is commonly +/// used to detach VM processes from the parent's session so they survive +/// if the parent is killed. +/// +/// # Arguments +/// +/// * `child_fn` - A closure to run in the child process. The closure must +/// never return - it should either call `std::process::exit()` or exec +/// another program. +/// +/// # Returns +/// +/// * `Ok(pid)` - The child's PID if this is the parent process +/// * `Err` - If the fork failed +/// +/// # Example +/// +/// ```ignore +/// let child_pid = fork_session_leader(|| { +/// // This runs in the child process as a session leader +/// launch_vm(...); +/// std::process::exit(0); +/// })?; +/// ``` +#[cfg(unix)] +pub fn fork_session_leader(child_fn: F) -> Result +where + F: FnOnce(), +{ + // SAFETY: fork() creates a new process. The child inherits the parent's + // memory space as copy-on-write. We must be careful not to: + // - Hold any locks across fork (we don't) + // - Use async-signal-unsafe functions in the child before exec + // + // The child immediately calls setsid() and then the user-provided closure, + // which is expected to exec or exit. + let pid = unsafe { libc::fork() }; + + match pid { + -1 => { + // Fork failed + let err = std::io::Error::last_os_error(); + Err(Error::vm_creation(format!("fork failed: {}", err))) + } + 0 => { + // Child process + // + // SAFETY: setsid() is safe to call immediately after fork. + // It creates a new session and makes this process the session leader, + // detaching it from the parent's controlling terminal. + unsafe { + libc::setsid(); + } + + // Close inherited file descriptors to prevent holding parent's + // database locks, sockets, and other resources. Keep stdin(0), + // stdout(1), stderr(2) for error output during child setup. + // The child opens fresh fds for everything it needs. + close_inherited_fds_from(3); + + // Run the user-provided closure + child_fn(); + + // If the closure returns (it shouldn't), exit with error + // + // SAFETY: _exit() is safe in the child after fork. We use _exit() + // instead of exit() to avoid running atexit handlers and flushing + // stdio buffers that were inherited from the parent. + unsafe { + libc::_exit(1); + } + } + child_pid => { + // Parent process + Ok(child_pid) + } + } +} + +/// `fork()`-based session-leader launch is a Unix-only mechanism. The Windows +/// host never uses the in-process fork launch path; it always launches the VM +/// boot as a `smolvm _boot-vm` subprocess via `Command::new`. +#[cfg(not(unix))] +pub fn fork_session_leader(_child_fn: F) -> Result +where + F: FnOnce(), +{ + Err(Error::vm_creation( + "fork-based launch is not supported on Windows; use the subprocess launch path", + )) +} + +/// Redirect stdin, stdout, and stderr to `/dev/null`. +/// +/// Call this in a forked child process before launching a long-running +/// background task (e.g. a VM via `krun_start_enter`). Without this, +/// the child inherits the parent's terminal file descriptors and libkrun's +/// internal threads may read from stdin or set terminal attributes, +/// stealing keystrokes from the user's shell. +/// +/// Must be called **after** any `eprintln!()` diagnostics that need the +/// real stderr, but **before** the point of no return (`krun_start_enter`). +#[cfg(unix)] +pub fn detach_stdio() { + unsafe { + let devnull = libc::open(c"/dev/null".as_ptr(), libc::O_RDWR); + if devnull >= 0 { + libc::dup2(devnull, 0); // stdin + libc::dup2(devnull, 1); // stdout + libc::dup2(devnull, 2); // stderr + if devnull > 2 { + libc::close(devnull); + } + } + } +} + +/// No-op on Windows: there is no `/dev/null` fd-redirection model; the +/// subprocess launch path configures the child's stdio via `Command` instead. +#[cfg(not(unix))] +pub fn detach_stdio() {} + +/// Redirect stdin/stdout to `/dev/null` and stderr to a log file. +/// +/// This keeps background children detached from the user's terminal while +/// preserving boot-time diagnostics for later inspection. +#[cfg(unix)] +pub fn detach_stdio_to_stderr_file(path: &std::path::Path) -> std::io::Result<()> { + let stderr_file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + let stderr_fd = stderr_file.into_raw_fd(); + + unsafe { + let devnull = libc::open(c"/dev/null".as_ptr(), libc::O_RDWR); + if devnull < 0 { + libc::close(stderr_fd); + return Err(std::io::Error::last_os_error()); + } + + libc::dup2(devnull, 0); // stdin + libc::dup2(devnull, 1); // stdout + libc::dup2(stderr_fd, 2); // stderr + + if devnull > 2 { + libc::close(devnull); + } + if stderr_fd > 2 { + libc::close(stderr_fd); + } + } + + Ok(()) +} + +/// Windows: redirect this process's stderr to a log file via `SetStdHandle`. +/// +/// The fd-level `dup2` of stdin/stdout to `/dev/null` has no portable analog +/// and is unnecessary here (the boot subprocess is launched with inherited or +/// null handles by the parent `Command`); only the stderr log redirection is +/// reproduced so boot diagnostics are captured. +#[cfg(not(unix))] +pub fn detach_stdio_to_stderr_file(path: &std::path::Path) -> std::io::Result<()> { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::Console::{SetStdHandle, STD_ERROR_HANDLE}; + + let stderr_file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path)?; + // Keep the file handle alive for the process lifetime by leaking it; the + // OS handle now backs stderr. + let handle = stderr_file.as_raw_handle(); + std::mem::forget(stderr_file); + // SAFETY: `handle` is a valid file handle that we have intentionally leaked + // so it outlives this call and remains valid as the process's stderr. + let ok = unsafe { SetStdHandle(STD_ERROR_HANDLE, handle as _) }; + if ok == 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + +/// Exit the current process immediately without cleanup. +/// +/// On Unix this is a safe wrapper around `libc::_exit()` for use in forked +/// child processes — it avoids running atexit handlers and flushing stdio +/// buffers inherited from the parent. On Windows (no fork) it is a plain +/// process exit. +/// +/// # Safety +/// +/// This function never returns. +#[cfg(unix)] +pub fn exit_child(code: i32) -> ! { + // SAFETY: _exit() is safe in a forked child process. Using _exit() instead + // of exit() ensures we don't run atexit handlers or flush stdio buffers + // that were inherited from the parent process. + unsafe { + libc::_exit(code); + } +} + +/// Exit the current process immediately (Windows). No fork is involved, so a +/// normal `process::exit` is correct. +#[cfg(not(unix))] +pub fn exit_child(code: i32) -> ! { + std::process::exit(code) +} + +/// A handle to a running child process. +/// +/// Provides methods to check status, stop, and kill the process. +#[derive(Debug)] +pub struct ChildProcess { + pid: Pid, + /// Start time captured at creation for PID reuse detection. + start_time: Option, + exit_code: Option, +} + +impl ChildProcess { + /// Create a new child process handle, capturing start time immediately. + pub fn new(pid: Pid) -> Self { + Self { + pid, + start_time: process_start_time(pid), + exit_code: None, + } + } + + /// Get the process ID. + pub fn pid(&self) -> Pid { + self.pid + } + + /// Get the start time captured when this handle was created. + pub fn start_time(&self) -> Option { + self.start_time + } + + /// Check if the process is still running. + pub fn is_running(&mut self) -> bool { + if self.exit_code.is_some() { + return false; + } + + if let Some(code) = try_wait(self.pid) { + self.exit_code = Some(code); + false + } else { + is_alive(self.pid) + } + } + + /// Get the exit code if the process has exited. + pub fn exit_code(&mut self) -> Option { + if self.exit_code.is_none() { + self.exit_code = try_wait(self.pid); + } + self.exit_code + } + + /// Wait for the process to exit (blocking). + pub fn wait(&mut self) -> i32 { + if let Some(code) = self.exit_code { + return code; + } + + let code = wait(self.pid); + self.exit_code = Some(code); + code + } + + /// Send SIGTERM to the process. + pub fn terminate(&self) -> bool { + terminate(self.pid) + } + + /// Send SIGKILL to the process. + pub fn kill(&self) -> bool { + kill(self.pid) + } + + /// Gracefully stop the process. + /// + /// Sends SIGTERM, waits for `timeout`, then SIGKILL if `force` is true. + pub fn stop(&mut self, timeout: Duration, force: bool) -> Result { + if let Some(code) = self.exit_code { + return Ok(code); + } + + let code = stop_process(self.pid, timeout, force)?; + self.exit_code = Some(code); + Ok(code) + } +} + +// ============================================================================ +// SIGINT guard — kill a VM child process on Ctrl+C +// ============================================================================ + +/// PID for the SIGINT handler to kill on Ctrl+C. +/// Set by [`SigintGuard::new`], cleared on drop/disarm. +static SIGINT_CHILD_PID: std::sync::atomic::AtomicI32 = std::sync::atomic::AtomicI32::new(0); + +/// RAII guard that ensures a VM child process is killed on Ctrl+C. +/// +/// Without this, SIGINT terminates the parent immediately (default handler) +/// without running Rust destructors, so [`AgentManager::drop`] never fires +/// and the separate-process-group VM child is orphaned. +/// +/// The signal handler only calls `kill()` and `_exit()` (async-signal-safe). +pub struct SigintGuard(()); + +impl SigintGuard { + /// Install a SIGINT handler that will SIGTERM+SIGKILL the given PID. + #[cfg(unix)] + pub fn new(pid: Pid) -> Self { + SIGINT_CHILD_PID.store(pid, Ordering::SeqCst); + unsafe { + libc::signal( + libc::SIGINT, + sigint_kill_handler as *const () as libc::sighandler_t, + ); + } + Self(()) + } + + /// Windows: POSIX signal-handler installation has no direct equivalent and + /// the orphaned-process-group concern does not apply. The guard is inert. + #[cfg(not(unix))] + pub fn new(_pid: Pid) -> Self { + Self(()) + } + + /// Disarm the guard: clear the PID, restore default handler, skip Drop. + /// + /// Use when transitioning to a phase with its own SIGINT handling + /// (e.g., interactive exec). + pub fn disarm(self) { + SIGINT_CHILD_PID.store(0, Ordering::SeqCst); + #[cfg(unix)] + unsafe { + libc::signal(libc::SIGINT, libc::SIG_DFL); + } + std::mem::forget(self); + } +} + +impl Drop for SigintGuard { + fn drop(&mut self) { + SIGINT_CHILD_PID.store(0, Ordering::SeqCst); + #[cfg(unix)] + unsafe { + libc::signal(libc::SIGINT, libc::SIG_DFL); + } + } +} + +/// SIGINT handler: SIGTERM the child, brief busy-wait, escalate to SIGKILL, then _exit. +/// +/// SAFETY: Only calls `kill()` and `_exit()`, both async-signal-safe. +#[cfg(unix)] +extern "C" fn sigint_kill_handler(_sig: libc::c_int) { + let pid = SIGINT_CHILD_PID.load(Ordering::SeqCst); + if pid > 0 { + unsafe { + libc::kill(pid, libc::SIGTERM); + for _ in 0..10 { + if libc::kill(pid, 0) != 0 { + break; + } + } + if libc::kill(pid, 0) == 0 { + libc::kill(pid, libc::SIGKILL); + } + } + } + unsafe { + libc::_exit(128 + libc::SIGINT); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_alive_self() { + // Current process should be alive + let pid = unsafe { libc::getpid() }; + assert!(is_alive(pid)); + } + + #[test] + fn test_is_alive_nonexistent() { + // PID 99999999 is unlikely to exist + assert!(!is_alive(99999999)); + } + + #[test] + fn test_process_start_time_self() { + let pid = unsafe { libc::getpid() }; + let start_time = process_start_time(pid); + // On macOS and Linux this should return Some; on other platforms None + #[cfg(any(target_os = "macos", target_os = "linux"))] + assert!( + start_time.is_some(), + "should get start time on this platform" + ); + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + assert!(start_time.is_none()); + } + + #[test] + fn test_process_start_time_nonexistent() { + assert!(process_start_time(99999999).is_none()); + } + + #[test] + fn test_is_our_process_self() { + let pid = unsafe { libc::getpid() }; + let start_time = process_start_time(pid); + assert!(is_our_process(pid, start_time)); + } + + #[test] + fn test_is_our_process_wrong_start_time() { + let pid = unsafe { libc::getpid() }; + // A start time far in the future should not match + assert!(!is_our_process(pid, Some(u64::MAX))); + } + + #[test] + fn test_is_our_process_nonexistent() { + assert!(!is_our_process(99999999, None)); + assert!(!is_our_process(99999999, Some(12345))); + } + + #[test] + fn test_is_our_process_strict_requires_start_time() { + let pid = unsafe { libc::getpid() }; + // Strict refuses to verify without start time + assert!(!is_our_process_strict(pid, None)); + // But works with valid start time + let start_time = process_start_time(pid); + if start_time.is_some() { + assert!(is_our_process_strict(pid, start_time)); + } + } + + #[test] + fn test_start_time_matches_exact() { + assert!(start_time_matches(12345, 12345)); + assert!(!start_time_matches(12345, 12346)); + } + + #[test] + fn test_start_time_matches_backward_compat() { + // Old record stored seconds (1_700_000_000), new code returns microseconds + let old_seconds = 1_700_000_000u64; + let new_micros = old_seconds * 1_000_000 + 500_000; // same second, different usec + assert!(start_time_matches(new_micros, old_seconds)); + + // Different second should not match + assert!(!start_time_matches(new_micros, old_seconds + 1)); + } + + /// The seccomp allowlist must actually *deny* — a syscall outside it + /// (`kexec_load`, an escape-amplifying one we never allow) must kill the + /// process with SIGSYS. The allowed-path is covered by the live boot+exec + /// test on a Linux/KVM host. + #[cfg(all( + target_os = "linux", + any(target_arch = "x86_64", target_arch = "aarch64") + ))] + #[test] + fn seccomp_denies_forbidden_syscall() { + // Build the filter in the parent (allocating), then fork a child that + // applies it (allocation-free) and attempts the forbidden syscall. + let program = build_seccomp_program(true).expect("build seccomp program"); + unsafe { + let pid = libc::fork(); + assert!(pid >= 0, "fork failed"); + if pid == 0 { + if seccompiler::apply_filter(&program).is_err() { + libc::_exit(2); + } + libc::syscall(libc::SYS_kexec_load, 0, 0, 0, 0); + // Reached only if the filter did NOT kill us. + libc::_exit(0); + } + let mut status: libc::c_int = 0; + libc::waitpid(pid, &mut status, 0); + assert!( + libc::WIFSIGNALED(status) && libc::WTERMSIG(status) == libc::SIGSYS, + "forbidden syscall (kexec_load) should trigger SIGSYS, status={status:#x}" + ); + } + } + + /// Landlock must actually *deny* — after restricting to `/usr` only, opening + /// an ungranted path (`/etc/hostname`) must fail with EACCES. Runs in a forked + /// child so the restriction doesn't affect the test runner. + #[cfg(target_os = "linux")] + #[test] + fn landlock_denies_ungranted_path() { + use std::path::PathBuf; + unsafe { + let pid = libc::fork(); + assert!(pid >= 0, "fork failed"); + if pid == 0 { + // Landlock requires no_new_privs (set in production by harden_self). + libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + let ro = [PathBuf::from("/usr")]; + if restrict_filesystem(&ro, &[]).is_err() { + libc::_exit(3); // Landlock unavailable on this kernel — tolerate. + } + // /etc is NOT granted -> opening it must be denied. + let path = c"/etc/hostname"; + let fd = libc::open(path.as_ptr(), libc::O_RDONLY); + if fd < 0 { + let err = *libc::__errno_location(); + libc::_exit(if err == libc::EACCES { 0 } else { 4 }); + } + libc::_exit(5); // opened -> NOT restricted -> failure + } + let mut status: libc::c_int = 0; + libc::waitpid(pid, &mut status, 0); + let code = libc::WEXITSTATUS(status); + assert!( + code == 0 || code == 3, + "expected EACCES denial (0) or no-landlock (3), got exit {code}" + ); + } + } + + /// `drop_privileges` must fail closed: an unprivileged caller cannot drop to + /// another identity, so it returns Err (the boot path then refuses to run + /// over-privileged). `setgroups` fails first, so the test process's own + /// identity is left unchanged. Skipped when running as root (where it could + /// actually drop and disrupt the test runner). + #[cfg(target_os = "linux")] + #[test] + fn drop_privileges_fails_closed_without_capability() { + if unsafe { libc::geteuid() } == 0 { + return; // privileged runner — skip to avoid dropping the test process + } + assert!( + drop_privileges(1, 1).is_err(), + "unprivileged drop_privileges must fail (fail-closed contract)" + ); + // Sanity: our identity is unchanged (setgroups failed before any setuid). + assert_ne!( + unsafe { libc::getuid() }, + 1, + "drop must not have taken effect" + ); + } + + /// `(base, registry, vms)` laid out as production: `/smolvm/{uids,vms}`, + /// so the allocator's `registry.parent()/vms` resolves to `vms`. A VM's data + /// dir is `vms/`. + #[cfg(target_os = "linux")] + fn tmp_uid_dirs(tag: &str) -> (std::path::PathBuf, std::path::PathBuf, std::path::PathBuf) { + let base = + std::env::temp_dir().join(format!("smolvm-uidtest-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&base); + let reg = base.join("smolvm").join("uids"); + let vms = base.join("smolvm").join("vms"); + std::fs::create_dir_all(®).unwrap(); + std::fs::create_dir_all(&vms).unwrap(); + (base, reg, vms) + } + + #[cfg(target_os = "linux")] + #[test] + fn allocate_vm_uid_is_stable_collision_free_and_freeable() { + let (base, reg, vms) = tmp_uid_dirs("alloc"); + let a = vms.join("aaaa"); + let b = vms.join("bbbb"); + std::fs::create_dir_all(&a).unwrap(); + std::fs::create_dir_all(&b).unwrap(); + + let ua = allocate_vm_uid(®, &a, "aaaa").unwrap(); + let ub = allocate_vm_uid(®, &b, "bbbb").unwrap(); + // In range, distinct (collision-free), and stable on re-allocation. + assert!((VM_UID_BASE..VM_UID_BASE + VM_UID_SPAN).contains(&ua)); + assert_ne!(ua, ub, "distinct VMs must get distinct uids"); + assert_eq!( + ua, + allocate_vm_uid(®, &a, "aaaa").unwrap(), + "must be stable" + ); + + // Free A, then a new VM reuses A's released uid (lowest-free). + free_vm_uid(®, &a); + let c = vms.join("cccc"); + std::fs::create_dir_all(&c).unwrap(); + assert_eq!( + allocate_vm_uid(®, &c, "cccc").unwrap(), + ua, + "freed uid is reused" + ); + + let _ = std::fs::remove_dir_all(&base); + } + + #[cfg(target_os = "linux")] + #[test] + fn registered_uid_recovers_assignment_when_cache_lost() { + let (base, reg, vms) = tmp_uid_dirs("recover"); + let a = vms.join("key-a"); + std::fs::create_dir_all(&a).unwrap(); + let ua = allocate_vm_uid(®, &a, "key-a").unwrap(); + // Drop the per-VM cache; the registry still maps key -> uid. + let _ = std::fs::remove_file(a.join(".vm-uid")); + assert_eq!(allocate_vm_uid(®, &a, "key-a").unwrap(), ua); + let _ = std::fs::remove_dir_all(&base); + } + + #[cfg(target_os = "linux")] + #[test] + fn allocator_self_heals_leaked_uid_when_vm_dir_gone() { + let (base, reg, vms) = tmp_uid_dirs("selfheal"); + let a = vms.join("aaaa"); + std::fs::create_dir_all(&a).unwrap(); + let ua = allocate_vm_uid(®, &a, "aaaa").unwrap(); + + // Simulate a delete path that removed the VM's data dir WITHOUT calling + // free_vm_uid: the registry marker is now leaked. + std::fs::remove_dir_all(&a).unwrap(); + assert!(reg.join(ua.to_string()).exists(), "marker is leaked"); + + // A new VM reclaims that stale uid (its VM dir is gone) — no permanent + // leak even though the delete path forgot to free it. + let b = vms.join("bbbb"); + std::fs::create_dir_all(&b).unwrap(); + assert_eq!( + allocate_vm_uid(®, &b, "bbbb").unwrap(), + ua, + "stale (data-dir-gone) uid must be reclaimed" + ); + + let _ = std::fs::remove_dir_all(&base); + } + + /// The selective reaper drains registered VM PIDs once they exit, and a + /// non-child PID (would-be ECHILD) is dropped without affecting others. + #[cfg(target_os = "linux")] + #[test] + fn reap_vm_children_is_scoped_and_drains() { + // A real short-lived child: register its PID and forget the handle so the + // reaper (not Child::drop) reaps it. + let child = std::process::Command::new("true") + .spawn() + .expect("spawn true"); + let real_pid = child.id() as i32; + std::mem::forget(child); + register_vm_child(real_pid); + // A PID we never parented → waitpid returns ECHILD → must be dropped. + register_vm_child(i32::MAX); + + // Sweep until the registry drains (the real child exits ~immediately). + let mut drained = false; + for _ in 0..50 { + reap_vm_children(); + if vm_children().lock().unwrap().is_empty() { + drained = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!( + drained, + "registry should drain: real child reaped, bogus PID dropped on ECHILD" + ); + } + + /// `process_stats` must report live CPU/RSS for our own (alive) process on + /// every supported host — macOS, Linux, and Windows (where it previously + /// returned None). RSS is always > 0 for a running process; CPU time is + /// cumulative and may legitimately read 0 very early, so we only assert RSS. + #[cfg(any(target_os = "macos", target_os = "linux", windows))] + #[test] + fn process_stats_samples_self() { + let me = std::process::id() as Pid; + let stats = process_stats(me).expect("process_stats must sample the current process"); + assert!( + stats.rss_bytes > 0, + "a live process must have non-zero RSS, got {}", + stats.rss_bytes + ); + } + + /// A PID that does not exist must yield None, not a bogus sample. + #[cfg(any(target_os = "macos", target_os = "linux", windows))] + #[test] + fn process_stats_dead_pid_is_none() { + assert!( + process_stats(i32::MAX as Pid).is_none(), + "stats for a non-existent PID must be None" + ); + } + + /// A live process's argv is matchable by a unique substring, and a + /// non-matching needle / dead PID / empty needle all return false — the + /// identity fallback must be specific enough not to misfire. + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn cmdline_contains_matches_only_its_own_argv() { + let token = format!("smolvm-cmdline-probe-{}", std::process::id()); + // A long-lived process whose argv carries our unique token. The token is + // embedded in the `-c` script itself, and the trailing `; :` makes it a + // COMPOUND command so the shell can't exec-optimize into `sleep` (which + // would drop the shell's argv, and with it our token, on macOS). + let mut child = std::process::Command::new("sh") + .arg("-c") + .arg(format!("sleep 30; : {token}")) + .spawn() + .expect("spawn sh"); + let pid = child.id() as Pid; + // Give the OS a beat to expose argv. + std::thread::sleep(std::time::Duration::from_millis(50)); + + assert!( + cmdline_contains(pid, &token), + "argv containing the unique token must match" + ); + assert!( + !cmdline_contains(pid, "a-token-that-is-not-in-argv"), + "a non-matching needle must not match" + ); + assert!( + !cmdline_contains(pid, ""), + "an empty needle must never match" + ); + + let _ = child.kill(); + let _ = child.wait(); + assert!(!cmdline_contains(pid, &token), "a dead PID must not match"); + } +} diff --git a/src/registry.rs b/src/registry.rs new file mode 100644 index 0000000..7b15a49 --- /dev/null +++ b/src/registry.rs @@ -0,0 +1,1063 @@ +//! OCI registry credentials and reference parsing. +//! +//! This module provides: +//! - [`RegistryConfig`] — per-registry credential storage with env-var resolution +//! - [`Reference`] — OCI image reference parsing (registry/repo:tag@digest) +//! - Registry mirrors for pull-through caching +//! +//! Configuration is stored in `~/.config/smolvm/config.toml` under the +//! `[machines]` and `[images]` sections. See [`crate::settings::SmolSettings`]. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// Registry credentials and defaults for a set of OCI registries. +/// +/// Used as the `[machines]` and `[images]` sections within [`SmolSettings`](crate::settings::SmolSettings). +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RegistryConfig { + /// Per-registry configuration entries. + #[serde(default)] + pub registries: HashMap, + /// Default settings. + #[serde(default)] + pub defaults: RegistryDefaults, +} + +/// Configuration for a single registry. +/// +/// Two distinct auth paths — only one should be set per entry: +/// +/// **Identity token path** (`identity_token`): an upstream credential (e.g. Auth0 JWT) +/// exchanged with a token service per-operation to obtain a short-lived OCI bearer token. +/// Used for smolmachines registries where a Cloudflare token service sits in front. +/// `RegistryClient::with_identity_token` implements the exchange. +/// +/// **Direct bearer path** (`password` / `password_env`): an OCI bearer token sent +/// directly to the registry. Used for Docker Hub, GHCR, and other standard OCI +/// registries that accept static credentials. +/// +/// `identity_token` takes precedence over `password`/`password_env` when both are set. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RegistryEntry { + /// Username for authentication (direct bearer path only). + pub username: Option, + /// Direct OCI bearer token or password (not recommended for secrets; use password_env). + pub password: Option, + /// Environment variable containing the direct OCI bearer token or password. + pub password_env: Option, + /// Mirror URL to use instead of this registry. + pub mirror: Option, + /// Upstream identity credential (e.g. Auth0 JWT) exchanged with a token service + /// to obtain a short-lived OCI bearer token per-operation. + /// When set, takes precedence over password/password_env. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub identity_token: Option, + /// OAuth refresh token used to silently renew identity_token when it expires. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub refresh_token: Option, + /// Unix timestamp when identity_token expires. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expires_at: Option, +} + +/// Default registry settings. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct RegistryDefaults { + /// Default registry when none specified (defaults to docker.io). + pub registry: Option, +} + +// Re-export RegistryAuth from protocol to avoid duplication +pub use smolvm_protocol::RegistryAuth; + +impl RegistryConfig { + /// Get credentials for a registry, resolving environment variables. + /// + /// Returns `Some((username, password))` if credentials are configured and available. + /// Returns `None` if: + /// - No entry for this registry + /// - No username configured + /// - Password not available (env var not set, no direct password) + pub fn get_credentials(&self, registry: &str) -> Option { + let entry = self.registries.get(registry)?; + let username = entry.username.as_ref()?; + + // Try password_env first, then fall back to direct password + let password = entry + .password_env + .as_ref() + .and_then(|env| { + std::env::var(env).ok().or_else(|| { + tracing::debug!( + registry = %registry, + env_var = %env, + "password environment variable not set" + ); + None + }) + }) + .or_else(|| { + if entry.password.is_some() { + tracing::warn!( + registry = %registry, + "using plaintext password from config — use password_env instead" + ); + } + entry.password.clone() + })?; + + Some(RegistryAuth { + username: username.clone(), + password, + }) + } + + /// Get mirror URL for a registry if configured. + pub fn get_mirror(&self, registry: &str) -> Option<&str> { + self.registries.get(registry)?.mirror.as_deref() + } + + /// Get the default registry (defaults to "docker.io"). + pub fn default_registry(&self) -> &str { + self.defaults + .registry + .as_deref() + .unwrap_or(DEFAULT_REGISTRY) + } + + /// Check if any registries are configured. + pub fn has_registries(&self) -> bool { + !self.registries.is_empty() + } + + /// Set credentials for a registry, creating or updating the entry. + /// + /// Clears `password_env` when a direct password is provided. Preserves any + /// existing `mirror` setting so callers do not need to re-supply it. + pub fn set_credentials(&mut self, registry: &str, username: String, password: String) { + let entry = self.registries.entry(registry.to_string()).or_default(); + entry.username = Some(username); + entry.password = Some(password); + entry.password_env = None; + // Clear all upstream-credential fields. identity_token takes precedence + // over password in build_registry_client(); leaving a stale identity_token + // would silently ignore the new direct credentials. + entry.identity_token = None; + entry.refresh_token = None; + entry.expires_at = None; + } + + /// Set a token for a registry using the `username="token"` convention. + /// + /// Suitable for API keys and short-lived JWTs produced by `smol login`. + /// Delegates to [`Self::set_credentials`] so mirror is preserved. + pub fn set_token(&mut self, registry: &str, token: &str) { + self.set_credentials(registry, "token".to_string(), token.to_string()); + } + + /// Store an upstream identity credential (e.g. an Auth0 JWT) for the + /// registry. Unlike [`Self::set_token`], the credential is NOT sent to the + /// registry directly — `build_registry_client` exchanges it at the + /// registry's token service per operation (the OCI challenge flow), which + /// is the only form the smolmachines registry accepts. Clears any + /// direct-bearer fields so the exchange path always wins; preserves mirror. + pub fn set_identity_token(&mut self, registry: &str, token: &str) { + let entry = self.registries.entry(registry.to_string()).or_default(); + entry.identity_token = Some(token.to_string()); + entry.username = None; + entry.password = None; + entry.password_env = None; + } +} + +/// Default registry when none specified in image reference. +pub const DEFAULT_REGISTRY: &str = "docker.io"; + +/// Extract the registry hostname from an image reference. +/// +/// # Examples +/// +/// ```ignore +/// extract_registry("alpine") == "docker.io" +/// extract_registry("library/alpine") == "docker.io" +/// extract_registry("docker.io/library/alpine") == "docker.io" +/// extract_registry("ghcr.io/owner/repo") == "ghcr.io" +/// extract_registry("registry.example.com:5000/image") == "registry.example.com:5000" +/// ``` +pub fn extract_registry(image: &str) -> String { + // Check if the image starts with a registry (contains . or : before first /) + if let Some(slash_pos) = image.find('/') { + let potential_registry = &image[..slash_pos]; + + // A registry hostname contains a dot (.) or a port (:) + // This distinguishes "ghcr.io/owner/repo" from "library/alpine" + if potential_registry.contains('.') || potential_registry.contains(':') { + return potential_registry.to_string(); + } + } + + // No explicit registry - use default + DEFAULT_REGISTRY.to_string() +} + +/// Hosts to add to a hostname egress allow-list so a scoped machine can still +/// PULL this image in-guest on first boot. +/// +/// The DNS filter matches exact-or-subdomain, so a single apex covers a +/// registry's sub-hosts. Docker Hub is the exception: its manifests live under +/// `docker.io` but blobs are served from a `docker.com` CDN, so both apexes are +/// needed. Every other registry needs only its own host. +/// +/// Mirrors the control plane's `source_registry_hosts` so a machine's pull +/// behaves identically whether launched locally or via smolfleet. +pub fn registry_pull_hosts(image: &str) -> Vec { + match extract_registry(image).as_str() { + "docker.io" | "index.docker.io" | "registry-1.docker.io" => { + vec!["docker.io".to_string(), "docker.com".to_string()] + } + host if host.ends_with("smolmachines.com") => vec!["smolmachines.com".to_string()], + host => vec![host.to_string()], + } +} + +/// Rewrite an image reference to use a different registry. +/// +/// # Examples +/// +/// ```ignore +/// rewrite_image_registry("alpine", "mirror.example.com") == "mirror.example.com/library/alpine" +/// rewrite_image_registry("docker.io/library/alpine", "mirror.example.com") == "mirror.example.com/library/alpine" +/// rewrite_image_registry("ghcr.io/owner/repo", "mirror.example.com") == "mirror.example.com/owner/repo" +/// ``` +pub fn rewrite_image_registry(image: &str, new_registry: &str) -> String { + let current_registry = extract_registry(image); + + if image.starts_with(¤t_registry) { + // Explicit registry - replace it + format!("{}{}", new_registry, &image[current_registry.len()..]) + } else { + // Implicit docker.io - need to add "library/" for single-name images + if image.contains('/') { + format!("{}/{}", new_registry, image) + } else { + format!("{}/library/{}", new_registry, image) + } + } +} + +/// Default registry for smolmachines artifacts. +pub const SMOLMACHINES_REGISTRY: &str = "registry.smolmachines.com"; + +/// The control-plane API paired with [`SMOLMACHINES_REGISTRY`] — its OCI token +/// realm (`/v2/auth`) and identity endpoint (`/v1/me`) live here. Overridable +/// via the `SMOL_CLOUD_API` env var for self-hosted control planes. +pub const SMOLMACHINES_API: &str = "https://api.smolmachines.com"; + +/// Error parsing an artifact reference. +#[derive(Debug, Clone, PartialEq)] +pub struct ReferenceError { + /// The original input that failed to parse. + pub input: String, + /// What went wrong. + pub reason: String, +} + +impl std::fmt::Display for ReferenceError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "invalid reference '{}': {}", self.input, self.reason) + } +} + +impl std::error::Error for ReferenceError {} + +/// A parsed OCI-style reference for smolmachine artifacts. +/// +/// Supports both tag and digest references: +/// ```text +/// smolmachines.com/python-dev:latest # tag +/// smolmachines.com/python-dev@sha256:abc123... # digest (immutable) +/// smolmachines.com/binsquare/custom:v1 # user namespace +/// python-dev:latest # shorthand (default registry) +/// python-dev # bare (default registry + "latest") +/// ``` +#[derive(Debug, Clone, PartialEq)] +pub struct Reference { + /// Registry hostname (e.g., "registry.smolmachines.com"). + pub registry: String, + /// User namespace, if present (e.g., "binsquare"). None for official. + pub namespace: Option, + /// Machine name (e.g., "python-dev"). + pub name: String, + /// Tag (e.g., "latest", "v1.0"). None if digest is set. + pub tag: Option, + /// Digest (e.g., "sha256:abc123..."). None if tag is set. + pub digest: Option, +} + +impl Reference { + /// Parse a reference string. + /// + /// Returns a [`ReferenceError`] if the reference is empty or malformed. + pub fn parse(input: &str) -> std::result::Result { + let raw_input = input; + let input = input.trim(); + + let err = |reason: &str| ReferenceError { + input: raw_input.to_string(), + reason: reason.to_string(), + }; + + if input.is_empty() { + return Err(err("empty reference")); + } + + // Split off digest (@sha256:...) or tag (:tag) from the end. + // Digest takes precedence: `image@sha256:...` is a digest ref even if + // there's a `:` in the name part. + let (path, tag, digest) = if let Some(at_pos) = input.find('@') { + let path = &input[..at_pos]; + let digest_str = &input[at_pos + 1..]; + validate_digest(raw_input, digest_str)?; + (path, None, Some(digest_str.to_string())) + } else { + // No digest — check for tag after the last colon. + // But we must not split on `:` inside a registry hostname with port + // (e.g., "localhost:5000/image:tag"). + // Strategy: find the last `/`, then look for `:` after it. + let tag_split_from = input.rfind('/').map(|p| p + 1).unwrap_or(0); + let after_last_slash = &input[tag_split_from..]; + + if let Some(colon_pos) = after_last_slash.find(':') { + let abs_colon = tag_split_from + colon_pos; + let path = &input[..abs_colon]; + let tag = &input[abs_colon + 1..]; + if tag.is_empty() { + return Err(err("empty tag")); + } + (path, Some(tag.to_string()), None) + } else { + (input, None, None) + } + }; + + // Now parse the path into registry / namespace / name. + let parts: Vec<&str> = path.split('/').collect(); + + let (registry, namespace, name) = match parts.len() { + 1 => { + // "python-dev" — bare name, default registry + ( + SMOLMACHINES_REGISTRY.to_string(), + None, + parts[0].to_string(), + ) + } + 2 => { + let first = parts[0]; + if first.contains('.') || first.contains(':') { + // "smolmachines.com/python-dev" — registry + name (official) + (first.to_string(), None, parts[1].to_string()) + } else { + // "binsquare/custom" — namespace + name (default registry) + ( + SMOLMACHINES_REGISTRY.to_string(), + Some(first.to_string()), + parts[1].to_string(), + ) + } + } + 3 => { + // "smolmachines.com/binsquare/custom" — registry + namespace + name + let first = parts[0]; + if !first.contains('.') && !first.contains(':') { + return Err(ReferenceError { + input: raw_input.to_string(), + reason: format!( + "first component '{}' doesn't look like a registry hostname", + first + ), + }); + } + ( + first.to_string(), + Some(parts[1].to_string()), + parts[2].to_string(), + ) + } + n => { + // 4+ components — only valid when the first component is an explicit + // registry hostname (contains '.' or ':'). All middle components + // become the namespace, preserving arbitrary repository depth. + // Examples: + // ghcr.io/org/team/machine:latest → ns="org/team", name="machine" + // us-docker.pkg.dev/proj/repo/img → ns="proj/repo", name="img" + // localhost:5000/a/b/c:dev → ns="a/b", name="c" + let first = parts[0]; + if !first.contains('.') && !first.contains(':') { + return Err(ReferenceError { + input: raw_input.to_string(), + reason: format!( + "first component '{}' doesn't look like a registry hostname", + first + ), + }); + } + let name = parts[n - 1].to_string(); + let namespace = parts[1..n - 1].join("/"); + (first.to_string(), Some(namespace), name) + } + }; + + if name.is_empty() { + return Err(err("empty name")); + } + + // Reject empty path components (e.g. ghcr.io/org//machine from a double slash). + if let Some(ns) = &namespace { + for component in ns.split('/') { + if component.is_empty() { + return Err(err("empty repository component (check for double slashes)")); + } + } + } + + Ok(Reference { + registry, + namespace, + name, + tag, + digest, + }) + } + + /// The full repository path (namespace/name or just name). + pub fn repository(&self) -> String { + match &self.namespace { + Some(ns) => format!("{}/{}", ns, self.name), + None => self.name.clone(), + } + } +} + +impl std::fmt::Display for Reference { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let repo = self.repository(); + let suffix = if let Some(ref d) = self.digest { + format!("@{}", d) + } else if let Some(ref t) = self.tag { + format!(":{}", t) + } else { + ":latest".to_string() + }; + write!(f, "{}/{}{}", self.registry, repo, suffix) + } +} + +/// Validate a digest string: must be `sha256:` followed by exactly 64 hex chars. +fn validate_digest(raw_input: &str, digest: &str) -> std::result::Result<(), ReferenceError> { + let hex = match digest.strip_prefix("sha256:") { + Some(h) => h, + None => { + return Err(ReferenceError { + input: raw_input.to_string(), + reason: format!( + "unsupported digest algorithm in '{}': only sha256 is supported", + digest + ), + }); + } + }; + + if hex.len() != 64 { + return Err(ReferenceError { + input: raw_input.to_string(), + reason: format!("digest has {} hex chars, expected 64", hex.len()), + }); + } + + if !hex.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(ReferenceError { + input: raw_input.to_string(), + reason: "digest contains non-hex characters".to_string(), + }); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_registry_implicit_dockerhub() { + assert_eq!(extract_registry("alpine"), "docker.io"); + assert_eq!(extract_registry("alpine:latest"), "docker.io"); + assert_eq!(extract_registry("library/alpine"), "docker.io"); + assert_eq!(extract_registry("myuser/myimage"), "docker.io"); + } + + #[test] + fn test_extract_registry_explicit() { + assert_eq!(extract_registry("docker.io/library/alpine"), "docker.io"); + assert_eq!(extract_registry("ghcr.io/owner/repo"), "ghcr.io"); + assert_eq!(extract_registry("gcr.io/project/image"), "gcr.io"); + assert_eq!( + extract_registry("registry.example.com/image"), + "registry.example.com" + ); + assert_eq!(extract_registry("localhost:5000/image"), "localhost:5000"); + } + + #[test] + fn test_registry_pull_hosts_dockerhub_two_apexes() { + // Docker Hub serves blobs from a docker.com CDN, so both apexes are + // needed; a bare/library ref defaults to Docker Hub. + let expect = vec!["docker.io".to_string(), "docker.com".to_string()]; + assert_eq!(registry_pull_hosts("alpine"), expect); + assert_eq!(registry_pull_hosts("library/alpine:3.19"), expect); + assert_eq!(registry_pull_hosts("docker.io/library/nginx"), expect); + assert_eq!(registry_pull_hosts("registry-1.docker.io/foo/bar"), expect); + } + + #[test] + fn test_registry_pull_hosts_explicit_registry() { + // A single apex covers the registry's sub-hosts via subdomain matching. + assert_eq!(registry_pull_hosts("ghcr.io/acme/app:1"), vec!["ghcr.io"]); + assert_eq!( + registry_pull_hosts("registry.example.com:5000/img"), + vec!["registry.example.com:5000"] + ); + } + + #[test] + fn test_registry_pull_hosts_smolmachines_apex() { + assert_eq!( + registry_pull_hosts("registry.smolmachines.com/lib/alpine"), + vec!["smolmachines.com"] + ); + } + + #[test] + fn test_rewrite_image_registry() { + // Implicit docker.io + assert_eq!( + rewrite_image_registry("alpine", "mirror.example.com"), + "mirror.example.com/library/alpine" + ); + assert_eq!( + rewrite_image_registry("myuser/myimage", "mirror.example.com"), + "mirror.example.com/myuser/myimage" + ); + + // Explicit registry + assert_eq!( + rewrite_image_registry("docker.io/library/alpine", "mirror.example.com"), + "mirror.example.com/library/alpine" + ); + assert_eq!( + rewrite_image_registry("ghcr.io/owner/repo", "mirror.example.com"), + "mirror.example.com/owner/repo" + ); + } + + #[test] + fn test_registry_config_default() { + let config = RegistryConfig::default(); + assert!(config.registries.is_empty()); + assert_eq!(config.default_registry(), "docker.io"); + } + + #[test] + fn test_get_credentials_with_direct_password() { + let mut config = RegistryConfig::default(); + config.registries.insert( + "docker.io".to_string(), + RegistryEntry { + username: Some("testuser".to_string()), + password: Some("testpass".to_string()), + password_env: None, + mirror: None, + ..Default::default() + }, + ); + + let creds = config.get_credentials("docker.io"); + assert!(creds.is_some()); + let creds = creds.unwrap(); + assert_eq!(creds.username, "testuser"); + assert_eq!(creds.password, "testpass"); + } + + #[test] + fn test_get_credentials_missing_username() { + let mut config = RegistryConfig::default(); + config.registries.insert( + "docker.io".to_string(), + RegistryEntry { + username: None, + password: Some("testpass".to_string()), + password_env: None, + mirror: None, + ..Default::default() + }, + ); + + assert!(config.get_credentials("docker.io").is_none()); + } + + #[test] + fn test_get_credentials_missing_password() { + let mut config = RegistryConfig::default(); + config.registries.insert( + "docker.io".to_string(), + RegistryEntry { + username: Some("testuser".to_string()), + password: None, + password_env: None, + mirror: None, + ..Default::default() + }, + ); + + assert!(config.get_credentials("docker.io").is_none()); + } + + #[test] + fn test_get_mirror() { + let mut config = RegistryConfig::default(); + config.registries.insert( + "docker.io".to_string(), + RegistryEntry { + username: None, + password: None, + password_env: None, + mirror: Some("mirror.example.com".to_string()), + ..Default::default() + }, + ); + + assert_eq!(config.get_mirror("docker.io"), Some("mirror.example.com")); + assert_eq!(config.get_mirror("ghcr.io"), None); + } + + #[test] + fn test_parse_config() { + let toml_content = r#" +[defaults] +registry = "docker.io" + +[registries."docker.io"] +username = "myuser" +password_env = "DOCKER_TOKEN" + +[registries."ghcr.io"] +username = "github_user" +password = "direct_password" +mirror = "ghcr-mirror.example.com" +"#; + + let config: RegistryConfig = toml::from_str(toml_content).unwrap(); + assert_eq!(config.registries.len(), 2); + assert_eq!(config.default_registry(), "docker.io"); + + let docker_entry = config.registries.get("docker.io").unwrap(); + assert_eq!(docker_entry.username.as_deref(), Some("myuser")); + assert_eq!(docker_entry.password_env.as_deref(), Some("DOCKER_TOKEN")); + + let ghcr_entry = config.registries.get("ghcr.io").unwrap(); + assert_eq!(ghcr_entry.username.as_deref(), Some("github_user")); + assert_eq!(ghcr_entry.password.as_deref(), Some("direct_password")); + assert_eq!( + ghcr_entry.mirror.as_deref(), + Some("ghcr-mirror.example.com") + ); + } + + #[test] + fn test_get_credentials_with_env_password() { + // Set environment variable for this test + std::env::set_var("SMOLVM_TEST_TOKEN", "env_password_123"); + + let mut config = RegistryConfig::default(); + config.registries.insert( + "test.io".to_string(), + RegistryEntry { + username: Some("envuser".to_string()), + password: None, + password_env: Some("SMOLVM_TEST_TOKEN".to_string()), + mirror: None, + ..Default::default() + }, + ); + + let creds = config.get_credentials("test.io"); + assert!(creds.is_some()); + let creds = creds.unwrap(); + assert_eq!(creds.username, "envuser"); + assert_eq!(creds.password, "env_password_123"); + + // Clean up + std::env::remove_var("SMOLVM_TEST_TOKEN"); + } + + #[test] + fn test_get_credentials_env_var_not_set() { + let mut config = RegistryConfig::default(); + config.registries.insert( + "test.io".to_string(), + RegistryEntry { + username: Some("user".to_string()), + password: None, + password_env: Some("SMOLVM_NONEXISTENT_VAR".to_string()), + mirror: None, + ..Default::default() + }, + ); + + // Should return None when env var is not set + assert!(config.get_credentials("test.io").is_none()); + } + + #[test] + fn test_has_registries() { + let mut config = RegistryConfig::default(); + assert!(!config.has_registries()); + + config + .registries + .insert("docker.io".to_string(), RegistryEntry::default()); + assert!(config.has_registries()); + } + + #[test] + fn test_extract_registry_edge_cases() { + // Image with tag containing colon (version) + assert_eq!(extract_registry("alpine:3.18.0"), "docker.io"); + + // Image with digest + assert_eq!(extract_registry("alpine@sha256:abc123"), "docker.io"); + + // Registry with port and path + assert_eq!( + extract_registry("registry.example.com:5000/myorg/myimage:latest"), + "registry.example.com:5000" + ); + } + + #[test] + fn test_rewrite_image_registry_with_tag() { + assert_eq!( + rewrite_image_registry("alpine:3.18", "mirror.example.com"), + "mirror.example.com/library/alpine:3.18" + ); + + assert_eq!( + rewrite_image_registry("nginx:latest", "mirror.example.com"), + "mirror.example.com/library/nginx:latest" + ); + } + + #[test] + fn test_default_registry_custom() { + let mut config = RegistryConfig::default(); + config.defaults.registry = Some("custom.registry.io".to_string()); + assert_eq!(config.default_registry(), "custom.registry.io"); + } + + // --- Reference parser tests --- + + #[test] + fn test_reference_bare_name() { + let r = Reference::parse("python-dev").unwrap(); + assert_eq!(r.registry, SMOLMACHINES_REGISTRY); + assert_eq!(r.namespace, None); + assert_eq!(r.name, "python-dev"); + assert_eq!(r.tag, None); + assert_eq!(r.digest, None); + } + + #[test] + fn test_reference_name_with_tag() { + let r = Reference::parse("python-dev:latest").unwrap(); + assert_eq!(r.registry, SMOLMACHINES_REGISTRY); + assert_eq!(r.namespace, None); + assert_eq!(r.name, "python-dev"); + assert_eq!(r.tag, Some("latest".to_string())); + assert_eq!(r.digest, None); + } + + #[test] + fn test_reference_registry_and_name() { + let r = Reference::parse("smolmachines.com/python-dev:latest").unwrap(); + assert_eq!(r.registry, "smolmachines.com"); + assert_eq!(r.namespace, None); + assert_eq!(r.name, "python-dev"); + assert_eq!(r.tag, Some("latest".to_string())); + } + + #[test] + fn test_reference_registry_namespace_name() { + let r = Reference::parse("smolmachines.com/binsquare/custom:v1").unwrap(); + assert_eq!(r.registry, "smolmachines.com"); + assert_eq!(r.namespace, Some("binsquare".to_string())); + assert_eq!(r.name, "custom"); + assert_eq!(r.tag, Some("v1".to_string())); + } + + #[test] + fn test_reference_namespace_without_registry() { + let r = Reference::parse("binsquare/custom:v1").unwrap(); + assert_eq!(r.registry, SMOLMACHINES_REGISTRY); + assert_eq!(r.namespace, Some("binsquare".to_string())); + assert_eq!(r.name, "custom"); + assert_eq!(r.tag, Some("v1".to_string())); + } + + #[test] + fn test_reference_digest() { + let digest = "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + let input = format!("python-dev@{}", digest); + let r = Reference::parse(&input).unwrap(); + assert_eq!(r.registry, SMOLMACHINES_REGISTRY); + assert_eq!(r.name, "python-dev"); + assert_eq!(r.tag, None); + assert_eq!(r.digest, Some(digest.to_string())); + } + + #[test] + fn test_reference_registry_with_port() { + let r = Reference::parse("localhost:5000/myimage:latest").unwrap(); + assert_eq!(r.registry, "localhost:5000"); + assert_eq!(r.namespace, None); + assert_eq!(r.name, "myimage"); + assert_eq!(r.tag, Some("latest".to_string())); + } + + #[test] + fn test_reference_display() { + let r = Reference::parse("python-dev:latest").unwrap(); + assert_eq!( + r.to_string(), + format!("{}/python-dev:latest", SMOLMACHINES_REGISTRY) + ); + + let r = Reference::parse("python-dev").unwrap(); + // Bare name gets :latest in display + assert_eq!( + r.to_string(), + format!("{}/python-dev:latest", SMOLMACHINES_REGISTRY) + ); + } + + #[test] + fn test_reference_display_digest() { + let digest = "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + let input = format!("smolmachines.com/python-dev@{}", digest); + let r = Reference::parse(&input).unwrap(); + assert_eq!( + r.to_string(), + format!("smolmachines.com/python-dev@{}", digest) + ); + } + + #[test] + fn test_reference_error_empty() { + let err = Reference::parse("").unwrap_err(); + assert_eq!(err.reason, "empty reference"); + assert!(Reference::parse(" ").is_err()); + } + + #[test] + fn test_reference_error_invalid_digest_algorithm() { + let err = Reference::parse("python-dev@md5:abc123").unwrap_err(); + assert!(err.reason.contains("unsupported digest algorithm")); + } + + #[test] + fn test_reference_error_digest_too_short() { + let err = Reference::parse("python-dev@sha256:tooshort").unwrap_err(); + assert!(err.reason.contains("hex chars, expected 64")); + } + + #[test] + fn test_reference_error_digest_non_hex() { + // 64 chars but contains 'g' which is not hex + let bad = format!( + "python-dev@sha256:{}", + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234567g9" + ); + let err = Reference::parse(&bad).unwrap_err(); + assert!(err.reason.contains("non-hex")); + } + + #[test] + fn test_reference_error_empty_tag() { + let err = Reference::parse("python-dev:").unwrap_err(); + assert_eq!(err.reason, "empty tag"); + } + + #[test] + fn test_reference_deep_path_ghcr() { + // ghcr.io/org/team/machine:latest + let r = Reference::parse("ghcr.io/org/team/machine:latest").unwrap(); + assert_eq!(r.registry, "ghcr.io"); + assert_eq!(r.namespace, Some("org/team".to_string())); + assert_eq!(r.name, "machine"); + assert_eq!(r.tag, Some("latest".to_string())); + assert_eq!(r.repository(), "org/team/machine"); + } + + #[test] + fn test_reference_deep_path_gcr() { + // us-docker.pkg.dev/project/repo/image:tag (GCR Artifact Registry style) + let r = Reference::parse("us-docker.pkg.dev/project/repo/image:v2").unwrap(); + assert_eq!(r.registry, "us-docker.pkg.dev"); + assert_eq!(r.namespace, Some("project/repo".to_string())); + assert_eq!(r.name, "image"); + assert_eq!(r.tag, Some("v2".to_string())); + assert_eq!(r.repository(), "project/repo/image"); + } + + #[test] + fn test_reference_deep_path_localhost() { + // localhost:5000/a/b/c:dev + let r = Reference::parse("localhost:5000/a/b/c:dev").unwrap(); + assert_eq!(r.registry, "localhost:5000"); + assert_eq!(r.namespace, Some("a/b".to_string())); + assert_eq!(r.name, "c"); + assert_eq!(r.tag, Some("dev".to_string())); + } + + #[test] + fn test_reference_deep_path_explicit_registry_required() { + // 4+ components without a registry hostname — must fail + let err = Reference::parse("org/team/repo/image:latest").unwrap_err(); + assert!( + err.reason.contains("doesn't look like a registry hostname"), + "unexpected reason: {}", + err.reason + ); + } + + #[test] + fn test_reference_rejects_empty_path_components() { + // Double slashes produce empty components — must be caught + let err = Reference::parse("ghcr.io/org//machine:latest").unwrap_err(); + assert!( + err.reason.contains("empty repository component"), + "unexpected reason: {}", + err.reason + ); + } + + // ── set_credentials / set_token / save ────────────────────────────────── + + #[test] + fn test_set_credentials_new_entry() { + let mut config = RegistryConfig::default(); + config.set_credentials("registry.example.com", "user".into(), "pass".into()); + + let creds = config.get_credentials("registry.example.com").unwrap(); + assert_eq!(creds.username, "user"); + assert_eq!(creds.password, "pass"); + } + + #[test] + fn test_set_credentials_overwrites_password_env() { + let mut config = RegistryConfig::default(); + config.registries.insert( + "registry.example.com".to_string(), + RegistryEntry { + username: Some("old".to_string()), + password: None, + password_env: Some("OLD_TOKEN_ENV".to_string()), + mirror: Some("mirror.example.com".to_string()), + ..Default::default() + }, + ); + + config.set_credentials("registry.example.com", "new".into(), "newpass".into()); + + let entry = config.registries.get("registry.example.com").unwrap(); + assert_eq!(entry.username.as_deref(), Some("new")); + assert_eq!(entry.password.as_deref(), Some("newpass")); + assert_eq!(entry.password_env, None, "password_env must be cleared"); + assert_eq!( + entry.mirror.as_deref(), + Some("mirror.example.com"), + "mirror must be preserved" + ); + } + + #[test] + fn test_set_credentials_clears_identity_token() { + // If an entry previously had an identity_token, switching to direct credentials + // via set_credentials must clear it. identity_token takes precedence in + // build_registry_client(); a stale one would silently ignore the new password. + let mut config = RegistryConfig::default(); + config.registries.insert( + "registry.smolmachines.com".to_string(), + RegistryEntry { + identity_token: Some("eyJ_old_jwt".to_string()), + refresh_token: Some("old_refresh".to_string()), + expires_at: Some(9999999999), + ..Default::default() + }, + ); + + config.set_credentials( + "registry.smolmachines.com", + "user".into(), + "direct_bearer".into(), + ); + + let entry = config.registries.get("registry.smolmachines.com").unwrap(); + assert_eq!(entry.password.as_deref(), Some("direct_bearer")); + assert_eq!( + entry.identity_token, None, + "stale identity_token must be cleared" + ); + assert_eq!( + entry.refresh_token, None, + "stale refresh_token must be cleared" + ); + assert_eq!(entry.expires_at, None, "stale expires_at must be cleared"); + } + + #[test] + fn test_set_token_uses_token_username() { + let mut config = RegistryConfig::default(); + config.set_token("registry.smolmachines.com", "eyJhbGci.test"); + + let creds = config.get_credentials("registry.smolmachines.com").unwrap(); + assert_eq!(creds.username, "token"); + assert_eq!(creds.password, "eyJhbGci.test"); + } + + #[test] + fn test_save_roundtrip_preserves_all_fields() { + let mut config = RegistryConfig::default(); + config.defaults.registry = Some("custom.io".to_string()); + config.registries.insert( + "ghcr.io".to_string(), + RegistryEntry { + username: Some("gh_user".to_string()), + password: Some("gh_pass".to_string()), + password_env: None, + mirror: Some("ghcr-mirror.example.com".to_string()), + ..Default::default() + }, + ); + + let serialized = toml::to_string_pretty(&config).unwrap(); + let reloaded: RegistryConfig = toml::from_str(&serialized).unwrap(); + + assert_eq!(reloaded.default_registry(), "custom.io"); + let entry = reloaded.registries.get("ghcr.io").unwrap(); + assert_eq!(entry.username.as_deref(), Some("gh_user")); + assert_eq!(entry.password.as_deref(), Some("gh_pass")); + assert_eq!(entry.mirror.as_deref(), Some("ghcr-mirror.example.com")); + } +} diff --git a/src/secrets.rs b/src/secrets.rs new file mode 100644 index 0000000..a7f8ef7 --- /dev/null +++ b/src/secrets.rs @@ -0,0 +1,700 @@ +//! Host-side secret *references*. +//! +//! smolvm does not store secret material. A [`SecretRef`] is a pointer to +//! a secret that already lives somewhere on the host — a host environment +//! variable (`from_env`) or a host file (`from_file`) — and resolution +//! reads that source fresh at workload launch, injecting the value into +//! the guest env as part of the existing `env` field in +//! `AgentRequest::Run`/`VmExec` (no protocol change). +//! +//! Bring your own secrets manager: have Vault/1Password/AWS/sops/your +//! shell render the secret into an env var or a file, then reference it. +//! Because resolution is late-bound (per launch), rotating the underlying +//! source takes effect on the next run with nothing to re-sync. +//! +//! # Scope and non-goals +//! +//! This is defense-in-depth, not zero-knowledge. The target guest process +//! sees plaintext in its own environment. Isolation comes from: +//! (a) never persisting resolved plaintext to the VM record / DB / pack, +//! (b) not logging values, (c) scrubbing the immediate resolve buffers. +//! +//! # What Zeroize actually covers here +//! +//! Resolved plaintext lives in a `Zeroizing` from the moment it's +//! read until the resolution helper copies it into the caller-visible env +//! vector; those intermediate buffers scrub on drop. After that copy, +//! plaintext lives in a regular `String` inside a `Vec<(String, String)>` +//! until the outer request is serialized to vsock. We do not attempt to +//! zero that storage — doing so would not change the fundamental property +//! that the guest process's env contains plaintext. If you need true +//! "use without access," use a broker pattern (e.g. SSH agent forwarding). + +use crate::error::{Error, Result}; +use std::collections::BTreeMap; +use zeroize::Zeroizing; + +/// Maximum number of bytes read when resolving a `from_file` source. A +/// file larger than this is almost never a credential; we refuse rather +/// than silently injecting a huge env var into the workload. +pub const MAX_FROM_FILE_BYTES: u64 = 1024 * 1024; + +// The *shape* of a ref lives in `smolvm-protocol` (it's what flows +// across wire / on-disk boundaries). The *policy* — which source kinds +// are allowed at which trust boundary — lives here, alongside the code +// that enforces it at validation and resolution time. +pub use smolvm_protocol::{SecretRef, SecretSourceKind}; + +// ============================================================================ +// File-source reading +// ============================================================================ + +/// Structured error from bounded-read / file-source helpers so +/// callers can classify without string-matching the inner `io::Error`. +#[derive(Debug)] +enum FileSourceError { + /// The stored path points at a symlink; we refuse to follow. Only + /// constructed on Unix (via `O_NOFOLLOW`/`ELOOP`); Windows omits the check. + #[cfg_attr(not(unix), allow(dead_code))] + Symlink, + /// File size exceeded the per-call cap. + TooLarge, + /// Any other I/O failure (missing, permission denied, non-UTF-8 + /// content, etc.). The inner error is available for callers that + /// want to log it; classifiers treat this as `FileReadFailed`. + Io(std::io::Error), +} + +impl std::fmt::Display for FileSourceError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Symlink => write!( + f, + "from_file path is a symlink; refusing to follow \ + (store the canonicalized target instead)" + ), + Self::TooLarge => write!(f, "file size exceeds maximum"), + Self::Io(e) => write!(f, "{}", e), + } + } +} + +impl From for FileSourceError { + fn from(e: std::io::Error) -> Self { + Self::Io(e) + } +} + +/// Read a file to string while refusing to allocate more than `cap` +/// bytes. The cap is enforced against *bytes actually read*, not against +/// the length reported by `metadata()` — this closes a TOCTOU where the +/// file grows between stat and read. +fn read_file_bounded( + mut f: std::fs::File, + cap: u64, +) -> std::result::Result { + use std::io::Read; + let mut buf = String::new(); + // `take` caps the actual read. We request `cap + 1` so an over-cap + // file fills the reader by one byte, letting us detect the breach. + (&mut f).take(cap + 1).read_to_string(&mut buf)?; + if buf.len() as u64 > cap { + return Err(FileSourceError::TooLarge); + } + Ok(buf) +} + +/// Read a `from_file` source with both the size cap and a symlink refusal. +/// +/// The leaf is opened with `O_NOFOLLOW` so a symlink at the final path +/// component is refused ATOMICALLY at open time — there is no lstat-then-open +/// window where the target could be swapped for a symlink to (e.g.) +/// `/etc/shadow`. We then read from that fd only, never re-opening by path, so +/// the bytes come from exactly what we opened. (Parent-directory symlinks are +/// not covered — that would need a component-by-component `openat` walk.) +fn read_from_file_source(path: &std::path::Path) -> std::result::Result { + let mut opts = std::fs::OpenOptions::new(); + opts.read(true); + // O_NOFOLLOW refuses a symlink leaf atomically at open time. There is no + // portable Windows equivalent here; Windows symlink creation is privileged + // and the threat model (swapping the leaf for a symlink to /etc/shadow) is + // POSIX-specific, so the flag is simply omitted there. + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.custom_flags(libc::O_NOFOLLOW); + } + let f = opts.open(path).map_err(|e| { + // O_NOFOLLOW on a symlink leaf fails with ELOOP — surface it as the + // dedicated Symlink class rather than a generic I/O error. + #[cfg(unix)] + if e.raw_os_error() == Some(libc::ELOOP) { + return FileSourceError::Symlink; + } + FileSourceError::Io(e) + })?; + read_file_bounded(f, MAX_FROM_FILE_BYTES) +} + +/// Map a [`FileSourceError`] into the resolution-failure taxonomy. No +/// string matching; the classes are structural, so this is a pure enum +/// translation. +fn classify_file_source_error(e: FileSourceError) -> ResolutionFailure { + match e { + FileSourceError::TooLarge => ResolutionFailure::FileTooLarge, + FileSourceError::Symlink | FileSourceError::Io(_) => ResolutionFailure::FileReadFailed, + } +} + +// ============================================================================ +// Trust scope and validation +// ============================================================================ + +/// Trust level of the actor that supplied a [`SecretRef`]. +/// +/// The smolvm process resolves secrets against its own host. Different +/// input surfaces present refs with different trust; the scope chosen at +/// the validation site determines which source kinds can be honored. +/// +/// | Scope | `from_env` | `from_file` | +/// |---|:-:|:-:| +/// | `TrustedLocal` | yes | yes (absolute paths only) | +/// | `RecordReplay` | yes | yes | +/// | `Untrusted` | no | no | +/// +/// Both source kinds dereference *this host's* environment / filesystem, +/// so they are only meaningful for a trusted-local actor (the CLI running +/// as the host user) or for refs that actor persisted earlier +/// (`RecordReplay`). An `Untrusted` surface — an HTTP request body or a +/// portable `.smolmachine` pack authored elsewhere — must not be able to +/// read this host's env (`from_env`) or files (`from_file`), so it can +/// carry no resolvable secret at all. +/// +/// Callers must call [`validate_ref`] before acting on a ref received +/// from any source. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResolutionScope { + /// Caller is trusted equivalently to the smolvm process itself + /// (typically: the CLI running as the host user). All source kinds + /// are accepted; `from_file` still requires an absolute path for + /// defense-in-depth against CWD-dependent surprises. + TrustedLocal, + + /// Ref was persisted by a `TrustedLocal` actor in a prior session + /// (e.g., read back from a VM record). Trust is preserved across time. + RecordReplay, + + /// Ref came in from an unauthenticated or semi-trusted source: an + /// HTTP request body, or a portable pack manifest. Neither source + /// kind is honored — `from_env` would leak the smolvm process's + /// environment and `from_file` would turn the ref field into an + /// arbitrary host-file read primitive. + Untrusted, +} + +impl ResolutionScope { + fn allows_env(self) -> bool { + !matches!(self, Self::Untrusted) + } + fn allows_file(self) -> bool { + !matches!(self, Self::Untrusted) + } +} + +/// Structural or scope-policy violations rejected by [`validate_ref`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SecretRefError { + /// Neither `from_env` nor `from_file` was set. + NoSource, + /// More than one source field was set. + MultipleSources, + /// `from_file` was given a relative path. Persisted refs must be + /// absolute to avoid CWD-dependent resolution surprises. + RelativeFilePath(std::path::PathBuf), + /// The source kind is valid in general but not in the given scope — + /// e.g., `from_file` submitted via an untrusted HTTP request body. + SourceNotAllowedInScope { + /// Which source kind was attempted. + kind: SecretSourceKind, + /// Which scope rejected it. + scope: ResolutionScope, + }, +} + +impl std::fmt::Display for SecretRefError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NoSource => write!( + f, + "no source set: exactly one of from_env, from_file must be specified" + ), + Self::MultipleSources => write!( + f, + "multiple sources set: exactly one of from_env, from_file must be specified" + ), + Self::RelativeFilePath(p) => write!( + f, + "from_file path '{}' is relative; absolute paths are required", + p.display() + ), + Self::SourceNotAllowedInScope { kind, scope } => write!( + f, + "source kind '{}' is not allowed in scope {:?}; \ + secret refs may only be resolved on the trusted local host", + kind.as_str(), + scope + ), + } + } +} + +impl std::error::Error for SecretRefError {} + +/// Validate structure and scope policy for a [`SecretRef`]. +/// +/// Call before persisting or acting on any ref received from an outside +/// source. There are no "partial" refs in this codebase — if it wasn't +/// validated, it shouldn't be stored or resolved. +pub fn validate_ref( + r: &SecretRef, + scope: ResolutionScope, +) -> std::result::Result<(), SecretRefError> { + let count = [r.from_env.is_some(), r.from_file.is_some()] + .into_iter() + .filter(|b| *b) + .count(); + + match count { + 0 => return Err(SecretRefError::NoSource), + 1 => {} + _ => return Err(SecretRefError::MultipleSources), + } + + if r.from_env.is_some() && !scope.allows_env() { + return Err(SecretRefError::SourceNotAllowedInScope { + kind: SecretSourceKind::Env, + scope, + }); + } + if let Some(path) = &r.from_file { + if !scope.allows_file() { + return Err(SecretRefError::SourceNotAllowedInScope { + kind: SecretSourceKind::File, + scope, + }); + } + if !path.is_absolute() { + return Err(SecretRefError::RelativeFilePath(path.clone())); + } + } + + Ok(()) +} + +// ============================================================================ +// Resolution failure taxonomy +// ============================================================================ + +/// Classified reasons a resolution can fail. +/// +/// Distinct from [`SecretRefError`]: `SecretRefError` is a validation +/// failure (the ref itself is malformed or rejected by policy), while +/// [`ResolutionFailure`] happens *after* validation, when we actually +/// try to turn an accepted ref into a value. +/// +/// The classification drives two things: audit log records and HTTP +/// status-code mapping for the API (client-side 400 vs. server-side 500). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResolutionFailure { + /// `from_env` references an env var that isn't set on the host. + EnvUnset, + /// `from_file` path is missing, not readable, or not a regular + /// file. Also covers the "target is a symlink" refusal. + FileReadFailed, + /// `from_file` target exceeds the size cap. + FileTooLarge, + /// Any other failure: an internal/unexpected condition (e.g. a ref + /// with no source reaching resolution despite validation). + Internal, +} + +impl ResolutionFailure { + /// Whether the failure reflects something the caller can fix. + /// Determines whether the HTTP layer should return 4xx or 5xx. + pub fn is_client_error(self) -> bool { + matches!( + self, + Self::EnvUnset | Self::FileReadFailed | Self::FileTooLarge + ) + } + + /// Stable short identifier suitable for logs and public-facing + /// error payloads. Never includes path, env-var name, or value. + pub fn as_str(self) -> &'static str { + match self { + Self::EnvUnset => "env_unset", + Self::FileReadFailed => "file_read_failed", + Self::FileTooLarge => "file_too_large", + Self::Internal => "internal", + } + } +} + +impl std::fmt::Display for ResolutionFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// A resolution failure attached to the secret key it happened on. +/// +/// API handlers want this form so they can produce a response body +/// that names the bad secret without exposing the underlying error +/// text (which may include paths or env-var names). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolutionError { + /// Secret key (the map key in `[secrets]` or `req.secrets`). + pub key: String, + /// Classified failure kind. + pub kind: ResolutionFailure, +} + +impl std::fmt::Display for ResolutionError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "secret '{}': {}", self.key, self.kind) + } +} + +impl std::error::Error for ResolutionError {} + +// ============================================================================ +// Resolution +// ============================================================================ + +/// Tracing target for secret-resolution audit records. +/// +/// Operators tail this stream with +/// `RUST_LOG=smolvm::secrets::audit=info` to see every resolution event. +/// Records include the secret key, source kind, scope, and outcome — +/// never the resolved value, never the `from_file` path, never the +/// `from_env` variable name (those can themselves be revealing). +pub const AUDIT_TARGET: &str = "smolvm::secrets::audit"; + +fn scope_label(scope: ResolutionScope) -> &'static str { + match scope { + ResolutionScope::TrustedLocal => "trusted-local", + ResolutionScope::RecordReplay => "record-replay", + ResolutionScope::Untrusted => "untrusted", + } +} + +/// Resolve a single ref into plaintext and emit an audit record. +/// +/// **Caller responsibility:** validate the ref at its trust boundary +/// (`validate_ref(scope)`) before calling this. Resolution assumes the +/// ref has already been accepted by policy; the `scope` argument here +/// is recorded for forensics, not re-checked. +/// +/// Every call emits exactly one tracing event at `AUDIT_TARGET`: fields +/// are `secret_key`, `source_kind`, `scope`, and `result` (`ok` | +/// `error`). The full error message is deliberately omitted from the +/// log because it may contain `from_file` paths or `from_env` names. +pub fn resolve_secret_ref_classified( + name: &str, + r: &SecretRef, + scope: ResolutionScope, +) -> std::result::Result, ResolutionFailure> { + let kind_str = r.source_kind().map(|k| k.as_str()).unwrap_or("unknown"); + let scope_str = scope_label(scope); + + let result: std::result::Result, ResolutionFailure> = + if let Some(env_var) = &r.from_env { + std::env::var(env_var) + .map(Zeroizing::new) + .map_err(|_| ResolutionFailure::EnvUnset) + } else if let Some(path) = &r.from_file { + read_from_file_source(path) + .map(|v| Zeroizing::new(v.trim_end_matches(['\n', '\r']).to_string())) + .map_err(classify_file_source_error) + } else { + // Never reachable if validate_ref was called at the boundary. + Err(ResolutionFailure::Internal) + }; + + match &result { + Ok(_) => tracing::info!( + target: AUDIT_TARGET, + secret_key = %name, + source_kind = %kind_str, + scope = %scope_str, + result = "ok", + ), + Err(f) => tracing::info!( + target: AUDIT_TARGET, + secret_key = %name, + source_kind = %kind_str, + scope = %scope_str, + result = "error", + error_kind = %f.as_str(), + ), + } + + result +} + +/// Resolve a single ref, returning the project-wide [`Error`] type. +fn resolve_secret_ref( + name: &str, + r: &SecretRef, + scope: ResolutionScope, +) -> Result> { + resolve_secret_ref_classified(name, r, scope) + .map_err(|f| Error::config(format!("resolve secret '{}'", name), f.as_str().to_string())) +} + +/// Resolve a map of secret refs and flatten into `(name, value)` pairs +/// ready to append to an agent-bound env vector. +/// +/// Returns an empty vec for empty input. +/// +/// **No caching.** Every call redoes all resolutions. This is +/// deliberate: rotating the underlying env var / file takes effect at the +/// next resolution with no restart. Do not add a cache here without +/// replacing the rotation semantics with an explicit invalidation path. +/// +/// Callers choose the scope based on where the refs came from: +/// `TrustedLocal` for refs a CLI user just supplied, `RecordReplay` for +/// refs read out of a VM record. +pub fn resolve_refs_to_env( + refs: &BTreeMap, + scope: ResolutionScope, +) -> Result> { + let mut out = Vec::with_capacity(refs.len()); + for (name, r) in refs { + out.push((name.clone(), Secret(resolve_secret_ref(name, r, scope)?))); + } + Ok(out) +} + +/// Like [`resolve_refs_to_env`] but surfaces a classified +/// [`ResolutionError`] on the first failure, so API handlers can map +/// failure kinds to HTTP status codes and name the offending secret. +pub fn resolve_refs_to_env_classified( + refs: &BTreeMap, + scope: ResolutionScope, +) -> std::result::Result, ResolutionError> { + let mut out = Vec::with_capacity(refs.len()); + for (name, r) in refs { + let value = + resolve_secret_ref_classified(name, r, scope).map_err(|kind| ResolutionError { + key: name.clone(), + kind, + })?; + out.push((name.clone(), Secret(value))); + } + Ok(out) +} + +// ============================================================================ +// Resolved secret value +// ============================================================================ + +/// A resolved secret plaintext value. +/// +/// Deliberately has NO `Serialize`, `Display`, or `Deref` impl, and a REDACTING +/// `Debug`, so resolved plaintext cannot leak into logs, error messages, the +/// DB, or a pack by accident — the type system enforces the "plaintext never +/// persists / never logs" invariant. Zeroized on drop. Cross it into the guest +/// env (the one legitimate boundary) only via the explicit, greppable +/// `expose` / `into_plaintext`. +#[derive(Clone)] +pub struct Secret(Zeroizing); + +impl Secret { + /// Borrow the plaintext. Each call site is a reviewable point where a + /// secret crosses a trust boundary. + pub fn expose(&self) -> &str { + &self.0 + } + + /// Consume into the plaintext `String` for the one place it must go plain: + /// the agent env vector serialized to the guest over vsock. Moves the inner + /// buffer out, leaving nothing extra to scrub. + pub fn into_plaintext(mut self) -> String { + std::mem::take(&mut *self.0) + } +} + +impl std::fmt::Debug for Secret { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Secret()") + } +} + +/// Expose resolved secrets into plain `(name, value)` tuples for the agent env +/// vector. This is THE boundary where secret plaintext deliberately crosses into +/// the guest's environment (serialized over vsock to the guest). Every call site +/// is a greppable point where plaintext leaves the `Secret` type's protection — +/// keep them few and obvious. +pub fn expose_into_env(secrets: Vec<(String, Secret)>) -> Vec<(String, String)> { + secrets + .into_iter() + .map(|(k, v)| (k, v.into_plaintext())) + .collect() +} + +/// Build a `from_env` ref pointing at the named host environment variable. +pub fn env_ref(env_var: impl Into) -> SecretRef { + SecretRef { + from_env: Some(env_var.into()), + from_file: None, + } +} + +/// Build a `from_file` ref pointing at the given host path. +pub fn file_ref(path: impl Into) -> SecretRef { + SecretRef { + from_env: None, + from_file: Some(path.into()), + } +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + fn env_ref_for(name: &str) -> SecretRef { + env_ref(name) + } + + fn file_ref_for(path: &Path) -> SecretRef { + file_ref(path) + } + + #[test] + fn secret_debug_redacts_and_exposes_only_explicitly() { + let s = Secret(Zeroizing::new("hunter2".to_string())); + let dbg = format!("{:?}", s); + assert_eq!(dbg, "Secret()"); + assert!(!dbg.contains("hunter2"), "Debug leaked the secret: {}", dbg); + assert_eq!(s.expose(), "hunter2"); + assert_eq!(s.into_plaintext(), "hunter2"); + } + + #[test] + fn env_ref_resolves() { + std::env::set_var("SMOLVM_TEST_SECRET_ENV", "from-env-value"); + let r = env_ref_for("SMOLVM_TEST_SECRET_ENV"); + assert_eq!( + &*resolve_secret_ref("DST", &r, ResolutionScope::TrustedLocal).unwrap(), + "from-env-value" + ); + std::env::remove_var("SMOLVM_TEST_SECRET_ENV"); + assert!(resolve_secret_ref("DST", &r, ResolutionScope::TrustedLocal).is_err()); + } + + #[test] + fn file_ref_resolves_and_trims_newline() { + let tmp = tempfile::tempdir().unwrap(); + let p = tmp.path().join("cred"); + std::fs::write(&p, "hunter2\n").unwrap(); + let r = file_ref_for(&p); + assert_eq!( + &*resolve_secret_ref("FILE_SECRET", &r, ResolutionScope::TrustedLocal).unwrap(), + "hunter2" + ); + } + + #[test] + fn validate_rejects_no_source_and_multiple_sources() { + let none = SecretRef { + from_env: None, + from_file: None, + }; + assert_eq!( + validate_ref(&none, ResolutionScope::TrustedLocal), + Err(SecretRefError::NoSource) + ); + let both = SecretRef { + from_env: Some("X".into()), + from_file: Some("/abs".into()), + }; + assert_eq!( + validate_ref(&both, ResolutionScope::TrustedLocal), + Err(SecretRefError::MultipleSources) + ); + } + + #[test] + fn validate_requires_absolute_file_paths() { + let rel = file_ref_for(Path::new("relative/path")); + assert!(matches!( + validate_ref(&rel, ResolutionScope::TrustedLocal), + Err(SecretRefError::RelativeFilePath(_)) + )); + let abs = file_ref_for(Path::new("/etc/hostname")); + assert!(validate_ref(&abs, ResolutionScope::TrustedLocal).is_ok()); + } + + #[test] + fn untrusted_scope_rejects_every_source_kind() { + let env = env_ref_for("X"); + assert!(matches!( + validate_ref(&env, ResolutionScope::Untrusted), + Err(SecretRefError::SourceNotAllowedInScope { + kind: SecretSourceKind::Env, + .. + }) + )); + let file = file_ref_for(Path::new("/etc/hostname")); + assert!(matches!( + validate_ref(&file, ResolutionScope::Untrusted), + Err(SecretRefError::SourceNotAllowedInScope { + kind: SecretSourceKind::File, + .. + }) + )); + } + + #[test] + fn record_replay_allows_env_and_file() { + let env = env_ref_for("X"); + assert!(validate_ref(&env, ResolutionScope::RecordReplay).is_ok()); + let file = file_ref_for(Path::new("/etc/hostname")); + assert!(validate_ref(&file, ResolutionScope::RecordReplay).is_ok()); + } + + #[test] + fn resolution_failure_client_vs_server_classification() { + assert!(ResolutionFailure::EnvUnset.is_client_error()); + assert!(ResolutionFailure::FileReadFailed.is_client_error()); + assert!(ResolutionFailure::FileTooLarge.is_client_error()); + assert!(!ResolutionFailure::Internal.is_client_error()); + } + + #[test] + fn resolve_refs_to_env_classified_names_failing_key() { + let mut refs = BTreeMap::new(); + refs.insert( + "MISSING".to_string(), + env_ref_for("SMOLVM_TEST_DEFINITELY_UNSET"), + ); + std::env::remove_var("SMOLVM_TEST_DEFINITELY_UNSET"); + let err = resolve_refs_to_env_classified(&refs, ResolutionScope::TrustedLocal).unwrap_err(); + assert_eq!(err.key, "MISSING"); + assert_eq!(err.kind, ResolutionFailure::EnvUnset); + } + + #[test] + fn empty_refs_resolve_to_empty_vec() { + let refs = BTreeMap::new(); + assert!(resolve_refs_to_env(&refs, ResolutionScope::TrustedLocal) + .unwrap() + .is_empty()); + } +} diff --git a/src/settings.rs b/src/settings.rs new file mode 100644 index 0000000..3c637b4 --- /dev/null +++ b/src/settings.rs @@ -0,0 +1,541 @@ +//! Unified user settings for smolvm and the smol CLI. +//! +//! All user-facing configuration lives in `~/.config/smolvm/config.toml`: +//! +//! ```toml +//! [cloud] +//! endpoint = "https://api.smolmachines.com" +//! api_key = "smk_live_abc123" +//! +//! [machines.defaults] +//! registry = "registry.smolmachines.com" +//! +//! [machines.registries."registry.smolmachines.com"] +//! username = "token" +//! password = "eyJ..." +//! +//! [images.defaults] +//! registry = "docker.io" +//! +//! [images.registries."docker.io"] +//! username = "myuser" +//! password_env = "DOCKER_HUB_TOKEN" +//! ``` +//! +//! - `[cloud]` — smolfleet API credentials +//! - `[machines]` — credentials for OCI registries storing .smolmachine artifacts +//! - `[images]` — credentials for container image registries (base images for VMs) + +use crate::error::{Error, Result}; +use crate::registry::RegistryConfig; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +/// Unified settings loaded from `~/.config/smolvm/config.toml`. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct SmolSettings { + /// smolfleet API configuration. + #[serde(default)] + pub cloud: CloudSection, + /// Credentials for .smolmachine artifact registries. + #[serde(default)] + pub machines: RegistryConfig, + /// Credentials for container image registries. + #[serde(default)] + pub images: RegistryConfig, +} + +/// smolfleet API configuration. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct CloudSection { + /// smolfleet API endpoint (e.g., "https://api.smolmachines.com"). + pub endpoint: Option, + /// API key for authentication (e.g., "smk_..." or a JWT). + pub api_key: Option, + /// OAuth refresh token for silent token renewal. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub refresh_token: Option, + /// Unix timestamp when the access token expires. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub token_expires_at: Option, +} + +impl SmolSettings { + /// Get the path to the unified configuration file. + /// + /// Respects `SMOLVM_CONFIG` environment variable for overriding the default + /// path. This is useful for CI/CD and server deployments where the config + /// file is placed at a non-standard location. + pub fn config_path() -> Result { + if let Ok(custom) = std::env::var("SMOLVM_CONFIG") { + if custom.is_empty() { + return Err(Error::config( + "resolve path", + "SMOLVM_CONFIG is set but empty; provide a full path or unset it", + )); + } + return Ok(PathBuf::from(custom)); + } + let home = dirs::home_dir() + .ok_or_else(|| Error::config("resolve path", "no home directory found"))?; + Ok(home.join(".config").join("smolvm").join("config.toml")) + } + + /// Load settings from the config file, migrating from old format if needed. + /// + /// Returns empty settings if no config file exists. Migrates legacy + /// `registries.toml` into the `[images]` section on first load. + /// Load settings from the config file, then overlay environment overrides + /// (`SMOL_CLOUD_TOKEN`). Missing/invalid config falls back to defaults. + pub fn load() -> Result { + let mut settings = Self::load_from_disk()?; + settings.apply_env_overrides(); + Ok(settings) + } + + /// Overlay environment-provided credentials onto the loaded settings. + /// `SMOL_CLOUD_TOKEN` → `cloud.api_key` (only when not already configured), + /// so a CI job can authenticate the CLI — push, cloud ops — with a secret + /// env var alone, matching how the SDKs already read `SMOL_CLOUD_TOKEN`. A + /// config-file `api_key` (e.g. written by `smol login`) always wins. + fn apply_env_overrides(&mut self) { + self.apply_cloud_token(std::env::var("SMOL_CLOUD_TOKEN").ok().as_deref()); + } + + /// Pure core of the `SMOL_CLOUD_TOKEN` fallback (env read split out so it is + /// testable without touching process env): set `cloud.api_key` from the token + /// only when it is absent and the token is non-empty. + fn apply_cloud_token(&mut self, env_token: Option<&str>) { + if self.cloud.api_key.is_some() { + return; + } + if let Some(token) = env_token { + let token = token.trim(); + if !token.is_empty() { + self.cloud.api_key = Some(token.to_string()); + } + } + } + + /// Load settings purely from the config file (or a legacy-format migration), + /// without environment overrides. + fn load_from_disk() -> Result { + let config_path = match Self::config_path() { + Ok(p) => p, + Err(e) => { + tracing::debug!(error = %e, "could not determine settings config path"); + return Ok(Self::default()); + } + }; + + if config_path.exists() { + let contents = std::fs::read_to_string(&config_path).map_err(|e| { + Error::config( + format!("read config at {}", config_path.display()), + e.to_string(), + ) + })?; + + let settings: Self = toml::from_str(&contents).map_err(|e| { + Error::config( + format!("parse config at {}", config_path.display()), + e.to_string(), + ) + })?; + + tracing::debug!( + path = %config_path.display(), + machines_count = settings.machines.registries.len(), + images_count = settings.images.registries.len(), + "loaded settings" + ); + + return Ok(settings); + } + + // Migrate from legacy registries.toml if it exists + if let Some(dir) = config_path.parent() { + let old_path = dir.join("registries.toml"); + if old_path.exists() { + tracing::info!( + old = %old_path.display(), + new = %config_path.display(), + "migrating legacy registries.toml to config.toml" + ); + if let Ok(contents) = std::fs::read_to_string(&old_path) { + if let Ok(old_config) = toml::from_str::(&contents) { + let settings = SmolSettings { + images: old_config, + ..Default::default() + }; + // Best-effort save; don't fail the load on write errors + if let Err(e) = settings.save() { + tracing::warn!(error = %e, "failed to save migrated config"); + } + return Ok(settings); + } + } + } + } + + tracing::debug!( + path = %config_path.display(), + "config file not found, using defaults" + ); + Ok(Self::default()) + } + + /// Persist the settings back to the config file. + /// + /// Sets file permissions to `0600` (owner read/write only) since this + /// file may contain secrets (API keys, registry tokens). + pub fn save(&self) -> Result<()> { + let config_path = Self::config_path()?; + if let Some(parent) = config_path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| Error::config("create config directory", e.to_string()))?; + } + let contents = toml::to_string_pretty(self) + .map_err(|e| Error::config("serialize settings", e.to_string()))?; + + // On Unix, create the file with 0o600 from the start so credentials + // are never world-readable, even briefly. On non-Unix fall back to + // write + chmod (Windows has no meaningful file-mode concept). + #[cfg(unix)] + { + use std::io::Write as _; + use std::os::unix::fs::OpenOptionsExt as _; + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o600) + .open(&config_path) + .map_err(|e| { + Error::config( + format!("write config to {}", config_path.display()), + e.to_string(), + ) + })?; + file.write_all(contents.as_bytes()).map_err(|e| { + Error::config( + format!("write config to {}", config_path.display()), + e.to_string(), + ) + })?; + // Also fix any pre-existing file that may have wrong permissions. + use std::os::unix::fs::PermissionsExt as _; + let perms = std::fs::Permissions::from_mode(0o600); + if let Err(e) = std::fs::set_permissions(&config_path, perms) { + tracing::warn!(error = %e, "could not restrict config file permissions to 0600"); + } + } + #[cfg(not(unix))] + { + std::fs::write(&config_path, &contents).map_err(|e| { + Error::config( + format!("write config to {}", config_path.display()), + e.to_string(), + ) + })?; + } + + Ok(()) + } +} + +/// Buffer in seconds before actual expiry to trigger refresh. +const TOKEN_EXPIRY_BUFFER_SECS: i64 = 60; + +/// Hosted smolfleet endpoint used when none is configured. The CLI targets the +/// managed cloud by default; self-hosters override with `smol config set cloud `. +pub const DEFAULT_CLOUD_ENDPOINT: &str = "https://api.smolmachines.com"; + +impl CloudSection { + /// Get the configured endpoint, falling back to the hosted default + /// ([`DEFAULT_CLOUD_ENDPOINT`]) so a fresh install talks to the managed + /// cloud without any `config set`. Self-hosters override via + /// `smol config set cloud `. + pub fn endpoint(&self) -> Result<&str> { + Ok(self.endpoint.as_deref().unwrap_or(DEFAULT_CLOUD_ENDPOINT)) + } + + /// Check if the stored access token has expired or will expire within 60 seconds. + /// + /// Returns `false` if no expiry is set (assume valid). + pub fn is_token_expired(&self) -> bool { + match self.token_expires_at { + Some(expires_at) => { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64; + now >= expires_at - TOKEN_EXPIRY_BUFFER_SECS + } + None => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cloud_token_env_fills_unset_api_key() { + let mut s = SmolSettings::default(); + s.apply_cloud_token(Some("smk_ci_token")); + assert_eq!(s.cloud.api_key.as_deref(), Some("smk_ci_token")); + } + + #[test] + fn config_api_key_takes_precedence_over_env() { + let mut s = SmolSettings::default(); + s.cloud.api_key = Some("smk_from_config".to_string()); + s.apply_cloud_token(Some("smk_from_env")); + assert_eq!(s.cloud.api_key.as_deref(), Some("smk_from_config")); + } + + #[test] + fn blank_or_absent_env_token_is_ignored() { + let mut s = SmolSettings::default(); + s.apply_cloud_token(Some(" ")); + assert!(s.cloud.api_key.is_none()); + s.apply_cloud_token(None); + assert!(s.cloud.api_key.is_none()); + } + + #[test] + fn default_settings_are_empty() { + let settings = SmolSettings::default(); + assert!(settings.cloud.endpoint.is_none()); + assert!(settings.cloud.api_key.is_none()); + assert!(settings.machines.registries.is_empty()); + assert!(settings.images.registries.is_empty()); + } + + #[test] + fn settings_roundtrip_toml() { + let mut settings = SmolSettings::default(); + settings.cloud.endpoint = Some("https://api.smolmachines.com".to_string()); + settings.cloud.api_key = Some("smk_test".to_string()); + settings + .machines + .set_token("registry.smolmachines.com", "eyJhbGci.test"); + settings.images.registries.insert( + "docker.io".to_string(), + crate::registry::RegistryEntry { + username: Some("user".to_string()), + password: None, + password_env: Some("DOCKER_TOKEN".to_string()), + mirror: None, + identity_token: None, + refresh_token: None, + expires_at: None, + }, + ); + + let serialized = toml::to_string_pretty(&settings).unwrap(); + let reloaded: SmolSettings = toml::from_str(&serialized).unwrap(); + + assert_eq!( + reloaded.cloud.endpoint.as_deref(), + Some("https://api.smolmachines.com") + ); + assert_eq!(reloaded.cloud.api_key.as_deref(), Some("smk_test")); + + let machine_creds = reloaded + .machines + .get_credentials("registry.smolmachines.com") + .unwrap(); + assert_eq!(machine_creds.username, "token"); + assert_eq!(machine_creds.password, "eyJhbGci.test"); + + let image_entry = reloaded.images.registries.get("docker.io").unwrap(); + assert_eq!(image_entry.username.as_deref(), Some("user")); + assert_eq!(image_entry.password_env.as_deref(), Some("DOCKER_TOKEN")); + } + + #[test] + fn settings_parses_target_format() { + let toml_str = r#" +[cloud] +endpoint = "https://api.smolmachines.com" +api_key = "smk_live_abc123" + +[machines.defaults] +registry = "registry.smolmachines.com" + +[machines.registries."registry.smolmachines.com"] +username = "token" +password = "eyJhbGci..." + +[images.defaults] +registry = "docker.io" + +[images.registries."docker.io"] +username = "myuser" +password_env = "DOCKER_HUB_TOKEN" + +[images.registries."ghcr.io"] +username = "github_user" +password_env = "GHCR_TOKEN" +mirror = "ghcr-mirror.example.com" +"#; + + let settings: SmolSettings = toml::from_str(toml_str).unwrap(); + assert_eq!( + settings.cloud.endpoint.as_deref(), + Some("https://api.smolmachines.com") + ); + assert_eq!(settings.cloud.api_key.as_deref(), Some("smk_live_abc123")); + assert_eq!( + settings.machines.default_registry(), + "registry.smolmachines.com" + ); + assert_eq!(settings.images.default_registry(), "docker.io"); + assert_eq!(settings.images.registries.len(), 2); + + let ghcr = settings.images.registries.get("ghcr.io").unwrap(); + assert_eq!(ghcr.mirror.as_deref(), Some("ghcr-mirror.example.com")); + } + + #[test] + fn config_path_default_ends_with_expected_components() { + // Do not mutate SMOLVM_CONFIG — env vars are process-global and unsafe + // to set in parallel tests. Skip rather than fail when another test + // holds the var (including config_path_rejects_empty_smolvm_config which + // temporarily sets it to ""). We also accept an Err return for the same + // reason: the var could be set between our guard check and the call. + if std::env::var("SMOLVM_CONFIG").is_ok() { + return; + } + let path = match SmolSettings::config_path() { + Ok(p) => p, + Err(_) => return, // SMOLVM_CONFIG was set between our check and the call + }; + assert!( + path.ends_with(".config/smolvm/config.toml"), + "unexpected default config path: {}", + path.display() + ); + } + + #[test] + fn config_path_rejects_empty_smolvm_config() { + // Set SMOLVM_CONFIG to an empty string and verify we get a clear error + // rather than silently resolving to the current directory. + // config_path_default_ends_with_expected_components skips when this var + // is present, so the only parallelism risk is that test skipping, not failing. + std::env::set_var("SMOLVM_CONFIG", ""); + let result = SmolSettings::config_path(); + std::env::remove_var("SMOLVM_CONFIG"); + + let err = result.unwrap_err(); + assert!( + err.to_string().contains("empty"), + "expected 'empty' in error, got: {}", + err + ); + } + + #[test] + fn load_from_custom_path_via_env() { + // This test verifies SMOLVM_CONFIG override by parsing directly + // rather than using env vars (which interfere with parallel tests). + let dir = tempfile::tempdir().unwrap(); + let config_path = dir.path().join("test-config.toml"); + std::fs::write( + &config_path, + r#" +[cloud] +api_key = "env-override-key" +"#, + ) + .unwrap(); + + let contents = std::fs::read_to_string(&config_path).unwrap(); + let settings: SmolSettings = toml::from_str(&contents).unwrap(); + assert_eq!(settings.cloud.api_key.as_deref(), Some("env-override-key")); + } + + #[test] + fn save_and_reload_preserves_refresh_token_fields() { + // Test roundtrip via direct file write + parse (avoids env var races) + let dir = tempfile::tempdir().unwrap(); + let config_path = dir.path().join("config.toml"); + + let mut settings = SmolSettings::default(); + settings.cloud.api_key = Some("access123".to_string()); + settings.cloud.refresh_token = Some("refresh456".to_string()); + settings.cloud.token_expires_at = Some(1700000000); + + let contents = toml::to_string_pretty(&settings).unwrap(); + std::fs::write(&config_path, &contents).unwrap(); + + let reloaded_contents = std::fs::read_to_string(&config_path).unwrap(); + let reloaded: SmolSettings = toml::from_str(&reloaded_contents).unwrap(); + assert_eq!(reloaded.cloud.api_key.as_deref(), Some("access123")); + assert_eq!(reloaded.cloud.refresh_token.as_deref(), Some("refresh456")); + assert_eq!(reloaded.cloud.token_expires_at, Some(1700000000)); + } + + #[test] + fn is_token_expired_returns_false_when_no_expiry() { + let cloud = CloudSection::default(); + assert!(!cloud.is_token_expired()); + } + + #[test] + fn is_token_expired_returns_true_for_past_timestamp() { + let cloud = CloudSection { + token_expires_at: Some(1000000000), // year 2001 + ..Default::default() + }; + assert!(cloud.is_token_expired()); + } + + #[test] + fn is_token_expired_returns_false_for_far_future() { + let cloud = CloudSection { + token_expires_at: Some(4000000000), // year 2096 + ..Default::default() + }; + assert!(!cloud.is_token_expired()); + } + + #[test] + fn registry_entry_refresh_fields_roundtrip() { + let toml_str = r#" +[machines.registries."registry.smolmachines.com"] +username = "token" +password = "access123" +refresh_token = "refresh456" +expires_at = 1700000000 +"#; + + let settings: SmolSettings = toml::from_str(toml_str).unwrap(); + let entry = settings + .machines + .registries + .get("registry.smolmachines.com") + .unwrap(); + assert_eq!(entry.refresh_token.as_deref(), Some("refresh456")); + assert_eq!(entry.expires_at, Some(1700000000)); + + // Roundtrip + let serialized = toml::to_string_pretty(&settings).unwrap(); + assert!(serialized.contains("refresh_token")); + assert!(serialized.contains("1700000000")); + } + + #[test] + fn skip_serializing_none_refresh_fields() { + let settings = SmolSettings::default(); + let serialized = toml::to_string_pretty(&settings).unwrap(); + // None fields should not appear + assert!(!serialized.contains("refresh_token")); + assert!(!serialized.contains("expires_at")); + assert!(!serialized.contains("token_expires_at")); + } +} diff --git a/src/smolfile.rs b/src/smolfile.rs new file mode 100644 index 0000000..82a2703 --- /dev/null +++ b/src/smolfile.rs @@ -0,0 +1,192 @@ +//! Smolfile support for smolvm. +//! +//! Re-exports all types and parsing from the standalone [`smolfile`] crate, +//! plus smolvm-specific helpers (file loading with smolvm error types, +//! network policy resolution). +//! +//! See the [`smolfile`] crate documentation for the full Smolfile specification. + +// Re-export everything from the standalone crate. +pub use smolfile::*; + +use std::path::Path; + +// ============================================================================ +// smolvm-specific loading (wraps crate error into smolvm::Error) +// ============================================================================ + +/// Load and parse a Smolfile from the given path. +/// +/// This is a convenience wrapper that converts [`smolfile::SmolfileError`] +/// into [`crate::Error`] for use within smolvm. +pub fn load(path: &Path) -> crate::Result { + smolfile::load(path).map_err(|e| crate::Error::config("load smolfile", e.to_string())) +} + +// ============================================================================ +// Network helpers (smolvm-specific, depend on ipnet and std::net) +// ============================================================================ + +/// Resolve a hostname or IP address to CIDRs suitable for TSI egress policy. +/// +/// IPv4 addresses become `/32` CIDRs; IPv6 addresses become `/128` CIDRs. +/// Hostnames are resolved via the host DNS at VM start time and each result +/// address is formatted with the correct prefix length. +/// +/// Rejects `host:port` syntax — port filtering is not supported by the TSI +/// egress policy (all ports to a resolved IP are implicitly allowed). +pub fn resolve_host_to_cidrs(host: &str) -> Result, String> { + use ipnet::IpNet; + use std::net::{IpAddr, ToSocketAddrs}; + + // Try parsing as a bare IP first — must come before the ':' check so that + // IPv6 addresses like "::1" or "2001:db8::1" (which contain ':') are + // handled correctly rather than being rejected as host:port syntax. + if let Ok(ip) = host.parse::() { + return Ok(vec![IpNet::from(ip).to_string()]); + } + + // Reject host:port syntax (e.g. "example.com:443"). + if host.contains(':') { + return Err(format!( + "invalid hostname '{}': port suffixes are not supported. \ + Use the hostname only (all ports are allowed to resolved IPs).", + host + )); + } + + // Resolve hostname with a timeout to avoid hanging on slow/unreachable DNS. + let host_owned = host.to_string(); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let result = format!("{}:0", host_owned).to_socket_addrs().map(|addrs| { + addrs + .map(|addr| IpNet::from(addr.ip()).to_string()) + .collect::>() + }); + let _ = tx.send(result); + }); + + let addrs: Vec = rx + .recv_timeout(std::time::Duration::from_secs(10)) + .map_err(|_| format!("DNS resolution for '{}' timed out after 10 seconds", host))? + .map_err(|e| format!("failed to resolve '{}': {}", host, e))?; + + if addrs.is_empty() { + return Err(format!("'{}' resolved to no addresses", host)); + } + + Ok(addrs) +} + +/// Parse and validate a CIDR specification (e.g., `"10.0.0.0/8"`, `"1.1.1.1"`). +/// +/// Accepts `IP/prefix` or bare `IP` (auto-appends /32 for IPv4, /128 for IPv6). +/// Returns the normalized CIDR string. +pub fn parse_cidr(s: &str) -> Result { + use ipnet::IpNet; + use std::net::IpAddr; + + let net: IpNet = match s.parse::() { + Ok(net) => net, + Err(_) => match s.parse::() { + Ok(ip) => IpNet::from(ip), + Err(_) => { + return Err(format!( + "invalid CIDR '{}': expected format like 10.0.0.0/8 or 1.1.1.1", + s + )) + } + }, + }; + + Ok(net.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolve_host_bare_ipv4() { + let cidrs = resolve_host_to_cidrs("1.2.3.4").unwrap(); + assert_eq!(cidrs, vec!["1.2.3.4/32"]); + } + + #[test] + fn resolve_host_bare_ipv6() { + let cidrs = resolve_host_to_cidrs("::1").unwrap(); + assert_eq!(cidrs, vec!["::1/128"]); + + let cidrs = resolve_host_to_cidrs("2001:db8::1").unwrap(); + assert_eq!(cidrs, vec!["2001:db8::1/128"]); + } + + #[test] + fn resolve_host_rejects_port_suffix() { + let err = resolve_host_to_cidrs("example.com:443").unwrap_err(); + assert!(err.contains("port suffixes are not supported"), "{}", err); + + // Bracketed IPv6 with port must also be rejected. + let err = resolve_host_to_cidrs("[::1]:80").unwrap_err(); + assert!(err.contains("port suffixes are not supported"), "{}", err); + } + + #[test] + fn parse_cidr_valid() { + assert_eq!(parse_cidr("10.0.0.0/8").unwrap(), "10.0.0.0/8"); + assert_eq!(parse_cidr("1.1.1.1").unwrap(), "1.1.1.1/32"); + } + + #[test] + fn parse_cidr_invalid() { + assert!(parse_cidr("not-a-cidr").is_err()); + } + + #[test] + fn load_basic_smolfile() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("Smolfile"); + std::fs::write( + &path, + r#" +image = "alpine" +cpus = 2 +memory = 1024 +net = true + +[dev] +volumes = ["./src:/app"] +init = ["echo hello"] +"#, + ) + .unwrap(); + let sf = load(&path).unwrap(); + assert_eq!(sf.image.as_deref(), Some("alpine")); + assert_eq!(sf.cpus, Some(2)); + assert_eq!(sf.dev.unwrap().volumes, vec!["./src:/app"]); + } + + #[test] + fn smolfile_gpu_field() { + let dir = tempfile::tempdir().unwrap(); + + // With gpu = true + let path = dir.path().join("gpu.smolfile"); + std::fs::write(&path, "image = \"alpine\"\ngpu = true\n").unwrap(); + let sf = load(&path).unwrap(); + assert_eq!(sf.gpu, Some(true)); + + // Without gpu field (defaults to None) + let path = dir.path().join("nogpu.smolfile"); + std::fs::write(&path, "image = \"alpine\"\n").unwrap(); + let sf = load(&path).unwrap(); + assert_eq!(sf.gpu, None); + + // With gpu = false + let path = dir.path().join("gpuoff.smolfile"); + std::fs::write(&path, "image = \"alpine\"\ngpu = false\n").unwrap(); + let sf = load(&path).unwrap(); + assert_eq!(sf.gpu, Some(false)); + } +} diff --git a/src/storage.rs b/src/storage.rs new file mode 100644 index 0000000..edca1c0 --- /dev/null +++ b/src/storage.rs @@ -0,0 +1,854 @@ +//! Persistent storage management. +//! +//! This module provides [`StorageDisk`] for managing persistent storage. +//! Each VM (default or named) gets its own sparse ext4 disk image that stores +//! OCI layers, container overlays, and cached manifests. +//! +//! # Storage Locations +//! +//! - Default VM: `~/Library/Application Support/smolvm/storage.raw` (macOS) +//! - Named VMs: `~/Library/Caches/smolvm/vms/{name}/storage.raw` (macOS) +//! +//! # Architecture +//! +//! The storage disk is a sparse raw disk image formatted with ext4. +//! It's mounted inside the agent VM which handles OCI layer extraction +//! and overlay filesystem management. + +use crate::data::consts::BYTES_PER_GIB; +pub use crate::data::disk::{DiskFormat, DiskType, Overlay, Storage}; +pub use crate::data::storage::{ + DEFAULT_OVERLAY_SIZE_GIB, DEFAULT_STORAGE_SIZE_GIB, OVERLAY_DISK_FILENAME, + STORAGE_DISK_FILENAME, +}; +use crate::disk_utils; +use crate::error::{Error, Result}; +use serde::{Deserialize, Serialize}; +use std::marker::PhantomData; +use std::path::{Path, PathBuf}; + +/// Seed a VM-mode machine's overlay+storage disks from extracted pack templates so +/// it boots the source VM's filesystem rather than a freshly-mkfs'd empty overlay. +/// +/// VM-mode (`--from-vm`) packs carry the source VM's rootfs DISKS, but the packed +/// template has its trailing zero extent stripped, so the disk must be grown back to +/// its logical size before boot. The caller's [`AgentManager`] has already created +/// default `.qcow2` overlays backed by the EMPTY default template; this removes them +/// and writes the seeded RAW disks under their place so [`resolve_disk_image`] picks +/// these at start. A `.formatted` marker stops the host from reformatting the +/// inherited (already-formatted) filesystem; the guest still grows it with resize2fs. +/// +/// The template → disk copy uses [`crate::disk_utils::clone_or_copy_file`] (clonefile +/// CoW on macOS, `SEEK_HOLE` sparse copy on Linux), NOT a dense `fs::copy`: the +/// templates are sparse (~25 MiB of real data in a multi-GiB logical file), so a dense +/// copy would balloon every machine to its full logical size (~28 GiB) on disk. +/// +/// `disk_dir` is the machine's data dir (the parent of `manager.storage_path()`). +/// Shared by both the serve API create path and the `smolvm machine create --from` +/// CLI path so a VM-mode pack restores identically through either entry point. +/// +/// [`AgentManager`]: crate::agent::AgentManager +/// [`resolve_disk_image`]: crate::agent::manager::resolve_disk_image +pub fn seed_vm_mode_disks( + disk_dir: &Path, + cache_dir: &Path, + overlay_template: Option<&str>, + storage_template: Option<&str>, + overlay_logical_size: Option, + overlay_gb: Option, + storage_gb: Option, +) -> std::io::Result<()> { + // Drop the manager's default qcow2 overlays (backed by the empty default + // template) so the seeded raw disks below are what start resolves. + for raw_filename in [STORAGE_DISK_FILENAME, OVERLAY_DISK_FILENAME] { + let stem = Path::new(raw_filename); + let _ = std::fs::remove_file(disk_dir.join(stem.with_extension("qcow2"))); + } + + // The overlay carries the rootfs and is grown back to its (pre-truncation) + // logical size; storage to the requested size. Both VM-mode templates are + // normally present, but tolerate a missing one with an empty disk. + seed_one_disk( + cache_dir, + overlay_template, + &disk_dir.join(OVERLAY_DISK_FILENAME), + overlay_logical_size, + overlay_gb, + )?; + seed_one_disk( + cache_dir, + storage_template, + &disk_dir.join(STORAGE_DISK_FILENAME), + None, + storage_gb, + )?; + Ok(()) +} + +/// Sparse-copy one packed template into `dest`, grow it to the largest of its copied +/// size / the source's logical size / any requested size, and mark it formatted +/// (the copy carries the source's ext4; the guest grows it with resize2fs). With no +/// template, write an empty sparse disk for the guest to format (no formatted marker). +fn seed_one_disk( + cache_dir: &Path, + template: Option<&str>, + dest: &Path, + logical_size: Option, + size_gb_override: Option, +) -> std::io::Result<()> { + let _ = std::fs::remove_file(dest); + let formatted_marker = dest.with_extension("formatted"); + + let Some(rel) = template else { + // No template (rare for VM mode): an empty sparse disk the guest formats. + let _ = std::fs::remove_file(&formatted_marker); + let size = size_gb_override + .map(|gb| gb * BYTES_PER_GIB) + .unwrap_or(512 * 1024 * 1024); + std::fs::File::create(dest)?.set_len(size)?; + return Ok(()); + }; + + let src = resolve_template_in_cache(cache_dir, rel)?; + if !src.exists() { + return Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("VM-mode template not found: {}", src.display()), + )); + } + crate::disk_utils::clone_or_copy_file(&src, dest) + .map_err(|e| std::io::Error::other(e.to_string()))?; + + // Grow back to the logical/requested size (set_len extends sparsely; the guest + // resize2fs expands the ext4 to fill it at boot). + let copied = std::fs::metadata(dest)?.len(); + let target = [ + Some(copied), + logical_size, + size_gb_override.map(|gb| gb * BYTES_PER_GIB), + ] + .into_iter() + .flatten() + .max() + .unwrap_or(copied); + if target > copied { + std::fs::OpenOptions::new() + .write(true) + .open(dest)? + .set_len(target)?; + } + + std::fs::write(&formatted_marker, b"")?; + Ok(()) +} + +/// Resolve a pack template's relative path within `cache_dir`, rejecting anything +/// that isn't a plain in-tree path (no `..`, absolute, or other escaping component) +/// so a hostile pack manifest can't redirect the copy outside the cache. +fn resolve_template_in_cache(cache_dir: &Path, rel: &str) -> std::io::Result { + let rel_path = Path::new(rel); + let safe = !rel.is_empty() + && rel_path + .components() + .all(|c| matches!(c, std::path::Component::Normal(_))); + if !safe { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("invalid VM-mode template path: {rel:?}"), + )); + } + Ok(cache_dir.join(rel_path)) +} + +/// Disk format version info (stored at `/.smolvm/version.json` in ext4 disk). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DiskVersion { + /// Format version (currently: 1). + pub format_version: u32, + + /// Timestamp when the disk was created. + pub created_at: String, + + /// Digest of the base rootfs image. + pub base_digest: String, + + /// smolvm version that created this disk. + pub smolvm_version: String, +} + +impl DiskVersion { + /// Current format version. + pub const CURRENT_VERSION: u32 = 1; + + /// Create a new disk version with current settings. + pub fn new(base_digest: impl Into) -> Self { + Self { + format_version: Self::CURRENT_VERSION, + created_at: crate::util::current_timestamp().to_string(), + base_digest: base_digest.into(), + smolvm_version: env!("CARGO_PKG_VERSION").to_string(), + } + } + + /// Check if this version is compatible with the current smolvm. + pub fn is_compatible(&self) -> bool { + self.format_version <= Self::CURRENT_VERSION + } +} + +/// Shared disk implementation for storage and overlay disks. +#[derive(Debug, Clone)] +pub struct VmDisk { + path: PathBuf, + size_bytes: u64, + format: DiskFormat, + _kind: PhantomData, +} + +impl VmDisk { + /// Get the default path for the disk. + pub fn default_path() -> Result { + let data_dir = dirs::data_local_dir() + .or_else(dirs::data_dir) + .ok_or_else(|| { + Error::storage( + format!("resolve {} path", K::NAME), + "could not determine data directory", + ) + })?; + + Ok(data_dir.join("smolvm").join(K::DEFAULT_FILENAME)) + } + + /// Open or create the disk of the default size at the default location. + pub fn open_or_create() -> Result { + let path = Self::default_path()?; + Self::open_or_create_at(&path, K::DEFAULT_SIZE_GIB) + } + + /// Open or create the disk of the custom size at a custom path. + pub fn open_or_create_at(path: &Path, size_gb: u64) -> Result { + if size_gb == 0 { + return Err(Error::config( + format!("validate {} size", K::NAME), + "disk size must be greater than 0 GiB", + )); + } + + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + + let size_bytes = size_gb * BYTES_PER_GIB; + + if path.exists() { + let metadata = std::fs::metadata(path)?; + Ok(Self { + path: path.to_path_buf(), + size_bytes: metadata.len(), + format: DiskFormat::Raw, + _kind: PhantomData, + }) + } else { + disk_utils::create_sparse_disk::(path, size_bytes)?; + Ok(Self { + path: path.to_path_buf(), + size_bytes, + format: DiskFormat::Raw, + _kind: PhantomData, + }) + } + } + + /// Open an existing disk image with an explicit on-disk format, without + /// creating or formatting it. Used for fork-clone qcow2 overlays, which are + /// created up front by the fork path and inherit the backing disk's + /// already-formatted filesystem. + pub fn open_existing_with_format(path: &Path, format: DiskFormat) -> Result { + let metadata = std::fs::metadata(path)?; + Ok(Self { + path: path.to_path_buf(), + size_bytes: metadata.len(), + format, + _kind: PhantomData, + }) + } + + /// Open the disk for a fresh (non-clone) VM. On Linux at this disk type's + /// DEFAULT size, when the shipped template has already been sized to the + /// default virtual size **at install time**, create an instant qcow2 + /// copy-on-write overlay over the read-only template (O(metadata), no byte + /// copy) — so N concurrent boots don't thrash the host disk the way N full + /// template copies do. Anything else (a custom size, a pre-existing disk, a + /// not-yet-sized legacy template, non-Linux, or an overlay-create failure) + /// falls back to the raw create + format-time copy. The template is treated + /// as immutable here: sizing it is an install-time concern (see + /// `scripts/install.sh`), never a per-boot mutation of a shared file. + pub fn open_or_overlay_at(raw_path: &Path, size_gb: u64) -> Result { + #[cfg(target_os = "linux")] + if !raw_path.exists() && size_gb == K::DEFAULT_SIZE_GIB { + let size_bytes = size_gb * BYTES_PER_GIB; + // Only overlay when the template already presents the full virtual + // size. A legacy/too-small template would make the overlay inherit an + // undersized device, so degrade to the copy path rather than mutate + // the shared template. + if Self::template_at_least(size_bytes) { + let overlay_path = raw_path.with_extension(DiskFormat::Qcow2.extension()); + match Self::create_overlay_from_template(&overlay_path, size_bytes) { + Ok(disk) => return Ok(disk), + Err(error) => { + tracing::warn!( + disk_type = K::NAME, %error, + "qcow2 overlay create failed; falling back to raw template copy" + ); + } + } + } + } + Self::open_or_create_at(raw_path, size_gb) + } + + /// True if a disk template exists and already presents at least `size_bytes` + /// of virtual size (sized at install). Used to decide whether the fast qcow2 + /// overlay path is safe without ever mutating the shared template. + #[cfg(target_os = "linux")] + fn template_at_least(size_bytes: u64) -> bool { + Self::template_path() + .and_then(|p| std::fs::metadata(p).ok()) + .map(|m| m.len() >= size_bytes) + .unwrap_or(false) + } + + /// Create a `.qcow2` CoW overlay backed by the read-only template instead of + /// copying it. The overlay inherits the backing file's VIRTUAL size; the + /// template is sized to the default at install time (see + /// `scripts/install.sh`) while its ext4 stays 512 MiB, so the guest grows the + /// inherited filesystem with resize2fs at boot, exactly as on the copy path. + /// The template is opened read-only and never mutated — if it is not yet + /// sized to `size_bytes`, this refuses (the caller falls back to the copy + /// path) rather than truncate a shared file under concurrent boots. Marks the + /// disk formatted so it is never reformatted (the overlay shares the + /// template's filesystem). Linux only; called only via [`Self::open_or_overlay_at`]. + #[cfg(target_os = "linux")] + fn create_overlay_from_template(overlay_path: &Path, size_bytes: u64) -> Result { + let template = Self::template_path() + .ok_or_else(|| Error::storage("overlay from template", "no disk template found"))?; + + // The template must already present the full virtual size (install-time + // sizing). Refuse rather than mutate a shared file under concurrent boots. + let template_len = std::fs::metadata(&template) + .map_err(|e| Error::storage("read template size", e.to_string()))? + .len(); + if template_len < size_bytes { + return Err(Error::storage( + "overlay from template", + format!("template virtual size {template_len} < required {size_bytes}; not sized at install"), + )); + } + + if let Some(parent) = overlay_path.parent() { + std::fs::create_dir_all(parent)?; + } + // The backing path is written verbatim into the overlay header, so it + // must be absolute (canonicalized). + let base = template + .canonicalize() + .map_err(|e| Error::storage("canonicalize template", e.to_string()))?; + crate::agent::create_disk_overlays(&[(overlay_path.to_path_buf(), base, DiskFormat::Raw)])?; + + let disk = Self { + path: overlay_path.to_path_buf(), + size_bytes, + format: DiskFormat::Qcow2, + _kind: PhantomData, + }; + disk.mark_formatted()?; + Ok(disk) + } + + /// Pre-format the disk with ext4 on the host. + /// + /// This tries multiple approaches in order: + /// 1. Copy from pre-formatted template (no dependencies, fastest) + /// 2. Format with mkfs.ext4 (requires e2fsprogs) + /// + /// The template approach eliminates the e2fsprogs dependency for end users. + pub fn ensure_formatted(&self) -> Result<()> { + if self.format == DiskFormat::Qcow2 { + // A qcow2 CoW overlay inherits its backing disk's already-formatted + // filesystem; formatting it would write a fresh fs into the overlay + // and diverge from the backing image. + return Ok(()); + } + + if !self.needs_format() { + tracing::debug!( + path = %self.path.display(), + disk_type = K::NAME, + "disk already formatted" + ); + return Ok(()); + } + + if let Some(template_path) = Self::template_path() { + disk_utils::copy_disk_from_template::(&self.path, self.size_bytes, &template_path)?; + } else { + disk_utils::format_disk_with_mkfs::(&self.path)?; + } + + self.mark_formatted() + } + + /// Get the path to the disk image. + pub fn path(&self) -> &Path { + &self.path + } + + /// Get the on-disk image format (raw, or qcow2 for fork-clone overlays). + pub fn format(&self) -> DiskFormat { + self.format + } + + /// Get the disk size in bytes. + pub fn size_bytes(&self) -> u64 { + self.size_bytes + } + + /// Get the disk size in GiB. + pub fn size_gib(&self) -> u64 { + self.size_bytes / BYTES_PER_GIB + } + + /// Check if the disk needs to be formatted. + /// + /// Fast path: if the format marker and the disk file both exist, the disk + /// was formatted successfully, so skip the expensive `file` command check. + pub fn needs_format(&self) -> bool { + if !self.disk_marker_path().exists() { + return true; + } + + if !self.path.exists() { + let marker_path = self.disk_marker_path(); + if let Err(error) = std::fs::remove_file(&marker_path) { + tracing::warn!( + path = %marker_path.display(), + disk_type = K::NAME, + %error, + "failed to remove stale disk marker" + ); + } + return true; + } + + false + } + + /// Mark a disk as formatted by creating its marker file. + pub fn mark_formatted(&self) -> Result<()> { + std::fs::write(self.disk_marker_path(), "1")?; + Ok(()) + } + + /// Delete a disk image and its marker file. + pub fn delete(&self) -> Result<()> { + if self.path.exists() { + std::fs::remove_file(&self.path)?; + } + + let marker_path = self.disk_marker_path(); + if marker_path.exists() { + std::fs::remove_file(marker_path)?; + } + Ok(()) + } + + /// Find a pre-formatted disk template. + /// + /// Searches in order: + /// 1. `~/.smolvm/{filename}` (installed location) + /// 2. Next to the current executable (development) + fn template_path() -> Option { + if let Some(home) = dirs::home_dir() { + let installed_path = home.join(".smolvm").join(K::TEMPLATE_FILENAME); + if installed_path.exists() { + tracing::debug!( + path = %installed_path.display(), + disk_type = K::NAME, + "found disk template" + ); + return Some(installed_path); + } + } + + if let Ok(exe_path) = std::env::current_exe() { + if let Some(exe_dir) = exe_path.parent() { + let dev_path = exe_dir.join(K::TEMPLATE_FILENAME); + if dev_path.exists() { + tracing::debug!( + path = %dev_path.display(), + disk_type = K::NAME, + "found disk template (dev)" + ); + return Some(dev_path); + } + } + } + + None + } + + /// Get the path to the format marker file for a disk. + fn disk_marker_path(&self) -> PathBuf { + self.path.with_extension("formatted") + } +} + +impl VmDisk { + /// Open or create the storage disk at the default location with a custom size. + pub fn open_or_create_with_size(size_gb: u64) -> Result { + let path = Self::default_path()?; + Self::open_or_create_at(&path, size_gb) + } +} + +// ============================================================================ +// Storage Disk +// ============================================================================ + +/// Shared storage disk for OCI layers. +/// +/// This is a sparse raw disk image that the helper VM mounts to store +/// OCI image layers and overlay filesystems. +/// +/// # Directory Structure (inside ext4) +/// +/// ```text +/// / +/// ├── .smolvm_formatted # Marker file +/// ├── layers/ # Extracted OCI layers (content-addressed) +/// │ └── sha256:{digest}/ # Each layer as a directory +/// ├── configs/ # OCI image configs +/// │ └── {digest}.json +/// ├── overlays/ # Workload overlay directories +/// │ └── {workload_id}/ +/// │ ├── upper/ # Writable layer +/// │ ├── work/ # Overlay work directory +/// │ └── merged/ # Mount point (optional) +/// └── manifests/ # Cached image manifests +/// └── {image_ref}.json +/// ``` +pub type StorageDisk = VmDisk; + +// ============================================================================ +// Overlay Disk +// ============================================================================ + +/// Persistent rootfs overlay disk. +/// +/// A sparse ext4 disk image used as the upper layer of an overlayfs +/// on top of the initramfs. Changes to the root filesystem (e.g., +/// `apk add git`) persist across VM reboots. +/// +/// The overlay is set up by the agent's `setup_persistent_rootfs()` +/// function early in boot, before the vsock listener starts. +pub type OverlayDisk = VmDisk; + +/// Expand a disk image at an arbitrary path for a specific disk type. +pub fn expand_disk(path: &Path, new_size_gb: u64) -> Result<()> { + disk_utils::expand_sparse_disk::(path, new_size_gb) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_disk_version_compatibility() { + let version = DiskVersion::new("sha256:abc123"); + assert!(version.is_compatible()); + + let future_version = DiskVersion { + format_version: 999, + created_at: "0".to_string(), + base_digest: "sha256:abc123".to_string(), + smolvm_version: "99.0.0".to_string(), + }; + assert!(!future_version.is_compatible()); + } + + #[test] + fn test_disk_version_serialization() { + let version = DiskVersion::new("sha256:abc123"); + let json = serde_json::to_string(&version).unwrap(); + let deserialized: DiskVersion = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.format_version, version.format_version); + assert_eq!(deserialized.base_digest, version.base_digest); + } + + #[test] + fn test_storage_disk_create_and_delete() { + let temp_dir = std::env::temp_dir().join("smolvm_test_basic"); + std::fs::create_dir_all(&temp_dir).unwrap(); + let disk_path = temp_dir.join("test_storage.raw"); + + let _ = std::fs::remove_file(&disk_path); + let _ = std::fs::remove_file(disk_path.with_extension("formatted")); + + let disk = StorageDisk::open_or_create_at(&disk_path, 1).unwrap(); + + assert!(disk_path.exists()); + assert_eq!(disk.size_gib(), 1); + assert!(disk.needs_format()); + + write_ext4_magic(&disk_path); + + disk.mark_formatted().unwrap(); + assert!(!disk.needs_format()); + + disk.delete().unwrap(); + assert!(!disk_path.exists()); + + let _ = std::fs::remove_dir(&temp_dir); + } + + #[test] + fn test_default_paths_use_expected_filenames() { + assert_eq!( + StorageDisk::default_path().unwrap().file_name().unwrap(), + STORAGE_DISK_FILENAME + ); + assert_eq!( + OverlayDisk::default_path().unwrap().file_name().unwrap(), + OVERLAY_DISK_FILENAME + ); + } + + #[test] + fn test_corruption_detection() { + let temp_dir = std::env::temp_dir().join("smolvm_test_corrupt"); + std::fs::create_dir_all(&temp_dir).unwrap(); + let disk_path = temp_dir.join("corrupt_storage.raw"); + let marker_path = disk_path.with_extension("formatted"); + + let _ = std::fs::remove_file(&disk_path); + let _ = std::fs::remove_file(&marker_path); + + let disk = StorageDisk::open_or_create_at(&disk_path, 1).unwrap(); + write_ext4_magic(&disk_path); + disk.mark_formatted().unwrap(); + + assert!(!disk.needs_format()); + assert!(disk_appears_valid_ext4(&disk_path)); + + corrupt_ext4_magic(&disk_path); + + assert!(!disk_appears_valid_ext4(&disk_path)); + + let disk2 = StorageDisk::open_or_create_at(&disk_path, 1).unwrap(); + assert!(!disk2.needs_format()); + + let _ = std::fs::remove_file(&disk_path); + assert!(disk2.needs_format()); + assert!(!marker_path.exists()); + + let _ = std::fs::remove_dir(&temp_dir); + } + + #[test] + fn test_overlay_disk_create_and_delete() { + let temp_dir = std::env::temp_dir().join("smolvm_test_overlay"); + std::fs::create_dir_all(&temp_dir).unwrap(); + let disk_path = temp_dir.join("test_overlay.raw"); + + let _ = std::fs::remove_file(&disk_path); + let _ = std::fs::remove_file(disk_path.with_extension("formatted")); + + let disk = OverlayDisk::open_or_create_at(&disk_path, 1).unwrap(); + + assert!(disk_path.exists()); + assert!(disk.needs_format()); + + write_ext4_magic(&disk_path); + + disk.mark_formatted().unwrap(); + assert!(!disk.needs_format()); + + disk.delete().unwrap(); + assert!(!disk_path.exists()); + + let _ = std::fs::remove_dir(&temp_dir); + } + + #[test] + fn test_overlay_disk_zero_size_rejected() { + let temp_dir = std::env::temp_dir().join("smolvm_test_overlay_zero"); + let disk_path = temp_dir.join("zero_overlay.raw"); + assert!(OverlayDisk::open_or_create_at(&disk_path, 0).is_err()); + } + + #[test] + fn test_overlay_disk_ensure_formatted() { + if disk_utils::find_e2fsprogs_tool("mkfs.ext4").is_none() { + eprintln!("skipping test_overlay_disk_ensure_formatted: mkfs.ext4 not found"); + return; + } + + let temp_dir = std::env::temp_dir().join("smolvm_test_overlay_fmt"); + std::fs::create_dir_all(&temp_dir).unwrap(); + let disk_path = temp_dir.join("fmt_overlay.raw"); + + let _ = std::fs::remove_file(&disk_path); + let _ = std::fs::remove_file(disk_path.with_extension("formatted")); + + let disk = OverlayDisk::open_or_create_at(&disk_path, 1).unwrap(); + assert!(disk.needs_format()); + + disk.ensure_formatted().unwrap(); + assert!(!disk.needs_format()); + + disk.ensure_formatted().unwrap(); + + disk.delete().unwrap(); + assert!(!disk_path.exists()); + let _ = std::fs::remove_dir(&temp_dir); + } + + #[test] + fn test_typed_expand_updates_cached_size() { + let temp_dir = std::env::temp_dir().join("smolvm_test_typed_expand"); + std::fs::create_dir_all(&temp_dir).unwrap(); + let disk_path = temp_dir.join("typed_expand_test.raw"); + + let _ = std::fs::remove_file(&disk_path); + + let _disk = StorageDisk::open_or_create_at(&disk_path, 1).unwrap(); + expand_disk::(&disk_path, 2).unwrap(); + + let disk = StorageDisk::open_or_create_at(&disk_path, 2).unwrap(); + assert_eq!(disk.size_gib(), 2); + let metadata = std::fs::metadata(&disk_path).unwrap(); + assert_eq!(metadata.len(), 2 * BYTES_PER_GIB); + + let _ = std::fs::remove_file(&disk_path); + let _ = std::fs::remove_dir(&temp_dir); + } + + #[test] + fn test_expand_disk_basic() { + let temp_dir = std::env::temp_dir().join("smolvm_test_expand"); + std::fs::create_dir_all(&temp_dir).unwrap(); + let disk_path = temp_dir.join("expand_test.raw"); + + let _ = std::fs::remove_file(&disk_path); + + let initial_size = BYTES_PER_GIB; + disk_utils::create_sparse_disk::(&disk_path, initial_size).unwrap(); + + let metadata = std::fs::metadata(&disk_path).unwrap(); + assert_eq!(metadata.len(), initial_size); + + expand_disk::(&disk_path, 2).unwrap(); + + let metadata = std::fs::metadata(&disk_path).unwrap(); + assert_eq!(metadata.len(), 2 * BYTES_PER_GIB); + + let _ = std::fs::remove_file(&disk_path); + let _ = std::fs::remove_dir(&temp_dir); + } + + #[test] + fn test_expand_disk_reject_shrink() { + let temp_dir = std::env::temp_dir().join("smolvm_test_shrink"); + std::fs::create_dir_all(&temp_dir).unwrap(); + let disk_path = temp_dir.join("shrink_test.raw"); + + let _ = std::fs::remove_file(&disk_path); + + let initial_size = 10 * BYTES_PER_GIB; + disk_utils::create_sparse_disk::(&disk_path, initial_size).unwrap(); + + let result = expand_disk::(&disk_path, 5); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err.to_string().contains("must be larger")); + + let metadata = std::fs::metadata(&disk_path).unwrap(); + assert_eq!(metadata.len(), initial_size); + + let _ = std::fs::remove_file(&disk_path); + let _ = std::fs::remove_dir(&temp_dir); + } + + #[test] + fn test_expand_disk_same_size_is_idempotent() { + let temp_dir = std::env::temp_dir().join("smolvm_test_same"); + std::fs::create_dir_all(&temp_dir).unwrap(); + let disk_path = temp_dir.join("same_test.raw"); + + let _ = std::fs::remove_file(&disk_path); + + let initial_size = 10 * BYTES_PER_GIB; + disk_utils::create_sparse_disk::(&disk_path, initial_size).unwrap(); + + // Same size is a no-op (idempotent for retry after partial failure) + let result = expand_disk::(&disk_path, 10); + assert!(result.is_ok()); + + // Shrink is still rejected + let result = expand_disk::(&disk_path, 5); + assert!(result.is_err()); + + let _ = std::fs::remove_file(&disk_path); + let _ = std::fs::remove_dir(&temp_dir); + } + + /// Write ext4 magic bytes to make `file` command recognize it as ext4. + /// ext4 superblock is at offset 1024, magic number 0xEF53 is at offset 56. + fn write_ext4_magic(path: &std::path::Path) { + use std::io::{Seek, SeekFrom, Write}; + let mut file = std::fs::OpenOptions::new().write(true).open(path).unwrap(); + + file.seek(SeekFrom::Start(1080)).unwrap(); + file.write_all(&[0x53, 0xEF]).unwrap(); + file.sync_all().unwrap(); + } + + /// Corrupt the ext4 magic bytes by zeroing them. + fn corrupt_ext4_magic(path: &std::path::Path) { + use std::io::{Seek, SeekFrom, Write}; + let mut file = std::fs::OpenOptions::new().write(true).open(path).unwrap(); + + file.seek(SeekFrom::Start(1080)).unwrap(); + file.write_all(&[0x00, 0x00]).unwrap(); + file.sync_all().unwrap(); + } + + /// Check if a disk file appears to be a valid ext4 filesystem. + fn disk_appears_valid_ext4(disk_path: &Path) -> bool { + let output = std::process::Command::new("file") + .arg("-b") + .arg(disk_path) + .output(); + + match output { + Ok(output) if output.status.success() => { + let desc = String::from_utf8_lossy(&output.stdout); + let is_ext4 = + desc.contains("ext4") || desc.contains("ext2") || desc.contains("ext3"); + if !is_ext4 { + tracing::debug!( + path = %disk_path.display(), + file_type = %desc.trim(), + "disk is not ext4" + ); + } + is_ext4 + } + _ => { + tracing::debug!(path = %disk_path.display(), "could not verify disk type, assuming valid"); + true + } + } + } +} diff --git a/src/systemd_scope.rs b/src/systemd_scope.rs new file mode 100644 index 0000000..e4f7512 --- /dev/null +++ b/src/systemd_scope.rs @@ -0,0 +1,350 @@ +//! Place a VM process in its own **systemd transient scope** so it survives a +//! `serve` restart. +//! +//! Today a VM is a child process inside `smolvm-node.service`'s delegated cgroup. +//! On `systemctl restart`, a surviving VM left in that cgroup makes systemd fail +//! to recreate the unit (`status=219/CGROUP`) → serve crash-loops. Adopting the +//! VM into its own `smolvm-vm-.scope` (a sibling unit owned by PID1) moves it +//! out of the service cgroup, so serve can restart and reconnect to the still- +//! running VM. See `docs/lossless-serve-restart.md`. +//! +//! Implemented by shelling out to `busctl` (ships with systemd — no D-Bus crate +//! dependency, and absent exactly where scopes wouldn't work anyway). The caller +//! forks the VM normally (retaining stdio/fd/process-group control), then calls +//! [`adopt_into_scope`] on the resulting PID. systemd's `StartTransientUnit` +//! with a `PIDs=` property moves the process into the scope's cgroup and applies +//! the resource caps as unit properties. Scopes auto-remove when their last +//! process exits, so machine stop/delete needs no extra teardown. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::time::{Duration, Instant}; + +use crate::error::{Error, Result}; + +/// Hard wall-clock bound on any single `busctl` call. +/// +/// `busctl` is on both VM hot paths — `start` → [`adopt_into_scope`] and +/// `stop`/delete → [`kill_scope`] — and these run on the request-serving blocking +/// pool. systemd's `StartTransientUnit`/`KillUnit` can block for MINUTES when a VM +/// is wedged in an uninterruptible (D) kernel state (its cgroup won't die), and the +/// call had no timeout — so one stuck VM could pin blocking-pool threads until they +/// were exhausted, starving `start`/`exec` across the whole node while `/health` +/// (pure-async, no busctl) stayed green so auto-cordon never fired. This bounds the +/// call so the thread is freed in seconds; the teardown is retried out-of-band +/// instead of hanging the node. See the 2026-07-05 worker-1 wedge. +const BUSCTL_TIMEOUT: Duration = Duration::from_secs(10); + +/// Poll cadence while waiting for a `busctl` call to finish. Small relative to the +/// timeout; adds at most one interval of latency to a normal (fast) call. +const BUSCTL_POLL_INTERVAL: Duration = Duration::from_millis(25); + +/// Run a `busctl` command with a hard wall-clock bound ([`BUSCTL_TIMEOUT`]). +/// +/// On timeout the child is killed (busctl is a normal userspace process, so it dies +/// at once, freeing the thread) and a timeout error is returned for the caller to +/// handle out-of-band — never leaving a request-serving thread pinned on a wedged +/// systemd. std-only (no libc) so it compiles on every platform the callers do; +/// busctl's replies are tiny, so leaving stdout/stderr buffered until the poll loop +/// exits can't fill the pipe. +fn busctl_bounded(mut cmd: Command, timeout: Duration) -> Result { + let mut child = cmd + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| Error::agent("vm scope", format!("busctl spawn failed: {e}")))?; + let deadline = Instant::now() + timeout; + loop { + match child.try_wait() { + Ok(Some(_)) => break, + Ok(None) => { + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + return Err(Error::agent( + "vm scope", + format!( + "busctl timed out after {}s (systemd/cgroup wedged)", + timeout.as_secs() + ), + )); + } + std::thread::sleep(BUSCTL_POLL_INTERVAL); + } + Err(e) => return Err(Error::agent("vm scope", format!("busctl wait failed: {e}"))), + } + } + child + .wait_with_output() + .map_err(|e| Error::agent("vm scope", format!("busctl output failed: {e}"))) +} + +/// Resource caps applied to the VM's scope (as systemd unit properties). +/// +/// Mirrors the per-VM cgroup limits set by [`crate::process::place_in_cgroup`]: +/// `MemoryMax` ↔ `memory.max`, `CPUQuotaPerSecUSec` ↔ `cpu.max`, `TasksMax` ↔ +/// `pids.max`. +#[derive(Debug, Default, Clone)] +pub struct ScopeCaps { + /// Hard memory ceiling in bytes (`MemoryMax`). `None` = uncapped. + pub memory_max_bytes: Option, + /// CPU quota in microseconds-of-CPU-time per real second + /// (`CPUQuotaPerSecUSec`). For N vCPUs uncapped-overcommit, pass + /// `N * 1_000_000`. `None` = uncapped. + pub cpu_quota_usec_per_sec: Option, + /// Max number of tasks/PIDs (`TasksMax`). `None` = systemd default. + pub tasks_max: Option, +} + +/// True iff we can actually create system-bus transient scopes here: a systemd +/// host (`/run/systemd/system`, à la `sd_booted()`), `busctl` present, AND we run +/// as root. +/// +/// The root check matters: `StartTransientUnit` on the **system** bus needs root +/// (or polkit), and serve runs as root on the cloud worker. Without it, an +/// unprivileged local `serve` would pass the systemd check, enter scope-mode, +/// then have every adopt rejected — leaving VMs uncapped (scope-mode skips the +/// cgroup fallback). Returning false instead routes the caller to direct cgroup +/// placement, which keeps resource caps (it just isn't lossless — fine for dev). +/// +/// Non-systemd hosts (macOS dev, OpenRC, bare containers) also return false. +/// Future: support the per-user systemd bus (`busctl --user`) so an unprivileged +/// local serve can still get scopes. +pub fn is_available() -> bool { + // systemd scopes are a Linux-only concept; never available elsewhere. + #[cfg(not(target_os = "linux"))] + { + false + } + #[cfg(target_os = "linux")] + { + // SAFETY: geteuid() is always-safe (no args, no global state mutation). + let is_root = unsafe { libc::geteuid() } == 0; + is_root && Path::new("/run/systemd/system").is_dir() && busctl_path().is_some() + } +} + +/// Locate `busctl` (PATH, then the usual absolute locations — serve may run with +/// a minimal `PATH`). +fn busctl_path() -> Option { + for cand in ["/usr/bin/busctl", "/bin/busctl", "/usr/local/bin/busctl"] { + let p = Path::new(cand); + if p.exists() { + return Some(p.to_path_buf()); + } + } + // Fall back to PATH resolution via the shell-less which-equivalent. + std::env::var_os("PATH").and_then(|paths| { + std::env::split_paths(&paths) + .map(|d| d.join("busctl")) + .find(|p| p.exists()) + }) +} + +/// systemd unit names allow `[A-Za-z0-9:_.\-]`; sanitize the machine id and clamp +/// length so the scope name is always valid. Collisions are avoided by the caller +/// using unique machine ids; a dead scope self-removes when its VM exits. +pub fn scope_name(machine_id: &str) -> String { + let safe: String = machine_id + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | ':') { + c + } else { + '_' + } + }) + .take(200) + .collect(); + format!("smolvm-vm-{safe}.scope") +} + +/// Adopt an already-forked VM `pid` into its own transient scope with `caps`. +/// +/// Returns `Err` if systemd rejects the request (e.g. the PID already exited, or +/// the bus call failed); the caller should fall back to plain cgroup placement +/// rather than failing the launch. +pub fn adopt_into_scope(machine_id: &str, pid: i32, caps: &ScopeCaps) -> Result<()> { + let busctl = busctl_path() + .ok_or_else(|| Error::agent("vm scope", "busctl not found; cannot create scope"))?; + let name = scope_name(machine_id); + + // Build the a(sv) property list in busctl's positional encoding: + // + // PIDs is an array of u32 (`au`): "au ". + let mut props: Vec = Vec::new(); + let mut nprops: u32 = 0; + + props.extend(["PIDs".into(), "au".into(), "1".into(), pid.to_string()]); + nprops += 1; + props.extend([ + "Description".into(), + "s".into(), + format!("smolvm VM {machine_id}"), + ]); + nprops += 1; + if let Some(m) = caps.memory_max_bytes { + props.extend(["MemoryMax".into(), "t".into(), m.to_string()]); + nprops += 1; + } + if let Some(q) = caps.cpu_quota_usec_per_sec { + props.extend(["CPUQuotaPerSecUSec".into(), "t".into(), q.to_string()]); + nprops += 1; + } + if let Some(t) = caps.tasks_max { + props.extend(["TasksMax".into(), "t".into(), t.to_string()]); + nprops += 1; + } + + // StartTransientUnit(name: s, mode: s, properties: a(sv), aux: a(sa(sv))). + // mode "fail": error if the unit already exists (a stale same-name scope must + // have been GC'd first — it would have been, when its VM exited). + let mut args: Vec = vec![ + "call".into(), + "org.freedesktop.systemd1".into(), + "/org/freedesktop/systemd1".into(), + "org.freedesktop.systemd1.Manager".into(), + "StartTransientUnit".into(), + "ssa(sv)a(sa(sv))".into(), + name.clone(), + "fail".into(), + nprops.to_string(), + ]; + args.extend(props); + args.push("0".into()); // empty aux array + + let mut cmd = Command::new(&busctl); + cmd.args(&args); + let out = busctl_bounded(cmd, BUSCTL_TIMEOUT)?; + + if !out.status.success() { + let stderr = String::from_utf8_lossy(&out.stderr); + return Err(Error::agent( + "vm scope", + format!("StartTransientUnit {name} failed: {}", stderr.trim()), + )); + } + tracing::info!(scope = %name, pid, "adopted VM into systemd transient scope"); + Ok(()) +} + +/// Force-kill a VM's transient scope: SIGKILL every process in its cgroup. +/// +/// This is the AUTHORITATIVE teardown when the pid-based delete can't confirm +/// death — no recorded pid, or a process that outlived SIGKILL-by-pid. The scope +/// owns every process the VM spawned regardless of the pid we happened to record, +/// so killing the cgroup is what stops a stuck/crash-looping VM that would +/// otherwise become an un-deletable orphan (control marks it deleted, the node +/// keeps running it). Returns `Ok(true)` on a successful kill OR a missing scope +/// (already gone == teardown done). Non-Linux / no-busctl hosts return `Ok(false)` +/// — nothing scope-based to stop; the caller falls back to its pid path. +pub fn kill_scope(machine_id: &str) -> Result { + #[cfg(not(target_os = "linux"))] + { + let _ = machine_id; + Ok(false) + } + #[cfg(target_os = "linux")] + { + let Some(busctl) = busctl_path() else { + return Ok(false); + }; + let name = scope_name(machine_id); + // KillUnit(name: s, who: s, signal: i): SIGKILL (9) every process in the + // scope ("all"). SIGKILL is uncatchable, so a wedged VM dies immediately; + // the emptied scope then self-GCs. `ssi` is the busctl arg signature. + let mut cmd = Command::new(&busctl); + cmd.args([ + "call", + "org.freedesktop.systemd1", + "/org/freedesktop/systemd1", + "org.freedesktop.systemd1.Manager", + "KillUnit", + "ssi", + &name, + "all", + "9", + ]); + let out = busctl_bounded(cmd, BUSCTL_TIMEOUT)?; + if out.status.success() { + tracing::info!(scope = %name, "SIGKILLed VM transient scope (forced teardown)"); + return Ok(true); + } + // A scope that already exited is not loaded — teardown is effectively done. + let stderr = String::from_utf8_lossy(&out.stderr); + if stderr.contains("not loaded") + || stderr.contains("NoSuchUnit") + || stderr.contains("not found") + { + tracing::debug!(scope = %name, "scope already gone on kill"); + return Ok(true); + } + Err(Error::agent( + "vm scope", + format!("KillUnit {name} failed: {}", stderr.trim()), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // A wedged busctl (systemd stuck on a dying cgroup) must not pin the thread: + // the call is bounded and returns promptly, well under the hang it replaces. + #[cfg(unix)] + #[test] + fn busctl_bounded_kills_a_hung_call() { + let mut cmd = Command::new("sleep"); + cmd.arg("30"); // stand-in for a busctl call that never returns + let start = Instant::now(); + let out = busctl_bounded(cmd, Duration::from_millis(150)); + let elapsed = start.elapsed(); + assert!( + out.is_err(), + "a call exceeding the bound must error, not hang" + ); + assert!( + out.unwrap_err().to_string().contains("timed out"), + "the error must identify a timeout" + ); + // Freed in ~the bound, not the child's 30s runtime — a generous ceiling. + assert!( + elapsed < Duration::from_secs(2), + "bounded call took {elapsed:?}; should return near the 150ms bound" + ); + } + + // A fast call returns its real output and status, unaffected by the bound. + #[cfg(unix)] + #[test] + fn busctl_bounded_returns_fast_call_output() { + let mut cmd = Command::new("printf"); + cmd.arg("ok"); + let out = busctl_bounded(cmd, Duration::from_secs(5)).expect("fast call"); + assert!(out.status.success()); + assert_eq!(String::from_utf8_lossy(&out.stdout), "ok"); + } + + #[test] + fn scope_name_sanitizes_and_suffixes() { + assert_eq!( + scope_name("machine-abc123"), + "smolvm-vm-machine-abc123.scope" + ); + // Illegal chars (slash, space) collapse to underscore. + assert_eq!(scope_name("a/b c"), "smolvm-vm-a_b_c.scope"); + // Allowed punctuation is preserved. + assert_eq!(scope_name("m_1.2:3"), "smolvm-vm-m_1.2:3.scope"); + } + + #[test] + fn scope_name_is_bounded() { + let long = "x".repeat(500); + let n = scope_name(&long); + assert!(n.starts_with("smolvm-vm-")); + assert!(n.ends_with(".scope")); + // sanitized body clamped to 200 chars + fixed prefix/suffix. + assert!(n.len() <= "smolvm-vm-".len() + 200 + ".scope".len()); + } +} diff --git a/src/util.rs b/src/util.rs new file mode 100644 index 0000000..fcc6801 --- /dev/null +++ b/src/util.rs @@ -0,0 +1,159 @@ +//! Shared utility functions. + +use std::time::{SystemTime, UNIX_EPOCH}; + +// Re-export retry utilities from the protocol crate for convenience. +// This provides a single source of truth for retry logic across the codebase. +pub use smolvm_protocol::retry::{ + is_transient_io_error, is_transient_network_error, retry_with_backoff, RetryConfig, +}; + +/// Generate a short random ID for auto-naming machines. +/// +/// Produces an 8-character hex string (e.g., "a1b2c3d4") from 4 bytes +/// of OS entropy. Falls back to time+pid if /dev/urandom is unavailable. +pub fn generate_short_id() -> String { + let mut buf = [0u8; 4]; + if std::fs::File::open("/dev/urandom") + .and_then(|mut f| { + use std::io::Read; + f.read_exact(&mut buf) + }) + .is_ok() + { + return format!("{:08x}", u32::from_le_bytes(buf)); + } + // Fallback: time + pid (less random but functional) + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + format!("{:08x}", (nanos as u32) ^ std::process::id()) +} + +/// Generate an auto machine name (e.g., "vm-a1b2c3d4"). +pub fn generate_machine_name() -> String { + format!("vm-{}", generate_short_id()) +} + +/// Get current timestamp as seconds since Unix epoch. +pub fn current_timestamp() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +/// Get the filename of libkrunfw dynamic lib +pub fn libkrunfw_filename() -> &'static str { + #[cfg(target_os = "macos")] + let lib_name = "libkrunfw.5.dylib"; + #[cfg(target_os = "linux")] + let lib_name = "libkrunfw.so.5"; + #[cfg(target_os = "windows")] + let lib_name = "libkrunfw.dll"; + lib_name +} + +/// Get the filename of the libkrun dynamic lib +pub fn libkrun_filename() -> &'static str { + #[cfg(target_os = "macos")] + let lib_name = "libkrun.dylib"; + #[cfg(target_os = "linux")] + let lib_name = "libkrun.so"; + #[cfg(target_os = "windows")] + let lib_name = "krun.dll"; + lib_name +} + +/// Parse a single environment variable specification. +/// +/// - `KEY=VALUE` → `Some((KEY, VALUE))`. +/// - `KEY` (no `=`) → forwards the host's current value for `KEY` (Docker +/// `-e KEY` semantics); `None` if the host variable is unset. +/// - empty key (`=VALUE`) or empty spec → `None`. +pub fn parse_env_spec(spec: &str) -> Option<(String, String)> { + match spec.split_once('=') { + Some((key, value)) => { + if key.is_empty() { + None + } else { + Some((key.to_string(), value.to_string())) + } + } + None => { + if spec.is_empty() { + None + } else { + // Key-only form: forward the value from the host environment. + std::env::var(spec).ok().map(|v| (spec.to_string(), v)) + } + } + } +} + +/// Parse a list of `KEY=VALUE` strings into `(key, value)` tuples. +/// +/// Silently skips malformed entries (no `=` or empty key). +pub fn parse_env_list(env_args: &[String]) -> Vec<(String, String)> { + env_args.iter().filter_map(|e| parse_env_spec(e)).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + #[test] + fn parse_env_spec_key_value() { + assert_eq!( + parse_env_spec("FOO=bar"), + Some(("FOO".to_string(), "bar".to_string())) + ); + // Empty value is allowed. + assert_eq!( + parse_env_spec("FOO="), + Some(("FOO".to_string(), String::new())) + ); + // Empty key is rejected. + assert_eq!(parse_env_spec("=bar"), None); + assert_eq!(parse_env_spec(""), None); + } + + #[test] + fn parse_env_spec_key_only_forwards_host_value() { + let key = "SMOLVM_TEST_ENV_FWD_XYZ"; + std::env::set_var(key, "host_value"); + assert_eq!( + parse_env_spec(key), + Some((key.to_string(), "host_value".to_string())) + ); + std::env::remove_var(key); + // Unset host var → skipped. + assert_eq!(parse_env_spec(key), None); + } + + #[test] + fn test_generate_ids() { + // Generate 100 IDs and validate all of them + let ids: Vec = (0..100).map(|_| generate_short_id()).collect(); + + for id in &ids { + assert_eq!(id.len(), 8, "should be 8 hex chars: {id}"); + assert!(id.chars().all(|c| c.is_ascii_hexdigit()), "not hex: {id}"); + } + + // All unique + let unique: HashSet<&String> = ids.iter().collect(); + assert_eq!(unique.len(), 100, "100 IDs should all be unique"); + + // Machine name wraps the ID correctly + let name = generate_machine_name(); + assert!(name.starts_with("vm-"), "prefix: {name}"); + assert_eq!(name.len(), 11, "length: {name}"); + assert!(name.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')); + + // Two names differ + assert_ne!(generate_machine_name(), generate_machine_name()); + } +} diff --git a/src/vm/backend/libkrun.rs b/src/vm/backend/libkrun.rs new file mode 100644 index 0000000..ff9e842 --- /dev/null +++ b/src/vm/backend/libkrun.rs @@ -0,0 +1,697 @@ +//! libkrun backend implementation. +//! +//! This module provides VM creation and management using libkrun, +//! which uses Hypervisor.framework on macOS and KVM on Linux. + +use std::ffi::CString; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +use crate::agent::{find_lib_dir, KrunFunctions}; +use crate::data::storage::HostMount; +use crate::error::{Error, Result}; +use crate::platform::{self, VmExecutor}; +use crate::vm::config::{NetworkPolicy, RootfsSource, VmConfig}; +use crate::vm::rosetta; +use crate::vm::state::{ExitReason, VmState}; +use crate::vm::{VmBackend, VmHandle, VmId}; + +/// libkrun backend for VM creation. +pub struct LibkrunBackend { + /// Whether libkrun appears to be available. + available: bool, +} + +impl LibkrunBackend { + /// Create a new libkrun backend. + pub fn new() -> Result { + Ok(Self { + available: find_lib_dir().is_some(), + }) + } +} + +impl VmBackend for LibkrunBackend { + fn name(&self) -> &'static str { + "libkrun" + } + + fn is_available(&self) -> bool { + self.available + } + + fn create(&self, config: VmConfig) -> Result> { + LibkrunVm::create(config).map(|vm| Box::new(vm) as Box) + } +} + +/// A VM instance managed by libkrun. +pub struct LibkrunVm { + id: VmId, + state: VmState, + exit_reason: Option, + /// Child process running the VM. + child: Option, +} + +impl LibkrunVm { + /// Create and start a VM with the given configuration. + fn create(config: VmConfig) -> Result { + let id = config.id.clone(); + + // Resolve rootfs to a path + let rootfs_path = resolve_rootfs(&config.rootfs)?; + + // Inject init.krun into rootfs (required by libkrunfw kernel) + inject_init_krun(&rootfs_path)?; + + // Setup DNS if network egress is enabled + if let NetworkPolicy::Egress { dns, .. } = &config.network { + setup_dns(&rootfs_path, dns.map(|ip| ip.to_string()).as_deref())?; + } + + let mut vm = Self { + id, + state: VmState::Created, + exit_reason: None, + child: None, + }; + + // Execute VM (this blocks until VM exits) + vm.state = VmState::Booting; + let result = vm.exec_vm(&rootfs_path, &config); + + match result { + Ok(code) => { + vm.state = VmState::Stopped; + vm.exit_reason = Some(ExitReason::exited(code)); + } + Err(e) => { + vm.state = VmState::Failed { + reason: e.to_string(), + }; + vm.exit_reason = Some(ExitReason::vm_crash(e.to_string())); + } + } + + Ok(vm) + } + + /// Execute the VM using libkrun FFI. + fn exec_vm(&mut self, rootfs_path: &Path, config: &VmConfig) -> Result { + // Raise file descriptor limits (required by libkrun) + set_rlimits(); + + let lib_dir = + find_lib_dir().ok_or_else(|| Error::vm_creation("libkrun/libkrunfw not found"))?; + let krun = unsafe { KrunFunctions::load(&lib_dir) } + .map_err(|e| Error::vm_creation(format!("failed to load libkrun: {e}")))?; + + unsafe { + let krun_set_log_level = krun.set_log_level; + let krun_create_ctx = krun.create_ctx; + let krun_free_ctx = krun.free_ctx; + let krun_set_vm_config = krun.set_vm_config; + let krun_set_workdir = krun.set_workdir; + let krun_set_exec = krun.set_exec; + let krun_add_virtiofs = krun.add_virtiofs; + let krun_add_virtiofs3 = krun.add_virtiofs3; + let krun_set_port_map = krun.set_port_map; + let krun_add_disk2 = krun.add_disk2; + let krun_add_vsock = krun.add_vsock; + let krun_add_vsock_port2 = krun.add_vsock_port2; + let krun_start_enter = krun.start_enter; + + // Initialize libkrun logging (0 = off, 1 = error, 2 = warn, 3 = info, 4 = debug) + // Use 0 (off) in production - smolvm has its own logging via tracing + krun_set_log_level(0); + + // Create VM context + let ctx = krun_create_ctx(); + if ctx < 0 { + return Err(Error::vm_creation("failed to create libkrun context")); + } + let ctx = ctx as u32; + + // Set VM resources + if krun_set_vm_config(ctx, config.cpus, config.memory_mib) < 0 { + krun_free_ctx(ctx); + return Err(Error::vm_creation("failed to set VM config")); + } + + // Set root filesystem via the root virtiofs tag (upstream removed + // krun_set_root in favor of krun_add_virtiofs with KRUN_FS_ROOT_TAG). + let root = path_to_cstring(rootfs_path)?; + let root_tag = CString::new("/dev/root").expect("static tag"); + tracing::debug!("[libkrun] rootfs_path: {:?}", rootfs_path); + // 512 MB root DAX window (matches the removed krun_set_root); plain + // krun_add_virtiofs uses shm_size=0 (no DAX) → writeback caching that + // delays host visibility of guest writes (e.g. the agent ready marker). + let Some(krun_add_virtiofs3) = krun_add_virtiofs3 else { + krun_free_ctx(ctx); + return Err(Error::vm_creation( + "root DAX requires libkrun with krun_add_virtiofs3", + )); + }; + let ret = krun_add_virtiofs3(ctx, root_tag.as_ptr(), root.as_ptr(), 1 << 29, false); + tracing::debug!("[libkrun] krun_add_virtiofs3(root) returned: {}", ret); + if ret < 0 { + krun_free_ctx(ctx); + return Err(Error::vm_creation("failed to set root filesystem")); + } + + // Upstream libkrun no longer creates an implicit vsock; both + // krun_set_port_map and krun_add_vsock_port2 require a vsock device + // (else -ENODEV). Add it explicitly (no TSI hijacking) before they run. + if krun_add_vsock(ctx, 0) < 0 { + krun_free_ctx(ctx); + return Err(Error::vm_creation("failed to add vsock device")); + } + + // Set empty port map (required by libkrun) + let empty_ports: Vec<*const libc::c_char> = vec![std::ptr::null()]; + if krun_set_port_map(ctx, empty_ports.as_ptr()) < 0 { + krun_free_ctx(ctx); + return Err(Error::vm_creation("failed to set port map")); + } + + // Note: libkrun's implicit console connects stdin/stdout/stderr automatically. + // In libkrun 1.15.x, krun_add_virtio_console_default is not available. + // Console output should work via the implicit console mechanism. + + // Set working directory if specified + if let Some(ref wd) = config.workdir { + let workdir = path_to_cstring(wd)?; + if krun_set_workdir(ctx, workdir.as_ptr()) < 0 { + tracing::warn!(workdir = %wd.display(), "failed to set workdir"); + } + } + + // Build environment with defaults + let (envp, _env_cstrings) = build_env_args(&config.env, &self.id)?; + + // Build mounts list for wrapper script: (tag, guest_path) + let mount_specs: Vec<(String, String)> = config + .mounts + .iter() + .enumerate() + .map(|(i, m)| { + ( + HostMount::mount_tag(i), + m.target.to_string_lossy().to_string(), + ) + }) + .collect(); + + // Determine if Rosetta should be enabled + // Only enable if requested in config AND Rosetta is actually available + let rosetta_enabled = config.rosetta && rosetta::is_available(); + if config.rosetta && !rosetta_enabled { + tracing::warn!( + "Rosetta requested but not available - x86_64 binaries will not work" + ); + } + if rosetta_enabled { + tracing::info!("Rosetta enabled for x86_64 binary support"); + } + + // Build exec command using platform-specific executor + // On macOS, this wraps the command with a mount script for virtiofs + // On Linux, the kernel handles virtiofs mounting automatically + let executor = platform::vm_executor(); + let (exec_path, argv, _argv_cstrings) = executor.build_exec_command( + &config.command, + &mount_specs, + rootfs_path, + rosetta_enabled, + )?; + + tracing::debug!("[libkrun] exec_path: {:?}", exec_path); + tracing::debug!("[libkrun] command: {:?}", config.command); + tracing::debug!("[libkrun] argv count (excluding null): {}", argv.len() - 1); + tracing::debug!("[libkrun] argv_cstrings: {:?}", _argv_cstrings); + + let ret = krun_set_exec(ctx, exec_path.as_ptr(), argv.as_ptr(), envp.as_ptr()); + tracing::debug!("[libkrun] krun_set_exec returned: {}", ret); + if ret < 0 { + krun_free_ctx(ctx); + return Err(Error::vm_creation("failed to set exec command")); + } + + // Add virtiofs mounts for host directories. Prefer krun_add_virtiofs3 + // with a DAX window (like the root fs): without DAX, virtiofs falls + // back to writeback caching where every file access is a FUSE + // round-trip over the virtio queue — pathological for read-heavy + // mounts with many files (e.g. a multi-GB Python venv taking minutes + // just to import). DAX maps file contents through a coherent host + // window, giving near-native read throughput. Fall back to the plain + // API only on an older libkrun that lacks virtiofs3. + // 2 GiB: the window must exceed the largest single file mapped from + // the volume, or mapping that file stalls (a 902 MB libtorch_cuda.so + // hung against a 512 MB window). The window is virtual host address + // space backed on demand, so oversizing costs nothing until touched. + const VIRTIOFS_DAX_WINDOW: u64 = 1 << 31; + for (i, mount) in config.mounts.iter().enumerate() { + let tag = CString::new(HostMount::mount_tag(i)) + .map_err(|_| Error::mount("create mount tag", "tag contains null byte"))?; + let path = path_to_cstring(&mount.source)?; + + // `krun_add_virtiofs3` is already unwrapped above (the root fs + // requires it), so use it directly with a DAX window. + let ret = krun_add_virtiofs3( + ctx, + tag.as_ptr(), + path.as_ptr(), + VIRTIOFS_DAX_WINDOW, + mount.read_only, + ); + if ret < 0 { + tracing::warn!( + "failed to add virtiofs mount: {} -> {}", + mount.source.display(), + mount.target.display() + ); + } + } + + // Add Rosetta virtiofs mount if enabled + if rosetta_enabled { + if let Some(runtime_path) = rosetta::runtime_path() { + let tag = CString::new(rosetta::ROSETTA_TAG).map_err(|_| { + Error::mount("create rosetta tag", "tag contains null byte") + })?; + let path = CString::new(runtime_path).map_err(|_| { + Error::mount("create rosetta path", "path contains null byte") + })?; + + if krun_add_virtiofs(ctx, tag.as_ptr(), path.as_ptr()) < 0 { + tracing::warn!("failed to add Rosetta virtiofs mount"); + } else { + tracing::debug!( + tag = rosetta::ROSETTA_TAG, + path = runtime_path, + "added Rosetta virtiofs mount" + ); + } + } + } + + // Add block devices + for disk in &config.disks { + let block_id = CString::new(disk.block_id.as_str()) + .map_err(|_| Error::vm_creation("invalid block_id"))?; + let disk_path = path_to_cstring(&disk.path)?; + let format = disk.format as u32; + + if krun_add_disk2( + ctx, + block_id.as_ptr(), + disk_path.as_ptr(), + format, + disk.read_only, + ) < 0 + { + tracing::warn!( + "failed to add disk: {} ({})", + disk.block_id, + disk.path.display() + ); + } else { + tracing::debug!( + block_id = %disk.block_id, + path = %disk.path.display(), + "added block device" + ); + } + } + + // Add vsock ports + for vsock in &config.vsock_ports { + let socket_path = path_to_cstring(&vsock.socket_path)?; + + let ret = krun_add_vsock_port2(ctx, vsock.port, socket_path.as_ptr(), vsock.listen); + if ret < 0 { + tracing::warn!( + "failed to add vsock port {}: {}", + vsock.port, + vsock.socket_path.display() + ); + } else { + tracing::debug!( + port = vsock.port, + socket = %vsock.socket_path.display(), + listen = vsock.listen, + "added vsock port" + ); + } + } + + // Redirect console output to a file if specified, via the upstream + // virtio-console API (krun_set_console_output was removed). + if let Some(ref log_path) = config.console_log { + if krun.console_output_to_file(ctx, log_path) < 0 { + tracing::warn!("failed to set console output: {}", log_path.display()); + } else { + tracing::debug!(path = %log_path.display(), "console output enabled"); + } + } + + // Register a control socket (pause/resume/checkpoint/restore) when + // requested via SMOLVM_CONTROL_SOCKET. Best-effort: a missing symbol + // (older libkrun) or failure just leaves the VM without a control + // channel rather than failing the boot. + if let Ok(ctl_path) = std::env::var("SMOLVM_CONTROL_SOCKET") { + if !ctl_path.is_empty() { + match krun.set_control_socket { + Some(set_control_socket) => { + if let Ok(ctl_c) = std::ffi::CString::new(ctl_path.clone()) { + let ret = set_control_socket(ctx, ctl_c.as_ptr()); + if ret < 0 { + tracing::warn!("krun_set_control_socket failed: {ret}"); + } else { + tracing::info!(socket = %ctl_path, "control socket enabled"); + } + } + } + None => tracing::warn!( + "SMOLVM_CONTROL_SOCKET set but libkrun lacks krun_set_control_socket" + ), + } + } + } + + // Update state to running + self.state = VmState::Running; + + // Fork before starting VM because krun_start_enter calls exit() directly + // The child runs the VM, parent waits and returns the exit code + tracing::info!(vm_id = %self.id, "starting VM"); + + let pid = libc::fork(); + if pid < 0 { + Err(Error::vm_creation("fork failed")) + } else if pid == 0 { + // Child process: run the VM + // This will call exit() when the VM exits + krun_start_enter(ctx); + // If we get here, something went wrong + libc::_exit(1); + } else { + // Parent process: store child process and wait + let mut child = crate::process::ChildProcess::new(pid); + let exit_code = child.wait(); + self.child = Some(child); + + // Clear child reference after exit + self.child = None; + + Ok(exit_code) + } + } + } +} + +/// Raise file descriptor limits (required by libkrun). +fn set_rlimits() { + unsafe { + let mut limit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + + if libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) == 0 { + limit.rlim_cur = limit.rlim_max; + libc::setrlimit(libc::RLIMIT_NOFILE, &limit); + } + } +} + +impl VmHandle for LibkrunVm { + fn id(&self) -> &VmId { + &self.id + } + + fn state(&self) -> VmState { + self.state.clone() + } + + fn wait(&mut self) -> Result { + // VM already exited since krun_start_enter blocks + self.exit_reason + .clone() + .ok_or_else(|| Error::vm_not_found(&self.id.0)) + } + + fn stop(&mut self) -> Result<()> { + if let Some(ref mut child) = self.child { + if child.is_running() { + tracing::info!(pid = child.pid(), "stopping VM with SIGTERM"); + child.stop(crate::process::DEFAULT_STOP_TIMEOUT, true)?; + self.state = VmState::Stopped; + } + } + Ok(()) + } + + fn kill(&mut self) -> Result<()> { + if let Some(ref child) = self.child { + if crate::process::is_alive(child.pid()) { + tracing::info!(pid = child.pid(), "killing VM with SIGKILL"); + crate::process::kill(child.pid()); + } + } + self.state = VmState::Stopped; + Ok(()) + } +} + +// Helper functions + +/// Resolve a rootfs source to an actual path. +fn resolve_rootfs(source: &RootfsSource) -> Result { + match source { + RootfsSource::Path { path } => { + if path.exists() { + Ok(path.clone()) + } else { + Err(Error::RootfsNotFound { path: path.clone() }) + } + } + } +} + +/// Setup DNS configuration in the rootfs. +fn setup_dns(rootfs: &Path, dns: Option<&str>) -> Result<()> { + let resolv_path = rootfs.join("etc/resolv.conf"); + + // Only write if etc directory exists + if let Some(parent) = resolv_path.parent() { + if parent.exists() { + let content = format!("nameserver {}\n", dns.unwrap_or("1.1.1.1")); + std::fs::write(&resolv_path, content)?; + tracing::debug!("wrote DNS config to {:?}", resolv_path); + } + } + + Ok(()) +} + +/// Inject init.krun into the rootfs. +/// +/// libkrunfw's kernel is built without initramfs support, so it expects +/// /init.krun to be present in the virtiofs root filesystem. +fn inject_init_krun(rootfs: &Path) -> Result<()> { + let target = rootfs.join("init.krun"); + tracing::debug!("[libkrun] inject_init_krun: target = {:?}", target); + + // Check if init.krun already exists + if target.exists() { + tracing::debug!("[libkrun] init.krun already present in rootfs"); + tracing::debug!("init.krun already present in rootfs"); + return Ok(()); + } + + // Look for init.krun in standard locations + let sources = [ + // User's local smolvm data directory (macOS: ~/Library/Application Support) + dirs::data_local_dir().map(|d| d.join("smolvm/init.krun")), + // XDG data home (Linux: ~/.local/share, works on macOS too) + dirs::home_dir().map(|d| d.join(".local/share/smolvm/init.krun")), + // System-wide location + Some(PathBuf::from("/usr/local/share/smolvm/init.krun")), + // Homebrew location + Some(PathBuf::from("/opt/homebrew/share/smolvm/init.krun")), + ]; + + for source in sources.into_iter().flatten() { + tracing::debug!("[libkrun] checking for init.krun at {:?}", source); + if source.exists() { + tracing::debug!( + "[libkrun] found init.krun at {:?}, copying to {:?}", + source, + target + ); + std::fs::copy(&source, &target).map_err(|e| { + Error::vm_creation(format!("failed to copy init.krun to rootfs: {}", e)) + })?; + // Make executable + let perms = std::fs::Permissions::from_mode(0o755); + std::fs::set_permissions(&target, perms)?; + tracing::debug!("[libkrun] successfully injected init.krun"); + tracing::debug!("injected init.krun from {:?} to {:?}", source, target); + return Ok(()); + } + } + + Err(Error::vm_creation( + "init.krun not found. Please install smolvm-init or build from source. \ + See https://github.com/smolvm/smolvm#init-krun for details.", + )) +} + +/// Convert a Path to a CString. +fn path_to_cstring(path: &Path) -> Result { + CString::new(path.to_string_lossy().as_bytes()) + .map_err(|_| Error::vm_creation("path contains null byte")) +} + +/// Build exec arguments for libkrun (test helper). +/// +/// libkrun passes exec_path as KRUN_INIT env var, and argv is appended to the +/// kernel cmdline after " -- ". init.krun replaces argv[0] with KRUN_INIT. +/// Therefore, argv should NOT include the command name (argv[0]) - only the +/// arguments starting from argv[1]. +/// +/// Note: Production code uses `platform::vm_executor().build_exec_command()` instead. +#[cfg(test)] +fn build_exec_args( + command: &Option>, +) -> Result<(CString, Vec<*const libc::c_char>, Vec)> { + let default_cmd = vec!["/bin/sh".to_string()]; + let cmd = command.as_ref().unwrap_or(&default_cmd); + + if cmd.is_empty() { + return Err(Error::vm_creation("command cannot be empty")); + } + + let exec_path = + CString::new(cmd[0].as_str()).map_err(|_| Error::vm_creation("invalid command path"))?; + + // Do NOT include argv[0] - libkrun/init.krun handles it via KRUN_INIT. + // Only pass arguments (cmd[1..]). + let cstrings: Vec = cmd + .iter() + .skip(1) // Skip argv[0], it's passed via exec_path/KRUN_INIT + .map(|s| CString::new(s.as_str())) + .collect::, _>>() + .map_err(|_| Error::vm_creation("invalid command argument"))?; + + let mut argv: Vec<*const libc::c_char> = cstrings.iter().map(|s| s.as_ptr()).collect(); + argv.push(std::ptr::null()); + + Ok((exec_path, argv, cstrings)) +} + +/// Build environment variables for libkrun. +fn build_env_args( + env: &[(String, String)], + vm_id: &VmId, +) -> Result<(Vec<*const libc::c_char>, Vec)> { + let mut cstrings: Vec = Vec::new(); + + // Add default environment variables + cstrings.push( + CString::new(format!("HOSTNAME={}", vm_id.as_str())) + .map_err(|_| Error::vm_creation("invalid hostname"))?, + ); + cstrings.push(CString::new("HOME=/root").map_err(|_| Error::vm_creation("invalid HOME"))?); + cstrings.push( + CString::new("PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") + .map_err(|_| Error::vm_creation("invalid PATH"))?, + ); + cstrings + .push(CString::new("TERM=xterm-256color").map_err(|_| Error::vm_creation("invalid TERM"))?); + + // Add user-provided environment variables + for (k, v) in env { + cstrings.push( + CString::new(format!("{}={}", k, v)) + .map_err(|_| Error::vm_creation("invalid environment variable"))?, + ); + } + + let mut envp: Vec<*const libc::c_char> = cstrings.iter().map(|s| s.as_ptr()).collect(); + envp.push(std::ptr::null()); + + Ok((envp, cstrings)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_exec_args_default() { + let (exec_path, argv, _) = build_exec_args(&None).unwrap(); + assert_eq!(exec_path.to_str().unwrap(), "/bin/sh"); + // argv should NOT include argv[0] - only the arguments after it + // For default "/bin/sh" with no args: argv = [null] + assert_eq!(argv.len(), 1); // just [null] + } + + #[test] + fn test_build_exec_args_custom() { + let cmd = Some(vec![ + "/bin/echo".to_string(), + "hello".to_string(), + "world".to_string(), + ]); + let (exec_path, argv, _) = build_exec_args(&cmd).unwrap(); + assert_eq!(exec_path.to_str().unwrap(), "/bin/echo"); + // argv should NOT include argv[0] - only ["hello", "world", null] + assert_eq!(argv.len(), 3); // ["hello", "world", null] + } + + #[test] + fn test_build_exec_args_empty() { + let cmd = Some(vec![]); + let result = build_exec_args(&cmd); + assert!(result.is_err()); + } + + #[test] + fn test_build_env_args() { + let env = vec![ + ("FOO".to_string(), "bar".to_string()), + ("BAZ".to_string(), "qux".to_string()), + ]; + let vm_id = VmId::new("test-vm"); + let (envp, cstrings) = build_env_args(&env, &vm_id).unwrap(); + // 4 defaults (HOSTNAME, HOME, PATH, TERM) + 2 user vars + null + assert_eq!(cstrings.len(), 6); + assert_eq!(envp.len(), 7); + assert_eq!(cstrings[0].to_str().unwrap(), "HOSTNAME=test-vm"); + assert_eq!(cstrings[1].to_str().unwrap(), "HOME=/root"); + assert!(cstrings[2].to_str().unwrap().starts_with("PATH=")); + assert!(cstrings[3].to_str().unwrap().starts_with("TERM=")); + assert_eq!(cstrings[4].to_str().unwrap(), "FOO=bar"); + assert_eq!(cstrings[5].to_str().unwrap(), "BAZ=qux"); + } + + #[test] + fn test_build_env_args_empty() { + let env: Vec<(String, String)> = vec![]; + let vm_id = VmId::new("test-vm"); + let (envp, cstrings) = build_env_args(&env, &vm_id).unwrap(); + // 4 defaults (HOSTNAME, HOME, PATH, TERM) + null + assert_eq!(cstrings.len(), 4); + assert_eq!(envp.len(), 5); + } + + #[test] + fn test_path_to_cstring() { + let path = Path::new("/some/path"); + let cstring = path_to_cstring(path).unwrap(); + assert_eq!(cstring.to_str().unwrap(), "/some/path"); + } +} diff --git a/src/vm/backend/mod.rs b/src/vm/backend/mod.rs new file mode 100644 index 0000000..0c1f793 --- /dev/null +++ b/src/vm/backend/mod.rs @@ -0,0 +1,27 @@ +//! VM backend implementations. +//! +//! This module provides hypervisor backend implementations for different platforms. + +#[cfg(any(target_os = "macos", target_os = "linux"))] +mod libkrun; + +use crate::error::{Error, Result}; +use crate::vm::VmBackend; + +#[cfg(any(target_os = "macos", target_os = "linux"))] +pub use libkrun::LibkrunBackend; + +/// Create the default backend for this platform. +pub fn create_default() -> Result> { + #[cfg(any(target_os = "macos", target_os = "linux"))] + { + let backend = LibkrunBackend::new()?; + if backend.is_available() { + return Ok(Box::new(backend)); + } + } + + Err(Error::HypervisorUnavailable( + "no available backend for this platform".into(), + )) +} diff --git a/src/vm/config.rs b/src/vm/config.rs new file mode 100644 index 0000000..e2a0f94 --- /dev/null +++ b/src/vm/config.rs @@ -0,0 +1,520 @@ +//! VM configuration types. + +pub use crate::data::storage::HostMount; + +use serde::{Deserialize, Serialize}; +use std::net::IpAddr; +use std::path::PathBuf; +use std::time::Duration; + +/// Unique identifier for a VM instance. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct VmId(pub String); + +impl VmId { + /// Create a new VmId from a string. + /// + /// The ID is sanitized to only allow alphanumeric characters, dashes, and underscores. + /// IDs are limited to 64 characters. If the input is empty or contains only invalid + /// characters, a generated ID is used instead. + pub fn new(id: impl Into) -> Self { + let id = id.into(); + // Sanitize: only allow alphanumeric, dash, underscore + let sanitized: String = id + .chars() + .filter(|c| c.is_alphanumeric() || *c == '-' || *c == '_') + .take(64) // Max length + .collect(); + if sanitized.is_empty() { + Self::generate() + } else { + Self(sanitized) + } + } + + /// Generate a unique VmId based on timestamp. + pub fn generate() -> Self { + use std::time::{SystemTime, UNIX_EPOCH}; + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system time before unix epoch") + .as_millis(); + Self(format!("vm-{:x}", ts)) + } + + /// Get the ID as a string slice. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for VmId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl AsRef for VmId { + fn as_ref(&self) -> &str { + &self.0 + } +} + +/// Timeout configuration (aligned with DESIGN.md defaults). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Timeouts { + /// Boot timeout (default: 30s). + #[serde(with = "humantime_serde")] + pub boot: Duration, + + /// Shutdown timeout (default: 10s). + #[serde(with = "humantime_serde")] + pub shutdown: Duration, + + /// Optional execution timeout. + #[serde(default, with = "option_duration")] + pub exec: Option, +} + +impl Default for Timeouts { + fn default() -> Self { + Self { + boot: Duration::from_secs(30), + shutdown: Duration::from_secs(10), + exec: None, + } + } +} + +mod humantime_serde { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use std::time::Duration; + + pub fn serialize(duration: &Duration, serializer: S) -> Result + where + S: Serializer, + { + duration.as_secs().serialize(serializer) + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let secs = u64::deserialize(deserializer)?; + Ok(Duration::from_secs(secs)) + } +} + +mod option_duration { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use std::time::Duration; + + pub fn serialize(duration: &Option, serializer: S) -> Result + where + S: Serializer, + { + match duration { + Some(d) => Some(d.as_secs()).serialize(serializer), + None => None::.serialize(serializer), + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let opt = Option::::deserialize(deserializer)?; + Ok(opt.map(Duration::from_secs)) + } +} + +/// Network policy (from DESIGN.md). +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum NetworkPolicy { + /// vsock only, no network access. + #[default] + None, + + /// Outbound internet via NAT. + Egress { + /// Custom DNS server (default: inherit from host). + dns: Option, + /// If set, only these CIDR ranges are reachable. Omit for unrestricted. + #[serde(default)] + allowed_cidrs: Option>, + }, +} + +/// Disk image format for block devices. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "lowercase")] +#[repr(u32)] +pub enum DiskFormat { + /// Raw disk image. + #[default] + Raw = 0, + /// QCOW2 format (copy-on-write). + Qcow2 = 1, +} + +/// Block device configuration. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct DiskConfig { + /// Unique identifier for this block device. + pub block_id: String, + /// Path to the disk image. + pub path: PathBuf, + /// Disk format. + pub format: DiskFormat, + /// Whether the disk is read-only. + pub read_only: bool, +} + +impl DiskConfig { + /// Create a new disk configuration. + pub fn new(block_id: impl Into, path: impl Into) -> Self { + Self { + block_id: block_id.into(), + path: path.into(), + format: DiskFormat::Raw, + read_only: false, + } + } + + /// Set the disk as read-only. + pub fn read_only(mut self) -> Self { + self.read_only = true; + self + } + + /// Set the disk format. + pub fn format(mut self, format: DiskFormat) -> Self { + self.format = format; + self + } +} + +/// vsock port configuration. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct VsockPort { + /// Port number (CID 3 is guest, 2 is host). + pub port: u32, + /// Unix socket path on the host. + pub socket_path: PathBuf, + /// If true, the host listens; if false, the guest listens. + pub listen: bool, +} + +impl VsockPort { + /// Create a new vsock port where host listens. + pub fn host_listen(port: u32, socket_path: impl Into) -> Self { + Self { + port, + socket_path: socket_path.into(), + listen: true, + } + } + + /// Create a new vsock port where guest listens. + pub fn guest_listen(port: u32, socket_path: impl Into) -> Self { + Self { + port, + socket_path: socket_path.into(), + listen: false, + } + } +} + +/// Source of the guest root filesystem. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum RootfsSource { + /// Direct path to a directory or image. + Path { + /// Path to the rootfs directory. + path: PathBuf, + }, +} + +impl RootfsSource { + /// Create a path-based rootfs source. + pub fn path(p: impl Into) -> Self { + Self::Path { path: p.into() } + } +} + +/// Complete VM configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VmConfig { + /// Unique VM identifier. + pub id: VmId, + + /// Root filesystem source. + pub rootfs: RootfsSource, + + /// Memory in MiB (default: 512). + pub memory_mib: u32, + + /// Number of vCPUs (default: 1). + pub cpus: u8, + + /// Timeouts. + pub timeouts: Timeouts, + + /// Network policy. + pub network: NetworkPolicy, + + /// Host mounts (virtiofs). + pub mounts: Vec, + + /// Block devices (virtio-blk). + pub disks: Vec, + + /// vsock ports for host-guest communication. + pub vsock_ports: Vec, + + /// Console output log file (for debugging). + pub console_log: Option, + + /// Enable Rosetta for x86_64 binaries on Apple Silicon. + pub rosetta: bool, + + /// Enable GPU acceleration (virtio-gpu with Venus/Vulkan). + pub gpu: bool, + + /// Command to execute (None = use rootfs default). + pub command: Option>, + + /// Working directory. + pub workdir: Option, + + /// Environment variables. + pub env: Vec<(String, String)>, +} + +impl VmConfig { + /// Create a builder for VmConfig. + pub fn builder(rootfs: RootfsSource) -> VmConfigBuilder { + VmConfigBuilder::new(rootfs) + } +} + +/// Builder for VmConfig. +#[derive(Debug)] +pub struct VmConfigBuilder { + config: VmConfig, +} + +impl VmConfigBuilder { + /// Create a new builder with the given rootfs source. + pub fn new(rootfs: RootfsSource) -> Self { + Self { + config: VmConfig { + id: VmId::generate(), + rootfs, + memory_mib: 512, + cpus: 1, + timeouts: Timeouts::default(), + network: NetworkPolicy::default(), + mounts: Vec::new(), + disks: Vec::new(), + vsock_ports: Vec::new(), + console_log: None, + rosetta: false, + gpu: false, + command: None, + workdir: None, + env: Vec::new(), + }, + } + } + + /// Set the VM ID. + pub fn id(mut self, id: VmId) -> Self { + self.config.id = id; + self + } + + /// Set the memory in MiB. + pub fn memory(mut self, mib: u32) -> Self { + self.config.memory_mib = mib; + self + } + + /// Set the number of CPUs. + pub fn cpus(mut self, cpus: u8) -> Self { + self.config.cpus = cpus; + self + } + + /// Set the network policy. + pub fn network(mut self, policy: NetworkPolicy) -> Self { + self.config.network = policy; + self + } + + /// Add a host mount. + pub fn mount(mut self, mount: HostMount) -> Self { + self.config.mounts.push(mount); + self + } + + /// Set the command to execute. + pub fn command(mut self, cmd: Vec) -> Self { + self.config.command = Some(cmd); + self + } + + /// Set the working directory. + pub fn workdir(mut self, path: impl Into) -> Self { + self.config.workdir = Some(path.into()); + self + } + + /// Add an environment variable. + pub fn env(mut self, key: impl Into, value: impl Into) -> Self { + self.config.env.push((key.into(), value.into())); + self + } + + /// Set the boot timeout. + pub fn boot_timeout(mut self, timeout: Duration) -> Self { + self.config.timeouts.boot = timeout; + self + } + + /// Set the shutdown timeout. + pub fn shutdown_timeout(mut self, timeout: Duration) -> Self { + self.config.timeouts.shutdown = timeout; + self + } + + /// Set the execution timeout. + pub fn exec_timeout(mut self, timeout: Duration) -> Self { + self.config.timeouts.exec = Some(timeout); + self + } + + /// Add a block device. + pub fn disk(mut self, disk: DiskConfig) -> Self { + self.config.disks.push(disk); + self + } + + /// Add a vsock port. + pub fn vsock(mut self, port: VsockPort) -> Self { + self.config.vsock_ports.push(port); + self + } + + /// Set the console log file. + pub fn console_log(mut self, path: impl Into) -> Self { + self.config.console_log = Some(path.into()); + self + } + + /// Enable Rosetta for x86_64 binaries. + pub fn rosetta(mut self, enabled: bool) -> Self { + self.config.rosetta = enabled; + self + } + + /// Build the VmConfig. + pub fn build(self) -> VmConfig { + self.config + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_vm_id_sanitization() { + // Normal ID passes through + let id = VmId::new("my-test_vm123"); + assert_eq!(id.as_str(), "my-test_vm123"); + + // Invalid characters are stripped (security: prevents directory traversal) + let id = VmId::new("../../../etc/passwd"); + assert_eq!(id.as_str(), "etcpasswd"); + + // Slashes and dots removed + let id = VmId::new("vm/name.with.dots"); + assert_eq!(id.as_str(), "vmnamewithdots"); + + // Empty after sanitization uses generated ID + let id = VmId::new("///"); + assert!(id.as_str().starts_with("vm-")); + + // Long IDs are truncated + let long_id = "a".repeat(100); + let id = VmId::new(long_id); + assert_eq!(id.as_str().len(), 64); + } + + #[test] + fn test_vm_config_builder() { + let config = VmConfig::builder(RootfsSource::path("/rootfs")) + .id(VmId::new("my-vm")) + .memory(1024) + .cpus(2) + .network(NetworkPolicy::Egress { + dns: None, + allowed_cidrs: None, + }) + .mount(HostMount { + source: "/host".into(), + target: "/guest".into(), + read_only: true, + }) + .command(vec!["/bin/sh".to_string()]) + .workdir("/app") + .env("FOO", "bar") + .build(); + + assert_eq!(config.id.as_str(), "my-vm"); + assert_eq!(config.memory_mib, 1024); + assert_eq!(config.cpus, 2); + assert!(matches!(config.network, NetworkPolicy::Egress { .. })); + assert_eq!(config.mounts.len(), 1); + assert_eq!(config.command, Some(vec!["/bin/sh".to_string()])); + assert_eq!(config.workdir, Some(PathBuf::from("/app"))); + assert_eq!(config.env, vec![("FOO".to_string(), "bar".to_string())]); + } + + #[test] + fn test_network_policy_serialization() { + let none = NetworkPolicy::None; + let json = serde_json::to_string(&none).unwrap(); + assert!(json.contains("none")); + + let egress = NetworkPolicy::Egress { + dns: Some("8.8.8.8".parse().unwrap()), + allowed_cidrs: None, + }; + let json = serde_json::to_string(&egress).unwrap(); + assert!(json.contains("egress")); + assert!(json.contains("8.8.8.8")); + } + + #[test] + fn test_network_policy_allowed_cidrs_roundtrip() { + let policy = NetworkPolicy::Egress { + dns: None, + allowed_cidrs: Some(vec!["10.0.0.0/8".to_string(), "1.1.1.1/32".to_string()]), + }; + let json = serde_json::to_string(&policy).unwrap(); + let roundtripped: NetworkPolicy = serde_json::from_str(&json).unwrap(); + match roundtripped { + NetworkPolicy::Egress { dns, allowed_cidrs } => { + assert!(dns.is_none()); + let cidrs = allowed_cidrs.expect("allowed_cidrs should be Some"); + assert_eq!(cidrs, vec!["10.0.0.0/8", "1.1.1.1/32"]); + } + _ => panic!("expected Egress variant"), + } + } +} diff --git a/src/vm/mod.rs b/src/vm/mod.rs new file mode 100644 index 0000000..e04caa5 --- /dev/null +++ b/src/vm/mod.rs @@ -0,0 +1,74 @@ +//! VM creation and lifecycle management. +//! +//! This module provides the core abstractions for creating and managing microVMs: +//! - [`VmConfig`]: Configuration for a VM instance +//! - [`VmHandle`]: Trait for controlling a running VM +//! - [`VmBackend`]: Trait for VM backend implementations (e.g., libkrun) + +pub mod backend; +pub mod config; +pub mod rosetta; +pub mod state; + +use crate::error::Result; +pub use config::{ + DiskConfig, DiskFormat, HostMount, NetworkPolicy, RootfsSource, Timeouts, VmConfig, VmId, + VsockPort, +}; +pub use state::{ExitReason, VmState}; + +/// Handle to a running or stopped VM. +/// +/// This trait provides the interface for controlling a VM's lifecycle. +/// Implementations handle the platform-specific details of VM management. +pub trait VmHandle: Send { + /// Get the VM ID. + fn id(&self) -> &VmId; + + /// Get current state. + fn state(&self) -> VmState; + + /// Wait for VM to exit (blocking). + /// + /// Returns the exit reason once the VM terminates. + fn wait(&mut self) -> Result; + + /// Request graceful shutdown. + /// + /// This sends a shutdown signal to the VM and waits for it to terminate + /// gracefully. The shutdown timeout from the VM config is used. + fn stop(&mut self) -> Result<()>; + + /// Force kill the VM. + /// + /// This immediately terminates the VM without waiting for graceful shutdown. + fn kill(&mut self) -> Result<()>; +} + +/// Factory for creating VMs. +/// +/// This trait abstracts over different hypervisor backends (e.g., libkrun, KVM). +pub trait VmBackend: Send + Sync { + /// Backend name (e.g., "libkrun", "kvm"). + fn name(&self) -> &'static str; + + /// Check if this backend is available on the current system. + fn is_available(&self) -> bool; + + /// Create and start a VM with the given configuration. + /// + /// This creates a new VM, starts it, and returns a handle for controlling it. + fn create(&self, config: VmConfig) -> Result>; +} + +/// Get the default backend for this platform. +/// +/// On macOS, this returns the libkrun backend. +/// On Linux, this also returns libkrun (KVM via libkrun). +/// +/// # Errors +/// +/// Returns an error if no suitable backend is available. +pub fn default_backend() -> Result> { + backend::create_default() +} diff --git a/src/vm/rosetta.rs b/src/vm/rosetta.rs new file mode 100644 index 0000000..a93e4bb --- /dev/null +++ b/src/vm/rosetta.rs @@ -0,0 +1,104 @@ +//! Rosetta 2 support for running x86_64 binaries on Apple Silicon. +//! +//! This module provides detection and configuration for Apple's Rosetta 2 +//! translation layer, allowing x86_64 container images to run on ARM Macs. +//! +//! # How it works +//! +//! When Rosetta is available and enabled, smolvm mounts the Rosetta runtime +//! directory into the guest VM via virtiofs. A ptrace wrapper +//! (`/usr/bin/rosetta-wrapper`) intercepts Rosetta's Virtualization.framework +//! validation ioctl — required because libkrun uses Hypervisor.framework, not +//! Virtualization.framework — and registers as the binfmt_misc interpreter for +//! x86_64 ELF binaries. +//! +//! # Requirements +//! +//! - Apple Silicon Mac (M1/M2/M3/M4) +//! - Rosetta 2 installed (`softwareupdate --install-rosetta`) +//! - macOS 11.0 or later + +use crate::platform::{self, RosettaSupport}; + +/// Virtiofs tag for the Rosetta mount. Re-exported from the wire protocol so the +/// host launcher and guest agent share one definition. +pub use smolvm_protocol::ROSETTA_TAG; + +/// Guest mount path for the Rosetta runtime. Re-exported from the wire protocol. +pub use smolvm_protocol::ROSETTA_GUEST_PATH; + +/// binfmt_misc registration command for the guest. +/// +/// Registers the ptrace wrapper as the interpreter for x86_64 ELF binaries. +/// The wrapper intercepts Rosetta's validation ioctl, then detaches for +/// full-speed execution. The `F` flag keeps the interpreter fd open at +/// registration time (required since the wrapper lives on the rootfs, not +/// the virtiofs mount). +pub const BINFMT_REGISTER_CMD: &str = r#" +if [ -d /mnt/rosetta ] && [ -f /mnt/rosetta/rosetta ] && [ -x /usr/bin/rosetta-wrapper ]; then + if [ -d /proc/sys/fs/binfmt_misc ]; then + mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc 2>/dev/null || true + echo ':rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/rosetta-wrapper:F' > /proc/sys/fs/binfmt_misc/register 2>/dev/null || true + fi +fi +"#; + +/// Check if Rosetta 2 is available on this system. +/// +/// Returns `true` only on Apple Silicon Macs with Rosetta installed. +pub fn is_available() -> bool { + platform::rosetta().is_available() +} + +/// Get the path to the Rosetta runtime directory. +/// +/// Returns `None` if Rosetta is not available. +pub fn runtime_path() -> Option<&'static str> { + platform::rosetta().runtime_path() +} + +/// Guest init script snippet for enabling Rosetta. +/// +/// This should be included in the guest's init process when Rosetta is enabled. +pub fn init_script() -> &'static str { + BINFMT_REGISTER_CMD +} + +/// Platform strings that require Rosetta on ARM Macs. +pub fn needs_rosetta(platform_str: &str) -> bool { + platform::rosetta().needs_rosetta(platform_str) +} + +/// Get the native platform string for this system. +/// +/// Returns "linux/arm64" or "linux/amd64" based on host architecture. +/// Note: Always returns a Linux platform since VMs run Linux guests. +pub fn native_platform() -> &'static str { + platform::native_platform() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_needs_rosetta() { + // Platform detection logic for cross-architecture support + assert!(needs_rosetta("linux/amd64")); + assert!(needs_rosetta("linux/x86_64")); + assert!(needs_rosetta("LINUX/AMD64")); + assert!(!needs_rosetta("linux/arm64")); + assert!(!needs_rosetta("linux/aarch64")); + } + + #[test] + fn test_native_platform_format() { + let platform = native_platform(); + assert!(platform.starts_with("linux/")); + assert!( + platform == "linux/arm64" || platform == "linux/amd64", + "unexpected platform: {}", + platform + ); + } +} diff --git a/src/vm/state.rs b/src/vm/state.rs new file mode 100644 index 0000000..1535273 --- /dev/null +++ b/src/vm/state.rs @@ -0,0 +1,221 @@ +//! VM lifecycle state types. + +use serde::{Deserialize, Serialize}; + +/// VM lifecycle states (from DESIGN.md). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "lowercase")] +pub enum VmState { + /// VM created but not started. + Created, + + /// VM is booting. + Booting, + + /// VM is ready (guest agent connected). + Ready, + + /// VM is running workload. + Running, + + /// VM is shutting down. + Stopping, + + /// VM has stopped cleanly. + Stopped, + + /// VM failed with error. + Failed { + /// Reason for failure. + reason: String, + }, +} + +impl VmState { + /// Check if the VM is in a terminal state. + pub fn is_terminal(&self) -> bool { + matches!(self, VmState::Stopped | VmState::Failed { .. }) + } + + /// Check if the VM is currently running (Ready or Running). + pub fn is_running(&self) -> bool { + matches!(self, VmState::Running | VmState::Ready) + } + + /// Check if the VM can be started. + pub fn can_start(&self) -> bool { + matches!(self, VmState::Created) + } + + /// Check if the VM can be stopped. + pub fn can_stop(&self) -> bool { + matches!(self, VmState::Booting | VmState::Ready | VmState::Running) + } + + /// Get the state name as a string. + pub fn name(&self) -> &'static str { + match self { + VmState::Created => "created", + VmState::Booting => "booting", + VmState::Ready => "ready", + VmState::Running => "running", + VmState::Stopping => "stopping", + VmState::Stopped => "stopped", + VmState::Failed { .. } => "failed", + } + } +} + +impl std::fmt::Display for VmState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + VmState::Failed { reason } => write!(f, "failed: {}", reason), + _ => write!(f, "{}", self.name()), + } + } +} + +/// Exit reason for a VM (from DESIGN.md). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ExitReason { + /// Clean exit with code. + Exited { + /// Exit code from the process. + code: i32, + }, + + /// Killed by signal. + Signaled { + /// Signal number. + signal: i32, + }, + + /// Execution timeout reached. + Timeout, + + /// Out of memory. + OomKilled, + + /// Disk full (exit code 200 per DESIGN.md). + DiskFull, + + /// VM crashed. + VmCrash { + /// Details about the crash. + details: String, + }, + + /// Protocol/communication error. + ProtocolError { + /// Details about the error. + details: String, + }, +} + +impl ExitReason { + /// Map to exit code for CLI (per DESIGN.md conventions). + pub fn exit_code(&self) -> i32 { + match self { + ExitReason::Exited { code } => *code, + ExitReason::Signaled { signal } => 128 + signal, + ExitReason::Timeout => 124, // Standard timeout exit code + ExitReason::OomKilled => 137, // 128 + SIGKILL(9) + ExitReason::DiskFull => 200, // Per DESIGN.md + ExitReason::VmCrash { .. } => 1, + ExitReason::ProtocolError { .. } => 1, + } + } + + /// Check if this represents a successful exit. + pub fn is_success(&self) -> bool { + matches!(self, ExitReason::Exited { code: 0 }) + } + + /// Create an exited reason with the given code. + pub fn exited(code: i32) -> Self { + Self::Exited { code } + } + + /// Create a signaled reason with the given signal. + pub fn signaled(signal: i32) -> Self { + Self::Signaled { signal } + } + + /// Create a VM crash reason with details. + pub fn vm_crash(details: impl Into) -> Self { + Self::VmCrash { + details: details.into(), + } + } + + /// Create a protocol error reason with details. + pub fn protocol_error(details: impl Into) -> Self { + Self::ProtocolError { + details: details.into(), + } + } +} + +impl std::fmt::Display for ExitReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ExitReason::Exited { code } => write!(f, "exited with code {}", code), + ExitReason::Signaled { signal } => write!(f, "killed by signal {}", signal), + ExitReason::Timeout => write!(f, "execution timeout"), + ExitReason::OomKilled => write!(f, "out of memory"), + ExitReason::DiskFull => write!(f, "disk full"), + ExitReason::VmCrash { details } => write!(f, "vm crash: {}", details), + ExitReason::ProtocolError { details } => write!(f, "protocol error: {}", details), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_vm_state_transitions() { + // (state, is_terminal, can_start, can_stop) + let cases = [ + (VmState::Created, false, true, false), + (VmState::Booting, false, false, true), + (VmState::Running, false, false, true), + (VmState::Stopped, true, false, false), + ( + VmState::Failed { + reason: "test".to_string(), + }, + true, + false, + false, + ), + ]; + + for (state, terminal, start, stop) in cases { + assert_eq!(state.is_terminal(), terminal, "{:?}.is_terminal()", state); + assert_eq!(state.can_start(), start, "{:?}.can_start()", state); + assert_eq!(state.can_stop(), stop, "{:?}.can_stop()", state); + } + } + + #[test] + fn test_exit_reason_exit_codes() { + // Documents DESIGN.md exit code contract + assert_eq!(ExitReason::exited(0).exit_code(), 0); + assert_eq!(ExitReason::exited(1).exit_code(), 1); + assert_eq!(ExitReason::signaled(9).exit_code(), 137); // 128 + SIGKILL + assert_eq!(ExitReason::Timeout.exit_code(), 124); + assert_eq!(ExitReason::OomKilled.exit_code(), 137); + assert_eq!(ExitReason::DiskFull.exit_code(), 200); + } + + #[test] + fn test_exit_reason_serialization() { + let reason = ExitReason::exited(42); + let json = serde_json::to_string(&reason).unwrap(); + let deserialized: ExitReason = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized, reason); + } +} diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..01fc020 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,97 @@ +# smolvm Tests + +Integration tests for smolvm. All tests invoke the `smolvm` binary directly +against real VMs — no mocking. + +--- + +## Runner: `run_tests.sh` + +Single entry point for all test suites. + +```bash +./tests/run_tests.sh # all 12 feature suites (~10 min) +SMOLVM_SKIP_SLOW=1 ./tests/run_tests.sh # skip tests with long sleeps (~5 min) +./tests/run_tests.sh bare network # specific groups only +``` + +### Feature suites (run by default) + +| Group | Suite file | Tests | Notes | +|---|---|---|---| +| `bare` | `test_machine_bare.sh` | 22 | Lifecycle, exec, shell, file I/O, observability | +| `db` | `test_db.sh` | 6 | DB state persistence, VM state transitions | +| `network` | `test_network.sh` | 17 | Network disable, DNS, egress, DNS filter, Smolfile allow_hosts | +| `volumes` | `test_volumes.sh` | 6 | virtiofs mounts, /workspace priority | +| `ports` | `test_ports.sh` | 2 | Port mapping, cross-VM conflict detection | +| `storage` | `test_storage.sh` | 12 | Overlay, image list, prune, storage resize | +| `resources` | `test_resources.sh` | 8 | CLI validation — no VMs required | +| `reliability` | `test_reliability.sh` | 5 | Concurrency, state probe, ls-does-not-kill-vm | +| `run` | `test_machine_run.sh` | 25 | Ephemeral `machine run` scenarios | +| `image` | `test_machine_image.sh` | 13 | Image-based VMs, exec-join, large stdout | +| `local-image` | `test_machine_local_image.sh` | 6 | Local images: save-archive (file/stdin), rootfs dir, stdin + Dockerfile guards (needs docker) | +| `packed` | `test_machine_packed.sh` | 2 | `.smolmachine` create and cp | + +### Extended suites (opt-in only) + +These are not included in the default run. Pass the group name explicitly: + +```bash +./tests/run_tests.sh cli +./tests/run_tests.sh api +./tests/run_tests.sh virtio-net +./tests/run_tests.sh smolfile +./tests/run_tests.sh pack +./tests/run_tests.sh pack-quick # pack tests, skip large image pulls +./tests/run_tests.sh gpu # requires GPU hardware +``` + +| Group | Suite file | Tests | +|---|---|---| +| `cli` | `test_cli.sh` | 10 | +| `api` | `test_api.sh` | 25 | +| `virtio-net` | `test_virtio_net.sh` | 6 | +| `smolfile` | `test_smolfile.sh` | 49 | +| `pack` / `pack-quick` | `test_pack.sh` | 39 | +| `gpu` | `test_gpu.sh` | 14 | + +### Environment variables + +| Variable | Effect | +|---|---| +| `SMOLVM` | Override binary path | +| `SMOLVM_SKIP_SLOW=1` | Skip tests with intentional sleeps ≥ 25 s (docker-in-vm, ls-probe loops, egress refresh) | +| `TEST_FILTER` | Only run tests whose display name contains this substring | +| `FAIL_FAST=1` | Stop on first failure | + +--- + +## Unit tests + +No VM required. Run via cargo: + +```bash +cargo test +``` + +--- + +## Benchmarks + +```bash +./tests/bench_vm_startup.sh # VM cold/warm start time +./tests/run_tests.sh bench # same, via runner +``` + +--- + +## Binary discovery + +Tests find the `smolvm` binary in this order: + +1. `$SMOLVM` environment variable +2. `target/release/smolvm` (cargo build output) +3. `dist/smolvm-*-darwin-*/smolvm` or `dist/smolvm-*-linux-*/smolvm` + +--- + diff --git a/tests/bench_vm_startup.sh b/tests/bench_vm_startup.sh new file mode 100755 index 0000000..af46c40 --- /dev/null +++ b/tests/bench_vm_startup.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Benchmark: microVM startup time +# +# Measures the time to start the smolvm agent VM from cold state. +# This includes: VM creation, kernel boot, init execution, agent ready. +# +# Usage: ./tests/bench_vm_startup.sh [iterations] +# or: ./tests/run_tests.sh bench + +set -euo pipefail + +ITERATIONS="${1:-5}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Millisecond timestamp using bash builtin (no subprocess overhead) +now_ms() { + # EPOCHREALTIME gives seconds.microseconds — convert to milliseconds + local t="${EPOCHREALTIME}" + local secs="${t%%.*}" + local frac="${t#*.}" + # Pad/truncate fractional part to 3 digits (milliseconds) + frac="${frac:0:3}" + echo $(( secs * 1000 + 10#$frac )) +} + +echo "========================================" +echo " smolvm microVM Startup Benchmark" +echo "========================================" +echo "" +echo "Iterations: $ITERATIONS" +echo "" + +# Check if smolvm is available +if [[ -n "${SMOLVM:-}" ]] && [[ -x "$SMOLVM" ]]; then + : # use provided SMOLVM +elif command -v smolvm &> /dev/null; then + SMOLVM="smolvm" +elif [ -f "$PROJECT_ROOT/target/release/smolvm" ]; then + SMOLVM="$PROJECT_ROOT/target/release/smolvm" +elif [ -f "$PROJECT_ROOT/target/debug/smolvm" ]; then + SMOLVM="$PROJECT_ROOT/target/debug/smolvm" +else + echo -e "${RED}Error: smolvm not found. Build with 'cargo build --release'${NC}" + exit 1 +fi + +echo "Using: $SMOLVM" +echo "" + +# Cleanup function +cleanup() { + echo "" + echo "Cleaning up..." + $SMOLVM machine stop 2>/dev/null || true + pkill -f "smolvm-bin machine" 2>/dev/null || true + pkill -f "smolvm machine start" 2>/dev/null || true +} + +trap cleanup EXIT + +# Kill any existing smolvm processes before benchmarking +echo "Cleaning up any existing smolvm processes..." +pkill -f "smolvm-bin machine" 2>/dev/null || true +pkill -f "smolvm machine start" 2>/dev/null || true +$SMOLVM machine stop 2>/dev/null || true +sleep 1 + +# ============================================ +# Test 1: MicroVM Start (VM boot + agent ready) +# ============================================ +echo -e "${BLUE}Test 1: VM Cold Start (machine start)${NC}" +echo " Measures: fork → kernel boot → init → agent ready" +echo "" + +declare -a START_TIMES + +for i in $(seq 1 $ITERATIONS); do + $SMOLVM machine stop 2>/dev/null || true + sleep 0.5 + + START_TIME=$(now_ms) + $SMOLVM machine start > /dev/null 2>&1 + END_TIME=$(now_ms) + + DURATION=$(( END_TIME - START_TIME )) + START_TIMES+=($DURATION) + echo " Run $i: ${DURATION}ms" +done + +# ============================================ +# Test 2: MicroVM Start + First Command +# ============================================ +echo "" +echo -e "${BLUE}Test 2: VM Start + First Command (exec)${NC}" +echo " Measures: cold start + first vsock round-trip" +echo "" + +declare -a PING_TIMES + +for i in $(seq 1 $ITERATIONS); do + $SMOLVM machine stop 2>/dev/null || true + sleep 0.5 + + START_TIME=$(now_ms) + $SMOLVM machine start > /dev/null 2>&1 + $SMOLVM machine exec -- echo hello > /dev/null 2>&1 + END_TIME=$(now_ms) + + DURATION=$(( END_TIME - START_TIME )) + PING_TIMES+=($DURATION) + echo " Run $i: ${DURATION}ms" +done + +# ============================================ +# Results Summary (pure bash — no Python) +# ============================================ +stats() { + local label="$1" + shift + local times=("$@") + local n=${#times[@]} + local sum=0 min=${times[0]} max=${times[0]} + + for t in "${times[@]}"; do + (( sum += t )) + (( t < min )) && min=$t + (( t > max )) && max=$t + done + + local avg=$(( sum / n )) + # Standard deviation (integer approximation) + local var_sum=0 + for t in "${times[@]}"; do + local diff=$(( t - avg )) + (( var_sum += diff * diff )) + done + local variance=$(( var_sum / n )) + # Integer square root approximation + local std_dev=0 + if (( variance > 0 )); then + std_dev=1 + while (( std_dev * std_dev < variance )); do + (( std_dev++ )) + done + fi + + echo " $label:" + echo " Min: ${min}ms" + echo " Max: ${max}ms" + echo " Average: ${avg}ms" + echo " Std Dev: ~${std_dev}ms" + # Return average via global + _STAT_AVG=$avg +} + +echo "" +echo "========================================" +echo " Results Summary" +echo "========================================" +echo "" + +stats "VM Cold Start (machine start)" "${START_TIMES[@]}" +START_AVG=$_STAT_AVG + +echo "" +stats "VM Start + First Command" "${PING_TIMES[@]}" +PING_AVG=$_STAT_AVG + +echo "" +echo "----------------------------------------" +echo "Breakdown:" +echo "----------------------------------------" +echo " VM boot to agent ready: ${START_AVG}ms" +echo " First command overhead: $(( PING_AVG - START_AVG ))ms" + +echo "" +echo -e "${GREEN}Benchmark complete.${NC}" +echo "" diff --git a/tests/common.sh b/tests/common.sh new file mode 100755 index 0000000..1094bb1 --- /dev/null +++ b/tests/common.sh @@ -0,0 +1,431 @@ +#!/usr/bin/env bash +# +# Common test utilities for smolvm integration tests. +# +# Source this file in test scripts: +# source "$(dirname "$0")/common.sh" + +set -euo pipefail + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Test counters +TESTS_RUN=0 +TESTS_PASSED=0 +TESTS_FAILED=0 + +# Find the script directory and project root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Find smolvm binary +find_smolvm() { + if [[ -n "${SMOLVM:-}" ]] && [[ -x "$SMOLVM" ]]; then + echo "$SMOLVM" + return + fi + + # Prefer cargo build output (latest build) over dist + local target_release="$PROJECT_ROOT/target/release/smolvm" + if [[ -x "$target_release" ]]; then + echo "$target_release" + return + fi + + # Fall back to dist directory + local dist_dir="$PROJECT_ROOT/dist" + if [[ -d "$dist_dir" ]]; then + # Find the extracted distribution directory + local smolvm_dir=$(find "$dist_dir" -maxdepth 1 -type d \( -name 'smolvm-*-darwin-*' -o -name 'smolvm-*-linux-*' \) 2>/dev/null | head -1) + if [[ -n "$smolvm_dir" ]] && [[ -x "$smolvm_dir/smolvm" ]]; then + echo "$smolvm_dir/smolvm" + return + fi + fi + + echo "" +} + +# Initialize SMOLVM variable +init_smolvm() { + SMOLVM=$(find_smolvm) + + # Resolve to absolute path (tests cd into temp dirs) + if [[ -n "$SMOLVM" ]] && [[ "$SMOLVM" != /* ]]; then + SMOLVM="$(cd "$(dirname "$SMOLVM")" && pwd)/$(basename "$SMOLVM")" + fi + + if [[ -z "$SMOLVM" ]]; then + echo -e "${RED}Error: Could not find smolvm binary${NC}" + echo "Either:" + echo " 1. Build and extract the distribution: ./scripts/build-dist.sh" + echo " 2. Set SMOLVM environment variable to the binary path" + exit 1 + fi + + # Set library path to ensure we use bundled libkrun/libkrunfw. + # This is needed when running from target/release since the system + # may not have libkrun on its default library search path. + if [[ "$(uname -s)" == "Darwin" ]]; then + local lib_dir="$PROJECT_ROOT/lib" + if [[ -d "$lib_dir" ]]; then + export DYLD_LIBRARY_PATH="${lib_dir}${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" + fi + else + local lib_dir="$PROJECT_ROOT/lib/linux-$(uname -m)" + if [[ -d "$lib_dir" ]]; then + export LD_LIBRARY_PATH="${lib_dir}:${LD_LIBRARY_PATH:-}" + fi + fi + + echo "Using smolvm: $SMOLVM" +} + +# Resolve a hostname to a single IPv4 address on the host, portably across +# Linux and macOS. Linux glibc has `getent`; macOS does not, so fall back to +# dig/python3/host (all present on a stock macOS or with bind tools). Prints the +# IP on stdout, or nothing if resolution fails. +resolve_host_ipv4() { + local host="$1" ip="" + if command -v getent >/dev/null 2>&1; then + ip=$(getent ahostsv4 "$host" 2>/dev/null | awk 'NR==1{print $1}') + fi + if [[ -z "$ip" ]] && command -v dig >/dev/null 2>&1; then + ip=$(dig +short "$host" A 2>/dev/null | grep -m1 -E '^[0-9]+(\.[0-9]+){3}$') + fi + if [[ -z "$ip" ]] && command -v python3 >/dev/null 2>&1; then + ip=$(python3 -c 'import socket,sys; print(socket.gethostbyname(sys.argv[1]))' "$host" 2>/dev/null) + fi + if [[ -z "$ip" ]] && command -v host >/dev/null 2>&1; then + ip=$(host -t A "$host" 2>/dev/null | awk '/has address/{print $NF; exit}') + fi + printf '%s' "$ip" +} + +# Log helpers +log_test() { + echo -e "${YELLOW}[TEST]${NC} $1" +} + +log_pass() { + echo -e "${GREEN}[PASS]${NC} $1" + ((TESTS_PASSED++)) +} + +log_fail() { + echo -e "${RED}[FAIL]${NC} $1" + ((TESTS_FAILED++)) +} + +log_skip() { + echo -e "${BLUE}[SKIP]${NC} $1" +} + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +# Track failed test names for summary +FAILED_TESTS=() + +# Fail-fast mode: stop on first failure. +# Set FAIL_FAST=1 or use TEST_FILTER with run_tests.sh. +FAIL_FAST="${FAIL_FAST:-0}" + +# Single test filter: only run tests whose name contains this string. +# Usage: TEST_FILTER="port mapping" ./tests/run_tests.sh ports +TEST_FILTER="${TEST_FILTER:-}" + +# Skip slow tests (≥25s intentional sleeps) when SMOLVM_SKIP_SLOW=1. +# Usage inside a test function: skip_if_slow && return 0 +SMOLVM_SKIP_SLOW="${SMOLVM_SKIP_SLOW:-0}" + +skip_if_slow() { + if [[ "$SMOLVM_SKIP_SLOW" == "1" ]]; then + log_skip "slow test skipped (SMOLVM_SKIP_SLOW=1)" + return 0 + fi + return 1 +} + +# Run a test function, capturing output and showing it on failure. +run_test() { + local test_name="$1" + local test_func="$2" + + # Skip if filter is set and test name doesn't match + if [[ -n "$TEST_FILTER" ]] && [[ "$test_name" != *"$TEST_FILTER"* ]]; then + return 0 + fi + + # Skip remaining tests if fail-fast triggered + if [[ "$FAIL_FAST" == "1" ]] && [[ $TESTS_FAILED -gt 0 ]]; then + return 0 + fi + + ((TESTS_RUN++)) + log_test "$test_name" + + local output_file + output_file=$(mktemp) + + if $test_func 2>&1 | tee "$output_file"; then + log_pass "$test_name" + rm -f "$output_file" + return 0 + else + log_fail "$test_name" + FAILED_TESTS+=("$test_name") + + # Show last 10 lines on failure (may already be visible, but + # repeating under the FAIL marker makes it easy to find) + local output + output=$(tail -10 "$output_file" 2>/dev/null || true) + if [[ -n "$output" ]]; then + echo -e " ${RED}Error output:${NC}" + echo "$output" | sed 's/^/ /' + fi + rm -f "$output_file" + + if [[ "$FAIL_FAST" == "1" ]]; then + echo -e "\n${RED}Stopping: --fail-fast is set${NC}" + fi + return 1 + fi +} + +# Print test summary +print_summary() { + local test_suite="${1:-Tests}" + + echo "" + echo "==========================================" + echo " $test_suite Summary" + echo "==========================================" + echo "" + echo "Tests run: $TESTS_RUN" + echo -e "Tests passed: ${GREEN}$TESTS_PASSED${NC}" + echo -e "Tests failed: ${RED}$TESTS_FAILED${NC}" + + if [[ $TESTS_FAILED -gt 0 ]] && [[ ${#FAILED_TESTS[@]} -gt 0 ]]; then + echo "" + echo -e "${RED}Failed tests:${NC}" + for test_name in "${FAILED_TESTS[@]}"; do + echo -e " ${RED}✗${NC} $test_name" + done + fi + + echo "" + + if [[ $TESTS_FAILED -eq 0 ]]; then + echo -e "${GREEN}All tests passed!${NC}" + return 0 + else + echo -e "${RED}Some tests failed.${NC}" + return 1 + fi +} + +# Get the data directory for a named machine. +# +# Delegates to `smolvm machine data-dir --name ` so the test helper never +# duplicates the hash logic. If Rust changes the on-disk layout, the test +# suite automatically picks it up via this CLI call. +vm_data_dir() { + local name="${1:-default}" + $SMOLVM machine data-dir --name "$name" 2>/dev/null +} + +# Cleanup helper - stop machine and remove named "default" from DB +# so tests start from a clean slate (no leftover DB records from +# manual testing or previous test runs). +cleanup_machine() { + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true +} + +# Verify that a VM's data directory was removed after deletion. +# Returns non-zero if the directory still exists. +ensure_data_dir_deleted() { + local name="${1:?vm name required}" + local data_dir + data_dir=$(vm_data_dir "$name") + if [[ -d "$data_dir" ]]; then + echo -e "${RED}ERROR: data directory still exists after delete: $data_dir${NC}" >&2 + return 1 + fi +} + +# Ensure machine is running and reachable. +# If net=true, recreate with --net (needed for container image pulls). +# Handles stale "already running" state from previous tests by verifying +# connectivity and doing a full cleanup cycle if the VM is unreachable. +ensure_machine_running() { + local with_net="${1:-false}" + if [[ "$with_net" == "true" ]]; then + # Stop and delete existing default VM, recreate with --net + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine create --name default --net 2>/dev/null || true + fi + $SMOLVM machine start 2>/dev/null || true + + # Verify the VM is actually reachable. If it reports "running" but + # the process is dead (stale PID), do a full cleanup and restart. + if ! $SMOLVM machine exec -- true 2>/dev/null; then + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + if [[ "$with_net" == "true" ]]; then + $SMOLVM machine create --name default --net 2>/dev/null || true + fi + $SMOLVM machine start 2>/dev/null || true + fi +} + +# Poll until a VM responds to exec (i.e., the agent is ready to accept commands). +# Replaces fixed sleep N readiness waits after machine start. +# +# Usage: wait_vm_ready [--name NAME] [TIMEOUT_SECS] +# --name NAME Named VM (default: unnamed default VM) +# TIMEOUT_SECS Give up after this many seconds (default: 10) +# +# Returns 0 when ready, 1 on timeout. +wait_vm_ready() { + local name_flag="" timeout=10 + while [[ $# -gt 0 ]]; do + case "$1" in + --name) name_flag="--name $2"; shift 2 ;; + *) timeout="$1"; shift ;; + esac + done + local i=0 + while [[ $i -lt $((timeout * 2)) ]]; do + $SMOLVM machine exec $name_flag -- true 2>/dev/null && return 0 + sleep 0.5 + ((i++)) || true + done + return 1 +} + +# Extract container ID from output +extract_container_id() { + local output="$1" + echo "$output" | grep -oE 'smolvm-[a-f0-9]+' | head -1 +} + +# Cleanup container by ID +cleanup_container() { + local container_id="$1" + $SMOLVM container rm --container "$container_id" -f 2>/dev/null || true +} + +# Run a command with a timeout (default 60 seconds). +# Usage: run_with_timeout [timeout_seconds] command [args...] +# Returns the command's exit code, or 124 if timed out. +# Output is written to stdout. +run_with_timeout() { + local timeout_secs="${1:-60}" + shift + + # Create temp file for output + local tmpfile + tmpfile=$(mktemp) + + # Run command in background, redirecting output to temp file + "$@" > "$tmpfile" 2>&1 & + local pid=$! + + # Wait with timeout + local count=0 + while kill -0 "$pid" 2>/dev/null; do + sleep 1 + ((count++)) + if [[ $count -ge $timeout_secs ]]; then + echo "[TIMEOUT] Command timed out after ${timeout_secs}s: $*" >&2 + # Kill the process and all its children + kill -9 "$pid" 2>/dev/null + pkill -9 -P "$pid" 2>/dev/null + wait "$pid" 2>/dev/null + cat "$tmpfile" + rm -f "$tmpfile" + return 124 + fi + done + + # Get exit code and output + wait "$pid" + local exit_code=$? + cat "$tmpfile" + rm -f "$tmpfile" + return $exit_code +} + +# Kill any orphaned smolvm processes that might be holding the database lock. +# This includes: +# - smolvm serve (API server) +# - smolvm-bin machine start (VM processes from previous test runs) +# - Packed binaries running as daemons +# +# When SMOLVM_ORCHESTRATED=1 (set by run_tests.sh), this is a no-op: the +# orchestrator does a single pre-flight kill before launching any suites, so +# per-suite kills are skipped to avoid parallel suites killing each other's +# in-flight VM starts. +# +# Call this before running tests to ensure clean state. +kill_orphan_smolvm_processes() { + if [[ "${SMOLVM_ORCHESTRATED:-0}" == "1" ]]; then + return 0 + fi + local killed=0 + + # Kill any smolvm serve processes + if pkill -f "smolvm serve" 2>/dev/null; then + ((killed++)) || true + fi + if pkill -f "smolvm-bin serve" 2>/dev/null; then + ((killed++)) || true + fi + + # Kill any orphaned machine processes (from smolvm-bin in dist/) + if pkill -f "smolvm-bin machine start" 2>/dev/null; then + ((killed++)) || true + fi + + # Kill any orphaned machine processes (from target/release) + if pkill -f "smolvm machine start" 2>/dev/null; then + ((killed++)) || true + fi + + # Wait briefly for processes to die + if [[ $killed -gt 0 ]]; then + sleep 1 + fi +} + +# Check if any smolvm processes are running that might interfere with tests +check_smolvm_processes() { + local procs + procs=$(pgrep -f "(smolvm serve|smolvm-bin machine start|smolvm machine start)" 2>/dev/null || true) + if [[ -n "$procs" ]]; then + return 1 # Processes found + fi + return 0 # No interfering processes +} + +# Ensure clean test environment - call at start of test suite +ensure_clean_test_environment() { + # First, try to kill any orphan processes + kill_orphan_smolvm_processes + + # Verify they're gone + if ! check_smolvm_processes; then + log_info "Warning: Some smolvm processes are still running after cleanup" + log_info "Processes:" + ps aux | grep -E "(smolvm serve|smolvm-bin machine|smolvm machine)" | grep -v grep || true + fi +} diff --git a/tests/idmap_e2e_boot.sh b/tests/idmap_e2e_boot.sh new file mode 100644 index 0000000..2145e84 --- /dev/null +++ b/tests/idmap_e2e_boot.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Full-VM e2e for the shared content-addressed pack store + per-VM idmapped +# bind mount. Drives the *serve API* (the cloud/node path that the shared store +# targets) as root so the per-VM uid drop (#456) is active and the idmap mount +# fires. Validates: (1) shared store extracts ONCE into _shared/, +# root-owned 0700; (2) a second machine from the same .smolmachine REUSES it +# (no second decode); (3) each machine leaves an empty `pack` mountpoint + a +# `.pack-shared` pointer; (4) both guests BOOT and exec reads files (proving the +# idmap mount presented the root-owned pack as the VM's dropped uid); (5) the +# on-disk shared copy stays root:root (isolation preserved). +# +# Run as root on a Linux/KVM box: sudo SMOLVM=... SMOLVM_LIB_DIR=... ./idmap_e2e_boot.sh /path/to/x.smolmachine +set -u + +SMOLVM="${SMOLVM:?set SMOLVM to the smolvm binary}" +SIDECAR="${1:?usage: idmap_e2e_boot.sh }" +DATA_DIR="${SMOLVM_DATA_DIR:-/tmp/idmap-e2e-data}" +SOCK="/tmp/idmap-e2e.sock" +CURL=(curl -s --unix-socket "$SOCK") +URL="http://localhost" +PASS=0; FAIL=0 +ok() { echo " PASS: $*"; PASS=$((PASS+1)); } +bad() { echo " FAIL: $*"; FAIL=$((FAIL+1)); } + +# Fresh, world-traversable data root so the dropped VM uid can reach its dirs. +rm -rf "$DATA_DIR"; mkdir -p "$DATA_DIR"; chmod 755 "$DATA_DIR" +export SMOLVM_DATA_DIR="$DATA_DIR" + +# Seed the node-constant guest agent-rootfs (the serve API gates `start` on it +# via "verify rootfs"). It's install-state, not part of the per-image pack the +# shared store optimizes, so we copy a known-good tree in rather than re-install. +AGENT_ROOTFS_SRC="${AGENT_ROOTFS_SRC:-$HOME/.local/share/smolvm/agent-rootfs}" +if [ -d "$AGENT_ROOTFS_SRC" ]; then + mkdir -p "$DATA_DIR/.local/share/smolvm" + cp -a "$AGENT_ROOTFS_SRC" "$DATA_DIR/.local/share/smolvm/agent-rootfs" + chmod -R a+rX "$DATA_DIR/.local/share/smolvm" +else + echo "[!] no agent-rootfs at $AGENT_ROOTFS_SRC — boot steps will fail (set AGENT_ROOTFS_SRC)" +fi + +echo "[*] euid=$(id -u) (expect 0 for uid-drop), data=$DATA_DIR, sidecar=$SIDECAR" + +rm -f "$SOCK" +"$SMOLVM" serve start -l "unix://$SOCK" >"$DATA_DIR/serve.log" 2>&1 & +SERVE_PID=$! +cleanup() { + "${CURL[@]}" -X DELETE "$URL/api/v1/machines/e2e-a" >/dev/null 2>&1 || true + "${CURL[@]}" -X DELETE "$URL/api/v1/machines/e2e-b" >/dev/null 2>&1 || true + kill "$SERVE_PID" 2>/dev/null || true; wait "$SERVE_PID" 2>/dev/null || true +} +trap cleanup EXIT + +for i in $(seq 1 50); do "${CURL[@]}" "$URL/health" >/dev/null 2>&1 && break; sleep 0.2; done +"${CURL[@]}" "$URL/health" | grep -q '"status":"ok"' && ok "serve up" || { bad "serve did not start"; cat "$DATA_DIR/serve.log"; exit 1; } + +create() { + "${CURL[@]}" -o /dev/null -w "%{http_code}" -X POST "$URL/api/v1/machines" \ + -H 'Content-Type: application/json' -d "{\"name\":\"$1\",\"from\":\"$SIDECAR\",\"cpus\":1,\"mem\":512}" +} + +# --- Create machine A: first extraction populates the shared store ---------- +echo "[*] creating e2e-a" +[ "$(create e2e-a)" = "200" ] && ok "create e2e-a 200" || bad "create e2e-a !=200" + +# Layout: SMOLVM_DATA_DIR becomes $HOME, so the store is $HOME/.cache/smolvm/vms. +VMS_ROOT="$(find "$DATA_DIR" -type d -path '*/smolvm/vms' | head -1)" +SHARED_ROOT="$VMS_ROOT/_shared" +[ -d "$SHARED_ROOT" ] && ok "shared root exists: $SHARED_ROOT" || bad "no shared root (vms=$VMS_ROOT)" +CKDIR="$(find "$SHARED_ROOT" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | head -1)" +[ -n "$CKDIR" ] && ok "shared checksum dir: $(basename "$CKDIR")" || bad "no shared checksum dir" +# Root-owned + 0700 (isolation): a sibling dropped-uid cannot read it directly. +own="$(stat -c '%U:%G %a' "$CKDIR" 2>/dev/null)" +[ "$own" = "root:root 700" ] && ok "shared dir is root:root 0700" || bad "shared dir perms = $own (want root:root 700)" +# Per-machine: empty `pack` mountpoint + a pointer to the shared dir. +PACK_A="$(find "$VMS_ROOT" -mindepth 2 -maxdepth 2 -name pack -type d 2>/dev/null | grep -v _shared | head -1)" +PTR_A="$(dirname "$PACK_A")/.pack-shared" +[ -f "$PTR_A" ] && ok "pointer written: $PTR_A -> $(cat "$PTR_A")" || bad "no .pack-shared pointer" +[ -z "$(ls -A "$PACK_A" 2>/dev/null)" ] && ok "pack mountpoint is empty (pre-boot)" || bad "pack mountpoint not empty" + +# --- Create machine B from the SAME sidecar: must REUSE, not re-extract ------ +echo "[*] creating e2e-b (same sidecar)" +[ "$(create e2e-b)" = "200" ] && ok "create e2e-b 200" || bad "create e2e-b !=200" +NDIRS="$(find "$SHARED_ROOT" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')" +[ "$NDIRS" = "1" ] && ok "still ONE shared dir after 2 creates (reuse, no re-decode)" || bad "shared dirs=$NDIRS (expected 1)" + +# --- Boot both: idmap mount presents the root-owned pack as each VM's uid ---- +boot_and_exec() { + local n="$1" + local st; st="$("${CURL[@]}" -X POST "$URL/api/v1/machines/$n/start")" + echo "$st" | grep -q '"state":"running"' && ok "$n booted (state=running)" || { bad "$n did not boot"; echo "$st"; } + local ex; ex="$("${CURL[@]}" -X POST "$URL/api/v1/machines/$n/exec" -H 'Content-Type: application/json' -d '{"command":["cat","/etc/os-release"]}')" + echo "$ex" | grep -qi 'alpine\|linux' && ok "$n exec read rootfs through idmap mount" || { bad "$n exec failed"; echo "$ex" | head -c 400; } +} +boot_and_exec e2e-a +boot_and_exec e2e-b + +# --- Isolation: shared copy still root-owned after both VMs booted ----------- +own2="$(stat -c '%U:%G %a' "$CKDIR" 2>/dev/null)" +[ "$own2" = "root:root 700" ] && ok "shared copy still root:root 0700 post-boot" || bad "shared perms changed: $own2" + +# --- The two VMMs dropped to DIFFERENT uids (per-VM isolation #456) ---------- +uids="$(ps -eo uid,args | grep '[_]boot-vm' | awk '{print $1}' | sort -u)" +nuid="$(echo "$uids" | grep -c .)" +echo "[*] live _boot-vm uids: $(echo $uids | tr '\n' ' ')" +[ "$nuid" -ge 2 ] && ok "VMMs run under >=2 distinct dropped uids" || echo " INFO: distinct boot-vm uids=$nuid (VMs may share or have exited)" + +echo "" +echo "==== idmap e2e: $PASS passed, $FAIL failed ====" +[ "$FAIL" = "0" ] diff --git a/tests/idmap_mount.rs b/tests/idmap_mount.rs new file mode 100644 index 0000000..e3edc05 --- /dev/null +++ b/tests/idmap_mount.rs @@ -0,0 +1,84 @@ +//! Validates the shared-pack idmapped-bind-mount mechanism on a real Linux +//! kernel (>= 5.12). This is the novel piece of the shared content-addressed +//! pack store: one root-owned copy of the build-constant pack is extracted per +//! node, then presented at each VM's `pack` mountpoint via an idmapped bind +//! mount that maps the on-disk uid/gid 0 down to the VM's dropped uid (#456). +//! +//! Root-only (needs CAP_SYS_ADMIN for unshare/open_tree/mount_setattr/move_mount) +//! and Linux-only, so it is `#[ignore]`d by default. Run on the test box with: +//! sudo -E cargo test --release --test idmap_mount -- --ignored --test-threads=1 +#![cfg(target_os = "linux")] + +use std::os::unix::fs::MetadataExt; + +/// A distinctive uid/gid well outside any real account, matching the +/// 2_000_000+ range #456 drops VMM processes into. +const TEST_UID: u32 = 2_000_123; +const TEST_GID: u32 = 2_000_123; + +#[test] +#[ignore = "requires root (CAP_SYS_ADMIN) and Linux >= 5.12"] +fn idmap_mount_presents_root_owned_pack_as_vm_uid() { + // SAFETY of the in-process mount-ns mutation: setup_pack_idmap_mount does + // unshare(CLONE_NEWNS), so the bind mount is visible only to this thread's + // private mount namespace and vanishes when the test process exits. Run with + // --test-threads=1 so no sibling test races on the namespace. + let root = std::env::temp_dir().join(format!("smolvm-idmap-test-{}", std::process::id())); + let shared = root.join("shared"); + let target = root.join("target"); + std::fs::create_dir_all(&shared).expect("mkdir shared"); + std::fs::create_dir_all(&target).expect("mkdir target"); + + // A root-owned (extraction-as-root => uid 0) file + nested dir, mirroring the + // agent-rootfs tree the real pack contains. + let file = shared.join("hello"); + std::fs::write(&file, b"pack contents").expect("write shared file"); + let nested_dir = shared.join("sub"); + std::fs::create_dir_all(&nested_dir).expect("mkdir nested"); + let nested_file = nested_dir.join("deep"); + std::fs::write(&nested_file, b"deep contents").expect("write nested file"); + + // Precondition: on disk the files are owned by uid/gid 0. + let pre = std::fs::metadata(&file).expect("stat pre"); + assert_eq!(pre.uid(), 0, "shared file must start root-owned"); + assert_eq!(pre.gid(), 0, "shared file must start root-group"); + + // Exercise the mechanism under test. + smolvm::process::setup_pack_idmap_mount(&shared, &target, TEST_UID, TEST_GID) + .expect("setup_pack_idmap_mount failed"); + + // Through the idmapped mount, on-disk uid 0 must surface as the VM uid — this + // is what lets the soon-to-drop VMM read every pack file as its owner. + let mapped = std::fs::metadata(target.join("hello")).expect("stat mapped file"); + assert_eq!( + mapped.uid(), + TEST_UID, + "idmapped mount must surface on-disk uid 0 as the VM uid" + ); + assert_eq!( + mapped.gid(), + TEST_GID, + "idmapped mount must surface on-disk gid 0 as the VM gid" + ); + // Contents must read through unchanged (it is the same inode, just remapped). + let body = std::fs::read(target.join("hello")).expect("read mapped file"); + assert_eq!(body, b"pack contents", "mapped file contents must match"); + + // AT_RECURSIVE must remap nested entries too, not just the top level. + let mapped_deep = + std::fs::metadata(target.join("sub").join("deep")).expect("stat mapped nested"); + assert_eq!( + mapped_deep.uid(), + TEST_UID, + "nested entries must be remapped (AT_RECURSIVE)" + ); + + // Isolation invariant: the underlying shared copy is untouched on disk — a + // sibling VM dropped to a *different* uid still sees it as root-only and so + // cannot read it directly (only its own idmapped view maps it). + let post = std::fs::metadata(&file).expect("stat post"); + assert_eq!(post.uid(), 0, "on-disk shared copy must stay root-owned"); + + // Best-effort cleanup; the private mount ns tears down on process exit anyway. + let _ = std::fs::remove_dir_all(&root); +} diff --git a/tests/run_tests.sh b/tests/run_tests.sh new file mode 100755 index 0000000..81a8240 --- /dev/null +++ b/tests/run_tests.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# +# run_tests.sh -- primary test runner for smolvm +# +# Usage: +# ./tests/run_tests.sh # run all 11 feature suites (~10 min) +# ./tests/run_tests.sh bare network # run specific groups +# SMOLVM_SKIP_SLOW=1 ./tests/run_tests.sh # skip tests with long sleeps +# +# Feature suites (run by default with no args): +# bare test_machine_bare.sh +# db test_db.sh +# network test_network.sh +# volumes test_volumes.sh +# ports test_ports.sh +# storage test_storage.sh +# resources test_resources.sh +# reliability test_reliability.sh +# run test_machine_run.sh +# image test_machine_image.sh +# local-image test_machine_local_image.sh +# fork-base test_fork_base_guards.sh +# packed test_machine_packed.sh +# +# Extended suites (opt-in only, not run by default): +# cli test_cli.sh +# api test_api.sh +# virtio-net test_virtio_net.sh +# smolfile test_smolfile.sh +# secrets test_secrets.sh +# pack test_pack.sh +# pack-quick test_pack.sh --quick +# gpu test_gpu.sh (requires GPU hardware) +# +# Non-pass/fail: +# bench bench_vm_startup.sh (prints timing, always exits 0) +# +# Environment: +# SMOLVM_SKIP_SLOW=1 Skip long-running tests (>=25 s intentional sleeps) +# +# Parallelism (no-args mode only): +# resources runs in the background (pure CLI, no VMs -- always safe). +# All VM-starting suites run sequentially to avoid host resource contention +# that causes intermittent machine-start failures under concurrent load. +# + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# Map a group name to its script path and optional extra args. +# Prints "script [args...]" on stdout; returns non-zero for unknown groups. +get_suite() { + case "$1" in + bare) echo "$SCRIPT_DIR/test_machine_bare.sh" ;; + db) echo "$SCRIPT_DIR/test_db.sh" ;; + network) echo "$SCRIPT_DIR/test_network.sh" ;; + volumes) echo "$SCRIPT_DIR/test_volumes.sh" ;; + ports) echo "$SCRIPT_DIR/test_ports.sh" ;; + storage) echo "$SCRIPT_DIR/test_storage.sh" ;; + resources) echo "$SCRIPT_DIR/test_resources.sh" ;; + reliability) echo "$SCRIPT_DIR/test_reliability.sh" ;; + run) echo "$SCRIPT_DIR/test_machine_run.sh" ;; + image) echo "$SCRIPT_DIR/test_machine_image.sh" ;; + local-image) echo "$SCRIPT_DIR/test_machine_local_image.sh" ;; + fork-base) echo "$SCRIPT_DIR/test_fork_base_guards.sh" ;; + packed) echo "$SCRIPT_DIR/test_machine_packed.sh" ;; + cli) echo "$SCRIPT_DIR/test_cli.sh" ;; + api) echo "$SCRIPT_DIR/test_api.sh" ;; + virtio-net) echo "$SCRIPT_DIR/test_virtio_net.sh" ;; + smolfile) echo "$SCRIPT_DIR/test_smolfile.sh" ;; + secrets) echo "$SCRIPT_DIR/test_secrets.sh" ;; + pack) echo "$SCRIPT_DIR/test_pack.sh" ;; + pack-quick) echo "$SCRIPT_DIR/test_pack.sh --quick" ;; + gpu) echo "$SCRIPT_DIR/test_gpu.sh" ;; + scale) echo "$SCRIPT_DIR/test_scale.sh" ;; + *) return 1 ;; + esac +} + +FAILED_SUITES=() +PASSED_SUITES=() + +# Tell each suite to skip its own kill_orphan_smolvm_processes pre-flight. +# We do a single pre-flight kill here before launching anything, so per-suite +# kills are skipped to avoid parallel suites killing each other's VMs. +export SMOLVM_ORCHESTRATED=1 + +run_suite() { + local name="$1" + local suite_line + suite_line="$(get_suite "$name")" + echo "" + echo "------------------------------------------------" + echo " Running: $name" + echo "------------------------------------------------" + # shellcheck disable=SC2086 + if bash $suite_line; then + PASSED_SUITES+=("$name") + else + FAILED_SUITES+=("$name") + fi +} + +# Single pre-flight orphan kill before any suite touches the system. +echo "Pre-flight: killing orphan smolvm processes..." +pkill -f "smolvm serve" 2>/dev/null || true +pkill -f "smolvm-bin machine start" 2>/dev/null || true +pkill -f "smolvm machine start" 2>/dev/null || true +sleep 1 + +if [[ $# -eq 0 ]]; then + # resources: pure CLI validation (no VMs) -- safe to run concurrently with anything. + # All other suites start VMs, which under concurrent load causes intermittent + # machine-start failures in the sequential group, so they run sequentially. + PARALLEL_ORDER=(resources) + PARALLEL_PIDS=() + PARALLEL_OUTFILES=() + + for i in "${!PARALLEL_ORDER[@]}"; do + _name="${PARALLEL_ORDER[$i]}" + _outfile=$(mktemp) + PARALLEL_OUTFILES[$i]="$_outfile" + _suite_line="$(get_suite "$_name")" + # shellcheck disable=SC2086 + bash $_suite_line > "$_outfile" 2>&1 & + PARALLEL_PIDS[$i]=$! + done + + # Sequential suites -- run one at a time. + for _name in bare db reliability storage run image local-image fork-base network volumes ports packed; do + run_suite "$_name" + done + + # Print buffered output from parallel suites and collect results. + for i in "${!PARALLEL_ORDER[@]}"; do + _name="${PARALLEL_ORDER[$i]}" + echo "" + echo "------------------------------------------------" + echo " Results: $_name" + echo "------------------------------------------------" + cat "${PARALLEL_OUTFILES[$i]}" + if wait "${PARALLEL_PIDS[$i]}"; then + PASSED_SUITES+=("$_name") + else + FAILED_SUITES+=("$_name") + fi + rm -f "${PARALLEL_OUTFILES[$i]}" + done +else + for group in "$@"; do + # bench is not a pass/fail suite -- run directly and exit + if [[ "$group" == "bench" ]]; then + bash "$SCRIPT_DIR/bench_vm_startup.sh" + continue + fi + get_suite "$group" > /dev/null || { + echo "Unknown group: $group" + echo "Feature suites: bare db network volumes ports storage resources reliability run image local-image fork-base packed" + echo "Extended suites: cli api virtio-net smolfile pack pack-quick gpu scale" + echo "Other: bench" + exit 1 + } + run_suite "$group" + done +fi + +echo "" +echo "================================================" +echo " Suite Summary" +echo "================================================" +[[ ${#PASSED_SUITES[@]} -gt 0 ]] && echo " PASSED: ${PASSED_SUITES[*]}" +if [[ ${#FAILED_SUITES[@]} -gt 0 ]]; then + echo " FAILED: ${FAILED_SUITES[*]}" + exit 1 +fi diff --git a/tests/test_api.sh b/tests/test_api.sh new file mode 100755 index 0000000..01a4cb9 --- /dev/null +++ b/tests/test_api.sh @@ -0,0 +1,603 @@ +#!/usr/bin/env bash +# +# End-to-end HTTP API tests for smolvm. +# +# Tests the `smolvm serve` command with real VM operations. +# +# Usage: +# ./tests/test_api.sh + +source "$(dirname "$0")/common.sh" +init_smolvm + +echo "" +echo "==========================================" +echo " smolvm HTTP API Tests (End-to-End)" +echo "==========================================" +echo "" + +# Pre-flight: Kill any existing smolvm processes that might hold database lock +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +# API server configuration +API_SOCKET="${XDG_RUNTIME_DIR:-/tmp}/smolvm.sock" +API_URL="http://localhost" +SERVER_PID="" +MACHINE_NAME="api-test-machine" +REGISTRY_TEST_NAME="registry-coherence-test" + +# API client shortcut +CURL=(curl --unix-socket "$API_SOCKET") + +# ============================================================================= +# Setup / Teardown +# ============================================================================= + +start_server() { + log_info "Starting API server on unix://$API_SOCKET..." + rm -f "$API_SOCKET" + $SMOLVM serve start & + SERVER_PID=$! + + local retries=30 + while [[ $retries -gt 0 ]]; do + if "${CURL[@]}" -s "$API_URL/health" >/dev/null 2>&1; then + log_info "Server started (PID: $SERVER_PID)" + return 0 + fi + sleep 0.1 + ((retries--)) + done + + log_fail "Server failed to start" + return 1 +} + +stop_server() { + if [[ -n "$SERVER_PID" ]]; then + log_info "Stopping API server (PID: $SERVER_PID)..." + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + SERVER_PID="" + fi +} + +cleanup() { + # Delete machines via API (this stops the VMs properly) + if "${CURL[@]}" -s "$API_URL/health" >/dev/null 2>&1; then + "${CURL[@]}" -s -X DELETE "$API_URL/api/v1/machines/$MACHINE_NAME" >/dev/null 2>&1 || true + "${CURL[@]}" -s -X DELETE "$API_URL/api/v1/machines/$REGISTRY_TEST_NAME" >/dev/null 2>&1 || true + fi + stop_server + + # Fallback: if server died unexpectedly, try to stop any orphan VMs + # This handles cases where tests were interrupted + $SMOLVM machine stop 2>/dev/null || true +} + +trap cleanup EXIT + +# ============================================================================= +# Tests +# ============================================================================= + +test_health() { + local response + response=$("${CURL[@]}" -s "$API_URL/health") + [[ "$response" == *'"status":"ok"'* ]] +} + +test_create_and_start_machine() { + # Create machine + local status + status=$("${CURL[@]}" -s -o /dev/null -w "%{http_code}" -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d "{\"name\": \"$MACHINE_NAME\", \"network\": true, \"cpus\": 1, \"mem\": 512}") + [[ "$status" != "200" ]] && return 1 + + # Start machine (boots VM) + local response + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/start") + [[ "$response" == *'"state":"running"'* ]] +} + +test_exec_echo() { + local response + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/exec" \ + -H "Content-Type: application/json" \ + -d '{"command": ["echo", "api-test-marker"]}') + [[ "$response" == *"api-test-marker"* ]] +} + +test_exec_reads_vm_filesystem() { + local response + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/exec" \ + -H "Content-Type: application/json" \ + -d '{"command": ["cat", "/etc/os-release"]}') + [[ "$response" == *"Alpine"* ]] || [[ "$response" == *"alpine"* ]] +} + +test_exec_exit_codes() { + # Test exit code 0 + local response exit_code + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/exec" \ + -H "Content-Type: application/json" \ + -d '{"command": ["sh", "-c", "exit 0"]}') + exit_code=$(echo "$response" | grep -o '"exitCode":[0-9]*' | cut -d: -f2) + [[ "$exit_code" != "0" ]] && return 1 + + # Test exit code 42 + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/exec" \ + -H "Content-Type: application/json" \ + -d '{"command": ["sh", "-c", "exit 42"]}') + exit_code=$(echo "$response" | grep -o '"exitCode":[0-9]*' | cut -d: -f2) + [[ "$exit_code" == "42" ]] +} + +test_exec_with_env() { + local response + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/exec" \ + -H "Content-Type: application/json" \ + -d '{"command": ["sh", "-c", "echo $MY_VAR"], "env": [{"name": "MY_VAR", "value": "hello_from_api"}]}') + [[ "$response" == *"hello_from_api"* ]] +} + +test_exec_with_workdir() { + local response + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/exec" \ + -H "Content-Type: application/json" \ + -d '{"command": ["pwd"], "workdir": "/tmp"}') + [[ "$response" == *"/tmp"* ]] +} + +test_exec_shell_pipeline() { + local response + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/exec" \ + -H "Content-Type: application/json" \ + -d '{"command": ["sh", "-c", "echo hello world | wc -w"]}') + [[ "$response" == *"2"* ]] +} + +test_pull_and_run_image() { + # Pull image + "${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/images/pull" \ + -H "Content-Type: application/json" \ + -d '{"image": "alpine:latest"}' >/dev/null + + # Run in image + local response + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/run" \ + -H "Content-Type: application/json" \ + -d '{"image": "alpine:latest", "command": ["echo", "container-test"]}') + [[ "$response" == *"container-test"* ]] +} + +test_stop_machine() { + local response + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$MACHINE_NAME/stop") + [[ "$response" == *'"state":"stopped"'* ]] || [[ "$response" == *'"name":'* ]] +} + +test_delete_machine() { + local status + status=$("${CURL[@]}" -s -o /dev/null -w "%{http_code}" -X DELETE "$API_URL/api/v1/machines/$MACHINE_NAME") + [[ "$status" == "200" ]] +} + +test_error_not_found() { + local status + status=$("${CURL[@]}" -s -o /dev/null -w "%{http_code}" "$API_URL/api/v1/machines/nonexistent-12345") + [[ "$status" == "404" ]] +} + +test_error_bad_request() { + local status + status=$("${CURL[@]}" -s -o /dev/null -w "%{http_code}" -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d '{"name": ""}') + [[ "$status" == "400" ]] +} + +# ============================================================================= +# Registry Coherence Tests +# Validates that create → start → exec works in a single server session +# without restart. This was a known bug where ApiState and DB were out of sync. +# ============================================================================= + +test_registry_create_start_exec() { + # Create a fresh machine + local status + status=$("${CURL[@]}" -s -o /dev/null -w "%{http_code}" -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d "{\"name\": \"$REGISTRY_TEST_NAME\", \"network\": true, \"cpus\": 1, \"mem\": 512}") + [[ "$status" != "200" ]] && { echo "create failed: $status"; return 1; } + + # Start it + local response + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$REGISTRY_TEST_NAME/start") + [[ "$response" != *'"state":"running"'* ]] && { echo "start failed: $response"; return 1; } + + # Exec immediately — this is the key test. Before the registry fix, this returned 404. + response=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$REGISTRY_TEST_NAME/exec" \ + -H "Content-Type: application/json" \ + -d '{"command": ["echo", "registry-ok"]}') + [[ "$response" == *"registry-ok"* ]] +} + +test_registry_get_machine() { + local response + response=$("${CURL[@]}" -s "$API_URL/api/v1/machines/$REGISTRY_TEST_NAME") + [[ "$response" == *"\"name\":\"$REGISTRY_TEST_NAME\""* ]] && \ + [[ "$response" == *'"state":"running"'* ]] +} + +test_registry_cleanup() { + # Stop + delete the registry test machine + "${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$REGISTRY_TEST_NAME/stop" >/dev/null 2>&1 || true + local status + status=$("${CURL[@]}" -s -o /dev/null -w "%{http_code}" -X DELETE "$API_URL/api/v1/machines/$REGISTRY_TEST_NAME") + [[ "$status" == "200" ]] +} + +# ============================================================================= +# Run Tests +# ============================================================================= + +if ! start_server; then + echo -e "${RED}Failed to start server, aborting tests${NC}" + exit 1 +fi + +run_test "Health check" test_health || true +run_test "Create and start machine" test_create_and_start_machine || true +run_test "Exec echo" test_exec_echo || true +run_test "Exec reads VM filesystem" test_exec_reads_vm_filesystem || true +run_test "Exec exit codes" test_exec_exit_codes || true +run_test "Exec with environment variable" test_exec_with_env || true +run_test "Exec with workdir" test_exec_with_workdir || true +run_test "Exec shell pipeline" test_exec_shell_pipeline || true +run_test "Pull and run image" test_pull_and_run_image || true +run_test "Stop machine" test_stop_machine || true +run_test "Delete machine" test_delete_machine || true +run_test "Error: not found (404)" test_error_not_found || true +run_test "Error: bad request (400)" test_error_bad_request || true + +# Registry coherence tests (validates create→start→exec without restart) +run_test "Registry: create→start→exec in one session" test_registry_create_start_exec || true +run_test "Registry: get machine after create" test_registry_get_machine || true +run_test "Registry: cleanup test machine" test_registry_cleanup || true + +# Auto-generated names via API +test_api_auto_generated_names() { + # Without name → auto-generated vm-* name + local response + response=$("${CURL[@]}" -sf -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d '{"cpus": 1, "memoryMb": 512}' 2>&1) || { echo "Create failed: $response"; return 1; } + + local auto_name + auto_name=$(echo "$response" | jq -r '.name // empty') + [[ "$auto_name" == vm-* ]] || { echo "Expected vm-* name, got: $auto_name"; return 1; } + + # With explicit name → uses it + local explicit="api-name-test-$$" + response=$("${CURL[@]}" -sf -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d "{\"name\": \"$explicit\", \"cpus\": 1, \"memoryMb\": 512}" 2>&1) || return 1 + local name + name=$(echo "$response" | jq -r '.name') + [[ "$name" == "$explicit" ]] || { echo "Expected $explicit, got: $name"; return 1; } + + # Cleanup + "${CURL[@]}" -sf -X DELETE "$API_URL/api/v1/machines/$auto_name" 2>/dev/null + "${CURL[@]}" -sf -X DELETE "$API_URL/api/v1/machines/$explicit" 2>/dev/null +} + +echo "" +echo "--- Auto-Generated Names (API) ---" +echo "" + +run_test "API: auto-generated names" test_api_auto_generated_names || true + +# ============================================================================= +# Observability (Trace ID correlation) +# Tests verify that the API server returns trace IDs and they propagate +# through to the agent for end-to-end request correlation. +# ============================================================================= + +test_trace_id_in_response_header() { + # Every API response should have an X-Trace-Id header + local headers + headers=$("${CURL[@]}" -sI "$API_URL/health" 2>&1) + echo "$headers" | grep -qi "x-trace-id" || { echo "Missing X-Trace-Id header"; return 1; } + + # Trace ID should be a hex string + local trace_id + trace_id=$(echo "$headers" | grep -i "x-trace-id" | tr -d '\r' | awk '{print $2}') + [[ "$trace_id" =~ ^[0-9a-f]{16}$ ]] || { echo "Invalid trace_id format: '$trace_id'"; return 1; } +} + +test_trace_id_unique_per_request() { + # Two requests should get different trace IDs + local tid1 tid2 + tid1=$("${CURL[@]}" -sI "$API_URL/health" 2>&1 | grep -i "x-trace-id" | tr -d '\r' | awk '{print $2}') + tid2=$("${CURL[@]}" -sI "$API_URL/health" 2>&1 | grep -i "x-trace-id" | tr -d '\r' | awk '{print $2}') + + [[ -n "$tid1" ]] && [[ -n "$tid2" ]] && [[ "$tid1" != "$tid2" ]] || { + echo "Trace IDs not unique: '$tid1' vs '$tid2'" + return 1 + } +} + +echo "" +echo "--- Observability Tests ---" +echo "" + +run_test "Trace ID: present in response header" test_trace_id_in_response_header || true +run_test "Trace ID: unique per request" test_trace_id_unique_per_request || true + +test_trace_id_end_to_end() { + # Create and start a machine via API + local vm_name="trace-e2e-test-$$" + "${CURL[@]}" -sf -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d "{\"name\": \"$vm_name\", \"cpus\": 1, \"memoryMb\": 512}" >/dev/null 2>&1 || return 1 + "${CURL[@]}" -sf -X POST "$API_URL/api/v1/machines/$vm_name/start" >/dev/null 2>&1 || { + "${CURL[@]}" -sf -X DELETE "$API_URL/api/v1/machines/$vm_name" >/dev/null 2>&1 + return 1 + } + + # Exec a command and capture the trace ID from the response header + local trace_id + trace_id=$("${CURL[@]}" -sD - -X POST "$API_URL/api/v1/machines/$vm_name/exec" \ + -H "Content-Type: application/json" \ + -d '{"command": ["echo", "trace-e2e"]}' 2>&1 | grep -i "x-trace-id" | tr -d '\r' | awk '{print $2}') + + # Cleanup + "${CURL[@]}" -sf -X POST "$API_URL/api/v1/machines/$vm_name/stop" >/dev/null 2>&1 + "${CURL[@]}" -sf -X DELETE "$API_URL/api/v1/machines/$vm_name" >/dev/null 2>&1 + + # Verify we got a trace ID back + [[ -n "$trace_id" ]] || { echo "No trace ID returned from exec"; return 1; } + [[ "$trace_id" =~ ^[0-9a-f]{16}$ ]] || { echo "Invalid trace ID: '$trace_id'"; return 1; } + + echo "End-to-end trace ID: $trace_id" +} + +run_test "Trace ID: end-to-end with running VM" test_trace_id_end_to_end || true + +test_metrics_endpoint() { + local response + response=$("${CURL[@]}" -s "$API_URL/metrics" 2>&1) + + # Should return Prometheus text format + [[ -n "$response" ]] || { echo "Empty metrics response"; return 1; } + + # After making requests, the counter should exist + echo "$response" | grep -q "smolvm_api_requests_total" || { echo "Missing request counter"; return 1; } +} + +test_health_enriched() { + local response + response=$("${CURL[@]}" -s "$API_URL/health" 2>&1) + + # Should have version + echo "$response" | grep -q '"version"' || { echo "Missing version"; return 1; } + + # Should have machine counts + echo "$response" | grep -q '"machines"' || { echo "Missing machines"; return 1; } + + # Should have uptime + echo "$response" | grep -q '"uptime_seconds"' || { echo "Missing uptime"; return 1; } +} + +run_test "Prometheus: /metrics endpoint" test_metrics_endpoint || true +run_test "Health: enriched response" test_health_enriched || true + +# ============================================================================= +# Create from .smolmachine via API +# ============================================================================= + +test_api_create_from_smolmachine() { + local tmpdir + tmpdir=$(mktemp -d) + local pack_output="$tmpdir/api-from-pack" + + # Pack alpine + $SMOLVM pack create --image alpine:latest -o "$pack_output" --cpus 1 --mem 512 2>&1 >/dev/null || { + echo "SKIP: pack create failed" + rm -rf "$tmpdir" + return 0 + } + local sidecar + sidecar=$(cd "$tmpdir" && pwd)/api-from-pack.smolmachine + [[ -f "$sidecar" ]] || { echo "FAIL: no sidecar"; rm -rf "$tmpdir"; return 1; } + + local vm_name="api-from-$$" + + # Create via API with from field + local create_resp + create_resp=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d "{\"name\": \"$vm_name\", \"from\": \"$sidecar\", \"memoryMb\": 512}") + echo "$create_resp" | grep -q "$vm_name" || { + echo "FAIL: create response missing name: $create_resp" + rm -rf "$tmpdir"; return 1 + } + + # Start + local start_resp + start_resp=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$vm_name/start") + echo "$start_resp" | grep -q "running" || { + echo "FAIL: start failed: $start_resp" + "${CURL[@]}" -s -X DELETE "$API_URL/api/v1/machines/$vm_name" >/dev/null + rm -rf "$tmpdir"; return 1 + } + + # Exec + local exec_resp + exec_resp=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$vm_name/exec" \ + -H "Content-Type: application/json" \ + -d '{"command": ["echo", "api-from-ok"]}') + echo "$exec_resp" | grep -q "api-from-ok" || { + echo "FAIL: exec failed: $exec_resp" + "${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$vm_name/stop" >/dev/null + "${CURL[@]}" -s -X DELETE "$API_URL/api/v1/machines/$vm_name" >/dev/null + rm -rf "$tmpdir"; return 1 + } + + # Cleanup + "${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$vm_name/stop" >/dev/null + "${CURL[@]}" -s -X DELETE "$API_URL/api/v1/machines/$vm_name" >/dev/null + rm -rf "$tmpdir" +} + +# Regression test for the .smolmachine layer-drop bug. Unlike the /exec test +# above (which runs in the base agent rootfs and passes even when the bundled +# layers are dropped — a false negative), /run enters an overlay built from the +# bundled image, so it requires the packed layers to be mounted into the guest. +# The machine has no network, so /run cannot fall back to a registry pull: if the +# start path drops packed_layers_dir, /run hard-fails "image not found". This is +# RED on a start path that passes Default::default() and GREEN once the packed +# layers are wired through. It also proves the image tag resolves *through* the +# packed layers, not merely that packed_layers_dir is set. +test_api_run_from_smolmachine_offline() { + local tmpdir + tmpdir=$(mktemp -d) + local pack_output="$tmpdir/api-run-from-pack" + + # Pack alpine:latest into a .smolmachine bundle. + $SMOLVM pack create --image alpine:latest -o "$pack_output" --cpus 1 --mem 512 2>&1 >/dev/null || { + echo "SKIP: pack create failed" + rm -rf "$tmpdir" + return 0 + } + local sidecar + sidecar=$(cd "$tmpdir" && pwd)/api-run-from-pack.smolmachine + [[ -f "$sidecar" ]] || { echo "FAIL: no sidecar"; rm -rf "$tmpdir"; return 1; } + + local vm_name="api-run-from-$$" + + # Create from bundle (network defaults off → no registry fallback for /run). + local create_resp + create_resp=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d "{\"name\": \"$vm_name\", \"from\": \"$sidecar\", \"memoryMb\": 512}") + echo "$create_resp" | grep -q "$vm_name" || { + echo "FAIL: create response missing name: $create_resp" + rm -rf "$tmpdir"; return 1 + } + + # Start + local start_resp + start_resp=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$vm_name/start") + echo "$start_resp" | grep -q "running" || { + echo "FAIL: start failed: $start_resp" + "${CURL[@]}" -s -X DELETE "$API_URL/api/v1/machines/$vm_name" >/dev/null + rm -rf "$tmpdir"; return 1 + } + + # Bug 1 (layer-drop on start): /run the bundled image offline. Reaches the + # guest's image overlay, which needs the packed layers mounted. + local run_resp + run_resp=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$vm_name/run" \ + -H "Content-Type: application/json" \ + -d '{"image": "alpine:latest", "command": ["cat", "/etc/os-release"]}') + + # Bug 2 (eager sidecar extraction on the running-VM preflight): remove the + # .smolmachine while the VM is running, then exercise the implicit-start + # preflight again. The running guest already has its layers mounted, so + # losing the original bundle must NOT fail the request. + rm -f "$sidecar" + local rerun_resp + rerun_resp=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$vm_name/run" \ + -H "Content-Type: application/json" \ + -d '{"image": "alpine:latest", "command": ["true"]}') + + "${CURL[@]}" -s -X POST "$API_URL/api/v1/machines/$vm_name/stop" >/dev/null + "${CURL[@]}" -s -X DELETE "$API_URL/api/v1/machines/$vm_name" >/dev/null + rm -rf "$tmpdir" + + if echo "$run_resp" | grep -qi "image not found"; then + echo "FAIL: bundled layers were dropped on start — /run could not find the image offline: $run_resp" + return 1 + fi + echo "$run_resp" | grep -qi "alpine" || { + echo "FAIL: /run did not enter the bundled image overlay: $run_resp" + return 1 + } + if echo "$rerun_resp" | grep -qi "source .smolmachine not found"; then + echo "FAIL: running-VM preflight re-extracted the .smolmachine and failed after the bundle was removed: $rerun_resp" + return 1 + fi +} + +test_api_from_and_image_conflict() { + local resp + resp=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d '{"from": "/tmp/test.smolmachine", "image": "alpine"}') + echo "$resp" | grep -q "mutually exclusive" || { + echo "FAIL: expected conflict error: $resp" + return 1 + } +} + +test_api_from_nonexistent_sidecar() { + local resp + resp=$("${CURL[@]}" -s -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d '{"from": "/nonexistent/file.smolmachine"}') + echo "$resp" | grep -q "not found" || { + echo "FAIL: expected not found error: $resp" + return 1 + } +} + +echo "" +echo "--- Create from .smolmachine via API ---" +echo "" + +run_test "API: create from .smolmachine" test_api_create_from_smolmachine || true +run_test "API: run from .smolmachine (offline)" test_api_run_from_smolmachine_offline || true +run_test "API: from + image conflict" test_api_from_and_image_conflict || true +run_test "API: from nonexistent sidecar" test_api_from_nonexistent_sidecar || true + +# ============================================================================= +# Created-state machines survive server restart +# ============================================================================= + +test_created_machine_survives_restart() { + local name="created-survive-$$" + + # Create a machine (never start it) + local status + status=$("${CURL[@]}" -s -o /dev/null -w "%{http_code}" -X POST "$API_URL/api/v1/machines" \ + -H "Content-Type: application/json" \ + -d "{\"name\": \"$name\", \"cpus\": 1, \"memoryMb\": 512}") + [[ "$status" == "200" ]] || { echo "FAIL: create returned $status"; return 1; } + + # Verify it exists + local get_status + get_status=$("${CURL[@]}" -s -o /dev/null -w "%{http_code}" "$API_URL/api/v1/machines/$name") + [[ "$get_status" == "200" ]] || { echo "FAIL: machine not found after create"; return 1; } + + # Restart the server + stop_server + start_server || { echo "FAIL: server restart failed"; return 1; } + + # Machine should still exist + get_status=$("${CURL[@]}" -s -o /dev/null -w "%{http_code}" "$API_URL/api/v1/machines/$name") + [[ "$get_status" == "200" ]] || { + echo "FAIL: created machine deleted on server restart (got $get_status)" + return 1 + } + + # Cleanup + "${CURL[@]}" -s -X DELETE "$API_URL/api/v1/machines/$name" >/dev/null 2>&1 || true +} + +run_test "API: created machine survives server restart" test_created_machine_survives_restart || true + +print_summary "HTTP API Tests" diff --git a/tests/test_cli.sh b/tests/test_cli.sh new file mode 100755 index 0000000..bccecd2 --- /dev/null +++ b/tests/test_cli.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# +# CLI tests for smolvm. +# +# Tests basic CLI functionality like --version, --help, and subcommand structure. +# Does not require VM environment. +# +# Usage: +# ./tests/test_cli.sh + +source "$(dirname "$0")/common.sh" +init_smolvm + +echo "" +echo "==========================================" +echo " smolvm CLI Tests" +echo "==========================================" +echo "" + +# ============================================================================= +# Version and Help +# ============================================================================= + +test_version() { + local output + output=$($SMOLVM --version 2>&1) + [[ "$output" == *"smolvm"* ]] +} + +test_help() { + local output + output=$($SMOLVM --help 2>&1) + [[ "$output" == *"machine"* ]] && \ + [[ "$output" == *"container"* ]] && \ + [[ "$output" == *"pack"* ]] +} + +test_machine_help() { + local output + output=$($SMOLVM machine --help 2>&1) + [[ "$output" == *"run"* ]] && \ + [[ "$output" == *"create"* ]] && \ + [[ "$output" == *"start"* ]] && \ + [[ "$output" == *"stop"* ]] && \ + [[ "$output" == *"exec"* ]] && \ + [[ "$output" == *"images"* ]] && \ + [[ "$output" == *"prune"* ]] +} + +test_machine_run_help() { + local output + output=$($SMOLVM machine run --help 2>&1) + [[ "$output" == *"IMAGE"* ]] && \ + [[ "$output" == *"--net"* ]] && \ + [[ "$output" == *"--detach"* ]] && \ + [[ "$output" == *"--oci-platform"* ]] +} + +test_no_container_command() { + # Container subcommand was removed — should not exist + local exit_code=0 + $SMOLVM container --help 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] +} + +test_pack_help() { + local output + output=$($SMOLVM pack create --help 2>&1) + [[ "$output" == *"--oci-platform"* ]] && \ + [[ "$output" == *"--output"* ]] +} + +# ============================================================================= +# Removed Commands +# ============================================================================= + + +# ============================================================================= +# Machine Aliases +# ============================================================================= + +test_vm_alias() { + local output + output=$($SMOLVM vm --help 2>&1) + [[ "$output" == *"run"* ]] && \ + [[ "$output" == *"create"* ]] +} + +# ============================================================================= +# Invalid Commands +# ============================================================================= + +test_invalid_subcommand() { + ! $SMOLVM nonexistent-command 2>/dev/null +} + +# ============================================================================= +# Flag Presence +# ============================================================================= + +test_machine_create_flags() { + local output + output=$($SMOLVM machine create --help 2>&1) + [[ "$output" == *"--overlay"* ]] && \ + [[ "$output" == *"--storage"* ]] && \ + [[ "$output" == *"--net"* ]] && \ + [[ "$output" == *"--smolfile"* ]] +} + +test_machine_run_flags() { + local output + output=$($SMOLVM machine run --help 2>&1) + [[ "$output" == *"--overlay"* ]] && \ + [[ "$output" == *"--volume"* ]] && \ + [[ "$output" == *"--port"* ]] && \ + [[ "$output" == *"--smolfile"* ]] +} + +# ============================================================================= +# Run Tests +# ============================================================================= + +run_test "Version command" test_version || true +run_test "Help command" test_help || true +run_test "Machine help" test_machine_help || true +run_test "Machine run help" test_machine_run_help || true +run_test "No container command" test_no_container_command || true +run_test "Pack help" test_pack_help || true +run_test "vm alias works" test_vm_alias || true +run_test "Invalid subcommand fails" test_invalid_subcommand || true +run_test "Machine create flags" test_machine_create_flags || true +run_test "Machine run flags" test_machine_run_flags || true + +print_summary "CLI Tests" diff --git a/tests/test_db.sh b/tests/test_db.sh new file mode 100755 index 0000000..f9b92f1 --- /dev/null +++ b/tests/test_db.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# +# Database State Tests +# +# Part of the smolvm test suite. Run with: ./tests/test_db.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Database State Tests" +echo "==========================================" +echo "" + +test_db_persistence_across_restart() { + local vm_name="db-test-vm-$$" + + # Clean up any existing + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create a named VM with specific configuration + $SMOLVM machine create --name "$vm_name" --cpus 2 --mem 1024 2>&1 + + # Verify it was created with correct config + local list_output + list_output=$($SMOLVM machine ls --json 2>&1) + if [[ "$list_output" != *"$vm_name"* ]]; then + echo "VM was not created" + return 1 + fi + + if [[ "$list_output" != *'"cpus": 2'* ]] || [[ "$list_output" != *'"memory_mib": 1024'* ]]; then + echo "VM configuration not persisted correctly" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Clean up + $SMOLVM machine delete --name "$vm_name" -f 2>&1 + ensure_data_dir_deleted "$vm_name" +} + +test_db_vm_state_update() { + local vm_name="db-state-test-$$" + + # Clean up any existing + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create a named VM + $SMOLVM machine create --name "$vm_name" 2>&1 + + # Check initial state is "created" + local initial_state + initial_state=$($SMOLVM machine ls --json 2>&1) + if [[ "$initial_state" != *'"state": "created"'* ]]; then + echo "Initial state should be 'created'" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Start the VM + $SMOLVM machine start --name "$vm_name" 2>&1 + + # Check state changed to "running" + local running_state + running_state=$($SMOLVM machine ls --json 2>&1) + if [[ "$running_state" != *'"state": "running"'* ]]; then + echo "State should be 'running' after start" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Stop the VM + $SMOLVM machine stop --name "$vm_name" 2>&1 + + # Check state changed to "stopped" + local stopped_state + stopped_state=$($SMOLVM machine ls --json 2>&1) + + # Clean up + $SMOLVM machine delete --name "$vm_name" -f 2>&1 + ensure_data_dir_deleted "$vm_name" + + [[ "$stopped_state" == *'"state": "stopped"'* ]] +} + +test_db_delete_removes_from_db() { + local vm_name="db-delete-test-$$" + + # Clean up any existing + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create a VM + $SMOLVM machine create --name "$vm_name" 2>&1 + + # Verify it exists + local before_delete + before_delete=$($SMOLVM machine ls --json 2>&1) + if [[ "$before_delete" != *"$vm_name"* ]]; then + echo "VM should exist before delete" + return 1 + fi + + # Delete it + $SMOLVM machine delete --name "$vm_name" -f 2>&1 + ensure_data_dir_deleted "$vm_name" + + # Verify it's gone + local after_delete + after_delete=$($SMOLVM machine ls --json 2>&1) + + [[ "$after_delete" != *"$vm_name"* ]] +} + +test_db_default_vm_appears_in_list_on_start() { + cleanup_machine + + # Start the default VM (no name) + $SMOLVM machine start 2>&1 || return 1 + + # Verify "default" appears in machine ls --json as running + local list_output + list_output=$($SMOLVM machine ls --json 2>&1) + + # Clean up + $SMOLVM machine stop 2>/dev/null || true + + [[ "$list_output" == *'"name": "default"'* ]] && \ + [[ "$list_output" == *'"state": "running"'* ]] +} + +test_db_default_vm_shows_stopped_after_stop() { + cleanup_machine + + # Start then stop the default VM + $SMOLVM machine start 2>&1 || return 1 + $SMOLVM machine stop 2>&1 || return 1 + + # Verify "default" shows as stopped + local list_output + list_output=$($SMOLVM machine ls --json 2>&1) + + [[ "$list_output" == *'"name": "default"'* ]] && \ + [[ "$list_output" == *'"state": "stopped"'* ]] +} + +test_db_default_vm_state_transitions() { + cleanup_machine + + # Start default VM + $SMOLVM machine start 2>&1 || return 1 + + # Check running state + local running_state + running_state=$($SMOLVM machine ls --json 2>&1) + if [[ "$running_state" != *'"state": "running"'* ]]; then + echo "State should be 'running' after start" + $SMOLVM machine stop 2>/dev/null || true + return 1 + fi + + # Stop default VM + $SMOLVM machine stop 2>&1 || return 1 + + # Check stopped state + local stopped_state + stopped_state=$($SMOLVM machine ls --json 2>&1) + if [[ "$stopped_state" != *'"state": "stopped"'* ]]; then + echo "State should be 'stopped' after stop" + return 1 + fi + + # Restart and check running again + $SMOLVM machine start 2>&1 || return 1 + local restarted_state + restarted_state=$($SMOLVM machine ls --json 2>&1) + + # Clean up + $SMOLVM machine stop 2>/dev/null || true + + [[ "$restarted_state" == *'"state": "running"'* ]] +} + + +run_test "DB persistence across restart" test_db_persistence_across_restart || true +run_test "DB VM state update" test_db_vm_state_update || true +run_test "DB delete removes from database" test_db_delete_removes_from_db || true +run_test "DB default VM appears in list on start" test_db_default_vm_appears_in_list_on_start || true +run_test "DB default VM shows stopped after stop" test_db_default_vm_shows_stopped_after_stop || true +run_test "DB default VM state transitions" test_db_default_vm_state_transitions || true + +print_summary "DB State Tests" diff --git a/tests/test_fork_base_guards.sh b/tests/test_fork_base_guards.sh new file mode 100755 index 0000000..60219d3 --- /dev/null +++ b/tests/test_fork_base_guards.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# +# Fork-base guard tests (regression for BUG-142/143/144) +# +# A forkable golden is snapshot-frozen once it has clones whose disks are +# copy-on-write overlays backed by its disks. It must therefore: +# - report state "frozen" (not the misleading "unreachable") [BUG-144] +# - answer `status` instantly, without probing its paused agent [BUG-143] +# - refuse `stop`/`start` while clones exist (no silent reap) [BUG-142] +# - stay usable as a fork base (new clones can still fork) [BUG-142] +# - reap cleanly under `delete --force` with no orphaned VMM +# +# Part of the smolvm test suite. Run with: ./tests/test_fork_base_guards.sh + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +GOLD="forkbase-gold-$$" +C1="forkbase-c1-$$" +C2="forkbase-c2-$$" + +cleanup_fork() { + for m in "$C1" "$C2" "$GOLD"; do + "$SMOLVM" machine delete --name "$m" -f >/dev/null 2>&1 || true + done +} +trap cleanup_fork EXIT + +echo "" +echo "==========================================" +echo " Fork-base Guard Tests (BUG-142/143/144)" +echo "==========================================" +echo "" + +# Bring up a frozen golden with one live clone. If fork isn't supported on +# this platform/build, skip the whole suite cleanly. +setup_frozen_golden() { + cleanup_fork + "$SMOLVM" machine create --name "$GOLD" >/dev/null 2>&1 || return 1 + "$SMOLVM" machine start --name "$GOLD" --forkable >/dev/null 2>&1 || return 1 + "$SMOLVM" machine fork --golden "$GOLD" --name "$C1" >/dev/null 2>&1 || return 1 +} + +if ! setup_frozen_golden; then + log_info "fork/--forkable not available on this platform/build — skipping suite" + print_summary "Fork-base Guard Tests" + exit 0 +fi + +# BUG-144: golden reports "frozen", not the misleading "unreachable". +# Use single-object `status --json` so there's no array-association +# fragility, and tolerate pretty-printed spacing (`"state": "frozen"`). +test_golden_state_is_frozen() { + local out + out=$(run_with_timeout 8 "$SMOLVM" machine status --name "$GOLD" --json 2>/dev/null) + echo "golden state json: $(echo "$out" | grep -oE '"state":[[:space:]]*"[^"]*"')" + echo "$out" | grep -qE '"state":[[:space:]]*"frozen"' +} + +# BUG-143: status on the frozen golden returns fast (was a >12s hang) and +# reports frozen. Assert it completes well under the old timeout. +test_status_is_fast_and_frozen() { + local start_ms end_ms out + start_ms=$(date +%s%N) + out=$(run_with_timeout 8 "$SMOLVM" machine status --name "$GOLD" 2>&1) + end_ms=$(date +%s%N) + local took_ms=$(( (end_ms - start_ms) / 1000000 )) + echo "status: '${out}' in ${took_ms}ms" + [[ "$out" == *"frozen"* ]] && [[ $took_ms -lt 3000 ]] +} + +# BUG-142: stop must refuse (not silently reap the golden). +test_stop_refuses() { + local out exit_code=0 + out=$("$SMOLVM" machine stop --name "$GOLD" 2>&1) || exit_code=$? + echo "stop output: $out (exit $exit_code)" + [[ $exit_code -ne 0 ]] && [[ "$out" == *"fork base"* ]] +} + +# BUG-142: start must refuse (not reap-then-error). +test_start_refuses() { + local out exit_code=0 + out=$("$SMOLVM" machine start --name "$GOLD" 2>&1) || exit_code=$? + echo "start output: $out (exit $exit_code)" + [[ $exit_code -ne 0 ]] && [[ "$out" == *"fork base"* ]] +} + +# BUG-142: after the refused stop/start, the golden is still a usable fork +# base — a brand-new clone can be forked from it. +test_golden_still_forkable() { + "$SMOLVM" machine delete --name "$C2" -f >/dev/null 2>&1 || true + local out exit_code=0 + out=$(run_with_timeout 30 "$SMOLVM" machine fork --golden "$GOLD" --name "$C2" 2>&1) || exit_code=$? + echo "fork output: $out (exit $exit_code)" + [[ $exit_code -eq 0 ]] && [[ "$out" == *"Forked"* ]] +} + +# The original clone keeps running through all of the above. +test_original_clone_alive() { + local out + out=$(run_with_timeout 15 "$SMOLVM" machine exec --name "$C1" -- echo "clone-alive" 2>&1) + echo "clone exec: $out" + [[ "$out" == *"clone-alive"* ]] +} + +# delete --force breaks the chain and reaps the golden's VMM (no orphan). +test_force_delete_reaps_golden() { + "$SMOLVM" machine delete --name "$GOLD" -f >/dev/null 2>&1 || return 1 + # Golden record is gone. + "$SMOLVM" machine ls --json 2>/dev/null | grep -q "\"name\":\"$GOLD\"" && { + echo "FAIL: golden record still present after force delete"; return 1; } + echo "golden record removed" + return 0 +} + +run_test "BUG-144: golden state is 'frozen'" test_golden_state_is_frozen || true +run_test "BUG-143: status is fast and reports frozen" test_status_is_fast_and_frozen || true +run_test "BUG-142: stop refuses on a fork base" test_stop_refuses || true +run_test "BUG-142: start refuses on a fork base" test_start_refuses || true +run_test "BUG-142: golden still forkable after refusal" test_golden_still_forkable || true +run_test "Original clone stays alive throughout" test_original_clone_alive || true +run_test "delete --force reaps golden (no orphan)" test_force_delete_reaps_golden || true + +print_summary "Fork-base Guard Tests" diff --git a/tests/test_gpu.sh b/tests/test_gpu.sh new file mode 100755 index 0000000..9d724ed --- /dev/null +++ b/tests/test_gpu.sh @@ -0,0 +1,372 @@ +#!/usr/bin/env bash +# +# GPU acceleration integration tests for smolvm. +# +# Tests the --gpu flag across two tiers: +# 1. virtio-gpu device enumeration (Alpine — fast, no Mesa needed) +# 2. Full Vulkan workloads via patched Mesa on Fedora 42 +# +# The suite is SKIPPED (not failed) if GPU support is not compiled into +# libkrun or virglrenderer is unavailable on the host. This keeps CI green +# on machines without GPU infrastructure. +# +# Usage: +# ./tests/test_gpu.sh +# ./tests/run_tests.sh gpu + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +# Name for the shared Fedora machine used across Vulkan workload tests. +GPU_FEDORA_MACHINE="gpu-fedora-$$" + +cleanup_gpu() { + "$SMOLVM" machine stop --name "$GPU_FEDORA_MACHINE" 2>/dev/null || true + "$SMOLVM" machine delete --name "$GPU_FEDORA_MACHINE" -f 2>/dev/null || true +} +trap cleanup_gpu EXIT + +echo "" +echo "==========================================" +echo " smolvm GPU Tests" +echo "==========================================" +echo "" + +# ============================================================================= +# GPU availability probe +# ============================================================================= +# Run a trivial --gpu command in Alpine to confirm GPU support is compiled in +# and virglrenderer is present. If either is missing, smolvm emits one of: +# - "libkrun was built without GPU support (krun_set_gpu_options2 not found)" +# - "krun_set_gpu_options2 failed ... Check that virglrenderer is installed." +# On those messages we skip the whole suite (not fail) — GPU is optional. +# +# Note: --net is required here because image pulls happen inside the VM. + +log_info "Probing GPU support..." +_PROBE_EXIT=0 +_PROBE_OUT=$(run_with_timeout 120 "$SMOLVM" machine run --gpu --net --image alpine:latest -- true 2>&1) \ + || _PROBE_EXIT=$? + +if [[ $_PROBE_EXIT -ne 0 ]]; then + if echo "$_PROBE_OUT" | grep -qE "krun_set_gpu_options2|without GPU support|virglrenderer"; then + echo "" + log_skip "GPU not available on this host — skipping all GPU tests" + log_skip " Cause: $(echo "$_PROBE_OUT" | grep -E "krun_set_gpu_options2|without GPU|virglrenderer" | head -1)" + log_skip " To enable: rebuild libkrun with GPU=1 and install virglrenderer" + echo "" + print_summary "GPU Tests" + exit 0 + fi + # Probe failed for an unrelated reason (e.g. transient network). Proceed + # and let the individual tests surface a meaningful error. + log_info "Probe failed (exit $_PROBE_EXIT) — not a GPU capability error, proceeding" +else + log_info "GPU probe passed — virglrenderer and krun_set_gpu_options2 both available" +fi + +# ============================================================================= +# Helper +# ============================================================================= + +# Returns 0 if the shared Fedora GPU machine is currently running. +_fedora_running() { + "$SMOLVM" machine status --name "$GPU_FEDORA_MACHINE" 2>&1 | grep -q "running" +} + +# ============================================================================= +# Section 1: virtio-gpu device enumeration (Alpine, no Mesa) +# ============================================================================= +# When --gpu is passed, libkrun creates a virtio-gpu PCI device in the VM. +# The guest kernel (CONFIG_DRM_VIRTIO_GPU=y in libkrunfw) initialises the DRM +# subsystem and creates two device nodes: +# /dev/dri/renderD128 — render node (Vulkan/OpenGL compute, no modesetting) +# /dev/dri/card0 — DRM card (modesetting, display) +# +# The agent's add_gpu_devices_if_available() (crates/smolvm-agent/src/oci.rs) +# then forwards both nodes into the OCI container spec with mode 0o666 so +# unprivileged processes can open them. +# +# Alpine has no Vulkan userspace, but the kernel device nodes are enough to +# verify the full host→guest→container forwarding pipeline. + +echo "" +echo "Running virtio-gpu device tests (Alpine)..." + +test_dri_renderD128_present() { + local out + out=$(run_with_timeout 120 "$SMOLVM" machine run --gpu --net --image alpine:latest -- \ + ls /dev/dri/renderD128 2>&1) || return 1 + [[ "$out" == *"renderD128"* ]] || { echo "FAIL: renderD128 absent (got: $out)"; return 1; } +} + +test_dri_card0_present() { + local out + out=$(run_with_timeout 120 "$SMOLVM" machine run --gpu --net --image alpine:latest -- \ + ls /dev/dri/card0 2>&1) || return 1 + [[ "$out" == *"card0"* ]] || { echo "FAIL: card0 absent (got: $out)"; return 1; } +} + +test_renderD128_world_readable() { + # The OCI spec sets mode 0o666 on the render node. Verify an unprivileged + # stat succeeds (open would need actual Vulkan libs, but stat is sufficient). + local out + out=$(run_with_timeout 120 "$SMOLVM" machine run --gpu --net --image alpine:latest -- \ + stat /dev/dri/renderD128 2>&1) || { echo "FAIL: stat failed (got: $out)"; return 1; } + [[ "$out" == *"renderD128"* ]] || { echo "FAIL: stat output unexpected (got: $out)"; return 1; } +} + +test_interactive_run_has_dri() { + # `-i`/`-t` uses the agent's interactive OCI bundle path. It must wire + # GPU devices just like non-interactive runs, otherwise shells started with + # `-it` cannot use Vulkan even though the VM itself has virtio-gpu. + local out + out=$(run_with_timeout 120 "$SMOLVM" machine run --gpu --net -i --image alpine:latest -- \ + ls /dev/dri/renderD128 2>&1) || { echo "FAIL: interactive run missing renderD128 (got: $out)"; return 1; } + [[ "$out" == *"renderD128"* ]] || { echo "FAIL: interactive run output unexpected (got: $out)"; return 1; } +} + +test_no_dri_without_gpu() { + # Without --gpu, no virtio-gpu device → no /dev/dri in guest → no DRI + # devices in OCI container spec. ls /dev/dri should fail or produce nothing. + local out exit_code=0 + out=$(run_with_timeout 120 "$SMOLVM" machine run --net --image alpine:latest -- \ + ls /dev/dri 2>&1) || exit_code=$? + if [[ $exit_code -eq 0 ]] && echo "$out" | grep -qE "render|card"; then + echo "FAIL: /dev/dri device nodes visible without --gpu: $out" + return 1 + fi +} + +test_gpu_vram_zero_rejected() { + # Validation in src/data/resources.rs rejects gpu_vram_mib == 0 before + # any VM starts, so --net is not required but included for clarity. + local out exit_code=0 + out=$("$SMOLVM" machine run --gpu --gpu-vram 0 --net --image alpine:latest -- true 2>&1) \ + || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "FAIL: --gpu-vram 0 was accepted (exit 0)"; return 1; } + echo " (correctly rejected with exit $exit_code)" +} + +test_named_machine_gpu_persists() { + # Create a named machine with --gpu, stop+start it, verify DRI still present. + # This exercises the full DB round-trip: VmResources{gpu:true} → sqlite → + # deserialise → launcher → krun_set_gpu_options2. + local name="gpu-named-$$" + "$SMOLVM" machine stop --name "$name" 2>/dev/null || true + "$SMOLVM" machine delete --name "$name" -f 2>/dev/null || true + + "$SMOLVM" machine create --name "$name" --gpu --net --image alpine:latest 2>&1 || return 1 + "$SMOLVM" machine start --name "$name" 2>&1 || { + "$SMOLVM" machine delete --name "$name" -f 2>/dev/null; return 1 + } + + local out rc=0 + out=$("$SMOLVM" machine exec --name "$name" -- ls /dev/dri/renderD128 2>&1) || rc=$? + + "$SMOLVM" machine stop --name "$name" 2>/dev/null || true + "$SMOLVM" machine delete --name "$name" -f 2>/dev/null || true + + [[ $rc -eq 0 ]] && [[ "$out" == *"renderD128"* ]] || { + echo "FAIL: renderD128 absent in named GPU machine (got: $out, exit $rc)" + return 1 + } +} + +run_test "GPU: /dev/dri/renderD128 present with --gpu" test_dri_renderD128_present || true +run_test "GPU: /dev/dri/card0 present with --gpu" test_dri_card0_present || true +run_test "GPU: renderD128 accessible (stat succeeds)" test_renderD128_world_readable || true +run_test "GPU: interactive run exposes /dev/dri" test_interactive_run_has_dri || true +run_test "GPU: no /dev/dri without --gpu (isolation)" test_no_dri_without_gpu || true +run_test "GPU: --gpu-vram 0 rejected by validation" test_gpu_vram_zero_rejected || true +run_test "GPU: named machine DB persistence of --gpu flag" test_named_machine_gpu_persists || true + +# ============================================================================= +# Section 1b: pack create --gpu (manifest embedding) +# ============================================================================= +# pack create --gpu must write gpu=true into the .smolmachine manifest so the +# packed binary boots with a virtio-gpu device. We verify the round-trip: +# pack create --gpu → .smolmachine → pack run → /dev/dri/renderD128 visible. + +echo "" +echo "Running pack create --gpu tests..." + +test_pack_create_gpu_manifest() { + local tmp_dir + tmp_dir=$(mktemp -d) + local out_path="$tmp_dir/gpu-alpine" + + echo " Packing alpine:latest with --gpu..." + run_with_timeout 300 "$SMOLVM" pack create \ + --image alpine:latest --gpu \ + --output "$out_path" 2>&1 || { + rm -rf "$tmp_dir" + return 1 + } + [[ -f "$out_path.smolmachine" ]] || { + echo "FAIL: no .smolmachine produced" + rm -rf "$tmp_dir" + return 1 + } + + # Run the packed binary and verify /dev/dri/renderD128 is present. + # pack run reads manifest.gpu and passes it to VmResources. + echo " Running packed binary, checking for /dev/dri/renderD128..." + local run_out rc=0 + run_out=$(run_with_timeout 120 "$SMOLVM" pack run \ + --sidecar "$out_path.smolmachine" -- \ + ls /dev/dri/renderD128 2>&1) || rc=$? + + rm -rf "$tmp_dir" + [[ $rc -eq 0 ]] && [[ "$run_out" == *"renderD128"* ]] || { + echo "FAIL: renderD128 not present in GPU-packed binary (exit $rc, got: $run_out)" + return 1 + } +} + +test_pack_create_no_gpu_manifest() { + # Without --gpu, the packed binary must NOT have /dev/dri. + local tmp_dir + tmp_dir=$(mktemp -d) + local out_path="$tmp_dir/nogpu-alpine" + + run_with_timeout 300 "$SMOLVM" pack create \ + --image alpine:latest \ + --output "$out_path" 2>&1 || { + rm -rf "$tmp_dir" + return 1 + } + + local run_out exit_code=0 + run_out=$(run_with_timeout 120 "$SMOLVM" pack run \ + --sidecar "$out_path.smolmachine" -- \ + ls /dev/dri 2>&1) || exit_code=$? + + rm -rf "$tmp_dir" + if [[ $exit_code -eq 0 ]] && echo "$run_out" | grep -qE "render|card"; then + echo "FAIL: /dev/dri present in non-GPU packed binary: $run_out" + return 1 + fi +} + +run_test "GPU: pack create --gpu embeds gpu=true in manifest" test_pack_create_gpu_manifest || true +run_test "GPU: pack create without --gpu has no /dev/dri (isolation)" test_pack_create_no_gpu_manifest || true + +# ============================================================================= +# Section 2: Vulkan workloads (Fedora 42 + patched Mesa) +# ============================================================================= +# Standard Fedora Mesa has a 16KB page-alignment bug that crashes Venus ICD +# initialisation on Apple Silicon (host pages are 16KB; guest expects 4KB). +# The slp/mesa-libkrun-vulkan COPR carries the upstream patch. +# +# We create one shared Fedora 42 machine and install Mesa once, then run all +# Vulkan assertions against the same running container to avoid paying the +# dnf install cost (~60s) for each individual test. +# +# Key commands: +# vulkaninfo --summary → lists ICDs, device names, API versions +# stat /dev/dri/renderD128 → confirms DRI forwarding inside Fedora container + +echo "" +echo "Running Vulkan workload tests (Fedora 42 + patched Mesa)..." + +test_fedora_gpu_setup() { + "$SMOLVM" machine stop --name "$GPU_FEDORA_MACHINE" 2>/dev/null || true + "$SMOLVM" machine delete --name "$GPU_FEDORA_MACHINE" -f 2>/dev/null || true + + echo " Creating Fedora 42 GPU machine (this pulls ~600 MB on first run)..." + "$SMOLVM" machine create --name "$GPU_FEDORA_MACHINE" \ + --image fedora:42 --gpu --net 2>&1 || return 1 + + echo " Starting machine..." + "$SMOLVM" machine start --name "$GPU_FEDORA_MACHINE" 2>&1 || { + "$SMOLVM" machine delete --name "$GPU_FEDORA_MACHINE" -f 2>/dev/null + return 1 + } + + echo " Enabling slp/mesa-libkrun-vulkan COPR (Apple Silicon 16KB page-alignment fix)..." + run_with_timeout 120 "$SMOLVM" machine exec --name "$GPU_FEDORA_MACHINE" -- \ + dnf copr enable -y slp/mesa-libkrun-vulkan 2>&1 || { + echo "FAIL: dnf copr enable failed" + return 1 + } + + echo " Installing mesa-vulkan-drivers + vulkan-tools (~60s)..." + run_with_timeout 300 "$SMOLVM" machine exec --name "$GPU_FEDORA_MACHINE" -- \ + dnf install -y --allowerasing mesa-vulkan-drivers vulkan-tools 2>&1 || { + echo "FAIL: dnf install failed" + return 1 + } + + echo " Fedora GPU machine ready." +} + +test_vulkaninfo_venus_icd() { + # Venus is the guest-side Vulkan-over-virtio-gpu driver in Mesa. + # It should appear as an ICD in vulkaninfo --summary output. + _fedora_running || { echo "SKIP: Fedora GPU machine not running (setup failed)"; return 1; } + local out + out=$("$SMOLVM" machine exec --name "$GPU_FEDORA_MACHINE" -- \ + vulkaninfo --summary 2>&1) || { echo "FAIL: vulkaninfo exited non-zero"; echo "$out"; return 1; } + echo "$out" | grep -qi "Venus" || { + echo "FAIL: 'Venus' not found in vulkaninfo --summary" + echo "$out" | head -30 | sed 's/^/ /' + return 1 + } +} + +test_vulkaninfo_virtio_gpu_device() { + # The device name seen by the guest reflects the libkrun virtio-gpu backend. + _fedora_running || { echo "SKIP: Fedora GPU machine not running (setup failed)"; return 1; } + local out + out=$("$SMOLVM" machine exec --name "$GPU_FEDORA_MACHINE" -- \ + vulkaninfo --summary 2>&1) || { echo "FAIL: vulkaninfo exited non-zero"; echo "$out"; return 1; } + echo "$out" | grep -qi "Virtio-GPU" || { + echo "FAIL: 'Virtio-GPU' not found in vulkaninfo --summary" + echo "$out" | head -30 | sed 's/^/ /' + return 1 + } +} + +test_vulkan_api_version() { + # Venus exposes Vulkan 1.2+ through the virtio-gpu transport. + _fedora_running || { echo "SKIP: Fedora GPU machine not running (setup failed)"; return 1; } + local out + out=$("$SMOLVM" machine exec --name "$GPU_FEDORA_MACHINE" -- \ + vulkaninfo --summary 2>&1) || { echo "FAIL: vulkaninfo exited non-zero"; echo "$out"; return 1; } + echo "$out" | grep -qiE "apiVersion|Vulkan [0-9]+\." || { + echo "FAIL: No Vulkan API version in vulkaninfo --summary" + echo "$out" | head -30 | sed 's/^/ /' + return 1 + } +} + +test_render_node_in_fedora_container() { + # Confirms that add_gpu_devices_if_available() forwards /dev/dri into the + # Fedora OCI container, not just Alpine. Different base images, same result. + _fedora_running || { echo "SKIP: Fedora GPU machine not running (setup failed)"; return 1; } + local out rc=0 + out=$("$SMOLVM" machine exec --name "$GPU_FEDORA_MACHINE" -- \ + stat /dev/dri/renderD128 2>&1) || rc=$? + [[ $rc -eq 0 ]] && [[ "$out" == *"renderD128"* ]] || { + echo "FAIL: /dev/dri/renderD128 inaccessible in Fedora container (exit $rc, got: $out)" + return 1 + } +} + +test_fedora_gpu_cleanup() { + "$SMOLVM" machine stop --name "$GPU_FEDORA_MACHINE" 2>&1 || true + "$SMOLVM" machine delete --name "$GPU_FEDORA_MACHINE" -f 2>/dev/null || true +} + +run_test "GPU: Fedora 42 + patched Mesa setup" test_fedora_gpu_setup || true +run_test "GPU: vulkaninfo reports Venus ICD" test_vulkaninfo_venus_icd || true +run_test "GPU: vulkaninfo reports Virtio-GPU device name" test_vulkaninfo_virtio_gpu_device || true +run_test "GPU: Vulkan API version reported (1.x)" test_vulkan_api_version || true +run_test "GPU: /dev/dri/renderD128 accessible in Fedora container" test_render_node_in_fedora_container || true +run_test "GPU: Fedora cleanup" test_fedora_gpu_cleanup || true + +print_summary "GPU Tests" diff --git a/tests/test_machine_bare.sh b/tests/test_machine_bare.sh new file mode 100755 index 0000000..6a19f61 --- /dev/null +++ b/tests/test_machine_bare.sh @@ -0,0 +1,842 @@ +#!/usr/bin/env bash +# +# Bare VM Tests (lifecycle, exec, shell, file I/O, observability) +# +# Part of the smolvm test suite. Run with: ./tests/test_machine_bare.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Bare VM Tests (lifecycle, exec, shell, file I/O, observability)" +echo "==========================================" +echo "" + +test_machine_start() { + cleanup_machine + $SMOLVM machine start 2>&1 +} + +test_machine_stop() { + ensure_machine_running + $SMOLVM machine stop 2>&1 +} + +test_machine_status_running() { + ensure_machine_running + local status + status=$($SMOLVM machine status 2>&1) + [[ "$status" == *"running"* ]] +} + +test_machine_status_stopped() { + cleanup_machine + local status exit_code=0 + status=$($SMOLVM machine status 2>&1) || exit_code=$? + # When stopped, status command either: + # - Returns non-zero exit code, OR + # - Returns status containing "not running" or "stopped" + [[ $exit_code -ne 0 ]] || [[ "$status" == *"not running"* ]] || [[ "$status" == *"stopped"* ]] +} + +test_machine_start_stop_cycle() { + cleanup_machine + + # Start + $SMOLVM machine start 2>&1 || return 1 + + # Verify running + local status exit_code=0 + status=$($SMOLVM machine status 2>&1) || exit_code=$? + if [[ $exit_code -ne 0 ]] || [[ "$status" != *"running"* ]]; then + return 1 + fi + + # Stop + $SMOLVM machine stop 2>&1 || return 1 + + # Verify stopped - either non-zero exit or status message indicates stopped + exit_code=0 + status=$($SMOLVM machine status 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || [[ "$status" == *"not running"* ]] || [[ "$status" == *"stopped"* ]] +} + +test_machine_exec() { + ensure_machine_running + local output + output=$($SMOLVM machine exec -- cat /etc/os-release 2>&1) + [[ "$output" == *"Alpine"* ]] +} + +test_machine_exec_echo() { + ensure_machine_running + local output + output=$($SMOLVM machine exec -- echo "test-marker-xyz" 2>&1) + [[ "$output" == *"test-marker-xyz"* ]] +} + +test_machine_exec_binary_output_preserved() { + ensure_machine_running + + # Fetch first 4 bytes of /bin/busybox — the ELF magic. + # Pipe through xxd to render as hex so we're comparing ASCII strings + # (bash can't easily compare binary blobs, but the agent→client→CLI + # path is what we're exercising; the xxd happens host-side after the + # bytes are already through the protocol). + local hex + hex=$($SMOLVM machine exec -- head -c 4 /bin/busybox 2>&1 | xxd -p | tr -d '\n') + + # ELF magic: 7f 45 4c 46 (.ELF) + # If the 0x7f byte was dropped/replaced, we'd see "454c46" or "efbfbd454c46". + [[ "$hex" == "7f454c46" ]] || { + echo "expected ELF magic '7f454c46', got '$hex' — binary output corrupted" + return 1 + } +} + +test_machine_exec_exit_code() { + ensure_machine_running + + # Test exit 0 + $SMOLVM machine exec -- sh -c "exit 0" 2>&1 || return 1 + + # Test exit 1 + local exit_code=0 + $SMOLVM machine exec -- sh -c "exit 1" 2>&1 || exit_code=$? + [[ $exit_code -eq 1 ]] +} + +test_machine_exec_failed_does_not_kill_vm() { + ensure_machine_running + + # Nonexistent binary — should fail but VM stays alive + local exit_code=0 + $SMOLVM machine exec -- /nonexistent_binary_xyz 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "expected failure for nonexistent binary"; return 1; } + + # VM must still be running + local status + status=$($SMOLVM machine status 2>&1) + [[ "$status" == *"running"* ]] || { echo "VM died after failed exec: $status"; return 1; } + + # Next exec must succeed + local output + output=$($SMOLVM machine exec -- echo "survived-failed-exec" 2>&1) + [[ "$output" == *"survived-failed-exec"* ]] || { echo "exec after failure returned: $output"; return 1; } + + # Empty string command — should fail but VM stays alive + exit_code=0 + $SMOLVM machine exec -- "" 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "expected failure for empty command"; return 1; } + + status=$($SMOLVM machine status 2>&1) + [[ "$status" == *"running"* ]] || { echo "VM died after empty command exec: $status"; return 1; } + + # Final verification + output=$($SMOLVM machine exec -- echo "still-alive" 2>&1) + [[ "$output" == *"still-alive"* ]] +} + +test_sigterm_during_exec_does_not_stall_vm() { + local name="bug12-sigterm-$$" + $SMOLVM machine stop --name "$name" 2>/dev/null || true + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true + $SMOLVM machine create --name "$name" 2>&1 | tail -1 || return 1 + $SMOLVM machine start --name "$name" 2>&1 | tail -1 || { + $SMOLVM machine delete --name "$name" -f 2>/dev/null; return 1 + } + wait_vm_ready --name "$name" || { + $SMOLVM machine delete --name "$name" -f 2>/dev/null; return 1 + } + + # Start a long-running exec, then SIGTERM the client mid-flight. + $SMOLVM machine exec --name "$name" -- sh -c 'sleep 30' & + local client_pid=$! + # Wait long enough for the exec to have registered with the agent (vsock round-trip). + sleep 1 + kill -TERM "$client_pid" 2>/dev/null + wait "$client_pid" 2>/dev/null + # Give the agent's 10ms poll loop a moment to detect the disconnect + sleep 1 + + # VM must still be reachable. Before the fix, state would be "unreachable" + # and the next exec would take ~30s. Time the exec to detect the stall. + local t_start t_end elapsed + t_start=$(python3 -c 'import time; print(time.time())' 2>/dev/null || date +%s) + local result + result=$($SMOLVM machine exec --name "$name" -- echo "survived" 2>&1) + t_end=$(python3 -c 'import time; print(time.time())' 2>/dev/null || date +%s) + elapsed=$(python3 -c "print(f'{$t_end - $t_start:.1f}')" 2>/dev/null || echo "?") + + echo " Next exec after SIGTERM: ${elapsed}s" + + [[ "$result" == *"survived"* ]] || { + echo "FAIL: exec after SIGTERM failed: $result" + $SMOLVM machine stop --name "$name" 2>/dev/null + $SMOLVM machine delete --name "$name" -f 2>/dev/null + return 1 + } + + # Must be fast — the old path took ~30s while the orphan sleep finished. + local over_threshold + over_threshold=$(python3 -c "print('yes' if $t_end - $t_start > 5 else 'no')" 2>/dev/null || echo "no") + if [[ "$over_threshold" == "yes" ]]; then + echo "FAIL: next exec took ${elapsed}s (>5s) — agent stalled on orphan child?" + $SMOLVM machine stop --name "$name" 2>/dev/null + $SMOLVM machine delete --name "$name" -f 2>/dev/null + return 1 + fi + + $SMOLVM machine stop --name "$name" 2>/dev/null || true + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true +} + +test_exec_timeout_does_not_stall_vm() { + local name="bug20-timeout-$$" + $SMOLVM machine stop --name "$name" 2>/dev/null || true + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true + $SMOLVM machine create --name "$name" 2>&1 | tail -1 || return 1 + $SMOLVM machine start --name "$name" 2>&1 | tail -1 || { + $SMOLVM machine delete --name "$name" -f 2>/dev/null; return 1 + } + wait_vm_ready --name "$name" || { + $SMOLVM machine delete --name "$name" -f 2>/dev/null; return 1 + } + + # Short timeout, long-running command with sub-processes — timeout should + # kill the whole tree without stalling the agent. + local result + result=$($SMOLVM machine exec --name "$name" --timeout 2s -- sh -c 'sleep 30' 2>&1) + # Exit code 124 = timeout, expected behavior (don't assert — just note) + + # Next exec must be fast. If the agent stalled, this will take ~30s. + local t_start t_end elapsed + t_start=$(python3 -c 'import time; print(time.time())' 2>/dev/null || date +%s) + result=$($SMOLVM machine exec --name "$name" -- echo "alive" 2>&1) + t_end=$(python3 -c 'import time; print(time.time())' 2>/dev/null || date +%s) + elapsed=$(python3 -c "print(f'{$t_end - $t_start:.1f}')" 2>/dev/null || echo "?") + + echo " Next exec after timeout: ${elapsed}s" + + [[ "$result" == *"alive"* ]] || { + echo "FAIL: exec after timeout failed: $result" + $SMOLVM machine stop --name "$name" 2>/dev/null + $SMOLVM machine delete --name "$name" -f 2>/dev/null + return 1 + } + + local over_threshold + over_threshold=$(python3 -c "print('yes' if $t_end - $t_start > 5 else 'no')" 2>/dev/null || echo "no") + if [[ "$over_threshold" == "yes" ]]; then + echo "FAIL: next exec took ${elapsed}s (>5s) — agent stalled after timeout?" + $SMOLVM machine stop --name "$name" 2>/dev/null + $SMOLVM machine delete --name "$name" -f 2>/dev/null + return 1 + fi + + $SMOLVM machine stop --name "$name" 2>/dev/null || true + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true +} + +test_machine_named_vm() { + local vm_name="test-vm-named" + + # Clean up any existing + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create the named VM first + $SMOLVM machine create --name "$vm_name" 2>&1 || return 1 + + # Start + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Check status + local status + status=$($SMOLVM machine status --name "$vm_name" 2>&1) + if [[ "$status" != *"running"* ]]; then + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Stop and delete + $SMOLVM machine stop --name "$vm_name" 2>&1 + $SMOLVM machine delete --name "$vm_name" -f 2>&1 + ensure_data_dir_deleted "$vm_name" +} + +test_machine_create_prints_named_start_hint() { + local vm_name="create-hint-test-$$" + local output + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + output=$($SMOLVM machine create --name "$vm_name" 2>&1) || return 1 + + [[ "$output" == *"Use 'smolvm machine start --name $vm_name' to start the machine"* ]] || { + echo "$output" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + $SMOLVM machine delete --name "$vm_name" -f 2>&1 + ensure_data_dir_deleted "$vm_name" +} + +test_machine_exec_when_stopped() { + cleanup_machine + + local exit_code=0 + $SMOLVM machine exec -- echo "should-fail" 2>&1 || exit_code=$? + + # Should fail with non-zero exit code (don't check specific message) + [[ $exit_code -ne 0 ]] +} + +test_bare_vm_workspace() { + ensure_machine_running + local output + output=$($SMOLVM machine exec -- ls -d /workspace 2>&1) + [[ "$output" == *"/workspace"* ]] || { echo "FAIL: /workspace missing on bare VM"; return 1; } + + # Write and read back + $SMOLVM machine exec -- sh -c 'echo ws-bare > /workspace/bare.txt' 2>&1 || return 1 + output=$($SMOLVM machine exec -- cat /workspace/bare.txt 2>&1) + [[ "$output" == *"ws-bare"* ]] +} + +test_file_upload_download() { + local vm_name="cp-test-$$" + + # Create and start a machine + $SMOLVM machine create --name "$vm_name" 2>&1 || return 1 + run_with_timeout 30 $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + # Upload a file + local upload_content="hello from host $(date +%s)" + echo "$upload_content" > /tmp/smolvm-cp-test.txt + $SMOLVM machine cp /tmp/smolvm-cp-test.txt "$vm_name":/tmp/uploaded.txt 2>&1 || { + echo "Upload failed" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f /tmp/smolvm-cp-test.txt + return 1 + } + + # Verify upload via exec + local exec_result + exec_result=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/uploaded.txt 2>&1) || { + echo "Exec after upload failed" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f /tmp/smolvm-cp-test.txt + return 1 + } + [[ "$exec_result" == *"$upload_content"* ]] || { + echo "Upload content mismatch: expected '$upload_content', got '$exec_result'" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f /tmp/smolvm-cp-test.txt + return 1 + } + + # Create file in VM and download + $SMOLVM machine exec --name "$vm_name" -- sh -c "echo 'hello from VM' > /tmp/to-download.txt" 2>&1 + $SMOLVM machine cp "$vm_name":/tmp/to-download.txt /tmp/smolvm-downloaded.txt 2>&1 || { + echo "Download failed" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f /tmp/smolvm-cp-test.txt /tmp/smolvm-downloaded.txt + return 1 + } + local downloaded + downloaded=$(cat /tmp/smolvm-downloaded.txt) + [[ "$downloaded" == *"hello from VM"* ]] || { + echo "Download content mismatch: '$downloaded'" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f /tmp/smolvm-cp-test.txt /tmp/smolvm-downloaded.txt + return 1 + } + + # Cleanup + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f /tmp/smolvm-cp-test.txt /tmp/smolvm-downloaded.txt +} + +test_streaming_exec() { + # Start a machine, run a command with --stream, verify output arrives + $SMOLVM machine stop 2>/dev/null || true + + $SMOLVM machine create --name stream-test-$$ 2>&1 || return 1 + run_with_timeout 30 $SMOLVM machine start --name stream-test-$$ 2>&1 || { + $SMOLVM machine delete --name stream-test-$$ -f 2>/dev/null; return 1 + } + + # Streaming exec — output should contain the echoed text + local result + result=$(run_with_timeout 15 $SMOLVM machine exec --stream --name stream-test-$$ -- sh -c "echo 'stream-line-1' && echo 'stream-line-2' && echo 'done'" 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT"; $SMOLVM machine stop --name stream-test-$$ 2>/dev/null; $SMOLVM machine delete --name stream-test-$$ -f 2>/dev/null; return 1; } + + [[ "$result" == *"stream-line-1"* ]] && [[ "$result" == *"stream-line-2"* ]] && [[ "$result" == *"done"* ]] || { + echo "Missing streaming output: $result" + $SMOLVM machine stop --name stream-test-$$ 2>/dev/null + $SMOLVM machine delete --name stream-test-$$ -f 2>/dev/null + return 1 + } + + # Cleanup + $SMOLVM machine stop --name stream-test-$$ 2>/dev/null + $SMOLVM machine delete --name stream-test-$$ -f 2>/dev/null +} + +test_agent_json_logs() { + local vm_name="observability-test-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + $SMOLVM machine create --name "$vm_name" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Run a command to generate agent log entries + $SMOLVM machine exec --name "$vm_name" -- echo "observability-test" 2>&1 || true + + # Find the console log using the platform-aware vm_data_dir helper + local data_dir + data_dir=$(vm_data_dir "$vm_name") + local console_log="${data_dir}/agent-console.log" + + # Copy the log before stopping (stop/delete may remove the data dir) + local saved_log + saved_log=$(mktemp) + cp "$console_log" "$saved_log" 2>/dev/null || true + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + if [[ ! -s "$saved_log" ]]; then + echo "Console log not found or empty at $console_log" + rm -f "$saved_log" + return 1 + fi + + # Agent should write JSON — verify at least one line parses as JSON + local json_lines + json_lines=$(grep -c '^{' "$saved_log" 2>/dev/null || echo "0") + if [[ "$json_lines" -eq 0 ]]; then + echo "No JSON lines in console log ($console_log)" + rm -f "$saved_log" + return 1 + fi + + # Verify a tracing-formatted JSON line has expected structured fields. + # Skip early boot_log lines (target=smolvm_agent::boot) which use a + # simpler format without timestamps — they run before tracing is initialized. + local first_json + first_json=$(grep '^{' "$saved_log" | grep '"timestamp"' | head -1) + rm -f "$saved_log" + if [[ -z "$first_json" ]]; then + echo "No tracing JSON lines with timestamp found in console log" + return 1 + fi + echo "$first_json" | python3 -c " +import sys, json +line = json.load(sys.stdin) +assert 'timestamp' in line, 'missing timestamp' +assert 'level' in line, 'missing level' +" 2>&1 || { echo "JSON log missing structured fields: $first_json"; return 1; } +} + +test_machine_shell() { + ensure_machine_running + + # shell opens an interactive PTY. Pipe "echo X; exit" through it. + # Use a subshell with a watchdog kill to avoid hanging if the PTY blocks. + local tmpout + tmpout=$(mktemp) + (echo "echo shell-test-ok; exit" | $SMOLVM machine shell > "$tmpout" 2>&1) & + local pid=$! + sleep 5 + kill $pid 2>/dev/null; wait $pid 2>/dev/null + + local output + output=$(cat "$tmpout") + rm -f "$tmpout" + + [[ "$output" == *"shell-test-ok"* ]] || { + echo "FAIL: expected shell-test-ok, got: $output" + return 1 + } +} + +test_exec_large_stdout_does_not_crash_vm() { + ensure_machine_running "true" + + # Generate 128KB of output — well above the ~64KB pipe buffer + local output + output=$(run_with_timeout 30 $SMOLVM machine exec -- sh -c 'dd if=/dev/urandom bs=1024 count=128 2>/dev/null | base64' 2>&1) + local exit_code=$? + + [[ $exit_code -eq 124 ]] && { echo "FAIL: timed out (pipe deadlock?)"; return 1; } + + local output_size=${#output} + [[ $output_size -gt 100000 ]] || { + echo "FAIL: expected >100KB output, got ${output_size} bytes" + return 1 + } + + # VM must still be responsive after large output + local check + check=$(run_with_timeout 10 $SMOLVM machine exec -- echo "still-alive" 2>&1) || { + echo "FAIL: VM unreachable after large stdout" + return 1 + } + echo "$check" | grep -q "still-alive" || { + echo "FAIL: expected 'still-alive', got: $check" + return 1 + } +} + +_docker_in_vm_start_dockerd() { + local vm_name="$1" + $SMOLVM machine exec --name "$vm_name" -- sh -c ' + mkdir -p /storage/docker /var/lib/docker + mount --bind /storage/docker /var/lib/docker + rm -f /var/run/docker.pid + dockerd --storage-driver=overlay2 >/tmp/dockerd.log 2>&1 & + for i in $(seq 1 40); do + docker info >/dev/null 2>&1 && echo "dockerd-ready" && exit 0 + sleep 1 + done + echo "FAIL: dockerd did not become ready" + tail -5 /tmp/dockerd.log + exit 1 + ' 2>&1 +} + +test_docker_in_vm() { + skip_if_slow && return 0 + local vm_name="docker-in-vm-$$" + local smolfile="$PROJECT_ROOT/examples/docker-in-vm/docker.smolfile" + + echo "phase: pre-cleanup" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # ── Create ──────────────────────────────────────────────────────────────── + echo "phase: create" + local output + output=$($SMOLVM machine create --name "$vm_name" \ + -s "$smolfile" --net-backend virtio-net 2>&1) || { + echo "FAIL: machine create --name failed" + echo "$output" + return 1 + } + [[ "$output" == *"Init commands: 4"* ]] || { + echo "FAIL: expected 4 init commands, got: $output" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + # ── First start (runs init: apk add docker + bind-mount) ───────────────── + echo "phase: first-start (apk add docker — expect 30-90s)" + output=$($SMOLVM machine start --name "$vm_name" 2>&1) || { + echo "FAIL: first machine start failed" + echo "$output" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + [[ "$output" == *"Running 4 init command"* ]] || { + echo "FAIL: expected 4 init commands to run on first start" + echo "$output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + # ── Start dockerd ───────────────────────────────────────────────────────── + echo "phase: start-dockerd" + output=$(_docker_in_vm_start_dockerd "$vm_name") || { + echo "FAIL: dockerd failed to start" + echo "$output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + [[ "$output" == *"dockerd-ready"* ]] || { + echo "FAIL: dockerd did not report ready" + echo "$output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + # ── Assert: overlay2 storage driver on ext4 ─────────────────────────────── + echo "phase: assert-overlay2-on-ext4" + output=$($SMOLVM machine exec --name "$vm_name" -- sh -c ' + driver=$(docker info 2>/dev/null | grep "Storage Driver:" | awk "{print \$3}") + [ "$driver" = "overlay2" ] || { echo "FAIL: storage driver=$driver"; exit 1; } + echo "storage-driver-ok" + mount | grep -q "/dev/vda on /var/lib/docker type ext4" || { + echo "FAIL: /var/lib/docker not on ext4" + mount | grep "var/lib/docker" || true + exit 1 + } + echo "bind-mount-ok" + ' 2>&1) || { + echo "FAIL: storage driver or bind-mount check failed" + echo "$output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + [[ "$output" == *"storage-driver-ok"* ]] || { echo "FAIL: $output"; $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true; return 1; } + [[ "$output" == *"bind-mount-ok"* ]] || { echo "FAIL: $output"; $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true; return 1; } + + # ── Assert: docker run executes a container ─────────────────────────────── + echo "phase: docker-run" + output=$($SMOLVM machine exec --name "$vm_name" -- \ + docker run --rm alpine echo "docker-in-vm-ok" 2>&1) || { + echo "FAIL: docker run failed" + echo "$output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + [[ "$output" == *"docker-in-vm-ok"* ]] || { + echo "FAIL: expected docker-in-vm-ok, got: $output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + # ── Assert: bridge networking reaches the internet ──────────────────────── + echo "phase: docker-bridge-networking" + output=$($SMOLVM machine exec --name "$vm_name" -- \ + docker run --rm alpine wget -qO- https://httpbin.org/get 2>&1) || { + echo "FAIL: docker bridge networking failed" + echo "$output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + [[ "$output" == *'"url"'* ]] || { + echo "FAIL: expected JSON response with url field, got: $output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + # ── Assert: docker build produces a runnable image ──────────────────────── + echo "phase: docker-build" + output=$($SMOLVM machine exec --name "$vm_name" -- sh -c ' + printf "FROM alpine\nRUN echo build-layer-ok" > /tmp/Dockerfile + docker build -t smolvm-test-build /tmp 2>&1 | tail -3 + docker run --rm smolvm-test-build echo "built-image-run-ok" + ' 2>&1) || { + echo "FAIL: docker build or run of built image failed" + echo "$output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + [[ "$output" == *"built-image-run-ok"* ]] || { + echo "FAIL: expected built-image-run-ok, got: $output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + # ── Stop → restart cycle ────────────────────────────────────────────────── + echo "phase: stop" + $SMOLVM machine stop --name "$vm_name" 2>&1 || { + echo "FAIL: machine stop failed" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + echo "phase: restart" + output=$($SMOLVM machine start --name "$vm_name" 2>&1) || { + echo "FAIL: machine restart failed" + echo "$output" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + [[ "$output" == *"Init already completed"* ]] || { + echo "FAIL: init should be skipped on restart, got: $output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + # ── Re-start dockerd after restart ──────────────────────────────────────── + echo "phase: start-dockerd-after-restart" + output=$(_docker_in_vm_start_dockerd "$vm_name") || { + echo "FAIL: dockerd failed to start after VM restart" + echo "$output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + [[ "$output" == *"dockerd-ready"* ]] || { + echo "FAIL: dockerd not ready after VM restart" + echo "$output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + # ── Assert: images persist across restart ───────────────────────────────── + echo "phase: assert-persistence" + output=$($SMOLVM machine exec --name "$vm_name" -- sh -c ' + docker images | grep -q "^alpine " || { echo "FAIL: alpine image missing after restart"; docker images; exit 1; } + echo "alpine-persists" + docker images | grep -q "smolvm-test-build" || { echo "FAIL: built image missing after restart"; docker images; exit 1; } + echo "built-image-persists" + docker run --rm alpine echo "post-restart-run-ok" + ' 2>&1) || { + echo "FAIL: post-restart image persistence check failed" + echo "$output" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + [[ "$output" == *"alpine-persists"* ]] || { echo "FAIL: $output"; $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true; return 1; } + [[ "$output" == *"built-image-persists"* ]] || { echo "FAIL: $output"; $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true; return 1; } + [[ "$output" == *"post-restart-run-ok"* ]] || { echo "FAIL: $output"; $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true; return 1; } + + # ── Cleanup ─────────────────────────────────────────────────────────────── + echo "phase: cleanup" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true +} + + +run_test "Machine start" test_machine_start || true +run_test "Machine stop" test_machine_stop || true +run_test "Machine status (running)" test_machine_status_running || true +run_test "Machine status (stopped)" test_machine_status_stopped || true +run_test "Machine start/stop cycle" test_machine_start_stop_cycle || true +run_test "Machine exec" test_machine_exec || true +run_test "Machine exec: echo" test_machine_exec_echo || true +run_test "Machine exec: binary output preserved (BUG-23)" test_machine_exec_binary_output_preserved || true +run_test "Machine exec exit code" test_machine_exec_exit_code || true +run_test "Failed exec does not kill VM" test_machine_exec_failed_does_not_kill_vm || true +run_test "SIGTERM during exec does not stall VM" test_sigterm_during_exec_does_not_stall_vm || true +run_test "Exec timeout does not stall VM" test_exec_timeout_does_not_stall_vm || true +run_test "Named machine" test_machine_named_vm || true +run_test "Create prints named start hint" test_machine_create_prints_named_start_hint || true +run_test "Exec when stopped fails" test_machine_exec_when_stopped || true +run_test "Bare VM: /workspace exists" test_bare_vm_workspace || true +run_test "File upload and download" test_file_upload_download || true +run_test "Streaming exec" test_streaming_exec || true +run_test "Agent: structured JSON logs" test_agent_json_logs || true +run_test "Shell: machine shell opens interactive shell" test_machine_shell || true +run_test "Exec: large stdout does not crash VM (bare)" test_exec_large_stdout_does_not_crash_vm || true +run_test "Docker-in-VM: overlay2 + bridge networking + build + restart persistence" test_docker_in_vm || true + +# ============================================================================= +# Exec stdin null — stdin-blocking commands exit cleanly +# ============================================================================= + +_EXEC_STDIN_MACHINE="exec-stdin-$$" + +test_exec_cat_no_interactive() { + "$SMOLVM" machine stop --name "$_EXEC_STDIN_MACHINE" 2>/dev/null || true + "$SMOLVM" machine delete --name "$_EXEC_STDIN_MACHINE" -f 2>/dev/null || true + "$SMOLVM" machine create --name "$_EXEC_STDIN_MACHINE" 2>/dev/null || return 1 + "$SMOLVM" machine start --name "$_EXEC_STDIN_MACHINE" 2>/dev/null || return 1 + + local out exit_code=0 + out=$(run_with_timeout 10 "$SMOLVM" machine exec --name "$_EXEC_STDIN_MACHINE" -- cat 2>&1) \ + || exit_code=$? + + "$SMOLVM" machine stop --name "$_EXEC_STDIN_MACHINE" 2>/dev/null || true + "$SMOLVM" machine delete --name "$_EXEC_STDIN_MACHINE" -f 2>/dev/null || true + + if echo "$out" | grep -qi "connection closed"; then + echo "FAIL: got 'connection closed'" + return 1 + fi + [[ $exit_code -eq 0 ]] || { echo "FAIL: exit $exit_code (expected 0)"; return 1; } +} + +run_test "Exec: 'exec -- cat' exits cleanly with null stdin" test_exec_cat_no_interactive || true + +test_exec_tty_piped_stdin_terminates() { + # `--tty` runs the child on a PTY. A PTY cannot have one direction + # closed, so when the feeding pipe (`echo`) closes, end-of-input must + # still reach the child. Without EOF propagation a stdin reader (cat) + # never terminates and the exec session hangs. + "$SMOLVM" machine stop --name "$_EXEC_STDIN_MACHINE" 2>/dev/null || true + "$SMOLVM" machine delete --name "$_EXEC_STDIN_MACHINE" -f 2>/dev/null || true + "$SMOLVM" machine create --name "$_EXEC_STDIN_MACHINE" 2>/dev/null || return 1 + "$SMOLVM" machine start --name "$_EXEC_STDIN_MACHINE" 2>/dev/null || return 1 + + # Newline-terminated input: the PTY line buffer is empty when EOF + # arrives, so a single VEOF would already yield the zero-length read. + local exit_code=0 + run_with_timeout 15 sh -c \ + "echo tty-input | '$SMOLVM' machine exec --name '$_EXEC_STDIN_MACHINE' --tty -i -- cat" \ + >/dev/null 2>&1 || exit_code=$? + + # Unterminated input (no trailing newline): the first VEOF only flushes + # the partial line as data, so a second VEOF is required to deliver the + # zero-length read. This is the case a single VEOF fails to terminate. + local exit_code_partial=0 + run_with_timeout 15 sh -c \ + "printf no-newline | '$SMOLVM' machine exec --name '$_EXEC_STDIN_MACHINE' --tty -i -- cat" \ + >/dev/null 2>&1 || exit_code_partial=$? + + "$SMOLVM" machine stop --name "$_EXEC_STDIN_MACHINE" 2>/dev/null || true + "$SMOLVM" machine delete --name "$_EXEC_STDIN_MACHINE" -f 2>/dev/null || true + + [[ $exit_code -ne 124 ]] || { echo "FAIL: 'exec --tty -i' with piped stdin timed out (PTY EOF not propagated)"; return 1; } + [[ $exit_code_partial -ne 124 ]] || { echo "FAIL: 'exec --tty -i' with unterminated stdin timed out (PTY EOF not propagated)"; return 1; } +} + +run_test "Exec: 'exec --tty -i' with piped stdin terminates (PTY EOF)" test_exec_tty_piped_stdin_terminates || true + +# ============================================================================= +# Named machine survives observer Drop +# ============================================================================= + +_DROP_MACHINE="drop-safe-$$" + +test_machine_survives_rapid_exec() { + "$SMOLVM" machine stop --name "$_DROP_MACHINE" 2>/dev/null || true + "$SMOLVM" machine delete --name "$_DROP_MACHINE" -f 2>/dev/null || true + "$SMOLVM" machine create --name "$_DROP_MACHINE" 2>/dev/null || return 1 + "$SMOLVM" machine start --name "$_DROP_MACHINE" 2>/dev/null || return 1 + + local i + for i in 1 2 3 4 5; do + local out exit_code=0 + out=$(run_with_timeout 15 "$SMOLVM" machine exec --name "$_DROP_MACHINE" -- echo "alive-$i" 2>/dev/null) \ + || exit_code=$? + [[ $exit_code -eq 0 ]] || { + "$SMOLVM" machine delete --name "$_DROP_MACHINE" -f 2>/dev/null || true + echo "FAIL: exec #$i failed (exit $exit_code)"; return 1 + } + done + + "$SMOLVM" machine stop --name "$_DROP_MACHINE" 2>/dev/null || true + "$SMOLVM" machine delete --name "$_DROP_MACHINE" -f 2>/dev/null || true +} + +run_test "Drop-safety: machine survives 5 rapid execs" test_machine_survives_rapid_exec || true + +print_summary "Bare VM Tests" diff --git a/tests/test_machine_image.sh b/tests/test_machine_image.sh new file mode 100755 index 0000000..df3bec3 --- /dev/null +++ b/tests/test_machine_image.sh @@ -0,0 +1,393 @@ +#!/usr/bin/env bash +# +# Image-Backed Machine Tests +# +# Part of the smolvm test suite. Run with: ./tests/test_machine_image.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Image-Backed Machine Tests" +echo "==========================================" +echo "" + +test_create_with_image() { + local vm_name="create-image-test-$$" + + # Create with --image (new feature), start, exec, verify, cleanup + $SMOLVM machine create --name "$vm_name" --image alpine:latest --net 2>&1 || return 1 + + # Should appear in list (use --json for full names) + $SMOLVM machine ls --json 2>&1 | grep -q "$vm_name" || { + echo "Machine not in list" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + + # Start — should auto-pull the image + local start_result + start_result=$(run_with_timeout 60 $SMOLVM machine start --name "$vm_name" 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT on start"; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + [[ "$start_result" == *"Pulling"* ]] || [[ "$start_result" == *"Started"* ]] || [[ "$start_result" == *"already running"* ]] || { + echo "Start failed: $start_result" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + + # Exec — verify we're in the right image + local exec_result + exec_result=$(run_with_timeout 30 $SMOLVM machine exec --name "$vm_name" -- cat /etc/os-release 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT on exec"; $SMOLVM machine stop --name "$vm_name" 2>/dev/null; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + [[ "$exec_result" == *"Alpine"* ]] || { + echo "Not running Alpine: $exec_result" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + + # Cleanup + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null +} + +test_create_with_image_and_env() { + local vm_name="create-env-test-$$" + + # Create with --image + env + workdir + $SMOLVM machine create --name "$vm_name" --image alpine:latest --net \ + -e TEST_VAR=from_create -w /tmp 2>&1 || return 1 + + run_with_timeout 60 $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + + # Verify workdir was persisted (init commands run in /tmp) + local pwd_result + pwd_result=$(run_with_timeout 30 $SMOLVM machine exec --name "$vm_name" -- pwd 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT"; $SMOLVM machine stop --name "$vm_name" 2>/dev/null; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Cleanup + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null +} + +test_update_settings_applied_on_start() { + # Verify update changes cpus, ports, network, and that they take effect. + # Also verifies update refuses a running VM. + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine create --name default 2>&1 || return 1 + + # Update multiple settings at once + local output + output=$($SMOLVM machine update --name default --cpus 2 --mem 1024 -p 9090:9090 --net 2>&1) || { + echo "update failed: $output"; return 1 + } + [[ "$output" == *"cpus"* ]] || { echo "expected cpus in output: $output"; return 1; } + + # Start and verify cpus applied + $SMOLVM machine start 2>&1 || return 1 + local cpus + cpus=$($SMOLVM machine exec -- nproc 2>&1) + [[ "$cpus" == "2" ]] || { echo "expected 2 cpus, got: $cpus"; return 1; } + + # Update on running VM should fail + local exit_code=0 + $SMOLVM machine update --name default --mem 2048 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "update should fail on running VM"; return 1; } + + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true +} + +test_update_env_applied_on_start() { + # Env vars from the DB record are applied in image-based exec. + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine create --net --image alpine --name default 2>&1 || return 1 + + $SMOLVM machine update --name default -e MY_VAR=hello 2>&1 || return 1 + + $SMOLVM machine start 2>&1 || return 1 + local val + val=$($SMOLVM machine exec -- sh -c 'echo $MY_VAR' 2>&1) + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + + [[ "$val" == "hello" ]] || { echo "expected MY_VAR=hello, got: $val"; return 1; } +} + +test_exec_image_large_stdout_does_not_crash_vm() { + # Same test but for image-backed exec (the actual bug path) + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine create --net --image alpine --name default 2>&1 || return 1 + $SMOLVM machine start 2>&1 || return 1 + + local output + output=$(run_with_timeout 30 $SMOLVM machine exec -- sh -c 'dd if=/dev/urandom bs=1024 count=128 2>/dev/null | base64' 2>&1) + local exit_code=$? + + [[ $exit_code -eq 124 ]] && { echo "FAIL: timed out (pipe deadlock?)"; return 1; } + + local output_size=${#output} + [[ $output_size -gt 100000 ]] || { + echo "FAIL: expected >100KB output, got ${output_size} bytes" + return 1 + } + + # VM must still respond + local check + check=$(run_with_timeout 10 $SMOLVM machine exec -- echo "still-alive" 2>&1) || { + echo "FAIL: VM unreachable after large stdout (image-backed exec)" + return 1 + } + echo "$check" | grep -q "still-alive" || { + echo "FAIL: expected 'still-alive', got: $check" + return 1 + } + + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true +} + +test_exec_joined_large_stdout_does_not_crash_vm() { + # Same test but through the joined crun exec path (detached main container) + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine run -d --net --image alpine -- sleep 300 2>&1 || return 1 + + local output + output=$(run_with_timeout 30 $SMOLVM machine exec -- sh -c 'dd if=/dev/urandom bs=1024 count=128 2>/dev/null | base64' 2>&1) + local exit_code=$? + + [[ $exit_code -eq 124 ]] && { echo "FAIL: timed out (pipe deadlock in joined exec?)"; return 1; } + + local output_size=${#output} + [[ $output_size -gt 100000 ]] || { + echo "FAIL: expected >100KB output, got ${output_size} bytes" + return 1 + } + + # VM and main container must still be responsive + local check + check=$(run_with_timeout 10 $SMOLVM machine exec -- echo "still-alive" 2>&1) || { + echo "FAIL: VM unreachable after large stdout via joined exec" + return 1 + } + echo "$check" | grep -q "still-alive" || { + echo "FAIL: expected 'still-alive', got: $check" + return 1 + } + + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true +} + +test_exec_joins_main_container() { + # machine run -d creates a fresh VM, so no need for ensure_machine_running + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + + log_info "Starting detached workload: sleep 300..." + local run_output + if ! run_output=$(run_with_timeout 120 $SMOLVM machine run -d --net --image alpine -- sleep 300 2>&1); then + echo "FAIL: machine run -d failed: $run_output" + return 1 + fi + + # exec should join the running container — PID 1 is sleep 300 + local ps_output + if ! ps_output=$(run_with_timeout 30 $SMOLVM machine exec -- ps -ef 2>&1); then + echo "FAIL: machine exec failed: $ps_output" + return 1 + fi + + echo "$ps_output" | grep -q "sleep" || { + echo "FAIL: expected 'sleep' in ps output (shared PID namespace), got:" + echo "$ps_output" + return 1 + } +} + +test_repeated_exec_joins_same_container() { + # Relies on the detached container from the previous test still running + local out1 out2 + out1=$(run_with_timeout 30 $SMOLVM machine exec -- ps -ef 2>&1) || { + echo "FAIL: first repeated exec failed"; return 1 + } + out2=$(run_with_timeout 30 $SMOLVM machine exec -- ps -ef 2>&1) || { + echo "FAIL: second repeated exec failed"; return 1 + } + + for out in "$out1" "$out2"; do + echo "$out" | grep -q "sleep" || { + echo "FAIL: exec did not see 'sleep' in ps output" + echo "$out" + return 1 + } + done +} + +test_background_process_visible_across_execs() { + # Spawn sleep 90 in the background; it should be visible from the next exec + run_with_timeout 15 $SMOLVM machine exec -- sh -c 'sleep 90 &' 2>/dev/null || true + sleep 1 + + local ps_output + ps_output=$(run_with_timeout 30 $SMOLVM machine exec -- ps -ef 2>&1) || { + echo "FAIL: exec after background spawn failed"; return 1 + } + + local sleep_count + sleep_count=$(echo "$ps_output" | grep -c "sleep" || true) + [[ "$sleep_count" -ge 2 ]] || { + echo "FAIL: expected >=2 sleep processes, got $sleep_count" + echo "$ps_output" + return 1 + } +} + +test_ephemeral_run_is_isolated() { + # Ephemeral machine run (no -d) must NOT see the main container's processes + local ps_output + if ! ps_output=$(run_with_timeout 60 $SMOLVM machine run --net --image alpine -- ps -ef 2>&1); then + echo "FAIL: ephemeral machine run failed: $ps_output" + return 1 + fi + + if echo "$ps_output" | grep -q "sleep 300"; then + echo "FAIL: ephemeral run leaked into main container namespace" + echo "$ps_output" + return 1 + fi + + [[ -n "$ps_output" ]] || { echo "FAIL: ps returned empty output"; return 1; } +} + +test_exec_recovers_after_main_container_exits() { + # Start a detached container with a short-lived command + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine run -d --net --image alpine -- sleep 2 2>&1 || return 1 + + # Wait for the main container to exit naturally + sleep 4 + + # Exec should detect the stale container ID and start a fresh container + local output + if ! output=$(run_with_timeout 30 $SMOLVM machine exec -- echo "recovered" 2>&1); then + echo "FAIL: exec failed after main container exit: $output" + return 1 + fi + + echo "$output" | grep -q "recovered" || { + echo "FAIL: expected 'recovered', got: $output" + return 1 + } +} + +test_exec_join_timeout_does_not_kill_main_container() { + # A timed-out exec must not destroy the main workload container. + # The exec'd process may survive as an orphan reparented to PID 1 + # (Docker-compatible: docker exec timeout kills the exec wrapper, + # not the inner process or the container). + + # Ensure the detached container from earlier tests is still running + local ps_before + ps_before=$(run_with_timeout 10 $SMOLVM machine exec -- ps -ef 2>&1) || true + if ! echo "$ps_before" | grep -q "sleep"; then + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine run -d --net --image alpine -- sleep 300 2>&1 || return 1 + fi + + # Run a command with a short timeout — it will time out + local output exit_code=0 + output=$($SMOLVM machine exec --timeout 1 -- sleep 30 2>&1) || exit_code=$? + + # Exit code 124 = timeout (expected) + [[ $exit_code -eq 124 ]] || [[ "$output" == *"timed out"* ]] || { + echo "WARN: unexpected exit code $exit_code (expected 124 for timeout)" + } + + sleep 1 + + # The main container should still be running — sleep 300 visible + local ps_after + ps_after=$(run_with_timeout 10 $SMOLVM machine exec -- ps -ef 2>&1) || { + echo "FAIL: exec failed after timeout — main container may have been killed" + return 1 + } + + echo "$ps_after" | grep -q "sleep 300" || { + echo "FAIL: main container (sleep 300) not found after timed-out exec" + echo "ps output: $ps_after" + return 1 + } + + # Document whether the timed-out sleep 30 survives as an orphan. The main + # assertion above is that the main container remains alive. + if echo "$ps_after" | grep -q "sleep 30"; then + echo "WARN: timed-out 'sleep 30' still visible as orphan (Docker-compatible behavior)" + fi +} + +test_exec_join_documents_user_behavior() { + # Document current crun exec user behavior for images with a non-root USER. + # Some crun versions may run joined exec as root unless --user is explicit. + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine run -d --net --image nginxinc/nginx-unprivileged:stable-alpine -- sleep 300 2>&1 || { + echo "SKIP: could not start nginx-unprivileged image" + return 0 + } + + # Check what user the main container's PID 1 runs as + local id_output + id_output=$(run_with_timeout 10 $SMOLVM machine exec -- id -u 2>&1) || { + echo "FAIL: id -u failed: $id_output" + return 1 + } + + # nginx-unprivileged image has USER 101 + # NOTE: crun exec may run as root by default, not inheriting the image USER. + # If id returns 0 (root), this is a known gap — crun exec doesn't inherit + # the OCI process user without explicit --user. Document and accept. + if echo "$id_output" | grep -q "^101$"; then + echo "Joined exec runs as UID 101 — image USER preserved" + elif echo "$id_output" | grep -q "^0$"; then + echo "WARN: joined exec runs as root (UID 0), not image USER 101" + echo "This is a known limitation: crun exec does not inherit OCI process user" + # Not a test failure — documenting current behavior + else + echo "FAIL: unexpected UID: $id_output" + return 1 + fi +} + + +run_test "Create with --image" test_create_with_image || true +run_test "Create with --image + env" test_create_with_image_and_env || true +run_test "Update: settings applied on next start + refuses running VM" test_update_settings_applied_on_start || true +run_test "Update: env var applied on next start (image-based)" test_update_env_applied_on_start || true +run_test "Exec: large stdout does not crash VM (image-backed)" test_exec_image_large_stdout_does_not_crash_vm || true +run_test "Exec: large stdout does not crash VM (joined exec)" test_exec_joined_large_stdout_does_not_crash_vm || true +run_test "Exec-join: exec joins main workload container" test_exec_joins_main_container || true +run_test "Exec-join: repeated exec joins same container" test_repeated_exec_joins_same_container || true +run_test "Exec-join: background process visible across execs" test_background_process_visible_across_execs || true +run_test "Exec-join: ephemeral run is namespace-isolated" test_ephemeral_run_is_isolated || true +run_test "Exec-join: exec recovers after main container exits" test_exec_recovers_after_main_container_exits || true +run_test "Exec-join: timeout does not kill main container (orphan documented)" test_exec_join_timeout_does_not_kill_main_container || true +run_test "Exec-join: joined exec user behavior (crun exec runs as root)" test_exec_join_documents_user_behavior || true + +print_summary "Image-Backed Tests" diff --git a/tests/test_machine_image_proc_storage.sh b/tests/test_machine_image_proc_storage.sh new file mode 100755 index 0000000..b693ea0 --- /dev/null +++ b/tests/test_machine_image_proc_storage.sh @@ -0,0 +1,145 @@ +#!/bin/bash +# +# Image-Backed /proc and /storage Parity Tests +# +# Regression coverage for two asymmetries between bare-VM and --image machines: +# +# 1. --image froze a chunk of /proc read-only (runc's readonlyPaths), so +# docker-in-VM could not write /proc/sys/net/ipv4/ip_forward. +# 2. --image did not expose the per-VM /storage disk inside the container, so +# the docker-in-VM bind-mount pattern +# (`mount --bind /storage/docker /var/lib/docker`) broke. +# +# Both are fixed by treating a privileged (default) image machine like a bare +# VM: /proc is unrestricted and /storage is bind-mounted into the container. +# An unprivileged image machine keeps the full runc hardening. +# +# Part of the smolvm test suite. Run with: +# ./tests/test_machine_image_proc_storage.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Image-Backed /proc + /storage Parity" +echo "==========================================" +echo "" + +# A privileged --image VM must have a writable /proc/sys, like a bare VM, so an +# init system / dockerd can set sysctls. Before the fix crun applied runc's +# readonlyPaths and this write failed with EROFS. +test_image_proc_sys_is_writable() { + local out + out=$(run_with_timeout 120 $SMOLVM machine run --net --image alpine -- \ + sh -c 'echo 1 > /proc/sys/net/ipv4/ip_forward && cat /proc/sys/net/ipv4/ip_forward' 2>&1) + local rc=$? + [[ $rc -eq 124 ]] && { echo "FAIL: timed out booting image VM"; return 1; } + echo "$out" | grep -qx '1' || { + echo "FAIL: /proc/sys/net/ipv4/ip_forward not writable under --image" + echo " output: $out" + return 1 + } +} + +# Corollary: /proc/sys must not be a read-only bind in privileged mode. runc's +# readonlyPaths shows up as a `ro` mount line on the path; its absence means the +# path inherits the writable top-level /proc mount. +test_image_proc_sys_not_readonly_bind() { + local out + out=$(run_with_timeout 90 $SMOLVM machine run --net --image alpine -- \ + sh -c 'mount | grep " /proc/sys " || true' 2>&1) + local rc=$? + [[ $rc -eq 124 ]] && { echo "FAIL: timed out booting image VM"; return 1; } + if echo "$out" | grep -q '\bro\b'; then + echo "FAIL: /proc/sys still mounted read-only under --image" + echo " output: $out" + return 1 + fi +} + +# A privileged --image VM must see the per-VM /storage ext4 disk inside the +# container, so /storage/* resolves to the disk exactly as in a bare VM. +test_image_storage_disk_visible() { + local out + out=$(run_with_timeout 90 $SMOLVM machine run --net --image alpine -- \ + sh -c 'mount | grep " /storage " || true' 2>&1) + local rc=$? + [[ $rc -eq 124 ]] && { echo "FAIL: timed out booting image VM"; return 1; } + echo "$out" | grep -q ' /storage ' || { + echo "FAIL: /storage not mounted inside --image container" + echo " output: $out" + return 1 + } + echo "$out" | grep -q 'ext4' || { + echo "FAIL: /storage is not the ext4 disk under --image" + echo " output: $out" + return 1 + } +} + +# The docker-in-VM bind-mount pattern works under --image. Binding +# /var/lib/docker onto /storage/docker must land on the ext4 disk (so dockerd's +# overlay2 nests correctly), proven by a file written through the bind appearing +# on the /storage side. +test_image_docker_bind_resolves_to_storage() { + local out + out=$(run_with_timeout 90 $SMOLVM machine run --net --image alpine -- sh -c ' + set -e + mkdir -p /storage/docker /var/lib/docker + mount --bind /storage/docker /var/lib/docker + : > /var/lib/docker/probe + test -f /storage/docker/probe + # Confirm /var/lib/docker now resolves onto the ext4 storage disk. + mount | grep " /var/lib/docker " | grep -q ext4 + echo BIND_OK' 2>&1) + local rc=$? + [[ $rc -eq 124 ]] && { echo "FAIL: timed out booting image VM"; return 1; } + echo "$out" | grep -q 'BIND_OK' || { + echo "FAIL: bind-mount of /var/lib/docker onto /storage did not resolve to ext4" + echo " output: $out" + return 1 + } +} + +# Guard the security boundary: an UNPRIVILEGED image container must NOT get the +# relaxed /proc or the /storage disk — the hardening is retained for untrusted +# code. `machine run --unprivileged` opts into the restricted profile. +test_unprivileged_image_keeps_hardening() { + # /proc/sys read-only (write must fail) and /storage absent. + local out + out=$(run_with_timeout 90 $SMOLVM machine run --net --unprivileged --image alpine -- sh -c ' + if echo 1 > /proc/sys/net/ipv4/ip_forward 2>/dev/null; then echo PROC_WRITABLE; else echo PROC_RO; fi + if mount | grep -q " /storage "; then echo STORAGE_VISIBLE; else echo STORAGE_HIDDEN; fi' 2>&1) + local rc=$? + [[ $rc -eq 124 ]] && { echo "FAIL: timed out booting unprivileged image VM"; return 1; } + # If --unprivileged is unsupported by this build, skip rather than fail. + if echo "$out" | grep -qiE 'error|unexpected argument|unknown'; then + echo "SKIP: --unprivileged not supported by this build: $out" + return 0 + fi + echo "$out" | grep -q 'PROC_RO' || { + echo "FAIL: unprivileged container got a writable /proc/sys (hardening lost)" + echo " output: $out" + return 1 + } + echo "$out" | grep -q 'STORAGE_HIDDEN' || { + echo "FAIL: unprivileged container saw /storage (disk leak)" + echo " output: $out" + return 1 + } +} + +run_test "Image /proc/sys is writable (ip_forward settable)" test_image_proc_sys_is_writable || true +run_test "Image /proc/sys is not a read-only bind" test_image_proc_sys_not_readonly_bind || true +run_test "Image container sees the /storage ext4 disk" test_image_storage_disk_visible || true +run_test "Docker bind-mount onto /storage resolves to ext4" test_image_docker_bind_resolves_to_storage || true +run_test "Unprivileged image keeps /proc + /storage hardening" test_unprivileged_image_keeps_hardening || true + +print_summary "Image /proc + /storage Parity Tests" diff --git a/tests/test_machine_local_image.sh b/tests/test_machine_local_image.sh new file mode 100755 index 0000000..be88796 --- /dev/null +++ b/tests/test_machine_local_image.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# +# Local / Offline Image Tests +# +# Exercises booting a machine from a local image source with NO registry: +# - a `docker save` archive passed as a file (--image ./image.tar) +# - the same archive streamed on stdin (--image -) +# - an unpacked rootfs directory (--image ./rootfs/) +# plus persistent create/start/restart from an archive and the stdin-conflict +# guard. None of these tests pass --net: booting without networking proves the +# image is sourced locally and no pull happens. +# +# Requires docker (to produce the archive/rootfs fixtures). Skips cleanly when +# docker is unavailable. +# +# Part of the smolvm test suite. Run with: ./tests/test_machine_local_image.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +# Fixtures built once for the whole suite. +FIXTURE_DIR="" +ARCHIVE="" +ROOTFS_DIR="" + +build_fixtures() { + FIXTURE_DIR=$(mktemp -d) + ARCHIVE="$FIXTURE_DIR/alpine-save.tar" + ROOTFS_DIR="$FIXTURE_DIR/alpine-rootfs" + + # docker-save archive (OCI image tarball, multi-layer with config). + docker save alpine:latest -o "$ARCHIVE" 2>/dev/null || return 1 + + # Unpacked rootfs (flat filesystem) via docker export. + mkdir -p "$ROOTFS_DIR" + local cid + cid=$(docker create alpine:latest 2>/dev/null) || return 1 + docker export "$cid" 2>/dev/null | tar -x -C "$ROOTFS_DIR" 2>/dev/null + docker rm "$cid" >/dev/null 2>&1 || true + + [[ -s "$ARCHIVE" ]] && [[ -d "$ROOTFS_DIR/bin" ]] +} + +cleanup_local_image() { + $SMOLVM machine stop --name "$PERSIST_VM" 2>/dev/null || true + $SMOLVM machine delete --name "$PERSIST_VM" -f 2>/dev/null || true + [[ -n "$FIXTURE_DIR" ]] && rm -rf "$FIXTURE_DIR" +} +PERSIST_VM="local-image-persist-$$" +trap cleanup_local_image EXIT + +echo "" +echo "==========================================" +echo " Local / Offline Image Tests" +echo "==========================================" +echo "" + +if ! command -v docker >/dev/null 2>&1; then + log_skip "docker not available — local-image fixtures cannot be built" + print_summary "Local Image Tests" + exit 0 +fi + +if ! build_fixtures; then + log_skip "could not build docker fixtures (is the docker daemon running?)" + print_summary "Local Image Tests" + exit 0 +fi + +# Ephemeral run from a docker-save archive file. No --net: proves offline. +test_ephemeral_from_archive_file() { + local output + output=$(run_with_timeout 90 $SMOLVM machine run --image "$ARCHIVE" \ + -- sh -c 'cat /etc/os-release | head -1' 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT booting from archive file"; return 1; } + echo "$output" | grep -q "Alpine" || { + echo "FAIL: expected Alpine rootfs from archive, got: $output" + return 1 + } +} + +# Same archive, streamed on stdin via --image -. +test_ephemeral_from_stdin() { + local output + output=$(run_with_timeout 90 bash -c \ + "cat '$ARCHIVE' | '$SMOLVM' machine run --image - -- sh -c 'cat /etc/alpine-release'" 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT booting from stdin archive"; return 1; } + # alpine-release is a bare version string like 3.24.0 + echo "$output" | grep -qE '^[0-9]+\.[0-9]+' || { + echo "FAIL: expected an alpine-release version from stdin archive, got: $output" + return 1 + } +} + +# Unpacked rootfs directory (#398). The directory IS the rootfs (single lowerdir). +test_ephemeral_from_rootfs_dir() { + local output + output=$(run_with_timeout 90 $SMOLVM machine run --image "$ROOTFS_DIR" \ + -- sh -c 'cat /etc/os-release | head -1' 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT booting from rootfs dir"; return 1; } + echo "$output" | grep -q "Alpine" || { + echo "FAIL: expected Alpine rootfs from directory, got: $output" + return 1 + } +} + +# `--image -` and -i/-t both consume stdin: reject before any VM boots. +test_stdin_guard_rejects_interactive() { + local output exit_code=0 + output=$(echo "" | $SMOLVM machine run --image - -it 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || { + echo "FAIL: --image - with -it should have been rejected" + return 1 + } + echo "$output" | grep -q "cannot be" || { + echo "FAIL: expected a stdin-conflict error, got: $output" + return 1 + } + # The rejection must happen before launch — no VM should have started. + echo "$output" | grep -q "Starting ephemeral machine" && { + echo "FAIL: VM started before the guard fired" + return 1 + } + return 0 +} + +# A Dockerfile is not an image — reject with a build-first hint, not a flatten error. +test_dockerfile_rejected_with_hint() { + local dockerfile="$FIXTURE_DIR/Dockerfile" + printf 'FROM alpine:3.20\nRUN apk add curl\n' > "$dockerfile" + + local output exit_code=0 + output=$($SMOLVM machine run --image "$dockerfile" -- true 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || { + echo "FAIL: a Dockerfile should be rejected" + return 1 + } + echo "$output" | grep -qi "Dockerfile" || { + echo "FAIL: expected a build-first hint, got: $output" + return 1 + } +} + +# Persistent lifecycle from an archive: create, start, exec, stop, restart. +# Restart re-derives the local source and reopens the same on-disk rootfs. +test_persistent_create_start_restart() { + $SMOLVM machine delete --name "$PERSIST_VM" -f 2>/dev/null || true + + $SMOLVM machine create --name "$PERSIST_VM" --image "$ARCHIVE" 2>&1 || { + echo "FAIL: create from archive failed"; return 1 + } + + run_with_timeout 90 $SMOLVM machine start --name "$PERSIST_VM" 2>&1 || { + echo "FAIL: first start failed"; return 1 + } + + local first + first=$(run_with_timeout 30 $SMOLVM machine exec --name "$PERSIST_VM" \ + -- cat /etc/os-release 2>&1) + echo "$first" | grep -q "Alpine" || { + echo "FAIL: not Alpine after first start: $first"; return 1 + } + + # Write a marker, then stop + start: a persistent local image keeps its disk. + run_with_timeout 30 $SMOLVM machine exec --name "$PERSIST_VM" \ + -- sh -c 'echo persisted > /root/marker' 2>&1 || { + echo "FAIL: could not write marker"; return 1 + } + + $SMOLVM machine stop --name "$PERSIST_VM" 2>&1 || { echo "FAIL: stop failed"; return 1; } + run_with_timeout 90 $SMOLVM machine start --name "$PERSIST_VM" 2>&1 || { + echo "FAIL: restart failed (overlay/disk did not reopen)"; return 1 + } + + local marker + marker=$(run_with_timeout 30 $SMOLVM machine exec --name "$PERSIST_VM" \ + -- cat /root/marker 2>&1) + echo "$marker" | grep -q "persisted" || { + echo "FAIL: marker not persisted across restart: $marker"; return 1 + } + + $SMOLVM machine stop --name "$PERSIST_VM" 2>/dev/null || true + $SMOLVM machine delete --name "$PERSIST_VM" -f 2>/dev/null || true +} + +run_test "Ephemeral: boot from docker-save archive file (offline)" test_ephemeral_from_archive_file || true +run_test "Ephemeral: boot from archive on stdin (--image -)" test_ephemeral_from_stdin || true +run_test "Ephemeral: boot from unpacked rootfs dir (#398)" test_ephemeral_from_rootfs_dir || true +run_test "Guard: --image - with -it rejected before boot" test_stdin_guard_rejects_interactive || true +run_test "Guard: Dockerfile rejected with build-first hint" test_dockerfile_rejected_with_hint || true +run_test "Persistent: create/start/restart from archive" test_persistent_create_start_restart || true + +print_summary "Local Image Tests" diff --git a/tests/test_machine_packed.sh b/tests/test_machine_packed.sh new file mode 100755 index 0000000..9f48aa7 --- /dev/null +++ b/tests/test_machine_packed.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# +# Packed Machine Tests +# +# Part of the smolvm test suite. Run with: ./tests/test_machine_packed.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Packed Machine Tests" +echo "==========================================" +echo "" + +test_create_from_smolmachine() { + local vm_name="from-smolmachine-$$" + local tmpdir + tmpdir=$(mktemp -d) + local pack_output="$tmpdir/from-sm-pack" + + # 1. Pack alpine into a .smolmachine + $SMOLVM pack create --image alpine:latest -o "$pack_output" --cpus 1 --mem 512 2>&1 || { + echo "SKIP: pack create failed" + return 0 + } + [[ -f "$pack_output.smolmachine" ]] || { echo "FAIL: no sidecar"; return 1; } + + # 2. Create a named machine from it + $SMOLVM machine create --name "$vm_name" --from "$pack_output.smolmachine" 2>&1 || return 1 + + # 3. Start the machine (should NOT pull — uses extracted layers) + $SMOLVM machine start --name "$vm_name" 2>&1 || { + echo "FAIL: start failed" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$tmpdir"; return 1 + } + + # 4. Exec works + local exec_result + exec_result=$($SMOLVM machine exec --name "$vm_name" -- echo "from-sm-ok" 2>&1) + [[ "$exec_result" == *"from-sm-ok"* ]] || { + echo "FAIL: exec failed: $exec_result" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$tmpdir"; return 1 + } + + # 5. Persistence: write then read. Write to the container's persistent + # overlay (/sm-persist.txt), NOT /tmp — /tmp is a per-container tmpfs that is + # reset for each `machine exec` once the workload has exited, so it cannot + # test overlay persistence. + $SMOLVM machine exec --name "$vm_name" -- sh -c 'echo persist > /sm-persist.txt' 2>&1 || true + local read_result + read_result=$($SMOLVM machine exec --name "$vm_name" -- cat /sm-persist.txt 2>&1) + [[ "$read_result" == *"persist"* ]] || { + echo "FAIL: persistence failed" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$tmpdir"; return 1 + } + + # 6. Stop and restart — persistence survives + $SMOLVM machine stop --name "$vm_name" 2>&1 || true + $SMOLVM machine start --name "$vm_name" 2>&1 || { + echo "FAIL: restart failed" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$tmpdir"; return 1 + } + read_result=$($SMOLVM machine exec --name "$vm_name" -- cat /sm-persist.txt 2>&1) + [[ "$read_result" == *"persist"* ]] || { + echo "FAIL: persistence across restart failed" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$tmpdir"; return 1 + } + + # 7. Shows in ls + $SMOLVM machine ls --json 2>&1 | grep -q "$vm_name" || { + echo "FAIL: not in ls" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$tmpdir"; return 1 + } + + # 8. Cleanup + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$tmpdir" +} + +test_cp_preserves_state_on_packed_vm() { + local vm_name="cp-pack-$$" + local pack_dir + pack_dir=$(mktemp -d) + + # Create source VM, write marker, pack it + $SMOLVM machine create --name "cp-pack-src-$$" --image alpine:latest --net 2>&1 || return 1 + $SMOLVM machine start --name "cp-pack-src-$$" 2>&1 || { + $SMOLVM machine delete --name "cp-pack-src-$$" -f 2>/dev/null; return 1 + } + $SMOLVM machine exec --name "cp-pack-src-$$" -- sh -c 'echo marker > /etc/pack-marker' 2>&1 + $SMOLVM machine stop --name "cp-pack-src-$$" 2>&1 + $SMOLVM pack create --from-vm "cp-pack-src-$$" -o "$pack_dir/packed" 2>&1 || { + $SMOLVM machine delete --name "cp-pack-src-$$" -f 2>/dev/null; rm -rf "$pack_dir"; return 1 + } + $SMOLVM machine delete --name "cp-pack-src-$$" -f 2>/dev/null + + # Create destination from pack, start, mutate via exec + $SMOLVM machine create --name "$vm_name" --from "$pack_dir/packed.smolmachine" --net 2>&1 || { + rm -rf "$pack_dir"; return 1 + } + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$pack_dir"; return 1 + } + $SMOLVM machine exec --name "$vm_name" -- adduser -D alice 2>&1 + + # Verify alice exists before cp + local before + before=$($SMOLVM machine exec --name "$vm_name" -- id alice 2>&1) || { + echo "alice not found before cp"; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$pack_dir"; return 1 + } + + # Run cp + echo "probe" > "$pack_dir/probe.txt" + $SMOLVM machine cp "$pack_dir/probe.txt" "$vm_name":/tmp/probe.txt 2>&1 || { + echo "cp failed"; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$pack_dir"; return 1 + } + + # Verify alice STILL exists after cp + local after + after=$($SMOLVM machine exec --name "$vm_name" -- id alice 2>&1) || { + echo "FAIL: alice gone after cp" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$pack_dir" + return 1 + } + + # Verify probe file landed + local probe + probe=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/probe.txt 2>&1) + [[ "$probe" == *"probe"* ]] || { + echo "FAIL: probe file missing after cp" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$pack_dir" + return 1 + } + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$pack_dir" +} + + +run_test "Create from .smolmachine" test_create_from_smolmachine || true +run_test "File cp preserves state on packed VM" test_cp_preserves_state_on_packed_vm || true + +print_summary "Packed Machine Tests" diff --git a/tests/test_machine_run.sh b/tests/test_machine_run.sh new file mode 100755 index 0000000..6bbf3b1 --- /dev/null +++ b/tests/test_machine_run.sh @@ -0,0 +1,679 @@ +#!/usr/bin/env bash +# +# Machine Run (Ephemeral) Tests +# +# Part of the smolvm test suite. Run with: ./tests/test_machine_run.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Machine Run (Ephemeral) Tests" +echo "==========================================" +echo "" + +test_machine_run_echo() { + local output + output=$($SMOLVM machine run --net --image alpine:latest -- echo "run-test-marker" 2>&1) + [[ "$output" == *"run-test-marker"* ]] +} + +test_machine_run_exit_code() { + $SMOLVM machine run --net --image alpine:latest -- sh -c "exit 0" 2>&1 + local exit_code=0 + $SMOLVM machine run --net --image alpine:latest -- sh -c "exit 42" 2>&1 || exit_code=$? + [[ $exit_code -eq 42 ]] +} + +test_machine_run_env() { + local output + output=$($SMOLVM machine run --net -e TEST_VAR=hello_run --image alpine:latest -- sh -c 'echo $TEST_VAR' 2>&1) + [[ "$output" == *"hello_run"* ]] +} + +test_machine_run_volume() { + local tmpdir + tmpdir=$(mktemp -d) + echo "run-mount-test" > "$tmpdir/testfile.txt" + + local output + output=$($SMOLVM machine run --net -v "$tmpdir:/hostmnt" --image alpine:latest -- cat /hostmnt/testfile.txt 2>&1) + + rm -rf "$tmpdir" + [[ "$output" == *"run-mount-test"* ]] +} + +test_machine_run_volume_readonly() { + local tmpdir + tmpdir=$(mktemp -d) + echo "readonly-data" > "$tmpdir/readonly.txt" + + local output + output=$($SMOLVM machine run --net -v "$tmpdir:/hostmnt:ro" --image alpine:latest -- cat /hostmnt/readonly.txt 2>&1) + + # Should be able to read + [[ "$output" == *"readonly-data"* ]] || { rm -rf "$tmpdir"; return 1; } + + # Should fail to write + local write_exit=0 + $SMOLVM machine run --net -v "$tmpdir:/hostmnt:ro" --image alpine:latest -- sh -c "echo fail > /hostmnt/newfile.txt" 2>&1 || write_exit=$? + + rm -rf "$tmpdir" + [[ $write_exit -ne 0 ]] +} + +test_machine_run_volume_multiple() { + local tmpdir1 tmpdir2 + tmpdir1=$(mktemp -d) + tmpdir2=$(mktemp -d) + echo "data1" > "$tmpdir1/file1.txt" + echo "data2" > "$tmpdir2/file2.txt" + + local output + output=$($SMOLVM machine run --net -v "$tmpdir1:/data1" -v "$tmpdir2:/data2" --image alpine:latest -- sh -c "cat /data1/file1.txt && cat /data2/file2.txt" 2>&1) + + rm -rf "$tmpdir1" "$tmpdir2" + [[ "$output" == *"data1"* ]] && [[ "$output" == *"data2"* ]] +} + +test_machine_run_workdir() { + local output + output=$($SMOLVM machine run --net -w /tmp --image alpine:latest -- pwd 2>&1) + [[ "$output" == *"/tmp"* ]] +} + +test_machine_run_image_default_workdir() { + local output exit_code=0 + output=$(run_with_timeout 180 \ + "$SMOLVM" machine run -I docker.io/library/redis:7.4 --net -- \ + sh -lc "pwd") || exit_code=$? + + if [[ $exit_code -ne 0 ]]; then + case "$output" in + *"no matching manifest"*|*"exec format error"*|*"platform"* ) + log_skip "Skipping image default workdir regression: image unsupported on this host" + return 0 + ;; + esac + + echo "machine run failed:" + echo "$output" + return 1 + fi + + local pwd_result + pwd_result=$(printf '%s\n' "$output" | tail -n 1 | tr -d '\r') + [[ "$pwd_result" == "/data" ]] || { + echo "expected image workdir /data, got: $pwd_result" + echo "$output" + return 1 + } +} + +test_machine_run_image_default_user() { + local output exit_code=0 + output=$(run_with_timeout 180 \ + "$SMOLVM" machine run -I docker.io/nginxinc/nginx-unprivileged:stable-alpine --net -- \ + sh -lc "id -u") || exit_code=$? + + if [[ $exit_code -ne 0 ]]; then + case "$output" in + *"no matching manifest"*|*"exec format error"*|*"platform"*|*"not found"* ) + log_skip "Skipping image default user regression: image unsupported on this host" + return 0 + ;; + esac + + echo "machine run failed:" + echo "$output" + return 1 + fi + + local user_result + user_result=$(printf '%s\n' "$output" | tail -n 1 | tr -d '\r') + [[ "$user_result" == "101" ]] || { + echo "expected image user id 101, got: $user_result" + echo "$output" + return 1 + } +} + +test_machine_run_detached() { + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + + local run_output exit_code=0 + run_output=$($SMOLVM machine run -d --net --image alpine:latest 2>&1) || exit_code=$? + + if [[ $exit_code -ne 0 ]]; then + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + echo "Setup failed: machine run -d returned $exit_code: $run_output" + return 1 + fi + + # Should appear in machine ls + local list_output + list_output=$($SMOLVM machine ls --json 2>&1) + + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + + [[ "$list_output" == *'"name": "default"'* ]] && \ + [[ "$list_output" == *'"state": "running"'* ]] +} + +test_machine_run_detached_with_command() { + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + + # The detached command writes to a host-mounted volume so execution is + # verified on the host. Do NOT write to the container's /tmp and read it back + # via `machine exec`: /tmp is a per-container tmpfs, and once the detached + # workload exits a later `machine exec` runs in a fresh container, so the + # tmpfs file would not be visible (false failure). + local outdir + outdir=$(mktemp -d) + + local run_output exit_code=0 + run_output=$($SMOLVM machine run -d --net -v "$outdir:/out" --image alpine:latest -- \ + sh -c "echo issue198_fixed > /out/run-d-cmd-test.txt" 2>&1) || exit_code=$? + + if [[ $exit_code -ne 0 ]]; then + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + rm -rf "$outdir" + echo "Setup failed: machine run -d returned $exit_code: $run_output" + return 1 + fi + + # Poll until the background command has written the file (usually <1s). + local file_output="" i=0 + while [[ $i -lt 20 ]]; do + file_output=$(cat "$outdir/run-d-cmd-test.txt" 2>/dev/null) && [[ -n "$file_output" ]] && break + sleep 0.5 + ((i++)) || true + done + + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + rm -rf "$outdir" + + if [[ "$file_output" != *"issue198_fixed"* ]]; then + echo "FAIL: command was not executed by 'machine run -d --image X -- cmd'" + echo "Expected file contents containing 'issue198_fixed', got: $file_output" + return 1 + fi +} + +test_machine_run_image_volume_workspace() { + local tmpdir + tmpdir=$(mktemp -d) + echo "workspace-volume-marker" > "$tmpdir/probe.txt" + + local output exit_code=0 + output=$(run_with_timeout 60 $SMOLVM machine run --net \ + -v "$tmpdir:/workspace" --image alpine:latest -- cat /workspace/probe.txt 2>&1) || exit_code=$? + + rm -rf "$tmpdir" + + if [[ $exit_code -ne 0 ]]; then + echo "FAIL: machine run failed (exit $exit_code): $output" + return 1 + fi + if ! echo "$output" | grep -q "workspace-volume-marker"; then + echo "FAIL: host file not visible at /workspace in image container" + echo "output: $output" + return 1 + fi +} + +test_machine_run_image_volume_workspace_smolfile() { + local tmpdir + tmpdir=$(mktemp -d) + echo "smolfile-workspace-marker" > "$tmpdir/probe.txt" + + cat > "$tmpdir/Smolfile.toml" <<'EOF' +image = "alpine:latest" +net = true + +[dev] +volumes = [".:/workspace"] +EOF + + local output exit_code=0 + output=$( + cd "$tmpdir" + run_with_timeout 60 $SMOLVM machine run -s Smolfile.toml -- cat /workspace/probe.txt 2>&1 + ) || exit_code=$? + + rm -rf "$tmpdir" + + if [[ $exit_code -ne 0 ]]; then + echo "FAIL: machine run -s Smolfile.toml failed (exit $exit_code): $output" + return 1 + fi + if ! echo "$output" | grep -q "smolfile-workspace-marker"; then + echo "FAIL: host file not visible at /workspace via Smolfile [dev].volumes" + echo "output: $output" + return 1 + fi +} + +test_machine_run_detached_volume_workspace() { + local vm_name="det-vol-ws-$$" + local tmpdir + tmpdir=$(mktemp -d) + echo "detached-workspace-marker" > "$tmpdir/probe.txt" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + local run_output exit_code=0 + run_output=$(run_with_timeout 120 $SMOLVM machine run -d --net \ + --name "$vm_name" \ + -v "$tmpdir:/workspace" \ + --image alpine:latest -- sleep infinity 2>&1) || exit_code=$? + + if [[ $exit_code -ne 0 ]]; then + echo "FAIL: machine run -d failed: $run_output" + rm -rf "$tmpdir" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Keep tmpdir alive until after exec — the virtiofs mount points to it + local exec_out + exec_out=$(run_with_timeout 30 $SMOLVM machine exec --name "$vm_name" -- cat /workspace/probe.txt 2>&1) || { + echo "FAIL: exec failed: $exec_out" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$tmpdir" + return 1 + } + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$tmpdir" + + if ! echo "$exec_out" | grep -q "detached-workspace-marker"; then + echo "FAIL: host file not visible at /workspace in detached container" + echo "exec output: $exec_out" + return 1 + fi +} + +test_machine_start_restores_workload() { + local vm_name="start-restores-workload-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create persistent VM with a long-running workload + local run_output exit_code=0 + run_output=$(run_with_timeout 120 $SMOLVM machine run -d --net \ + --name "$vm_name" --image alpine:latest -- sleep infinity 2>&1) || exit_code=$? + if [[ $exit_code -ne 0 ]]; then + echo "FAIL: machine run -d failed: $run_output" + return 1 + fi + + # Verify workload is PID 1 before stop + local ps_before + ps_before=$(run_with_timeout 30 $SMOLVM machine exec --name "$vm_name" -- ps -ef 2>&1) || { + echo "FAIL: exec before stop failed: $ps_before" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + if ! echo "$ps_before" | grep -q "sleep"; then + echo "FAIL: expected 'sleep infinity' in ps before stop, got:" + echo "$ps_before" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Stop and restart + $SMOLVM machine stop --name "$vm_name" 2>&1 || { + echo "FAIL: machine stop failed" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + local start_output + start_output=$(run_with_timeout 120 $SMOLVM machine start --name "$vm_name" 2>&1) || { + echo "FAIL: machine start failed: $start_output" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + # Verify workload is PID 1 after restart — the regression check + local ps_after + ps_after=$(run_with_timeout 30 $SMOLVM machine exec --name "$vm_name" -- ps -ef 2>&1) || { + echo "FAIL: exec after restart failed: $ps_after" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + if ! echo "$ps_after" | grep -q "sleep"; then + echo "FAIL: workload 'sleep infinity' not present after stop+start" + echo "ps before stop:" + echo "$ps_before" + echo "ps after restart:" + echo "$ps_after" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true +} + +test_machine_start_restores_workload_smolfile() { + local vm_name="start-restores-sf-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + local tmpdir + tmpdir=$(mktemp -d) + + cat > "$tmpdir/Smolfile.toml" <<'EOF' +image = "alpine:latest" +net = true +cmd = ["sleep", "infinity"] +EOF + + # Create via Smolfile, then run detached (exact repro shape) + local run_output exit_code=0 + run_output=$( + cd "$tmpdir" + run_with_timeout 120 $SMOLVM machine run -d --name "$vm_name" -s Smolfile.toml 2>&1 + ) || exit_code=$? + + rm -rf "$tmpdir" + + if [[ $exit_code -ne 0 ]]; then + echo "FAIL: machine run -d -s Smolfile.toml failed: $run_output" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Verify workload is running before stop + local ps_before + ps_before=$(run_with_timeout 30 $SMOLVM machine exec --name "$vm_name" -- ps -ef 2>&1) || { + echo "FAIL: exec before stop failed: $ps_before" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + if ! echo "$ps_before" | grep -q "sleep"; then + echo "FAIL: expected 'sleep infinity' in ps before stop, got:" + echo "$ps_before" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Stop and restart + $SMOLVM machine stop --name "$vm_name" 2>&1 || { + echo "FAIL: machine stop failed" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + local start_output + start_output=$(run_with_timeout 120 $SMOLVM machine start --name "$vm_name" 2>&1) || { + echo "FAIL: machine start failed: $start_output" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + # Workload must still be present — the Smolfile cmd regression check + local ps_after + ps_after=$(run_with_timeout 30 $SMOLVM machine exec --name "$vm_name" -- ps -ef 2>&1) || { + echo "FAIL: exec after restart failed: $ps_after" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + if ! echo "$ps_after" | grep -q "sleep"; then + echo "FAIL: Smolfile cmd 'sleep infinity' not present after stop+start" + echo "ps before stop:" + echo "$ps_before" + echo "ps after restart:" + echo "$ps_after" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true +} + +test_machine_run_timeout() { + local output + output=$($SMOLVM machine run --net --timeout 5s --image alpine:latest -- sleep 60 2>&1 || true) + # Should be killed before 60s + [[ "$output" == *"timed out"* ]] || [[ "$output" == *"Killed"* ]] || [[ $? -ne 0 ]] +} + +test_machine_run_pipeline() { + local output + output=$($SMOLVM machine run --net --image alpine:latest -- sh -c "echo 'hello world' | wc -w" 2>&1) + [[ "$output" == *"2"* ]] +} + +test_machine_run_cmd_not_found() { + ! $SMOLVM machine run --net --image alpine:latest -- nonexistent_command_12345 2>/dev/null +} + +test_ephemeral_vm_tracking() { + # Ephemeral machine run should appear in list while running, disappear after exit + local result + result=$(run_with_timeout 30 $SMOLVM machine run --net --image alpine -- echo "ephemeral-tracking-test" 2>&1) + local exit_code=$? + [[ $exit_code -eq 124 ]] && { echo "TIMEOUT"; return 1; } + [[ "$result" == *"ephemeral-tracking-test"* ]] || { echo "Command failed: $result"; return 1; } + + # After clean exit, the ephemeral record should be gone + local list_result + list_result=$($SMOLVM machine ls 2>&1) + # Should NOT contain any ephemeral VMs from this run (they deregister on exit) + if echo "$list_result" | grep -q "(eph).*running"; then + echo "Ephemeral VM still in list after clean exit" + return 1 + fi + + # Verify orphan cleanup works: list should not error + [[ $? -eq 0 ]] +} + +test_ephemeral_shows_in_list_while_running() { + # Start a detached run with an explicit name and verify it appears in list. + # An unnamed `-d` run lands as "default"; naming it makes the assertion + # deterministic (grep the exact name) and the cleanup unambiguous. + local name="run-visible-$$" + $SMOLVM machine run --net -d --name "$name" --image alpine -- sleep 30 2>&1 || { + echo "Detached run failed" + return 1 + } + + # Poll until the detached VM appears in the list (usually <1s). + local list_result i=0 + while [[ $i -lt 20 ]]; do + list_result=$($SMOLVM machine ls 2>&1) + echo "$list_result" | grep -q "$name" && break + sleep 0.5 + ((i++)) || true + done + echo "$list_result" | grep -q "$name" || { + echo "Detached run not in list: $list_result" + $SMOLVM machine stop --name "$name" 2>/dev/null || true + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true + return 1 + } + + # Clean up + $SMOLVM machine stop --name "$name" 2>/dev/null || true + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true +} + +test_run_no_command_errors() { + # machine run with no command and no -it should error with guidance + local result + local exit_code=0 + result=$($SMOLVM machine run 2>&1) || exit_code=$? + + # Should fail with non-zero exit + [[ $exit_code -ne 0 ]] || { echo "Should have failed"; return 1; } + + # Should contain usage guidance + [[ "$result" == *"no command specified"* ]] || { + echo "Missing usage guidance: $result" + return 1 + } +} + +test_ephemeral_runs_do_not_share_state() { + # Run 1: create a marker file + $SMOLVM machine run --net --image alpine:latest -- \ + sh -c 'echo ephemeral-leak-test > /tmp/ephemeral-marker.txt' 2>&1 + + # Run 2: marker must not exist + local output exit_code=0 + output=$($SMOLVM machine run --net --image alpine:latest -- \ + sh -c 'cat /tmp/ephemeral-marker.txt 2>&1 || echo MARKER_NOT_FOUND' 2>&1) || exit_code=$? + + [[ "$output" == *"MARKER_NOT_FOUND"* ]] || { + echo "FAIL: ephemeral run found file from previous run" + echo "$output" + return 1 + } +} + +test_ephemeral_volume_mount_reflects_host() { + local tmpdir + tmpdir=$(mktemp -d) + echo "file-a" > "$tmpdir/a.txt" + echo "file-b" > "$tmpdir/b.txt" + echo "file-c" > "$tmpdir/c.txt" + + local output + output=$($SMOLVM machine run --net -v "$tmpdir:/hostmnt" --image alpine:latest -- \ + ls /hostmnt 2>&1) + + rm -rf "$tmpdir" + + [[ "$output" == *"a.txt"* ]] || { echo "missing a.txt: $output"; return 1; } + [[ "$output" == *"b.txt"* ]] || { echo "missing b.txt: $output"; return 1; } + [[ "$output" == *"c.txt"* ]] || { echo "missing c.txt: $output"; return 1; } +} + +test_init_skipped_on_restart() { + local vm_name="init-skip-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create with init command and start — first boot should run init + $SMOLVM machine create --name "$vm_name" --net --init "echo INIT_RAN" 2>&1 + local first_start + first_start=$($SMOLVM machine start --name "$vm_name" 2>&1) + echo "$first_start" + + if [[ "$first_start" != *"Running 1 init command"* ]]; then + echo "FAIL: first start should run init" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Stop and restart — should skip init + $SMOLVM machine stop --name "$vm_name" 2>&1 + local second_start + second_start=$($SMOLVM machine start --name "$vm_name" 2>&1) + echo "$second_start" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + if [[ "$second_start" == *"Running"*"init command"* ]]; then + echo "FAIL: second start should NOT re-run init" + return 1 + fi + if [[ "$second_start" != *"Init already completed"* ]]; then + echo "FAIL: second start should print skip message" + return 1 + fi +} + + +run_test "Machine run: echo" test_machine_run_echo || true +run_test "Machine run: exit code" test_machine_run_exit_code || true +run_test "Machine run: env variable" test_machine_run_env || true +run_test "Machine run: volume mount" test_machine_run_volume || true +run_test "Machine run: volume readonly" test_machine_run_volume_readonly || true +run_test "Machine run: multiple volumes" test_machine_run_volume_multiple || true +run_test "Machine run: workdir" test_machine_run_workdir || true +run_test "Machine run: image default workdir" test_machine_run_image_default_workdir || true +run_test "Machine run: image default user" test_machine_run_image_default_user || true +run_test "Machine run: detached" test_machine_run_detached || true +run_test "Machine run: detached with command (issue #198)" test_machine_run_detached_with_command || true +run_test "Volume: image container -v /workspace not overridden by storage disk" test_machine_run_image_volume_workspace || true +run_test "Volume: Smolfile [dev] volumes /workspace not overridden by storage disk" test_machine_run_image_volume_workspace_smolfile || true +run_test "Volume: detached -v /workspace not overridden by storage disk" test_machine_run_detached_volume_workspace || true +run_test "Machine start: restores workload after stop+start" test_machine_start_restores_workload || true +run_test "Machine start: restores Smolfile cmd workload after stop+start" test_machine_start_restores_workload_smolfile || true +run_test "Machine run: timeout" test_machine_run_timeout || true +run_test "Machine run: pipeline" test_machine_run_pipeline || true +run_test "Machine run: cmd not found" test_machine_run_cmd_not_found || true +run_test "Ephemeral VM: clean exit deregisters" test_ephemeral_vm_tracking || true +run_test "Ephemeral VM: visible while running" test_ephemeral_shows_in_list_while_running || true +run_test "Run with no command errors" test_run_no_command_errors || true +run_test "Ephemeral run: no state leaks between runs" test_ephemeral_runs_do_not_share_state || true +run_test "Ephemeral run: volume mount shows correct host contents" test_ephemeral_volume_mount_reflects_host || true +run_test "Init: skipped on restart after first successful run" test_init_skipped_on_restart || true + +# ============================================================================= +# Piped stdin EOF detection +# ============================================================================= + +test_piped_stdin_cat() { + local out exit_code=0 + out=$(echo "hello" | run_with_timeout 15 "$SMOLVM" machine run -i -- cat 2>/dev/null) \ + || exit_code=$? + [[ $exit_code -eq 0 ]] || { echo "FAIL: exit $exit_code (expected 0)"; return 1; } + [[ "$out" == *"hello"* ]] || { echo "FAIL: expected 'hello', got: $out"; return 1; } +} + +run_test "Stdin: piped 'echo hello | run -i cat' returns data" test_piped_stdin_cat || true + +# ============================================================================= +# Status messages go to stderr, not stdout +# ============================================================================= + +test_stdout_no_status_messages() { + local out + # Pass --net so the run can pull alpine if it isn't already cached. Without + # it this test depends on a warm image cache, which an earlier suite's + # `prune --all` can wipe (suites share the host image cache) — making the + # run fail to pull and the assertion flake on ordering. + out=$("$SMOLVM" machine run --net --image alpine -- echo PAYLOAD_ONLY 2>/dev/null) || true + if echo "$out" | grep -qiE "^Starting|^Pulling"; then + echo "FAIL: status messages in stdout: $(echo "$out" | grep -iE 'Starting|Pulling' | head -1)" + return 1 + fi + [[ "$out" == *"PAYLOAD_ONLY"* ]] || { echo "FAIL: missing payload in: $out"; return 1; } +} + +run_test "Output: stdout has only command output, no status" test_stdout_no_status_messages || true + +print_summary "Machine Run Tests" diff --git a/tests/test_network.sh b/tests/test_network.sh new file mode 100755 index 0000000..e5fb50e --- /dev/null +++ b/tests/test_network.sh @@ -0,0 +1,736 @@ +#!/usr/bin/env bash +# +# Network and Egress Tests +# +# Part of the smolvm test suite. Run with: ./tests/test_network.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Network and Egress Tests" +echo "==========================================" +echo "" + +# Assert that egress from a VM to a destination outside its allowlist is +# blocked by the egress policy (EACCES) — not merely failing or timing out. +# +# The probe deliberately targets a real, reachable, NON-resolver web host on +# port 443 (resolved on the host, where egress is open). A public-DNS IP such +# as 8.8.8.8 is NOT a valid "blocked" example: the policy auto-adds the host's +# own DNS resolver to every allowlist (so VMs can resolve names), so on a host +# whose resolver is 8.8.8.8 that IP is silently permitted. A web IP is never +# auto-added, so it is a sound block target regardless of the host's DNS config. +# +# A policy block returns EACCES immediately; an unreachable host would hang +# until the timeout. We require BOTH a nonzero exit AND a fast failure, so a +# timeout (or any non-policy failure) can never masquerade as a real block. +assert_egress_blocked() { + local vm_name="$1" + + local web_ip + web_ip=$(resolve_host_ipv4 example.com) + if [[ -z "$web_ip" ]]; then + echo "FAIL: could not resolve a non-resolver block-probe target on host" + return 1 + fi + + # Probe inside the guest; emit " ". + local result + result=$($SMOLVM machine exec --name "$vm_name" -- sh -c \ + "s=\$(date +%s%N); nc -w 4 -z $web_ip 443 >/dev/null 2>&1; r=\$?; e=\$(date +%s%N); echo \"\$r \$(((e-s)/1000000))\"" \ + 2>/dev/null | tail -1) + if [[ -z "$result" ]]; then + echo "FAIL: egress probe to $web_ip:443 produced no result (exec failed?)" + return 1 + fi + + local rc=${result%% *} ms=${result##* } + if [[ "$rc" == "0" ]]; then + echo "FAIL: egress to non-allowlisted $web_ip:443 was permitted (policy not enforced)" + return 1 + fi + if [[ -z "$ms" ]] || [[ "$ms" -ge 2000 ]]; then + echo "FAIL: egress to $web_ip:443 failed by timeout (${ms}ms), not a policy block (EACCES)" + return 1 + fi + return 0 +} + +test_machine_network_disabled_by_default() { + local vm_name="net-disabled-test-$$" + + # Clean up any existing + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM without --net (network disabled by default) + $SMOLVM machine create --name "$vm_name" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # DNS resolution should fail when network is disabled + local exit_code=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup cloudflare.com 2>&1 || exit_code=$? + + # Clean up + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + # Should fail (non-zero exit code) because network is disabled + [[ $exit_code -ne 0 ]] +} + +test_machine_network_dns_resolution() { + local vm_name="net-dns-test-$$" + + # Clean up any existing + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM with --net (network enabled) + $SMOLVM machine create --name "$vm_name" --net 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Test DNS resolution + local output exit_code=0 + output=$($SMOLVM machine exec --name "$vm_name" -- nslookup cloudflare.com 2>&1) || exit_code=$? + + # Clean up + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + # Should succeed and contain resolved address info + [[ $exit_code -eq 0 ]] && [[ "$output" == *"Address"* ]] +} + +test_machine_network_multiple_dns_lookups() { + local vm_name="net-multi-dns-test-$$" + + # Clean up any existing + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM with --net (network enabled) + $SMOLVM machine create --name "$vm_name" --net 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Test multiple DNS lookups + local output exit_code=0 + output=$($SMOLVM machine exec --name "$vm_name" -- sh -c "nslookup google.com && nslookup github.com" 2>&1) || exit_code=$? + + # Clean up + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + # Should succeed and contain addresses for both + [[ $exit_code -eq 0 ]] && [[ "$output" == *"Address"* ]] +} + +test_machine_egress_allow_cidr_permitted() { + local vm_name="egress-allow-test-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM allowing only Cloudflare DNS + $SMOLVM machine create --name "$vm_name" --allow-cidr 1.1.1.1/32 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # DNS lookup to allowed IP should succeed + local output exit_code=0 + output=$($SMOLVM machine exec --name "$vm_name" -- nslookup cloudflare.com 1.1.1.1 2>&1) || exit_code=$? + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ $exit_code -eq 0 ]] && [[ "$output" == *"Address"* ]] +} + +test_machine_egress_allow_cidr_blocked() { + local vm_name="egress-block-test-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM allowing only a private range. ensure_dns_in_cidrs also + # auto-adds the host's DNS resolver so the VM can still resolve names. + $SMOLVM machine create --name "$vm_name" --allow-cidr 10.0.0.0/8 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # A destination outside 10.0.0.0/8 (and not the auto-added resolver) blocked. + local blocked_rc=0 + assert_egress_blocked "$vm_name" || blocked_rc=1 + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ $blocked_rc -eq 0 ]] +} + +test_machine_egress_outbound_localhost_only() { + local vm_name="egress-localhost-test-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + $SMOLVM machine create --name "$vm_name" --outbound-localhost-only 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Loopback-only egress: every external destination must be blocked. + local blocked_rc=0 + assert_egress_blocked "$vm_name" || blocked_rc=1 + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ $blocked_rc -eq 0 ]] +} + +test_machine_egress_invalid_cidr_rejected() { + local vm_name="egress-invalid-test-$$" + local output exit_code=0 + output=$($SMOLVM machine create --name "$vm_name" --allow-cidr "not-a-cidr" 2>&1) || exit_code=$? + + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + [[ $exit_code -ne 0 ]] && [[ "$output" == *"invalid"* ]] +} + +test_machine_egress_allow_host_permitted() { + local vm_name="egress-host-allow-test-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM allowing only one.one.one.one (resolves to 1.1.1.1) + $SMOLVM machine create --name "$vm_name" --allow-host one.one.one.one 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Resolving the ALLOWED host must succeed. (Query one.one.one.one — the host + # we allow-listed — not some other domain: the allow-host filter correctly + # returns NXDOMAIN for any non-allowed name, so querying e.g. cloudflare.com + # here would be a self-inflicted failure, not a policy bug.) + local output exit_code=0 + output=$($SMOLVM machine exec --name "$vm_name" -- nslookup one.one.one.one 1.1.1.1 2>&1) || exit_code=$? + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ $exit_code -eq 0 ]] && [[ "$output" == *"Address"* ]] +} + +test_machine_egress_allow_host_blocked() { + local vm_name="egress-host-block-test-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM allowing only one.one.one.one. A host outside the allowlist + # (and not the auto-added resolver) must be blocked. + $SMOLVM machine create --name "$vm_name" --allow-host one.one.one.one 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + local blocked_rc=0 + assert_egress_blocked "$vm_name" || blocked_rc=1 + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ $blocked_rc -eq 0 ]] +} + +test_machine_egress_allow_host_invalid_rejected() { + local vm_name="egress-host-invalid-test-$$" + local output exit_code=0 + output=$($SMOLVM machine create --name "$vm_name" --allow-host "this-does-not-exist.invalid" 2>&1) || exit_code=$? + + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Should fail with a resolution error (hard error, not warning) + [[ $exit_code -ne 0 ]] && [[ "$output" == *"failed to resolve"* ]] +} + +test_machine_egress_allow_host_port_rejected() { + local vm_name="egress-host-port-test-$$" + local output exit_code=0 + output=$($SMOLVM machine create --name "$vm_name" --allow-host "example.com:443" 2>&1) || exit_code=$? + + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Should fail — port suffixes are not supported + [[ $exit_code -ne 0 ]] && [[ "$output" == *"port suffixes are not supported"* ]] +} + +test_machine_dns_filter_blocks_resolution() { + local vm_name="dns-filter-test-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM allowing only one.one.one.one + $SMOLVM machine create --name "$vm_name" --allow-host one.one.one.one 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Resolving an allowed domain should work + local exit_code_allowed=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup one.one.one.one 1.1.1.1 2>&1 || exit_code_allowed=$? + + # Resolving a non-allowed domain should fail (DNS proxy returns NXDOMAIN, + # or if agent doesn't have DNS proxy, TSI still blocks the IP) + local exit_code_blocked=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup attacker-test.example 2>&1 || exit_code_blocked=$? + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ $exit_code_allowed -eq 0 ]] && [[ $exit_code_blocked -ne 0 ]] +} + +# Helper: send a raw TCP DNS query to 127.0.0.1:53 inside a running VM and +# return the RCODE (0–15) from the response, or 255 on error. +# +# Usage: dns_tcp_rcode +# +# The query string must be a printf-compatible byte sequence including the +# 2-byte big-endian length prefix followed by the raw DNS query bytes. +# Response RCODE lives in the lower nibble of byte 5 of the TCP stream +# (= 2-byte length prefix + 3 DNS header bytes: ID×2, flags-byte-0). +_dns_tcp_rcode() { + local vm_name="$1" + local query_printf="$2" + + local resp_file + resp_file=$(mktemp) + + # Query the configured resolver (resolv.conf nameserver = the smolvm gateway, + # which enforces allow-host filtering) over TCP, exactly as a real client + # would on DNS/TCP fallback. The dst is resolved inside the guest so the test + # is independent of the gateway IP. + $SMOLVM machine exec --name "$vm_name" -- sh -c \ + "ns=\$(awk '/^nameserver/{print \$2; exit}' /etc/resolv.conf); printf '$query_printf' | nc -w 2 \"\$ns\" 53" \ + >"$resp_file" 2>/dev/null || true + + local rcode=255 + if [[ -s "$resp_file" ]]; then + local hex + hex=$(dd if="$resp_file" bs=1 skip=5 count=1 2>/dev/null | od -An -tx1 | tr -d ' \n') + [[ -n "$hex" ]] && rcode=$(( 16#${hex} & 0x0F )) + fi + + rm -f "$resp_file" + echo "$rcode" +} + +test_dns_filter_tcp_allowed() { + local vm_name="dns-tcp-allowed-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + $SMOLVM machine create --name "$vm_name" --allow-host one.one.one.one 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 \ + || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Minimal A query for "one.one.one.one" (33 DNS bytes = 0x21). + # TCP DNS framing: 2-byte BE length prefix + raw DNS query. + # Header: ID=0x1234, RD=1, QDCOUNT=1. Name: \x03one×4 + \x00. + # Host filter allows one.one.one.one → should return RCODE=0 (NOERROR). + local rcode + rcode=$(_dns_tcp_rcode "$vm_name" \ + '\x00\x21\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03one\x03one\x03one\x03one\x00\x00\x01\x00\x01') + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ "$rcode" -eq 0 ]] || { echo "FAIL: expected RCODE=0 (NOERROR), got RCODE=$rcode"; return 1; } +} + +test_dns_filter_tcp_blocked() { + local vm_name="dns-tcp-blocked-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + $SMOLVM machine create --name "$vm_name" --allow-host one.one.one.one 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 \ + || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # A query for "attacker.invalid" (34 DNS bytes = 0x22). + # Name: \x08attacker (9 bytes) + \x07invalid (8 bytes) + \x00 (1 byte) = 18 bytes. + # Not in the allowlist → host filter returns NXDOMAIN → RCODE=3. + local rcode + rcode=$(_dns_tcp_rcode "$vm_name" \ + '\x00\x22\x12\x34\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x08attacker\x07invalid\x00\x00\x01\x00\x01') + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ "$rcode" -eq 3 ]] || { echo "FAIL: expected RCODE=3 (NXDOMAIN), got RCODE=$rcode"; return 1; } +} + +test_machine_allow_host_persists_across_restart() { + local vm_name="dns-persist-test-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create with --allow-host, start, stop, start again + $SMOLVM machine create --name "$vm_name" --allow-host one.one.one.one 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Verify egress works + local exit_code=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup one.one.one.one 1.1.1.1 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] && { $SMOLVM machine stop --name "$vm_name" 2>/dev/null; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Stop and restart — config should persist from VmRecord + $SMOLVM machine stop --name "$vm_name" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Egress restriction must still hold after the restart. + local blocked_rc=0 + assert_egress_blocked "$vm_name" || blocked_rc=1 + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ $blocked_rc -eq 0 ]] +} + +test_smolfile_allow_hosts_stale_cidr_regression() { + local vm_name="allow-hosts-stale-test-$$" + + # sqlite3 is required to inject stale CIDRs into the DB + if ! command -v sqlite3 >/dev/null 2>&1; then + echo "SKIP: sqlite3 not available" + return 0 + fi + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + local tmpdir + tmpdir=$(mktemp -d) + + # Smolfile with allow_hosts — no explicit allow_cidrs. + # one.one.one.one resolves to 1.1.1.1 / 1.0.0.1 (Cloudflare DNS). + cat > "$tmpdir/Smolfile.toml" <<'EOF' +[network] +allow_hosts = ["one.one.one.one"] +EOF + + ( + cd "$tmpdir" + $SMOLVM machine create --name "$vm_name" -s Smolfile.toml 2>&1 + ) || { rm -rf "$tmpdir"; return 1; } + + # Determine DB path (matches SmolvmDb::default_path logic) + local db_path + if [[ "$(uname)" == "Darwin" ]]; then + db_path="$HOME/Library/Application Support/smolvm/server/smolvm.db" + else + db_path="$HOME/.local/share/smolvm/server/smolvm.db" + fi + + # Inject stale CIDRs — 192.0.2.0/24 is RFC 5737 TEST-NET, never routed. + # This simulates the old bug: CIDRs resolved at create time that are now + # stale due to CDN IP rotation. + # The data column is stored as BLOB. json_set returns TEXT, so we must + # CAST both ways: TEXT for JSON manipulation, then back to BLOB for storage. + sqlite3 "$db_path" \ + "UPDATE vms SET data = CAST(json_set(CAST(data AS TEXT), '$.allowed_cidrs', json('[\"192.0.2.0/24\"]')) AS BLOB) WHERE name = '$vm_name'" \ + 2>&1 || { echo "sqlite3 update failed"; rm -rf "$tmpdir"; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Start — fix must re-resolve allow_hosts and override the stale CIDRs + $SMOLVM machine start --name "$vm_name" 2>&1 \ + || { rm -rf "$tmpdir"; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Probe egress using 1.0.0.1 as the resolver — also a valid IP for + # one.one.one.one, but NOT auto-added by ensure_dns_in_cidrs (which only + # injects 1.1.1.1). With stale CIDRs and no re-resolution, 1.0.0.1 is + # blocked; with the fix, fresh resolution adds it and the query succeeds. + local exit_code=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup one.one.one.one 1.0.0.1 2>&1 || exit_code=$? + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + # Fallback: if machine delete --name failed due to a corrupt DB row (e.g. from a + # botched sqlite3 update), remove the row directly so it doesn't poison + # subsequent test runs that scan all rows. + sqlite3 "$db_path" "DELETE FROM vms WHERE name = '$vm_name'" 2>/dev/null || true + rm -rf "$tmpdir" + ensure_data_dir_deleted "$vm_name" + + [[ $exit_code -eq 0 ]] +} + +test_smolfile_allow_hosts_egress_basic() { + local vm_name="allow-hosts-sf-basic-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + local tmpdir + tmpdir=$(mktemp -d) + + cat > "$tmpdir/Smolfile.toml" <<'EOF' +[network] +allow_hosts = ["one.one.one.one"] +EOF + + ( + cd "$tmpdir" + $SMOLVM machine create --name "$vm_name" -s Smolfile.toml 2>&1 + ) || { rm -rf "$tmpdir"; return 1; } + + $SMOLVM machine start --name "$vm_name" 2>&1 \ + || { rm -rf "$tmpdir"; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Probe 1: 1.1.1.1 — always in the egress policy via ensure_dns_in_cidrs. + # Verifies the policy doesn't block what it should allow, but does NOT + # prove that hostname resolution ran (1.1.1.1 is auto-added regardless). + local exit_code_allowed=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup one.one.one.one 1.1.1.1 2>&1 || exit_code_allowed=$? + + # Probe 2: 1.0.0.1 — a real IP of one.one.one.one, but NOT auto-added by + # ensure_dns_in_cidrs. This probe passes only if allow_hosts DNS resolution + # actually ran and added 1.0.0.1 to the CIDR list. It is the definitive + # proof that the hostname-to-CIDR path works end-to-end. + local exit_code_resolution=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup one.one.one.one 1.0.0.1 2>&1 || exit_code_resolution=$? + + # Egress to a destination outside the allowlist must be blocked. + local blocked_rc=0 + assert_egress_blocked "$vm_name" || blocked_rc=1 + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$tmpdir" + ensure_data_dir_deleted "$vm_name" + + [[ $exit_code_allowed -eq 0 ]] && [[ $exit_code_resolution -eq 0 ]] && [[ $blocked_rc -eq 0 ]] +} + +test_egress_refresh_thread_stability() { + skip_if_slow && return 0 + local vm_name="egress-refresh-smoke-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + local tmpdir + tmpdir=$(mktemp -d) + + cat > "$tmpdir/Smolfile.toml" <<'EOF' +[network] +allow_hosts = ["one.one.one.one"] +EOF + + ( + cd "$tmpdir" + $SMOLVM machine create --name "$vm_name" -s Smolfile.toml 2>&1 + ) || { rm -rf "$tmpdir"; return 1; } + + # Start with a 10-second refresh interval so the thread fires twice during + # the test window without making the test slow. + SMOLVM_EGRESS_REFRESH_SECS=10 $SMOLVM machine start --name "$vm_name" 2>&1 \ + || { rm -rf "$tmpdir"; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Verify egress works immediately after start. + local exit_code_before=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup one.one.one.one 1.1.1.1 2>&1 \ + || exit_code_before=$? + + # Wait for two refresh cycles to fire (2 × 10 s + 5 s buffer). + echo " Waiting 25s for two egress refresh cycles..." + sleep 25 + + # Egress must still work after refreshes — the thread must not have + # wiped or corrupted the existing CIDR list. + local exit_code_after=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup one.one.one.one 1.1.1.1 2>&1 \ + || exit_code_after=$? + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$tmpdir" + ensure_data_dir_deleted "$vm_name" + + [[ $exit_code_before -eq 0 ]] && [[ $exit_code_after -eq 0 ]] +} + +test_proxy_flag_routes_pull_through_proxy() { + skip_if_slow && return 0 + + if ! command -v python3 >/dev/null 2>&1; then + echo "SKIP: python3 not available" + return 0 + fi + + local proxy_log proxy_pid proxy_port pull_output pull_exit + proxy_log=$(mktemp) + proxy_port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()') + + # Real CONNECT-tunneling proxy: logs each request line, opens a TCP + # connection to the target host:port, returns 200, and bidirectionally + # relays bytes. Exercises the full path the user hits in production — + # a corporate forward proxy doing CONNECT for TLS image pulls. + python3 - "$proxy_port" "$proxy_log" <<'PYEOF' & +import socketserver, socket, sys, threading +port = int(sys.argv[1]); log_path = sys.argv[2] + +def relay(src, dst): + try: + while True: + data = src.recv(65536) + if not data: + break + dst.sendall(data) + except (BrokenPipeError, ConnectionResetError, OSError): + pass + finally: + try: dst.shutdown(socket.SHUT_WR) + except OSError: pass + +class P(socketserver.StreamRequestHandler): + timeout = 60 + def handle(self): + line = self.rfile.readline().decode('ascii', errors='replace').strip() + with open(log_path, 'a') as f: + f.write(line + "\n"); f.flush() + while True: + h = self.rfile.readline() + if not h or h in (b"\r\n", b"\n"): break + parts = line.split() + if len(parts) < 2 or parts[0].upper() != "CONNECT": + self.wfile.write(b"HTTP/1.1 400 Bad Request\r\n\r\n"); return + host, _, port_s = parts[1].rpartition(":") + try: + target = socket.create_connection((host, int(port_s)), timeout=30) + except Exception: + self.wfile.write(b"HTTP/1.1 502 Bad Gateway\r\n\r\n"); return + self.wfile.write(b"HTTP/1.1 200 Connection Established\r\n\r\n") + self.wfile.flush() + client = self.request + t1 = threading.Thread(target=relay, args=(client, target), daemon=True) + t2 = threading.Thread(target=relay, args=(target, client), daemon=True) + t1.start(); t2.start() + t1.join(); t2.join() + target.close() + +class S(socketserver.ThreadingTCPServer): + allow_reuse_address = True + daemon_threads = True + +S(("127.0.0.1", port), P).serve_forever() +PYEOF + proxy_pid=$! + + # Wait for the proxy to bind (up to ~1s). + local ready=0 i + for i in 1 2 3 4 5 6 7 8 9 10; do + if (echo > /dev/tcp/127.0.0.1/$proxy_port) 2>/dev/null; then + ready=1; break + fi + sleep 0.1 + done + if [[ $ready -ne 1 ]]; then + kill $proxy_pid 2>/dev/null + rm -f "$proxy_log" + echo "FAIL: proxy never bound to 127.0.0.1:$proxy_port" + return 1 + fi + + # Run the pull. With a real tunneling proxy, this must succeed — + # both the manifest fetch and the blob downloads flow through CONNECT. + pull_output=$($SMOLVM machine run --net \ + --proxy "http://127.0.0.1:$proxy_port" \ + --image alpine -- echo ok 2>&1) + pull_exit=$? + + kill $proxy_pid 2>/dev/null + wait $proxy_pid 2>/dev/null + + # Two conditions must hold for the feature to be considered working: + # 1. The pull succeeded (exit 0 + workload printed "ok") + # 2. The proxy log contains a CONNECT for a docker registry, proving + # the request actually went through the proxy (not direct DNS). + local connect_seen=0 + if grep -qiE '^CONNECT[[:space:]]+([a-z0-9.-]*\.)?docker\.io:443' "$proxy_log"; then + connect_seen=1 + fi + + local result=0 + if [[ $pull_exit -ne 0 ]] || [[ "$pull_output" != *"ok"* ]] || [[ $connect_seen -ne 1 ]]; then + result=1 + echo " pull exit code: $pull_exit" + echo " workload output: $(echo "$pull_output" | tail -1)" + echo " CONNECT logged: $connect_seen" + echo " proxy log contents:" + sed 's/^/ /' "$proxy_log" + fi + rm -f "$proxy_log" + return $result +} + +test_grpcio_channel_ready() { + skip_if_slow && return 0 + local output + output=$($SMOLVM machine run --net --mem 4096 --image python:3.12-alpine -- sh -c ' + pip install grpcio > /dev/null 2>&1 + python3 -c " +import os +os.environ[\"GRPC_DNS_RESOLVER\"] = \"native\" +import grpc +ch = grpc.secure_channel(\"google.com:443\", grpc.ssl_channel_credentials()) +grpc.channel_ready_future(ch).result(timeout=10) +print(\"grpcio_channel_ready: PASS\") +" + ' 2>&1) + echo "$output" + [[ "$output" == *"grpcio_channel_ready: PASS"* ]] +} + + +run_test "Network: disabled by default" test_machine_network_disabled_by_default || true +run_test "Network: DNS resolution" test_machine_network_dns_resolution || true +run_test "Network: multiple DNS lookups" test_machine_network_multiple_dns_lookups || true +run_test "Egress: allow-cidr permits matching traffic" test_machine_egress_allow_cidr_permitted || true +run_test "Egress: allow-cidr blocks non-matching traffic" test_machine_egress_allow_cidr_blocked || true +run_test "Egress: --outbound-localhost-only blocks external" test_machine_egress_outbound_localhost_only || true +run_test "Egress: invalid CIDR rejected at create" test_machine_egress_invalid_cidr_rejected || true +run_test "Egress: allow-host permits matching traffic" test_machine_egress_allow_host_permitted || true +run_test "Egress: allow-host blocks non-matching traffic" test_machine_egress_allow_host_blocked || true +run_test "Egress: invalid hostname rejected at create" test_machine_egress_allow_host_invalid_rejected || true +run_test "Egress: host:port syntax rejected" test_machine_egress_allow_host_port_rejected || true +run_test "DNS filter: blocks resolution of non-allowed domains" test_machine_dns_filter_blocks_resolution || true +run_test "DNS filter: TCP/53 allowed domain returns NOERROR" test_dns_filter_tcp_allowed || true +run_test "DNS filter: TCP/53 blocked domain returns NXDOMAIN" test_dns_filter_tcp_blocked || true +run_test "DNS filter: allow-host persists across restart" test_machine_allow_host_persists_across_restart || true +run_test "Smolfile: allow_hosts basic egress permitted/blocked" test_smolfile_allow_hosts_egress_basic || true +run_test "Smolfile: allow_hosts re-resolves stale CIDRs on start (issue #124)" test_smolfile_allow_hosts_stale_cidr_regression || true +run_test "Egress refresh thread: stability across refresh cycles" test_egress_refresh_thread_stability || true +run_test "Proxy: --proxy flag routes image pull through proxy" test_proxy_flag_routes_pull_through_proxy || true +run_test "grpcio: secure channel ready (ilyaterin grpc test)" test_grpcio_channel_ready || true + + +print_summary "Network Tests" diff --git a/tests/test_pack.sh b/tests/test_pack.sh new file mode 100755 index 0000000..e92fb8c --- /dev/null +++ b/tests/test_pack.sh @@ -0,0 +1,1777 @@ +#!/usr/bin/env bash +# +# Pack tests for smolvm. +# +# Tests the `smolvm pack` command and packed binary execution. +# Requires VM environment and sufficient disk space (~500MB for images). +# +# Usage: +# ./tests/test_pack.sh [--quick] +# +# Options: +# --quick Skip slow tests (large image packing, daemon mode) + +source "$(dirname "$0")/common.sh" +init_smolvm + +# Pack tests must not inherit the repo-local libkrun search path from +# common.sh. Packed stubs are expected to start without host-provided libkrun. +if [[ "$(uname -s)" == "Linux" ]]; then + repo_lib_dir="$PROJECT_ROOT/lib/linux-$(uname -m)" + if [[ -n "${LD_LIBRARY_PATH:-}" ]]; then + filtered_ld_library_path="" + IFS=':' read -r -a ld_library_path_entries <<< "$LD_LIBRARY_PATH" + for entry in "${ld_library_path_entries[@]}"; do + [[ "$entry" == "$repo_lib_dir" ]] && continue + if [[ -z "$filtered_ld_library_path" ]]; then + filtered_ld_library_path="$entry" + else + filtered_ld_library_path="${filtered_ld_library_path}:$entry" + fi + done + + if [[ -n "$filtered_ld_library_path" ]]; then + export LD_LIBRARY_PATH="$filtered_ld_library_path" + else + unset LD_LIBRARY_PATH + fi + fi +fi + +# Pre-flight: Kill any existing smolvm processes that might hold database lock +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +QUICK_MODE=false +if [[ "${1:-}" == "--quick" ]]; then + QUICK_MODE=true +fi + +echo "" +echo "==========================================" +echo " smolvm Pack Tests" +echo "==========================================" +echo "" + +# Test output directory (cleaned up at end) +TEST_DIR=$(mktemp -d) +trap "rm -rf '$TEST_DIR'; $SMOLVM pack prune --all 2>/dev/null || true" EXIT + +# ============================================================================= +# Pack Command - Basic Tests +# ============================================================================= + +test_pack_help() { + # Verify pack command exists and shows help + $SMOLVM pack --help 2>&1 | grep -q "Package an OCI image" +} + +test_pack_requires_output() { + # Pack should fail without -o flag + local exit_code=0 + $SMOLVM pack create --image alpine:latest 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] +} + +test_pack_alpine() { + # Pack a minimal image + local output="$TEST_DIR/test-alpine" + local result + result=$($SMOLVM pack create --image alpine:latest -o "$output" 2>&1) + + # Binary should exist + [[ -f "$output" ]] + + # Sidecar should exist + [[ -f "$output.smolmachine" ]] + + # Binary should be executable + [[ -x "$output" ]] +} + +test_pack_with_custom_resources() { + # Pack with custom CPU/memory defaults + local output="$TEST_DIR/test-resources" + $SMOLVM pack create --image alpine:latest -o "$output" --cpus 2 --mem 512 2>&1 + + # Verify manifest has custom values + local info + info=$("$output" info 2>&1) + [[ "$info" == *"CPUs:"*"2"* ]] && [[ "$info" == *"Memory:"*"512"* ]] +} + +test_pack_with_platform() { + # Pack with explicit platform + local output="$TEST_DIR/test-platform" + + # Determine host platform for the test + local host_arch + if [[ "$(uname -m)" == "arm64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then + host_arch="linux/arm64" + else + host_arch="linux/amd64" + fi + + $SMOLVM pack create --image alpine:latest -o "$output" --oci-platform "$host_arch" 2>&1 + + # Binary should exist + [[ -f "$output" ]] + + # Verify manifest shows correct platform + local info + info=$("$output" info 2>&1) + [[ "$info" == *"Platform:"* ]] +} + +# ============================================================================= +# Packed Binary - Info +# ============================================================================= + +test_packed_info() { + local output="$TEST_DIR/test-alpine" + + # Ensure we have a packed binary + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Test info subcommand + local info_output + info_output=$("$output" info 2>&1) + [[ "$info_output" == *"Image:"* ]] && \ + [[ "$info_output" == *"Platform:"* ]] && \ + [[ "$info_output" == *"Checksum:"* ]] || return 1 +} + +test_packed_version() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # --version should succeed and print the smolvm version + local result + local exit_code=0 + result=$("$output" --version 2>&1) || exit_code=$? + [[ $exit_code -eq 0 ]] || return 1 + # Should contain a version-like string (e.g., "packed-binary 0.2.0") + [[ "$result" =~ [0-9]+\.[0-9]+ ]] +} + +test_packed_help() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + local result + result=$("$output" --help 2>&1) || true + [[ "$result" == *"run"* ]] || [[ "$result" == *"start"* ]] +} + +test_sidecar_has_magic() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output.smolmachine" ]]; then + echo "SKIP: No sidecar" + return 0 + fi + + # Check last 64 bytes (footer) for SMOLPACK magic + local magic + magic=$(tail -c 64 "$output.smolmachine" | head -c 8 2>/dev/null) || true + [[ "$magic" == "SMOLPACK" ]] +} + +test_binary_is_clean_macho() { + if [[ "$(uname)" != "Darwin" ]]; then + return 0 + fi + + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output" ]]; then + echo "SKIP: No packed binary" + return 0 + fi + + local file_result + file_result=$(file "$output" 2>&1) || true + [[ "$file_result" == *"Mach-O"* ]] +} + +test_sidecar_has_no_libs() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output.smolmachine" ]]; then + echo "SKIP: No sidecar" + return 0 + fi + + # Extract sidecar and verify no lib/ directory (V3: libs are in stub) + local tmpdir + tmpdir=$(mktemp -d) + # Sidecar is: [assets.tar.zst][manifest json][64-byte footer] + # Read footer to get assets_size, then extract just the tar.zst + # Simpler: just list the tar contents and check for lib/ + local footer_size=64 + local file_size + file_size=$(stat -f%z "$output.smolmachine" 2>/dev/null || stat -c%s "$output.smolmachine" 2>/dev/null) + + # The assets tar.zst starts at offset 0 in the sidecar + # Check that decompressed tar has no lib/ entries + if command -v zstd >/dev/null 2>&1; then + # Use head to get just the compressed portion (before manifest+footer) + # This is approximate but sufficient — if lib/ is in the tar, tar -t will find it + zstd -d < "$output.smolmachine" 2>/dev/null | tar -t 2>/dev/null | grep -q "^lib/" && return 1 + else + echo "SKIP: zstd not installed" + fi + + rm -rf "$tmpdir" + return 0 +} + +test_stub_has_libs_footer() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output" ]]; then + echo "SKIP: No packed binary" + return 0 + fi + + # Check last 32 bytes for SMOLLIBS magic + local magic + magic=$(tail -c 32 "$output" | head -c 8 2>/dev/null) || true + [[ "$magic" == "SMOLLIBS" ]] +} + +# ============================================================================= +# Packed Binary - Library Compatibility +# ============================================================================= + +test_pack_bundled_libkrun_has_required_symbols() { + local output="$TEST_DIR/test-alpine" + + # Ensure we have a packed binary + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Boot the VM briefly to trigger lib extraction, capture debug output + local debug_output + debug_output=$(run_with_timeout 60 "$output" --debug run -- true 2>&1) || true + + # Extract the lib_dir from debug output (e.g., "lib_dir=/path/to/lib") + local lib_dir + lib_dir=$(echo "$debug_output" | grep -o 'lib_dir=[^ ]*' | head -1 | cut -d= -f2) + + if [[ -z "$lib_dir" ]]; then + echo "FAIL: could not determine lib_dir from --debug output" + return 1 + fi + + local libkrun="$lib_dir/libkrun.dylib" + if [[ "$(uname)" != "Darwin" ]]; then + libkrun="$lib_dir/libkrun.so" + fi + + if [[ ! -f "$libkrun" ]]; then + echo "FAIL: bundled libkrun not found at $libkrun" + return 1 + fi + + # Verify all required symbols exist in the bundled library. + # This catches the bug where an older system libkrun gets bundled + # instead of the one smolvm was built against. + # + # Symbol inspection is platform-specific here: + # - macOS uses Mach-O, where `nm -gU` lists external symbols and C exports + # appear with a leading underscore (for example `_krun_create_ctx`) + # - Linux uses ELF, where `nm -D --defined-only` lists dynamic exports and + # the same symbol appears without the underscore (`krun_create_ctx`) + local symbols nm_prefix + if [[ "$(uname)" == "Darwin" ]]; then + symbols=$(nm -gU "$libkrun" 2>/dev/null) || { + echo "FAIL: nm failed on $libkrun" + return 1 + } + nm_prefix="_" + else + symbols=$(nm -D --defined-only "$libkrun" 2>/dev/null) || { + echo "FAIL: nm failed on $libkrun" + return 1 + } + nm_prefix="" + fi + + local missing=0 + for sym in krun_create_ctx krun_add_disk2 krun_add_vsock_port2 krun_start_enter; do + if ! echo "$symbols" | grep -q "${nm_prefix}${sym}$"; then + echo "FAIL: bundled libkrun missing required symbol: $sym" + missing=1 + fi + done + + [[ $missing -eq 0 ]] +} + +test_pack_uses_loaded_libkrun() { + # Verify the packer prefers dladdr over directory search + local output="$TEST_DIR/test-dladdr" + local pack_output + pack_output=$(RUST_LOG=debug $SMOLVM pack create --image alpine:latest -o "$output" 2>&1) + + # Debug log should show "using libkrun from running process" + if echo "$pack_output" | grep -q "using libkrun from running process"; then + return 0 + else + echo "WARN: dladdr path not used — falling back to directory search" + # Not a hard failure: dladdr may not work in all link configurations. + # The symbol check test above is the real safety net. + return 0 + fi +} + +# ============================================================================= +# Packed Binary - Ephemeral Execution (Requires VM) +# ============================================================================= + +test_packed_run_echo() { + local output="$TEST_DIR/test-alpine" + + # Ensure we have a packed binary with sidecar + if [[ ! -f "$output" ]] || [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Run with 60s timeout to prevent indefinite hangs + local result + result=$(run_with_timeout 60 "$output" run -- echo "pack-test-marker-12345" 2>&1) + local exit_code=$? + + if [[ $exit_code -eq 124 ]]; then + echo "TIMEOUT: packed binary hung" + return 1 + fi + + [[ "$result" == *"pack-test-marker-12345"* ]] +} + +test_packed_exit_code() { + local output="$TEST_DIR/test-alpine" + + # Ensure we have a packed binary + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Exit code 0 (with timeout) + run_with_timeout 60 "$output" run -- sh -c "exit 0" 2>&1 + local exit_zero=$? + [[ $exit_zero -eq 124 ]] && { echo "TIMEOUT"; return 1; } + + # Exit code 42 (with timeout) + local exit_42=0 + run_with_timeout 60 "$output" run -- sh -c "exit 42" 2>&1 || exit_42=$? + [[ $exit_42 -eq 124 ]] && { echo "TIMEOUT"; return 1; } + + [[ $exit_zero -eq 0 ]] && [[ $exit_42 -eq 42 ]] +} + +test_packed_env_var() { + local output="$TEST_DIR/test-alpine" + + # Ensure we have a packed binary + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + local result + result=$(run_with_timeout 60 "$output" run -e TEST_VAR=hello_pack -- sh -c 'echo $TEST_VAR' 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT"; return 1; } + [[ "$result" == *"hello_pack"* ]] +} + +test_packed_workdir() { + local output="$TEST_DIR/test-alpine" + + # Ensure we have a packed binary + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + local result + result=$(run_with_timeout 60 "$output" run -w /tmp -- pwd 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT"; return 1; } + [[ "$result" == *"/tmp"* ]] +} + +# ============================================================================= +# Sidecar File Tests +# ============================================================================= + +test_sidecar_required() { + local output="$TEST_DIR/test-sidecar" + + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Remove sidecar + rm -f "$output.smolmachine" + + # Binary should fail without sidecar + local exit_code=0 + "$output" info 2>&1 || exit_code=$? + + # Restore sidecar for other tests + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 >/dev/null + + [[ $exit_code -ne 0 ]] +} + +# ============================================================================= +# Single-File Mode Tests (--single-file) +# ============================================================================= + +test_single_file_pack() { + # Pack with --single-file flag + local output="$TEST_DIR/test-single-file" + $SMOLVM pack create --image alpine:latest -o "$output" --single-file 2>&1 + + # Binary should exist and be executable + [[ -f "$output" ]] || return 1 + [[ -x "$output" ]] || return 1 + + # Sidecar should NOT exist + [[ ! -f "$output.smolmachine" ]] || return 1 + + # Should work when moved (no sidecar needed) + local new_dir="$TEST_DIR/standalone-test" + mkdir -p "$new_dir" + cp "$output" "$new_dir/myapp" + local info_output + info_output=$("$new_dir/myapp" info 2>&1) + [[ "$info_output" == *"Image:"* ]] +} + +test_single_file_run_echo() { + local output="$TEST_DIR/test-single-file" + + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" --single-file 2>&1 + fi + + # Run with 60s timeout to prevent indefinite hangs + local result + result=$(run_with_timeout 60 "$output" run -- echo "single-file-test-marker" 2>&1) + local exit_code=$? + + if [[ $exit_code -eq 124 ]]; then + echo "TIMEOUT: packed binary hung" + return 1 + fi + + [[ "$result" == *"single-file-test-marker"* ]] +} + +# ============================================================================= +# pack run subcommand - Basic Tests +# ============================================================================= + +test_pack_run_help() { + # Verify pack run subcommand exists and shows help + $SMOLVM pack run --help 2>&1 | grep -q "Run a VM from a packed" +} + +test_pack_run_info() { + local output="$TEST_DIR/test-alpine" + + # Ensure we have a packed binary with sidecar + if [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Test --info via pack run + local info_output + info_output=$($SMOLVM pack run --sidecar "$output.smolmachine" --info 2>&1) + [[ "$info_output" == *"Image:"* ]] && \ + [[ "$info_output" == *"Platform:"* ]] && \ + [[ "$info_output" == *"Checksum:"* ]] || return 1 +} + +test_pack_run_info_no_sidecar() { + # Should error clearly when sidecar doesn't exist + local exit_code=0 + $SMOLVM pack run --sidecar /tmp/nonexistent-file.smolmachine --info 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] +} + +test_pack_run_auto_detect() { + # Test auto-detection of .smolmachine file in current directory + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Create a temp dir with a single .smolmachine file + local detect_dir="$TEST_DIR/auto-detect" + mkdir -p "$detect_dir" + cp "$output.smolmachine" "$detect_dir/myapp.smolmachine" + + # pack run --info from that directory should auto-detect + local info_output + info_output=$(cd "$detect_dir" && $SMOLVM pack run --info 2>&1) + [[ "$info_output" == *"Image:"* ]] +} + +test_pack_run_auto_detect_ambiguous() { + # Should error when multiple .smolmachine files exist and no --sidecar given + local detect_dir="$TEST_DIR/multi-detect" + mkdir -p "$detect_dir" + + # Create two dummy .smolmachine files (just need them to exist for detection) + touch "$detect_dir/app1.smolmachine" + touch "$detect_dir/app2.smolmachine" + + local exit_code=0 + (cd "$detect_dir" && $SMOLVM pack run --info 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] +} + +# ============================================================================= +# pack run subcommand - Execution Tests (Requires VM) +# ============================================================================= + +test_pack_run_resource_override() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Verify resource override flags are accepted (boot with custom resources) + # We use --debug to see the config, and run a quick command + local result + result=$(run_with_timeout 60 $SMOLVM pack run --sidecar "$output.smolmachine" --cpus 2 --mem 512 --debug -- echo "resource-test" 2>&1) + local exit_code=$? + + [[ $exit_code -eq 124 ]] && { echo "TIMEOUT"; return 1; } + + # Should contain the debug output showing the resource overrides + [[ "$result" == *"cpus=2"* ]] && [[ "$result" == *"mem=512"* ]] && \ + [[ "$result" == *"resource-test"* ]] +} + +test_pack_run_force_extract() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Run with --force-extract and --debug to verify re-extraction + local result + result=$(run_with_timeout 60 $SMOLVM pack run --sidecar "$output.smolmachine" --force-extract --debug -- echo "re-extracted" 2>&1) + local exit_code=$? + + [[ $exit_code -eq 124 ]] && { echo "TIMEOUT"; return 1; } + + # Debug output should show extraction happening + [[ "$result" == *"extract"* ]] && [[ "$result" == *"re-extracted"* ]] +} + +test_pack_run_cached_fast() { + # Second run should use cached assets (no extraction) + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # First run ensures cache exists + run_with_timeout 60 $SMOLVM pack run --sidecar "$output.smolmachine" -- true 2>&1 || true + + # Second run with --debug should show "using cached assets" + local result + result=$(run_with_timeout 60 $SMOLVM pack run --sidecar "$output.smolmachine" --debug -- echo "cached-run" 2>&1) + local exit_code=$? + + [[ $exit_code -eq 124 ]] && { echo "TIMEOUT"; return 1; } + [[ "$result" == *"cached"* ]] && [[ "$result" == *"cached-run"* ]] +} + +test_pack_run_python() { + if [[ "$QUICK_MODE" == "true" ]]; then + echo "SKIP: --quick mode" + return 0 + fi + + local output="$TEST_DIR/test-python" + + if [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image python:3.12-slim -o "$output" 2>&1 + fi + + local result + result=$(run_with_timeout 90 $SMOLVM pack run --sidecar "$output.smolmachine" -- python -c "print('Hello from pack run Python')" 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT"; return 1; } + [[ "$result" == *"Hello from pack run Python"* ]] +} + +# ============================================================================= +# Pack --from-vm Tests (Requires VM + Network) +# ============================================================================= + +# Shared VM name for --from-vm tests (cleaned up in test_from_vm_cleanup) +FROM_VM_NAME="pack-from-vm-test-$$" +FROM_VM_OUTPUT="$TEST_DIR/test-from-vm" + +test_from_vm_setup() { + # Create a named VM with network, install a package, then stop it + $SMOLVM machine stop --name "$FROM_VM_NAME" 2>/dev/null || true + $SMOLVM machine delete --name "$FROM_VM_NAME" -f 2>/dev/null || true + + $SMOLVM machine create --name "$FROM_VM_NAME" --net 2>&1 || return 1 + $SMOLVM machine start --name "$FROM_VM_NAME" 2>&1 || { + $SMOLVM machine delete --name "$FROM_VM_NAME" -f 2>/dev/null + return 1 + } + + # Install curl so we can verify it persists into the packed binary. + # apk add may exit non-zero due to busybox trigger errors (busybox was + # baked into the rootfs, not installed from a repo, so re-extraction + # fails). The package itself installs fine — verify with `which curl`. + $SMOLVM machine exec --name "$FROM_VM_NAME" -- apk add --no-cache curl 2>&1 || true + + # Verify curl was installed + local which_output + which_output=$($SMOLVM machine exec --name "$FROM_VM_NAME" -- which curl 2>&1) || { + $SMOLVM machine stop --name "$FROM_VM_NAME" 2>/dev/null || true + $SMOLVM machine delete --name "$FROM_VM_NAME" -f 2>/dev/null || true + return 1 + } + [[ "$which_output" == *"/usr/bin/curl"* ]] || { + $SMOLVM machine stop --name "$FROM_VM_NAME" 2>/dev/null || true + $SMOLVM machine delete --name "$FROM_VM_NAME" -f 2>/dev/null || true + return 1 + } + + # Stop the VM (pack requires it to be stopped) + $SMOLVM machine stop --name "$FROM_VM_NAME" 2>&1 +} + +test_from_vm_rejects_running() { + # --from-vm should fail if the VM is still running + local vm_name="pack-running-test-$$" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + $SMOLVM machine create --name "$vm_name" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + + local exit_code=0 + $SMOLVM pack create --from-vm "$vm_name" -o "$TEST_DIR/should-fail" 2>&1 || exit_code=$? + + # Clean up + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + [[ $exit_code -ne 0 ]] +} + +test_from_vm_pack() { + # Pack the stopped VM snapshot + $SMOLVM pack create --from-vm "$FROM_VM_NAME" -o "$FROM_VM_OUTPUT" 2>&1 || return 1 + + # Binary and sidecar should exist + [[ -f "$FROM_VM_OUTPUT" ]] || return 1 + [[ -f "$FROM_VM_OUTPUT.smolmachine" ]] || return 1 + [[ -x "$FROM_VM_OUTPUT" ]] || return 1 +} + +test_from_vm_run_finds_installed_package() { + if [[ ! -f "$FROM_VM_OUTPUT" ]]; then + echo "SKIP: no packed binary (setup or pack failed)" + return 1 + fi + + # The packed binary should have curl from the VM snapshot + local result + result=$(run_with_timeout 60 "$FROM_VM_OUTPUT" run -- which curl 2>&1) + local exit_code=$? + + [[ $exit_code -eq 124 ]] && { echo "TIMEOUT"; return 1; } + [[ "$result" == *"/usr/bin/curl"* ]] +} + +test_from_vm_cleanup() { + $SMOLVM machine stop --name "$FROM_VM_NAME" 2>/dev/null || true + $SMOLVM machine delete --name "$FROM_VM_NAME" -f 2>/dev/null || true + rm -f "$FROM_VM_OUTPUT" "$FROM_VM_OUTPUT.smolmachine" + return 0 +} + +# End-to-end test: --from-vm on an IMAGE-BASED VM captures container overlay. +# +# BUG-17 regression: `pack create --from-vm` for image-based VMs only captured +# the base OCI layers, missing packages installed via apk/apt. The container +# overlay (upper dir on the storage disk) wasn't exported. This test verifies +# the full round-trip: +# 1. Create image-based VM → install package → stop +# 2. Pack with --from-vm (must export overlay as additional layer) +# 3. Run packed binary — installed package must be present +# 4. Create machine from .smolmachine — installed package must be present +# 5. Stop/start machine — package persists across restart +test_from_vm_image_overlay() { + local vm_name="from-vm-img-$$" + local pack_output="$TEST_DIR/from-vm-img-pack" + local machine_name="from-vm-img-machine-$$" + + # Cleanup any leftovers + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + $SMOLVM machine stop --name "$machine_name" 2>/dev/null || true + $SMOLVM machine delete --name "$machine_name" -f 2>/dev/null || true + rm -f "$pack_output" "$pack_output.smolmachine" + + # 1. Create image-based VM, install curl, verify, stop + echo " Step 1: Create image-based VM and install curl..." + $SMOLVM machine create --name "$vm_name" --image alpine:latest --net 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + $SMOLVM machine exec --name "$vm_name" -- apk add --no-cache curl 2>&1 || true + local which_result + which_result=$($SMOLVM machine exec --name "$vm_name" -- which curl 2>&1) + [[ "$which_result" == *"/usr/bin/curl"* ]] || { + echo "FAIL: curl not installed in source VM" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + $SMOLVM machine stop --name "$vm_name" 2>&1 || return 1 + + # 2. Pack with --from-vm (this must export the container overlay) + echo " Step 2: Pack image-based VM with --from-vm..." + $SMOLVM pack create --from-vm "$vm_name" -o "$pack_output" 2>&1 || { + echo "FAIL: pack --from-vm failed" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + [[ -f "$pack_output.smolmachine" ]] || { + echo "FAIL: no .smolmachine sidecar produced" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + # 3. Run packed binary — curl must be present + echo " Step 3: Verify packed binary has curl..." + local run_result + run_result=$(run_with_timeout 60 $SMOLVM pack run --sidecar "$pack_output.smolmachine" -- which curl 2>&1) + [[ "$run_result" == *"/usr/bin/curl"* ]] || { + echo "FAIL: curl not found in packed binary (got: $run_result)" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f "$pack_output" "$pack_output.smolmachine"; return 1 + } + + # 4. Create machine from .smolmachine — curl must be present + echo " Step 4: Create machine from .smolmachine and verify curl..." + $SMOLVM machine create --name "$machine_name" --from "$pack_output.smolmachine" --net 2>&1 || { + echo "FAIL: machine create --from failed" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f "$pack_output" "$pack_output.smolmachine"; return 1 + } + $SMOLVM machine start --name "$machine_name" 2>&1 || { + echo "FAIL: machine start failed" + $SMOLVM machine delete --name "$machine_name" -f 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f "$pack_output" "$pack_output.smolmachine"; return 1 + } + local exec_result + exec_result=$($SMOLVM machine exec --name "$machine_name" -- which curl 2>&1) + [[ "$exec_result" == *"/usr/bin/curl"* ]] || { + echo "FAIL: curl not found in machine from .smolmachine (got: $exec_result)" + $SMOLVM machine stop --name "$machine_name" 2>/dev/null + $SMOLVM machine delete --name "$machine_name" -f 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f "$pack_output" "$pack_output.smolmachine"; return 1 + } + + # 5. Stop/start — curl persists + echo " Step 5: Verify persistence across restart..." + $SMOLVM machine stop --name "$machine_name" 2>&1 || true + $SMOLVM machine start --name "$machine_name" 2>&1 || { + echo "FAIL: restart failed" + $SMOLVM machine delete --name "$machine_name" -f 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f "$pack_output" "$pack_output.smolmachine"; return 1 + } + exec_result=$($SMOLVM machine exec --name "$machine_name" -- which curl 2>&1) + [[ "$exec_result" == *"/usr/bin/curl"* ]] || { + echo "FAIL: curl not found after restart (got: $exec_result)" + $SMOLVM machine stop --name "$machine_name" 2>/dev/null + $SMOLVM machine delete --name "$machine_name" -f 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -f "$pack_output" "$pack_output.smolmachine"; return 1 + } + + # Cleanup + $SMOLVM machine stop --name "$machine_name" 2>/dev/null || true + $SMOLVM machine delete --name "$machine_name" -f 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -f "$pack_output" "$pack_output.smolmachine" +} + +# Regression: default --from-vm repack preserves imported artifact layers. +# +# A machine created from a .smolmachine has two kinds of container state: +# imported layers from the source artifact, and the current writable overlay. +# Repacking it without --rebase-from-image must keep both. +test_from_vm_imported_layers_preserved_on_repack() { + local vm_a="from-vm-nested-a-$$" + local vm_b="from-vm-nested-b-$$" + local vm_c="from-vm-nested-c-$$" + local pack_a="$TEST_DIR/from-vm-nested-a" + local pack_b="$TEST_DIR/from-vm-nested-b" + + cleanup_from_vm_imported_layers_preserved() { + $SMOLVM machine stop --name "$vm_c" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_c" -f 2>/dev/null || true + $SMOLVM machine stop --name "$vm_b" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_b" -f 2>/dev/null || true + $SMOLVM machine stop --name "$vm_a" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_a" -f 2>/dev/null || true + rm -f "$pack_a" "$pack_a.smolmachine" "$pack_b" "$pack_b.smolmachine" + } + + # Cleanup any leftovers + cleanup_from_vm_imported_layers_preserved + + echo " Step 1: Create source image VM with artifact marker..." + $SMOLVM machine create --name "$vm_a" --image alpine:latest --net 2>&1 || { + cleanup_from_vm_imported_layers_preserved; return 1 + } + $SMOLVM machine start --name "$vm_a" 2>&1 || { + cleanup_from_vm_imported_layers_preserved; return 1 + } + $SMOLVM machine exec --name "$vm_a" -- sh -c "echo inherited > /artifact-origin.txt" 2>&1 || { + echo "FAIL: could not write source artifact marker" + cleanup_from_vm_imported_layers_preserved; return 1 + } + $SMOLVM machine stop --name "$vm_a" 2>&1 || { + cleanup_from_vm_imported_layers_preserved; return 1 + } + + echo " Step 2: Pack source VM..." + $SMOLVM pack create --from-vm "$vm_a" -o "$pack_a" 2>&1 || { + echo "FAIL: pack source VM failed" + cleanup_from_vm_imported_layers_preserved; return 1 + } + [[ -f "$pack_a.smolmachine" ]] || { + echo "FAIL: no source .smolmachine sidecar produced" + cleanup_from_vm_imported_layers_preserved; return 1 + } + + echo " Step 3: Create nested VM and verify inherited marker..." + $SMOLVM machine create --name "$vm_b" --from "$pack_a.smolmachine" --net 2>&1 || { + echo "FAIL: nested machine create --from failed" + cleanup_from_vm_imported_layers_preserved; return 1 + } + $SMOLVM machine start --name "$vm_b" 2>&1 || { + echo "FAIL: nested machine start failed" + cleanup_from_vm_imported_layers_preserved; return 1 + } + local inherited_result + inherited_result=$($SMOLVM machine exec --name "$vm_b" -- cat /artifact-origin.txt 2>&1) + [[ "$inherited_result" == *"inherited"* ]] || { + echo "FAIL: inherited marker missing in nested VM (got: $inherited_result)" + cleanup_from_vm_imported_layers_preserved; return 1 + } + $SMOLVM machine exec --name "$vm_b" -- sh -c "echo nested > /nested-mutation.txt" 2>&1 || { + echo "FAIL: could not write nested mutation marker" + cleanup_from_vm_imported_layers_preserved; return 1 + } + $SMOLVM machine stop --name "$vm_b" 2>&1 || { + cleanup_from_vm_imported_layers_preserved; return 1 + } + + echo " Step 4: Repack nested VM without rebase..." + $SMOLVM pack create --from-vm "$vm_b" -o "$pack_b" 2>&1 || { + echo "FAIL: repack nested VM failed" + cleanup_from_vm_imported_layers_preserved; return 1 + } + [[ -f "$pack_b.smolmachine" ]] || { + echo "FAIL: no nested .smolmachine sidecar produced" + cleanup_from_vm_imported_layers_preserved; return 1 + } + + echo " Step 5: Create final VM and verify both markers..." + $SMOLVM machine create --name "$vm_c" --from "$pack_b.smolmachine" --net 2>&1 || { + echo "FAIL: final machine create --from failed" + cleanup_from_vm_imported_layers_preserved; return 1 + } + $SMOLVM machine start --name "$vm_c" 2>&1 || { + echo "FAIL: final machine start failed" + cleanup_from_vm_imported_layers_preserved; return 1 + } + inherited_result=$($SMOLVM machine exec --name "$vm_c" -- cat /artifact-origin.txt 2>&1) + [[ "$inherited_result" == *"inherited"* ]] || { + echo "FAIL: inherited marker missing after repack (got: $inherited_result)" + cleanup_from_vm_imported_layers_preserved; return 1 + } + local nested_result + nested_result=$($SMOLVM machine exec --name "$vm_c" -- cat /nested-mutation.txt 2>&1) + [[ "$nested_result" == *"nested"* ]] || { + echo "FAIL: nested marker missing after repack (got: $nested_result)" + cleanup_from_vm_imported_layers_preserved; return 1 + } + + cleanup_from_vm_imported_layers_preserved +} + +# Regression: --rebase-from-image intentionally rebuilds lower image layers and +# drops imported artifact layers, while keeping the current writable overlay. +test_from_vm_rebase_from_image_drops_imported_layers() { + local vm_a="from-vm-rebase-a-$$" + local vm_b="from-vm-rebase-b-$$" + local vm_c="from-vm-rebase-c-$$" + local pack_a="$TEST_DIR/from-vm-rebase-a" + local pack_b="$TEST_DIR/from-vm-rebase-b" + + cleanup_from_vm_rebase_layers() { + $SMOLVM machine stop --name "$vm_c" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_c" -f 2>/dev/null || true + $SMOLVM machine stop --name "$vm_b" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_b" -f 2>/dev/null || true + $SMOLVM machine stop --name "$vm_a" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_a" -f 2>/dev/null || true + rm -f "$pack_a" "$pack_a.smolmachine" "$pack_b" "$pack_b.smolmachine" + } + + # Cleanup any leftovers + cleanup_from_vm_rebase_layers + + echo " Step 1: Create source image VM with artifact marker..." + $SMOLVM machine create --name "$vm_a" --image alpine:latest --net 2>&1 || { + cleanup_from_vm_rebase_layers; return 1 + } + $SMOLVM machine start --name "$vm_a" 2>&1 || { + cleanup_from_vm_rebase_layers; return 1 + } + $SMOLVM machine exec --name "$vm_a" -- sh -c "echo inherited > /artifact-origin.txt" 2>&1 || { + echo "FAIL: could not write source artifact marker" + cleanup_from_vm_rebase_layers; return 1 + } + $SMOLVM machine stop --name "$vm_a" 2>&1 || { + cleanup_from_vm_rebase_layers; return 1 + } + + echo " Step 2: Pack source VM..." + $SMOLVM pack create --from-vm "$vm_a" -o "$pack_a" 2>&1 || { + echo "FAIL: pack source VM failed" + cleanup_from_vm_rebase_layers; return 1 + } + [[ -f "$pack_a.smolmachine" ]] || { + echo "FAIL: no source .smolmachine sidecar produced" + cleanup_from_vm_rebase_layers; return 1 + } + + echo " Step 3: Create nested VM and write mutation marker..." + $SMOLVM machine create --name "$vm_b" --from "$pack_a.smolmachine" --net 2>&1 || { + echo "FAIL: nested machine create --from failed" + cleanup_from_vm_rebase_layers; return 1 + } + $SMOLVM machine start --name "$vm_b" 2>&1 || { + echo "FAIL: nested machine start failed" + cleanup_from_vm_rebase_layers; return 1 + } + local inherited_result + inherited_result=$($SMOLVM machine exec --name "$vm_b" -- cat /artifact-origin.txt 2>&1) + [[ "$inherited_result" == *"inherited"* ]] || { + echo "FAIL: inherited marker missing in nested VM (got: $inherited_result)" + cleanup_from_vm_rebase_layers; return 1 + } + $SMOLVM machine exec --name "$vm_b" -- sh -c "echo nested > /nested-mutation.txt" 2>&1 || { + echo "FAIL: could not write nested mutation marker" + cleanup_from_vm_rebase_layers; return 1 + } + $SMOLVM machine stop --name "$vm_b" 2>&1 || { + cleanup_from_vm_rebase_layers; return 1 + } + + echo " Step 4: Repack nested VM with --rebase-from-image..." + $SMOLVM pack create --from-vm "$vm_b" --rebase-from-image -o "$pack_b" 2>&1 || { + echo "FAIL: rebase repack nested VM failed" + cleanup_from_vm_rebase_layers; return 1 + } + [[ -f "$pack_b.smolmachine" ]] || { + echo "FAIL: no rebased .smolmachine sidecar produced" + cleanup_from_vm_rebase_layers; return 1 + } + + echo " Step 5: Create final VM and verify rebase semantics..." + $SMOLVM machine create --name "$vm_c" --from "$pack_b.smolmachine" --net 2>&1 || { + echo "FAIL: final machine create --from failed" + cleanup_from_vm_rebase_layers; return 1 + } + $SMOLVM machine start --name "$vm_c" 2>&1 || { + echo "FAIL: final machine start failed" + cleanup_from_vm_rebase_layers; return 1 + } + local presence_result + presence_result=$($SMOLVM machine exec --name "$vm_c" -- sh -c 'if [ -e /artifact-origin.txt ]; then echo PRESENT; else echo ABSENT; fi' 2>&1) + [[ "$presence_result" == *"ABSENT"* ]] || { + echo "FAIL: inherited marker present after --rebase-from-image (got: $presence_result)" + cleanup_from_vm_rebase_layers; return 1 + } + local nested_result + nested_result=$($SMOLVM machine exec --name "$vm_c" -- cat /nested-mutation.txt 2>&1) + [[ "$nested_result" == *"nested"* ]] || { + echo "FAIL: nested marker missing after --rebase-from-image (got: $nested_result)" + cleanup_from_vm_rebase_layers; return 1 + } + + cleanup_from_vm_rebase_layers +} + +# ============================================================================= +# Debian Pack Roundtrip (Char device entry regression) +# +# Debian's update-alternatives creates Char device entries in /etc/alternatives/ +# in the overlayfs upper layer. These entries caused safe_unpack() to abort +# mid-stream, silently dropping files that appeared after the Char entry in the +# tar. This test verifies the full round-trip: +# 1. Create image-based VM with Debian image +# 2. Install a package that triggers update-alternatives (creates Char entries) +# 3. Write a test file +# 4. Pack the VM +# 5. Create a new machine from the pack +# 6. Verify the test file survived (it appears after Char entries in the tar) +# ============================================================================= + +test_from_vm_debian_roundtrip() { + local vm_name="debian-pack-$$" + local from_name="debian-from-$$" + local pack_output="$TEST_DIR/test-debian-roundtrip" + + # Cleanup any leftovers + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + $SMOLVM machine stop --name "$from_name" 2>/dev/null || true + $SMOLVM machine delete --name "$from_name" -f 2>/dev/null || true + + # 1. Create a Debian-based VM + echo " Step 1: Creating Debian VM..." + $SMOLVM machine create --name "$vm_name" --image python:3.12-slim --net 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + # 2. Install a package that creates Char entries via update-alternatives + echo " Step 2: Installing man-db (creates Char entries in /etc/alternatives/)..." + $SMOLVM machine exec --name "$vm_name" -- \ + bash -c "apt-get update -qq && apt-get install -y -qq man-db >/dev/null 2>&1" || { + echo "FAIL: apt-get install failed" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + # 3. Write a test file (appears late in tar stream — after Char entries) + echo " Step 3: Writing test marker..." + $SMOLVM machine exec --name "$vm_name" -- \ + bash -c "echo 'debian-roundtrip-ok' > /test-marker.txt" + + local content + content=$($SMOLVM machine exec --name "$vm_name" -- cat /test-marker.txt 2>&1) + [[ "$content" == *"debian-roundtrip-ok"* ]] || { + echo "FAIL: test marker not written (got: '$content')" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + # 4. Stop and pack + echo " Step 4: Packing VM..." + $SMOLVM machine stop --name "$vm_name" 2>&1 + $SMOLVM pack create --from-vm "$vm_name" -o "$pack_output" 2>&1 || { + echo "FAIL: pack create --from-vm failed" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + # 5. Create a new machine from the pack + echo " Step 5: Creating machine from pack..." + $SMOLVM machine create --name "$from_name" --from "$pack_output.smolmachine" 2>&1 || { + echo "FAIL: machine create --from failed (Char device extraction bug)" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + $SMOLVM machine start --name "$from_name" 2>&1 || { + echo "FAIL: machine start failed" + $SMOLVM machine delete --name "$from_name" -f 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + # 6. Verify test file survived the roundtrip + echo " Step 6: Verifying test marker survived..." + local result + result=$($SMOLVM machine exec --name "$from_name" -- cat /test-marker.txt 2>&1) || { + echo "FAIL: test marker missing after roundtrip (Char device data loss)" + $SMOLVM machine stop --name "$from_name" 2>/dev/null + $SMOLVM machine delete --name "$from_name" -f 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + [[ "$result" == *"debian-roundtrip-ok"* ]] || { + echo "FAIL: test marker content mismatch (got: '$result')" + $SMOLVM machine stop --name "$from_name" 2>/dev/null + $SMOLVM machine delete --name "$from_name" -f 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + echo " Test marker survived Debian pack/unpack roundtrip" + + # Cleanup + $SMOLVM machine stop --name "$from_name" 2>/dev/null || true + $SMOLVM machine delete --name "$from_name" -f 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -f "$pack_output" "$pack_output.smolmachine" +} + +# Regression: short VM names (1-3 chars) must not panic in pack create --from-vm. +# The overlay layer digest is derived from the VM name via SHA-256, so any length works. +test_from_vm_short_name() { + local vm_name="x" + local pack_output="$TEST_DIR/from-vm-short" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -f "$pack_output" "$pack_output.smolmachine" + + # Create, start, modify, stop + $SMOLVM machine create --name "$vm_name" --image alpine:latest --net 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + $SMOLVM machine exec --name "$vm_name" -- touch /etc/short-name-marker 2>&1 || true + $SMOLVM machine stop --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + # Pack — this is where the panic used to occur + local exit_code=0 + $SMOLVM pack create --from-vm "$vm_name" -o "$pack_output" 2>&1 || exit_code=$? + + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -f "$pack_output" "$pack_output.smolmachine" + + [[ $exit_code -eq 0 ]] || { + echo "FAIL: pack create --from-vm with 1-char name failed (exit $exit_code)" + return 1 + } +} + +# ============================================================================= +# Case-Insensitive Collision Test (macOS regression) +# +# ============================================================================= +# Case-Insensitive Collision Test (macOS regression) +# +# Regression test for macOS case-insensitive APFS. Linux OCI layers may +# contain paths that differ only in case (e.g., "gdebi" script vs "GDebi/" +# directory). On case-insensitive macOS, these collide during host-side layer +# extraction and previously caused a fatal "failed to unpack" error. +# +# This test builds a minimal Docker image with intentional case-conflicting +# paths, pushes it to a local registry, packs it, and verifies the packed +# binary runs successfully. +# ============================================================================= + +test_pack_case_collision() { + if [[ "$(uname)" != "Darwin" ]]; then + echo "SKIP: case-insensitive collision test only relevant on macOS" + return 0 + fi + + # Check if docker is available + if ! command -v docker >/dev/null 2>&1; then + echo "SKIP: docker not installed" + return 0 + fi + + local img_tag="smolvm-case-test:latest" + local registry_port=5051 + local registry_name="smolvm-test-registry-$$" + local registry_img="localhost:${registry_port}/smolvm-case-test:latest" + local output="$TEST_DIR/test-case-collision" + + # 1. Build a minimal image with case-conflicting paths. + # Creates both "mymod" (file) and "MyMod/" (directory) under /usr/share/pkg/ — + # these are identical on case-insensitive APFS. + local dockerfile_dir + dockerfile_dir=$(mktemp -d) + cat > "$dockerfile_dir/Dockerfile" <<'DOCKERFILE' +FROM alpine:latest +RUN mkdir -p /usr/share/pkg/MyMod && \ + echo '#!/bin/sh' > /usr/share/pkg/mymod && \ + chmod +x /usr/share/pkg/mymod && \ + echo 'print("init")' > /usr/share/pkg/MyMod/__init__.py && \ + echo 'print("module")' > /usr/share/pkg/MyMod/Core.py +DOCKERFILE + + echo " Building test image with case-conflicting paths..." + docker build --platform linux/arm64 -t "$img_tag" "$dockerfile_dir" >/dev/null 2>&1 || { + echo "SKIP: docker build failed" + rm -rf "$dockerfile_dir" + return 0 + } + rm -rf "$dockerfile_dir" + + # 2. Start a temporary local registry to push the image to. + echo " Starting temporary registry on port $registry_port..." + docker run -d -p "${registry_port}:5000" --name "$registry_name" registry:2 >/dev/null 2>&1 || { + echo "SKIP: could not start local registry" + return 0 + } + + # Ensure cleanup on exit (registry container + image) + local cleanup_done=false + cleanup_case_test() { + if [[ "$cleanup_done" == "true" ]]; then return; fi + cleanup_done=true + docker stop "$registry_name" >/dev/null 2>&1 || true + docker rm "$registry_name" >/dev/null 2>&1 || true + docker rmi "$img_tag" "$registry_img" >/dev/null 2>&1 || true + } + trap cleanup_case_test RETURN + + # 3. Push to the local registry. + docker tag "$img_tag" "$registry_img" >/dev/null 2>&1 + docker push "$registry_img" >/dev/null 2>&1 || { + echo "SKIP: could not push to local registry" + cleanup_case_test + return 0 + } + + # 4. Pack the image. + echo " Packing image from local registry..." + $SMOLVM pack create --image "$registry_img" -o "$output" 2>&1 || { + echo "FAIL: pack create failed" + cleanup_case_test + return 1 + } + + [[ -f "$output" ]] || { echo "FAIL: packed binary not created"; return 1; } + [[ -f "$output.smolmachine" ]] || { echo "FAIL: sidecar not created"; return 1; } + + # 5. Run the packed binary — this is the critical test. + # Previously this failed with: "failed to unpack .../GDebi" + # Verify BOTH case-conflicting paths exist in the guest filesystem. + # Just proving "echo works" is not enough — we need to confirm the + # image contents are faithful (both mymod file AND MyMod/ directory). + echo " Running packed binary (verifying filesystem fidelity)..." + local result + result=$(run_with_timeout 60 "$output" run -- /bin/sh -c \ + "test -f /usr/share/pkg/mymod && test -d /usr/share/pkg/MyMod && test -f /usr/share/pkg/MyMod/__init__.py && echo 'case-fidelity-ok'" 2>&1) + local exit_code=$? + + [[ $exit_code -eq 124 ]] && { echo "TIMEOUT: packed binary hung"; return 1; } + [[ "$result" == *"case-fidelity-ok"* ]] || { + echo "FAIL: case-conflicting paths not preserved in guest. Output: $result" + return 1 + } +} + +# ============================================================================= +# Error Handling Tests +# ============================================================================= + +test_pack_nonexistent_image() { + local output="$TEST_DIR/test-nonexistent" + local exit_code=0 + $SMOLVM pack create --image nonexistent-image-that-does-not-exist:v999 -o "$output" 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] +} + +# ============================================================================= +# Python Image Test (Larger image, skip in quick mode) +# ============================================================================= + +test_pack_python() { + if [[ "$QUICK_MODE" == "true" ]]; then + echo "SKIP: --quick mode" + return 0 + fi + + local output="$TEST_DIR/test-python" + $SMOLVM pack create --image python:3.12-slim -o "$output" 2>&1 + + [[ -f "$output" ]] && [[ -f "$output.smolmachine" ]] +} + +test_packed_python_run() { + if [[ "$QUICK_MODE" == "true" ]]; then + echo "SKIP: --quick mode" + return 0 + fi + + local output="$TEST_DIR/test-python" + + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image python:3.12-slim -o "$output" 2>&1 + fi + + local result + result=$(run_with_timeout 90 "$output" run -- python -c "print('Hello from packed Python')" 2>&1) + [[ $? -eq 124 ]] && { echo "TIMEOUT"; return 1; } + [[ "$result" == *"Hello from packed Python"* ]] +} + +# ============================================================================= +# Large Image: r-base +# +# Regression test for streaming layer export. r-base:latest is ~9 GB with 6 +# layers. Previously failed with "connection closed" because the agent wrote +# a temp tar file that filled the 20 GB storage disk. The fix pipes tar stdout +# directly to the vsock stream with zero temp files. +# ============================================================================= + +test_pack_rbase() { + if [[ "$QUICK_MODE" == "true" ]]; then + echo "SKIP: --quick mode" + return 0 + fi + + local output="$TEST_DIR/test-rbase" + $SMOLVM pack create --image r-base:latest -o "$output" 2>&1 + + [[ -f "$output" ]] && [[ -f "$output.smolmachine" ]] +} + +test_packed_rbase_run() { + if [[ "$QUICK_MODE" == "true" ]]; then + echo "SKIP: --quick mode" + return 0 + fi + + local output="$TEST_DIR/test-rbase" + + if [[ ! -f "$output" ]]; then + echo "SKIP: no packed binary (pack failed)" + return 1 + fi + + local result + result=$(run_with_timeout 120 "$output" run -- R --version 2>&1) + local exit_code=$? + + [[ $exit_code -eq 124 ]] && { echo "TIMEOUT"; return 1; } + [[ "$result" == *"R version"* ]] || [[ "$result" == *"r ('littler') version"* ]] +} + +test_packed_rbase_auto_storage() { + # Regression test: large images used to fail with "No space left on device" + # because the default 20 GiB storage wasn't enough. The fix auto-sizes the + # storage disk based on image_size in the manifest. --force-extract ensures + # a clean extraction (no cached overlay from a previous --storage 50 run). + if [[ "$QUICK_MODE" == "true" ]]; then + echo "SKIP: --quick mode" + return 0 + fi + + local output="$TEST_DIR/test-rbase" + + if [[ ! -f "$output.smolmachine" ]]; then + echo "SKIP: no sidecar (pack failed)" + return 1 + fi + + local result + result=$(run_with_timeout 120 $SMOLVM pack run --sidecar "$output.smolmachine" --force-extract -- echo "auto-storage-ok" 2>&1) + local exit_code=$? + + [[ $exit_code -eq 124 ]] && { echo "TIMEOUT"; return 1; } + [[ "$result" == *"auto-storage-ok"* ]] +} + +# ============================================================================= +# Multi-Layer First Exec Performance Regression Test +# +# Regression test: images with many layers (e.g., rocker/tidyverse has 20) +# caused a 16-second first exec because overlayfs multi-lowerdir mount fails +# on virtiofs-backed layers. The fallback physically merged all layers on +# every first exec after restart. Fix: pre-merge layers at pack time so the +# packed binary has a single lowerdir that mounts instantly. +# +# Uses python:3.12 (7 layers) — enough to verify multi-layer handling while +# keeping test time reasonable. The threshold is 5 seconds; the old path +# took 10-16s, the fixed path takes <1s. +# ============================================================================= + +test_multi_layer_first_exec_fast() { + local pack_output="$TEST_DIR/test-multilayer" + local vm_name="multilayer-perf-$$" + local MAX_FIRST_EXEC_SECS=5 + + # Pack a multi-layer image (python:3.12 has 7 layers) + if [[ ! -f "$pack_output.smolmachine" ]]; then + $SMOLVM pack create --image python:3.12 -o "$pack_output" 2>&1 || { + echo "SKIP: pack create failed" + return 0 + } + fi + [[ -f "$pack_output.smolmachine" ]] || { echo "SKIP: no sidecar"; return 0; } + + # Create machine from .smolmachine + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + $SMOLVM machine create --name "$vm_name" --from "$pack_output.smolmachine" --net 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + sleep 2 + + # Time the first exec (the critical measurement) + local t_start t_end elapsed + t_start=$(python3 -c 'import time; print(time.time())' 2>/dev/null || date +%s) + local result + result=$($SMOLVM machine exec --name "$vm_name" -- python3 -c "print('fast')" 2>&1) + t_end=$(python3 -c 'import time; print(time.time())' 2>/dev/null || date +%s) + elapsed=$(python3 -c "print(f'{$t_end - $t_start:.1f}')" 2>/dev/null || echo "?") + + echo " First exec: ${elapsed}s (threshold: ${MAX_FIRST_EXEC_SECS}s)" + + # Verify it worked + [[ "$result" == *"fast"* ]] || { + echo "FAIL: exec failed: $result" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + # Verify it was fast + local over_threshold + over_threshold=$(python3 -c "print('yes' if $t_end - $t_start > $MAX_FIRST_EXEC_SECS else 'no')" 2>/dev/null || echo "no") + if [[ "$over_threshold" == "yes" ]]; then + echo "FAIL: first exec took ${elapsed}s (>${MAX_FIRST_EXEC_SECS}s) — multi-layer overlay merge regression?" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + fi + + # Also verify stop/start doesn't regress + $SMOLVM machine stop --name "$vm_name" 2>&1 || true + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + sleep 2 + + t_start=$(python3 -c 'import time; print(time.time())' 2>/dev/null || date +%s) + result=$($SMOLVM machine exec --name "$vm_name" -- python3 -c "print('still-fast')" 2>&1) + t_end=$(python3 -c 'import time; print(time.time())' 2>/dev/null || date +%s) + elapsed=$(python3 -c "print(f'{$t_end - $t_start:.1f}')" 2>/dev/null || echo "?") + + echo " After restart: ${elapsed}s" + + [[ "$result" == *"still-fast"* ]] || { + echo "FAIL: exec after restart failed: $result" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + over_threshold=$(python3 -c "print('yes' if $t_end - $t_start > $MAX_FIRST_EXEC_SECS else 'no')" 2>/dev/null || echo "no") + if [[ "$over_threshold" == "yes" ]]; then + echo "FAIL: exec after restart took ${elapsed}s (>${MAX_FIRST_EXEC_SECS}s)" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + fi + + # Cleanup + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true +} + +# ============================================================================= +# Run Tests +# ============================================================================= + +echo "Running Pack Command Tests..." +echo "" + +run_test "Pack help" test_pack_help || true +run_test "Pack requires output" test_pack_requires_output || true +run_test "Pack alpine" test_pack_alpine || true +run_test "Pack with custom resources" test_pack_with_custom_resources || true +run_test "Pack with --oci-platform" test_pack_with_platform || true + +echo "" +echo "Running Packed Binary Info Tests..." +echo "" + +run_test "Packed --info" test_packed_info || true +run_test "Packed --version" test_packed_version || true +run_test "Packed --help" test_packed_help || true +run_test "Sidecar has SMOLPACK magic" test_sidecar_has_magic || true +run_test "Sidecar has no libs (V3)" test_sidecar_has_no_libs || true +run_test "Stub has SMOLLIBS footer" test_stub_has_libs_footer || true +run_test "Binary is clean Mach-O" test_binary_is_clean_macho || true + +echo "" +echo "Running Library Compatibility Tests..." +echo "" + +run_test "Bundled libkrun has required symbols" test_pack_bundled_libkrun_has_required_symbols || true +run_test "Pack uses loaded libkrun (dladdr)" test_pack_uses_loaded_libkrun || true + +echo "" +echo "Running Sidecar Tests..." +echo "" + +run_test "Sidecar required" test_sidecar_required || true + +echo "" +echo "Running Single-File Mode Tests..." +echo "" + +run_test "Single-file pack" test_single_file_pack || true +run_test "Single-file run echo (requires VM)" test_single_file_run_echo || true + +echo "" +echo "Running Packed Binary Execution Tests (requires VM)..." +echo "" + +run_test "Packed run echo" test_packed_run_echo || true +run_test "Packed exit code" test_packed_exit_code || true +run_test "Packed env variable" test_packed_env_var || true +run_test "Packed workdir" test_packed_workdir || true + +echo "" +echo "Running pack run subcommand Tests..." +echo "" + +run_test "pack run help" test_pack_run_help || true +run_test "pack run --info" test_pack_run_info || true +run_test "pack run --info with missing sidecar" test_pack_run_info_no_sidecar || true +run_test "pack run auto-detect sidecar" test_pack_run_auto_detect || true +run_test "pack run auto-detect ambiguous" test_pack_run_auto_detect_ambiguous || true + +echo "" +echo "Running pack run execution tests (requires VM)..." +echo "" + +run_test "pack run resource override" test_pack_run_resource_override || true +run_test "pack run --force-extract" test_pack_run_force_extract || true +run_test "pack run cached fast" test_pack_run_cached_fast || true + +if [[ "$QUICK_MODE" != "true" ]]; then + echo "" + echo "Running --from-vm Tests (requires VM + network)..." + echo "" + + run_test "from-vm: rejects running VM" test_from_vm_rejects_running || true + run_test "from-vm: setup VM with curl" test_from_vm_setup || true + run_test "from-vm: pack stopped VM" test_from_vm_pack || true + run_test "from-vm: finds installed package" test_from_vm_run_finds_installed_package || true + run_test "from-vm: cleanup" test_from_vm_cleanup || true + + echo "" + echo "Running --from-vm Image-Based Tests (BUG-17 regression)..." + echo "" + + run_test "from-vm-image: container overlay captured" test_from_vm_image_overlay || true + run_test "from-vm-image: imported layers preserved on repack" test_from_vm_imported_layers_preserved_on_repack || true + run_test "from-vm-image: rebase flag drops imported layers" test_from_vm_rebase_from_image_drops_imported_layers || true + + echo "" + echo "Running Debian Pack Roundtrip Tests..." + echo "" + + run_test "from-vm-debian: Char device entries don't break extraction" test_from_vm_debian_roundtrip || true + + echo "" + echo "Running --from-vm Short Name Tests..." + echo "" + + run_test "from-vm: short VM name (1 char) does not panic" test_from_vm_short_name || true +fi + +# ============================================================================= +# Packed Binary - Bare Invocation (no subcommand) +# ============================================================================= + +test_packed_bare_invocation() { + local output="$TEST_DIR/test-alpine" + + # Ensure we have a packed binary + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # ./my-app with no subcommand should run the manifest entrypoint. + # Alpine's default is /bin/sh — without -it it reads stdin, gets EOF, + # and exits 0. That exit-0 proves the VM booted and ran the entrypoint + # instead of printing clap usage (which would exit non-zero). + local exit_code=0 + run_with_timeout 30 "$output" &1 || exit_code=$? + + if [[ $exit_code -eq 124 ]]; then + echo "TIMEOUT: bare invocation hung" + return 1 + fi + + [[ $exit_code -eq 0 ]] +} + +# ============================================================================= +# Packed Binary - Daemon Lifecycle (start/exec/status/stop) +# ============================================================================= + +test_packed_daemon_lifecycle() { + local output="$TEST_DIR/test-alpine" + + # Ensure we have a packed binary + if [[ ! -f "$output" ]] || [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # 1. Start the daemon + local start_result + start_result=$(run_with_timeout 60 "$output" start 2>&1) + local start_exit=$? + [[ $start_exit -eq 124 ]] && { echo "TIMEOUT on start"; return 1; } + [[ "$start_result" == *"Daemon started"* ]] || { echo "Start failed: $start_result"; return 1; } + + # 2. Check status + local status_result + status_result=$("$output" status 2>&1) || true + [[ "$status_result" == *"running"* ]] || { echo "Status failed: $status_result"; "$output" stop 2>/dev/null || true; return 1; } + + # 3. Exec a command + local exec_result + exec_result=$(run_with_timeout 30 "$output" exec -- echo "daemon-exec-marker" 2>&1) + local exec_exit=$? + [[ $exec_exit -eq 124 ]] && { echo "TIMEOUT on exec"; "$output" stop 2>/dev/null || true; return 1; } + [[ "$exec_result" == *"daemon-exec-marker"* ]] || { echo "Exec failed: $exec_result"; "$output" stop 2>/dev/null || true; return 1; } + + # 4. Stop the daemon + local stop_result + stop_result=$("$output" stop 2>&1) || true + [[ "$stop_result" == *"Daemon stopped"* ]] || { echo "Stop failed: $stop_result"; return 1; } + + # 5. Verify stopped + local status_after + status_after=$("$output" status 2>&1) || true + [[ "$status_after" == *"not running"* ]] +} + +test_packed_daemon_already_running() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output" ]] || [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Start the daemon + run_with_timeout 60 "$output" start 2>&1 || { echo "Initial start failed"; return 1; } + + # Starting again should say already running (not error) + local result + result=$("$output" start 2>&1) || true + [[ "$result" == *"already running"* ]] || { "$output" stop 2>/dev/null || true; echo "Expected 'already running': $result"; return 1; } + + # Clean up + "$output" stop 2>/dev/null || true +} + +test_packed_exec_without_daemon() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output" ]] || [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Make sure daemon is not running + "$output" stop 2>/dev/null || true + + # exec without a running daemon should fail with a clear message + local result + local exit_code=0 + result=$("$output" exec -- echo hello 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || return 1 + [[ "$result" == *"not running"* ]] +} + +test_packed_stop_without_daemon() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output" ]] || [[ ! -f "$output.smolmachine" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Make sure daemon is not running + "$output" stop 2>/dev/null || true + + # stop without a running daemon should succeed gracefully + local result + result=$("$output" stop 2>&1) || true + [[ "$result" == *"not running"* ]] || [[ "$result" == *"Daemon stopped"* ]] +} + +test_packed_info_subcommand() { + local output="$TEST_DIR/test-alpine" + + if [[ ! -f "$output" ]]; then + $SMOLVM pack create --image alpine:latest -o "$output" 2>&1 + fi + + # Test info as subcommand + local info_output + info_output=$("$output" info 2>&1) + [[ "$info_output" == *"Image:"* ]] && \ + [[ "$info_output" == *"Platform:"* ]] && \ + [[ "$info_output" == *"Checksum:"* ]] || return 1 +} + +echo "" +echo "Running Packed Binary Bare Invocation Tests (requires VM)..." +echo "" + +run_test "Packed bare invocation (no subcommand)" test_packed_bare_invocation || true + +echo "" +echo "Running Packed Daemon Lifecycle Tests (requires VM)..." +echo "" + +run_test "Daemon lifecycle: start -> exec -> status -> stop" test_packed_daemon_lifecycle || true +run_test "Daemon already running" test_packed_daemon_already_running || true +run_test "Exec without daemon" test_packed_exec_without_daemon || true +run_test "Stop without daemon" test_packed_stop_without_daemon || true +run_test "Info subcommand" test_packed_info_subcommand || true + +echo "" +echo "Running Error Handling Tests..." +echo "" + +run_test "Pack nonexistent image" test_pack_nonexistent_image || true + +echo "" +echo "Running Case-Insensitive Collision Tests (macOS)..." +echo "" + +run_test "Pack image with case-conflicting paths" test_pack_case_collision || true + +if [[ "$QUICK_MODE" != "true" ]]; then + echo "" + echo "Running Large Image Tests..." + echo "" + + run_test "Pack Python image" test_pack_python || true + run_test "Packed Python run" test_packed_python_run || true + run_test "pack run Python" test_pack_run_python || true + run_test "Pack r-base (large image, streaming export)" test_pack_rbase || true + run_test "Packed r-base run" test_packed_rbase_run || true + run_test "Packed r-base auto-sized storage (force-extract)" test_packed_rbase_auto_storage || true + run_test "First exec instant on multi-layer .smolmachine" test_multi_layer_first_exec_fast || true +fi + +print_summary "Pack Tests" diff --git a/tests/test_pack_registry.sh b/tests/test_pack_registry.sh new file mode 100755 index 0000000..c1b3606 --- /dev/null +++ b/tests/test_pack_registry.sh @@ -0,0 +1,279 @@ +#!/usr/bin/env bash +# +# Registry push/pull integration tests for smolvm pack. +# +# Tests smolvm pack push, pack pull, and pack inspect against a local +# registry:2 container, verifying the full OCI distribution spec roundtrip. +# +# Requires: +# - Docker (skip gracefully if not available) +# - A working smolvm binary (for pack create + push + pull) +# +# Usage: +# ./tests/test_pack_registry.sh + +source "$(dirname "$0")/common.sh" +init_smolvm + +# --------------------------------------------------------------------------- +# Registry helpers +# --------------------------------------------------------------------------- + +REGISTRY_PORT=5099 # port unlikely to conflict +REGISTRY_HOST="localhost:${REGISTRY_PORT}" +REGISTRY_CONTAINER="smolvm-test-reg-$$" +TEST_DIR=$(mktemp -d) + +# Start cleanup trap — always remove the container and temp dir on exit. +trap 'stop_local_registry; rm -rf "$TEST_DIR"' EXIT + +start_local_registry() { + docker run -d \ + -p "${REGISTRY_PORT}:5000" \ + --name "$REGISTRY_CONTAINER" \ + registry:2 >/dev/null 2>&1 +} + +stop_local_registry() { + docker stop "$REGISTRY_CONTAINER" >/dev/null 2>&1 || true + docker rm "$REGISTRY_CONTAINER" >/dev/null 2>&1 || true +} + +wait_for_registry() { + local i + for i in $(seq 1 20); do + if curl -sf "http://${REGISTRY_HOST}/v2/" >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + return 1 +} + +# --------------------------------------------------------------------------- +# Pre-flight: Docker availability +# --------------------------------------------------------------------------- + +if ! command -v docker >/dev/null 2>&1; then + log_skip "Docker not installed — registry tests skipped" + exit 0 +fi + +if ! docker info >/dev/null 2>&1; then + log_skip "Docker daemon not running — registry tests skipped" + exit 0 +fi + +log_info "Starting local registry:2 on ${REGISTRY_HOST}..." +if ! start_local_registry; then + log_skip "Failed to start local registry — skipping" + exit 0 +fi + +if ! wait_for_registry; then + log_skip "Registry did not become ready — skipping" + exit 0 +fi +log_info "Registry ready at ${REGISTRY_HOST}" + +# --------------------------------------------------------------------------- +# Shared fixture: one .smolmachine created once for all tests +# --------------------------------------------------------------------------- + +FIXTURE_SIDECAR="$TEST_DIR/fixture.smolmachine" + +log_info "Creating fixture .smolmachine (alpine:latest)..." +if ! $SMOLVM pack create --image alpine:latest -o "$TEST_DIR/fixture" 2>/dev/null; then + log_skip "pack create failed — cannot run registry tests (is the VM running?)" + exit 0 +fi + +if [[ ! -f "$FIXTURE_SIDECAR" ]]; then + log_skip "No fixture sidecar produced — skipping" + exit 0 +fi + +log_info "Fixture ready: $FIXTURE_SIDECAR" + +FIXTURE_SHA=$(shasum -a 256 "$FIXTURE_SIDECAR" | cut -d' ' -f1) + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +echo "" +echo "==========================================" +echo " smolvm Pack Registry Tests" +echo "==========================================" +echo "" + +# Push a .smolmachine artifact, then pull it back and verify SHA256 equality. +test_push_pull_roundtrip() { + local ref="${REGISTRY_HOST}/roundtrip-test:latest" + local pull_out="$TEST_DIR/pulled-roundtrip.smolmachine" + + # Push + $SMOLVM pack push "$ref" -f "$FIXTURE_SIDECAR" 2>/dev/null || { + echo "FAIL: pack push failed" + return 1 + } + + # Pull + $SMOLVM pack pull "$ref" -o "$pull_out" 2>/dev/null || { + echo "FAIL: pack pull failed" + return 1 + } + + [[ -f "$pull_out" ]] || { echo "FAIL: output file not created"; return 1; } + + # Byte-for-byte verification + local pull_sha + pull_sha=$(shasum -a 256 "$pull_out" | cut -d' ' -f1) + + if [[ "$FIXTURE_SHA" != "$pull_sha" ]]; then + echo "FAIL: SHA256 mismatch after roundtrip" + echo " original: $FIXTURE_SHA" + echo " pulled: $pull_sha" + return 1 + fi + + echo " SHA256 matches: $pull_sha" +} + +# Push once, inspect twice — inspect must not download the layer blob. +test_inspect_returns_metadata() { + local ref="${REGISTRY_HOST}/inspect-test:v1" + + $SMOLVM pack push "$ref" -f "$FIXTURE_SIDECAR" 2>/dev/null || { + echo "FAIL: push failed" + return 1 + } + + local inspect_out + inspect_out=$($SMOLVM pack inspect "$ref" --json 2>/dev/null) + + [[ "$inspect_out" == *'"image"'* ]] || { echo "FAIL: missing 'image' field"; return 1; } + [[ "$inspect_out" == *'"platform"'* ]] || { echo "FAIL: missing 'platform' field"; return 1; } + [[ "$inspect_out" == *'"smolvm_version"'* ]] || { echo "FAIL: missing 'smolvm_version' field"; return 1; } + [[ "$inspect_out" == *'"layer_digest"'* ]] || { echo "FAIL: missing 'layer_digest' field"; return 1; } + [[ "$inspect_out" == *'"layer_size"'* ]] || { echo "FAIL: missing 'layer_size' field"; return 1; } + + echo " Inspect JSON contains all expected fields" +} + +# Push by tag, pull back by the returned manifest digest instead of tag. +test_pull_by_digest() { + local tag_ref="${REGISTRY_HOST}/digest-test:tagged" + local pull_out="$TEST_DIR/pulled-by-digest.smolmachine" + + # Push and capture manifest digest from output + local push_out + push_out=$($SMOLVM pack push "$tag_ref" -f "$FIXTURE_SIDECAR" 2>&1) || { + echo "FAIL: push failed" + return 1 + } + + local manifest_digest + manifest_digest=$(echo "$push_out" | grep -oE 'sha256:[a-f0-9]{64}' | tail -1) + + if [[ -z "$manifest_digest" ]]; then + echo "SKIP: could not extract manifest digest from push output" + return 0 + fi + + local digest_ref="${REGISTRY_HOST}/digest-test@${manifest_digest}" + + $SMOLVM pack pull "$digest_ref" -o "$pull_out" 2>/dev/null || { + echo "FAIL: pull by digest failed (ref: $digest_ref)" + return 1 + } + + [[ -f "$pull_out" ]] || { echo "FAIL: output file not created"; return 1; } + + local pull_sha + pull_sha=$(shasum -a 256 "$pull_out" | cut -d' ' -f1) + + if [[ "$FIXTURE_SHA" != "$pull_sha" ]]; then + echo "FAIL: SHA256 mismatch pulling by digest" + echo " original: $FIXTURE_SHA" + echo " pulled: $pull_sha" + return 1 + fi + + echo " Pulled by digest, SHA256 matches" +} + +# Push same artifact under two tags — second push must skip the blob (already exists). +test_push_deduplicates_blob() { + local ref1="${REGISTRY_HOST}/dedup-test:v1" + local ref2="${REGISTRY_HOST}/dedup-test:v2" + + local out1 out2 + out1=$($SMOLVM pack push "$ref1" -f "$FIXTURE_SIDECAR" 2>&1) || { echo "FAIL: first push failed"; return 1; } + out2=$($SMOLVM pack push "$ref2" -f "$FIXTURE_SIDECAR" 2>&1) || { echo "FAIL: second push failed"; return 1; } + + # Second push should report blob already exists (skipped upload) + if echo "$out2" | grep -qi "already exists\|skipping"; then + echo " Second push correctly skipped blob re-upload" + else + # Not fatal — the registry may accept duplicate uploads silently. + echo " (blob dedup message not found in output — may be registry-dependent)" + fi +} + +# Push an artifact, then pull it to a default-named output path. +# (pull without -o should write to .smolmachine in the current dir) +test_pull_default_output_path() { + local ref="${REGISTRY_HOST}/default-path-test:latest" + local work_dir="$TEST_DIR/default-path-work" + mkdir -p "$work_dir" + + $SMOLVM pack push "$ref" -f "$FIXTURE_SIDECAR" 2>/dev/null || { + echo "FAIL: push failed" + return 1 + } + + local pull_out + pull_out=$(cd "$work_dir" && $SMOLVM pack pull "$ref" 2>/dev/null && ls -1 *.smolmachine 2>/dev/null | head -1) + + if [[ -z "$pull_out" ]]; then + echo "SKIP: default output path test — no .smolmachine written to cwd" + return 0 + fi + + [[ -f "$work_dir/$pull_out" ]] || { echo "FAIL: expected $work_dir/$pull_out to exist"; return 1; } + + echo " Pull wrote default output: $pull_out" +} + +# Push to a nonexistent registry — must fail with a clear error. +test_push_to_nonexistent_registry() { + local ref="localhost:19999/no-such-registry/test:latest" + local exit_code=0 + $SMOLVM pack push "$ref" -f "$FIXTURE_SIDECAR" 2>/dev/null || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "FAIL: push to nonexistent registry should fail"; return 1; } + echo " Correctly failed with exit code $exit_code" +} + +# Pull a tag that was never pushed — must fail clearly. +test_pull_nonexistent_tag() { + local ref="${REGISTRY_HOST}/never-pushed-repo:v999" + local exit_code=0 + $SMOLVM pack pull "$ref" -o "$TEST_DIR/should-not-exist.smolmachine" 2>/dev/null || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "FAIL: pull of nonexistent tag should fail"; return 1; } + echo " Correctly failed with exit code $exit_code" +} + +# --------------------------------------------------------------------------- +# Run all tests +# --------------------------------------------------------------------------- + +run_test "push → pull roundtrip (SHA256 match)" test_push_pull_roundtrip +run_test "inspect returns manifest metadata" test_inspect_returns_metadata +run_test "pull by manifest digest" test_pull_by_digest +run_test "push deduplicates blob on re-push" test_push_deduplicates_blob +run_test "pull default output path" test_pull_default_output_path +run_test "push to nonexistent registry fails" test_push_to_nonexistent_registry +run_test "pull nonexistent tag fails" test_pull_nonexistent_tag + +print_summary "Pack Registry Tests" diff --git a/tests/test_ports.sh b/tests/test_ports.sh new file mode 100755 index 0000000..c010691 --- /dev/null +++ b/tests/test_ports.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# +# Port Mapping Tests +# +# Part of the smolvm test suite. Run with: ./tests/test_ports.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Port Mapping Tests" +echo "==========================================" +echo "" + +test_machine_port_mapping_http() { + local vm_name="test-vm-portmap" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create and start VM with port mapping (host 18199 -> guest 8080) + $SMOLVM machine create --name "$vm_name" -p 18199:8080 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + + # Start a simple HTTP responder inside the VM (background exec) + $SMOLVM machine exec --name "$vm_name" -- \ + sh -c 'echo -e "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok" | nc -l -p 8080 -w 5' & + local server_pid=$! + # nc serves exactly one connection; any TCP probe would consume it before + # curl gets a chance. A fixed sleep is the right wait here. + sleep 1 + + # Curl the mapped port from the host + local output + output=$(curl -s --connect-timeout 5 http://127.0.0.1:18199/ 2>&1) + local curl_rc=$? + + kill "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + + # Cleanup + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ $curl_rc -eq 0 ]] && [[ "$output" == *"ok"* ]] +} + +test_port_conflict_across_vms() { + local vm_a="port-conflict-a-$$" + local vm_b="port-conflict-b-$$" + + $SMOLVM machine create --name "$vm_a" -p 19876:80 --net 2>&1 >/dev/null || return 1 + $SMOLVM machine create --name "$vm_b" -p 19876:80 --net 2>&1 >/dev/null || return 1 + + $SMOLVM machine start --name "$vm_a" 2>&1 >/dev/null || { + $SMOLVM machine delete --name "$vm_a" -f 2>/dev/null + $SMOLVM machine delete --name "$vm_b" -f 2>/dev/null + return 1 + } + + # Second start should fail with port conflict + local exit_code=0 + local output + output=$($SMOLVM machine start --name "$vm_b" 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "expected port conflict error"; } + [[ "$output" == *"already in use"* ]] || { echo "expected 'already in use' message"; } + + $SMOLVM machine stop --name "$vm_a" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_a" -f 2>/dev/null || true + $SMOLVM machine delete --name "$vm_b" -f 2>/dev/null || true + + [[ $exit_code -ne 0 ]] +} + + +run_test "Port: mapping host to guest HTTP" test_machine_port_mapping_http || true +run_test "Port: cross-VM conflict detected" test_port_conflict_across_vms || true + +print_summary "Port Tests" diff --git a/tests/test_reliability.sh b/tests/test_reliability.sh new file mode 100755 index 0000000..b8c6f99 --- /dev/null +++ b/tests/test_reliability.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# +# Reliability and Concurrency Tests +# +# Part of the smolvm test suite. Run with: ./tests/test_reliability.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Reliability and Concurrency Tests" +echo "==========================================" +echo "" + +test_concurrent_machine_start() { + local vm_a="conc-start-a-$$" + local vm_b="conc-start-b-$$" + + $SMOLVM machine create --name "$vm_a" --cpus 1 --mem 256 2>&1 >/dev/null || return 1 + $SMOLVM machine create --name "$vm_b" --cpus 1 --mem 256 2>&1 >/dev/null || return 1 + + # Start both simultaneously — previously the second would fail with DB lock error + $SMOLVM machine start --name "$vm_a" 2>&1 >/dev/null & + local pid_a=$! + $SMOLVM machine start --name "$vm_b" 2>&1 >/dev/null & + local pid_b=$! + wait $pid_a; local exit_a=$? + wait $pid_b; local exit_b=$? + + # Both should succeed + [[ $exit_a -eq 0 ]] || { echo "FAIL: start a failed (exit $exit_a)"; } + [[ $exit_b -eq 0 ]] || { echo "FAIL: start b failed (exit $exit_b)"; } + + # Both should be running + local status_a status_b + status_a=$($SMOLVM machine status --name "$vm_a" 2>&1) + status_b=$($SMOLVM machine status --name "$vm_b" 2>&1) + + $SMOLVM machine stop --name "$vm_a" 2>/dev/null || true + $SMOLVM machine stop --name "$vm_b" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_a" -f 2>/dev/null || true + $SMOLVM machine delete --name "$vm_b" -f 2>/dev/null || true + + [[ "$status_a" == *"running"* ]] && [[ "$status_b" == *"running"* ]] +} + +test_machine_ls_does_not_kill_vm() { + skip_if_slow && return 0 + # Regression test: state_probe's probe_agent() used to create a temporary + # AgentManager without detaching it. When that manager was dropped, its + # Drop impl sent a Shutdown command to the agent, killing the VM. + # Every `machine ls` (and any state-checking command) triggered this. + # The old bug killed VMs within 10-20 seconds; we verify survival for 60s. + ensure_machine_running + + # Repeatedly call `machine ls` — each call probes the agent via + # resolve_state → probe_agent. Before the fix, the first call + # would kill the VM. + for i in 1 2 3 4 5 6; do + local output + output=$($SMOLVM machine ls 2>&1) + [[ "$output" == *"running"* ]] || { echo "VM died after ls call #$i: $output"; return 1; } + sleep 10 + done + + # Exec must still work after 6 ls calls over 60 seconds + local result + result=$($SMOLVM machine exec -- echo "survived-ls-probe" 2>&1) + [[ "$result" == *"survived-ls-probe"* ]] || { echo "exec failed after ls probes: $result"; return 1; } +} + +test_named_vm_survives_ls() { + skip_if_slow && return 0 + # Same regression test but with a named VM — the customer's exact scenario: + # machine create --name X --from .smolmachine → machine start → machine ls shows stopped. + # Verify over 60 seconds with interleaved ls + exec. + local name="ls-probe-test" + $SMOLVM machine stop --name "$name" 2>/dev/null || true + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true + $SMOLVM machine create --name "$name" 2>&1 || return 1 + $SMOLVM machine start --name "$name" 2>&1 || return 1 + + # Wait for agent to be fully ready + sleep 2 + + for i in 1 2 3 4 5 6; do + local state + state=$($SMOLVM machine ls 2>&1 | grep "$name" | awk '{print $2}') + [[ "$state" == "running" ]] || { echo "VM '$name' died after ls #$i (state: $state)"; $SMOLVM machine delete --name "$name" -f 2>/dev/null; return 1; } + sleep 10 + done + + # Exec must work after 60 seconds of ls probing + local result + result=$($SMOLVM machine exec --name "$name" -- echo "alive" 2>&1) + [[ "$result" == *"alive"* ]] || { echo "exec failed: $result"; $SMOLVM machine delete --name "$name" -f 2>/dev/null; return 1; } + + $SMOLVM machine stop --name "$name" 2>&1 || true + $SMOLVM machine delete --name "$name" -f 2>&1 || true +} + +test_state_probe_tolerates_busy_agent() { + ensure_machine_running + + # Fire a 1-second sleep exec in the background. The agent will be busy + # with `sh -c 'sleep 1'` → crun startup → child wait. + $SMOLVM machine exec -- sh -c 'sleep 1' & + local exec_pid=$! + + # Give the exec time to reach the agent's busy-with-request state. + sleep 0.2 + + # While the exec is still running, `machine ls` must show "running". + # With the old 100ms ping, it would show "unreachable". + local state + state=$($SMOLVM machine ls 2>&1 | grep "^default " | awk '{print $2}') + + # Wait for the background exec to finish before asserting, so we don't + # leave a zombie if the test fails. + wait "$exec_pid" 2>/dev/null + + [[ "$state" == "running" ]] || { + echo "expected 'running' during busy agent, got '$state' — state probe regressed?" + return 1 + } +} + +test_concurrent_exec_does_not_flip_unreachable() { + ensure_machine_running + + # Hold a long-running exec open in the background. + $SMOLVM machine exec -- sh -c 'sleep 5' & + local hold_pid=$! + + # Give it time to be accepted by the agent and block the old single thread. + sleep 1 + + # A second exec must succeed while the first is still running. + local second_output + second_output=$($SMOLVM machine exec -- echo concurrent_ok 2>&1) + local second_exit=$? + + # Also verify state did not flip to unreachable. + local state + state=$($SMOLVM machine ls 2>&1 | grep "^default " | awk '{print $2}') + + wait "$hold_pid" 2>/dev/null + + if [[ $second_exit -ne 0 ]]; then + echo "FAIL: second concurrent exec failed (exit $second_exit): $second_output" + return 1 + fi + if [[ "$second_output" != *"concurrent_ok"* ]]; then + echo "FAIL: second exec output unexpected: $second_output" + return 1 + fi + if [[ "$state" != "running" ]]; then + echo "FAIL: VM flipped to '$state' during concurrent exec (expected 'running')" + return 1 + fi +} + + +run_test "Concurrent machine starts" test_concurrent_machine_start || true +run_test "State probe tolerates busy agent (no false unreachable)" test_state_probe_tolerates_busy_agent || true +run_test "Concurrent exec does not flip VM to unreachable" test_concurrent_exec_does_not_flip_unreachable || true +run_test "Listing: machine ls does not kill VM" test_machine_ls_does_not_kill_vm || true +run_test "Listing: named VM survives repeated ls" test_named_vm_survives_ls || true + +print_summary "Reliability Tests" diff --git a/tests/test_resize.sh b/tests/test_resize.sh new file mode 100755 index 0000000..8bab02e --- /dev/null +++ b/tests/test_resize.sh @@ -0,0 +1,305 @@ +#!/usr/bin/env bash +# +# Resize feature tests for smolvm. +# Runs only the resize-related integration tests. +# +# Usage: +# ./tests/test_resize.sh + +set -euo pipefail + +source "$(dirname "$0")/common.sh" +init_smolvm + +# Pre-flight: Kill any existing smolvm processes and clean up ALL test VMs +log_info "Pre-flight cleanup: killing orphan processes and removing test VMs..." +kill_orphan_smolvm_processes + +# Clean up any leftover test VM directories +for vm_name in test-vm-resize-happy test-vm-resize-running test-vm-resize-shrink \ + nonexistent-vm-resize-test test-vm-resize-noparams test-vm-resize-storage-only \ + test-vm-resize-overlay-only; do + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$(vm_data_dir "$vm_name")" +done + +# Cleanup on exit - stop default VM only (don't delete test VMs, they should be cleaned by each test) +trap 'cleanup_machine' EXIT + +# Run only resize tests +echo "" +echo "==========================================" +echo " smolvm Resize Feature Tests" +echo "==========================================" +echo "" + +# ============================================================================= +# Resize Tests +# ============================================================================= + +test_machine_resize_happy_path() { + # Integration test: Happy path resize workflow (4.4) + local vm_name="test-vm-resize-happy" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$(vm_data_dir "$vm_name")" + + # Create VM with small resources + $SMOLVM machine create --name "$vm_name" --storage 5 --overlay 2 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$(vm_data_dir "$vm_name")" + return 1 + } + $SMOLVM machine stop --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$(vm_data_dir "$vm_name")" + return 1 + } + + # Resize + local resize_output + resize_output=$($SMOLVM machine resize --name "$vm_name" --storage 10 --overlay 5 2>&1) + + # Verify resize output contains expected messages + if [[ "$resize_output" != *"Resizing machine"* ]]; then + echo "Resize output should contain 'Resizing machine'" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$(vm_data_dir "$vm_name")" + return 1 + fi + + # Clean up + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$(vm_data_dir "$vm_name")" + ensure_data_dir_deleted "$vm_name" + + # Verify output mentions success + [[ "$resize_output" == *"resized successfully"* ]] +} + +test_machine_resize_running_vm_rejected() { + # Integration test: Resize rejection - running VM (4.5) + local vm_name="test-vm-resize-running" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create and start VM + $SMOLVM machine create --name "$vm_name" --storage 5 --overlay 2 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + + # Attempt resize on running VM - should fail + local exit_code=0 + local resize_output + resize_output=$($SMOLVM machine resize --name "$vm_name" --storage 10 2>&1) || exit_code=$? + + # Clean up + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + # Should fail with non-zero exit code + [[ $exit_code -ne 0 ]] +} + +test_machine_resize_shrink_rejected() { + # Integration test: Resize rejection - shrink disk (4.5) + local vm_name="test-vm-resize-shrink" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM with 10 GiB storage + $SMOLVM machine create --name "$vm_name" --storage 10 --overlay 5 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + $SMOLVM machine stop --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + + # Attempt to shrink to 5 GiB - should fail + local exit_code=0 + local resize_output + resize_output=$($SMOLVM machine resize --name "$vm_name" --storage 5 2>&1) || exit_code=$? + + # Clean up + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + # Should fail with non-zero exit code and error message about shrinking + [[ $exit_code -ne 0 ]] && [[ "$resize_output" == *"shrunk"* || "$resize_output" == *"larger"* ]] +} + +test_machine_resize_nonexistent_vm_rejected() { + # Integration test: Resize rejection - non-existent VM (4.5) + local vm_name="nonexistent-vm-resize-test" + + # Attempt resize on non-existent VM + local exit_code=0 + local resize_output + resize_output=$($SMOLVM machine resize --name "$vm_name" --storage 10 2>&1) || exit_code=$? + + # Should fail with non-zero exit code and "not found" message + [[ $exit_code -ne 0 ]] && [[ "$resize_output" == *"not found"* ]] +} + +test_machine_resize_default_vm() { + # Integration test: Default VM resize (4.6) + cleanup_machine + + # Start and stop default VM (created with defaults: 20 GiB storage, 10 GiB overlay) + $SMOLVM machine start 2>&1 || return 1 + $SMOLVM machine stop 2>&1 || return 1 + + # Resize default VM (no name argument) - must expand, not shrink + local resize_output + resize_output=$($SMOLVM machine resize --storage 30 --overlay 15 2>&1) + + # Verify resize output contains expected messages + if [[ "$resize_output" != *"Resizing machine"* ]]; then + echo "Resize output should contain 'Resizing machine'" + echo "Got: $resize_output" + $SMOLVM machine stop 2>/dev/null || true + return 1 + fi + + # Clean up + $SMOLVM machine stop 2>/dev/null || true + + # Verify output mentions success + [[ "$resize_output" == *"resized successfully"* ]] +} + +test_machine_resize_no_params_rejected() { + # Integration test: Resize with no parameters should fail + local vm_name="test-vm-resize-noparams" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM + $SMOLVM machine create --name "$vm_name" --storage 5 --overlay 2 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + $SMOLVM machine stop --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + + # Attempt resize with no parameters - should fail + local exit_code=0 + local resize_output + resize_output=$($SMOLVM machine resize --name "$vm_name" 2>&1) || exit_code=$? + + # Clean up + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + # Should fail with non-zero exit code + [[ $exit_code -ne 0 ]] +} + +test_machine_resize_storage_only() { + # Integration test: Resize storage only (partial update) + local vm_name="test-vm-resize-storage-only" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$(vm_data_dir "$vm_name")" + + # Create VM with small resources + $SMOLVM machine create --name "$vm_name" --storage 5 --overlay 2 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$(vm_data_dir "$vm_name")" + return 1 + } + $SMOLVM machine stop --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$(vm_data_dir "$vm_name")" + return 1 + } + + # Resize storage only + local resize_output + resize_output=$($SMOLVM machine resize --name "$vm_name" --storage 15 2>&1) + + # Clean up + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$(vm_data_dir "$vm_name")" + ensure_data_dir_deleted "$vm_name" + + # Should succeed and mention storage expansion + [[ "$resize_output" == *"Storage"* ]] && [[ "$resize_output" == *"resized successfully"* ]] +} + +test_machine_resize_overlay_only() { + # Integration test: Resize overlay only (partial update) + local vm_name="test-vm-resize-overlay-only" + + # Aggressive cleanup - stop, delete, and remove directory + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$(vm_data_dir "$vm_name")" + sleep 0.5 + + # Create VM with small resources + $SMOLVM machine create --name "$vm_name" --storage 5 --overlay 2 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$(vm_data_dir "$vm_name")" + return 1 + } + $SMOLVM machine stop --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$(vm_data_dir "$vm_name")" + return 1 + } + + # Verify disk sizes before resize (debug) + local overlay_size + overlay_size=$(ls -lh "$(vm_data_dir "$vm_name")/overlay.raw" 2>/dev/null | awk '{print $5}') + if [[ "$overlay_size" != "2.0G" ]]; then + echo "DEBUG: overlay disk is $overlay_size, expected 2.0G" + echo "DEBUG: listing directory:" + ls -lh "$(vm_data_dir "$vm_name")/" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$(vm_data_dir "$vm_name")" + return 1 + fi + + # Resize overlay only + local resize_output + resize_output=$($SMOLVM machine resize --name "$vm_name" --overlay 8 2>&1) + echo "RESIZE OUTPUT: '$resize_output'" + + # Clean up + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$(vm_data_dir "$vm_name")" + ensure_data_dir_deleted "$vm_name" + + # Should succeed and mention overlay expansion + [[ "$resize_output" == *"Overlay"* ]] && [[ "$resize_output" == *"resized successfully"* ]] +} + +run_test "Resize: happy path" test_machine_resize_happy_path || true +run_test "Resize: running VM rejected" test_machine_resize_running_vm_rejected || true +run_test "Resize: shrink rejected" test_machine_resize_shrink_rejected || true +run_test "Resize: non-existent VM rejected" test_machine_resize_nonexistent_vm_rejected || true +run_test "Resize: default VM" test_machine_resize_default_vm || true +run_test "Resize: no params rejected" test_machine_resize_no_params_rejected || true +run_test "Resize: storage only" test_machine_resize_storage_only || true +run_test "Resize: overlay only" test_machine_resize_overlay_only || true + +print_summary "Resize Tests" diff --git a/tests/test_resources.sh b/tests/test_resources.sh new file mode 100755 index 0000000..f1b7383 --- /dev/null +++ b/tests/test_resources.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# +# Resource Validation and Naming Tests +# +# Part of the smolvm test suite. Run with: ./tests/test_resources.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Resource Validation and Naming Tests" +echo "==========================================" +echo "" + +test_resource_cpus_zero_rejected() { + local exit_code=0 + $SMOLVM machine run --cpus 0 -- echo hello 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] || return 1 +} + +test_resource_mem_zero_rejected() { + local exit_code=0 + $SMOLVM machine run --mem 0 -- echo hello 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] || return 1 +} + +test_resource_mem_below_minimum_rejected() { + local exit_code=0 + local output + output=$($SMOLVM machine run --mem 1 -- echo hello 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || return 1 + [[ "$output" == *"at least"* ]] || return 1 +} + +test_name_length_44_chars_accepted() { + local name="sandbox-7f3e2d1c-9a8b-4e5f-b123-456789abcdef" + [[ ${#name} -eq 44 ]] || { echo "test bug: expected 44 chars, got ${#name}"; return 1; } + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true + + local output exit_code=0 + output=$($SMOLVM machine create --name "$name" 2>&1) || exit_code=$? + if [[ $exit_code -ne 0 ]]; then + echo "expected 44-char name to succeed, got error: $output" + return 1 + fi + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true +} + +test_name_length_75_chars_accepted_via_hash_path() { + local name + name=$(printf 'a%.0s' {1..75}) + [[ ${#name} -eq 75 ]] || return 1 + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true + + local output exit_code=0 + output=$($SMOLVM machine create --name "$name" 2>&1) || exit_code=$? + if [[ $exit_code -ne 0 ]]; then + echo "expected 75-char name to succeed (hash path keeps socket bounded), got: $output" + return 1 + fi + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true +} + +test_name_length_sanity_cap_rejects_absurd_names() { + local name + name=$(printf 'a%.0s' {1..200}) + [[ ${#name} -eq 200 ]] || return 1 + + local output exit_code=0 + output=$($SMOLVM machine create --name "$name" 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "expected 200-char name to be rejected"; return 1; } + [[ "$output" == *"too long"* ]] || { + echo "expected length-cap error, got: $output" + return 1 + } +} + +test_start_nonexistent_name_rejected() { + local exit_code=0 + $SMOLVM machine start --name nonexistent-vm-regression-test 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "expected error for nonexistent VM"; return 1; } + + # Verify no "default" VM was created + local list + list=$($SMOLVM machine ls --json 2>&1) + [[ "$list" != *"nonexistent-vm-regression-test"* ]] || { echo "VM should not exist"; return 1; } +} + +test_auto_generated_names() { + # Auto-generate: create with no --name at all (omitting --name triggers the + # vm-XXXXXXXX auto-name; `--name` with no value is a CLI error, not auto-gen). + local result1 result2 + result1=$($SMOLVM machine create 2>&1) || { echo "auto-name create #1 failed: $result1"; return 1; } + result2=$($SMOLVM machine create 2>&1) || { echo "auto-name create #2 failed: $result2"; return 1; } + + local name1 name2 + name1=$(echo "$result1" | grep "Created machine:" | grep -oE "vm-[a-f0-9]{8}" | head -1) + name2=$(echo "$result2" | grep "Created machine:" | grep -oE "vm-[a-f0-9]{8}" | head -1) + + # Both should produce valid names + [[ -n "$name1" ]] && [[ -n "$name2" ]] || { echo "No auto name found"; return 1; } + + # Names should differ + [[ "$name1" != "$name2" ]] || { echo "Names should be unique: $name1"; return 1; } + + # Both should appear in list (use --json for full names, avoids truncation) + local list_result + list_result=$($SMOLVM machine ls --json 2>&1) + [[ "$list_result" == *"$name1"* ]] && [[ "$list_result" == *"$name2"* ]] || { + echo "Auto-named machines not in list" + $SMOLVM machine delete --name "$name1" -f 2>/dev/null + $SMOLVM machine delete --name "$name2" -f 2>/dev/null + return 1 + } + + # Explicit name still works. Use a unique suffix so a dirty/concurrent box + # (leftover machines) can't collide or false-match. + local explicit="explicit-test-$$-$RANDOM" + $SMOLVM machine create --name "$explicit" 2>&1 || { echo "Explicit name failed"; return 1; } + list_result=$($SMOLVM machine ls --json 2>&1) + [[ "$list_result" == *"$explicit"* ]] || { echo "Explicit name not in list"; return 1; } + + # Cleanup + $SMOLVM machine delete --name "$name1" -f 2>/dev/null + $SMOLVM machine delete --name "$name2" -f 2>/dev/null + $SMOLVM machine delete --name "$explicit" -f 2>/dev/null +} + + +test_create_rejects_zero_valued_disks() { + # `machine create` must reject zero-sized storage/overlay up front and + # NOT persist a machine — otherwise the failure only surfaces later at + # `machine start`, leaving an unstartable machine in the list. + local name="zero-disk-test-$$" + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true + + local exit_code=0 + $SMOLVM machine create --name "$name" --storage 0 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] || { + echo "--storage 0 should be rejected at create" + $SMOLVM machine delete --name "$name" -f 2>/dev/null + return 1 + } + + exit_code=0 + $SMOLVM machine create --name "$name" --overlay 0 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] || { + echo "--overlay 0 should be rejected at create" + $SMOLVM machine delete --name "$name" -f 2>/dev/null + return 1 + } + + # Neither rejected create may have persisted the machine. + local list + list=$($SMOLVM machine ls --json 2>&1) + [[ "$list" != *"$name"* ]] || { + echo "a rejected create persisted the machine anyway" + $SMOLVM machine delete --name "$name" -f 2>/dev/null + return 1 + } +} + +test_exec_nonexistent_machine_reports_not_found() { + # exec/cp on a machine that does not exist must say "not found", not + # "is not running" — the latter wrongly tells the user to run `start`. + local name="missing-machine-test-$$" + local output exit_code=0 + + output=$($SMOLVM machine exec --name "$name" -- echo hi 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "exec on a nonexistent machine should fail"; return 1; } + [[ "$output" == *"not found"* ]] || { echo "expected 'not found', got: $output"; return 1; } + [[ "$output" != *"is not running"* ]] || { + echo "must not report 'is not running' for a machine that does not exist" + return 1 + } + + exit_code=0 + output=$($SMOLVM machine cp /etc/hostname "$name":/tmp/x 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "cp to a nonexistent machine should fail"; return 1; } + [[ "$output" == *"not found"* ]] || { echo "cp: expected 'not found', got: $output"; return 1; } +} + +test_status_json_output() { + # `machine status` supports `--json`, mirroring `machine list --json`. + local name="status-json-test-$$" + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true + $SMOLVM machine create --name "$name" 2>&1 || { echo "setup: create failed"; return 1; } + + local output exit_code=0 + output=$($SMOLVM machine status --name "$name" --json 2>&1) || exit_code=$? + if [[ $exit_code -ne 0 ]]; then + echo "status --json should succeed, got: $output" + $SMOLVM machine delete --name "$name" -f 2>/dev/null + return 1 + fi + [[ "$output" == "{"* && "$output" == *"\"name\""* && "$output" == *"$name"* ]] || { + echo "status --json did not return the expected JSON object: $output" + $SMOLVM machine delete --name "$name" -f 2>/dev/null + return 1 + } + $SMOLVM machine delete --name "$name" -f 2>/dev/null + + # `--json` on a machine that does not exist must error. + exit_code=0 + $SMOLVM machine status --name "missing-$$" --json 2>&1 || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "status --json on a nonexistent machine should fail"; return 1; } +} + +run_test "Resource: --cpus 0 rejected" test_resource_cpus_zero_rejected || true +run_test "Resource: --mem 0 rejected" test_resource_mem_zero_rejected || true +run_test "Name length: 44-char UUID name accepted (was rejected by old 40-char cap)" test_name_length_44_chars_accepted || true +run_test "Name length: 75-char name accepted via hash-derived socket path" test_name_length_75_chars_accepted_via_hash_path || true +run_test "Name length: absurd names rejected by sanity cap" test_name_length_sanity_cap_rejects_absurd_names || true +run_test "Resource: --mem below minimum rejected" test_resource_mem_below_minimum_rejected || true +run_test "Start --name nonexistent rejected" test_start_nonexistent_name_rejected || true +run_test "Auto-generated names" test_auto_generated_names || true +run_test "Create: zero-valued storage/overlay rejected, not persisted" test_create_rejects_zero_valued_disks || true +run_test "Exec/cp: nonexistent machine reports 'not found'" test_exec_nonexistent_machine_reports_not_found || true +run_test "Status: --json outputs a JSON object" test_status_json_output || true + +print_summary "Resource Tests" diff --git a/tests/test_scale.sh b/tests/test_scale.sh new file mode 100755 index 0000000..9460184 --- /dev/null +++ b/tests/test_scale.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +# +# Scale tests for smolvm. +# +# Verifies concurrent VM operations work correctly under contention: +# SQLite locking, per-VM flock, vsock connect retries, and concurrent exec. +# +# Usage: +# ./tests/test_scale.sh +# ./tests/run_tests.sh scale + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +echo "" +echo "==========================================" +echo " smolvm Scale Tests" +echo "==========================================" +echo "" + +# ============================================================================= +# 10 concurrent VMs: create, start, exec, stop +# ============================================================================= + +test_10_concurrent_vms() { + local COUNT=10 + local i + + for i in $(seq 1 $COUNT); do + $SMOLVM machine stop --name "scale-$i" 2>/dev/null || true + $SMOLVM machine delete --name "scale-$i" -f 2>/dev/null || true + done + + # Create + for i in $(seq 1 $COUNT); do + $SMOLVM machine create --name "scale-$i" --mem 512 --cpus 1 2>/dev/null || { + echo "FAIL: create scale-$i failed"; return 1 + } + done + + # Start concurrently + local pids=() + for i in $(seq 1 $COUNT); do + $SMOLVM machine start --name "scale-$i" 2>/dev/null & + pids+=($!) + done + local start_fails=0 + for pid in "${pids[@]}"; do + wait "$pid" || ((start_fails++)) + done + sleep 1 + + # Verify all running + local running=0 + for i in $(seq 1 $COUNT); do + if $SMOLVM machine status --name "scale-$i" 2>&1 | grep -q "running"; then + ((running++)) + fi + done + + # Exec on each + local exec_pass=0 + for i in $(seq 1 $COUNT); do + local out + out=$($SMOLVM machine exec --name "scale-$i" -- echo "ok-$i" 2>&1) || true + if echo "$out" | grep -q "ok-$i"; then + ((exec_pass++)) + fi + done + + # Cleanup + for i in $(seq 1 $COUNT); do + $SMOLVM machine stop --name "scale-$i" 2>/dev/null || true + $SMOLVM machine delete --name "scale-$i" -f 2>/dev/null || true + done + + echo " running: $running/$COUNT, exec: $exec_pass/$COUNT, start_fails: $start_fails" + [[ $running -eq $COUNT ]] || { echo "FAIL: only $running/$COUNT running"; return 1; } + [[ $exec_pass -eq $COUNT ]] || { echo "FAIL: only $exec_pass/$COUNT exec passed"; return 1; } +} + +run_test "Scale: 10 concurrent VMs start + exec" test_10_concurrent_vms || true + +# ============================================================================= +# Concurrent exec storm: 5 execs per VM across 10 VMs (50 total) +# ============================================================================= + +test_concurrent_exec_storm() { + local COUNT=10 + local EXECS_PER_VM=5 + local i j + + for i in $(seq 1 $COUNT); do + $SMOLVM machine stop --name "storm-$i" 2>/dev/null || true + $SMOLVM machine delete --name "storm-$i" -f 2>/dev/null || true + done + + # Create and start sequentially (scale test above covers concurrent start) + for i in $(seq 1 $COUNT); do + $SMOLVM machine create --name "storm-$i" --mem 512 --cpus 1 2>/dev/null || return 1 + $SMOLVM machine start --name "storm-$i" 2>/dev/null || return 1 + done + + # Storm: 50 concurrent execs + local tmpdir + tmpdir=$(mktemp -d) + for i in $(seq 1 $COUNT); do + for j in $(seq 1 $EXECS_PER_VM); do + ( + out=$($SMOLVM machine exec --name "storm-$i" -- echo "s-${i}-${j}" 2>&1) + if echo "$out" | grep -q "s-${i}-${j}"; then + touch "$tmpdir/pass-${i}-${j}" + fi + ) & + done + done + wait + local pass_count + pass_count=$(find "$tmpdir" -name "pass-*" 2>/dev/null | wc -l) + rm -rf "$tmpdir" + + # Cleanup + for i in $(seq 1 $COUNT); do + $SMOLVM machine stop --name "storm-$i" 2>/dev/null || true + $SMOLVM machine delete --name "storm-$i" -f 2>/dev/null || true + done + + local total=$((COUNT * EXECS_PER_VM)) + echo " exec pass: $pass_count/$total" + [[ $pass_count -eq $total ]] || { echo "FAIL: only $pass_count/$total execs passed"; return 1; } +} + +run_test "Scale: 50 concurrent execs across 10 VMs" test_concurrent_exec_storm || true + +# ============================================================================= +# Rapid lifecycle: create-start-stop-delete x5 +# ============================================================================= + +test_rapid_lifecycle() { + local i + for i in $(seq 1 5); do + $SMOLVM machine create --name "rapid-$$" --mem 512 --cpus 1 2>/dev/null || { + echo "FAIL: create iteration $i"; return 1 + } + $SMOLVM machine start --name "rapid-$$" 2>/dev/null || { + echo "FAIL: start iteration $i" + $SMOLVM machine delete --name "rapid-$$" -f 2>/dev/null; return 1 + } + $SMOLVM machine stop --name "rapid-$$" 2>/dev/null || true + $SMOLVM machine delete --name "rapid-$$" -f 2>/dev/null || true + done + echo " 5 create-start-stop-delete cycles completed" +} + +run_test "Scale: rapid lifecycle (5 iterations)" test_rapid_lifecycle || true + +print_summary "Scale Tests" diff --git a/tests/test_secrets.sh b/tests/test_secrets.sh new file mode 100755 index 0000000..0e584b6 --- /dev/null +++ b/tests/test_secrets.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# +# End-to-end tests for host-side secret references. +# +# smolvm stores no secret material itself: a `[secrets]` entry references a +# value that already lives on the host (here, a host environment variable). The +# security INVARIANT is that the referenced PLAINTEXT is resolved on the host at +# launch time and reaches the guest workload's environment, but NEVER persists +# in the machine's DB record or a portable `.smolmachine` pack — only the opaque +# ref does. These tests boot real VMs and verify both halves end to end (the +# unit tests cover the resolution logic; only an e2e run can prove the plaintext +# actually crosses to the guest and actually stays out of the persisted +# artifacts). +# +# Usage: +# ./tests/test_secrets.sh + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +echo "" +echo "==========================================" +echo " smolvm Secret Reference E2E Tests" +echo "==========================================" +echo "" + +SECRET_TMPDIR=$(mktemp -d) +# Unique name + value per run so the value is an unambiguous needle to grep for +# in the DB / pack, and parallel runs never collide. +SECRET_NAME="E2ESECRET_$$" +SECRET_VALUE="plaintext-needle-$$-do-not-leak" +# The host env var the ref points at. smolvm is launched as a child of this +# shell, so exporting it puts the value in smolvm's own environment, where the +# `from_env` ref resolves it at launch time. The value is never persisted. +export "$SECRET_NAME=$SECRET_VALUE" + +cleanup_vm() { + local name="$1" + $SMOLVM machine stop --name "$name" 2>/dev/null || true + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true +} + +cleanup_secrets_test() { + rm -rf "$SECRET_TMPDIR" + cleanup_machine +} +trap 'cleanup_secrets_test' EXIT + +# Write a Smolfile that references the stored secret as a guest env var of the +# same name. `extra` is appended verbatim (e.g. an `init = [...]` line). +write_smolfile() { + local path="$1" + local extra="${2:-}" + cat > "$path" <&1 || return 1 + $SMOLVM machine start --name "$vm" 2>&1 || { cleanup_vm "$vm"; return 1; } + # `env` (busybox) prints the whole environment — robust across rootfs builds. + local output + output=$($SMOLVM machine exec --name "$vm" -- env 2>&1) + cleanup_vm "$vm" + [[ "$output" == *"$SECRET_NAME=$SECRET_VALUE"* ]] +} + +# ============================================================================= +# 2. Plaintext reaches the guest at init time too. +# ============================================================================= +test_secret_reaches_guest_init() { + local vm="secret-init-$$" + cleanup_vm "$vm" + write_smolfile "$SECRET_TMPDIR/Smolfile.init" \ + "init = [\"printenv $SECRET_NAME > /tmp/secret-out.txt\"]" + $SMOLVM machine create --name "$vm" --smolfile "$SECRET_TMPDIR/Smolfile.init" 2>&1 || return 1 + $SMOLVM machine start --name "$vm" 2>&1 || { cleanup_vm "$vm"; return 1; } + local output + output=$($SMOLVM machine exec --name "$vm" -- cat /tmp/secret-out.txt 2>&1) + cleanup_vm "$vm" + [[ "$output" == *"$SECRET_VALUE"* ]] +} + +# ============================================================================= +# 3. INVARIANT: the plaintext never lands in the persisted DB record. +# ============================================================================= +test_plaintext_not_in_db() { + local vm="secret-nodb-$$" + cleanup_vm "$vm" + write_smolfile "$SECRET_TMPDIR/Smolfile.nodb" + $SMOLVM machine create --name "$vm" --smolfile "$SECRET_TMPDIR/Smolfile.nodb" 2>&1 || return 1 + # Start too, so any persist-on-run path also executes. + $SMOLVM machine start --name "$vm" 2>&1 || { cleanup_vm "$vm"; return 1; } + # The ref NAME may legitimately appear in the record; the VALUE must not. + local db leaked=0 + db=$(find "$HOME" -name 'smolvm.db' -path '*smolvm/server/*' 2>/dev/null | head -1) + if [[ -n "$db" ]] && strings "$db" 2>/dev/null | grep -qF "$SECRET_VALUE"; then + leaked=1 + log_info "LEAK: plaintext secret found in DB at $db" + fi + cleanup_vm "$vm" + [[ "$leaked" -eq 0 ]] +} + +# ============================================================================= +# 4. INVARIANT: the plaintext never lands in a portable .smolmachine pack. +# ============================================================================= +test_plaintext_not_in_pack() { + local vm="secret-nopack-$$" + cleanup_vm "$vm" + write_smolfile "$SECRET_TMPDIR/Smolfile.nopack" + $SMOLVM machine create --name "$vm" --smolfile "$SECRET_TMPDIR/Smolfile.nopack" 2>&1 || return 1 + $SMOLVM machine start --name "$vm" 2>&1 || { cleanup_vm "$vm"; return 1; } + $SMOLVM machine stop --name "$vm" 2>&1 || true + local pack="$SECRET_TMPDIR/out.smolmachine" + $SMOLVM pack create --from-vm "$vm" -o "$pack" 2>&1 || { cleanup_vm "$vm"; return 1; } + local leaked=0 + if strings "$pack" 2>/dev/null | grep -qF "$SECRET_VALUE"; then + leaked=1 + log_info "LEAK: plaintext secret found in pack $pack" + fi + cleanup_vm "$vm" + [[ -f "$pack" ]] && [[ "$leaked" -eq 0 ]] +} + +# ============================================================================= +# 5. The secret re-resolves on restart (refs persist, plaintext does not). +# ============================================================================= +test_secret_reresolves_after_restart() { + local vm="secret-restart-$$" + cleanup_vm "$vm" + write_smolfile "$SECRET_TMPDIR/Smolfile.restart" + $SMOLVM machine create --name "$vm" --smolfile "$SECRET_TMPDIR/Smolfile.restart" 2>&1 || return 1 + $SMOLVM machine start --name "$vm" 2>&1 || { cleanup_vm "$vm"; return 1; } + $SMOLVM machine stop --name "$vm" 2>&1 || true + $SMOLVM machine start --name "$vm" 2>&1 || { cleanup_vm "$vm"; return 1; } + local output + output=$($SMOLVM machine exec --name "$vm" -- env 2>&1) + cleanup_vm "$vm" + [[ "$output" == *"$SECRET_NAME=$SECRET_VALUE"* ]] +} + +# ============================================================================= +# Run +# ============================================================================= +log_info "Test secret exported as host env var '$SECRET_NAME' (from_env source)" + +run_test "Secret plaintext reaches guest via exec" test_secret_reaches_guest_exec || true +run_test "Secret plaintext reaches guest via init" test_secret_reaches_guest_init || true +run_test "INVARIANT: plaintext never in DB record" test_plaintext_not_in_db || true +run_test "INVARIANT: plaintext never in .smolmachine pack" test_plaintext_not_in_pack || true +run_test "Secret re-resolves after restart" test_secret_reresolves_after_restart || true + +print_summary "Secret Reference E2E Tests" diff --git a/tests/test_smolfile.sh b/tests/test_smolfile.sh new file mode 100755 index 0000000..6309bf2 --- /dev/null +++ b/tests/test_smolfile.sh @@ -0,0 +1,1395 @@ +#!/usr/bin/env bash +# +# Smolfile tests for smolvm. +# +# Tests the `--smolfile` and `--init` functionality for both +# machine create --name commands. +# +# Usage: +# ./tests/test_smolfile.sh + +source "$(dirname "$0")/common.sh" +init_smolvm + +# Pre-flight: Kill any existing smolvm processes that might hold database lock +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +echo "" +echo "==========================================" +echo " smolvm Smolfile Tests" +echo "==========================================" +echo "" + +# Temp directory for Smolfiles +SMOLFILE_TMPDIR=$(mktemp -d) +trap 'rm -rf "$SMOLFILE_TMPDIR"; cleanup_machine' EXIT + +# ============================================================================= +# Helpers +# ============================================================================= + +# Clean up a named VM, ignoring errors +cleanup_vm() { + local name="$1" + $SMOLVM machine stop --name "$name" 2>/dev/null || true + $SMOLVM machine delete --name "$name" -f 2>/dev/null || true +} + +# ============================================================================= +# --init flag (no Smolfile) +# ============================================================================= + +test_init_flag_creates_file() { + local vm_name="smolfile-init-flag-$$" + cleanup_vm "$vm_name" + + # Create VM with --init that creates a marker file + $SMOLVM machine create --name "$vm_name" --init "echo 'init-ran' > /tmp/init-marker.txt" 2>&1 || return 1 + + # Start VM (init should run) + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Verify the init command ran + local output + output=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/init-marker.txt 2>&1) + + cleanup_vm "$vm_name" + [[ "$output" == *"init-ran"* ]] +} + +test_init_flag_multiple_commands() { + local vm_name="smolfile-multi-init-$$" + cleanup_vm "$vm_name" + + # Create VM with multiple --init flags + $SMOLVM machine create --name "$vm_name" \ + --init "echo 'first' > /tmp/init1.txt" \ + --init "echo 'second' > /tmp/init2.txt" \ + 2>&1 || return 1 + + # Start VM + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Verify both init commands ran + local out1 out2 + out1=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/init1.txt 2>&1) + out2=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/init2.txt 2>&1) + + cleanup_vm "$vm_name" + [[ "$out1" == *"first"* ]] && [[ "$out2" == *"second"* ]] +} + +test_init_flag_with_env() { + local vm_name="smolfile-init-env-$$" + cleanup_vm "$vm_name" + + # Create VM with --init and -e + $SMOLVM machine create --name "$vm_name" \ + -e MY_VAR=hello_from_env \ + --init 'echo "$MY_VAR" > /tmp/env-test.txt' \ + 2>&1 || return 1 + + # Start VM + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Verify env was passed to init + local output + output=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/env-test.txt 2>&1) + + cleanup_vm "$vm_name" + [[ "$output" == *"hello_from_env"* ]] +} + +test_init_flag_with_workdir() { + local vm_name="smolfile-init-wd-$$" + cleanup_vm "$vm_name" + + # Create VM with --init and -w + $SMOLVM machine create --name "$vm_name" \ + -w /tmp \ + --init "pwd > cwd.txt" \ + 2>&1 || return 1 + + # Start VM + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Verify workdir was applied + local output + output=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/cwd.txt 2>&1) + + cleanup_vm "$vm_name" + [[ "$output" == *"/tmp"* ]] +} + +test_init_runs_on_every_start() { + local vm_name="smolfile-restart-$$" + cleanup_vm "$vm_name" + + # Create VM with --init that appends to a file + $SMOLVM machine create --name "$vm_name" \ + --init 'echo "boot" >> /tmp/boot-count.txt' \ + 2>&1 || return 1 + + # First start + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Check count after first start + local count1 + count1=$($SMOLVM machine exec --name "$vm_name" -- wc -l /tmp/boot-count.txt 2>&1) + + # Stop and start again + $SMOLVM machine stop --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Check count after second start + local count2 + count2=$($SMOLVM machine exec --name "$vm_name" -- wc -l /tmp/boot-count.txt 2>&1) + + cleanup_vm "$vm_name" + + # First boot should have 1 line, second boot should have 2 + # (boot-count.txt is in tmpfs, so it resets between VM stops — + # but the init command should run each time) + [[ "$count1" == *"1"* ]] +} + +# ============================================================================= +# Init in container (image-based machines) +# ============================================================================= +# +# When a machine is created with an image AND init commands, init runs +# inside the container's rootfs (not the bare Alpine agent), and the +# image is pulled before init runs. The bare-VM init tests above don't +# cover this — they pass even with init routed through the agent +# because they have no image. These tests boot a real VM, pull a real +# image, and observe init's actual execution context. + +# Init runs inside the container's rootfs, not the Alpine agent. Uses +# `command -v pacman` because pacman exists in archlinux but not in +# Alpine — if init ever regressed to running against the agent, this +# would fail with "command not found". +test_init_in_container_uses_image_rootfs() { + local vm_name="smolfile-init-container-$$" + cleanup_vm "$vm_name" + + # Uses debian:stable-slim because dpkg exists in Debian but not in the + # bare Alpine agent. If init ever regressed to running against the agent, + # `command -v dpkg` would fail with "command not found". + cat > "$SMOLFILE_TMPDIR/Smolfile.initcontainer" <<'EOF' +image = "debian:stable-slim" +cpus = 2 +memory = 1024 +net = true + +[dev] +init = ["command -v dpkg > /tmp/dpkg-path.txt"] +EOF + + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.initcontainer" 2>&1 || return 1 + + # Start should pull the image, then run init *in* the container. + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Init wrote dpkg's path inside the container's overlay. Verify + # via exec that it's there. + local output + output=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/dpkg-path.txt 2>&1) + + cleanup_vm "$vm_name" + + [[ "$output" == */dpkg ]] +} + +# Image pull happens before init, not after. The init command below +# depends on a file that only exists in the Debian rootfs — the +# container layers must be in place when init runs, or `test -f` would +# hit the bare Alpine agent and fail. Also asserts the user-visible +# ordering: "Pulling..." line precedes "Running N init command(s)..." +# in the start output. +test_init_runs_after_image_pull() { + local vm_name="smolfile-init-after-pull-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.initafter" <<'EOF' +image = "debian:stable-slim" +cpus = 2 +memory = 1024 +net = true + +[dev] +init = ["test -f /etc/debian_version"] +EOF + + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.initafter" 2>&1 || return 1 + + # Capture start output — must show pull happens before init runs. + local start_output + start_output=$($SMOLVM machine start --name "$vm_name" 2>&1) + local start_exit=$? + + cleanup_vm "$vm_name" + + # Either the start succeeded (init found arch-release, post-fix + # behavior), or the output contains "Pulling" before "Running". + if [[ $start_exit -ne 0 ]]; then + return 1 + fi + + # Sanity: pull line must precede init line in the output stream. + local pull_line init_line + pull_line=$(echo "$start_output" | grep -n "^Pulling " | head -1 | cut -d: -f1) + init_line=$(echo "$start_output" | grep -n "^Running .* init command" | head -1 | cut -d: -f1) + [[ -n "$pull_line" && -n "$init_line" && "$pull_line" -lt "$init_line" ]] +} + +# Init's filesystem changes persist into subsequent `machine exec`. +# An `apt-get install curl` during init must leave curl installed for +# follow-up exec calls — this is the whole point of running init at +# start time rather than expecting the operator to script it themselves. +# The persistent-overlay wiring in `build_init_run_config` is what makes +# this work; if the overlay ID ever drifts from the machine name, +# init's writes land in an overlay exec never sees. +test_init_in_container_persists_filesystem_changes() { + local vm_name="smolfile-init-persist-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.initpersist" <<'EOF' +image = "debian:stable-slim" +cpus = 2 +memory = 2048 +net = true + +[dev] +init = [ + "apt-get update -qq", + "apt-get install -y -qq curl", +] +EOF + + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.initpersist" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Exec must see the package installed by init. If the overlay ID + # is wrong, this exec runs in a fresh overlay and `curl --version` + # returns "command not found". + local output + output=$($SMOLVM machine exec --name "$vm_name" -- curl --version 2>&1) + + cleanup_vm "$vm_name" + + [[ "$output" == *"curl"* ]] +} + +# Init failure surfaces *both* stdout and stderr in the error message. +# Package managers commonly write failure diagnostics to stdout +# (apt's "Unable to locate package") — if the error only included +# stderr, the operator would be left with an exit code and no +# explanation. Asks apt for a package that doesn't exist; the error +# must contain "Unable to locate". +test_init_failure_surfaces_full_error() { + local vm_name="smolfile-init-fail-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.initfail" <<'EOF' +image = "debian:stable-slim" +cpus = 2 +memory = 1024 +net = true + +[dev] +init = ["apt-get install -y bogus-pkg-does-not-exist-xyz"] +EOF + + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.initfail" 2>&1 || return 1 + + # Start must fail; the error must include pacman's output explaining + # why. Capture stderr too — the error prints there. + local output + output=$($SMOLVM machine start --name "$vm_name" 2>&1) + local start_exit=$? + + cleanup_vm "$vm_name" + + # Failure expected, with apt's "Unable to locate package" diagnostic + # surfaced (was previously swallowed). + [[ $start_exit -ne 0 ]] && [[ "$output" == *"Unable to locate"* ]] +} + +# ============================================================================= +# --smolfile flag +# ============================================================================= + +test_smolfile_basic() { + local vm_name="smolfile-basic-$$" + cleanup_vm "$vm_name" + + # Write a Smolfile + cat > "$SMOLFILE_TMPDIR/Smolfile.basic" <<'EOF' +cpus = 2 +memory = 1024 + +init = [ + "echo 'smolfile-init-ran' > /tmp/smolfile-marker.txt", +] +EOF + + # Create VM from Smolfile + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.basic" 2>&1 || return 1 + + # Verify config was applied + local list_output + list_output=$($SMOLVM machine ls --json 2>&1) + if [[ "$list_output" != *'"cpus": 2'* ]] || [[ "$list_output" != *'"memory_mib": 1024'* ]]; then + echo "Smolfile cpus/memory not applied" + cleanup_vm "$vm_name" + return 1 + fi + + # Start and verify init ran + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + local output + output=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/smolfile-marker.txt 2>&1) + + cleanup_vm "$vm_name" + [[ "$output" == *"smolfile-init-ran"* ]] +} + +test_smolfile_with_env() { + local vm_name="smolfile-env-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.env" <<'EOF' +env = ["GREETING=hello_from_smolfile"] + +init = [ + 'echo "$GREETING" > /tmp/greeting.txt', +] +EOF + + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.env" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + local output + output=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/greeting.txt 2>&1) + + cleanup_vm "$vm_name" + [[ "$output" == *"hello_from_smolfile"* ]] +} + +test_smolfile_cli_overrides_scalars() { + local vm_name="smolfile-override-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.override" <<'EOF' +cpus = 2 +memory = 256 +EOF + + # CLI --mem should override Smolfile memory + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.override" --mem 1024 2>&1 || return 1 + + local list_output + list_output=$($SMOLVM machine ls --json 2>&1) + + cleanup_vm "$vm_name" + + # mem should be 1024 (CLI override), cpus should be 2 (from Smolfile) + [[ "$list_output" == *'"memory_mib": 1024'* ]] && [[ "$list_output" == *'"cpus": 2'* ]] +} + +test_smolfile_cli_extends_init() { + local vm_name="smolfile-extend-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.extend" <<'EOF' +init = [ + "echo 'from-smolfile' > /tmp/source.txt", +] +EOF + + # CLI --init should extend, not replace + $SMOLVM machine create --name "$vm_name" \ + --smolfile "$SMOLFILE_TMPDIR/Smolfile.extend" \ + --init "echo 'from-cli' > /tmp/cli-source.txt" \ + 2>&1 || return 1 + + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + local sf_out cli_out + sf_out=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/source.txt 2>&1) + cli_out=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/cli-source.txt 2>&1) + + cleanup_vm "$vm_name" + [[ "$sf_out" == *"from-smolfile"* ]] && [[ "$cli_out" == *"from-cli"* ]] +} + +test_smolfile_not_found_errors() { + local vm_name="smolfile-notfound-$$" + cleanup_vm "$vm_name" + + local exit_code=0 + $SMOLVM machine create --name "$vm_name" --smolfile "/nonexistent/Smolfile" 2>&1 || exit_code=$? + + cleanup_vm "$vm_name" + [[ $exit_code -ne 0 ]] +} + +test_smolfile_invalid_toml_errors() { + local vm_name="smolfile-invalid-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.bad" <<'EOF' +this is not valid toml {{{ +EOF + + local exit_code=0 + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.bad" 2>&1 || exit_code=$? + + cleanup_vm "$vm_name" + [[ $exit_code -ne 0 ]] +} + +test_smolfile_unknown_field_errors() { + local vm_name="smolfile-unknown-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.unknown" <<'EOF' +cpus = 2 +typo_field = "oops" +EOF + + local exit_code=0 + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.unknown" 2>&1 || exit_code=$? + + cleanup_vm "$vm_name" + [[ $exit_code -ne 0 ]] +} + +test_no_auto_detection() { + local vm_name="smolfile-noauto-$$" + cleanup_vm "$vm_name" + + # Create a Smolfile in the temp dir (simulating CWD) + cat > "$SMOLFILE_TMPDIR/Smolfile" <<'EOF' +cpus = 4 +memory = 2048 +init = ["echo 'should-not-run' > /tmp/noauto.txt"] +EOF + + # Create VM WITHOUT --smolfile, even though Smolfile exists in CWD + # The Smolfile should NOT be auto-detected + (cd "$SMOLFILE_TMPDIR" && $SMOLVM machine create --name "$vm_name" 2>&1) || return 1 + + # Verify default config was used (not Smolfile config) + local list_output + list_output=$($SMOLVM machine ls --json 2>&1) + + cleanup_vm "$vm_name" + + # cpus should be default (4), not overridden by the Smolfile's cpus=4 + # Since default and Smolfile value happen to match, also verify memory + # wasn't overridden (default 8192 vs Smolfile 2048) + [[ "$list_output" == *'"memory_mib": 8192'* ]] +} + +# ============================================================================= +# Smolfile v2: image, entrypoint, cmd fields +# ============================================================================= + +test_smolfile_image_field() { + local vm_name="smolfile-image-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.image" <<'EOF' +image = "alpine:latest" +cpus = 1 +memory = 512 +net = true +EOF + + # Create + start — image from Smolfile should be picked up + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.image" 2>&1 || return 1 + + # Verify image is persisted in the record + local list_output + list_output=$($SMOLVM machine ls --json 2>&1) + + cleanup_vm "$vm_name" + [[ "$list_output" == *"alpine:latest"* ]] +} + +test_smolfile_entrypoint_field() { + cat > "$SMOLFILE_TMPDIR/Smolfile.ep" <<'EOF' +entrypoint = ["/bin/echo"] +cmd = ["hello-from-smolfile"] +cpus = 1 +memory = 512 +EOF + + # Verify it parses without error + local exit_code=0 + local vm_name="smolfile-ep-$$" + cleanup_vm "$vm_name" + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.ep" 2>&1 || exit_code=$? + + cleanup_vm "$vm_name" + [[ $exit_code -eq 0 ]] +} + +test_smolfile_cmd_field() { + cat > "$SMOLFILE_TMPDIR/Smolfile.cmd" <<'EOF' +cmd = ["echo", "hello-from-cmd"] +cpus = 1 +memory = 512 +EOF + + local exit_code=0 + local vm_name="smolfile-cmd-$$" + cleanup_vm "$vm_name" + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.cmd" 2>&1 || exit_code=$? + + cleanup_vm "$vm_name" + [[ $exit_code -eq 0 ]] +} + +# ============================================================================= +# Smolfile v2: [artifact] and [dev] sections +# ============================================================================= + +test_smolfile_artifact_section_parses() { + cat > "$SMOLFILE_TMPDIR/Smolfile.artifact" <<'EOF' +image = "python:3.12-alpine" +cpus = 2 +memory = 1024 + +[artifact] +cpus = 4 +memory = 2048 +entrypoint = ["/app/run.sh"] +oci_platform = "linux/amd64" +EOF + + local exit_code=0 + local vm_name="smolfile-artifact-$$" + cleanup_vm "$vm_name" + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.artifact" 2>&1 || exit_code=$? + + cleanup_vm "$vm_name" + [[ $exit_code -eq 0 ]] +} + +test_smolfile_pack_alias_parses() { + cat > "$SMOLFILE_TMPDIR/Smolfile.pack" <<'EOF' +image = "alpine:latest" + +[pack] +cpus = 4 +memory = 2048 +EOF + + local exit_code=0 + local vm_name="smolfile-pack-$$" + cleanup_vm "$vm_name" + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.pack" 2>&1 || exit_code=$? + + cleanup_vm "$vm_name" + [[ $exit_code -eq 0 ]] +} + +test_smolfile_dev_section_parses() { + cat > "$SMOLFILE_TMPDIR/Smolfile.dev" <<'EOF' +image = "node:22-alpine" +cpus = 2 +memory = 512 +net = true + +[dev] +volumes = ["./src:/app"] +env = ["NODE_ENV=development"] +init = ["apk add --no-cache nodejs npm"] +workdir = "/app" +ports = ["3000:3000"] +EOF + + local exit_code=0 + local vm_name="smolfile-dev-$$" + cleanup_vm "$vm_name" + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.dev" 2>&1 || exit_code=$? + + cleanup_vm "$vm_name" + [[ $exit_code -eq 0 ]] +} + +test_smolfile_dev_init_used_for_machine() { + local vm_name="smolfile-devinit-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.devinit" <<'EOF' +cpus = 1 +memory = 512 + +[dev] +init = [ + "echo 'dev-init-ran' > /tmp/dev-marker.txt", +] +EOF + + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.devinit" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + local output + output=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/dev-marker.txt 2>&1) + + cleanup_vm "$vm_name" + [[ "$output" == *"dev-init-ran"* ]] +} + +# ============================================================================= +# Smolfile v2: [service], [health], [restart], [deploy] parse without error +# ============================================================================= + +test_smolfile_full_spec_parses() { + cat > "$SMOLFILE_TMPDIR/Smolfile.full" <<'EOF' +image = "ghcr.io/acme/api:1.2.3" +entrypoint = ["/app/api"] +cmd = ["serve"] +workdir = "/app" +env = ["PORT=8080"] +cpus = 2 +memory = 1024 +net = true + +[health] +exec = ["curl", "-f", "http://127.0.0.1:8080/health"] +interval = "10s" +timeout = "2s" +retries = 3 +startup_grace = "20s" + +[restart] +policy = "always" +max_retries = 5 + +[dev] +volumes = ["./src:/app"] +env = ["APP_MODE=development"] +init = ["cargo build"] +ports = ["8080:8080"] + +[artifact] +cpus = 4 +memory = 2048 +entrypoint = ["/app/api"] +oci_platform = "linux/amd64" + +[network] +allow_hosts = ["one.one.one.one"] +allow_cidrs = ["10.0.0.0/8"] + +[auth] +ssh_agent = true +EOF + + local exit_code=0 + local vm_name="smolfile-full-$$" + cleanup_vm "$vm_name" + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.full" 2>&1 || exit_code=$? + + cleanup_vm "$vm_name" + [[ $exit_code -eq 0 ]] +} + +test_smolfile_bare_vm_no_image() { + # Bare Alpine VM with [dev].init + entrypoint/cmd — no image needed + cat > "$SMOLFILE_TMPDIR/Smolfile.bare" <<'EOF' +entrypoint = ["cat"] +cmd = ["/tmp/bare.txt"] +cpus = 1 +memory = 512 + +[dev] +init = [ + "echo 'bare-vm-works' > /tmp/bare.txt", +] +EOF + + local output + output=$($SMOLVM machine run -s "$SMOLFILE_TMPDIR/Smolfile.bare" 2>&1) + + [[ "$output" == *"bare-vm-works"* ]] +} + +test_smolfile_bare_vm_detached() { + # Detached bare VM should start, run init, return without hanging, + # and be visible in machine ls (persisted as default). + cat > "$SMOLFILE_TMPDIR/Smolfile.bare_detach" <<'EOF' +cpus = 1 +memory = 512 + +[dev] +init = [ + "echo 'detach-init-ran' > /tmp/detach-marker.txt", +] +EOF + + # Stop any existing default machine + $SMOLVM machine stop 2>/dev/null || true + + local output + output=$($SMOLVM machine run -d -s "$SMOLFILE_TMPDIR/Smolfile.bare_detach" 2>&1) + + # Should print PID and return (not hang) + [[ "$output" == *"Machine running"* ]] || return 1 + + # Verify the init command ran + local marker + marker=$($SMOLVM machine exec -- cat /tmp/detach-marker.txt 2>&1) || return 1 + [[ "$marker" == *"detach-init-ran"* ]] || return 1 + + # Verify the VM is visible in machine ls (persisted) + local ls_output + ls_output=$($SMOLVM machine ls 2>&1) || return 1 + [[ "$ls_output" == *"default"* ]] || return 1 + [[ "$ls_output" == *"running"* ]] || return 1 + + # Clean up + $SMOLVM machine stop 2>/dev/null || true +} + +test_smolfile_bare_vm_detached_with_cmd() { + # Detached bare VM with entrypoint/cmd should run workload in background + cat > "$SMOLFILE_TMPDIR/Smolfile.bare_detach_cmd" <<'EOF' +entrypoint = ["sh"] +cmd = ["-c", "echo bg-workload-ran > /tmp/bg-marker.txt"] +cpus = 1 +memory = 512 +EOF + + $SMOLVM machine stop 2>/dev/null || true + + local output + output=$($SMOLVM machine run -d -s "$SMOLFILE_TMPDIR/Smolfile.bare_detach_cmd" 2>&1) || return 1 + + # Should return immediately with "Machine running" + [[ "$output" == *"Machine running"* ]] || return 1 + + # Give background process a moment to complete + sleep 1 + + # Verify the background workload ran + local marker + marker=$($SMOLVM machine exec -- cat /tmp/bg-marker.txt 2>&1) || return 1 + [[ "$marker" == *"bg-workload-ran"* ]] || return 1 + + $SMOLVM machine stop 2>/dev/null || true +} + +test_smolfile_bare_vm_detached_with_cli_cmd() { + # Detached bare VM with CLI command should run workload in background + $SMOLVM machine stop 2>/dev/null || true + local output + output=$($SMOLVM machine run -d -- sh -c "echo cli-bg-ran > /tmp/cli-bg-marker.txt" 2>&1) || return 1 + + [[ "$output" == *"Machine running"* ]] || return 1 + + sleep 1 + + local marker + marker=$($SMOLVM machine exec -- cat /tmp/cli-bg-marker.txt 2>&1) || return 1 + [[ "$marker" == *"cli-bg-ran"* ]] || return 1 + + $SMOLVM machine stop 2>/dev/null || true +} + +test_smolfile_entrypoint_used_at_runtime() { + # Verify entrypoint + cmd from Smolfile are combined and used + cat > "$SMOLFILE_TMPDIR/Smolfile.ep_runtime" <<'EOF' +entrypoint = ["echo"] +cmd = ["hello-from-entrypoint"] +cpus = 1 +memory = 512 +EOF + + local output + output=$($SMOLVM machine run -s "$SMOLFILE_TMPDIR/Smolfile.ep_runtime" 2>&1) + + [[ "$output" == *"hello-from-entrypoint"* ]] +} + +test_smolfile_auto_container_on_start() { + local vm_name="smolfile-autoct-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.autoct" <<'EOF' +image = "alpine:latest" +cmd = ["echo", "auto-container-works"] +cpus = 1 +memory = 512 +net = true +EOF + + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.autoct" 2>&1 || return 1 + + # Start should auto-pull image and create container + local start_output + start_output=$($SMOLVM machine start --name "$vm_name" 2>&1) + + cleanup_vm "$vm_name" + + # Should mention container creation + [[ "$start_output" == *"container:"* ]] || [[ "$start_output" == *"Pulling"* ]] +} + +test_smolfile_image_cmd_only_overrides_when_set() { + # When Smolfile has image but no entrypoint/cmd, the image's own + # entrypoint/cmd should be used (empty command vec to agent). + # When Smolfile sets cmd, it should override the image default. + cat > "$SMOLFILE_TMPDIR/Smolfile.cmdonly" <<'EOF' +image = "alpine:latest" +cmd = ["echo", "smolfile-cmd-works"] +cpus = 1 +memory = 512 +net = true +EOF + + local output + output=$($SMOLVM machine run -s "$SMOLFILE_TMPDIR/Smolfile.cmdonly" 2>&1) + + [[ "$output" == *"smolfile-cmd-works"* ]] +} + +test_smolfile_image_no_cmd_uses_image_default() { + # When Smolfile has image but no entrypoint/cmd, the image's own + # defaults should be used. For alpine, that's /bin/sh which expects + # stdin, so we pass a command via CLI to verify the image runs. + cat > "$SMOLFILE_TMPDIR/Smolfile.imgdefault" <<'EOF' +image = "alpine:latest" +cpus = 1 +memory = 512 +net = true +EOF + + local output + output=$($SMOLVM machine run -s "$SMOLFILE_TMPDIR/Smolfile.imgdefault" -- echo "image-default-ok" 2>&1) + + [[ "$output" == *"image-default-ok"* ]] +} + +test_smolfile_unknown_section_errors() { + cat > "$SMOLFILE_TMPDIR/Smolfile.badsection" <<'EOF' +cpus = 2 + +[nonexistent_section] +foo = "bar" +EOF + + local exit_code=0 + local vm_name="smolfile-badsec-$$" + cleanup_vm "$vm_name" + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.badsection" 2>&1 || exit_code=$? + + cleanup_vm "$vm_name" + [[ $exit_code -ne 0 ]] +} + +# ============================================================================= +# Verbose output +# ============================================================================= + +test_ls_verbose_shows_init() { + local vm_name="smolfile-verbose-$$" + cleanup_vm "$vm_name" + + $SMOLVM machine create --name "$vm_name" \ + --init "echo hello" \ + --init "echo world" \ + -e FOO=bar \ + -w /app \ + 2>&1 || return 1 + + local verbose_output + verbose_output=$($SMOLVM machine ls --verbose 2>&1) + + cleanup_vm "$vm_name" + + # Should show init commands, env, and workdir in verbose output + [[ "$verbose_output" == *"Init:"* ]] && \ + [[ "$verbose_output" == *"echo hello"* ]] && \ + [[ "$verbose_output" == *"Env:"* ]] && \ + [[ "$verbose_output" == *"FOO=bar"* ]] && \ + [[ "$verbose_output" == *"Workdir:"* ]] && \ + [[ "$verbose_output" == *"/app"* ]] +} + +# ============================================================================= +# Restart & Health Config Tests +# ============================================================================= + +test_smolfile_restart_policy() { + cat > "$SMOLFILE_TMPDIR/Smolfile.restart" <<'EOF' +cpus = 1 +memory = 512 + +[restart] +policy = "always" +max_retries = 5 +max_backoff = "30s" +EOF + + local vm_name="smolfile-restart-$$" + cleanup_vm "$vm_name" + + # Create should succeed (restart config stored in VmRecord) + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.restart" 2>&1 || return 1 + + # Verify machine was created + $SMOLVM machine ls --json 2>&1 | grep -q "$vm_name" || return 1 + + cleanup_vm "$vm_name" +} + +test_smolfile_health_config() { + cat > "$SMOLFILE_TMPDIR/Smolfile.health" <<'EOF' +cpus = 1 +memory = 512 + +[health] +exec = ["echo", "ok"] +interval = "10s" +timeout = "3s" +retries = 5 + +[restart] +policy = "on-failure" +EOF + + local vm_name="smolfile-health-$$" + cleanup_vm "$vm_name" + + # Create should succeed (health + restart config stored) + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.health" 2>&1 || return 1 + + # Verify machine was created + $SMOLVM machine ls --json 2>&1 | grep -q "$vm_name" || return 1 + + cleanup_vm "$vm_name" +} + +test_smolfile_monitor_basic() { + local vm_name="smolfile-monitor-$$" + cleanup_vm "$vm_name" + + # Also stop the default VM in case a previous test left it running + $SMOLVM machine stop 2>/dev/null || true + + # Create and start a machine + $SMOLVM machine create --net --name "$vm_name" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || return 1 + + # Verify the VM is running before testing monitor + $SMOLVM machine exec --name "$vm_name" -- echo "ready" 2>&1 | grep -q "ready" || return 1 + + # Run monitor briefly — it should print the header then we kill it + local output + output=$(run_with_timeout 8 $SMOLVM machine monitor --name "$vm_name" --interval 1 2>&1 || true) + + # Verify it printed the monitoring header + echo "$output" | grep -q "Monitoring" || return 1 + + cleanup_vm "$vm_name" +} + +# ============================================================================= +# Run Tests +# ============================================================================= + +run_test "Init flag creates marker file" test_init_flag_creates_file || true +run_test "Init flag multiple commands" test_init_flag_multiple_commands || true +run_test "Init flag with env" test_init_flag_with_env || true +run_test "Init flag with workdir" test_init_flag_with_workdir || true +run_test "Init runs on every start" test_init_runs_on_every_start || true +# Container-init behavioral contract: init runs inside the container +# rootfs, after the image is pulled, with a persistent overlay shared +# with `machine exec`, and failure messages surface both stdout and +# stderr. Each test pins one of those four properties. +run_test "Init in container: runs inside image rootfs" test_init_in_container_uses_image_rootfs || true +run_test "Init in container: pull precedes init" test_init_runs_after_image_pull || true +run_test "Init in container: filesystem changes visible to exec" test_init_in_container_persists_filesystem_changes || true +run_test "Init in container: failure surfaces full error output" test_init_failure_surfaces_full_error || true +run_test "Smolfile basic (cpus + init)" test_smolfile_basic || true +run_test "Smolfile with env" test_smolfile_with_env || true +run_test "Smolfile CLI overrides scalars" test_smolfile_cli_overrides_scalars || true +run_test "Smolfile CLI extends init" test_smolfile_cli_extends_init || true +run_test "Smolfile not found errors" test_smolfile_not_found_errors || true +run_test "Smolfile invalid TOML errors" test_smolfile_invalid_toml_errors || true +run_test "Smolfile unknown field errors" test_smolfile_unknown_field_errors || true +run_test "No auto-detection of Smolfile" test_no_auto_detection || true +run_test "ls --verbose shows init/env/workdir" test_ls_verbose_shows_init || true + +echo "" +echo "--- Smolfile v2 Tests ---" +echo "" + +run_test "Smolfile v2: image field" test_smolfile_image_field || true +run_test "Smolfile v2: entrypoint field" test_smolfile_entrypoint_field || true +run_test "Smolfile v2: cmd field" test_smolfile_cmd_field || true +run_test "Smolfile v2: [artifact] section parses" test_smolfile_artifact_section_parses || true +run_test "Smolfile v2: [pack] alias parses" test_smolfile_pack_alias_parses || true +run_test "Smolfile v2: [dev] section parses" test_smolfile_dev_section_parses || true +run_test "Smolfile v2: [dev] init used for machine" test_smolfile_dev_init_used_for_machine || true +run_test "Smolfile v2: full spec parses" test_smolfile_full_spec_parses || true +run_test "Smolfile v2: bare VM (no image)" test_smolfile_bare_vm_no_image || true +run_test "Smolfile v2: bare VM detached" test_smolfile_bare_vm_detached || true +run_test "Smolfile v2: bare VM detached + Smolfile cmd" test_smolfile_bare_vm_detached_with_cmd || true +run_test "Smolfile v2: bare VM detached + CLI cmd" test_smolfile_bare_vm_detached_with_cli_cmd || true +run_test "Smolfile v2: entrypoint used at runtime" test_smolfile_entrypoint_used_at_runtime || true +run_test "Smolfile v2: auto-container on start" test_smolfile_auto_container_on_start || true +run_test "Smolfile v2: image+cmd overrides image default" test_smolfile_image_cmd_only_overrides_when_set || true +run_test "Smolfile v2: image without cmd uses image default" test_smolfile_image_no_cmd_uses_image_default || true +run_test "Smolfile v2: unknown section errors" test_smolfile_unknown_section_errors || true + +echo "" +echo "--- Restart & Health Config Tests ---" +echo "" + +run_test "Smolfile: [restart] policy persists" test_smolfile_restart_policy || true +run_test "Smolfile: [health] config persists" test_smolfile_health_config || true +run_test "Smolfile: monitor starts and exits" test_smolfile_monitor_basic || true + +# ============================================================================= +# SSH Agent Forwarding Tests +# ============================================================================= + +test_ssh_agent_flag_creates_socket() { + local vm_name="ssh-agent-flag-$$" + cleanup_vm "$vm_name" + + # Create VM with --ssh-agent + $SMOLVM machine create --name "$vm_name" --ssh-agent --net 2>&1 || return 1 + + # Start VM + $SMOLVM machine start --name "$vm_name" 2>&1 || return 1 + + # Verify SSH agent socket exists in guest + local result + result=$($SMOLVM machine exec --name "$vm_name" -- ls /tmp/ssh-agent.sock 2>&1) || return 1 + [[ "$result" == *"ssh-agent.sock"* ]] || return 1 + + # Verify SSH_AUTH_SOCK is set in guest environment + result=$($SMOLVM machine exec --name "$vm_name" -- sh -c 'echo $SSH_AUTH_SOCK' 2>&1) || return 1 + [[ "$result" == *"/tmp/ssh-agent.sock"* ]] || return 1 + + cleanup_vm "$vm_name" +} + +test_ssh_agent_lists_host_keys() { + local vm_name="ssh-agent-keys-$$" + cleanup_vm "$vm_name" + + # Skip if no SSH agent on host + if ! ssh-add -l >/dev/null 2>&1; then + echo " SKIP (no SSH keys loaded on host)" + return 0 + fi + + $SMOLVM machine create --name "$vm_name" --ssh-agent --net 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || return 1 + + # Install openssh-client. apk may return non-zero from trigger + # scripts (e.g., busybox post-install) even on successful install. + # Verify the binary exists instead of trusting the exit code. + $SMOLVM machine exec --name "$vm_name" -- apk add openssh-client 2>/dev/null || true + $SMOLVM machine exec --name "$vm_name" -- which ssh-add 2>/dev/null || return 1 + + # ssh-add -l should list the same keys as host + local guest_keys host_keys + guest_keys=$($SMOLVM machine exec --name "$vm_name" -- ssh-add -l 2>&1) || return 1 + host_keys=$(ssh-add -l 2>&1) + + # Both should contain the same key fingerprint + local host_fp + host_fp=$(echo "$host_keys" | head -1 | awk '{print $2}') + [[ "$guest_keys" == *"$host_fp"* ]] || return 1 + + cleanup_vm "$vm_name" +} + +test_ssh_agent_not_present_without_flag() { + local vm_name="ssh-agent-absent-$$" + cleanup_vm "$vm_name" + + # Create VM WITHOUT --ssh-agent + $SMOLVM machine create --name "$vm_name" --net 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || return 1 + + # Socket should NOT exist + if $SMOLVM machine exec --name "$vm_name" -- ls /tmp/ssh-agent.sock 2>/dev/null; then + return 1 # Socket exists but shouldn't + fi + + cleanup_vm "$vm_name" +} + +test_ssh_agent_smolfile() { + cat > "$SMOLFILE_TMPDIR/Smolfile.ssh" <<'EOF' +cpus = 1 +memory = 512 +net = true + +[auth] +ssh_agent = true +EOF + + local vm_name="ssh-agent-smolfile-$$" + cleanup_vm "$vm_name" + + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.ssh" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || return 1 + + # Socket should exist (enabled via Smolfile) + local result + result=$($SMOLVM machine exec --name "$vm_name" -- ls /tmp/ssh-agent.sock 2>&1) || return 1 + [[ "$result" == *"ssh-agent.sock"* ]] || return 1 + + cleanup_vm "$vm_name" +} + +test_ssh_agent_fails_without_host_socket() { + local vm_name="ssh-agent-nosock-$$" + cleanup_vm "$vm_name" + + # Temporarily unset SSH_AUTH_SOCK + local saved_sock="$SSH_AUTH_SOCK" + unset SSH_AUTH_SOCK + + # Create should succeed (flag is just stored) + $SMOLVM machine create --name "$vm_name" --ssh-agent 2>&1 || { export SSH_AUTH_SOCK="$saved_sock"; return 1; } + + # Start should fail with clear error + local output + output=$($SMOLVM machine start --name "$vm_name" 2>&1) || true + export SSH_AUTH_SOCK="$saved_sock" + + [[ "$output" == *"SSH_AUTH_SOCK"* ]] || return 1 + + cleanup_vm "$vm_name" +} + +# Regression test for SSH agent forwarding into container exec. +# +# Image-based machines route `machine exec` through crun — the container +# has its own mount namespace and env, so forwarding that works for bare +# VMs (direct exec inherits the agent's env) silently breaks unless we +# both bind-mount the bridge socket and set SSH_AUTH_SOCK in the OCI spec. +# This test catches regressions in either piece. +test_ssh_agent_forwarded_to_container_exec() { + local vm_name="ssh-agent-container-$$" + cleanup_vm "$vm_name" + + # Skip if no SSH agent on host (can't verify end-to-end key listing) + if ! ssh-add -l >/dev/null 2>&1; then + echo " SKIP (no SSH keys loaded on host)" + return 0 + fi + + # Image-based machine: exec goes through crun container, not direct VM exec. + $SMOLVM machine create --name "$vm_name" --image alpine:latest --ssh-agent --net 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || return 1 + + # SSH_AUTH_SOCK must be set inside the container. + local result + result=$($SMOLVM machine exec --name "$vm_name" -- sh -c 'echo $SSH_AUTH_SOCK' 2>&1) || return 1 + if [[ "$result" != *"/tmp/ssh-agent.sock"* ]]; then + echo " FAIL: SSH_AUTH_SOCK not set in container: $result" + return 1 + fi + + # Socket file must be visible inside the container. + result=$($SMOLVM machine exec --name "$vm_name" -- ls /tmp/ssh-agent.sock 2>&1) || return 1 + if [[ "$result" != *"ssh-agent.sock"* ]]; then + echo " FAIL: bridge socket not mounted into container: $result" + return 1 + fi + + # End-to-end: inside the container, `ssh-add -l` should list the same + # keys as the host. This proves the bind mount carries real bytes, not + # a dangling file entry. + $SMOLVM machine exec --name "$vm_name" -- apk add openssh-client 2>/dev/null || true + $SMOLVM machine exec --name "$vm_name" -- which ssh-add 2>/dev/null || return 1 + + local guest_keys host_keys host_fp + guest_keys=$($SMOLVM machine exec --name "$vm_name" -- ssh-add -l 2>&1) || return 1 + host_keys=$(ssh-add -l 2>&1) + host_fp=$(echo "$host_keys" | head -1 | awk '{print $2}') + + if [[ "$guest_keys" != *"$host_fp"* ]]; then + echo " FAIL: container ssh-add -l did not return host keys" + echo " host: $host_keys" + echo " container: $guest_keys" + return 1 + fi + + cleanup_vm "$vm_name" +} + +echo "" +echo "--- SSH Agent Forwarding Tests ---" +echo "" + +run_test "SSH agent: --ssh-agent creates socket in guest" test_ssh_agent_flag_creates_socket || true +run_test "SSH agent: guest can list host keys" test_ssh_agent_lists_host_keys || true +run_test "SSH agent: socket absent without flag" test_ssh_agent_not_present_without_flag || true +run_test "SSH agent: Smolfile [auth] ssh_agent" test_ssh_agent_smolfile || true +run_test "SSH agent: fails when SSH_AUTH_SOCK missing" test_ssh_agent_fails_without_host_socket || true +run_test "SSH agent: forwarded into container exec (image-based VM)" test_ssh_agent_forwarded_to_container_exec || true + +# ============================================================================= +# [network] section — egress policy via Smolfile +# ============================================================================= + +test_smolfile_network_allow_hosts() { + local vm_name="smolfile-network-hosts-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/network.smolfile" <<'EOF' +net = true + +[network] +allow_hosts = ["one.one.one.one"] +EOF + + # Create VM from Smolfile with [network] section + $SMOLVM machine create --name "$vm_name" -s "$SMOLFILE_TMPDIR/network.smolfile" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Allowed host's IP should be reachable + local exit_code_allowed=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup cloudflare.com 1.1.1.1 2>&1 || exit_code_allowed=$? + + # Non-allowed IP should be blocked + local exit_code_blocked=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup cloudflare.com 8.8.8.8 2>&1 || exit_code_blocked=$? + + cleanup_vm "$vm_name" + + [[ $exit_code_allowed -eq 0 ]] && [[ $exit_code_blocked -ne 0 ]] +} + +test_smolfile_network_allow_cidrs() { + local vm_name="smolfile-network-cidrs-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/network-cidr.smolfile" <<'EOF' +net = true + +[network] +allow_cidrs = ["1.1.1.1/32"] +EOF + + $SMOLVM machine create --name "$vm_name" -s "$SMOLFILE_TMPDIR/network-cidr.smolfile" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Allowed CIDR should work + local exit_code_allowed=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup cloudflare.com 1.1.1.1 2>&1 || exit_code_allowed=$? + + # Non-allowed IP should be blocked + local exit_code_blocked=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup cloudflare.com 8.8.8.8 2>&1 || exit_code_blocked=$? + + cleanup_vm "$vm_name" + + [[ $exit_code_allowed -eq 0 ]] && [[ $exit_code_blocked -ne 0 ]] +} + +test_smolfile_network_mixed() { + local vm_name="smolfile-network-mixed-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/network-mixed.smolfile" <<'EOF' +net = true + +[network] +allow_hosts = ["one.one.one.one"] +allow_cidrs = ["8.8.8.0/24"] +EOF + + $SMOLVM machine create --name "$vm_name" -s "$SMOLFILE_TMPDIR/network-mixed.smolfile" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # Both should be reachable + local exit_code_host=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup cloudflare.com 1.1.1.1 2>&1 || exit_code_host=$? + + local exit_code_cidr=0 + $SMOLVM machine exec --name "$vm_name" -- nslookup cloudflare.com 8.8.8.8 2>&1 || exit_code_cidr=$? + + cleanup_vm "$vm_name" + + [[ $exit_code_host -eq 0 ]] && [[ $exit_code_cidr -eq 0 ]] +} + +echo "" +echo "--- [network] Section Tests ---" +echo "" + +run_test "Smolfile: [network] allow_hosts egress filtering" test_smolfile_network_allow_hosts || true +run_test "Smolfile: [network] allow_cidrs egress filtering" test_smolfile_network_allow_cidrs || true +run_test "Smolfile: [network] mixed hosts + cidrs" test_smolfile_network_mixed || true + +# ============================================================================= +# Workdir applied to machine exec (issue #107 related) +# ============================================================================= + +test_exec_inherits_smolfile_workdir() { + local vm_name="smolfile-exec-wd-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.execwd" <<'EOF' +cpus = 1 +memory = 512 +workdir = "/tmp" +EOF + + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.execwd" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # exec without --workdir should inherit Smolfile workdir + local output + output=$($SMOLVM machine exec --name "$vm_name" -- pwd 2>&1) + + cleanup_vm "$vm_name" + [[ "$output" == *"/tmp"* ]] +} + +test_exec_cli_workdir_overrides_smolfile() { + local vm_name="smolfile-exec-wdover-$$" + cleanup_vm "$vm_name" + + cat > "$SMOLFILE_TMPDIR/Smolfile.execwdover" <<'EOF' +cpus = 1 +memory = 512 +workdir = "/tmp" +EOF + + $SMOLVM machine create --name "$vm_name" --smolfile "$SMOLFILE_TMPDIR/Smolfile.execwdover" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { cleanup_vm "$vm_name"; return 1; } + + # exec with explicit -w should override Smolfile workdir + local output + output=$($SMOLVM machine exec --name "$vm_name" -w /var -- pwd 2>&1) + + cleanup_vm "$vm_name" + [[ "$output" == *"/var"* ]] +} + +echo "" +echo "--- Exec Workdir Tests ---" +echo "" + +run_test "Exec inherits Smolfile workdir" test_exec_inherits_smolfile_workdir || true +run_test "Exec CLI --workdir overrides Smolfile" test_exec_cli_workdir_overrides_smolfile || true + +print_summary "Smolfile Tests" diff --git a/tests/test_storage.sh b/tests/test_storage.sh new file mode 100755 index 0000000..2f5cd4a --- /dev/null +++ b/tests/test_storage.sh @@ -0,0 +1,336 @@ +#!/usr/bin/env bash +# +# Storage Tests (overlay, prune, resize) +# +# Part of the smolvm test suite. Run with: ./tests/test_storage.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Storage Tests (overlay, prune, resize)" +echo "==========================================" +echo "" + +test_machine_overlay_root_active() { + local vm_name="overlay-active-test-$$" + + # Clean up any existing + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create and start VM + $SMOLVM machine create --name "$vm_name" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Check that root is an overlay mount + local output exit_code=0 + output=$($SMOLVM machine exec --name "$vm_name" -- mount 2>&1) || exit_code=$? + + # Clean up + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ $exit_code -eq 0 ]] && [[ "$output" == *"overlay on / type overlay"* ]] +} + +test_machine_rootfs_persists_across_reboot() { + local vm_name="overlay-persist-test-$$" + + # Clean up any existing + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create and start VM + $SMOLVM machine create --name "$vm_name" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Write a marker file to the rootfs + local exit_code=0 + $SMOLVM machine exec --name "$vm_name" -- sh -c "echo persistence-test-ok > /tmp/overlay-test-marker" 2>&1 || exit_code=$? + if [[ $exit_code -ne 0 ]]; then + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Verify file exists before reboot + local output + output=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/overlay-test-marker 2>&1) || { + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + if [[ "$output" != *"persistence-test-ok"* ]]; then + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + fi + + # Stop and restart the VM + $SMOLVM machine stop --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + $SMOLVM machine start --name "$vm_name" 2>&1 || { $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1; } + + # Verify the file survived the reboot + exit_code=0 + output=$($SMOLVM machine exec --name "$vm_name" -- cat /tmp/overlay-test-marker 2>&1) || exit_code=$? + + # Clean up + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + [[ $exit_code -eq 0 ]] && [[ "$output" == *"persistence-test-ok"* ]] +} + +test_machine_overlay_size() { + local vm_name="test-vm-overlay-size" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create VM with custom overlay size (4 GiB) + $SMOLVM machine create --name "$vm_name" --overlay 4 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + return 1 + } + + # Check the overlay disk size inside the VM via df + local df_output + df_output=$($SMOLVM machine exec --name "$vm_name" -- df -m / 2>&1) + + # Cleanup + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + ensure_data_dir_deleted "$vm_name" + + # The 4GB overlay should show ~3800-4096 MB total (ext4 overhead) + # Just verify it's > 3000 MB (not the old 2GB default) + local total_mb + total_mb=$(echo "$df_output" | tail -1 | awk '{print $2}') + [[ "$total_mb" -gt 3000 ]] +} + +test_machine_images() { + # Start default machine and check images command + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine create --net --name default 2>/dev/null || true + $SMOLVM machine start --name default 2>/dev/null || true + + local output + output=$($SMOLVM machine images --name default 2>&1) + + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + + # Should show storage info + [[ "$output" == *"Storage"* ]] || [[ "$output" == *"storage"* ]] +} + +test_machine_prune_dry_run() { + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine create --net --name default 2>/dev/null || true + $SMOLVM machine start --name default 2>/dev/null || true + + local output + output=$($SMOLVM machine prune --name default --dry-run 2>&1) + + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + + # Should complete without error + [[ $? -eq 0 ]] || [[ "$output" == *"unreferenced"* ]] || [[ "$output" == *"No unreferenced"* ]] +} + +assert_vm_stays_running() { + local desc="$1"; shift + + ensure_machine_running + + local status + status=$($SMOLVM machine status 2>&1) + [[ "$status" == *"running"* ]] || { echo "VM not running before '$desc'"; return 1; } + + "$@" 2>&1 || return 1 + + status=$($SMOLVM machine status 2>&1) + [[ "$status" == *"running"* ]] || { echo "VM stopped after '$desc'"; return 1; } +} + +test_images_does_not_stop_running_vm() { + assert_vm_stays_running "machine images" $SMOLVM machine images --name default +} + +test_prune_on_running_vm() { + ensure_machine_running + + local status + status=$($SMOLVM machine status 2>&1) + [[ "$status" == *"running"* ]] || { echo "VM not running before test"; return 1; } + + # Regular prune should work without stopping the VM + local output + output=$($SMOLVM machine prune --name default 2>&1) || true + [[ "$output" == *"unreferenced"* ]] || [[ "$output" == *"No unreferenced"* ]] || { + echo "unexpected prune output: $output" + return 1 + } + + # VM should still be running after prune + status=$($SMOLVM machine status 2>&1) + [[ "$status" == *"running"* ]] || { echo "VM stopped after prune"; return 1; } +} + +test_prune_dry_run_on_running_vm() { + ensure_machine_running + + local output + output=$($SMOLVM machine prune --name default --dry-run 2>&1) || true + [[ "$output" == *"unreferenced"* ]] || [[ "$output" == *"No unreferenced"* ]] || { + echo "unexpected prune --dry-run output: $output" + return 1 + } + + # VM should still be running + local status + status=$($SMOLVM machine status 2>&1) + [[ "$status" == *"running"* ]] || { echo "VM stopped after prune --dry-run"; return 1; } +} + +test_prune_all_refuses_on_running_vm() { + ensure_machine_running "true" + + local status + status=$($SMOLVM machine status 2>&1) + [[ "$status" == *"running"* ]] || { echo "VM not running before test"; return 1; } + + # --all should refuse while the VM is running + local output exit_code=0 + output=$($SMOLVM machine prune --name default --all 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || { echo "prune --all should have failed on running VM"; return 1; } + [[ "$output" == *"cannot prune --all while machine"* ]] || { + echo "unexpected error: $output" + return 1 + } + + # VM should still be running + status=$($SMOLVM machine status 2>&1) + [[ "$status" == *"running"* ]] || { echo "VM stopped after rejected prune --all"; return 1; } +} + +test_prune_all_keeps_in_use_image() { + # An image-backed machine needs its cached image to restart, so `prune --all` + # must keep it (and reclaim only unreferenced layers) rather than brick a + # stopped machine with "image not found" on the next start. + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true + $SMOLVM machine create --image alpine --net --name default 2>&1 || return 1 + $SMOLVM machine start 2>&1 || return 1 + + # Verify the image is cached and the VM is reachable + $SMOLVM machine exec -- true 2>&1 || { echo "VM not reachable"; return 1; } + + # Stop the VM (prune --all requires it) + $SMOLVM machine stop 2>&1 + + # Run prune --all: it must keep the in-use image, not remove it. + local output + output=$($SMOLVM machine prune --name default --all 2>&1) || { + echo "prune --all failed: $output" + $SMOLVM machine delete --name default -f 2>/dev/null + return 1 + } + [[ "$output" == *"Kept"* ]] || [[ "$output" == *"image-backed"* ]] || { + echo "expected prune --all to keep the in-use image, got: $output" + $SMOLVM machine delete --name default -f 2>/dev/null + return 1 + } + + # The machine must still restart — its image was kept, not pruned. + $SMOLVM machine start 2>&1 || { + echo "FAIL: machine could not restart after prune --all (image was pruned)" + $SMOLVM machine delete --name default -f 2>/dev/null + return 1 + } + $SMOLVM machine exec -- true 2>&1 || { + echo "FAIL: VM not reachable after prune + restart" + $SMOLVM machine delete --name default -f 2>/dev/null + return 1 + } + + $SMOLVM machine stop 2>/dev/null || true + $SMOLVM machine delete --name default -f 2>/dev/null || true +} + +test_storage_resize_and_large_pull() { + # Force a fresh storage disk by deleting the default VM data directory. + # This ensures we exercise the template → resize → mount path. + $SMOLVM machine stop 2>/dev/null || true + local data_dir + data_dir=$(vm_data_dir "default") + rm -rf "$data_dir" 2>/dev/null || true + + # Pull python:3.12 (full image: ~150MB compressed, ~1GB extracted). + # This exceeds the 512MB template size, so it will fail with ENOSPC + # if the storage disk was not properly resized from 512MB to 20GB. + local output exit_code=0 + output=$($SMOLVM machine run --net --image python:3.12 -- python3 -c 'import sys; print(f"python {sys.version_info.major}.{sys.version_info.minor}")' 2>&1) || exit_code=$? + + echo "$output" + + # Verify the command succeeded and Python ran + [[ $exit_code -eq 0 ]] || { echo "Exit code: $exit_code"; return 1; } + [[ "$output" == *"python 3.12"* ]] || { echo "Expected python 3.12 output"; return 1; } +} + +test_storage_mounted_as_ext4() { + # Verify /dev/vda is actually mounted at /storage as ext4 (not on overlay). + # This catches the bug where mount_storage_disk() silently fails and + # /storage is just a directory on the overlay rootfs. + local output + output=$($SMOLVM machine run --net -- sh -c ' + mount_line=$(mount | grep "/dev/vda") + if [ -z "$mount_line" ]; then + echo "FAIL: /dev/vda not mounted" + exit 1 + fi + echo "$mount_line" + # Verify the filesystem is large (>1GB = properly resized from 512MB template) + avail_kb=$(df /storage | tail -1 | awk "{print \$4}") + if [ "$avail_kb" -lt 1048576 ]; then + echo "FAIL: /storage too small (${avail_kb}KB available, expected >1GB)" + exit 1 + fi + echo "PASS: storage mounted and resized" + ' 2>&1) + + echo "$output" + [[ "$output" == *"PASS: storage mounted and resized"* ]] +} + + +run_test "Overlay: root is overlayfs" test_machine_overlay_root_active || true +run_test "Overlay: rootfs persists across reboot" test_machine_rootfs_persists_across_reboot || true +run_test "Overlay: custom size via --overlay" test_machine_overlay_size || true +run_test "Machine images" test_machine_images || true +run_test "Machine prune --dry-run" test_machine_prune_dry_run || true +run_test "Images: does not stop running VM" test_images_does_not_stop_running_vm || true +run_test "Prune: works on running VM without stopping it" test_prune_on_running_vm || true +run_test "Prune --dry-run: works on running VM" test_prune_dry_run_on_running_vm || true +run_test "Prune --all: refuses on running VM" test_prune_all_refuses_on_running_vm || true +run_test "Prune --all: keeps in-use image, machine still restarts" test_prune_all_keeps_in_use_image || true +run_test "Storage: resize + large image pull (fresh disk)" test_storage_resize_and_large_pull || true +run_test "Storage: /dev/vda mounted as ext4 with correct size" test_storage_mounted_as_ext4 || true + +print_summary "Storage Tests" diff --git a/tests/test_virtio_net.sh b/tests/test_virtio_net.sh new file mode 100755 index 0000000..c04c3fb --- /dev/null +++ b/tests/test_virtio_net.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +# +# virtio-net tests for smolvm. +# +# This suite covers the user-visible launcher/runtime behavior from the staged +# virtio-net transplant: +# - part 3: the guest sees a configured virtio NIC and can use the host-side +# gateway for DNS and outbound TCP +# - part 4: the `create -> start -> exec`, `machine run`, and `pack run` flows +# all drive real virtio-backed guest networking +# - part 5: published TCP ports work end-to-end on the virtio gateway +# - unsupported policy features still fail clearly + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +TEST_DIR=$(mktemp -d) +trap "rm -rf '$TEST_DIR'; cleanup_machine" EXIT + +echo "" +echo "==========================================" +echo " smolvm Virtio-Net Tests" +echo "==========================================" +echo "" + +VIRTIO_TEST_IMAGE="${VIRTIO_TEST_IMAGE:-alpine:latest}" +VIRTIO_PUBLISH_TEST_IMAGE="${VIRTIO_PUBLISH_TEST_IMAGE:-python:3.12-alpine}" + +virtio_guest_probe_script() { + cat <<'EOF' +ip route | grep -F 'default via 100.96.0.1 dev eth0' && +ip route | grep -F '100.96.0.0/30 dev eth0' && +ip addr show dev eth0 | grep -F 'link/ether 02:53:4d:00:00:02' && +ip addr show dev eth0 | grep -F 'inet 100.96.0.2/30' && +nslookup example.com >/tmp/virtio-nslookup.out && +grep -F '100.96.0.1' /tmp/virtio-nslookup.out && +apk add --no-cache curl bash >/dev/null && +command -v curl >/dev/null && +command -v bash >/dev/null && +echo virtio-net-ok +EOF +} + +probe_running_virtio_guest_network() { + local vm_name="$1" + local output + local script + script=$(virtio_guest_probe_script) + + output=$($SMOLVM machine exec --name "$vm_name" -- sh -c "$script" 2>&1) || { + echo "virtio-net guest networking probe failed" + echo "$output" + return 1 + } + + [[ "$output" == *"virtio-net-ok"* ]] || { + echo "expected guest networking probe to finish successfully" + echo "$output" + return 1 + } +} + +test_machine_create_virtio_net_works() { + cleanup_machine + local vm_name="virtio-create-test-$$" + local output + + output=$($SMOLVM machine create --name "$vm_name" --net --net-backend virtio-net 2>&1) || { + echo "expected virtio-net machine create --name to succeed" + echo "$output" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + local list_output + list_output=$($SMOLVM machine ls --json 2>&1) + [[ "$list_output" == *"$vm_name"* ]] || { + echo "virtio-net create should persist machine state" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true +} + +test_machine_create_start_exec_virtio_net_works() { + cleanup_machine + local vm_name="virtio-create-start-exec-test-$$" + + $SMOLVM machine create --name "$vm_name" --image "$VIRTIO_TEST_IMAGE" --net --net-backend virtio-net >/dev/null 2>&1 || { + echo "expected virtio-net machine create --name to succeed before start" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + + $SMOLVM machine start --name "$vm_name" >/dev/null 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f >/dev/null 2>&1 || true + return 1 + } + probe_running_virtio_guest_network "$vm_name" || { + $SMOLVM machine stop --name "$vm_name" >/dev/null 2>&1 || true + $SMOLVM machine delete --name "$vm_name" -f >/dev/null 2>&1 || true + return 1 + } + + $SMOLVM machine stop --name "$vm_name" >/dev/null 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f >/dev/null 2>&1 || true + return 1 + } + $SMOLVM machine start --name "$vm_name" >/dev/null 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f >/dev/null 2>&1 || true + return 1 + } + probe_running_virtio_guest_network "$vm_name" || { + $SMOLVM machine stop --name "$vm_name" >/dev/null 2>&1 || true + $SMOLVM machine delete --name "$vm_name" -f >/dev/null 2>&1 || true + return 1 + } + + $SMOLVM machine stop --name "$vm_name" >/dev/null 2>&1 || true + $SMOLVM machine delete --name "$vm_name" -f >/dev/null 2>&1 || true +} + +test_machine_run_virtio_net_works() { + cleanup_machine + local output + local script + script=$(virtio_guest_probe_script) + + output=$($SMOLVM machine run --image "$VIRTIO_TEST_IMAGE" --net --net-backend virtio-net -- sh -c "$script" 2>&1) || { + echo "virtio-net machine run probe failed" + echo "$output" + return 1 + } + + [[ "$output" == *"virtio-net-ok"* ]] || { + echo "expected machine run virtio-net probe to finish successfully" + echo "$output" + return 1 + } +} + +test_machine_run_virtio_net_port_publishing_works() { + cleanup_machine + local host_port=$((41000 + ($$ % 10000))) + local serve_dir="$TEST_DIR/virtio-port-publish" + local run_log="$TEST_DIR/virtio-port-publish.log" + local run_pid=0 + local curl_output="" + local ready=0 + + mkdir -p "$serve_dir" + printf 'virtio-port-ok\n' >"$serve_dir/index.html" + + trap 'if [[ ${run_pid:-0} -ne 0 ]]; then kill "$run_pid" 2>/dev/null || true; wait "$run_pid" 2>/dev/null || true; fi; cleanup_machine' RETURN + + $SMOLVM machine run \ + --image "$VIRTIO_PUBLISH_TEST_IMAGE" \ + --net \ + --net-backend virtio-net \ + -v "$serve_dir:/srv:ro" \ + -p "${host_port}:8080" \ + -- python -m http.server 8080 --directory /srv >"$run_log" 2>&1 & + run_pid=$! + + for _ in $(seq 1 30); do + if curl_output=$(curl -fsS --connect-timeout 2 "http://127.0.0.1:${host_port}/" 2>&1); then + ready=1 + break + fi + if ! kill -0 "$run_pid" 2>/dev/null; then + echo "virtio-net machine run exited before published port became reachable" + tail -20 "$run_log" 2>/dev/null || true + return 1 + fi + sleep 1 + done + + [[ $ready -eq 1 ]] || { + echo "virtio-net published TCP port did not become reachable" + if [[ -n "$curl_output" ]]; then + echo "$curl_output" + fi + tail -20 "$run_log" 2>/dev/null || true + return 1 + } + + [[ "$curl_output" == *"virtio-port-ok"* ]] || { + echo "unexpected HTTP response from virtio-net published port" + echo "$curl_output" + return 1 + } + + trap - RETURN + kill "$run_pid" 2>/dev/null || true + wait "$run_pid" 2>/dev/null || true + cleanup_machine +} + +test_machine_create_virtio_net_policy_rejected() { + cleanup_machine + local vm_name="virtio-policy-test-$$" + local exit_code=0 + local output + + output=$($SMOLVM machine create --name "$vm_name" --net --net-backend virtio-net --allow-cidr 1.1.1.1/32 2>&1) || exit_code=$? + [[ $exit_code -ne 0 ]] || { + echo "expected create failure for virtio-net policy request" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + return 1 + } + [[ "$output" == *"allow-cidr/allow-host policies are not supported"* ]] || { + echo "unexpected output: $output" + return 1 + } +} + +test_pack_run_virtio_net_works() { + local output_path="$TEST_DIR/virtio-pack" + local output + local script + script=$(virtio_guest_probe_script) + + if [[ ! -f "$output_path.smolmachine" ]]; then + $SMOLVM pack create --image "$VIRTIO_TEST_IMAGE" -o "$output_path" >/dev/null 2>&1 || { + echo "expected pack create to succeed before pack run" + return 1 + } + fi + + output=$($SMOLVM pack run --sidecar "$output_path.smolmachine" --net --net-backend virtio-net -- sh -c "$script" 2>&1) || { + echo "virtio-net pack run probe failed" + echo "$output" + return 1 + } + + [[ "$output" == *"virtio-net-ok"* ]] || { + echo "expected pack run virtio-net probe to finish successfully" + echo "$output" + return 1 + } +} + +run_test "Machine create: virtio-net works" test_machine_create_virtio_net_works || true +run_test "Machine create/start/exec: virtio-net guest networking works" test_machine_create_start_exec_virtio_net_works || true +run_test "Machine run: virtio-net guest networking works" test_machine_run_virtio_net_works || true +run_test "Machine run: virtio-net published TCP ports work" test_machine_run_virtio_net_port_publishing_works || true +run_test "Machine create: virtio-net + policy rejected" test_machine_create_virtio_net_policy_rejected || true +run_test "Pack run: virtio-net guest networking works" test_pack_run_virtio_net_works || true + +print_summary "Virtio-Net Tests" diff --git a/tests/test_volumes.sh b/tests/test_volumes.sh new file mode 100755 index 0000000..d326bef --- /dev/null +++ b/tests/test_volumes.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +# +# Volume Mount Tests +# +# Part of the smolvm test suite. Run with: ./tests/test_volumes.sh +# + +source "$(dirname "$0")/common.sh" +init_smolvm + +log_info "Pre-flight cleanup: killing orphan processes..." +kill_orphan_smolvm_processes + +trap cleanup_machine EXIT + +echo "" +echo "==========================================" +echo " Volume Mount Tests" +echo "==========================================" +echo "" + +test_machine_volume_mount_visible_to_exec() { + local vm_name="test-vm-volmnt" + + # Clean up any existing + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + # Create a host directory with a test file + local tmpdir + tmpdir=$(mktemp -d) + echo "volume-mount-marker-54321" > "$tmpdir/testfile.txt" + + # Create and start VM with volume mount + $SMOLVM machine create --name "$vm_name" -v "$tmpdir:/mnt/hostdata" 2>&1 || { + rm -rf "$tmpdir" + return 1 + } + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$tmpdir" + return 1 + } + + # Read the file via machine exec (VmExec) — this exercises boot-time mount + local output + output=$($SMOLVM machine exec --name "$vm_name" -- cat /mnt/hostdata/testfile.txt 2>&1) + + # Cleanup + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$tmpdir" + ensure_data_dir_deleted "$vm_name" + + [[ "$output" == *"volume-mount-marker-54321"* ]] +} + +test_volume_mount_workspace_is_virtiofs_not_symlink() { + local vm_name="vol-ws-virtiofs-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + local tmpdir + tmpdir=$(mktemp -d) + echo "host-workspace-content" > "$tmpdir/host.txt" + + $SMOLVM machine create --name "$vm_name" -v "$tmpdir:/workspace" 2>&1 || { + rm -rf "$tmpdir"; return 1 + } + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$tmpdir"; return 1 + } + + # /workspace must NOT be a symlink + local link_check + link_check=$($SMOLVM machine exec --name "$vm_name" -- sh -c '[ -L /workspace ] && echo SYMLINK || echo MOUNT' 2>&1) + if [[ "$link_check" == *"SYMLINK"* ]]; then + echo "FAIL: /workspace is a symlink, expected virtiofs bind mount" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$tmpdir" + return 1 + fi + + # Host file must be visible + local content + content=$($SMOLVM machine exec --name "$vm_name" -- cat /workspace/host.txt 2>&1) + if [[ "$content" != *"host-workspace-content"* ]]; then + echo "FAIL: host file not visible at /workspace: $content" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$tmpdir" + return 1 + fi + + # Guest write must propagate to host + $SMOLVM machine exec --name "$vm_name" -- sh -c 'echo guest-wrote > /workspace/guest.txt' 2>&1 + if [[ ! -f "$tmpdir/guest.txt" ]] || [[ "$(cat "$tmpdir/guest.txt")" != *"guest-wrote"* ]]; then + echo "FAIL: guest write not visible on host" + $SMOLVM machine stop --name "$vm_name" 2>/dev/null; $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; rm -rf "$tmpdir" + return 1 + fi + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$tmpdir" +} + +test_volume_mount_arbitrary_path() { + local vm_name="vol-arb-path-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + local tmpdir + tmpdir=$(mktemp -d) + echo "arbitrary-path-content" > "$tmpdir/data.txt" + + $SMOLVM machine create --name "$vm_name" -v "$tmpdir:/data" 2>&1 || { + rm -rf "$tmpdir"; return 1 + } + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$tmpdir"; return 1 + } + + local content + content=$($SMOLVM machine exec --name "$vm_name" -- cat /data/data.txt 2>&1) + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$tmpdir" + + [[ "$content" == *"arbitrary-path-content"* ]] +} + +test_default_workspace_symlink_without_volume() { + local vm_name="vol-ws-default-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + $SMOLVM machine create --name "$vm_name" 2>&1 || return 1 + $SMOLVM machine start --name "$vm_name" 2>&1 || { + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null; return 1 + } + + # /workspace should be a symlink to /storage/workspace + local target + target=$($SMOLVM machine exec --name "$vm_name" -- readlink /workspace 2>&1) + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + [[ "$target" == *"/storage/workspace"* ]] +} + +test_image_exec_volume_mount_visible() { + local vm_name="imgexec-vol-test-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + local tmpdir + tmpdir=$(mktemp -d) + echo "exec-volume-regression-marker" > "$tmpdir/marker.txt" + + $SMOLVM machine create --name "$vm_name" --image alpine:latest --net \ + -v "$tmpdir:/hostdata" 2>&1 || { rm -rf "$tmpdir"; return 1; } + + local start_out + start_out=$(run_with_timeout 90 $SMOLVM machine start --name "$vm_name" 2>&1) + if [[ $? -eq 124 ]]; then + echo "TIMEOUT on start" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$tmpdir" + return 1 + fi + + local exec_out + exec_out=$(run_with_timeout 30 $SMOLVM machine exec --name "$vm_name" -- cat /hostdata/marker.txt 2>&1) + local exec_rc=$? + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$tmpdir" + + [[ $exec_rc -eq 124 ]] && { echo "TIMEOUT on exec"; return 1; } + [[ "$exec_out" == *"exec-volume-regression-marker"* ]] +} + +test_image_exec_volume_mount_visible_smolfile() { + local vm_name="imgexec-sf-vol-test-$$" + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + + local tmpdir + tmpdir=$(mktemp -d) + echo "smolfile-exec-volume-regression-marker" > "$tmpdir/marker.txt" + + # Write a Smolfile that uses a relative path (.:/app) — same shape as the + # user's repro. We cd into tmpdir so "." resolves to it. + cat > "$tmpdir/Smolfile.toml" <<'EOF' +image = "alpine:latest" +net = true +cpus = 1 +memory = 512 + +[dev] +volumes = [".:/app"] +EOF + + ( + cd "$tmpdir" + $SMOLVM machine create --name "$vm_name" -s Smolfile.toml 2>&1 + ) || { rm -rf "$tmpdir"; return 1; } + + local start_out + start_out=$(run_with_timeout 90 $SMOLVM machine start --name "$vm_name" 2>&1) + if [[ $? -eq 124 ]]; then + echo "TIMEOUT on start" + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null + rm -rf "$tmpdir" + return 1 + fi + + local exec_out + exec_out=$(run_with_timeout 30 $SMOLVM machine exec --name "$vm_name" -- cat /app/marker.txt 2>&1) + local exec_rc=$? + + $SMOLVM machine stop --name "$vm_name" 2>/dev/null || true + $SMOLVM machine delete --name "$vm_name" -f 2>/dev/null || true + rm -rf "$tmpdir" + + [[ $exec_rc -eq 124 ]] && { echo "TIMEOUT on exec"; return 1; } + [[ "$exec_out" == *"smolfile-exec-volume-regression-marker"* ]] +} + + +run_test "Volume: mount visible to exec" test_machine_volume_mount_visible_to_exec || true +run_test "Volume: -v host:/workspace is virtiofs not symlink" test_volume_mount_workspace_is_virtiofs_not_symlink || true +run_test "Volume: arbitrary mount path (/data)" test_volume_mount_arbitrary_path || true +run_test "Volume: default /workspace symlink without -v" test_default_workspace_symlink_without_volume || true +run_test "Create with --image: volume mount visible to exec" test_image_exec_volume_mount_visible || true +run_test "Create with --image: Smolfile volumes visible to exec" test_image_exec_volume_mount_visible_smolfile || true + +print_summary "Volume Tests" diff --git a/tests/test_wrapper.sh b/tests/test_wrapper.sh new file mode 100644 index 0000000..47d5af4 --- /dev/null +++ b/tests/test_wrapper.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# +# Wrapper tests for release archive behavior. +# +# These tests use a fake release layout so they do not require a built smolvm +# binary, libkrun, KVM, or an agent rootfs with real contents. + +source "$(dirname "$0")/common.sh" + +echo "" +echo "==========================================" +echo " smolvm Wrapper Tests" +echo "==========================================" +echo "" + +make_release_fixture() { + local release_dir="$1" + + mkdir -p "$release_dir/lib" + cp "$PROJECT_ROOT/scripts/smolvm-wrapper.sh" "$release_dir/smolvm" + chmod +x "$release_dir/smolvm" + + cat > "$release_dir/smolvm-bin" <<'EOF' +#!/usr/bin/env bash +printf 'SMOLVM_AGENT_ROOTFS=%s\n' "${SMOLVM_AGENT_ROOTFS-}" +EOF + chmod +x "$release_dir/smolvm-bin" +} + +agent_rootfs_from_output() { + printf '%s\n' "$1" | sed -n 's/^SMOLVM_AGENT_ROOTFS=//p' +} + +test_bundled_rootfs_sets_env_when_unset() { + local tmp release_dir output actual expected + tmp=$(mktemp -d) + release_dir="$tmp/release" + make_release_fixture "$release_dir" + mkdir -p "$release_dir/agent-rootfs" + + output=$(env -u SMOLVM_AGENT_ROOTFS "$release_dir/smolvm") + actual=$(agent_rootfs_from_output "$output") + expected="$release_dir/agent-rootfs" + + rm -rf "$tmp" + [[ "$actual" == "$expected" ]] +} + +test_existing_agent_rootfs_env_wins() { + local tmp release_dir output actual expected + tmp=$(mktemp -d) + release_dir="$tmp/release" + make_release_fixture "$release_dir" + mkdir -p "$release_dir/agent-rootfs" + + expected="/custom/rootfs" + output=$(SMOLVM_AGENT_ROOTFS="$expected" "$release_dir/smolvm") + actual=$(agent_rootfs_from_output "$output") + + rm -rf "$tmp" + [[ "$actual" == "$expected" ]] +} + +test_missing_bundled_rootfs_leaves_env_unset() { + local tmp release_dir output actual + tmp=$(mktemp -d) + release_dir="$tmp/release" + make_release_fixture "$release_dir" + + output=$(env -u SMOLVM_AGENT_ROOTFS "$release_dir/smolvm") + actual=$(agent_rootfs_from_output "$output") + + rm -rf "$tmp" + [[ -z "$actual" ]] +} + +test_symlinked_wrapper_uses_real_release_dir() { + local tmp release_dir bin_dir output actual expected + tmp=$(mktemp -d) + release_dir="$tmp/release" + bin_dir="$tmp/bin" + make_release_fixture "$release_dir" + mkdir -p "$release_dir/agent-rootfs" "$bin_dir" + ln -s "$release_dir/smolvm" "$bin_dir/smolvm" + + output=$(env -u SMOLVM_AGENT_ROOTFS "$bin_dir/smolvm") + actual=$(agent_rootfs_from_output "$output") + expected="$release_dir/agent-rootfs" + + rm -rf "$tmp" + [[ "$actual" == "$expected" ]] +} + +run_test "Bundled rootfs sets SMOLVM_AGENT_ROOTFS when unset" test_bundled_rootfs_sets_env_when_unset || true +run_test "Existing SMOLVM_AGENT_ROOTFS wins" test_existing_agent_rootfs_env_wins || true +run_test "Missing bundled rootfs leaves env unset" test_missing_bundled_rootfs_leaves_env_unset || true +run_test "Symlinked wrapper uses real release dir" test_symlinked_wrapper_uses_real_release_dir || true + +print_summary "Wrapper Tests"