#!/usr/bin/env bun /** * RTK Full Integration Test Suite — Multipass VM * * Usage: * bun run scripts/benchmark/run.ts # Full suite * bun run scripts/benchmark/run.ts --quick # Skip slow phases (perf, concurrency) * bun run scripts/benchmark/run.ts --phase 3 # Run specific phase only * * Prerequisites: * brew install multipass */ import { $ } from "bun"; import { vmEnsureReady, vmBuildRtk, vmExec, RTK_BIN } from "./lib/vm"; import { testCmd, testSavings, testRewrite, skipTest, getCounts } from "./lib/test"; import { saveReport } from "./lib/report"; const args = process.argv.slice(2); const quick = args.includes("--quick"); const phaseArg = args.includes("--phase") ? parseInt(args[args.indexOf("--phase") + 1], 10) : null; const phaseOnly = phaseArg !== null && !Number.isNaN(phaseArg) ? phaseArg : null; if (args.includes("--phase") && phaseOnly === null) { console.error("Error: --phase requires a number (e.g. --phase 3)"); process.exit(1); } const reportPath = args.includes("--report") ? args[args.indexOf("--report") + 1] : `${new URL("../../", import.meta.url).pathname.replace(/\/$/, "")}/benchmark-report.txt`; const PROJECT_ROOT = new URL("../../", import.meta.url).pathname.replace(/\/$/, ""); const RTK = RTK_BIN; function shouldRun(phase: number): boolean { return phaseOnly === null || phaseOnly === phase; } function heading(phase: number, title: string) { console.log(`\n\x1b[34m[Phase ${phase}] ${title}\x1b[0m`); } // ══════════════════════════════════════════════════════════════ // Phase 0: VM Setup // ══════════════════════════════════════════════════════════════ console.log("\x1b[34m[rtk-test] RTK Full Integration Test Suite\x1b[0m"); console.log(`Project: ${PROJECT_ROOT}`); await vmEnsureReady(); // ══════════════════════════════════════════════════════════════ // Phase 1: Transfer & Build // ══════════════════════════════════════════════════════════════ heading(1, "Transfer & Build"); const branch = (await $`git -C ${PROJECT_ROOT} branch --show-current`.text()).trim(); const commit = (await $`git -C ${PROJECT_ROOT} log --oneline -1`.text()).trim(); const buildInfo = await vmBuildRtk(PROJECT_ROOT); // Binary size check // ARM Linux release binaries are ~6.5MB (vs ~4MB x86 stripped). // CLAUDE.md target is <5MB for stripped x86 release builds. // VM builds are ARM + not fully stripped, so we use a relaxed 8MB limit here. const sizeLimit = 8_388_608; // 8MB (relaxed for ARM Linux VM) if (buildInfo.binarySize < sizeLimit) { console.log(` \x1b[32mPASS\x1b[0m | binary size | ${buildInfo.binarySize} bytes < 8MB`); } else { console.log(` \x1b[31mFAIL\x1b[0m | binary size | ${buildInfo.binarySize} bytes >= 8MB`); } // ══════════════════════════════════════════════════════════════ // Phase 2: Cargo Quality (fmt, clippy, test) // ══════════════════════════════════════════════════════════════ if (shouldRun(2)) { heading(2, "Cargo Quality"); await testCmd( "quality:cargo fmt", "export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo fmt --all --check 2>&1" ); await testCmd( "quality:cargo clippy", "export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo clippy --all-targets -- -D warnings 2>&1" ); await testCmd( "quality:cargo test", "export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo test --all 2>&1" ); } // ══════════════════════════════════════════════════════════════ // Phase 3: Rust Built-in Commands // ══════════════════════════════════════════════════════════════ if (shouldRun(3)) { heading(3, "Rust Built-in Commands"); // Git await testCmd("git:status", `cd /tmp/test-git && ${RTK} git status`); await testCmd("git:log", `cd /tmp/test-git && ${RTK} git log -5`); await testCmd("git:log --oneline", `cd /tmp/test-git && ${RTK} git log --oneline -10`); await testCmd("git:diff", `cd /tmp/test-git && ${RTK} git diff`, "any"); await testCmd("git:branch", `cd /tmp/test-git && ${RTK} git branch`); await testCmd("git:add --dry-run", `cd /tmp/test-git && ${RTK} git add --dry-run .`, "any"); // Files await testCmd("files:ls", `${RTK} ls /home/ubuntu/rtk`); await testCmd("files:ls src/", `${RTK} ls /home/ubuntu/rtk/src/`); await testCmd("files:ls -R", `${RTK} ls -R /home/ubuntu/rtk/src/`); await testCmd("files:read", `${RTK} read /home/ubuntu/rtk/src/main.rs`); await testCmd("files:read aggressive", `${RTK} read /home/ubuntu/rtk/src/main.rs -l aggressive`); await testCmd("files:smart", `${RTK} smart /home/ubuntu/rtk/src/main.rs`); await testCmd("files:find *.rs", `${RTK} find '*.rs' /home/ubuntu/rtk/src/`); await testCmd("files:wc", `${RTK} wc /home/ubuntu/rtk/src/main.rs`); await testCmd("files:diff", `${RTK} diff /home/ubuntu/rtk/src/main.rs /home/ubuntu/rtk/src/utils.rs`); // Search await testCmd("search:grep", `${RTK} grep 'fn main' /home/ubuntu/rtk/src/`); // Data await testCmd("data:json", `${RTK} json /tmp/test-node/package.json`); await testCmd("data:deps", `cd /home/ubuntu/rtk && ${RTK} deps`); await testCmd("data:env", `${RTK} env`); // Runners await testCmd("runner:summary", `${RTK} summary 'echo hello world'`); // BUG: rtk err swallows exit code — tracked in #846 await testCmd("runner:err", `${RTK} err false`, "any"); await testCmd("runner:test", `${RTK} test 'echo ok'`, "any"); // Logs await testCmd("log:large", `${RTK} log /tmp/large.log`); // Network await testCmd("net:curl", `${RTK} curl https://mockhttp.org/get`, "any"); // GitHub await testCmd("gh:pr list", `cd /home/ubuntu/rtk && ${RTK} gh pr list`, "any"); // Cargo (test project has intentional test failure → exit 101) await testCmd("cargo:build", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo build`); await testCmd("cargo:test", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo test`, 101); await testCmd("cargo:clippy", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo clippy`); // Python (test project has intentional failures) await testCmd("python:pytest", `cd /tmp/test-python && ${RTK} pytest`, 1); await testCmd("python:ruff check", `cd /tmp/test-python && ${RTK} ruff check .`, 1); await testCmd("python:mypy", `cd /tmp/test-python && ${RTK} mypy .`, 1); await testCmd("python:pip list", `${RTK} pip list`); // Go (test project has intentional test failure) await testCmd("go:test", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go test ./...`, 1); await testCmd("go:build", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go build .`, 1); await testCmd("go:vet", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go vet ./...`, 1); await testCmd("go:golangci-lint", `export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin && cd /tmp/test-go && ${RTK} golangci-lint run`, 1); // TypeScript await testCmd("ts:tsc", `cd /tmp/test-node && ${RTK} tsc --noEmit`, "any"); // Linters await testCmd("lint:eslint", `cd /tmp/test-node && ${RTK} lint 'eslint src/'`, "any"); await testCmd("lint:prettier", `cd /tmp/test-node && ${RTK} prettier --check src/`, "any"); // Docker await testCmd("docker:ps", `${RTK} docker ps`, "any"); await testCmd("docker:images", `${RTK} docker images`, "any"); // Kubernetes await testCmd("k8s:pods", `${RTK} kubectl pods`, "any"); // .NET await testCmd("dotnet:build", `export DOTNET_ROOT=/usr/local/share/dotnet && export PATH=$PATH:$DOTNET_ROOT && cd /tmp/test-dotnet/TestApp 2>/dev/null && ${RTK} dotnet build || echo 'dotnet skip'`, "any"); // Meta await testCmd("meta:gain", `${RTK} gain`); await testCmd("meta:gain --history", `${RTK} gain --history`); await testCmd("meta:proxy", `${RTK} proxy echo 'proxy test'`); await testCmd("meta:verify", `${RTK} verify`, "any"); } // ══════════════════════════════════════════════════════════════ // Phase 4: TOML Filter Commands // ══════════════════════════════════════════════════════════════ if (shouldRun(4)) { heading(4, "TOML Filter Commands"); // System await testCmd("toml:df", `${RTK} df -h`); await testCmd("toml:du", `${RTK} du -sh /tmp`, "any"); await testCmd("toml:ps", `${RTK} ps aux`); await testCmd("toml:ping", `${RTK} ping -c 2 127.0.0.1`); // Build tools await testCmd("toml:make", `cd /tmp && ${RTK} make -f Makefile`, "any"); await testCmd("toml:rsync", `${RTK} rsync --version`); // Linters await testCmd("toml:shellcheck", `${RTK} shellcheck /tmp/test.sh`, "any"); await testCmd("toml:hadolint", `${RTK} hadolint /tmp/Dockerfile.bad`, "any"); await testCmd("toml:yamllint", `${RTK} yamllint /tmp/test.yaml`, "any"); await testCmd("toml:markdownlint", `${RTK} markdownlint /tmp/test.md`, "any"); // Cloud/Infra await testCmd("toml:terraform", `${RTK} terraform --version`, "any"); await testCmd("toml:helm", `${RTK} helm version`, "any"); await testCmd("toml:ansible", `${RTK} ansible-playbook --version`, "any"); // Mocked tools await testCmd("toml:gcloud", `${RTK} gcloud version`); await testCmd("toml:shopify", `${RTK} shopify theme check`, "any"); await testCmd("toml:pio", `${RTK} pio run`, "any"); await testCmd("toml:quarto", `${RTK} quarto render`, "any"); await testCmd("toml:sops", `${RTK} sops --version`); // Swift ecosystem await testCmd("toml:swift build", `${RTK} swift build`, "any"); await testCmd("toml:swift test", `${RTK} swift test`, "any"); await testCmd("toml:swift run", `${RTK} swift run`, "any"); await testCmd("toml:swift package", `${RTK} swift package resolve`, "any"); await testCmd("toml:swiftlint", `${RTK} swiftlint`, "any"); await testCmd("toml:swiftformat", `${RTK} swiftformat`, "any"); await testCmd("toml:kubectl", `${RTK} kubectl version --client`, "any"); } // ══════════════════════════════════════════════════════════════ // Phase 5: Hook Rewrite Engine // ══════════════════════════════════════════════════════════════ if (shouldRun(5)) { heading(5, "Hook Rewrite Engine"); // Basic rewrites await testRewrite("git status", "rtk git status"); await testRewrite("git log --oneline -10", "rtk git log --oneline -10"); await testRewrite("cargo test", "rtk cargo test"); await testRewrite("cargo build --release", "rtk cargo build --release"); await testRewrite("docker ps", "rtk docker ps"); // NOTE: rtk rewrites "kubectl get pods" to "rtk kubectl get pods" (preserves get) await testRewrite("kubectl get pods", "rtk kubectl get pods"); await testRewrite("ruff check", "rtk ruff check"); await testRewrite("pytest", "rtk pytest"); await testRewrite("go test", "rtk go test"); await testRewrite("pnpm list", "rtk pnpm list"); await testRewrite("gh pr list", "rtk gh pr list"); await testRewrite("df -h", "rtk df -h"); await testRewrite("ps aux", "rtk ps aux"); // Compound await testRewrite("cargo test && git status", "rtk cargo test && rtk git status"); // NOTE: shell strips single quotes in vmExec, so 'msg' becomes msg await testRewrite("git add . && git commit -m msg", "rtk git add . && rtk git commit -m msg"); // No rewrite (shell builtins) — rtk rewrite returns empty string + exit 1 // We test via testCmd since testRewrite expects non-empty output await testCmd("rewrite:cd (no rewrite)", `${RTK} rewrite 'cd /tmp'`, 1); await testCmd("rewrite:export (no rewrite)", `${RTK} rewrite 'export FOO=bar'`, 1); } // ══════════════════════════════════════════════════════════════ // Phase 6: Exit Code Preservation // ══════════════════════════════════════════════════════════════ if (shouldRun(6)) { heading(6, "Exit Code Preservation"); // Success await testCmd("exit:git status=0", `cd /tmp/test-git && ${RTK} git status`, 0); await testCmd("exit:ls=0", `${RTK} ls /tmp`, 0); await testCmd("exit:gain=0", `${RTK} gain`, 0); // Failures // rg returns exit 1 (no match) or 2 (error) — accept both await testCmd("exit:grep NOTFOUND", `${RTK} grep NOTFOUND_XYZ_123 /tmp`, "any"); } // ══════════════════════════════════════════════════════════════ // Phase 7: Token Savings // ══════════════════════════════════════════════════════════════ if (shouldRun(7)) { heading(7, "Token Savings"); await testSavings( "savings:git log", "cd /tmp/test-git && git log -20", `cd /tmp/test-git && ${RTK} git log -20`, 60 ); await testSavings( "savings:ls", "ls -la /home/ubuntu/rtk/src/", `${RTK} ls /home/ubuntu/rtk/src/`, 60 ); await testSavings( "savings:log dedup", "cat /tmp/large.log", `${RTK} log /tmp/large.log`, 80 ); await testSavings( "savings:read aggressive", "cat /home/ubuntu/rtk/src/main.rs", `${RTK} read /home/ubuntu/rtk/src/main.rs -l aggressive`, 50 ); await testSavings( "savings:swift test", "swift test", `${RTK} swift test`, 60 ); await testSavings( "savings:swiftlint", "swiftlint", `${RTK} swiftlint`, 20 ); } // ══════════════════════════════════════════════════════════════ // Phase 8: Pipe Compatibility // ══════════════════════════════════════════════════════════════ if (shouldRun(8)) { heading(8, "Pipe Compatibility"); await testCmd("pipe:git status|wc", `cd /tmp/test-git && ${RTK} git status | wc -l`); await testCmd("pipe:ls|wc", `${RTK} ls /home/ubuntu/rtk/src/ | wc -l`); await testCmd("pipe:grep|head", `${RTK} grep 'fn' /home/ubuntu/rtk/src/ | head -5`); } // ══════════════════════════════════════════════════════════════ // Phase 9: Edge Cases // ══════════════════════════════════════════════════════════════ if (shouldRun(9)) { heading(9, "Edge Cases"); await testCmd("edge:summary true", `${RTK} summary 'true'`, "any"); await testCmd("edge:grep NOTFOUND", `${RTK} grep NOTFOUND_XYZ /home/ubuntu/rtk/src/`, 1); await testCmd("edge:unicode", `echo 'hello world' > /tmp/uni.txt && ${RTK} grep 'hello' /tmp`, "any"); } // ══════════════════════════════════════════════════════════════ // Phase 10: Performance (skip with --quick) // ══════════════════════════════════════════════════════════════ if (shouldRun(10) && !quick) { heading(10, "Performance"); // hyperfine const { exitCode: hfExist } = await vmExec("command -v hyperfine"); if (hfExist === 0) { const { stdout: hfOut } = await vmExec( `cd /tmp/test-git && hyperfine --warmup 3 --min-runs 5 '${RTK} git status' 'git status' --export-json /dev/stdout 2>/dev/null` ); try { const hf = JSON.parse(hfOut); const rtkMean = (hf.results?.[0]?.mean * 1000).toFixed(1); const rawMean = (hf.results?.[1]?.mean * 1000).toFixed(1); console.log(` Startup: rtk=${rtkMean}ms raw=${rawMean}ms`); } catch { console.log(" hyperfine output parse failed"); } } else { skipTest("perf:hyperfine", "not installed"); } // Memory const { stdout: memOut } = await vmExec( `cd /tmp/test-git && /usr/bin/time -v ${RTK} git status 2>&1 | grep 'Maximum resident'` ); const memKb = parseInt(memOut.match(/(\d+)/)?.[1] ?? "0", 10); if (memKb > 0 && memKb < 20000) { await testCmd("perf:memory", `echo '${memKb} KB < 20MB'`); } else if (memKb > 0) { await testCmd("perf:memory", `echo '${memKb} KB >= 20MB' && exit 1`, 0); } } else if (quick && shouldRun(10)) { skipTest("perf:hyperfine", "--quick mode"); skipTest("perf:memory", "--quick mode"); } // ══════════════════════════════════════════════════════════════ // Phase 11: Concurrency (skip with --quick) // ══════════════════════════════════════════════════════════════ if (shouldRun(11) && !quick) { heading(11, "Concurrency"); await testCmd( "concurrency:10x git status", `cd /tmp/test-git && for i in $(seq 1 10); do ${RTK} git status >/dev/null & done; wait` ); } else if (quick && shouldRun(11)) { skipTest("concurrency:10x", "--quick mode"); } // ══════════════════════════════════════════════════════════════ // Report // ══════════════════════════════════════════════════════════════ const report = await saveReport( { ...buildInfo, branch, commit }, reportPath ); console.log("\n" + report); const { total, passed, failed, skipped } = getCounts(); const passRate = total > 0 ? Math.round((passed * 100) / total) : 0; if (failed === 0) { console.log(`\n\x1b[32m READY FOR RELEASE — ${passed}/${total} (${passRate}%)\x1b[0m\n`); process.exit(0); } else { console.log(`\n\x1b[31m NOT READY — ${failed} failures — ${passed}/${total} (${passRate}%)\x1b[0m\n`); process.exit(1); }