wehub-resource-sync
426 lines
19 KiB
TypeScript
426 lines
19 KiB
TypeScript
#!/usr/bin/env bun
|
|
/**
|
|
* RTK Full Integration Test Suite — Multipass VM
|
|
*
|
|
* Usage:
|
|
* bun run scripts/benchmark/run.ts # Full suite
|
|
* bun run scripts/benchmark/run.ts --quick # Skip slow phases (perf, concurrency)
|
|
* bun run scripts/benchmark/run.ts --phase 3 # Run specific phase only
|
|
*
|
|
* Prerequisites:
|
|
* brew install multipass
|
|
*/
|
|
|
|
import { $ } from "bun";
|
|
import { vmEnsureReady, vmBuildRtk, vmExec, RTK_BIN } from "./lib/vm";
|
|
import { testCmd, testSavings, testRewrite, skipTest, getCounts } from "./lib/test";
|
|
import { saveReport } from "./lib/report";
|
|
|
|
const args = process.argv.slice(2);
|
|
const quick = args.includes("--quick");
|
|
const phaseArg = args.includes("--phase")
|
|
? parseInt(args[args.indexOf("--phase") + 1], 10)
|
|
: null;
|
|
const phaseOnly = phaseArg !== null && !Number.isNaN(phaseArg) ? phaseArg : null;
|
|
if (args.includes("--phase") && phaseOnly === null) {
|
|
console.error("Error: --phase requires a number (e.g. --phase 3)");
|
|
process.exit(1);
|
|
}
|
|
const reportPath = args.includes("--report")
|
|
? args[args.indexOf("--report") + 1]
|
|
: `${new URL("../../", import.meta.url).pathname.replace(/\/$/, "")}/benchmark-report.txt`;
|
|
|
|
const PROJECT_ROOT = new URL("../../", import.meta.url).pathname.replace(/\/$/, "");
|
|
const RTK = RTK_BIN;
|
|
|
|
function shouldRun(phase: number): boolean {
|
|
return phaseOnly === null || phaseOnly === phase;
|
|
}
|
|
|
|
function heading(phase: number, title: string) {
|
|
console.log(`\n\x1b[34m[Phase ${phase}] ${title}\x1b[0m`);
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 0: VM Setup
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
console.log("\x1b[34m[rtk-test] RTK Full Integration Test Suite\x1b[0m");
|
|
console.log(`Project: ${PROJECT_ROOT}`);
|
|
|
|
await vmEnsureReady();
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 1: Transfer & Build
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
heading(1, "Transfer & Build");
|
|
const branch = (await $`git -C ${PROJECT_ROOT} branch --show-current`.text()).trim();
|
|
const commit = (await $`git -C ${PROJECT_ROOT} log --oneline -1`.text()).trim();
|
|
const buildInfo = await vmBuildRtk(PROJECT_ROOT);
|
|
|
|
// Binary size check
|
|
// ARM Linux release binaries are ~6.5MB (vs ~4MB x86 stripped).
|
|
// CLAUDE.md target is <5MB for stripped x86 release builds.
|
|
// VM builds are ARM + not fully stripped, so we use a relaxed 8MB limit here.
|
|
const sizeLimit = 8_388_608; // 8MB (relaxed for ARM Linux VM)
|
|
if (buildInfo.binarySize < sizeLimit) {
|
|
console.log(` \x1b[32mPASS\x1b[0m | binary size | ${buildInfo.binarySize} bytes < 8MB`);
|
|
} else {
|
|
console.log(` \x1b[31mFAIL\x1b[0m | binary size | ${buildInfo.binarySize} bytes >= 8MB`);
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 2: Cargo Quality (fmt, clippy, test)
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
if (shouldRun(2)) {
|
|
heading(2, "Cargo Quality");
|
|
|
|
await testCmd(
|
|
"quality:cargo fmt",
|
|
"export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo fmt --all --check 2>&1"
|
|
);
|
|
|
|
await testCmd(
|
|
"quality:cargo clippy",
|
|
"export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo clippy --all-targets -- -D warnings 2>&1"
|
|
);
|
|
|
|
await testCmd(
|
|
"quality:cargo test",
|
|
"export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo test --all 2>&1"
|
|
);
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 3: Rust Built-in Commands
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
if (shouldRun(3)) {
|
|
heading(3, "Rust Built-in Commands");
|
|
|
|
// Git
|
|
await testCmd("git:status", `cd /tmp/test-git && ${RTK} git status`);
|
|
await testCmd("git:log", `cd /tmp/test-git && ${RTK} git log -5`);
|
|
await testCmd("git:log --oneline", `cd /tmp/test-git && ${RTK} git log --oneline -10`);
|
|
await testCmd("git:diff", `cd /tmp/test-git && ${RTK} git diff`, "any");
|
|
await testCmd("git:branch", `cd /tmp/test-git && ${RTK} git branch`);
|
|
await testCmd("git:add --dry-run", `cd /tmp/test-git && ${RTK} git add --dry-run .`, "any");
|
|
|
|
// Files
|
|
await testCmd("files:ls", `${RTK} ls /home/ubuntu/rtk`);
|
|
await testCmd("files:ls src/", `${RTK} ls /home/ubuntu/rtk/src/`);
|
|
await testCmd("files:ls -R", `${RTK} ls -R /home/ubuntu/rtk/src/`);
|
|
await testCmd("files:read", `${RTK} read /home/ubuntu/rtk/src/main.rs`);
|
|
await testCmd("files:read aggressive", `${RTK} read /home/ubuntu/rtk/src/main.rs -l aggressive`);
|
|
await testCmd("files:smart", `${RTK} smart /home/ubuntu/rtk/src/main.rs`);
|
|
await testCmd("files:find *.rs", `${RTK} find '*.rs' /home/ubuntu/rtk/src/`);
|
|
await testCmd("files:wc", `${RTK} wc /home/ubuntu/rtk/src/main.rs`);
|
|
await testCmd("files:diff", `${RTK} diff /home/ubuntu/rtk/src/main.rs /home/ubuntu/rtk/src/utils.rs`);
|
|
|
|
// Search
|
|
await testCmd("search:grep", `${RTK} grep 'fn main' /home/ubuntu/rtk/src/`);
|
|
|
|
// Data
|
|
await testCmd("data:json", `${RTK} json /tmp/test-node/package.json`);
|
|
await testCmd("data:deps", `cd /home/ubuntu/rtk && ${RTK} deps`);
|
|
await testCmd("data:env", `${RTK} env`);
|
|
|
|
// Runners
|
|
await testCmd("runner:summary", `${RTK} summary 'echo hello world'`);
|
|
// BUG: rtk err swallows exit code — tracked in #846
|
|
await testCmd("runner:err", `${RTK} err false`, "any");
|
|
await testCmd("runner:test", `${RTK} test 'echo ok'`, "any");
|
|
|
|
// Logs
|
|
await testCmd("log:large", `${RTK} log /tmp/large.log`);
|
|
|
|
// Network
|
|
await testCmd("net:curl", `${RTK} curl https://mockhttp.org/get`, "any");
|
|
|
|
// GitHub
|
|
await testCmd("gh:pr list", `cd /home/ubuntu/rtk && ${RTK} gh pr list`, "any");
|
|
|
|
// Cargo (test project has intentional test failure → exit 101)
|
|
await testCmd("cargo:build", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo build`);
|
|
await testCmd("cargo:test", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo test`, 101);
|
|
await testCmd("cargo:clippy", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo clippy`);
|
|
|
|
// Python (test project has intentional failures)
|
|
await testCmd("python:pytest", `cd /tmp/test-python && ${RTK} pytest`, 1);
|
|
await testCmd("python:ruff check", `cd /tmp/test-python && ${RTK} ruff check .`, 1);
|
|
await testCmd("python:mypy", `cd /tmp/test-python && ${RTK} mypy .`, 1);
|
|
await testCmd("python:pip list", `${RTK} pip list`);
|
|
|
|
// Go (test project has intentional test failure)
|
|
await testCmd("go:test", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go test ./...`, 1);
|
|
await testCmd("go:build", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go build .`, 1);
|
|
await testCmd("go:vet", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go vet ./...`, 1);
|
|
await testCmd("go:golangci-lint", `export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin && cd /tmp/test-go && ${RTK} golangci-lint run`, 1);
|
|
|
|
// TypeScript
|
|
await testCmd("ts:tsc", `cd /tmp/test-node && ${RTK} tsc --noEmit`, "any");
|
|
|
|
// Linters
|
|
await testCmd("lint:eslint", `cd /tmp/test-node && ${RTK} lint 'eslint src/'`, "any");
|
|
await testCmd("lint:prettier", `cd /tmp/test-node && ${RTK} prettier --check src/`, "any");
|
|
|
|
// Docker
|
|
await testCmd("docker:ps", `${RTK} docker ps`, "any");
|
|
await testCmd("docker:images", `${RTK} docker images`, "any");
|
|
|
|
// Kubernetes
|
|
await testCmd("k8s:pods", `${RTK} kubectl pods`, "any");
|
|
|
|
// .NET
|
|
await testCmd("dotnet:build", `export DOTNET_ROOT=/usr/local/share/dotnet && export PATH=$PATH:$DOTNET_ROOT && cd /tmp/test-dotnet/TestApp 2>/dev/null && ${RTK} dotnet build || echo 'dotnet skip'`, "any");
|
|
|
|
// Meta
|
|
await testCmd("meta:gain", `${RTK} gain`);
|
|
await testCmd("meta:gain --history", `${RTK} gain --history`);
|
|
await testCmd("meta:proxy", `${RTK} proxy echo 'proxy test'`);
|
|
await testCmd("meta:verify", `${RTK} verify`, "any");
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 4: TOML Filter Commands
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
if (shouldRun(4)) {
|
|
heading(4, "TOML Filter Commands");
|
|
|
|
// System
|
|
await testCmd("toml:df", `${RTK} df -h`);
|
|
await testCmd("toml:du", `${RTK} du -sh /tmp`, "any");
|
|
await testCmd("toml:ps", `${RTK} ps aux`);
|
|
await testCmd("toml:ping", `${RTK} ping -c 2 127.0.0.1`);
|
|
|
|
// Build tools
|
|
await testCmd("toml:make", `cd /tmp && ${RTK} make -f Makefile`, "any");
|
|
await testCmd("toml:rsync", `${RTK} rsync --version`);
|
|
|
|
// Linters
|
|
await testCmd("toml:shellcheck", `${RTK} shellcheck /tmp/test.sh`, "any");
|
|
await testCmd("toml:hadolint", `${RTK} hadolint /tmp/Dockerfile.bad`, "any");
|
|
await testCmd("toml:yamllint", `${RTK} yamllint /tmp/test.yaml`, "any");
|
|
await testCmd("toml:markdownlint", `${RTK} markdownlint /tmp/test.md`, "any");
|
|
|
|
// Cloud/Infra
|
|
await testCmd("toml:terraform", `${RTK} terraform --version`, "any");
|
|
await testCmd("toml:helm", `${RTK} helm version`, "any");
|
|
await testCmd("toml:ansible", `${RTK} ansible-playbook --version`, "any");
|
|
|
|
// Mocked tools
|
|
await testCmd("toml:gcloud", `${RTK} gcloud version`);
|
|
await testCmd("toml:shopify", `${RTK} shopify theme check`, "any");
|
|
await testCmd("toml:pio", `${RTK} pio run`, "any");
|
|
await testCmd("toml:quarto", `${RTK} quarto render`, "any");
|
|
await testCmd("toml:sops", `${RTK} sops --version`);
|
|
// Swift ecosystem
|
|
await testCmd("toml:swift build", `${RTK} swift build`, "any");
|
|
await testCmd("toml:swift test", `${RTK} swift test`, "any");
|
|
await testCmd("toml:swift run", `${RTK} swift run`, "any");
|
|
await testCmd("toml:swift package", `${RTK} swift package resolve`, "any");
|
|
await testCmd("toml:swiftlint", `${RTK} swiftlint`, "any");
|
|
await testCmd("toml:swiftformat", `${RTK} swiftformat`, "any");
|
|
await testCmd("toml:kubectl", `${RTK} kubectl version --client`, "any");
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 5: Hook Rewrite Engine
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
if (shouldRun(5)) {
|
|
heading(5, "Hook Rewrite Engine");
|
|
|
|
// Basic rewrites
|
|
await testRewrite("git status", "rtk git status");
|
|
await testRewrite("git log --oneline -10", "rtk git log --oneline -10");
|
|
await testRewrite("cargo test", "rtk cargo test");
|
|
await testRewrite("cargo build --release", "rtk cargo build --release");
|
|
await testRewrite("docker ps", "rtk docker ps");
|
|
// NOTE: rtk rewrites "kubectl get pods" to "rtk kubectl get pods" (preserves get)
|
|
await testRewrite("kubectl get pods", "rtk kubectl get pods");
|
|
await testRewrite("ruff check", "rtk ruff check");
|
|
await testRewrite("pytest", "rtk pytest");
|
|
await testRewrite("go test", "rtk go test");
|
|
await testRewrite("pnpm list", "rtk pnpm list");
|
|
await testRewrite("gh pr list", "rtk gh pr list");
|
|
await testRewrite("df -h", "rtk df -h");
|
|
await testRewrite("ps aux", "rtk ps aux");
|
|
|
|
// Compound
|
|
await testRewrite("cargo test && git status", "rtk cargo test && rtk git status");
|
|
// NOTE: shell strips single quotes in vmExec, so 'msg' becomes msg
|
|
await testRewrite("git add . && git commit -m msg", "rtk git add . && rtk git commit -m msg");
|
|
|
|
// No rewrite (shell builtins) — rtk rewrite returns empty string + exit 1
|
|
// We test via testCmd since testRewrite expects non-empty output
|
|
await testCmd("rewrite:cd (no rewrite)", `${RTK} rewrite 'cd /tmp'`, 1);
|
|
await testCmd("rewrite:export (no rewrite)", `${RTK} rewrite 'export FOO=bar'`, 1);
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 6: Exit Code Preservation
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
if (shouldRun(6)) {
|
|
heading(6, "Exit Code Preservation");
|
|
|
|
// Success
|
|
await testCmd("exit:git status=0", `cd /tmp/test-git && ${RTK} git status`, 0);
|
|
await testCmd("exit:ls=0", `${RTK} ls /tmp`, 0);
|
|
await testCmd("exit:gain=0", `${RTK} gain`, 0);
|
|
|
|
// Failures
|
|
// rg returns exit 1 (no match) or 2 (error) — accept both
|
|
await testCmd("exit:grep NOTFOUND", `${RTK} grep NOTFOUND_XYZ_123 /tmp`, "any");
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 7: Token Savings
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
if (shouldRun(7)) {
|
|
heading(7, "Token Savings");
|
|
|
|
await testSavings(
|
|
"savings:git log",
|
|
"cd /tmp/test-git && git log -20",
|
|
`cd /tmp/test-git && ${RTK} git log -20`,
|
|
60
|
|
);
|
|
await testSavings(
|
|
"savings:ls",
|
|
"ls -la /home/ubuntu/rtk/src/",
|
|
`${RTK} ls /home/ubuntu/rtk/src/`,
|
|
60
|
|
);
|
|
await testSavings(
|
|
"savings:log dedup",
|
|
"cat /tmp/large.log",
|
|
`${RTK} log /tmp/large.log`,
|
|
80
|
|
);
|
|
await testSavings(
|
|
"savings:read aggressive",
|
|
"cat /home/ubuntu/rtk/src/main.rs",
|
|
`${RTK} read /home/ubuntu/rtk/src/main.rs -l aggressive`,
|
|
50
|
|
);
|
|
await testSavings(
|
|
"savings:swift test",
|
|
"swift test",
|
|
`${RTK} swift test`,
|
|
60
|
|
);
|
|
await testSavings(
|
|
"savings:swiftlint",
|
|
"swiftlint",
|
|
`${RTK} swiftlint`,
|
|
20
|
|
);
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 8: Pipe Compatibility
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
if (shouldRun(8)) {
|
|
heading(8, "Pipe Compatibility");
|
|
|
|
await testCmd("pipe:git status|wc", `cd /tmp/test-git && ${RTK} git status | wc -l`);
|
|
await testCmd("pipe:ls|wc", `${RTK} ls /home/ubuntu/rtk/src/ | wc -l`);
|
|
await testCmd("pipe:grep|head", `${RTK} grep 'fn' /home/ubuntu/rtk/src/ | head -5`);
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 9: Edge Cases
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
if (shouldRun(9)) {
|
|
heading(9, "Edge Cases");
|
|
|
|
await testCmd("edge:summary true", `${RTK} summary 'true'`, "any");
|
|
await testCmd("edge:grep NOTFOUND", `${RTK} grep NOTFOUND_XYZ /home/ubuntu/rtk/src/`, 1);
|
|
await testCmd("edge:unicode", `echo 'hello world' > /tmp/uni.txt && ${RTK} grep 'hello' /tmp`, "any");
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 10: Performance (skip with --quick)
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
if (shouldRun(10) && !quick) {
|
|
heading(10, "Performance");
|
|
|
|
// hyperfine
|
|
const { exitCode: hfExist } = await vmExec("command -v hyperfine");
|
|
if (hfExist === 0) {
|
|
const { stdout: hfOut } = await vmExec(
|
|
`cd /tmp/test-git && hyperfine --warmup 3 --min-runs 5 '${RTK} git status' 'git status' --export-json /dev/stdout 2>/dev/null`
|
|
);
|
|
try {
|
|
const hf = JSON.parse(hfOut);
|
|
const rtkMean = (hf.results?.[0]?.mean * 1000).toFixed(1);
|
|
const rawMean = (hf.results?.[1]?.mean * 1000).toFixed(1);
|
|
console.log(` Startup: rtk=${rtkMean}ms raw=${rawMean}ms`);
|
|
} catch {
|
|
console.log(" hyperfine output parse failed");
|
|
}
|
|
} else {
|
|
skipTest("perf:hyperfine", "not installed");
|
|
}
|
|
|
|
// Memory
|
|
const { stdout: memOut } = await vmExec(
|
|
`cd /tmp/test-git && /usr/bin/time -v ${RTK} git status 2>&1 | grep 'Maximum resident'`
|
|
);
|
|
const memKb = parseInt(memOut.match(/(\d+)/)?.[1] ?? "0", 10);
|
|
if (memKb > 0 && memKb < 20000) {
|
|
await testCmd("perf:memory", `echo '${memKb} KB < 20MB'`);
|
|
} else if (memKb > 0) {
|
|
await testCmd("perf:memory", `echo '${memKb} KB >= 20MB' && exit 1`, 0);
|
|
}
|
|
} else if (quick && shouldRun(10)) {
|
|
skipTest("perf:hyperfine", "--quick mode");
|
|
skipTest("perf:memory", "--quick mode");
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Phase 11: Concurrency (skip with --quick)
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
if (shouldRun(11) && !quick) {
|
|
heading(11, "Concurrency");
|
|
|
|
await testCmd(
|
|
"concurrency:10x git status",
|
|
`cd /tmp/test-git && for i in $(seq 1 10); do ${RTK} git status >/dev/null & done; wait`
|
|
);
|
|
} else if (quick && shouldRun(11)) {
|
|
skipTest("concurrency:10x", "--quick mode");
|
|
}
|
|
|
|
// ══════════════════════════════════════════════════════════════
|
|
// Report
|
|
// ══════════════════════════════════════════════════════════════
|
|
|
|
const report = await saveReport(
|
|
{ ...buildInfo, branch, commit },
|
|
reportPath
|
|
);
|
|
|
|
console.log("\n" + report);
|
|
|
|
const { total, passed, failed, skipped } = getCounts();
|
|
const passRate = total > 0 ? Math.round((passed * 100) / total) : 0;
|
|
|
|
if (failed === 0) {
|
|
console.log(`\n\x1b[32m READY FOR RELEASE — ${passed}/${total} (${passRate}%)\x1b[0m\n`);
|
|
process.exit(0);
|
|
} else {
|
|
console.log(`\n\x1b[31m NOT READY — ${failed} failures — ${passed}/${total} (${passRate}%)\x1b[0m\n`);
|
|
process.exit(1);
|
|
}
|