chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Delete the RTK test VM.
|
||||
* Usage: bun run scripts/benchmark/cleanup.ts
|
||||
*/
|
||||
|
||||
import { vmDelete } from "./lib/vm";
|
||||
|
||||
console.log("Deleting rtk-test VM...");
|
||||
await vmDelete();
|
||||
console.log("Done.");
|
||||
@@ -0,0 +1,315 @@
|
||||
#cloud-config
|
||||
# RTK Integration Test VM — Ubuntu 24.04
|
||||
# Installs all tools needed for comprehensive RTK testing (~200 commands)
|
||||
# Usage: multipass launch --name rtk-test --cloud-init scripts/benchmark/cloud-init.yaml --cpus 2 --memory 4G --disk 20G 24.04
|
||||
|
||||
package_update: true
|
||||
package_upgrade: false
|
||||
|
||||
packages:
|
||||
# System tools
|
||||
- curl
|
||||
- wget
|
||||
- jq
|
||||
- git
|
||||
- make
|
||||
- cmake
|
||||
- rsync
|
||||
- sqlite3
|
||||
- shellcheck
|
||||
- yamllint
|
||||
- postgresql-client
|
||||
- docker.io
|
||||
- containerd
|
||||
- python3
|
||||
- python3-pip
|
||||
- python3-venv
|
||||
- pipx
|
||||
# Build essentials (for Rust compilation)
|
||||
- build-essential
|
||||
- pkg-config
|
||||
- libssl-dev
|
||||
- libsqlite3-dev
|
||||
# Misc
|
||||
- hyperfine
|
||||
- unzip
|
||||
- tree
|
||||
|
||||
runcmd:
|
||||
# ── Rust toolchain ──
|
||||
- su - ubuntu -c 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y'
|
||||
|
||||
# ── Node.js 22 + package managers ──
|
||||
- curl -fsSL https://deb.nodesource.com/setup_22.x | bash -
|
||||
- apt-get install -y nodejs
|
||||
- npm install -g pnpm yarn
|
||||
- npm install -g eslint prettier typescript
|
||||
- npm install -g markdownlint-cli
|
||||
|
||||
# ── Go 1.22 ──
|
||||
- curl -fsSL https://go.dev/dl/go1.22.5.linux-amd64.tar.gz | tar -C /usr/local -xz
|
||||
- echo 'export PATH=$PATH:/usr/local/go/bin:/home/ubuntu/go/bin' >> /home/ubuntu/.bashrc
|
||||
- su - ubuntu -c 'export PATH=$PATH:/usr/local/go/bin && go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest'
|
||||
|
||||
# ── Python tools ──
|
||||
- pipx install ruff
|
||||
- pipx install mypy
|
||||
- pipx install poetry
|
||||
- pip3 install --break-system-packages pytest uv pre-commit
|
||||
|
||||
# ── .NET 8 SDK ──
|
||||
- |
|
||||
wget https://dot.net/v1/dotnet-install.sh -O /tmp/dotnet-install.sh
|
||||
chmod +x /tmp/dotnet-install.sh
|
||||
/tmp/dotnet-install.sh --channel 8.0 --install-dir /usr/local/share/dotnet
|
||||
ln -sf /usr/local/share/dotnet/dotnet /usr/local/bin/dotnet
|
||||
echo 'export DOTNET_ROOT=/usr/local/share/dotnet' >> /home/ubuntu/.bashrc
|
||||
|
||||
# ── Terraform ──
|
||||
- |
|
||||
wget -qO- https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
|
||||
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/hashicorp.list
|
||||
apt-get update && apt-get install -y terraform
|
||||
|
||||
# ── Helm ──
|
||||
- curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
|
||||
|
||||
# ── Hadolint ──
|
||||
- |
|
||||
wget -qO /usr/local/bin/hadolint https://github.com/hadolint/hadolint/releases/latest/download/hadolint-Linux-x86_64
|
||||
chmod +x /usr/local/bin/hadolint
|
||||
|
||||
# ── Docker setup ──
|
||||
- usermod -aG docker ubuntu
|
||||
- systemctl enable docker
|
||||
- systemctl start docker
|
||||
|
||||
# ── kubectl (standalone binary) ──
|
||||
- |
|
||||
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
|
||||
install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
|
||||
rm kubectl
|
||||
|
||||
# ── ansible ──
|
||||
- pip3 install --break-system-packages ansible-core
|
||||
|
||||
# ── Mock tools (too heavy to install) ──
|
||||
- |
|
||||
cat > /usr/local/bin/gcloud << 'MOCK'
|
||||
#!/bin/bash
|
||||
if [ "$1" = "version" ] || [ "$1" = "--version" ]; then
|
||||
echo "Google Cloud SDK 400.0.0"
|
||||
echo "bq 2.0.80"
|
||||
echo "core 2023.01.01"
|
||||
echo "gsutil 5.17"
|
||||
else
|
||||
echo "gcloud mock: $*"
|
||||
fi
|
||||
MOCK
|
||||
chmod +x /usr/local/bin/gcloud
|
||||
|
||||
- |
|
||||
cat > /usr/local/bin/shopify << 'MOCK'
|
||||
#!/bin/bash
|
||||
echo "Shopify CLI 3.0.0 (mock)"
|
||||
if [ "$1" = "theme" ] && [ "$2" = "check" ]; then
|
||||
echo "Running theme check..."
|
||||
echo " 1 issue found"
|
||||
echo " [warn] Missing alt text on image"
|
||||
fi
|
||||
MOCK
|
||||
chmod +x /usr/local/bin/shopify
|
||||
|
||||
- |
|
||||
cat > /usr/local/bin/pio << 'MOCK'
|
||||
#!/bin/bash
|
||||
if [ "$1" = "--version" ]; then echo "PlatformIO Core, version 6.1.0"
|
||||
elif [ "$1" = "run" ]; then
|
||||
echo "Processing esp32dev (platform: espressif32; board: esp32dev)"
|
||||
echo "Linking .pio/build/esp32dev/firmware.elf"
|
||||
echo "========================= [SUCCESS] ========================="
|
||||
fi
|
||||
MOCK
|
||||
chmod +x /usr/local/bin/pio
|
||||
|
||||
- |
|
||||
cat > /usr/local/bin/quarto << 'MOCK'
|
||||
#!/bin/bash
|
||||
if [ "$1" = "--version" ]; then echo "1.3.450"
|
||||
elif [ "$1" = "render" ]; then echo "Rendering document..."; echo "Output created: document.html"
|
||||
fi
|
||||
MOCK
|
||||
chmod +x /usr/local/bin/quarto
|
||||
|
||||
- |
|
||||
cat > /usr/local/bin/sops << 'MOCK'
|
||||
#!/bin/bash
|
||||
if [ "$1" = "--version" ]; then echo "sops 3.7.3"; fi
|
||||
MOCK
|
||||
chmod +x /usr/local/bin/sops
|
||||
|
||||
- |
|
||||
cat > /usr/local/bin/swift << 'MOCK'
|
||||
#!/bin/bash
|
||||
if [ "$1" = "--version" ]; then echo "Swift version 5.9.2 (swift-5.9.2-RELEASE)"
|
||||
elif [ "$1" = "build" ]; then echo "Compiling Swift module..."; echo "Build complete! (0.42s)"
|
||||
fi
|
||||
MOCK
|
||||
chmod +x /usr/local/bin/swift
|
||||
|
||||
# ── Fake test projects ──
|
||||
|
||||
# Node.js project with errors
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
mkdir -p /tmp/test-node/src && cd /tmp/test-node
|
||||
npm init -y >/dev/null 2>&1
|
||||
echo "{\"compilerOptions\":{\"strict\":true,\"noEmit\":true,\"target\":\"ES2020\",\"module\":\"ESNext\",\"moduleResolution\":\"node\"},\"include\":[\"src\"]}" > tsconfig.json
|
||||
echo "const x: number = \"not a number\";\nconst unused = 42;\nfunction greet(name: string): string { return name }\ngreet(123);" > src/index.ts
|
||||
echo "{\"rules\":{\"no-unused-vars\":\"error\",\"semi\":[\"error\",\"always\"]}}" > .eslintrc.json
|
||||
echo "const x = 1;const y=2; const z =3" > src/ugly.ts
|
||||
'
|
||||
|
||||
# Python project with errors
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
mkdir -p /tmp/test-python && cd /tmp/test-python
|
||||
cat > main.py << "PYEOF"
|
||||
import os
|
||||
import sys
|
||||
unused_import = 1
|
||||
def add(a: int, b: int) -> str:
|
||||
return a + b
|
||||
x: int = "hello"
|
||||
PYEOF
|
||||
cat > test_main.py << "PYEOF"
|
||||
def test_pass():
|
||||
assert 1 + 1 == 2
|
||||
def test_fail():
|
||||
assert 1 + 1 == 3, "math is broken"
|
||||
PYEOF
|
||||
cat > pyproject.toml << "PYEOF"
|
||||
[tool.ruff]
|
||||
line-length = 80
|
||||
select = ["E", "F", "W"]
|
||||
[tool.mypy]
|
||||
strict = true
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["."]
|
||||
PYEOF
|
||||
'
|
||||
|
||||
# Go project with errors
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
mkdir -p /tmp/test-go && cd /tmp/test-go
|
||||
go mod init test-go 2>/dev/null
|
||||
cat > main.go << "GOEOF"
|
||||
package main
|
||||
import "fmt"
|
||||
func main() { fmt.Println("hello") }
|
||||
func unused() { var x int; _ = x }
|
||||
GOEOF
|
||||
cat > main_test.go << "GOEOF"
|
||||
package main
|
||||
import "testing"
|
||||
func TestPass(t *testing.T) { if 1+1 != 2 { t.Fatal("math") } }
|
||||
func TestFail(t *testing.T) { t.Fatal("expected failure") }
|
||||
GOEOF
|
||||
'
|
||||
|
||||
# Rust project with errors
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
export PATH=$HOME/.cargo/bin:$PATH
|
||||
mkdir -p /tmp/test-rust && cd /tmp/test-rust
|
||||
cargo init --name test-rust 2>/dev/null
|
||||
cat > src/main.rs << "RSEOF"
|
||||
fn main() {
|
||||
let x = vec![1, 2, 3];
|
||||
let _y = x.iter().map(|i| i.clone()).collect::<Vec<_>>();
|
||||
println!("hello");
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test] fn test_pass() { assert_eq!(1 + 1, 2); }
|
||||
#[test] fn test_fail() { assert_eq!(1 + 1, 3); }
|
||||
}
|
||||
RSEOF
|
||||
'
|
||||
|
||||
# Dockerfiles for hadolint
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
cat > /tmp/Dockerfile.bad << "DEOF"
|
||||
FROM ubuntu:latest
|
||||
RUN apt-get update && apt-get install -y curl wget git
|
||||
RUN cd /tmp && wget http://example.com/script.sh && bash script.sh
|
||||
EXPOSE 80 443 8080
|
||||
DEOF
|
||||
'
|
||||
|
||||
# Shell/YAML/Markdown test files
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
printf "#!/bin/bash\necho \$foo\nls *.txt\ncd \$(pwd)\n[ -f file ] && rm file\n" > /tmp/test.sh
|
||||
printf "foo: bar\nbaz: qux\nlist:\n - item1\n - item2\ntruthy: yes\n" > /tmp/test.yaml
|
||||
printf "#Header without space\nSome text\n\n* List item\n+ Mixed markers\n" > /tmp/test.md
|
||||
'
|
||||
|
||||
# Git repo for testing
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
mkdir -p /tmp/test-git && cd /tmp/test-git
|
||||
git init && git config user.email "test@rtk.dev" && git config user.name "RTK Test"
|
||||
for i in $(seq 1 20); do echo "line $i" >> file.txt && git add file.txt && git commit -m "feat: commit number $i"; done
|
||||
echo "modified" >> file.txt && echo "new file" > new.txt
|
||||
'
|
||||
|
||||
# Large log file for dedup testing
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
for i in $(seq 1 500); do
|
||||
printf "[2026-03-25 10:00:00] INFO Starting service...\n[2026-03-25 10:00:01] WARN Connection timeout\n[2026-03-25 10:00:01] ERROR Failed to connect: refused\n"
|
||||
done > /tmp/large.log
|
||||
for i in $(seq 1 50); do echo "[2026-03-25 10:05:00] FATAL Out of memory"; done >> /tmp/large.log
|
||||
'
|
||||
|
||||
# .env file
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
printf "DATABASE_URL=postgres://user:pass@localhost:5432/db\nAPI_KEY=sk-1234567890abcdef\nSECRET_TOKEN=ghp_xxxx\nNODE_ENV=production\nPORT=3000\n" > /tmp/.env
|
||||
'
|
||||
|
||||
# Makefile
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
printf ".PHONY: all test\nall:\n\t@echo Building...\n\t@echo Build complete\ntest:\n\t@echo Running tests...\n\t@echo 2 tests passed\n" > /tmp/Makefile
|
||||
'
|
||||
|
||||
# Terraform project
|
||||
- |
|
||||
su - ubuntu -c '
|
||||
mkdir -p /tmp/test-terraform && cd /tmp/test-terraform
|
||||
printf "terraform {\n required_version = \">= 1.0\"\n}\nresource \"null_resource\" \"test\" {\n triggers = { always = timestamp() }\n}\noutput \"test\" { value = \"hello\" }\n" > main.tf
|
||||
'
|
||||
|
||||
# Helm chart
|
||||
- su - ubuntu -c 'mkdir -p /tmp/test-helm && cd /tmp/test-helm && helm create test-chart 2>/dev/null || true'
|
||||
|
||||
# .NET project
|
||||
- |
|
||||
export DOTNET_ROOT=/usr/local/share/dotnet
|
||||
su - ubuntu -c '
|
||||
export DOTNET_ROOT=/usr/local/share/dotnet && export PATH=$PATH:$DOTNET_ROOT
|
||||
mkdir -p /tmp/test-dotnet && cd /tmp/test-dotnet
|
||||
dotnet new console -n TestApp --force 2>/dev/null || true
|
||||
'
|
||||
|
||||
# Signal completion
|
||||
- touch /home/ubuntu/.cloud-init-complete
|
||||
- chown ubuntu:ubuntu /home/ubuntu/.cloud-init-complete
|
||||
- echo "RTK cloud-init setup complete" | tee /var/log/rtk-setup.log
|
||||
|
||||
final_message: "RTK test VM ready in $UPTIME seconds"
|
||||
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* Report generation for RTK integration test results.
|
||||
*/
|
||||
|
||||
import type { TestResult } from "./test";
|
||||
import { getCounts, getResults } from "./test";
|
||||
|
||||
interface BuildInfo {
|
||||
buildTime: number;
|
||||
binarySize: number;
|
||||
version: string;
|
||||
branch: string;
|
||||
commit: string;
|
||||
}
|
||||
|
||||
export function generateReport(buildInfo: BuildInfo): string {
|
||||
const { total, passed, failed, skipped } = getCounts();
|
||||
const results = getResults();
|
||||
const passRate = total > 0 ? Math.round((passed * 100) / total) : 0;
|
||||
|
||||
const lines: string[] = [];
|
||||
|
||||
lines.push("======================================================");
|
||||
lines.push(" RTK INTEGRATION TEST REPORT");
|
||||
lines.push("======================================================");
|
||||
lines.push("");
|
||||
lines.push(`Date: ${new Date().toISOString()}`);
|
||||
lines.push(`Branch: ${buildInfo.branch}`);
|
||||
lines.push(`Commit: ${buildInfo.commit}`);
|
||||
lines.push(`Version: ${buildInfo.version}`);
|
||||
lines.push(`Binary: ${buildInfo.binarySize} bytes`);
|
||||
lines.push(`Build: ${buildInfo.buildTime}s`);
|
||||
lines.push("");
|
||||
|
||||
// Summary
|
||||
lines.push("--- Summary ---");
|
||||
lines.push(`Total: ${total}`);
|
||||
lines.push(`Passed: ${passed} (${passRate}%)`);
|
||||
lines.push(`Failed: ${failed}`);
|
||||
lines.push(`Skipped: ${skipped}`);
|
||||
lines.push("");
|
||||
|
||||
// Group results by phase (name prefix before ":")
|
||||
const phases = new Map<string, TestResult[]>();
|
||||
for (const r of results) {
|
||||
const colonIdx = r.name.indexOf(":");
|
||||
const phase = colonIdx > 0 ? r.name.slice(0, colonIdx) : "misc";
|
||||
if (!phases.has(phase)) phases.set(phase, []);
|
||||
phases.get(phase)!.push(r);
|
||||
}
|
||||
|
||||
for (const [phase, phaseResults] of phases) {
|
||||
const pPassed = phaseResults.filter((r) => r.status === "PASS").length;
|
||||
const pTotal = phaseResults.length;
|
||||
lines.push(`--- ${phase} (${pPassed}/${pTotal}) ---`);
|
||||
|
||||
for (const r of phaseResults) {
|
||||
const shortName = r.name.includes(":") ? r.name.split(":")[1] : r.name;
|
||||
lines.push(` ${r.status.padEnd(4)} | ${shortName} | ${r.detail}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
// Failures detail
|
||||
const failures = results.filter((r) => r.status === "FAIL");
|
||||
if (failures.length > 0) {
|
||||
lines.push("--- Failures ---");
|
||||
for (const f of failures) {
|
||||
lines.push(` ${f.name}: ${f.detail}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
// Token savings summary
|
||||
const savingsResults = results.filter((r) => r.savings !== undefined);
|
||||
if (savingsResults.length > 0) {
|
||||
const avgSavings = Math.round(
|
||||
savingsResults.reduce((sum, r) => sum + (r.savings ?? 0), 0) /
|
||||
savingsResults.length
|
||||
);
|
||||
const minSavings = Math.min(
|
||||
...savingsResults.map((r) => r.savings ?? 100)
|
||||
);
|
||||
const maxSavings = Math.max(...savingsResults.map((r) => r.savings ?? 0));
|
||||
lines.push("--- Token Savings ---");
|
||||
lines.push(`Average: ${avgSavings}%`);
|
||||
lines.push(`Min: ${minSavings}%`);
|
||||
lines.push(`Max: ${maxSavings}%`);
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
// Verdict
|
||||
lines.push("======================================================");
|
||||
if (failed === 0) {
|
||||
lines.push(" Verdict: READY FOR RELEASE");
|
||||
} else {
|
||||
lines.push(` Verdict: NOT READY (${failed} failures)`);
|
||||
}
|
||||
lines.push("======================================================");
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
/** Save report to file */
|
||||
export async function saveReport(
|
||||
buildInfo: BuildInfo,
|
||||
outPath: string
|
||||
): Promise<string> {
|
||||
const report = generateReport(buildInfo);
|
||||
await Bun.write(outPath, report);
|
||||
console.log(`\nReport saved to: ${outPath}`);
|
||||
return report;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/**
|
||||
* Test helpers for RTK integration testing.
|
||||
*/
|
||||
|
||||
import { vmExec, RTK_BIN } from "./vm";
|
||||
|
||||
export type TestStatus = "PASS" | "FAIL" | "SKIP";
|
||||
|
||||
export interface TestResult {
|
||||
name: string;
|
||||
status: TestStatus;
|
||||
detail: string;
|
||||
exitCode?: number;
|
||||
outputSize?: number;
|
||||
savings?: number;
|
||||
duration?: number;
|
||||
}
|
||||
|
||||
const results: TestResult[] = [];
|
||||
|
||||
export function getResults(): TestResult[] {
|
||||
return results;
|
||||
}
|
||||
|
||||
export function getCounts() {
|
||||
const total = results.length;
|
||||
const passed = results.filter((r) => r.status === "PASS").length;
|
||||
const failed = results.filter((r) => r.status === "FAIL").length;
|
||||
const skipped = results.filter((r) => r.status === "SKIP").length;
|
||||
return { total, passed, failed, skipped };
|
||||
}
|
||||
|
||||
function record(result: TestResult) {
|
||||
results.push(result);
|
||||
const icon =
|
||||
result.status === "PASS"
|
||||
? "\x1b[32mPASS\x1b[0m"
|
||||
: result.status === "FAIL"
|
||||
? "\x1b[31mFAIL\x1b[0m"
|
||||
: "\x1b[33mSKIP\x1b[0m";
|
||||
console.log(` ${icon} | ${result.name} | ${result.detail}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test a command exits with expected code and doesn't crash.
|
||||
* expectedExit: number or "any" (just checks no signal death)
|
||||
*/
|
||||
export async function testCmd(
|
||||
name: string,
|
||||
cmd: string,
|
||||
expectedExit: number | "any" = 0
|
||||
): Promise<TestResult> {
|
||||
const start = Date.now();
|
||||
const { stdout, stderr, exitCode } = await vmExec(cmd);
|
||||
const duration = Date.now() - start;
|
||||
const outputSize = stdout.length + stderr.length;
|
||||
|
||||
let status: TestStatus;
|
||||
let detail: string;
|
||||
|
||||
if (expectedExit === "any") {
|
||||
// Just check it didn't die from signal (exit >= 128)
|
||||
if (exitCode < 128) {
|
||||
status = "PASS";
|
||||
detail = `exit=${exitCode} | ${outputSize}b | ${duration}ms`;
|
||||
} else {
|
||||
status = "FAIL";
|
||||
detail = `SIGNAL exit=${exitCode} | ${outputSize}b`;
|
||||
}
|
||||
} else if (exitCode === expectedExit) {
|
||||
status = "PASS";
|
||||
detail = `exit=${exitCode} | ${outputSize}b | ${duration}ms`;
|
||||
} else {
|
||||
status = "FAIL";
|
||||
detail = `expected exit=${expectedExit}, got ${exitCode} | ${outputSize}b`;
|
||||
}
|
||||
|
||||
const result: TestResult = {
|
||||
name,
|
||||
status,
|
||||
detail,
|
||||
exitCode,
|
||||
outputSize,
|
||||
duration,
|
||||
};
|
||||
record(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test token savings: compare raw command output vs RTK filtered output.
|
||||
*/
|
||||
export async function testSavings(
|
||||
name: string,
|
||||
rawCmd: string,
|
||||
rtkCmd: string,
|
||||
targetPct: number
|
||||
): Promise<TestResult> {
|
||||
const raw = await vmExec(rawCmd);
|
||||
const rtk = await vmExec(rtkCmd);
|
||||
|
||||
const rawSize = raw.stdout.length;
|
||||
const rtkSize = rtk.stdout.length;
|
||||
|
||||
if (rawSize === 0) {
|
||||
const result: TestResult = {
|
||||
name,
|
||||
status: "SKIP",
|
||||
detail: "raw output empty",
|
||||
};
|
||||
record(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
const savings = Math.round(100 - (rtkSize * 100) / rawSize);
|
||||
|
||||
let status: TestStatus;
|
||||
let detail: string;
|
||||
|
||||
if (savings >= targetPct) {
|
||||
status = "PASS";
|
||||
detail = `raw=${rawSize}b filtered=${rtkSize}b savings=${savings}% (target: >=${targetPct}%)`;
|
||||
} else {
|
||||
status = "FAIL";
|
||||
detail = `savings=${savings}% < target ${targetPct}% (raw=${rawSize}b filtered=${rtkSize}b)`;
|
||||
}
|
||||
|
||||
const result: TestResult = { name, status, detail, savings };
|
||||
record(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test rewrite engine: input -> expected output.
|
||||
*/
|
||||
export async function testRewrite(
|
||||
input: string,
|
||||
expected: string
|
||||
): Promise<TestResult> {
|
||||
const escaped = input.replace(/'/g, "'\\''");
|
||||
const { stdout } = await vmExec(`${RTK_BIN} rewrite '${escaped}'`);
|
||||
const actual = stdout.trim();
|
||||
|
||||
let status: TestStatus;
|
||||
let detail: string;
|
||||
|
||||
if (actual === expected) {
|
||||
status = "PASS";
|
||||
detail = `'${input}' -> '${actual}'`;
|
||||
} else {
|
||||
status = "FAIL";
|
||||
detail = `'${input}' -> expected '${expected}', got '${actual}'`;
|
||||
}
|
||||
|
||||
const result: TestResult = { name: `rewrite: ${input}`, status, detail };
|
||||
record(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Skip a test with a reason.
|
||||
*/
|
||||
export function skipTest(name: string, reason: string): TestResult {
|
||||
const result: TestResult = { name, status: "SKIP", detail: reason };
|
||||
record(result);
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
/**
|
||||
* Multipass VM management for RTK integration testing.
|
||||
*/
|
||||
|
||||
import { $ } from "bun";
|
||||
|
||||
const VM_NAME = "rtk-test";
|
||||
const CLOUD_INIT = "scripts/benchmark/cloud-init.yaml";
|
||||
|
||||
export interface VmInfo {
|
||||
name: string;
|
||||
state: string;
|
||||
ipv4: string;
|
||||
}
|
||||
|
||||
/** Check if VM exists and is running */
|
||||
export async function vmExists(): Promise<boolean> {
|
||||
const result = await $`multipass list --format json`.quiet();
|
||||
const data = JSON.parse(result.stdout.toString());
|
||||
return data.list?.some((vm: VmInfo) => vm.name === VM_NAME) ?? false;
|
||||
}
|
||||
|
||||
/** Check if VM is running */
|
||||
export async function vmRunning(): Promise<boolean> {
|
||||
const result = await $`multipass list --format json`.quiet();
|
||||
const data = JSON.parse(result.stdout.toString());
|
||||
const vm = data.list?.find((v: VmInfo) => v.name === VM_NAME);
|
||||
return vm?.state === "Running";
|
||||
}
|
||||
|
||||
/** Create a new VM with cloud-init (20 min timeout for full provisioning) */
|
||||
export async function vmCreate(): Promise<void> {
|
||||
console.log(`[vm] Creating ${VM_NAME} with cloud-init (this takes ~10-15 min)...`);
|
||||
// --timeout 1200 = 20 min for cloud-init to finish installing Rust, Go, Node, .NET, etc.
|
||||
await $`multipass launch --name ${VM_NAME} --cpus 2 --memory 4G --disk 20G --timeout 1200 --cloud-init ${CLOUD_INIT} 24.04`;
|
||||
}
|
||||
|
||||
/** Start existing VM */
|
||||
export async function vmStart(): Promise<void> {
|
||||
console.log(`[vm] Starting ${VM_NAME}...`);
|
||||
await $`multipass start ${VM_NAME}`;
|
||||
}
|
||||
|
||||
/** Execute a command in the VM, returns stdout (60s timeout per test by default) */
|
||||
export async function vmExec(
|
||||
cmd: string,
|
||||
timeoutMs = 60_000
|
||||
): Promise<{
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
exitCode: number;
|
||||
}> {
|
||||
const exec = $`multipass exec ${VM_NAME} -- bash -c ${cmd}`
|
||||
.quiet()
|
||||
.nothrow()
|
||||
.then((r) => ({
|
||||
stdout: r.stdout.toString(),
|
||||
stderr: r.stderr.toString(),
|
||||
exitCode: r.exitCode,
|
||||
}));
|
||||
|
||||
const timeout = new Promise<{ stdout: string; stderr: string; exitCode: number }>((_, reject) =>
|
||||
setTimeout(() => reject(new Error(`vmExec timed out after ${timeoutMs}ms: ${cmd}`)), timeoutMs)
|
||||
);
|
||||
|
||||
return Promise.race([exec, timeout]);
|
||||
}
|
||||
|
||||
/** Transfer a file to the VM */
|
||||
export async function vmTransfer(
|
||||
localPath: string,
|
||||
remotePath: string
|
||||
): Promise<void> {
|
||||
await $`multipass transfer ${localPath} ${VM_NAME}:${remotePath}`;
|
||||
}
|
||||
|
||||
/** Wait for cloud-init to complete (max 40 min — installs Rust, Go, Node, .NET, etc.) */
|
||||
export async function vmWaitReady(maxWaitSec = 2400): Promise<boolean> {
|
||||
console.log("[vm] Waiting for cloud-init...");
|
||||
const start = Date.now();
|
||||
while ((Date.now() - start) / 1000 < maxWaitSec) {
|
||||
const { exitCode } = await vmExec(
|
||||
"test -f /home/ubuntu/.cloud-init-complete"
|
||||
);
|
||||
if (exitCode === 0) {
|
||||
const elapsed = Math.round((Date.now() - start) / 1000);
|
||||
console.log(`[vm] Cloud-init complete after ${elapsed}s`);
|
||||
return true;
|
||||
}
|
||||
await Bun.sleep(10_000);
|
||||
}
|
||||
console.error("[vm] Cloud-init timed out!");
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Transfer RTK source and build in release mode */
|
||||
export async function vmBuildRtk(projectRoot: string): Promise<{
|
||||
buildTime: number;
|
||||
binarySize: number;
|
||||
version: string;
|
||||
}> {
|
||||
console.log("[vm] Transferring RTK source...");
|
||||
|
||||
// Create tarball excluding heavy dirs and macOS resource forks (._*)
|
||||
await $`COPYFILE_DISABLE=1 tar czf /tmp/rtk-src.tar.gz --exclude target --exclude .git --exclude node_modules --exclude "index.html*" --exclude "._*" -C ${projectRoot} .`;
|
||||
await vmTransfer("/tmp/rtk-src.tar.gz", "/tmp/rtk-src.tar.gz");
|
||||
await vmExec(
|
||||
"mkdir -p /home/ubuntu/rtk && cd /home/ubuntu/rtk && tar xzf /tmp/rtk-src.tar.gz"
|
||||
);
|
||||
|
||||
console.log("[vm] Building RTK (release)...");
|
||||
const start = Date.now();
|
||||
const { stdout, exitCode } = await vmExec(
|
||||
"export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo build --release 2>&1 | tail -5"
|
||||
);
|
||||
const buildTime = Math.round((Date.now() - start) / 1000);
|
||||
|
||||
if (exitCode !== 0) {
|
||||
throw new Error(`Build failed:\n${stdout}`);
|
||||
}
|
||||
|
||||
const { stdout: sizeStr } = await vmExec(
|
||||
"stat -c%s /home/ubuntu/rtk/target/release/rtk"
|
||||
);
|
||||
const binarySize = parseInt(sizeStr.trim(), 10);
|
||||
|
||||
const { stdout: version } = await vmExec(
|
||||
"/home/ubuntu/rtk/target/release/rtk --version"
|
||||
);
|
||||
|
||||
console.log(
|
||||
`[vm] Build OK in ${buildTime}s — ${binarySize} bytes — ${version.trim()}`
|
||||
);
|
||||
|
||||
return { buildTime, binarySize, version: version.trim() };
|
||||
}
|
||||
|
||||
/** Delete the VM */
|
||||
export async function vmDelete(): Promise<void> {
|
||||
console.log(`[vm] Deleting ${VM_NAME}...`);
|
||||
await $`multipass delete ${VM_NAME} --purge`.nothrow();
|
||||
}
|
||||
|
||||
/** Ensure VM is ready (create or reuse) */
|
||||
export async function vmEnsureReady(): Promise<void> {
|
||||
if (await vmExists()) {
|
||||
if (!(await vmRunning())) {
|
||||
await vmStart();
|
||||
}
|
||||
console.log(`[vm] Reusing existing VM ${VM_NAME}`);
|
||||
// Check if cloud-init is still running
|
||||
const { exitCode } = await vmExec(
|
||||
"test -f /home/ubuntu/.cloud-init-complete"
|
||||
);
|
||||
if (exitCode !== 0) {
|
||||
console.log("[vm] Cloud-init still running, waiting...");
|
||||
const ready = await vmWaitReady();
|
||||
if (!ready) {
|
||||
throw new Error(
|
||||
"Cloud-init timed out. Check: multipass exec rtk-test -- cat /var/log/cloud-init-output.log"
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
await vmCreate();
|
||||
// multipass launch --timeout should wait, but double-check
|
||||
const { exitCode } = await vmExec(
|
||||
"test -f /home/ubuntu/.cloud-init-complete"
|
||||
);
|
||||
if (exitCode !== 0) {
|
||||
const ready = await vmWaitReady();
|
||||
if (!ready) {
|
||||
throw new Error(
|
||||
"Cloud-init timed out. Check: multipass exec rtk-test -- cat /var/log/cloud-init-output.log"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const RTK_BIN = "/home/ubuntu/rtk/target/release/rtk";
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* Fast rebuild: reuse existing VM, just transfer source and recompile.
|
||||
* Usage: bun run scripts/benchmark/rebuild.ts
|
||||
*/
|
||||
|
||||
import { vmEnsureReady, vmBuildRtk } from "./lib/vm";
|
||||
|
||||
const PROJECT_ROOT = new URL("../../", import.meta.url).pathname.replace(/\/$/, "");
|
||||
|
||||
await vmEnsureReady();
|
||||
const info = await vmBuildRtk(PROJECT_ROOT);
|
||||
|
||||
console.log(`\nRebuild complete:`);
|
||||
console.log(` Version: ${info.version}`);
|
||||
console.log(` Binary: ${info.binarySize} bytes`);
|
||||
console.log(` Time: ${info.buildTime}s`);
|
||||
@@ -0,0 +1,425 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* RTK Full Integration Test Suite — Multipass VM
|
||||
*
|
||||
* Usage:
|
||||
* bun run scripts/benchmark/run.ts # Full suite
|
||||
* bun run scripts/benchmark/run.ts --quick # Skip slow phases (perf, concurrency)
|
||||
* bun run scripts/benchmark/run.ts --phase 3 # Run specific phase only
|
||||
*
|
||||
* Prerequisites:
|
||||
* brew install multipass
|
||||
*/
|
||||
|
||||
import { $ } from "bun";
|
||||
import { vmEnsureReady, vmBuildRtk, vmExec, RTK_BIN } from "./lib/vm";
|
||||
import { testCmd, testSavings, testRewrite, skipTest, getCounts } from "./lib/test";
|
||||
import { saveReport } from "./lib/report";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const quick = args.includes("--quick");
|
||||
const phaseArg = args.includes("--phase")
|
||||
? parseInt(args[args.indexOf("--phase") + 1], 10)
|
||||
: null;
|
||||
const phaseOnly = phaseArg !== null && !Number.isNaN(phaseArg) ? phaseArg : null;
|
||||
if (args.includes("--phase") && phaseOnly === null) {
|
||||
console.error("Error: --phase requires a number (e.g. --phase 3)");
|
||||
process.exit(1);
|
||||
}
|
||||
const reportPath = args.includes("--report")
|
||||
? args[args.indexOf("--report") + 1]
|
||||
: `${new URL("../../", import.meta.url).pathname.replace(/\/$/, "")}/benchmark-report.txt`;
|
||||
|
||||
const PROJECT_ROOT = new URL("../../", import.meta.url).pathname.replace(/\/$/, "");
|
||||
const RTK = RTK_BIN;
|
||||
|
||||
function shouldRun(phase: number): boolean {
|
||||
return phaseOnly === null || phaseOnly === phase;
|
||||
}
|
||||
|
||||
function heading(phase: number, title: string) {
|
||||
console.log(`\n\x1b[34m[Phase ${phase}] ${title}\x1b[0m`);
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 0: VM Setup
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
console.log("\x1b[34m[rtk-test] RTK Full Integration Test Suite\x1b[0m");
|
||||
console.log(`Project: ${PROJECT_ROOT}`);
|
||||
|
||||
await vmEnsureReady();
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 1: Transfer & Build
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
heading(1, "Transfer & Build");
|
||||
const branch = (await $`git -C ${PROJECT_ROOT} branch --show-current`.text()).trim();
|
||||
const commit = (await $`git -C ${PROJECT_ROOT} log --oneline -1`.text()).trim();
|
||||
const buildInfo = await vmBuildRtk(PROJECT_ROOT);
|
||||
|
||||
// Binary size check
|
||||
// ARM Linux release binaries are ~6.5MB (vs ~4MB x86 stripped).
|
||||
// CLAUDE.md target is <5MB for stripped x86 release builds.
|
||||
// VM builds are ARM + not fully stripped, so we use a relaxed 8MB limit here.
|
||||
const sizeLimit = 8_388_608; // 8MB (relaxed for ARM Linux VM)
|
||||
if (buildInfo.binarySize < sizeLimit) {
|
||||
console.log(` \x1b[32mPASS\x1b[0m | binary size | ${buildInfo.binarySize} bytes < 8MB`);
|
||||
} else {
|
||||
console.log(` \x1b[31mFAIL\x1b[0m | binary size | ${buildInfo.binarySize} bytes >= 8MB`);
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 2: Cargo Quality (fmt, clippy, test)
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
if (shouldRun(2)) {
|
||||
heading(2, "Cargo Quality");
|
||||
|
||||
await testCmd(
|
||||
"quality:cargo fmt",
|
||||
"export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo fmt --all --check 2>&1"
|
||||
);
|
||||
|
||||
await testCmd(
|
||||
"quality:cargo clippy",
|
||||
"export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo clippy --all-targets -- -D warnings 2>&1"
|
||||
);
|
||||
|
||||
await testCmd(
|
||||
"quality:cargo test",
|
||||
"export PATH=$HOME/.cargo/bin:$PATH && cd /home/ubuntu/rtk && cargo test --all 2>&1"
|
||||
);
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 3: Rust Built-in Commands
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
if (shouldRun(3)) {
|
||||
heading(3, "Rust Built-in Commands");
|
||||
|
||||
// Git
|
||||
await testCmd("git:status", `cd /tmp/test-git && ${RTK} git status`);
|
||||
await testCmd("git:log", `cd /tmp/test-git && ${RTK} git log -5`);
|
||||
await testCmd("git:log --oneline", `cd /tmp/test-git && ${RTK} git log --oneline -10`);
|
||||
await testCmd("git:diff", `cd /tmp/test-git && ${RTK} git diff`, "any");
|
||||
await testCmd("git:branch", `cd /tmp/test-git && ${RTK} git branch`);
|
||||
await testCmd("git:add --dry-run", `cd /tmp/test-git && ${RTK} git add --dry-run .`, "any");
|
||||
|
||||
// Files
|
||||
await testCmd("files:ls", `${RTK} ls /home/ubuntu/rtk`);
|
||||
await testCmd("files:ls src/", `${RTK} ls /home/ubuntu/rtk/src/`);
|
||||
await testCmd("files:ls -R", `${RTK} ls -R /home/ubuntu/rtk/src/`);
|
||||
await testCmd("files:read", `${RTK} read /home/ubuntu/rtk/src/main.rs`);
|
||||
await testCmd("files:read aggressive", `${RTK} read /home/ubuntu/rtk/src/main.rs -l aggressive`);
|
||||
await testCmd("files:smart", `${RTK} smart /home/ubuntu/rtk/src/main.rs`);
|
||||
await testCmd("files:find *.rs", `${RTK} find '*.rs' /home/ubuntu/rtk/src/`);
|
||||
await testCmd("files:wc", `${RTK} wc /home/ubuntu/rtk/src/main.rs`);
|
||||
await testCmd("files:diff", `${RTK} diff /home/ubuntu/rtk/src/main.rs /home/ubuntu/rtk/src/utils.rs`);
|
||||
|
||||
// Search
|
||||
await testCmd("search:grep", `${RTK} grep 'fn main' /home/ubuntu/rtk/src/`);
|
||||
|
||||
// Data
|
||||
await testCmd("data:json", `${RTK} json /tmp/test-node/package.json`);
|
||||
await testCmd("data:deps", `cd /home/ubuntu/rtk && ${RTK} deps`);
|
||||
await testCmd("data:env", `${RTK} env`);
|
||||
|
||||
// Runners
|
||||
await testCmd("runner:summary", `${RTK} summary 'echo hello world'`);
|
||||
// BUG: rtk err swallows exit code — tracked in #846
|
||||
await testCmd("runner:err", `${RTK} err false`, "any");
|
||||
await testCmd("runner:test", `${RTK} test 'echo ok'`, "any");
|
||||
|
||||
// Logs
|
||||
await testCmd("log:large", `${RTK} log /tmp/large.log`);
|
||||
|
||||
// Network
|
||||
await testCmd("net:curl", `${RTK} curl https://mockhttp.org/get`, "any");
|
||||
|
||||
// GitHub
|
||||
await testCmd("gh:pr list", `cd /home/ubuntu/rtk && ${RTK} gh pr list`, "any");
|
||||
|
||||
// Cargo (test project has intentional test failure → exit 101)
|
||||
await testCmd("cargo:build", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo build`);
|
||||
await testCmd("cargo:test", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo test`, 101);
|
||||
await testCmd("cargo:clippy", `export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/test-rust && ${RTK} cargo clippy`);
|
||||
|
||||
// Python (test project has intentional failures)
|
||||
await testCmd("python:pytest", `cd /tmp/test-python && ${RTK} pytest`, 1);
|
||||
await testCmd("python:ruff check", `cd /tmp/test-python && ${RTK} ruff check .`, 1);
|
||||
await testCmd("python:mypy", `cd /tmp/test-python && ${RTK} mypy .`, 1);
|
||||
await testCmd("python:pip list", `${RTK} pip list`);
|
||||
|
||||
// Go (test project has intentional test failure)
|
||||
await testCmd("go:test", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go test ./...`, 1);
|
||||
await testCmd("go:build", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go build .`, 1);
|
||||
await testCmd("go:vet", `export PATH=$PATH:/usr/local/go/bin && cd /tmp/test-go && ${RTK} go vet ./...`, 1);
|
||||
await testCmd("go:golangci-lint", `export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin && cd /tmp/test-go && ${RTK} golangci-lint run`, 1);
|
||||
|
||||
// TypeScript
|
||||
await testCmd("ts:tsc", `cd /tmp/test-node && ${RTK} tsc --noEmit`, "any");
|
||||
|
||||
// Linters
|
||||
await testCmd("lint:eslint", `cd /tmp/test-node && ${RTK} lint 'eslint src/'`, "any");
|
||||
await testCmd("lint:prettier", `cd /tmp/test-node && ${RTK} prettier --check src/`, "any");
|
||||
|
||||
// Docker
|
||||
await testCmd("docker:ps", `${RTK} docker ps`, "any");
|
||||
await testCmd("docker:images", `${RTK} docker images`, "any");
|
||||
|
||||
// Kubernetes
|
||||
await testCmd("k8s:pods", `${RTK} kubectl pods`, "any");
|
||||
|
||||
// .NET
|
||||
await testCmd("dotnet:build", `export DOTNET_ROOT=/usr/local/share/dotnet && export PATH=$PATH:$DOTNET_ROOT && cd /tmp/test-dotnet/TestApp 2>/dev/null && ${RTK} dotnet build || echo 'dotnet skip'`, "any");
|
||||
|
||||
// Meta
|
||||
await testCmd("meta:gain", `${RTK} gain`);
|
||||
await testCmd("meta:gain --history", `${RTK} gain --history`);
|
||||
await testCmd("meta:proxy", `${RTK} proxy echo 'proxy test'`);
|
||||
await testCmd("meta:verify", `${RTK} verify`, "any");
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 4: TOML Filter Commands
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
if (shouldRun(4)) {
|
||||
heading(4, "TOML Filter Commands");
|
||||
|
||||
// System
|
||||
await testCmd("toml:df", `${RTK} df -h`);
|
||||
await testCmd("toml:du", `${RTK} du -sh /tmp`, "any");
|
||||
await testCmd("toml:ps", `${RTK} ps aux`);
|
||||
await testCmd("toml:ping", `${RTK} ping -c 2 127.0.0.1`);
|
||||
|
||||
// Build tools
|
||||
await testCmd("toml:make", `cd /tmp && ${RTK} make -f Makefile`, "any");
|
||||
await testCmd("toml:rsync", `${RTK} rsync --version`);
|
||||
|
||||
// Linters
|
||||
await testCmd("toml:shellcheck", `${RTK} shellcheck /tmp/test.sh`, "any");
|
||||
await testCmd("toml:hadolint", `${RTK} hadolint /tmp/Dockerfile.bad`, "any");
|
||||
await testCmd("toml:yamllint", `${RTK} yamllint /tmp/test.yaml`, "any");
|
||||
await testCmd("toml:markdownlint", `${RTK} markdownlint /tmp/test.md`, "any");
|
||||
|
||||
// Cloud/Infra
|
||||
await testCmd("toml:terraform", `${RTK} terraform --version`, "any");
|
||||
await testCmd("toml:helm", `${RTK} helm version`, "any");
|
||||
await testCmd("toml:ansible", `${RTK} ansible-playbook --version`, "any");
|
||||
|
||||
// Mocked tools
|
||||
await testCmd("toml:gcloud", `${RTK} gcloud version`);
|
||||
await testCmd("toml:shopify", `${RTK} shopify theme check`, "any");
|
||||
await testCmd("toml:pio", `${RTK} pio run`, "any");
|
||||
await testCmd("toml:quarto", `${RTK} quarto render`, "any");
|
||||
await testCmd("toml:sops", `${RTK} sops --version`);
|
||||
// Swift ecosystem
|
||||
await testCmd("toml:swift build", `${RTK} swift build`, "any");
|
||||
await testCmd("toml:swift test", `${RTK} swift test`, "any");
|
||||
await testCmd("toml:swift run", `${RTK} swift run`, "any");
|
||||
await testCmd("toml:swift package", `${RTK} swift package resolve`, "any");
|
||||
await testCmd("toml:swiftlint", `${RTK} swiftlint`, "any");
|
||||
await testCmd("toml:swiftformat", `${RTK} swiftformat`, "any");
|
||||
await testCmd("toml:kubectl", `${RTK} kubectl version --client`, "any");
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 5: Hook Rewrite Engine
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
if (shouldRun(5)) {
|
||||
heading(5, "Hook Rewrite Engine");
|
||||
|
||||
// Basic rewrites
|
||||
await testRewrite("git status", "rtk git status");
|
||||
await testRewrite("git log --oneline -10", "rtk git log --oneline -10");
|
||||
await testRewrite("cargo test", "rtk cargo test");
|
||||
await testRewrite("cargo build --release", "rtk cargo build --release");
|
||||
await testRewrite("docker ps", "rtk docker ps");
|
||||
// NOTE: rtk rewrites "kubectl get pods" to "rtk kubectl get pods" (preserves get)
|
||||
await testRewrite("kubectl get pods", "rtk kubectl get pods");
|
||||
await testRewrite("ruff check", "rtk ruff check");
|
||||
await testRewrite("pytest", "rtk pytest");
|
||||
await testRewrite("go test", "rtk go test");
|
||||
await testRewrite("pnpm list", "rtk pnpm list");
|
||||
await testRewrite("gh pr list", "rtk gh pr list");
|
||||
await testRewrite("df -h", "rtk df -h");
|
||||
await testRewrite("ps aux", "rtk ps aux");
|
||||
|
||||
// Compound
|
||||
await testRewrite("cargo test && git status", "rtk cargo test && rtk git status");
|
||||
// NOTE: shell strips single quotes in vmExec, so 'msg' becomes msg
|
||||
await testRewrite("git add . && git commit -m msg", "rtk git add . && rtk git commit -m msg");
|
||||
|
||||
// No rewrite (shell builtins) — rtk rewrite returns empty string + exit 1
|
||||
// We test via testCmd since testRewrite expects non-empty output
|
||||
await testCmd("rewrite:cd (no rewrite)", `${RTK} rewrite 'cd /tmp'`, 1);
|
||||
await testCmd("rewrite:export (no rewrite)", `${RTK} rewrite 'export FOO=bar'`, 1);
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 6: Exit Code Preservation
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
if (shouldRun(6)) {
|
||||
heading(6, "Exit Code Preservation");
|
||||
|
||||
// Success
|
||||
await testCmd("exit:git status=0", `cd /tmp/test-git && ${RTK} git status`, 0);
|
||||
await testCmd("exit:ls=0", `${RTK} ls /tmp`, 0);
|
||||
await testCmd("exit:gain=0", `${RTK} gain`, 0);
|
||||
|
||||
// Failures
|
||||
// rg returns exit 1 (no match) or 2 (error) — accept both
|
||||
await testCmd("exit:grep NOTFOUND", `${RTK} grep NOTFOUND_XYZ_123 /tmp`, "any");
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 7: Token Savings
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
if (shouldRun(7)) {
|
||||
heading(7, "Token Savings");
|
||||
|
||||
await testSavings(
|
||||
"savings:git log",
|
||||
"cd /tmp/test-git && git log -20",
|
||||
`cd /tmp/test-git && ${RTK} git log -20`,
|
||||
60
|
||||
);
|
||||
await testSavings(
|
||||
"savings:ls",
|
||||
"ls -la /home/ubuntu/rtk/src/",
|
||||
`${RTK} ls /home/ubuntu/rtk/src/`,
|
||||
60
|
||||
);
|
||||
await testSavings(
|
||||
"savings:log dedup",
|
||||
"cat /tmp/large.log",
|
||||
`${RTK} log /tmp/large.log`,
|
||||
80
|
||||
);
|
||||
await testSavings(
|
||||
"savings:read aggressive",
|
||||
"cat /home/ubuntu/rtk/src/main.rs",
|
||||
`${RTK} read /home/ubuntu/rtk/src/main.rs -l aggressive`,
|
||||
50
|
||||
);
|
||||
await testSavings(
|
||||
"savings:swift test",
|
||||
"swift test",
|
||||
`${RTK} swift test`,
|
||||
60
|
||||
);
|
||||
await testSavings(
|
||||
"savings:swiftlint",
|
||||
"swiftlint",
|
||||
`${RTK} swiftlint`,
|
||||
20
|
||||
);
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 8: Pipe Compatibility
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
if (shouldRun(8)) {
|
||||
heading(8, "Pipe Compatibility");
|
||||
|
||||
await testCmd("pipe:git status|wc", `cd /tmp/test-git && ${RTK} git status | wc -l`);
|
||||
await testCmd("pipe:ls|wc", `${RTK} ls /home/ubuntu/rtk/src/ | wc -l`);
|
||||
await testCmd("pipe:grep|head", `${RTK} grep 'fn' /home/ubuntu/rtk/src/ | head -5`);
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 9: Edge Cases
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
if (shouldRun(9)) {
|
||||
heading(9, "Edge Cases");
|
||||
|
||||
await testCmd("edge:summary true", `${RTK} summary 'true'`, "any");
|
||||
await testCmd("edge:grep NOTFOUND", `${RTK} grep NOTFOUND_XYZ /home/ubuntu/rtk/src/`, 1);
|
||||
await testCmd("edge:unicode", `echo 'hello world' > /tmp/uni.txt && ${RTK} grep 'hello' /tmp`, "any");
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 10: Performance (skip with --quick)
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
if (shouldRun(10) && !quick) {
|
||||
heading(10, "Performance");
|
||||
|
||||
// hyperfine
|
||||
const { exitCode: hfExist } = await vmExec("command -v hyperfine");
|
||||
if (hfExist === 0) {
|
||||
const { stdout: hfOut } = await vmExec(
|
||||
`cd /tmp/test-git && hyperfine --warmup 3 --min-runs 5 '${RTK} git status' 'git status' --export-json /dev/stdout 2>/dev/null`
|
||||
);
|
||||
try {
|
||||
const hf = JSON.parse(hfOut);
|
||||
const rtkMean = (hf.results?.[0]?.mean * 1000).toFixed(1);
|
||||
const rawMean = (hf.results?.[1]?.mean * 1000).toFixed(1);
|
||||
console.log(` Startup: rtk=${rtkMean}ms raw=${rawMean}ms`);
|
||||
} catch {
|
||||
console.log(" hyperfine output parse failed");
|
||||
}
|
||||
} else {
|
||||
skipTest("perf:hyperfine", "not installed");
|
||||
}
|
||||
|
||||
// Memory
|
||||
const { stdout: memOut } = await vmExec(
|
||||
`cd /tmp/test-git && /usr/bin/time -v ${RTK} git status 2>&1 | grep 'Maximum resident'`
|
||||
);
|
||||
const memKb = parseInt(memOut.match(/(\d+)/)?.[1] ?? "0", 10);
|
||||
if (memKb > 0 && memKb < 20000) {
|
||||
await testCmd("perf:memory", `echo '${memKb} KB < 20MB'`);
|
||||
} else if (memKb > 0) {
|
||||
await testCmd("perf:memory", `echo '${memKb} KB >= 20MB' && exit 1`, 0);
|
||||
}
|
||||
} else if (quick && shouldRun(10)) {
|
||||
skipTest("perf:hyperfine", "--quick mode");
|
||||
skipTest("perf:memory", "--quick mode");
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Phase 11: Concurrency (skip with --quick)
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
if (shouldRun(11) && !quick) {
|
||||
heading(11, "Concurrency");
|
||||
|
||||
await testCmd(
|
||||
"concurrency:10x git status",
|
||||
`cd /tmp/test-git && for i in $(seq 1 10); do ${RTK} git status >/dev/null & done; wait`
|
||||
);
|
||||
} else if (quick && shouldRun(11)) {
|
||||
skipTest("concurrency:10x", "--quick mode");
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
// Report
|
||||
// ══════════════════════════════════════════════════════════════
|
||||
|
||||
const report = await saveReport(
|
||||
{ ...buildInfo, branch, commit },
|
||||
reportPath
|
||||
);
|
||||
|
||||
console.log("\n" + report);
|
||||
|
||||
const { total, passed, failed, skipped } = getCounts();
|
||||
const passRate = total > 0 ? Math.round((passed * 100) / total) : 0;
|
||||
|
||||
if (failed === 0) {
|
||||
console.log(`\n\x1b[32m READY FOR RELEASE — ${passed}/${total} (${passRate}%)\x1b[0m\n`);
|
||||
process.exit(0);
|
||||
} else {
|
||||
console.log(`\n\x1b[31m NOT READY — ${failed} failures — ${passed}/${total} (${passRate}%)\x1b[0m\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
Reference in New Issue
Block a user