#!/usr/bin/env bats # Tests for the isolation helpers in scripts/cli/_common.sh — the --isolate # runtime-state plumbing that lets multiple `showcase test --isolate` runs share # one host without colliding on ports, compose project names, or scratch dirs. # # Three behaviors are pinned here: # 1. Runtime state lives under XDG_STATE_HOME (NOT /tmp). The slot registry and # per-run scratch dir are rooted at $XDG_STATE_HOME/copilotkit/showcase, with # the documented $HOME/.local/state fallback when XDG_STATE_HOME is unset. # 2. Stale slots are reaped by compose-project liveness: a slot whose recorded # project has no live containers is reclaimed; one with a live container is # left alone (so a --keep'd stack is never stolen out from under). # 3. restore_isolation honors --keep: when ISOLATE_KEEP is set it does NOT # compose-down, does NOT remove the run dir, does NOT release the slot, and # prints a survival notice (project, slot, the three +offset host ports, and # the exact teardown command). ISOLATE_KEEP must be a GLOBAL that survives # cmd_test returning — the EXIT trap fires at top-level exit, where a # function-local flag has already unwound. # # NB on assertion style: bats-core runs test bodies with errexit semantics # (`set -eET` plus an ERR trap), so a bare `[[ ... ]]` that fails DOES fail the # test — even on a non-final line. We still write assertions as # `[[ ... ]] || fail "message"` for the diagnostic message on failure, and for # robustness in contexts where errexit is suppressed (e.g. inside && / || # chains or condition positions). # fail — print the message and abort the test. The message goes to # STDOUT: bats reliably displays a failed test's stdout, but not its stderr. fail() { echo "$1" return 1 } # require_python3 — skip tests that drive apply_isolation through its real # python3 ports/compose rewriters; without this, a python3-less machine fails # those tests with a bare "command not found" instead of a clear skip. require_python3() { command -v python3 >/dev/null 2>&1 || skip "python3 required (apply_isolation rewrites ports/compose files with it)" } # _touch_age — backdate a path's mtime portably. BSD # touch has no GNU-style `-d '-N seconds'`; compute the absolute `-t` stamp # with python3 instead (callers must require_python3). DST-safe: the stamp is # computed in UTC (time.gmtime) and interpreted in UTC (TZ=UTC for touch) — # a localtime stamp straddling a DST transition would be off by an hour (or # land in a nonexistent/ambiguous wall-clock time). _touch_age() { local stamp stamp="$(python3 -c "import time; print(time.strftime('%Y%m%d%H%M.%S', time.gmtime(time.time() - $1)))")" TZ=UTC touch -t "$stamp" "$2" } # _assert_stub_sentinel_logged — prove the docker-stub→DOCKER_LOG pipeline is # alive before drawing any conclusion from the log's contents. Several tests # assert the ABSENCE of compose-down lines; without this sentinel, a silently # broken pipeline (DOCKER_LOG unset/unwritable in the child, an export # regression, the stub falling off PATH) makes those absence checks pass # vacuously — "no evidence of wrongdoing" instead of "verified right-doing". # Each caller runs `docker version` before the code under test (and after any # log truncation); this asserts that invocation actually landed in the log. _assert_stub_sentinel_logged() { [ -f "$DOCKER_LOG" ] \ || fail "docker stub log missing — stub/DOCKER_LOG pipeline broken, absence assertions would be vacuous" grep -qx "version" "$DOCKER_LOG" \ || fail "sentinel 'docker version' invocation not logged — stub/DOCKER_LOG pipeline broken: $(cat "$DOCKER_LOG")" } # _seed_live_owner [pid] — record a LIVE, start-time-VERIFIED owner # in : write the pid file AND a matching pid.start (the new # anti-PID-reuse fingerprint). Under the post-Change-1 owner contract a bare # `pid` file with NO pid.start is "unverifiable" (treated as dead), so every # fixture that wants a slot to read `live` via its owning PID must seed BOTH # files — that is what this helper does. Defaults to $$ (the bats process, # definitely alive). The pid.start is produced by the SAME _pid_start_time # helper the classifier reads, so the recorded and current values match. The # pid file is written FIRST (preserving the claim-time write order the # classifier's "missing pid ⇒ owner gone" signal relies on). # # NB: this sources _common.sh into the CURRENT shell to reach _pid_start_time. # Tests that have already run load_common are unaffected; tests that have not # get a harmless extra source (the path vars are re-pointed by load_common). _seed_live_owner() { local slotdir="${1:?slot dir required}" local pid="${2:-$$}" # shellcheck disable=SC1090 type _pid_start_time >/dev/null 2>&1 || source "$COMMON" echo "$pid" > "$slotdir/pid" _pid_start_time "$pid" > "$slotdir/pid.start" } setup() { COMMON="$BATS_TEST_DIRNAME/../cli/_common.sh" # ── Fake docker on PATH ──────────────────────────────────────────────────── # apply_isolation runs `$COMPOSE_CMD down` and the reaper runs # `docker ps -q --filter ...`. Both must be stubbed so no real docker is hit. # The `ps` behavior is driven by $DOCKER_PS_OUTPUT so individual tests can make # a project look live (non-empty) or dead (empty), and by $DOCKER_PS_EXIT so # tests can simulate a docker daemon failure (non-zero exit, no output). # `compose ... down` failure is driven by $DOCKER_COMPOSE_DOWN_EXIT so tests # can simulate a teardown that leaves the stack standing. STUB_DIR="$BATS_TEST_TMPDIR/stub" mkdir -p "$STUB_DIR" cat > "$STUB_DIR/docker" <<'STUB' #!/usr/bin/env bash # Record every invocation so tests can assert on WHICH compose commands ran # (e.g. that a failed --isolate never composes down the default stack). printf '%s\n' "$*" >> "${DOCKER_LOG:-/dev/null}" case "$1" in ps) [ "${DOCKER_PS_EXIT:-0}" -eq 0 ] || exit "${DOCKER_PS_EXIT}" printf '%s' "${DOCKER_PS_OUTPUT:-}"; [ -z "${DOCKER_PS_OUTPUT:-}" ] || echo ;; compose) # Ordering probe: when REAP_ORDER_CHECK_DIR is set, snapshot whether that # dir still exists at the moment a compose-down is invoked. Lets tests pin # "compose-down runs BEFORE state deletion" (the reap's split-brain # guarantee) instead of only "compose-down ran at some point". if [[ " $* " == *" down "* ]] && [ -n "${REAP_ORDER_CHECK_DIR:-}" ]; then if [ -d "$REAP_ORDER_CHECK_DIR" ]; then echo "REAP_ORDER_CHECK dir-present" >> "${DOCKER_LOG:-/dev/null}" else echo "REAP_ORDER_CHECK dir-absent" >> "${DOCKER_LOG:-/dev/null}" fi fi # Match `down` as a discrete word: a bare *" down"* substring also matches # the PROJECT NAME in `--project-name downfail`, silently failing commands # that never ran `down` at all. Space-padding "$*" makes the word check # exact whether `down` is interior or the final argument. if [[ " $* " == *" down "* ]] && [ "${DOCKER_COMPOSE_DOWN_EXIT:-0}" -ne 0 ]; then echo "stub: compose down failed" >&2 exit "${DOCKER_COMPOSE_DOWN_EXIT}" fi ;; # otherwise swallow `docker compose ...` quietly *) : ;; esac exit 0 STUB chmod +x "$STUB_DIR/docker" # ── Fake lsof on PATH ────────────────────────────────────────────────────── # apply_isolation's port-conflict probe (and cmd-doctor.sh) shell out to # `lsof -i : -sTCP:LISTEN -P -n`. Real lsof on the dev box reports # whatever happens to be listening, which is both non-deterministic and # non-hermetic. The stub drives held-port reporting from $LSOF_HOLD_PORTS # (comma-separated): ports in the list emit the standard lsof header + a # single LISTEN row and exit 0; ports not in the list exit 1 with empty # stdout, mirroring real lsof's no-match semantics. Every invocation is # appended to $LSOF_LOG so tests can assert which ports got probed. cat > "$STUB_DIR/lsof" <<'STUB' #!/usr/bin/env bash # Record every invocation so tests can assert WHICH ports were probed. echo "lsof $*" >> "${LSOF_LOG:-/dev/null}" # Extract the port from `-i :`. cmd-doctor.sh and _common.sh both invoke # `lsof -i : -sTCP:LISTEN -P -n`, so the port arg is the one following # `-i`, with a leading colon. Other args (-sTCP:LISTEN, -P, -n) are tolerated # and ignored — the stub does not care about protocol/numeric flags. port="" while [ $# -gt 0 ]; do if [ "$1" = "-i" ] && [[ "${2:-}" =~ ^:([0-9]+)$ ]]; then port="${BASH_REMATCH[1]}" break fi shift done [ -n "$port" ] || exit 1 # Check $LSOF_HOLD_PORTS (comma-separated). Pad with commas so a port doesn't # substring-match a longer port (e.g. "341" vs "3410"). case ",${LSOF_HOLD_PORTS:-}," in *",${port},"*) echo "COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME" echo "${LSOF_HOLD_COMMAND:-Python} 12345 user 5u IPv4 0x0000000000000000 0t0 TCP 127.0.0.1:${port} (LISTEN)" exit 0 ;; *) exit 1 ;; esac STUB chmod +x "$STUB_DIR/lsof" PATH="$STUB_DIR:$PATH" # Invocation log for the docker stub (one line per call, "$*"-joined args). export DOCKER_LOG="$BATS_TEST_TMPDIR/docker-invocations.log" # Invocation log + held-port config for the lsof stub. Per-test tmpdir, so # each test sees an empty log at start — no cross-test contamination. export LSOF_LOG="$BATS_TEST_TMPDIR/lsof-invocations.log" export LSOF_HOLD_PORTS="" export LSOF_HOLD_COMMAND="Python" # ── Fake SHOWCASE_ROOT with the two files apply_isolation rewrites ────────── export SHOWCASE_ROOT_OVERRIDE="$BATS_TEST_TMPDIR/root" mkdir -p "$SHOWCASE_ROOT_OVERRIDE/shared" cat > "$SHOWCASE_ROOT_OVERRIDE/shared/local-ports.json" <<'JSON' { "mastra": 3104 } JSON cat > "$SHOWCASE_ROOT_OVERRIDE/docker-compose.local.yml" <<'YML' services: aimock: container_name: showcase-aimock ports: - "4010:4010" YML # ── XDG state base ───────────────────────────────────────────────────────── export XDG_STATE_HOME="$BATS_TEST_TMPDIR/xdg" # Default: dead project (no live containers), healthy docker daemon, compose # down succeeds, unless a test overrides them. export DOCKER_PS_OUTPUT="" export DOCKER_PS_EXIT=0 export DOCKER_COMPOSE_DOWN_EXIT=0 } # load_common — source _common.sh, then repoint the path vars at the fake root. # _common.sh computes SHOWCASE_ROOT from its own location at source time, so we # override the derived paths afterward to point at the per-test scratch root. load_common() { # shellcheck disable=SC1090 source "$COMMON" SHOWCASE_ROOT="$SHOWCASE_ROOT_OVERRIDE" COMPOSE_FILE="$SHOWCASE_ROOT/docker-compose.local.yml" PORTS_FILE="$SHOWCASE_ROOT/shared/local-ports.json" COMPOSE_CMD="docker compose -f $COMPOSE_FILE" } # ── Foundation smoke: lsof stub plumbing ──────────────────────────────────── # Pins the bats lsof stub used by the port-conflict tests: $LSOF_HOLD_PORTS # (comma-separated) controls which ports the stub reports as LISTEN'd; every # invocation is appended to $LSOF_LOG so callers can assert on what was probed. # Kept PERMANENT (test #0) so a silent stub regression — stub off PATH, env # vars unexported, log unwritable — fails LOUDLY here instead of corrupting # every downstream port-conflict test with vacuous "no listener" results. @test "lsof stub: emits listener line for held ports, exits 1 for free ports" { # held: stub returns LISTEN row, exit 0 LSOF_HOLD_PORTS=3410 run lsof -i :3410 -sTCP:LISTEN -P -n [ "$status" -eq 0 ] [[ "$output" == *"Python"*"127.0.0.1:3410"*"(LISTEN)"* ]] # log captured grep -q "lsof -i :3410" "$LSOF_LOG" # free: stub exits 1, empty stdout LSOF_HOLD_PORTS="" run lsof -i :9999 -sTCP:LISTEN -P -n [ "$status" -eq 1 ] [ -z "$output" ] } # ── Change 1: runtime state under XDG_STATE_HOME ───────────────────────────── @test "ISOLATE_SLOT_DIR is rooted under XDG_STATE_HOME, not /tmp" { load_common [[ "$ISOLATE_SLOT_DIR" == "$XDG_STATE_HOME/copilotkit/showcase/slots" ]] \ || fail "ISOLATE_SLOT_DIR not under XDG_STATE_HOME: $ISOLATE_SLOT_DIR" # (Regression note: the pre-XDG registry lived at /tmp/showcase-isolate-slots; # the exact-path equality above already rules that out.) } @test "state base falls back to \$HOME/.local/state when XDG_STATE_HOME unset" { unset XDG_STATE_HOME export HOME="$BATS_TEST_TMPDIR/home" load_common run _showcase_state_base [ "$status" -eq 0 ] || fail "_showcase_state_base failed: $output" [[ "$output" == "$HOME/.local/state/copilotkit/showcase" ]] \ || fail "wrong fallback base: $output" } @test "per-run dir lives under XDG state runs/, not /tmp" { require_python3 load_common apply_isolation foo [[ "$ISOLATE_TMPDIR" == "$XDG_STATE_HOME/copilotkit/showcase/runs/foo" ]] \ || fail "run dir not under XDG state runs/: $ISOLATE_TMPDIR" # (Regression note: the pre-XDG scratch dir was the PID-keyed # ${TMPDIR:-/tmp}/showcase-isolate-; the exact-path equality above # already rules that out.) [ -d "$ISOLATE_TMPDIR" ] || fail "run dir not created: $ISOLATE_TMPDIR" } @test "apply_isolation stamps the com.copilotkit.showcase.isolate label on every service" { # The forward-stack self-id label is what lets `showcase reap` identify a # harness-owned isolated project even when its slot record + run dir are both # gone (a user-supplied --isolate orphan). Drive the REAL rewrite and # assert the label landed on the (only) service, right under its rewritten # container_name, as a compose-native labels: block. require_python3 load_common apply_isolation foo local rewritten="$ISOLATE_TMPDIR/docker-compose.local.yml" [ -f "$rewritten" ] || fail "rewritten compose not created: $rewritten" grep -q 'com.copilotkit.showcase.isolate: "1"' "$rewritten" \ || fail "self-id label not stamped into the rewritten compose: $(cat "$rewritten")" # The label must sit under a labels: block, not be orphaned at the wrong # indent — assert the labels: key is present too. grep -q '^ labels:$' "$rewritten" \ || fail "labels: block missing/mis-indented in the rewritten compose: $(cat "$rewritten")" # And it never touched the source compose (originals stay pristine). grep -q 'com.copilotkit.showcase.isolate' "$SHOWCASE_ROOT/docker-compose.local.yml" \ && fail "source compose was mutated with the label (must rewrite a COPY only)" return 0 } # ── Change 2: stale-slot reaping by compose-project liveness ───────────────── @test "claim reaps a slot whose project has no live containers and no live owning PID" { load_common # Pre-seed slot 0 for a project with NO live containers (DOCKER_PS_OUTPUT="") # and NO pid file (so there is no live owning PID to protect it either — # under the three-signal order, the ABSENCE of live containers alone is not # enough to reap while the owning process is still alive). mkdir -p "$XDG_STATE_HOME/copilotkit/showcase/slots/0" echo "ghost-proj" > "$XDG_STATE_HOME/copilotkit/showcase/slots/0/project" export DOCKER_PS_OUTPUT="" _claim_isolate_slot # Slot 0 is reserved for the base stack, so the auto-picker lands on slot 1; # the dead slot 0 is still reaped by the sweep (its ghost project file is # gone), the claim just doesn't land there. [ "$ISOLATE_SLOT" = "1" ] || fail "expected to claim slot 1 (slot 0 reserved), got: $ISOLATE_SLOT" # The reap rm -rf'd the ghost slot dir: only apply_isolation (not the claim) # writes a project file, so after a genuine reap the ghost project file must # be GONE. (A `run cat`+`!= ghost-proj` check here would pass vacuously on # empty output even if the reap never happened.) [ ! -f "$XDG_STATE_HOME/copilotkit/showcase/slots/0/project" ] \ || fail "ghost project file survived the reap" } @test "claim does NOT reap a slot whose project has a live container" { load_common mkdir -p "$XDG_STATE_HOME/copilotkit/showcase/slots/0" echo "live-proj" > "$XDG_STATE_HOME/copilotkit/showcase/slots/0/project" # Make docker ps report a live container id -> slot 0 must be preserved. export DOCKER_PS_OUTPUT="abc123def456" _claim_isolate_slot [ "$ISOLATE_SLOT" = "1" ] || fail "expected to skip live slot 0 and claim 1, got: $ISOLATE_SLOT" # Slot 0 and its project file survive untouched. [ -d "$XDG_STATE_HOME/copilotkit/showcase/slots/0" ] || fail "live slot 0 was wrongly reaped" run cat "$XDG_STATE_HOME/copilotkit/showcase/slots/0/project" [ "$status" -eq 0 ] || fail "live slot 0 project file missing" [[ "$output" == "live-proj" ]] || fail "live slot 0 project mangled: $output" } @test "slot claim persists the project name alongside the pid file" { require_python3 load_common apply_isolation myproj local slotdir="$ISOLATE_SLOT_DIR/$ISOLATE_SLOT" [ -f "$slotdir/project" ] || fail "no project file written for claimed slot" run cat "$slotdir/project" [ "$status" -eq 0 ] || fail "project file unreadable" [[ "$output" == "myproj" ]] || fail "wrong project recorded: $output" [ -f "$slotdir/pid" ] || fail "pid file no longer written" } @test "claim does NOT reap a project-recorded slot with zero containers but a LIVE owning PID" { # The claim/start race: apply_isolation writes the project file minutes before # any container exists (image builds happen later). Zero live containers must # NOT be authoritative while the owning process is still alive. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "racer-proj" > "$slots/0/project" _seed_live_owner "$slots/0" # this bats process — definitely alive (pid + pid.start) export DOCKER_PS_OUTPUT="" # no containers yet (still building) _claim_isolate_slot [ "$ISOLATE_SLOT" = "1" ] || fail "live-owner slot 0 was stolen; claimed: $ISOLATE_SLOT" [ -d "$slots/0" ] || fail "live-owner slot 0 was reaped" run cat "$slots/0/project" [ "$status" -eq 0 ] || fail "slot 0 project file missing after sweep" [[ "$output" == "racer-proj" ]] || fail "slot 0 project mangled: $output" } @test "claim leaves a project-recorded slot alone when docker ps fails" { # A docker daemon failure must read as "cannot verify", never as "no # containers" — otherwise a daemon hiccup mass-reaps live slots. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "unknowable-proj" > "$slots/0/project" export DOCKER_PS_EXIT=1 _claim_isolate_slot [ "$ISOLATE_SLOT" = "1" ] || fail "unverifiable slot 0 was claimed: $ISOLATE_SLOT" [ -d "$slots/0" ] || fail "slot 0 reaped despite docker ps failure" run cat "$slots/0/project" [ "$status" -eq 0 ] || fail "slot 0 project file missing after sweep" [[ "$output" == "unknowable-proj" ]] || fail "slot 0 project mangled: $output" } @test "legacy slot without project file survives set -e and is reaped via dead-PID fallback" { # bin/showcase runs `set -euo pipefail` and sources _common.sh; a slot dir # with no project file must not kill the script (cat exits 1), and the # documented PID fallback must actually reap it. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" # legacy: NO project file bash -c 'exit 0' & local dead_pid=$! wait "$dead_pid" || true # PID-reuse guard: the fixture's premise is a DEAD owning pid; if the OS # recycled it to a live process between wait and the sweep, skip rather than # flake. if kill -0 "$dead_pid" 2>/dev/null; then skip "PID $dead_pid was recycled to a live process — dead-PID fixture invalid" fi echo "$dead_pid" > "$slots/0/pid" run bash -euo pipefail -c "source '$COMMON'; _claim_isolate_slot; echo \"CLAIMED=\$ISOLATE_SLOT\"" [ "$status" -eq 0 ] || fail "script died under set -euo pipefail on legacy slot: $output" # Slot 0 is reserved for the base stack, so the auto-picker lands on slot 1 # even though the dead legacy slot 0 was reaped by the sweep. [[ "$output" == *"CLAIMED=1"* ]] || fail "dead legacy slot 0 not reaped (claim should be 1, slot 0 reserved): $output" } @test "claim does NOT reap a kept-stack slot (owner dead, containers live)" { # --keep leaves the stack standing after the owning process exits; container # liveness alone must keep protecting the slot. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "keeper-proj" > "$slots/0/project" bash -c 'exit 0' & local dead_pid=$! wait "$dead_pid" || true # PID-reuse guard (see the legacy-slot test above): a recycled live pid # would invalidate the dead-owner premise. if kill -0 "$dead_pid" 2>/dev/null; then skip "PID $dead_pid was recycled to a live process — dead-PID fixture invalid" fi echo "$dead_pid" > "$slots/0/pid" export DOCKER_PS_OUTPUT="abc123def456" # containers still up _claim_isolate_slot [ "$ISOLATE_SLOT" = "1" ] || fail "kept-stack slot 0 was stolen: $ISOLATE_SLOT" [ -d "$slots/0" ] || fail "kept-stack slot 0 was reaped" run cat "$slots/0/project" [ "$status" -eq 0 ] || fail "kept-stack project file missing after sweep" [[ "$output" == "keeper-proj" ]] || fail "kept-stack project mangled: $output" } @test "claim reaps an over-age legacy slot whose pid file is empty (inconclusive)" { # An empty/garbage pid file used to dodge BOTH the pid check and the # `[ ! -f pid ]`-gated age fallback, leaking the slot forever. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" # legacy: no project file : > "$slots/0/pid" # empty pid file → pid check inconclusive touch -t 202001010000 "$slots/0" # mtime far beyond the 2h threshold _claim_isolate_slot # Slot 0 is reserved for the base stack, so the auto-picker lands on slot 1. # The empty-pid over-age slot 0 was still reaped by the sweep (verified by # the slot dir being gone), the claim just doesn't land there. [ "$ISOLATE_SLOT" = "1" ] || fail "empty-pid over-age slot 0 leaked or claim path broken; claimed: $ISOLATE_SLOT" [ ! -d "$slots/0" ] || fail "empty-pid over-age slot 0 was not reaped" } @test "claim does NOT reap a RECENT project-recorded slot whose pid file is garbage (inconclusive)" { # Truncated-pid-write race: a live owner mid-build (project recorded, zero # containers yet) whose pid file write was truncated/corrupted must NOT lose # its slot — "no live owner" was never actually verified. An unreadable/ # empty/non-numeric pid is INCONCLUSIVE and must fall through to the age # fallback; a RECENT slot therefore survives the sweep. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "truncated-proj" > "$slots/0/project" echo "not-a-pid" > "$slots/0/pid" # garbage contents — pid check inconclusive export DOCKER_PS_OUTPUT="" # zero containers (still building) _claim_isolate_slot [ "$ISOLATE_SLOT" = "1" ] || fail "inconclusive-pid slot 0 was stolen; claimed: $ISOLATE_SLOT" [ -d "$slots/0" ] || fail "inconclusive-pid slot 0 was reaped" run cat "$slots/0/project" [ "$status" -eq 0 ] || fail "slot 0 project file missing after sweep" [[ "$output" == "truncated-proj" ]] || fail "slot 0 project mangled: $output" } @test "claim reaps an OVER-AGE project-recorded slot whose pid file is garbage via the age fallback" { # Companion to the test above: inconclusive-pid slots are not protected # FOREVER — the age fallback still reclaims them (and their runs/ # scratch dir) once they exceed ISOLATE_STALE_THRESHOLD. load_common local base="$XDG_STATE_HOME/copilotkit/showcase" local slots="$base/slots" mkdir -p "$slots/0" echo "truncated-proj" > "$slots/0/project" echo "not-a-pid" > "$slots/0/pid" mkdir -p "$base/runs/truncated-proj" touch -t 202001010000 "$slots/0" # mtime far beyond the 2h threshold export DOCKER_PS_OUTPUT="" _claim_isolate_slot # Slot 0 is reserved for the base stack, so the auto-picker lands on slot 1. # The over-age inconclusive-pid slot 0 was still reaped by the sweep # (verified by the ghost project record and orphan run dir being gone), the # claim just doesn't land there. [ "$ISOLATE_SLOT" = "1" ] || fail "over-age inconclusive-pid slot 0 leaked or claim path broken; claimed: $ISOLATE_SLOT" [ ! -f "$slots/0/project" ] || fail "ghost project record survived the age-fallback reap" [ ! -d "$base/runs/truncated-proj" ] || fail "orphan run dir leaked: $base/runs/truncated-proj" } @test "reaping a dead slot also removes its runs/ scratch dir" { # Orphan-leak regression: the sweep used to remove only the slot dir; the # crashed run's runs/ scratch dir was cleaned by NOTHING (the owner # is dead, and restore_isolation only removes the CURRENT run's dir), so # orphans accumulated under XDG state forever. load_common local base="$XDG_STATE_HOME/copilotkit/showcase" mkdir -p "$base/slots/0" echo "ghost-proj" > "$base/slots/0/project" # dead project, no pid file mkdir -p "$base/runs/ghost-proj" touch "$base/runs/ghost-proj/docker-compose.local.yml" export DOCKER_PS_OUTPUT="" # no live containers → project-stale reap _claim_isolate_slot # Slot 0 is reserved for the base stack, so the auto-picker lands on slot 1. # The dead slot 0 is still reaped by the sweep (verified by the ghost slot # dir and orphan run dir being gone), the claim just doesn't land there. [ "$ISOLATE_SLOT" = "1" ] || fail "expected slot 1 (slot 0 reserved), got: $ISOLATE_SLOT" [ ! -f "$base/slots/0/project" ] || fail "ghost slot 0 was not reaped" [ ! -d "$base/runs/ghost-proj" ] || fail "orphan run dir leaked: $base/runs/ghost-proj" # The sweep released its own lock on the way out. [ ! -d "$base/slots/.sweep.lock" ] || fail "sweep lock left behind after a normal sweep" } @test "reaping a dead slot composes the project down before deleting its state" { # Stopped-but-present remnants: container-liveness protection is RUNNING # containers only (docker ps -q). A --keep'd stack whose containers were # stopped (manual docker stop, daemon restart, host reboot) reaches the reap # with stopped containers and named volumes still present — deleting the run # dir + slot while those remain would strand them with no compose state # (split-brain). The reap must therefore attempt a best-effort, project- # labeled compose-down FIRST. load_common local base="$XDG_STATE_HOME/copilotkit/showcase" mkdir -p "$base/slots/0" echo "stopped-keeper" > "$base/slots/0/project" # validated record, no pid file mkdir -p "$base/runs/stopped-keeper" export DOCKER_PS_OUTPUT="" # containers stopped → no RUNNING ones → reap path # Ordering probe (see the stub): snapshot the runs dir's existence at # compose-down time, so "down BEFORE state deletion" is pinned directly. export REAP_ORDER_CHECK_DIR="$base/runs/stopped-keeper" : > "$DOCKER_LOG" docker version _claim_isolate_slot # Slot 0 is reserved for the base stack, so the auto-picker lands on slot 1. # The dead slot 0 is still reaped by the sweep (verified by the ghost project # record and runs dir being gone), the claim just doesn't land there. [ "$ISOLATE_SLOT" = "1" ] || fail "expected slot 1 (slot 0 reserved), got: $ISOLATE_SLOT" [ ! -f "$base/slots/0/project" ] || fail "ghost project record survived the reap" [ ! -d "$base/runs/stopped-keeper" ] || fail "runs dir survived the reap" # The remnant cleanup ran: a project-labeled compose down carrying --volumes. _assert_stub_sentinel_logged run grep -E -- "compose -p stopped-keeper down --remove-orphans --volumes" "$DOCKER_LOG" [ "$status" -eq 0 ] \ || fail "reap did not compose the dead project down: $(cat "$DOCKER_LOG")" # …and it ran BEFORE the state deletion: the runs dir still existed when the # down was invoked (deleting state first would strand stopped containers / # named volumes with no compose state — the split-brain the order prevents). run grep -qx -- "REAP_ORDER_CHECK dir-present" "$DOCKER_LOG" [ "$status" -eq 0 ] \ || fail "compose-down ran AFTER the runs dir was deleted (reap order regression): $(cat "$DOCKER_LOG")" } @test "a slot whose project record fails the traversal guard is left intact for inspection" { # Path-traversal guard vs. orphaning: a corrupted/tampered project record # (e.g. "../..") must never be interpolated into rm -rf — but the OLD code # still rm -rf'd the SLOT dir, destroying the only pointer to whatever runs # dir the record names. The guard now leaves the whole slot in place for # manual inspection instead of half-destroying the evidence. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" printf '%s\n' '../evil' > "$slots/0/project" # fails [a-z0-9][a-z0-9_-]* export DOCKER_PS_OUTPUT="" # no live containers, no pid file → reap path _claim_isolate_slot # The suspicious slot survives — with its record — and the claim moves on. [ "$ISOLATE_SLOT" = "1" ] || fail "expected slot 1 (suspicious slot 0 preserved), got: $ISOLATE_SLOT" [ -d "$slots/0" ] || fail "suspicious slot 0 was reaped (record/evidence destroyed)" run cat "$slots/0/project" [ "$status" -eq 0 ] || fail "suspicious slot 0 record removed" [[ "$output" == "../evil" ]] || fail "suspicious record mangled: $output" } @test "a slot whose project record reads the RESERVED 'showcase' is left intact — reap never composes the default stack down" { # Reserved-name guard in the REAPER: 'showcase' IS the default stack's # compose project name, and it PASSES the charset regex — so a slot record # reading 'showcase' (a corrupt record, or one written by an older CLI # version before apply_isolation reserved the name) used to make the reap # run `compose -p showcase down --remove-orphans --volumes` against the # user's LIVE DEFAULT stack, destroying the PocketBase named volume. # apply_isolation reserves the name at claim time, but the reaper must not # trust records: the reserved name gets the same treatment as the # path-traversal guard — warn and leave the whole slot in place for manual # inspection (no compose-down, no state removal). load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "showcase" > "$slots/0/project" # passes [a-z0-9][a-z0-9_-]* export DOCKER_PS_OUTPUT="" # no live containers, no pid file → reap path : > "$DOCKER_LOG" docker version # sentinel: prove stub+DOCKER_LOG wiring before the claim run _claim_isolate_slot [ "$status" -eq 0 ] || fail "_claim_isolate_slot failed: $output" # A loud warning names the offending record and why it is dangerous. # (Checked before any `run grep` clobbers $output.) [[ "$output" == *"RESERVED"* ]] \ || fail "no reserved-name warning printed: $output" [[ "$output" == *"$slots/0"* ]] \ || fail "warning does not name the slot record: $output" # The reserved-name slot survives — with its record — for inspection, and # the claim moves on to the next free slot (the subshell's mkdir persists # on the filesystem even though `run` ran the claim in a subshell). [ -d "$slots/0" ] || fail "reserved-name slot 0 was reaped (default stack endangered)" run cat "$slots/0/project" [ "$status" -eq 0 ] || fail "reserved-name slot 0 record removed" [[ "$output" == "showcase" ]] || fail "reserved-name record mangled: $output" [ -d "$slots/1" ] || fail "claim did not proceed past the preserved slot 0" # NO compose-down of the 'showcase' project may have run — match BOTH # compose project-flag spellings (the reaper uses -p; other paths use # --project-name). The sentinel proves the stub→log pipeline is live, so # this absence check cannot pass vacuously. _assert_stub_sentinel_logged run grep -E -- "(--project-name|-p) showcase down" "$DOCKER_LOG" [ "$status" -ne 0 ] \ || fail "reap composed the RESERVED 'showcase' project down (live default stack): $output" } @test "a fresh pre-existing .sweep.lock skips the sweep; claiming still succeeds" { # TOCTOU sweep/claim race: two concurrent claimants could both observe slot 0 # stale — A reaps + re-claims it, then B reaps A's FRESH claim and claims the # same slot (two owners, identical port offsets). The sweep is serialized by # a .sweep.lock dir; a held (fresh) lock means another process is already # sweeping, so this claimant must SKIP the sweep — the stale-looking slot # survives — and proceed straight to claiming a free slot. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "ghost-proj" > "$slots/0/project" # stale-looking: dead project, no pid mkdir -p "$slots/.sweep.lock" # fresh lock — sweeper "active" export DOCKER_PS_OUTPUT="" _claim_isolate_slot # Sweep skipped: the stale-looking slot 0 survives untouched… [ -d "$slots/0" ] || fail "sweep ran despite a held .sweep.lock" run cat "$slots/0/project" [ "$status" -eq 0 ] || fail "slot 0 project file missing after skipped sweep" [[ "$output" == "ghost-proj" ]] || fail "slot 0 project mangled: $output" # …and the claim proceeds to the next free slot (lock name never collides # with the numeric claim loop). [ "$ISOLATE_SLOT" = "1" ] || fail "expected slot 1 with sweep skipped, got: $ISOLATE_SLOT" # The foreign sweeper's fresh lock is not ours to remove. [ -d "$slots/.sweep.lock" ] || fail "foreign (fresh) sweep lock was removed" } @test "an over-age .sweep.lock is taken over and the sweep proceeds" { # Crashed-sweeper recovery, with the DEDICATED lock threshold: the lock is # held for seconds, so a leftover lock aged ~10 minutes must be taken over — # under the old code the lock reused the 2-hour SLOT threshold # (ISOLATE_STALE_THRESHOLD), so a crashed sweeper disabled stale reaping for # 2 hours. The takeover is an atomic mv-to-tombstone (only one of two racing # observers can win the rename), so no tombstone may leak either. The # companion guarantee — a FRESH lock is NOT taken over — is pinned by the # "fresh pre-existing .sweep.lock skips the sweep" test above. require_python3 # _touch_age computes the backdated stamp with python3 load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "ghost-proj" > "$slots/0/project" # dead project, no pid file mkdir -p "$slots/.sweep.lock" _touch_age 600 "$slots/.sweep.lock" # >60s lock threshold, <<2h slot threshold export DOCKER_PS_OUTPUT="" _claim_isolate_slot # Takeover happened → the sweep ran → the dead slot 0 was reaped. The # auto-picker lands on slot 1 (slot 0 reserved for the base stack); the # absence of the ghost project record is what proves the sweep ran after the # takeover. [ "$ISOLATE_SLOT" = "1" ] \ || fail "over-age sweep lock not taken over (sweep skipped); claimed: $ISOLATE_SLOT" [ ! -f "$slots/0/project" ] \ || fail "ghost project record survived — sweep did not run after the takeover" # The winner's fresh lock was released on the way out, and no tombstone leaked. [ ! -d "$slots/.sweep.lock" ] || fail "sweep lock left behind after takeover sweep" run bash -c "ls -A '$slots' | grep -F '.sweep.lock.tomb'" [ "$status" -ne 0 ] || fail "takeover tombstone leaked: $output" } @test "sweep-lock release is own-lock-aware: a taken-over lock is left in place" { # Takeover/release race: if a slow sweeper's lock is taken over (heartbeat # stalled — e.g. a wedged docker daemon pushed it over-age), the ORIGINAL # holder's release must not remove the TAKEOVER's fresh lock — that would # open the door to a THIRD concurrent sweeper. Ownership is the pid file # written into the lock dir at acquisition; release only removes a lock # that still carries OUR pid. Tested via direct invocation of the release # helper (pausing a real sweep mid-flight is not deterministic in bats). load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" local lock="$slots/.sweep.lock" # Simulate the takeover: the lock exists but carries a DIFFERENT holder's # pid (the takeover rewrote it). PID 1 is never this test process. mkdir -p "$lock" echo "1" > "$lock/pid" run _release_sweep_lock "$lock" [ "$status" -eq 0 ] || fail "_release_sweep_lock failed on a foreign lock: $output" [ -d "$lock" ] || fail "release removed a lock owned by ANOTHER holder (takeover's lock destroyed)" [[ "$output" == *"taken over"* ]] || fail "no takeover warning printed: $output" # Control: a lock carrying OUR pid IS removed on release. echo "$$" > "$lock/pid" run _release_sweep_lock "$lock" [ "$status" -eq 0 ] || fail "_release_sweep_lock failed on our own lock: $output" [ ! -d "$lock" ] || fail "release left our own lock behind" } @test "an over-age takeover tombstone is reclaimed on the next claim; a fresh one survives" { # Crashed-takeover recovery: a sweeper that died between the takeover mv and # its rm -rf leaves .sweep.lock.tomb. behind — dot-named, invisible to # every glob, cleaned by nothing. The claim path must reclaim tombstones # older than the LOCK threshold, while leaving a FRESH tombstone alone (it # may belong to a takeover in flight: mv done, rm pending). load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/.sweep.lock.tomb.11111" touch -t 202001010000 "$slots/.sweep.lock.tomb.11111" # far over the 60s lock threshold mkdir -p "$slots/.sweep.lock.tomb.22222" # fresh — takeover may be in flight _claim_isolate_slot [ ! -d "$slots/.sweep.lock.tomb.11111" ] || fail "over-age tombstone leaked" [ -d "$slots/.sweep.lock.tomb.22222" ] || fail "fresh (possibly in-flight) tombstone was removed" } @test "sweep heartbeat never recreates a removed sweep lock (no plain-file resurrection)" { # Takeover/heartbeat race: the lock can be mv'd away (taken over) between # heartbeats. The OLD heartbeat (bare `touch "$sweep_lock"`) then RECREATED # the lock as a plain FILE — the takeover's `mkdir` failed against it and # sweeping wedged until the 60s over-age self-heal. The heartbeat must # refresh-only (touch -c behind a -d guard), never create. Direct # invocation: run the sweep with the lock already gone — exactly the # post-takeover view inside an iteration — and pin that no lock (file OR # dir) materializes. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" # one slot so the loop body (heartbeat) runs echo "ghost-proj" > "$slots/0/project" export DOCKER_PS_OUTPUT="" _sweep_isolate_slots [ ! -e "$slots/.sweep.lock" ] \ || fail "heartbeat recreated the removed sweep lock: $(ls -ld "$slots/.sweep.lock")" } @test "an over-age PLAIN-FILE .sweep.lock (old heartbeat artifact) is taken over; claiming proceeds" { # Forward-compat with the old bug's residue: a pre-fix heartbeat could leave # the lock behind as a plain FILE. mkdir fails against it just like against # a dir, so it must flow through the same over-age takeover (mv works on # files too) instead of wedging the sweep/claim forever. require_python3 # _touch_age computes the backdated stamp with python3 load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "ghost-proj" > "$slots/0/project" # dead project, no pid file : > "$slots/.sweep.lock" # plain FILE, not a dir _touch_age 600 "$slots/.sweep.lock" # over the 60s lock threshold export DOCKER_PS_OUTPUT="" _claim_isolate_slot # Takeover + sweep ran: the dead slot 0 was reaped. The auto-picker lands on # slot 1 (slot 0 reserved for the base stack); the absence of slot 0's # project file (asserted below alongside the lock-residue check) is what # proves the sweep ran past the plain-file lock. [ "$ISOLATE_SLOT" = "1" ] || fail "plain-file lock wedged the sweep; claimed: $ISOLATE_SLOT" [ ! -f "$slots/0/project" ] || fail "ghost project record survived — sweep did not run" # And nothing lock-shaped was left behind (takeover's own lock released). [ ! -e "$slots/.sweep.lock" ] || fail "stale plain-file lock (or its takeover) left behind" # No takeover tombstone leaked either (same guarantee the dir-lock variant # pins: the winner disposes of the mv'd-aside tombstone). run bash -c "ls -A '$slots' | grep -F '.sweep.lock.tomb'" [ "$status" -ne 0 ] || fail "takeover tombstone leaked: $output" } @test "claim survives an over-age tombstone whose removal fails (set -e safety)" { # The tombstone reclamation runs OUTSIDE the sweep lock by design, so two # claimants can race the same rm: both observe the tombstone over-age, one # wins, and the loser's mid-traversal ENOENT makes its rm exit nonzero — # under bin/showcase's `set -euo pipefail` that used to kill the whole CLI. # The race itself isn't deterministically reproducible in bats; an # unremovable tombstone (non-writable dir with contents) forces the same # nonzero rm exit deterministically. # # Root caveat: as root, `chmod a-w` does not block rm — the tombstone is # removed anyway, the failure can't be forced, and the cleanup chmod below # would then ERROR on the vanished path under bats errexit. Skip as root. [ "$(id -u)" -ne 0 ] || skip "rm -rf succeeds as root — cannot force the tombstone rm failure" load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/.sweep.lock.tomb.99999/inner" touch -t 202001010000 "$slots/.sweep.lock.tomb.99999" # far over the 60s threshold chmod a-w "$slots/.sweep.lock.tomb.99999" # rm -rf cannot unlink 'inner' run bash -euo pipefail -c "source '$COMMON'; _claim_isolate_slot; echo \"CLAIMED=\$ISOLATE_SLOT\"" # Let bats teardown clean up; guarded — the tombstone may be gone if rm # unexpectedly succeeded, and a chmod error here must not mask the real # assertions below. chmod u+w "$slots/.sweep.lock.tomb.99999" 2>/dev/null || true [ "$status" -eq 0 ] || fail "claim died under set -e on a failed tombstone rm: $output" # Slot 0 is reserved for the base stack, so the auto-picker lands on slot 1. # What this test really pins is that the claim PROCEEDED past the failed # tombstone rm under set -e — landing on any valid slot is sufficient proof. [[ "$output" == *"CLAIMED=1"* ]] \ || fail "claim did not proceed past the failed tombstone rm: $output" } @test "takeover survives a failed tombstone disposal (set -e safety) and still sweeps" { # The takeover winner's `rm -rf $lock_tombstone` is ACTIVELY raced: mv # preserves the (already over-age) lock mtime, so the fresh tombstone is # immediately over-age and concurrent claimants' tombstone-reclamation # loops legitimately race its removal — the loser's nonzero rm must not # kill the CLI under bin/showcase's `set -euo pipefail`. As in the # tombstone-reclamation test above, the race itself isn't deterministic in # bats; an unremovable tombstone (non-writable dir with contents, preserved # verbatim by the takeover mv) forces the same nonzero rm exit. require_python3 # _touch_age computes the backdated stamp with python3 [ "$(id -u)" -ne 0 ] || skip "rm -rf succeeds as root — cannot force the tombstone rm failure" load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "ghost-proj" > "$slots/0/project" # dead project, no pid file mkdir -p "$slots/.sweep.lock/inner" # over-age lock the takeover will mv aside _touch_age 600 "$slots/.sweep.lock" # >60s lock threshold chmod a-w "$slots/.sweep.lock" # rm -rf of the mv'd tombstone cannot unlink 'inner' export DOCKER_PS_OUTPUT="" run bash -euo pipefail -c "source '$COMMON'; _claim_isolate_slot; echo \"CLAIMED=\$ISOLATE_SLOT\"" # Let bats teardown clean up the unremovable tombstone (pid-suffixed by the # child, hence the glob); guarded — must not mask the assertions below. chmod u+w "$slots"/.sweep.lock.tomb.* 2>/dev/null || true [ "$status" -eq 0 ] || fail "takeover died under set -e on a failed tombstone rm: $output" # The takeover completed despite the failed disposal: fresh lock acquired, # sweep ran, dead slot 0 reaped. The auto-picker lands on slot 1 (slot 0 # reserved for the base stack); landing on any valid slot proves the sweep # progressed past the failed tombstone disposal. [[ "$output" == *"CLAIMED=1"* ]] \ || fail "sweep did not run after the failed tombstone disposal: $output" } @test "_release_sweep_lock distinguishes a vanished lock from a takeover" { # When the lock (or its pid ownership marker) is GONE entirely, the release # used to mis-report "taken over (current holder pid: unknown)" — there is # no holder to report. A vanished lock gets its own message; the takeover # warn (pinned by the own-lock-aware test above) stays reserved for a lock # that actually carries a DIFFERENT holder's pid. load_common local lock="$XDG_STATE_HOME/copilotkit/showcase/slots/.sweep.lock" [ ! -e "$lock" ] || fail "precondition: lock unexpectedly exists" run _release_sweep_lock "$lock" [ "$status" -eq 0 ] || fail "_release_sweep_lock failed on a missing lock: $output" [[ "$output" == *"vanished"* ]] || fail "missing lock not reported as vanished: $output" [[ "$output" != *"taken over"* ]] || fail "missing lock mis-reported as a takeover: $output" } # ── Change 3: restore_isolation honors --keep ──────────────────────────────── @test "restore_isolation with keep set preserves slot + run dir and prints survival notice" { require_python3 load_common apply_isolation keepme local slotdir="$ISOLATE_SLOT_DIR/$ISOLATE_SLOT" local rundir="$ISOLATE_TMPDIR" [ -d "$slotdir" ] || fail "precondition: slot dir missing after apply" [ -d "$rundir" ] || fail "precondition: run dir missing after apply" # Truncate the stub log: apply_isolation's idempotent pre-down already # logged a `--project-name keepme down ...` line, which would poison the # does-NOT-compose-down assertion below. The `docker version` sentinel then # proves the stub→DOCKER_LOG pipeline is still live AFTER the truncation, # so the absence check cannot pass vacuously. : > "$DOCKER_LOG" docker version ISOLATE_KEEP=true run restore_isolation [ "$status" -eq 0 ] || fail "restore_isolation failed under keep: $output" # Nothing torn down: slot + run dir survive, slot NOT released. [ -d "$slotdir" ] || fail "kept slot dir was removed" [ -d "$rundir" ] || fail "kept run dir was removed" # Survival notice content: project name, the 3 +offset host ports, teardown cmd. # Slot 0 is reserved for the base stack, so apply_isolation auto-picks slot 1 # (offset (1+1)*200 = 400): aimock 4010+400=4410, dashboard 3210+400=3610, # pocketbase 8090+400=8490. [[ "$output" == *"keepme"* ]] || fail "notice missing project name: $output" [[ "$output" == *"4410"* ]] || fail "notice missing aimock host port: $output" [[ "$output" == *"3610"* ]] || fail "notice missing dashboard host port: $output" [[ "$output" == *"8490"* ]] || fail "notice missing pocketbase host port: $output" [[ "$output" == *"docker compose -p keepme down"* ]] \ || fail "notice missing literal teardown command: $output" # The keep path's central promise: it does NOT compose-down the kept stack. # Match BOTH compose project-flag spellings — restore_isolation's own down # uses --project-name, but the reaper (and any keep-branch regression) # spells it -p, which a --project-name-only grep would let slip through. # (Last, because `run grep` clobbers the $output the notice checks read.) _assert_stub_sentinel_logged run grep -E -- "(--project-name|-p) keepme down" "$DOCKER_LOG" [ "$status" -ne 0 ] || fail "keep path composed the kept stack down: $output" } # The survival notice's human-readable hours must track ISOLATE_KEEP_TTL, not a # hardcoded "(4h)". SHOWCASE_ISOLATE_KEEP_TTL is read at source time, so it must # be exported BEFORE load_common sources _common.sh. @test "survival notice's human-readable hours track an overridden ISOLATE_KEEP_TTL (not a stale (4h))" { require_python3 export SHOWCASE_ISOLATE_KEEP_TTL=7200 # 2h, not the 4h default load_common [ "$ISOLATE_KEEP_TTL" = 7200 ] || fail "precondition: TTL override not picked up, got '$ISOLATE_KEEP_TTL'" apply_isolation keepttl ISOLATE_KEEP=true run restore_isolation [ "$status" -eq 0 ] || fail "restore_isolation failed under keep: $output" # The seconds value must reflect the override. [[ "$output" == *"7200s"* ]] || fail "notice missing overridden TTL seconds: $output" # And the parenthetical hours must NOT contradict it with a stale 4h. [[ "$output" != *"(4h)"* ]] || fail "notice still says (4h) for a 2h TTL: $output" # The computed hours should read sensibly (2h here). [[ "$output" == *"(2h)"* ]] || fail "notice missing computed (2h): $output" } @test "survival notice reads sensibly at the default ISOLATE_KEEP_TTL (4h)" { require_python3 load_common [ "$ISOLATE_KEEP_TTL" = 14400 ] || fail "precondition: expected default TTL 14400, got '$ISOLATE_KEEP_TTL'" apply_isolation keepdef ISOLATE_KEEP=true run restore_isolation [ "$status" -eq 0 ] || fail "restore_isolation failed under keep: $output" [[ "$output" == *"14400s"* ]] || fail "notice missing default TTL seconds: $output" [[ "$output" == *"(4h)"* ]] || fail "notice missing default (4h): $output" } # ── Change 4: a FAILED apply_isolation must never down the default stack ───── # # cmd-test.sh registers `trap restore_isolation EXIT` BEFORE calling # apply_isolation, so a die() inside apply_isolation (invalid name, slot # exhaustion, ...) fires the trap with half-initialized isolation state. If # ISOLATE_ACTIVE was already true while COMPOSE_CMD still pointed at the # ORIGINAL docker-compose.local.yml (no project name), restore_isolation would # silently `compose down` the user's live DEFAULT stack. This pins that a # failed apply_isolation results in NO compose-down against the default project. @test "failed apply_isolation (invalid name) does not compose-down the default stack via the EXIT trap" { load_common # Mirror cmd-test.sh's real registration order in a child bash process run # under bin/showcase's real shell options (-euo pipefail): a sentinel docker # call proves the stub+log pipeline, the EXIT trap is armed, then # apply_isolation dies on the invalid name ('MyName!' lowercases to # 'myname!', which still fails [a-z0-9][a-z0-9_-]*). run bash -euo pipefail -c " source '$COMMON' SHOWCASE_ROOT='$SHOWCASE_ROOT_OVERRIDE' COMPOSE_FILE=\"\$SHOWCASE_ROOT/docker-compose.local.yml\" PORTS_FILE=\"\$SHOWCASE_ROOT/shared/local-ports.json\" COMPOSE_CMD=\"docker compose -f \$COMPOSE_FILE\" docker version # sentinel: prove stub+DOCKER_LOG wiring BEFORE the trap trap restore_isolation EXIT apply_isolation 'MyName!' " [ "$status" -ne 0 ] || fail "apply_isolation unexpectedly succeeded with invalid name 'MyName!'" # Stub pipeline verified live — now absence of compose-down is meaningful. _assert_stub_sentinel_logged # The EXIT trap must NOT have run any `compose ... down` (word-matched, so a # project name merely CONTAINING "down" can't satisfy/poison the check, and # the bare minimal `compose down` — no args in between — is matched too) — # neither against the original compose file nor the (never-created) # isolated project. local default_compose="$SHOWCASE_ROOT_OVERRIDE/docker-compose.local.yml" run grep -E "(^|[[:space:]])compose([[:space:]].*)?[[:space:]]down([[:space:]]|\$)" "$DOCKER_LOG" [ "$status" -ne 0 ] \ || fail "EXIT trap composed down after failed apply_isolation: $output" run grep -F "$default_compose" "$DOCKER_LOG" [ "$status" -ne 0 ] \ || fail "EXIT trap touched the default compose file after failed apply_isolation: $output" } @test "apply_isolation dying AFTER the slot claim (python3 failure) does not compose-down the default stack via the EXIT trap" { load_common # The dangerous window is wider than the pre-claim die above: apply_isolation # can also die AFTER the slot claim but BEFORE ISOLATE_ACTIVE=true — e.g. the # python3 ports/compose rewriters failing. COMPOSE_CMD still points at the # ORIGINAL compose file in that window, so the armed EXIT trap must not run # any compose-down against the default stack. Poison python3 via a PATH stub # in the child to force the post-claim die. # # NB: this test pins ONLY the no-default-stack-compose-down invariant; what # happens to the slot/run dirs in this window is pinned elsewhere. local pybad="$BATS_TEST_TMPDIR/pybad" mkdir -p "$pybad" printf '#!/usr/bin/env bash\nexit 1\n' > "$pybad/python3" chmod +x "$pybad/python3" run bash -euo pipefail -c " PATH='$pybad':\"\$PATH\" # poison python3; docker stub stays on PATH source '$COMMON' SHOWCASE_ROOT='$SHOWCASE_ROOT_OVERRIDE' COMPOSE_FILE=\"\$SHOWCASE_ROOT/docker-compose.local.yml\" PORTS_FILE=\"\$SHOWCASE_ROOT/shared/local-ports.json\" COMPOSE_CMD=\"docker compose -f \$COMPOSE_FILE\" docker version # sentinel: prove stub+DOCKER_LOG wiring BEFORE the trap trap restore_isolation EXIT apply_isolation 'pyfail' " [ "$status" -ne 0 ] || fail "apply_isolation unexpectedly succeeded with a poisoned python3" # Stub pipeline verified live — now absence of compose-down is meaningful. _assert_stub_sentinel_logged # No compose-down of ANY kind may have run in this window: the idempotent # pre-down comes only after ISOLATE_ACTIVE=true (never reached here), and # the trap's half-init cleanup is state-only. The pattern also matches the # bare minimal `compose down` (no args between the two words), which a # `compose .*[[:space:]]down` regex would miss. run grep -E "(^|[[:space:]])compose([[:space:]].*)?[[:space:]]down([[:space:]]|\$)" "$DOCKER_LOG" [ "$status" -ne 0 ] \ || fail "EXIT trap composed down after post-claim die: $output" } # ── Leading-character rule: compose project names must START with [a-z0-9] ─── # # docker compose accepts [a-z0-9][a-z0-9_-]* — '_' and '-' are valid only in # the INTERIOR of a project name. Names like '_foo' or '-foo' used to slip # through the CLI check (which only looked for chars outside [a-z0-9_-]) and # then failed deep inside compose with exactly the opaque error this # validation exists to prevent. @test "apply_isolation rejects names with an invalid leading character" { load_common run apply_isolation '_foo' [ "$status" -ne 0 ] || fail "apply_isolation unexpectedly accepted '_foo'" [[ "$output" == *"must start with a lowercase letter or digit"* ]] \ || fail "die message does not describe the leading-character rule: $output" run apply_isolation '-foo' [ "$status" -ne 0 ] || fail "apply_isolation unexpectedly accepted '-foo'" [[ "$output" == *"must start with a lowercase letter or digit"* ]] \ || fail "die message does not describe the leading-character rule: $output" } @test "apply_isolation accepts a normal name with interior '_' and '-'" { require_python3 load_common apply_isolation 'foo_bar-9' [[ "$ISOLATE_TMPDIR" == "$XDG_STATE_HOME/copilotkit/showcase/runs/foo_bar-9" ]] \ || fail "valid name 'foo_bar-9' was not accepted: $ISOLATE_TMPDIR" } # ── Reserved name: 'showcase' IS the default stack's compose project ───────── # # docker compose defaults the project name to the directory name, so the # DEFAULT stack's project is literally "showcase". An accidental # `--isolate showcase` used to sail through the charset validation (the # container-name rewrite showcase- → showcase- is a no-op), and the idempotent # pre-down then ran `--project-name showcase down --remove-orphans --volumes` # — tearing down the user's live default stack, bypassing every guard this # code has. @test "apply_isolation rejects the reserved name 'showcase' before any compose command" { load_common docker version # sentinel: prove stub+DOCKER_LOG wiring BEFORE the call run apply_isolation showcase [ "$status" -ne 0 ] || fail "apply_isolation accepted the reserved name 'showcase'" [[ "$output" == *"reserved"* ]] \ || fail "die message does not say the name is reserved: $output" # The uppercase variant must be caught too: the check runs AFTER the # lowercase normalization, so 'Showcase' lowercases into the reserved name. run apply_isolation Showcase [ "$status" -ne 0 ] || fail "apply_isolation accepted 'Showcase' (lowercases to the reserved name)" [[ "$output" == *"reserved"* ]] \ || fail "die message for 'Showcase' does not say the name is reserved: $output" # No compose command of ANY kind may have run — the rejection must land # BEFORE the idempotent pre-down that would hit the default stack. Word- # matched so the sweep's `ps -q --filter label=com.docker.compose.project=…` # line can't satisfy/poison the check. The sentinel proves the stub→log # pipeline is live first, so this absence check cannot pass vacuously. _assert_stub_sentinel_logged run grep -E "(^|[[:space:]])compose([[:space:]]|\$)" "$DOCKER_LOG" [ "$status" -ne 0 ] \ || fail "reserved-name rejection ran a compose command: $output" } @test "restore_isolation without keep removes run dir and releases slot" { require_python3 load_common apply_isolation dropme local slotdir="$ISOLATE_SLOT_DIR/$ISOLATE_SLOT" local rundir="$ISOLATE_TMPDIR" [ -d "$slotdir" ] || fail "precondition: slot dir missing after apply" [ -d "$rundir" ] || fail "precondition: run dir missing after apply" # Truncate the stub log NOW: apply_isolation's idempotent pre-down logs a # BYTE-IDENTICAL `--project-name dropme down --remove-orphans --volumes` # line, so without the truncation the teardown assertion below would pass # vacuously even if restore_isolation dropped --volumes or never composed # down at all. After the truncation, any matching line can only have come # from the teardown itself. : > "$DOCKER_LOG" ISOLATE_KEEP=false run restore_isolation [ "$status" -eq 0 ] || fail "restore_isolation failed: $output" # NB `run` executes in a subshell, so the parent's ISOLATE_* are unchanged; we # assert on the filesystem effects, which DO persist across the subshell. [ ! -d "$rundir" ] || fail "run dir not removed on teardown: $rundir" [ ! -d "$slotdir" ] || fail "slot not released on teardown: $slotdir" # Volumes consistency: isolated stacks are ephemeral, and both printed # manual teardown commands (keep notice + failed-down recovery) include # --volumes. The AUTOMATIC teardown must agree, or project-scoped named # volumes leak (unbounded for explicit names). The log was truncated after # apply_isolation returned (see above), so this match is necessarily the # teardown's own project-named down carrying --volumes. run grep -E -- "--project-name dropme down --remove-orphans --volumes" "$DOCKER_LOG" [ "$status" -eq 0 ] \ || fail "automatic teardown down did not include --volumes: $(cat "$DOCKER_LOG")" } # ── Duplicate explicit --isolate name must not destroy a live run ──────────── # # apply_isolation's idempotent pre-down keys on the compose project NAME, but # the slot registry only enforces SLOT uniqueness. A second concurrent run # passing the same explicit --isolate used to get a different slot but # the same compose project — its pre-down silently tore down the first run's # containers mid-test (or a --keep-parked stack), and two slots then recorded # the same project name, corrupting the liveness-reaping signal. @test "apply_isolation dies on a duplicate explicit name without composing it down" { load_common # Seed slot 0 as another LIVE run already using "dupename": project recorded, # owning PID alive (this bats process), so the stale-sweep must not reap it. local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "dupename" > "$slots/0/project" _seed_live_owner "$slots/0" export DOCKER_PS_OUTPUT="" # zero containers (still building) — PID protects run apply_isolation dupename [ "$status" -ne 0 ] || fail "apply_isolation accepted a duplicate name" [[ "$output" == *"already in use by slot 0"* ]] \ || fail "die message does not name the conflicting slot: $output" # The live run's stack must be untouched: NO compose-down of any kind may # have run (the pre-down comes only after the duplicate guard). The sweep is # expected to have called `docker ps` for the recorded project — assert that # exact invocation actually landed in the log (the stub logs one # "$*"-joined line per call), not merely that the log file exists: a # silently broken stub/DOCKER_LOG pipeline would make the absence check # below pass vacuously. [ -f "$DOCKER_LOG" ] \ || fail "docker stub log missing — the sweep should have called docker ps; absence check would be vacuous" grep -qxF -- "ps -q --filter label=com.docker.compose.project=dupename" "$DOCKER_LOG" \ || fail "sweep's docker ps invocation not logged — stub/DOCKER_LOG pipeline broken, absence check would be vacuous: $(cat "$DOCKER_LOG")" run grep -E "(^|[[:space:]])compose([[:space:]].*)?[[:space:]]down([[:space:]]|\$)" "$DOCKER_LOG" [ "$status" -ne 0 ] \ || fail "duplicate-name run composed down the live project: $output" # Slot 0's record survives intact for the live owner. [ -d "$slots/0" ] || fail "live slot 0 was reaped during the duplicate claim" run cat "$slots/0/project" [ "$status" -eq 0 ] || fail "live slot 0 project file missing after duplicate claim" [[ "$output" == "dupename" ]] || fail "live slot 0 project mangled: $output" } @test "duplicate-name guard is claim-then-verify: loser backs off without touching the winner" { # TOCTOU regression: the guard used to SCAN the other slots and only then # write its own project record. Two concurrent same-name claims could both # pass the scan and both record the name — one run's pre-down then tore the # other's stack down mid-test, and reaping either slot deleted the SHARED # runs/ dir. The fix writes our record FIRST, then scans: the later # writer of any concurrent pair is guaranteed to see the earlier record and # back off (die) before runs/ is ever created or touched. load_common local base="$XDG_STATE_HOME/copilotkit/showcase" local slots="$base/slots" # Winner: slot 0 already holds 'dup' (live owning PID) plus its run dir. mkdir -p "$slots/0" echo "dup" > "$slots/0/project" _seed_live_owner "$slots/0" mkdir -p "$base/runs/dup" touch "$base/runs/dup/docker-compose.local.yml" export DOCKER_PS_OUTPUT="" # zero containers (still building) — PID protects # Ordering pin (claim BEFORE verify): `run` dies in a subshell with no trap, # so the loser's slot dir survives for inspection — and it must already # contain our own 'dup' record, written BEFORE the conflict scan. Under the # old scan-then-write order this record never exists (that unrecorded window # is exactly the TOCTOU hole). run apply_isolation dup [ "$status" -ne 0 ] || fail "apply_isolation accepted duplicate name 'dup'" [[ "$output" == *"already in use by slot 0"* ]] \ || fail "die message does not name the conflicting slot: $output" run cat "$slots/1/project" [ "$status" -eq 0 ] \ || fail "loser did not record its claim BEFORE verifying (scan-then-write TOCTOU order)" [[ "$output" == "dup" ]] || fail "loser recorded the wrong project: $output" rm -rf "$slots/1" # clear the no-trap leftovers; trap cleanup is pinned next # Real wiring (trap armed first, bin/showcase's shell options): the loser's # die must clean up its OWN slot — including the just-written record — via # the EXIT trap, and the winner's slot record AND runs/dup dir must be # untouched (ISOLATE_TMPDIR is set only after the verify passes, so the # loser's cleanup cannot reach the shared run dir). run bash -euo pipefail -c " source '$COMMON' SHOWCASE_ROOT='$SHOWCASE_ROOT_OVERRIDE' COMPOSE_FILE=\"\$SHOWCASE_ROOT/docker-compose.local.yml\" PORTS_FILE=\"\$SHOWCASE_ROOT/shared/local-ports.json\" COMPOSE_CMD=\"docker compose -f \$COMPOSE_FILE\" trap restore_isolation EXIT apply_isolation dup " [ "$status" -ne 0 ] || fail "apply_isolation accepted duplicate name 'dup' (trap run)" [ ! -d "$slots/1" ] || fail "loser's slot/record leaked after the EXIT trap: $slots/1" # Winner untouched: record intact, run dir (and its contents) intact. [ -d "$slots/0" ] || fail "winner's slot was reaped by the losing claim" run cat "$slots/0/project" [ "$status" -eq 0 ] || fail "winner's project record missing" [[ "$output" == "dup" ]] || fail "winner's project record mangled: $output" [ -f "$base/runs/dup/docker-compose.local.yml" ] \ || fail "winner's run dir was damaged by the losing claim" } @test "duplicate-name guard treats an EQUAL-mtime conflicting record as a conflict (tie = back off)" { # Tie semantics pin: the backoff comparison is `-le` — a conflicting record # that does NOT strictly postdate ours (older OR EQUAL epoch mtime) means we # lose. A regression to `-lt` would make two same-second claimants BOTH # treat the other's record as "strictly newer → the other backs off" and # BOTH proceed as dual owners of one compose project. The other duplicate # tests seed their conflicting record by wall clock, so whether they hit # the equal or the older branch depends on second-boundary timing; force # the tie DETERMINISTICALLY by stubbing _file_mtime to one fixed epoch for # every path (safe here: slot 0 is pid-protected, so the sweep never # consults mtimes, and no tombstones/locks exist). load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" echo "tiename" > "$slots/0/project" _seed_live_owner "$slots/0" # live owner — sweep must not reap it export DOCKER_PS_OUTPUT="" _file_mtime() { echo 1700000000; } # our record and theirs: EQUAL mtimes run apply_isolation tiename [ "$status" -ne 0 ] \ || fail "equal-mtime conflicting record did not back the claimant off (tie regression: -le became -lt?)" [[ "$output" == *"already in use by slot 0"* ]] \ || fail "die message does not name the conflicting slot: $output" } @test "re-using a name after clean teardown still works" { require_python3 load_common apply_isolation reusable ISOLATE_KEEP=false restore_isolation # The old slot was released and its record removed — the name is free again. # (In production restore_isolation only runs at EXIT; it does not repoint # COMPOSE_FILE/PORTS_FILE/COMPOSE_CMD at the originals. Re-point them here # the way a fresh process sourcing _common.sh would see them.) COMPOSE_FILE="$SHOWCASE_ROOT/docker-compose.local.yml" PORTS_FILE="$SHOWCASE_ROOT/shared/local-ports.json" COMPOSE_CMD="docker compose -f $COMPOSE_FILE" apply_isolation reusable [[ "$ISOLATE_NAME" == "reusable" ]] || fail "name not reusable after teardown" } # ── Half-initialized apply_isolation must clean its state via the EXIT trap ── # # If apply_isolation dies AFTER _claim_isolate_slot but BEFORE # ISOLATE_ACTIVE=true (python3 failure, mkdir failure), the EXIT trap's # restore_isolation must not compose-down anything (that guard protects the # user's default stack) — but it used to ALSO leak the claimed slot dir and # the runs/ dir silently. The slot self-healed via the stale-sweep; the # runs dir never did. Pin that the trap now cleans both, with NO compose # command at all. @test "half-initialized apply_isolation cleans slot and run dir via EXIT trap without any compose" { # Poison python3 so apply_isolation dies after the slot claim, project-file # write, and run-dir mkdir — i.e. genuinely half-initialized. local poison="$BATS_TEST_TMPDIR/poison" mkdir -p "$poison" printf '#!/usr/bin/env bash\nexit 1\n' > "$poison/python3" chmod +x "$poison/python3" # Mirror cmd-test.sh's real wiring in a child bash: trap armed first, then # apply_isolation fails partway through under set -euo pipefail. run bash -euo pipefail -c " PATH='$poison':\"\$PATH\" source '$COMMON' SHOWCASE_ROOT='$SHOWCASE_ROOT_OVERRIDE' COMPOSE_FILE=\"\$SHOWCASE_ROOT/docker-compose.local.yml\" PORTS_FILE=\"\$SHOWCASE_ROOT/shared/local-ports.json\" COMPOSE_CMD=\"docker compose -f \$COMPOSE_FILE\" docker version # sentinel: prove stub+DOCKER_LOG wiring BEFORE the trap trap restore_isolation EXIT apply_isolation halfinit " [ "$status" -ne 0 ] || fail "apply_isolation unexpectedly survived poisoned python3" # State cleaned: the claimed slot AND the runs/ dir are gone. [ ! -d "$XDG_STATE_HOME/copilotkit/showcase/slots/0" ] \ || fail "half-initialized slot dir leaked" [ ! -d "$XDG_STATE_HOME/copilotkit/showcase/runs/halfinit" ] \ || fail "half-initialized run dir leaked" # And the cleanup was STATE-ONLY: no compose command of any kind ran (the # compose-down hazard against the default stack is exactly what the # ISOLATE_ACTIVE guard prevents — cleanup must not reintroduce it). The # sentinel proves the stub→log pipeline is live first, so this absence # check cannot pass vacuously. Word-matched so the sweep's # `ps -q --filter label=com.docker.compose.project=…` line can't # satisfy/poison the check. _assert_stub_sentinel_logged run grep -E "(^|[[:space:]])compose([[:space:]]|\$)" "$DOCKER_LOG" [ "$status" -ne 0 ] \ || fail "half-init cleanup ran a compose command: $output" } @test "restore_isolation without keep preserves run dir and slot when compose down FAILS" { require_python3 # Split-brain regression: a silently failed `compose down` (stderr to # /dev/null, `|| true`) used to leave the stack RUNNING while the run dir # (the only copy of the rewritten compose file) and the slot were deleted — # live containers with no state and a re-claimable slot (port collisions). # On compose-down failure the evidence must survive and a loud warning with # the project name + manual teardown command must print. load_common apply_isolation downfail local slotdir="$ISOLATE_SLOT_DIR/$ISOLATE_SLOT" local rundir="$ISOLATE_TMPDIR" [ -d "$slotdir" ] || fail "precondition: slot dir missing after apply" [ -d "$rundir" ] || fail "precondition: run dir missing after apply" # Set AFTER apply_isolation so its idempotent pre-down is unaffected. export DOCKER_COMPOSE_DOWN_EXIT=1 ISOLATE_KEEP=false run restore_isolation [ "$status" -eq 0 ] || fail "restore_isolation must not propagate the failure: $output" # Evidence preserved: run dir kept, slot NOT released. [ -d "$rundir" ] || fail "run dir deleted despite failed compose down" [ -d "$slotdir" ] || fail "slot released despite failed compose down" # Loud warning: project name + manual teardown command for recovery. [[ "$output" == *"downfail"* ]] || fail "warning missing project name: $output" [[ "$output" == *"docker compose -p downfail down"* ]] \ || fail "warning missing manual teardown command: $output" } @test "restore_isolation guard mismatch warns and preserves run dir + slot (no fall-through deletion)" { # Belt-and-suspenders divergence: ISOLATE_ACTIVE=true but COMPOSE_CMD is NOT # repointed at the isolated project. The old code skipped the compose-down # (correct — never down an unknown target) but then FELL THROUGH to deleting # the run dir and releasing the slot, manufacturing the exact split-brain # the adjacent comment documents: a possibly-running stack whose only # compose state is deleted and whose slot is reclaimable. Unreachable # through today's call paths (apply_isolation sets ISOLATE_ACTIVE only # after the repoint), but the defensive path must be SAFE, not destructive: # warn loudly and preserve everything, mirroring the failed-down branch. load_common local base="$XDG_STATE_HOME/copilotkit/showcase" mkdir -p "$base/slots/3" "$base/runs/mismatched" echo "$$" > "$base/slots/3/pid" ISOLATE_ACTIVE=true ISOLATE_NAME="mismatched" ISOLATE_SLOT=3 ISOLATE_TMPDIR="$base/runs/mismatched" ISOLATE_KEEP=false # COMPOSE_CMD (from load_common) carries NO --project-name — the mismatch. : > "$DOCKER_LOG" docker version run restore_isolation [ "$status" -eq 0 ] || fail "restore_isolation failed on the mismatch path: $output" [ -d "$base/runs/mismatched" ] \ || fail "run dir deleted on the guard-mismatch path (split-brain)" [ -d "$base/slots/3" ] \ || fail "slot released on the guard-mismatch path (split-brain)" [[ "$output" == *"mismatch"* ]] \ || fail "no loud warning on the guard-mismatch path: $output" [[ "$output" == *"docker compose -p mismatched down"* ]] \ || fail "warning missing manual teardown command: $output" # No compose command of any kind ran — COMPOSE_CMD's target is unknown. _assert_stub_sentinel_logged run grep -E "(^|[[:space:]])compose([[:space:]].*)?[[:space:]]down([[:space:]]|\$)" "$DOCKER_LOG" [ "$status" -ne 0 ] || fail "guard-mismatch path ran a compose down: $output" } @test "apply_isolation warns (but does not die) when the idempotent pre-down fails" { require_python3 # The pre-down exists to clear a prior crashed run's containers/volumes. Its # failure must stay non-fatal (the common case is "nothing to tear down"), # but it must not be SILENT either — leftover state is exactly what the # pre-clean is for, so the user gets a warning that it may remain. load_common export DOCKER_COMPOSE_DOWN_EXIT=1 # set BEFORE apply: poison the pre-down run apply_isolation predownfail [ "$status" -eq 0 ] || fail "apply_isolation died on a failed pre-down: $output" [[ "$output" == *"pre-clean of project predownfail failed"* ]] \ || fail "no warning printed for the failed pre-down: $output" [[ "$output" == *"Isolation active"* ]] \ || fail "apply_isolation did not complete after the failed pre-down: $output" } @test "EXIT trap honors keep flag set inside a function that has returned (real cmd_test wiring)" { require_python3 # Regression test for the --keep teardown bug: cmd_test used to hold the keep # flag in a `local keep`. On the NORMAL path cmd_test returns, its locals # unwind, and the EXIT trap fires later at top-level script exit — where the # local no longer exists — so restore_isolation read `${keep:-false}` as # false and tore the kept stack down anyway. (The earlier tests above call # restore_isolation directly with the flag set at test scope, which is # exactly the pattern that masked this.) # # This test exercises the REAL wiring: a child bash sources _common.sh, # registers `trap restore_isolation EXIT`, then a function sets the keep flag # the same way cmd_test's --keep handler does, applies isolation, RETURNS, # and the script exits normally. The run dir and slot must survive the trap # and the survival notice must print. cat > "$BATS_TEST_TMPDIR/keep-wiring.sh" <<'FIXTURE' #!/usr/bin/env bash set -euo pipefail # bin/showcase's real shell options — fixture fidelity # shellcheck disable=SC1090 source "$COMMON_SH" # Repoint the derived paths at the per-test fake root (same as load_common). SHOWCASE_ROOT="$SHOWCASE_ROOT_OVERRIDE" COMPOSE_FILE="$SHOWCASE_ROOT/docker-compose.local.yml" PORTS_FILE="$SHOWCASE_ROOT/shared/local-ports.json" COMPOSE_CMD="docker compose -f $COMPOSE_FILE" fake_cmd_test() { ISOLATE_KEEP=true # what cmd_test's --keep arg handler does trap restore_isolation EXIT apply_isolation trapkeep return 0 } fake_cmd_test # Truncate the stub log: apply_isolation's idempotent pre-down already logged # a `--project-name trapkeep down ...` line that would poison the parent # test's does-NOT-compose-down assertion. The `docker version` sentinel then # proves the stub→DOCKER_LOG pipeline is still live after the truncation. : > "$DOCKER_LOG" docker version exit 0 # locals (if any) are long gone; the EXIT trap fires HERE FIXTURE export COMMON_SH="$COMMON" run bash "$BATS_TEST_TMPDIR/keep-wiring.sh" [ "$status" -eq 0 ] || fail "fixture script failed: $output" # Survival: the run dir and slot must still exist after the trap ran. # Slot 0 is reserved for the base stack, so apply_isolation auto-picks slot 1. local slotdir="$XDG_STATE_HOME/copilotkit/showcase/slots/1" local rundir="$XDG_STATE_HOME/copilotkit/showcase/runs/trapkeep" [ -d "$rundir" ] || fail "kept run dir was torn down by the EXIT trap: $output" [ -d "$slotdir" ] || fail "kept slot was released by the EXIT trap: $output" [[ "$output" == *"Kept isolated group standing"* ]] \ || fail "survival notice not printed at exit: $output" # The keep path's central promise, under the REAL trap wiring: the EXIT trap # did NOT compose-down the kept stack. Match BOTH compose project-flag # spellings (--project-name and -p) — a keep-branch regression via the -p # form would pass a --project-name-only grep undetected. The log was # truncated inside the fixture after apply_isolation's pre-down, and the # sentinel proves the stub→log pipeline stayed live, so this absence check # is not vacuous. _assert_stub_sentinel_logged run grep -E -- "(--project-name|-p) trapkeep down" "$DOCKER_LOG" [ "$status" -ne 0 ] || fail "EXIT trap composed the kept stack down: $output" } # ── SHOWCASE_ISO_SLOT pinning ──────────────────────────────────────────────── # # When SHOWCASE_ISO_SLOT is exported the picker pins to that slot instead of # auto-selecting the first free one. Three behaviors are pinned here: # 1. A free pinned slot is claimed verbatim, even if lower-numbered slots are # occupied — the pin BYPASSES auto-selection, doesn't merely seed it. # 2. A pinned slot that is currently LIVE refuses to steal it — the die # message names the liveness state so the user can clear it explicitly. # 3. Invalid pin values (reserved slot 0, non-numeric, out-of-range) are # rejected with the appropriate validation message BEFORE any slot is # claimed or reaped. @test "SHOWCASE_ISO_SLOT=9 claims slot 9 even when 1-3 are pre-occupied" { # Pinning bypass: when slots 1, 2, 3 are all live-occupied (live owning PID), # the auto-picker would land on slot 4; with SHOWCASE_ISO_SLOT=9 set the # picker must skip every lower free slot AND every lower live slot and land # on 9 verbatim. Live PID alone (no project file, no docker containers) # makes _slot_liveness return "live" via signal #2 (owning-PID liveness), # so the sweep leaves the pre-seeded slots intact. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" local n for n in 1 2 3; do mkdir -p "$slots/$n" _seed_live_owner "$slots/$n" # this bats process — definitely alive (pid + pid.start) done export SHOWCASE_ISO_SLOT=9 _claim_isolate_slot [ "$ISOLATE_SLOT" = "9" ] \ || fail "pinned SHOWCASE_ISO_SLOT=9 did not land on slot 9: got $ISOLATE_SLOT" # Offset formula is (slot + 1) * 200 — slot 9 → 2000. [ "$ISOLATE_PORT_OFFSET" = "2000" ] \ || fail "pinned slot 9 port offset wrong: expected 2000, got $ISOLATE_PORT_OFFSET" # Slot 9 actually got claimed on disk and the pid file was written by the # common post-claim block. [ -d "$slots/9" ] || fail "slot 9 dir not created on pin" [ -f "$slots/9/pid" ] || fail "slot 9 pid file not written on pin" # Pre-occupied slots survive — the pin must not steal/reap them. for n in 1 2 3; do [ -d "$slots/$n" ] || fail "pre-occupied slot $n was reaped by the pin" done } @test "SHOWCASE_ISO_SLOT pinned to live slot dies with liveness message" { # Live-pin refusal: a pinned slot that is currently live must NOT be stolen. # Seed slot 9 with a live owning PID (this bats process) so _slot_liveness 9 # classifies it as "live" via signal #2 — no docker containers needed. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/9" _seed_live_owner "$slots/9" # this bats process — _slot_liveness → live export SHOWCASE_ISO_SLOT=9 run _claim_isolate_slot [ "$status" -ne 0 ] || fail "pin to LIVE slot 9 unexpectedly succeeded: $output" [[ "$output" == *"liveness=live"* ]] \ || fail "die message does not mention the live liveness state: $output" # The live slot survives — pin must not touch it. [ -d "$slots/9" ] || fail "live slot 9 was reaped by the refused pin" run cat "$slots/9/pid" [ "$status" -eq 0 ] || fail "live slot 9 pid file vanished after refused pin" [[ "$output" == "$$" ]] || fail "live slot 9 pid file mangled: $output" } @test "SHOWCASE_ISO_SLOT validates input (0, foo, 99 all rejected)" { # Three input-validation branches, one test: # - SHOWCASE_ISO_SLOT=0 → reserved (slot 0 is the base stack) # - SHOWCASE_ISO_SLOT=foo → non-numeric # - SHOWCASE_ISO_SLOT=99 → exceeds ISOLATE_MAX_SLOT (45) # All three must die with a message naming the specific failure mode, # BEFORE any slot is claimed (so the slots dir stays empty across all # three calls). load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" # NB: the regex check runs FIRST, so non-numeric values trip the "positive # integer" branch even if they would also be out-of-range. Order: numeric # regex → >=1 → <=ISOLATE_MAX_SLOT. # Reserved slot 0. SHOWCASE_ISO_SLOT=0 run _claim_isolate_slot [ "$status" -ne 0 ] || fail "SHOWCASE_ISO_SLOT=0 unexpectedly succeeded: $output" [[ "$output" == *"slot 0 is reserved"* ]] \ || fail "SHOWCASE_ISO_SLOT=0 die message does not say 'slot 0 is reserved': $output" # Non-numeric. SHOWCASE_ISO_SLOT=foo run _claim_isolate_slot [ "$status" -ne 0 ] || fail "SHOWCASE_ISO_SLOT=foo unexpectedly succeeded: $output" [[ "$output" == *"positive integer"* ]] \ || fail "SHOWCASE_ISO_SLOT=foo die message does not mention 'positive integer': $output" # Out of range (ISOLATE_MAX_SLOT=45, so 99 is over). SHOWCASE_ISO_SLOT=99 run _claim_isolate_slot [ "$status" -ne 0 ] || fail "SHOWCASE_ISO_SLOT=99 unexpectedly succeeded: $output" [[ "$output" == *"exceeds ISOLATE_MAX_SLOT=45"* ]] \ || fail "SHOWCASE_ISO_SLOT=99 die message does not mention 'exceeds ISOLATE_MAX_SLOT=45': $output" # None of the rejected calls claimed a slot — the slots dir contains no # numeric entries (the sweep may have created/removed .sweep.lock-shaped # entries, but no integer slot dirs). if [ -d "$slots" ]; then run bash -c "ls -A '$slots' | grep -E '^[0-9]+\$' || true" [ -z "$output" ] || fail "rejected pins claimed a slot anyway: $output" fi } # ── Port-probe and stale-reap behavior (MT3.2c) ────────────────────────────── # # Four additional behaviors pinned here: # #4. Pinned-path reap-and-retry: a stale slot at the pinned number is reaped # and the slot dir re-mkdir'd by the picker on the same call. # #5. Post-shift _slot_offset_ports regression: slot 8's dashboard port is # now 5010 (3210 + 1800), NOT 5000 (the pre-shift 3200 + 1800). The # regression test pins both halves: 5010 present, 5000 absent. # #6. Auto-pick port-probe skip: a slot whose dashboard port is held by a # FOREIGN process (not docker) is skipped — the picker rmdirs the # provisional slot dir and advances to the next slot. # #7. Own-project docker-listener filter: a live slot's project sees a # com.docker.* listener on one of its ports as the slot's OWN binding # (treated as free); a non-docker process name on the same port is NOT # filtered (treated as held). This is the port-probe path that lets a # --keep'd stack's own docker-proxy listener not be re-flagged as a # foreign hold. @test "SHOWCASE_ISO_SLOT pinned-path reaps stale slot and re-claims successfully" { # Stale-reap on pinned: pre-create slot 9 with a project file but NO pid # file — _slot_liveness classifies this as "stale" (has_proj=true, # pid_file_present=false → stale). The pinned branch should reap-and-retry # (rm -rf the slot entry, then mkdir it fresh) and write our pid into the # rebuilt slot via the common post-claim block. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/9" echo "stale-proj" > "$slots/9/project" # project recorded, no pid file → stale export DOCKER_PS_OUTPUT="" # no live containers for stale-proj export SHOWCASE_ISO_SLOT=9 _claim_isolate_slot [ "$ISOLATE_SLOT" = "9" ] \ || fail "pinned stale slot 9 did not re-claim: got $ISOLATE_SLOT" # The reap removed the original slot dir + project file; the common # post-claim block re-mkdir'd it and wrote our pid. Prove BOTH halves: [ -f "$slots/9/pid" ] || fail "post-reap pid file missing on slot 9" run cat "$slots/9/pid" [ "$status" -eq 0 ] || fail "post-reap pid file unreadable on slot 9" [[ "$output" == "$$" ]] \ || fail "post-reap pid file does not contain reaper's pid: got $output, expected $$" # The stale project file should be gone (reap rm -rf'd it; the claim does # NOT re-write a project file — only apply_isolation does). [ ! -f "$slots/9/project" ] \ || fail "stale project file survived the reap-and-retry: $(cat "$slots/9/project")" } @test "_slot_offset_ports 8 includes 5010 and excludes 5000 (port-shift regression)" { # Pre-shift the dashboard base was 3200, so slot 8 (offset 1800) bound 5000. # Post-shift the dashboard base is 3210, so slot 8 binds 5010. Pin BOTH halves # of the regression: 5010 must be present (the new port emitted by the offset # function), AND 5000 must be absent (a stale assumption that would let a # foreign 5000 binder collide with slot 8 invisibly). One assertion without # the other lets a half-revert (e.g. emitting BOTH 5000 and 5010) slip past. load_common run _slot_offset_ports 8 [ "$status" -eq 0 ] || fail "_slot_offset_ports 8 failed: $output" [[ "$output" == *"5010"* ]] \ || fail "_slot_offset_ports 8 did not emit dashboard port 5010 (post-shift): $output" ! [[ "$output" == *"5000"* ]] \ || fail "_slot_offset_ports 8 still emits pre-shift dashboard port 5000: $output" } @test "auto-pick skips slot 1 when its dashboard port is held by a foreign process" { # Auto-pick port-probe skip: with no SHOWCASE_ISO_SLOT set, the picker walks # 1..ISOLATE_MAX_SLOT. Slot 1's dashboard port (3210 + 400 = 3610) is held # by a non-docker process (Python — not subject to the own-project filter, # which only fires for docker/com.docker process names). The picker must # mkdir slot 1, see the port held, rmdir slot 1, and advance to slot 2. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" export LSOF_HOLD_PORTS="3610" export LSOF_HOLD_COMMAND="Python" _claim_isolate_slot [ "$ISOLATE_SLOT" = "2" ] \ || fail "auto-pick did not skip port-held slot 1: claimed $ISOLATE_SLOT" # Slot 1 was rmdir'd after the port probe failed; slot 2 was claimed. [ ! -d "$slots/1" ] \ || fail "port-held slot 1 dir survived the picker's rmdir" [ -d "$slots/2" ] || fail "slot 2 was not claimed" [ -f "$slots/2/pid" ] || fail "slot 2 pid file not written" } @test "_slot_ports_free treats own-project docker listeners as free, but not Python" { # Own-project filter (port-probe path): a docker / com.docker listener on a # port owned by a slot whose project is recorded and live is treated as the # slot's OWN binding (returns 0 = "all free"). The same port held by a # non-docker process name is NOT filtered (returns 1 = "held"). This is # what lets a --keep'd stack's own docker-proxy not be re-flagged as a # foreign hold. # # The brief originally proposed exercising this via the pinned-path's # EEXIST branch with a live slot, but that path DIES on a live pin before # reaching the port probe — so the own-project filter is not observable # from _claim_isolate_slot directly. We test _slot_ports_free in isolation, # which is the function that actually implements the filter. # # Pinning slot 8 as live: write the project file + a live pid ($$). Slot 8's # dashboard port post-shift is 3210 + 1800 = 5010. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/8" echo "iso8-proj" > "$slots/8/project" _seed_live_owner "$slots/8" # _slot_liveness 8 → live (pid live + pid.start, no containers needed) # Case A: docker listener on slot 8's dashboard port → filtered as own-project. export DOCKER_PS_OUTPUT="" # no docker containers; pid-liveness still wins export LSOF_HOLD_PORTS="5010" export LSOF_HOLD_COMMAND="com.docker.backend" run _slot_ports_free 8 [ "$status" -eq 0 ] \ || fail "_slot_ports_free 8 should treat com.docker.backend on own port as free, got status=$status output=$output" # Case B: same port held by Python (non-docker) → NOT filtered, returns 1. export LSOF_HOLD_COMMAND="Python" run _slot_ports_free 8 [ "$status" -ne 0 ] \ || fail "_slot_ports_free 8 should treat Python listener on own port as held (foreign), got status=0 output=$output" [[ "$output" == *"Slot 8 port 5010 held by Python"* ]] \ || fail "_slot_ports_free 8 did not emit the foreign-hold info line for Python: $output" } @test "_slot_ports_free fails LOUDLY for a bad slot instead of falsely reporting all-free" { # Regression: _slot_ports_free fed its port list from # while read port; do ...; done < <(_slot_offset_ports "$slot") # When _slot_offset_ports dies on a bad slot (out-of-range / non-numeric), the # die only kills the process-substitution SUBSHELL — not _slot_ports_free. The # while loop then read ZERO ports, any_held stayed 0, and the function returned # 0 ("all ports free"), SILENTLY defeating the port-conflict guard for a bad # slot. Both claim paths treat 0 as "free, go claim it", so a bad slot would # have sailed past the guard. The fix captures the port list (and thus # _slot_offset_ports's exit status) BEFORE the loop, so a bad slot fails LOUDLY. load_common export LSOF_HOLD_PORTS="" # Out-of-range slot (ISOLATE_MAX_SLOT=45). _slot_offset_ports dies on this. run _slot_ports_free 99 [ "$status" -ne 0 ] \ || fail "_slot_ports_free 99 (out-of-range) wrongly reported success; the bad slot defeated the guard. status=$status output=$output" [[ "$output" == *"exceeds ISOLATE_MAX_SLOT"* ]] \ || fail "_slot_ports_free 99 did not surface the _slot_offset_ports die reason: $output" # Non-numeric slot. _slot_offset_ports dies on this too. run _slot_ports_free "bogus" [ "$status" -ne 0 ] \ || fail "_slot_ports_free bogus (non-numeric) wrongly reported success. status=$status output=$output" [[ "$output" == *"non-negative integer"* ]] \ || fail "_slot_ports_free bogus did not surface the _slot_offset_ports die reason: $output" } @test "_slot_ports_free still reports a valid slot correctly (free when free, held when held)" { # Guard the fix: valid slots must NOT regress. Slot 8's dashboard port post- # shift is 3210 + 1800 = 5010 (same arithmetic as the own-project test above). load_common # Free: nothing held → returns 0. export LSOF_HOLD_PORTS="" run _slot_ports_free 8 [ "$status" -eq 0 ] \ || fail "_slot_ports_free 8 should be free when no port is held, got status=$status output=$output" # Held by a foreign (non-docker) process on slot 8's dashboard port → returns 1. export LSOF_HOLD_PORTS="5010" export LSOF_HOLD_COMMAND="Python" run _slot_ports_free 8 [ "$status" -ne 0 ] \ || fail "_slot_ports_free 8 should be held when port 5010 is taken by Python, got status=0 output=$output" [[ "$output" == *"Slot 8 port 5010 held by Python"* ]] \ || fail "_slot_ports_free 8 did not emit the foreign-hold info line: $output" } # ── Composite tests (MT3.2d): _slot_state axes, bin/showcase slots, concurrent claim ── # # Three additional behaviors pinned here: # #8. _slot_state emits one pipe-delimited line carrying ALL 7 axes (slot, # dir, pid, liveness, ports, offset, project) for a populated slot — the # wire format that `bin/showcase slots` parses. # #9. `bin/showcase slots` enumerates ALL 46 slots (0..ISOLATE_MAX_SLOT=45) # with a header row and the special "showcase (base)" project label for # slot 0 — the table contract the CLI exposes to operators. # #10. Two concurrent `_claim_isolate_slot` invocations land on DISTINCT # slots — atomic mkdir is the synchronization primitive; no two # claimants ever end up owning the same slot. @test "_slot_state 5 reports all axes for a live populated slot" { # The wire format: one pipe-delimited line of 7 fields, exactly as # cmd-slots.sh's `IFS='|' read -r slot dir pid liveness ports offset project` # consumes them. Seed slot 5 with a project record AND a live owning pid # (this bats process) so _slot_liveness 5 returns "live" via signal #2 — no # docker containers needed. Slot 5's offset is (5+1)*200 = 1200. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/5" echo "iso5-proj" > "$slots/5/project" _seed_live_owner "$slots/5" # this bats process — _slot_liveness → live (pid + pid.start) # No held ports → _slot_ports_free returns 0 → ports="free". export LSOF_HOLD_PORTS="" run _slot_state 5 [ "$status" -eq 0 ] || fail "_slot_state 5 failed: $output" # Exactly one output line. [ "${#lines[@]}" -eq 1 ] \ || fail "_slot_state emitted ${#lines[@]} lines, expected 1: $output" # Parse the 7 pipe-delimited axes. `read -ra fields` on a single line under # IFS='|' gives one field per axis — independent of bats's line-splitting. local -a fields IFS='|' read -ra fields <<< "$output" [ "${#fields[@]}" -eq 7 ] \ || fail "_slot_state emitted ${#fields[@]} fields, expected 7 (slot|dir|pid|liveness|ports|offset|project): $output" [ "${fields[0]}" = "5" ] || fail "axis[0] slot wrong: got '${fields[0]}', expected '5'" [ "${fields[1]}" = "present" ] || fail "axis[1] dir wrong: got '${fields[1]}', expected 'present'" [ "${fields[2]}" = "$$" ] || fail "axis[2] pid wrong: got '${fields[2]}', expected '$$'" [ "${fields[3]}" = "live" ] || fail "axis[3] liveness wrong: got '${fields[3]}', expected 'live'" [ "${fields[4]}" = "free" ] || fail "axis[4] ports wrong: got '${fields[4]}', expected 'free' (no LSOF_HOLD_PORTS)" [ "${fields[5]}" = "1200" ] || fail "axis[5] offset wrong: got '${fields[5]}', expected '1200' (=(5+1)*200)" [ "${fields[6]}" = "iso5-proj" ] || fail "axis[6] project wrong: got '${fields[6]}', expected 'iso5-proj'" } @test "bin/showcase slots prints all 46 slots with header and special slot 0 project label" { # End-to-end contract for `showcase slots`: with no slots pre-claimed, the # default fixed-width table emits one header row + 46 data rows (slots # 0..ISOLATE_MAX_SLOT=45), and slot 0's project column shows the special # "showcase (base)" label that cmd-slots.sh injects (the on-disk record is # never set for slot 0 — it's reserved for the base stack). # # XDG_STATE_HOME is already set to the per-test scratch dir by setup(), so # the slots dir starts empty and every row reports dir=absent. # The docker/lsof stubs in $STUB_DIR are on PATH and stay on PATH for the # child shell, so no real docker/lsof is hit; _slot_liveness short-circuits # to "inconclusive" for absent dirs (no docker call). # bin/showcase lives at the repo's showcase/bin/showcase, two dirs up from # this test file (scripts/__tests__/ → showcase/, then bin/showcase). run bash "$BATS_TEST_DIRNAME/../../bin/showcase" slots [ "$status" -eq 0 ] || fail "bin/showcase slots failed: $output" # ≥47 lines: 1 header + 46 data rows. [ "${#lines[@]}" -ge 47 ] \ || fail "bin/showcase slots emitted ${#lines[@]} lines, expected >=47 (1 header + 46 rows): $output" # Header: first non-blank line starts with "SLOT" (the fixed-width table's # header row, per cmd-slots.sh: `printf '%-4s ... "SLOT" "DIR" ...`). local first_nonblank="" local line for line in "${lines[@]}"; do if [ -n "${line// /}" ]; then first_nonblank="$line" break fi done [[ "$first_nonblank" == SLOT* ]] \ || fail "first non-blank line is not the SLOT header: '$first_nonblank'" # Slot 0's special project label "showcase (base)" must appear (cmd-slots.sh # injects it for slot 0 regardless of the on-disk record). [[ "$output" == *"showcase (base)"* ]] \ || fail "slot 0 project label 'showcase (base)' missing: $output" # Exactly 46 data rows: lines starting with a digit + whitespace (the # printf format opens with %-4s for the slot number, so data rows start # with a digit followed by spaces; the header line starts with "SLOT"). local data_rows=0 for line in "${lines[@]}"; do if [[ "$line" =~ ^[0-9]+[[:space:]] ]]; then data_rows=$((data_rows + 1)) fi done [ "$data_rows" -eq 46 ] \ || fail "expected 46 data rows (slots 0..45), got $data_rows: $output" } @test "concurrent _claim_isolate_slot calls claim distinct slots (no double-claim)" { # Atomic-mkdir lock under concurrency: two _claim_isolate_slot invocations # running simultaneously must NEVER both land on the same slot. Bats runs # in a single process so we spawn the two claims as background subshells # (`&`) and wait for both — the atomic-mkdir lock in _claim_isolate_slot # is the synchronization primitive. Slots 1-8 are pre-occupied with live # owning PIDs so the auto-picker is forced to evaluate slots 9 and 10. load_common local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" local n for n in 1 2 3 4 5 6 7 8; do mkdir -p "$slots/$n" _seed_live_owner "$slots/$n" # this bats process — definitely alive (pid + pid.start) done # No held ports for any slot's offset range → _slot_ports_free returns 0 # for whichever slot the picker probes. export LSOF_HOLD_PORTS="" export DOCKER_PS_OUTPUT="" # Spawn two subshell claims concurrently. Each subshell writes its result # (the claimed ISOLATE_SLOT and its own subshell pid) to a file the parent # can read after wait(); ISOLATE_SLOT itself is a variable inside each # subshell and doesn't propagate back to the parent. The filesystem state # under $slots/ is the real source of truth for "did they double-claim". local out1="$BATS_TEST_TMPDIR/claim1.out" local out2="$BATS_TEST_TMPDIR/claim2.out" ( _claim_isolate_slot printf 'slot=%s pid=%s\n' "$ISOLATE_SLOT" "$BASHPID" > "$out1" ) & local p1=$! # Tiny stagger so both subshells exist before the second's mkdir loop — # without this the second can finish before the first ever enters the loop, # serializing them (still correct, but doesn't exercise the concurrent path). sleep 0.05 ( _claim_isolate_slot printf 'slot=%s pid=%s\n' "$ISOLATE_SLOT" "$BASHPID" > "$out2" ) & local p2=$! wait "$p1" || fail "subshell 1 (_claim_isolate_slot) failed" wait "$p2" || fail "subshell 2 (_claim_isolate_slot) failed" # Both wrote their result files. [ -f "$out1" ] || fail "subshell 1 did not produce $out1" [ -f "$out2" ] || fail "subshell 2 did not produce $out2" # Filesystem proof: BOTH slots 9 AND 10 exist with pid files. The picker # walks 1..ISOLATE_MAX_SLOT, skips the live-occupied 1-8 (each carries # this bats process's pid → liveness=live), and the two concurrent claims # land on the next two free slots. [ -d "$slots/9" ] || fail "slot 9 was not claimed (no concurrent landing): $(ls -1 "$slots")" [ -d "$slots/10" ] || fail "slot 10 was not claimed (no concurrent landing): $(ls -1 "$slots")" [ -f "$slots/9/pid" ] || fail "slot 9 has no pid file (claim did not write owner): $(ls -1 "$slots/9")" [ -f "$slots/10/pid" ] || fail "slot 10 has no pid file (claim did not write owner): $(ls -1 "$slots/10")" # No double-claim: the two subshells reported DIFFERENT slot numbers in # their own ISOLATE_SLOT — proving they followed disjoint paths through the # auto-pick loop. (The pid files themselves carry $$ — the parent bats PID, # which a subshell inherits unchanged — so identical pid file contents is # NOT evidence of double-claim; the slot-number divergence is.) local result1 result2 result1="$(cat "$out1" 2>/dev/null || true)" result2="$(cat "$out2" 2>/dev/null || true)" local slot1 slot2 slot1="$(printf '%s' "$result1" | sed -E 's/^slot=([0-9]+).*/\1/')" slot2="$(printf '%s' "$result2" | sed -E 's/^slot=([0-9]+).*/\1/')" [ -n "$slot1" ] || fail "subshell 1 did not record a slot: result1='$result1'" [ -n "$slot2" ] || fail "subshell 2 did not record a slot: result2='$result2'" [ "$slot1" != "$slot2" ] \ || fail "subshell 1 and subshell 2 both claimed slot $slot1 (double-claim): result1='$result1' result2='$result2'" # Slot-numbers must each be 9 OR 10 (the two slots the auto-picker reaches # after skipping the live-occupied 1-8). case "$slot1" in 9|10) ;; *) fail "subshell 1 claimed unexpected slot $slot1 (expected 9 or 10): $result1" ;; esac case "$slot2" in 9|10) ;; *) fail "subshell 2 claimed unexpected slot $slot2 (expected 9 or 10): $result2" ;; esac } @test "_slot_state emits ports=? when lsof is unavailable" { # Passive-inspector forgiveness: `bin/showcase slots` calls _slot_state for # every present slot, and _slot_state's port-probe goes through # _slot_ports_free — which dies hard when lsof is missing. That die is the # right contract for the picker (--isolate genuinely needs lsof), but it # makes the read-only `slots` table unusable on lsof-less hosts. Fix: # _slot_state short-circuits to ports="?" when lsof is unavailable, leaving # _slot_ports_free's die intact for the picker path. load_common # Seed slot 0 as present so _slot_state takes the probe branch (dir=absent # would leave ports="-" and bypass the lsof check entirely). local slots="$XDG_STATE_HOME/copilotkit/showcase/slots" mkdir -p "$slots/0" # Hide the stub lsof by pointing PATH at an empty dir — done AFTER # load_common so the harness's stub wiring is already in place and we are # specifically simulating "lsof not installed". local empty_bin="$BATS_TEST_TMPDIR/no-lsof-bin" mkdir -p "$empty_bin" local saved_path="$PATH" PATH="$empty_bin" # _slot_state must not die; the ports field (axis #5, between the 4th and # 5th pipes) must be "?". local state state=$(_slot_state 0) local rc=$? PATH="$saved_path" [ "$rc" -eq 0 ] || fail "_slot_state 0 failed (rc=$rc) with lsof unavailable: $state" [ -n "$state" ] || fail "_slot_state 0 emitted no output with lsof unavailable" [[ "$state" == *"|?|"* ]] \ || fail "_slot_state did not degrade to ports='?' when lsof was unavailable: $state" } # ── --isolate= arg-parser sugar (cmd-test.sh) ───────────────────────────── # # The `--isolate=` form is sugar over `SHOWCASE_ISO_SLOT= ... --isolate`: # the arg parser in cmd-test.sh sets SHOWCASE_ISO_SLOT from the = suffix and # exports it, then the existing picker (_claim_isolate_slot) handles ALL # validation. The tests here pin the parser→env→picker pipeline end-to-end by # replaying cmd-test.sh's --isolate=* branch verbatim and then invoking the # picker (the SAME wiring bin/showcase test would do). No new validation logic # exists in the arg parser, so we don't re-test validation here — we prove the # arg form drives the existing validation paths pinned above. # # We replay the parser branch rather than invoking `cmd_test` directly because # cmd_test runs heavy work (need_slug, trap restore_isolation, apply_isolation # which forks docker/python3) after parsing — orthogonal to what this test # pins. @test "--isolate= arg form exports SHOWCASE_ISO_SLOT and pins the slot through the picker" { load_common # Replay cmd-test.sh's `--isolate=*` branch verbatim, then run the picker. set -- --isolate=9 dummy-slug while [[ $# -gt 0 ]]; do case "$1" in --isolate=*) SHOWCASE_ISO_SLOT="${1#--isolate=}" export SHOWCASE_ISO_SLOT shift ;; *) shift ;; esac done [ "$SHOWCASE_ISO_SLOT" = "9" ] \ || fail "--isolate=9 did not export SHOWCASE_ISO_SLOT=9; got: $SHOWCASE_ISO_SLOT" # The exported pin drives the picker exactly as the SHOWCASE_ISO_SLOT=9 # pinned-path test does (pinned by "SHOWCASE_ISO_SLOT=9 claims slot 9 ..." # above): slot 9 claimed verbatim. _claim_isolate_slot [ "$ISOLATE_SLOT" = "9" ] \ || fail "picker did not honor --isolate=9-exported SHOWCASE_ISO_SLOT: got $ISOLATE_SLOT" } @test "--isolate=0 arg form drives the picker's reserved-slot rejection" { # Wiring proof: the arg parser does NO validation of N itself — it relies on # the picker. So --isolate=0 must trip the SAME 'slot 0 is reserved' die # pinned by "SHOWCASE_ISO_SLOT validates input (0, foo, 99 all rejected)" # above. Replay the parser branch, then invoke the picker. load_common set -- --isolate=0 while [[ $# -gt 0 ]]; do case "$1" in --isolate=*) SHOWCASE_ISO_SLOT="${1#--isolate=}" export SHOWCASE_ISO_SLOT shift ;; *) shift ;; esac done [ "$SHOWCASE_ISO_SLOT" = "0" ] || fail "parser did not export 0: got $SHOWCASE_ISO_SLOT" run _claim_isolate_slot [ "$status" -ne 0 ] || fail "--isolate=0 unexpectedly succeeded: $output" [[ "$output" == *"slot 0 is reserved"* ]] \ || fail "--isolate=0 did not surface the reserved-slot die: $output" } @test "cmd-test.sh --isolate= actually wires the arg through to SHOWCASE_ISO_SLOT" { # Drift guard: the three tests above replay the parser branch verbatim. This # test sources the REAL cmd-test.sh and runs cmd_test end-to-end with a stub # apply_isolation that snapshots SHOWCASE_ISO_SLOT — so if the parser branch # is ever removed or wired differently, this fails LOUDLY. load_common local snapshot="$BATS_TEST_TMPDIR/iso-slot-snapshot" local cmd_test_sh="$BATS_TEST_DIRNAME/../cli/cmd-test.sh" [ -f "$cmd_test_sh" ] || fail "cmd-test.sh not found: $cmd_test_sh" # Stub apply_isolation: snapshot SHOWCASE_ISO_SLOT, then exit cleanly via # `die` so the rest of cmd_test (which forks docker/etc) never runs. # Stub need_slug to a no-op so the parser path is what's exercised. # Stub everything else cmd_test invokes after the parser to no-ops. run bash -euo pipefail -c " source '$COMMON' source '$cmd_test_sh' apply_isolation() { printf '%s\n' \"\${SHOWCASE_ISO_SLOT:-}\" > '$snapshot'; exit 0; } need_slug() { :; } info() { :; } run_harness() { :; } cmd_test --isolate=7 dummy-slug " [ "$status" -eq 0 ] || fail "cmd_test --isolate=7 failed: $output" [ -f "$snapshot" ] || fail "stubbed apply_isolation never fired (parser/use_isolate wiring broken)" run cat "$snapshot" [[ "$output" == "7" ]] \ || fail "cmd-test.sh did not export SHOWCASE_ISO_SLOT=7 from --isolate=7: got '$output'" } @test "--isolate=99 arg form drives the picker's out-of-range rejection" { # Same wiring proof for the upper bound: ISOLATE_MAX_SLOT=45, so 99 is over. load_common set -- --isolate=99 while [[ $# -gt 0 ]]; do case "$1" in --isolate=*) SHOWCASE_ISO_SLOT="${1#--isolate=}" export SHOWCASE_ISO_SLOT shift ;; *) shift ;; esac done [ "$SHOWCASE_ISO_SLOT" = "99" ] || fail "parser did not export 99: got $SHOWCASE_ISO_SLOT" run _claim_isolate_slot [ "$status" -ne 0 ] || fail "--isolate=99 unexpectedly succeeded: $output" [[ "$output" == *"exceeds ISOLATE_MAX_SLOT=45"* ]] \ || fail "--isolate=99 did not surface the out-of-range die: $output" } # ── --isolate space-form arg-parser (cmd-test.sh) ───────────────────── # # The space-separated `--isolate ` form (vs the `--isolate=` sugar) # is parsed entirely inside cmd-test.sh's option loop, which must bind the # explicit name to ISOLATE's name slot and the service to the positional slug # REGARDLESS of token order — `--isolate ` (name first) and # ` --isolate ` (slug first) must both resolve correctly. The # auto-named form (`--isolate ` with NO explicit name) must still treat # the lone trailing token as the slug. And a bare/empty `--isolate=` (or # `--isolate` with no value) must be REJECTED loudly, never silently auto-pick. # # These tests drive the REAL cmd_test parser end-to-end (the same source + # stub-after-parser pattern as the --isolate= drift guard above): they stub # everything cmd_test invokes after parsing (apply_isolation, run-harness, etc.) # and snapshot the parsed slug + isolate_name so a parser regression fails # LOUDLY. apply_isolation receives ("$isolate_name" "$slug") in that order # (see _common.sh), so the snapshot records the two positional args it was # handed — exactly the contract downstream depends on. # _run_cmd_test_parse — source the real cmd-test.sh, stub the entire # post-parser body, and snapshot what the parser bound. Writes two files: # $PARSE_SNAP.name → the isolate name apply_isolation was handed ($1) # $PARSE_SNAP.slug → the slug apply_isolation was handed ($2) # When --isolate is NOT requested, apply_isolation never fires; the no-isolate # fallback stub records need_slug's argument as the slug and "" name. # Sets $status/$output via `run`. _run_cmd_test_parse() { local cmd_test_sh="$BATS_TEST_DIRNAME/../cli/cmd-test.sh" [ -f "$cmd_test_sh" ] || fail "cmd-test.sh not found: $cmd_test_sh" PARSE_SNAP="$BATS_TEST_TMPDIR/parse-snap" rm -f "$PARSE_SNAP".name "$PARSE_SNAP".slug # apply_isolation snapshots (name, slug) then exits cleanly so the docker/ # python3-forking remainder of cmd_test never runs. need_slug snapshots the # slug too (and preserves its real "slug required" die so the empty-slug # contract is exercised). info/success/warn are silenced; the harness call is # a no-op (it is never reached — apply_isolation exits first when isolating). run bash -euo pipefail -c " source '$COMMON' source '$cmd_test_sh' apply_isolation() { printf '%s' \"\${1:-}\" > '$PARSE_SNAP.name' printf '%s' \"\${2:-}\" > '$PARSE_SNAP.slug' exit 0 } need_slug() { [ -n \"\${1:-}\" ] || die 'slug required' printf '' > '$PARSE_SNAP.name' printf '%s' \"\${1:-}\" > '$PARSE_SNAP.slug' } info() { :; } success() { :; } npx() { :; } cmd_test $* " } @test "cmd-test.sh --isolate (name BEFORE slug) binds name and slug correctly" { load_common _run_cmd_test_parse --isolate myname mastra [ "$status" -eq 0 ] || fail "cmd_test --isolate myname mastra failed: $output" [ -f "$PARSE_SNAP.name" ] || fail "apply_isolation never fired (use_isolate wiring broken): $output" run cat "$PARSE_SNAP.name" [[ "$output" == "myname" ]] || fail "name-before-slug bound wrong isolate name: got '$output' (want myname)" run cat "$PARSE_SNAP.slug" [[ "$output" == "mastra" ]] || fail "name-before-slug bound wrong slug: got '$output' (want mastra)" } @test "cmd-test.sh --isolate (slug BEFORE name) still binds name and slug correctly" { load_common _run_cmd_test_parse mastra --isolate myname [ "$status" -eq 0 ] || fail "cmd_test mastra --isolate myname failed: $output" run cat "$PARSE_SNAP.name" [[ "$output" == "myname" ]] || fail "slug-before-name bound wrong isolate name: got '$output' (want myname)" run cat "$PARSE_SNAP.slug" [[ "$output" == "mastra" ]] || fail "slug-before-name bound wrong slug: got '$output' (want mastra)" } @test "cmd-test.sh --isolate (no explicit name) treats the lone token as the slug (auto-named)" { load_common _run_cmd_test_parse --isolate mastra [ "$status" -eq 0 ] || fail "cmd_test --isolate mastra failed: $output" run cat "$PARSE_SNAP.name" [[ "$output" == "" ]] || fail "auto-named --isolate wrongly bound an explicit name: got '$output' (want empty)" run cat "$PARSE_SNAP.slug" [[ "$output" == "mastra" ]] || fail "auto-named --isolate bound wrong slug: got '$output' (want mastra)" } @test "cmd-test.sh --isolate with surrounding flags binds name and slug correctly" { load_common _run_cmd_test_parse --d5 --isolate d5verify agno --verbose [ "$status" -eq 0 ] || fail "cmd_test with flags around --isolate failed: $output" run cat "$PARSE_SNAP.name" [[ "$output" == "d5verify" ]] || fail "flagged name-before-slug bound wrong name: got '$output' (want d5verify)" run cat "$PARSE_SNAP.slug" [[ "$output" == "agno" ]] || fail "flagged name-before-slug bound wrong slug: got '$output' (want agno)" } @test "cmd-test.sh bare --isolate= (empty value) is rejected loudly, never silently auto-picks" { load_common _run_cmd_test_parse --isolate= mastra [ "$status" -ne 0 ] || fail "bare --isolate= unexpectedly succeeded (silent auto-pick): $output" # It must NOT have reached apply_isolation with an empty pin and proceeded. [ ! -f "$PARSE_SNAP.slug" ] || fail "bare --isolate= fell through to apply_isolation instead of erroring: name='$(cat "$PARSE_SNAP.name" 2>/dev/null)' slug='$(cat "$PARSE_SNAP.slug" 2>/dev/null)'" [[ "$output" == *"isolate"* ]] || fail "bare --isolate= error did not mention isolate: $output" } @test "cmd-test.sh --isolate= (non-numeric sugar) binds the explicit name, not a slot" { load_common # The = sugar is equivalent to the space form `--isolate `: a # non-numeric value is an explicit isolate name, NOT a slot pin (a slot pin # would die in the picker on "must be a positive integer"). SHOWCASE_ISO_SLOT # must remain unset so the picker auto-picks a slot for the named project. local snap="$BATS_TEST_TMPDIR/iso-namesugar-snap" local cmd_test_sh="$BATS_TEST_DIRNAME/../cli/cmd-test.sh" run bash -euo pipefail -c " source '$COMMON' source '$cmd_test_sh' apply_isolation() { printf '%s' \"\${SHOWCASE_ISO_SLOT:-}\" > '$snap.slot' printf '%s' \"\${1:-}\" > '$snap.name' printf '%s' \"\${2:-}\" > '$snap.slug' exit 0 } need_slug() { :; } info() { :; } success() { :; } npx() { :; } cmd_test --isolate=d5verify agno " [ "$status" -eq 0 ] || fail "cmd_test --isolate=d5verify agno failed: $output" run cat "$snap.slot" [[ "$output" == "" ]] || fail "--isolate=d5verify wrongly pinned a slot: SHOWCASE_ISO_SLOT='$output'" run cat "$snap.name" [[ "$output" == "d5verify" ]] || fail "--isolate=d5verify bound wrong name: got '$output'" run cat "$snap.slug" [[ "$output" == "agno" ]] || fail "--isolate=d5verify bound wrong slug: got '$output'" } @test "cmd-test.sh --isolate= numeric pinned path is unchanged (still exports the slot)" { load_common # Regression guard for the existing numeric sugar: --isolate=7 must still # export SHOWCASE_ISO_SLOT=7 and reach apply_isolation with an EMPTY name and # the real slug (the picker, not the parser, owns numeric validation). local snap="$BATS_TEST_TMPDIR/iso-num-snap" local cmd_test_sh="$BATS_TEST_DIRNAME/../cli/cmd-test.sh" run bash -euo pipefail -c " source '$COMMON' source '$cmd_test_sh' apply_isolation() { printf '%s' \"\${SHOWCASE_ISO_SLOT:-}\" > '$snap.slot' printf '%s' \"\${1:-}\" > '$snap.name' printf '%s' \"\${2:-}\" > '$snap.slug' exit 0 } need_slug() { :; } info() { :; } success() { :; } npx() { :; } cmd_test --isolate=7 mastra " [ "$status" -eq 0 ] || fail "cmd_test --isolate=7 mastra failed: $output" run cat "$snap.slot" [[ "$output" == "7" ]] || fail "--isolate=7 did not export SHOWCASE_ISO_SLOT=7: got '$output'" run cat "$snap.name" [[ "$output" == "" ]] || fail "--isolate=7 wrongly bound a name: got '$output'" run cat "$snap.slug" [[ "$output" == "mastra" ]] || fail "--isolate=7 bound wrong slug: got '$output'" }