chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,156 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# CPU device test: runs both server bench and vLLM e2e tests
|
||||
# in the same environment to avoid repeated vLLM installation.
|
||||
#
|
||||
# Usage: cpu_device_test.sh [mode]
|
||||
# mode: server_bench, vllm_e2e, or all (default)
|
||||
#
|
||||
# Environment variables:
|
||||
# LMCACHE_BENCH_TRANSFER_MODE engine_driven|lmcache_driven
|
||||
# (default: engine_driven)
|
||||
# LMCACHE_E2E_TRANSPORT_MODE engine_driven|lmcache_driven|shm|pickle
|
||||
# (default: engine_driven)
|
||||
# LMCACHE_E2E_DATA_MODE shm|pickle (default: shm)
|
||||
# LMCACHE_HTTP_PORT_BENCH HTTP port for bench (default: 18080)
|
||||
# LMCACHE_ZMQ_PORT_BENCH ZMQ port for bench (default: 15555)
|
||||
# LMCACHE_HTTP_PORT_E2E HTTP port for e2e (default: 18081)
|
||||
# LMCACHE_ZMQ_PORT_E2E ZMQ port for e2e (default: 15557)
|
||||
# VLLM_PORT_E2E HTTP port for vLLM (default: 18000)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
OS="$(uname -s)"
|
||||
TEST_MODE="${1:-all}"
|
||||
|
||||
echo "==> CPU device test (OS: ${OS}, Mode: ${TEST_MODE})"
|
||||
echo " Python: $(python3 --version 2>&1 || true)"
|
||||
|
||||
# Configuration
|
||||
BENCH_TRANSFER_MODE="${LMCACHE_BENCH_TRANSFER_MODE:-engine_driven}"
|
||||
E2E_TRANSPORT_MODE="${LMCACHE_E2E_TRANSPORT_MODE:-engine_driven}"
|
||||
E2E_DATA_MODE="${LMCACHE_E2E_DATA_MODE:-shm}"
|
||||
HTTP_PORT_BENCH="${LMCACHE_HTTP_PORT_BENCH:-18080}"
|
||||
ZMQ_PORT_BENCH="${LMCACHE_ZMQ_PORT_BENCH:-15555}"
|
||||
HTTP_PORT_E2E="${LMCACHE_HTTP_PORT_E2E:-18081}"
|
||||
ZMQ_PORT_E2E="${LMCACHE_ZMQ_PORT_E2E:-15557}"
|
||||
VLLM_PORT_E2E="${VLLM_PORT_E2E:-18000}"
|
||||
|
||||
# Validate modes
|
||||
case "${BENCH_TRANSFER_MODE}" in
|
||||
lmcache_driven|engine_driven) ;;
|
||||
*)
|
||||
echo "!! Unknown LMCACHE_BENCH_TRANSFER_MODE='${BENCH_TRANSFER_MODE}'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Map user-facing LMCACHE_E2E_TRANSPORT_MODE to internal representation.
|
||||
# shm/pickle are aliases for lmcache_driven mode with sub-mode selection.
|
||||
case "${E2E_TRANSPORT_MODE}" in
|
||||
lmcache_driven|engine_driven)
|
||||
MAPPED_TRANSPORT_MODE="${E2E_TRANSPORT_MODE}"
|
||||
MAPPED_DATA_MODE="${E2E_DATA_MODE}"
|
||||
;;
|
||||
shm)
|
||||
MAPPED_TRANSPORT_MODE="engine_driven"
|
||||
MAPPED_DATA_MODE="shm"
|
||||
;;
|
||||
pickle)
|
||||
MAPPED_TRANSPORT_MODE="engine_driven"
|
||||
MAPPED_DATA_MODE="pickle"
|
||||
;;
|
||||
*)
|
||||
echo "!! Unknown LMCACHE_E2E_TRANSPORT_MODE='${E2E_TRANSPORT_MODE}'"
|
||||
echo " Valid values: engine_driven, lmcache_driven, shm, pickle"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo " Bench transfer mode: ${BENCH_TRANSFER_MODE}"
|
||||
echo " E2E transport mode: ${E2E_TRANSPORT_MODE}"
|
||||
echo " Ports: bench=${HTTP_PORT_BENCH}/${ZMQ_PORT_BENCH}, e2e=${HTTP_PORT_E2E}/${ZMQ_PORT_E2E}/${VLLM_PORT_E2E}"
|
||||
|
||||
# Reap any LMCache/vLLM children started by this run on exit so the
|
||||
# next workflow step does not collide on default ZMQ/HTTP ports.
|
||||
cleanup_processes_safe() {
|
||||
local rc=$?
|
||||
set +e
|
||||
# Kill child processes started by this shell first (e.g. lmcache server
|
||||
# backgrounded by the shared validation script).
|
||||
pkill -P $$ 2>/dev/null || true
|
||||
sleep 1
|
||||
pkill -9 -P $$ 2>/dev/null || true
|
||||
return $rc
|
||||
}
|
||||
trap cleanup_processes_safe EXIT
|
||||
|
||||
# Function to run server bench test
|
||||
run_server_bench() {
|
||||
echo ""
|
||||
echo "==> Running CPU server bench test"
|
||||
|
||||
# Set environment for bench test
|
||||
export LMCACHE_BENCH_TRANSFER_MODE="${BENCH_TRANSFER_MODE}"
|
||||
export LMCACHE_HTTP_PORT="${HTTP_PORT_BENCH}"
|
||||
export LMCACHE_ZMQ_PORT="${ZMQ_PORT_BENCH}"
|
||||
export LMCACHE_LOG_FILE="/tmp/cpu_device_bench_${BENCH_TRANSFER_MODE}_lmcache.log"
|
||||
export BENCH_OUTPUT_LOG="/tmp/cpu_device_bench_${BENCH_TRANSFER_MODE}_output.log"
|
||||
export LMCACHE_HEALTHCHECK_TIMEOUT="30"
|
||||
export BENCH_NUM_REQUESTS="3"
|
||||
export BENCH_NUM_TOKENS="512"
|
||||
|
||||
# Run bench test
|
||||
bash "${SCRIPT_DIR}/cpu_server_bench_test.sh"
|
||||
|
||||
echo "==> CPU server bench test completed successfully"
|
||||
}
|
||||
|
||||
# Function to run vLLM e2e test
|
||||
run_vllm_e2e() {
|
||||
echo ""
|
||||
echo "==> Running CPU vLLM e2e test"
|
||||
|
||||
# Set environment for e2e test
|
||||
export LMCACHE_TRANSPORT_MODE="${MAPPED_TRANSPORT_MODE}"
|
||||
export LMCACHE_DATA_MODE="${MAPPED_DATA_MODE}"
|
||||
export LMCACHE_HTTP_PORT="${HTTP_PORT_E2E}"
|
||||
export LMCACHE_ZMQ_PORT="${ZMQ_PORT_E2E}"
|
||||
export VLLM_PORT="${VLLM_PORT_E2E}"
|
||||
export LMCACHE_LOG_FILE="/tmp/cpu_device_e2e_${E2E_TRANSPORT_MODE}_lmcache.log"
|
||||
export VLLM_LOG_FILE="/tmp/cpu_device_e2e_${E2E_TRANSPORT_MODE}_vllm.log"
|
||||
export LMCACHE_HEALTHCHECK_TIMEOUT="30"
|
||||
export VLLM_READY_TIMEOUT="300"
|
||||
|
||||
# Run e2e test
|
||||
bash "${SCRIPT_DIR}/cpu_vllm_e2e_test.sh"
|
||||
|
||||
echo "==> CPU vLLM e2e test completed successfully"
|
||||
}
|
||||
|
||||
# Determine which tests to run
|
||||
case "${TEST_MODE}" in
|
||||
"server_bench")
|
||||
run_server_bench
|
||||
;;
|
||||
"vllm_e2e")
|
||||
run_vllm_e2e
|
||||
;;
|
||||
"all")
|
||||
run_server_bench
|
||||
run_vllm_e2e
|
||||
;;
|
||||
*)
|
||||
echo "!! Unknown test mode: ${TEST_MODE}"
|
||||
echo " Supported modes: server_bench, vllm_e2e, all"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "==> CPU device test passed for modes:"
|
||||
echo " Test mode: ${TEST_MODE}"
|
||||
echo " Server bench: ${BENCH_TRANSFER_MODE}"
|
||||
echo " vLLM e2e: ${E2E_TRANSPORT_MODE}"
|
||||
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# CPU server bench test: starts lmcache server, runs
|
||||
# ``lmcache bench server --mode cpu`` with the requested transfer mode,
|
||||
# then tears down.
|
||||
#
|
||||
# Transfer modes (LMCACHE_BENCH_TRANSFER_MODE):
|
||||
# lmcache_driven - worker-side gather/scatter via POSIX SHM pool
|
||||
# engine_driven - server-side copy via POSIX SHM IPC (shm_open/mmap)
|
||||
#
|
||||
# Environment variables (all optional, defaults shown):
|
||||
# LMCACHE_BENCH_TRANSFER_MODE lmcache_driven|engine_driven
|
||||
# (default: engine_driven)
|
||||
# LMCACHE_HTTP_PORT HTTP port (default: 18080)
|
||||
# LMCACHE_ZMQ_PORT ZMQ RPC port (default: 15555)
|
||||
# LMCACHE_LOG_FILE server log path (default: /tmp/...)
|
||||
# LMCACHE_HEALTHCHECK_TIMEOUT seconds (default: 60)
|
||||
# BENCH_NUM_REQUESTS requests to run (default: 3)
|
||||
# BENCH_NUM_TOKENS tokens per request (default: 512)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
OS="$(uname -s)"
|
||||
echo "==> CPU server bench test (OS: ${OS})"
|
||||
echo " Python: $(python3 --version 2>&1 || true)"
|
||||
|
||||
TRANSFER_MODE="${LMCACHE_BENCH_TRANSFER_MODE:-engine_driven}"
|
||||
HTTP_PORT="${LMCACHE_HTTP_PORT:-18080}"
|
||||
ZMQ_PORT="${LMCACHE_ZMQ_PORT:-15555}"
|
||||
LOG_FILE="${LMCACHE_LOG_FILE:-/tmp/cpu_server_bench_lmcache.log}"
|
||||
HEALTHCHECK_TIMEOUT="${LMCACHE_HEALTHCHECK_TIMEOUT:-60}"
|
||||
BENCH_NUM_REQUESTS="${BENCH_NUM_REQUESTS:-3}"
|
||||
BENCH_NUM_TOKENS="${BENCH_NUM_TOKENS:-512}"
|
||||
|
||||
case "${TRANSFER_MODE}" in
|
||||
lmcache_driven|engine_driven) ;;
|
||||
*)
|
||||
echo "!! Unknown LMCACHE_BENCH_TRANSFER_MODE='${TRANSFER_MODE}'"
|
||||
echo " Valid values: lmcache_driven, engine_driven"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo " TRANSFER_MODE=${TRANSFER_MODE}"
|
||||
echo " HTTP_PORT=${HTTP_PORT} ZMQ_PORT=${ZMQ_PORT}"
|
||||
echo " BENCH_NUM_REQUESTS=${BENCH_NUM_REQUESTS}"
|
||||
echo " BENCH_NUM_TOKENS=${BENCH_NUM_TOKENS}"
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Start lmcache server
|
||||
# ------------------------------------------------------------------ #
|
||||
echo ""
|
||||
echo "==> Starting lmcache server (log: ${LOG_FILE})"
|
||||
rm -f "${LOG_FILE}"
|
||||
|
||||
lmcache server \
|
||||
--port "${ZMQ_PORT}" \
|
||||
--http-port "${HTTP_PORT}" \
|
||||
--l1-size-gb 1 \
|
||||
--eviction-policy LRU \
|
||||
>"${LOG_FILE}" 2>&1 &
|
||||
SERVER_PID=$!
|
||||
|
||||
cleanup() {
|
||||
echo "==> Cleanup: stopping lmcache server (pid=${SERVER_PID})"
|
||||
if kill -0 "${SERVER_PID}" 2>/dev/null; then
|
||||
kill "${SERVER_PID}" 2>/dev/null || true
|
||||
for _ in $(seq 1 10); do
|
||||
kill -0 "${SERVER_PID}" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
kill -9 "${SERVER_PID}" 2>/dev/null || true
|
||||
fi
|
||||
echo "==> Last 50 lines of server log:"
|
||||
tail -n 50 "${LOG_FILE}" 2>/dev/null || true
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Wait for healthcheck
|
||||
# ------------------------------------------------------------------ #
|
||||
echo "==> Waiting for healthcheck (timeout: ${HEALTHCHECK_TIMEOUT}s)"
|
||||
READY=0
|
||||
for i in $(seq 1 "${HEALTHCHECK_TIMEOUT}"); do
|
||||
if ! kill -0 "${SERVER_PID}" 2>/dev/null; then
|
||||
echo "!! lmcache server exited prematurely after ${i}s"
|
||||
break
|
||||
fi
|
||||
if curl -fsS --max-time 2 \
|
||||
"http://127.0.0.1:${HTTP_PORT}/healthcheck" >/dev/null 2>&1; then
|
||||
READY=1
|
||||
echo " Server healthy after ${i}s"
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ "${READY}" != "1" ]; then
|
||||
echo "!! lmcache server did not become healthy within ${HEALTHCHECK_TIMEOUT}s"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Run bench and validate results
|
||||
# ------------------------------------------------------------------ #
|
||||
echo ""
|
||||
echo "==> Running: lmcache bench server" \
|
||||
"--mode cpu --transfer-mode ${TRANSFER_MODE}" \
|
||||
"--num-tokens ${BENCH_NUM_TOKENS}" \
|
||||
"--end ${BENCH_NUM_REQUESTS}"
|
||||
|
||||
BENCH_LOG="${BENCH_OUTPUT_LOG:-/tmp/cpu_server_bench_output.log}"
|
||||
lmcache bench server \
|
||||
--rpc-url "tcp://127.0.0.1:${ZMQ_PORT}" \
|
||||
--url "http://127.0.0.1:${HTTP_PORT}" \
|
||||
--mode cpu \
|
||||
--transfer-mode "${TRANSFER_MODE}" \
|
||||
--num-tokens "${BENCH_NUM_TOKENS}" \
|
||||
--end "${BENCH_NUM_REQUESTS}" \
|
||||
2>&1 | tee "${BENCH_LOG}"
|
||||
|
||||
echo ""
|
||||
echo "==> Validating bench results"
|
||||
|
||||
if grep -q "CHECKSUM MISMATCH" "${BENCH_LOG}"; then
|
||||
echo "!! CHECKSUM MISMATCH detected — store/retrieve data corruption"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
MATCH_COUNT="$(grep -c "CHECKSUM MATCH OK" "${BENCH_LOG}" || true)"
|
||||
if [ "${MATCH_COUNT}" -lt "${BENCH_NUM_REQUESTS}" ]; then
|
||||
echo "!! CHECKSUM MATCH count (${MATCH_COUNT}) < expected (${BENCH_NUM_REQUESTS})"
|
||||
exit 1
|
||||
fi
|
||||
echo " CHECKSUM MATCH: ${MATCH_COUNT}/${BENCH_NUM_REQUESTS} request(s) verified OK"
|
||||
|
||||
echo ""
|
||||
echo "==> CPU server bench (${TRANSFER_MODE}) passed."
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Generic CPU vLLM e2e wrapper for both macOS and Ubuntu.
|
||||
# Assumes vLLM CPU build and facebook/opt-125m are already installed/
|
||||
# downloaded by the CI workflow steps before this script is invoked.
|
||||
#
|
||||
# Delegates all logic to the shared run-cpu-e2e-validation.sh with:
|
||||
# SKIP_INSTALL=1 (install done by CI workflow steps)
|
||||
#
|
||||
# Transport mode is selected via LMCACHE_TRANSPORT_MODE:
|
||||
# engine_driven -> LMCACHE_MP_TRANSFER_MODE=engine_driven (server-side copy)
|
||||
# lmcache_driven -> LMCACHE_DATA_MODE selects shm (default) or pickle
|
||||
#
|
||||
# Environment variables (all optional, defaults shown):
|
||||
# LMCACHE_TRANSPORT_MODE Transport: engine_driven|lmcache_driven
|
||||
# (default: engine_driven)
|
||||
# LMCACHE_DATA_MODE Data transfer mode: shm|pickle (default: shm)
|
||||
# LMCACHE_HTTP_PORT HTTP port for LMCache server (default: 8080)
|
||||
# VLLM_PORT HTTP port for vLLM server (default: 8000)
|
||||
# LMCACHE_L1_SIZE_GB LMCache L1 cache size in GB (default: 2)
|
||||
# VLLM_READY_TIMEOUT Seconds to wait for vLLM (default: 300)
|
||||
# LMCACHE_HEALTHCHECK_TIMEOUT Seconds to wait for LMCache (default: 60)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
OS="$(uname -s)"
|
||||
echo "==> CPU vLLM e2e test (OS: ${OS})"
|
||||
echo " Python: $(python3 --version 2>&1 || true)"
|
||||
echo " uname: $(uname -a)"
|
||||
if [ "${OS}" = "Darwin" ]; then
|
||||
sw_vers 2>/dev/null || true
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
||||
SHARED_SCRIPT="${REPO_ROOT}/.github/scripts/run-cpu-e2e-validation.sh"
|
||||
|
||||
if [ ! -f "${SHARED_SCRIPT}" ]; then
|
||||
echo "!! Shared script not found: ${SHARED_SCRIPT}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LMCACHE_TRANSPORT_MODE="${LMCACHE_TRANSPORT_MODE:-engine_driven}"
|
||||
|
||||
# When LMCACHE_TRANSPORT_MODE=engine_driven, LMCACHE_DATA_MODE selects the
|
||||
# specific data transfer mechanism: shm (default) or pickle.
|
||||
LMCACHE_DATA_MODE="${LMCACHE_DATA_MODE:-shm}"
|
||||
|
||||
# Map LMCACHE_TRANSPORT_MODE to the vars expected by the shared script.
|
||||
case "${LMCACHE_TRANSPORT_MODE}" in
|
||||
engine_driven)
|
||||
case "${LMCACHE_DATA_MODE}" in
|
||||
shm)
|
||||
export LMCACHE_SHM_NAME="__default__"
|
||||
;;
|
||||
pickle)
|
||||
export LMCACHE_SHM_NAME=""
|
||||
;;
|
||||
esac
|
||||
export LMCACHE_MP_TRANSFER_MODE="engine_driven"
|
||||
;;
|
||||
lmcache_driven)
|
||||
export LMCACHE_MP_TRANSFER_MODE="lmcache_driven"
|
||||
;;
|
||||
*)
|
||||
echo "!! Unknown LMCACHE_TRANSPORT_MODE='${LMCACHE_TRANSPORT_MODE}'"
|
||||
echo " Valid values: engine_driven, lmcache_driven"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
export SKIP_INSTALL="${SKIP_INSTALL:-1}"
|
||||
export LMCACHE_HEALTHCHECK_TIMEOUT="${LMCACHE_HEALTHCHECK_TIMEOUT:-60}"
|
||||
export VLLM_READY_TIMEOUT="${VLLM_READY_TIMEOUT:-300}"
|
||||
export LMCACHE_LOG_FILE="${LMCACHE_LOG_FILE:-/tmp/cpu_e2e_lmcache.log}"
|
||||
export VLLM_LOG_FILE="${VLLM_LOG_FILE:-/tmp/cpu_e2e_vllm.log}"
|
||||
|
||||
echo " LMCACHE_TRANSPORT_MODE=${LMCACHE_TRANSPORT_MODE}"
|
||||
echo " LMCACHE_DATA_MODE=${LMCACHE_DATA_MODE}"
|
||||
echo " SKIP_INSTALL=${SKIP_INSTALL}"
|
||||
echo " Delegating to: ${SHARED_SCRIPT}"
|
||||
|
||||
exec bash "${SHARED_SCRIPT}"
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Download (or just verify the local cache of) a HuggingFace model
|
||||
# repo, with bounded retry + exponential backoff so flaky HF mirrors
|
||||
# don't fail the whole CI run.
|
||||
#
|
||||
# Usage:
|
||||
# download_model.sh <repo_id> [<repo_id> ...]
|
||||
# MODEL_ID=facebook/opt-125m download_model.sh
|
||||
#
|
||||
# Environment:
|
||||
# MODEL_ID fallback when no positional args given
|
||||
# HF_TOKEN optional HuggingFace token for authenticated
|
||||
# requests (avoids anonymous rate limits)
|
||||
# HF_DOWNLOAD_MAX_RETRIES default 5
|
||||
# HF_DOWNLOAD_RETRY_DELAY default 30 (seconds, doubled per retry)
|
||||
# HF_DOWNLOAD_FAIL_ON_ERROR default 0 (1 -> exit non-zero on failure)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [ "$#" -eq 0 ]; then
|
||||
if [ -z "${MODEL_ID:-}" ]; then
|
||||
echo "!! download_model.sh: no model id provided"
|
||||
echo " pass repo ids as args or set MODEL_ID=..."
|
||||
exit 2
|
||||
fi
|
||||
set -- "${MODEL_ID}"
|
||||
fi
|
||||
|
||||
MAX_RETRIES="${HF_DOWNLOAD_MAX_RETRIES:-5}"
|
||||
RETRY_DELAY="${HF_DOWNLOAD_RETRY_DELAY:-30}"
|
||||
FAIL_ON_ERROR="${HF_DOWNLOAD_FAIL_ON_ERROR:-0}"
|
||||
|
||||
MAX_RETRIES="${MAX_RETRIES}" RETRY_DELAY="${RETRY_DELAY}" \
|
||||
FAIL_ON_ERROR="${FAIL_ON_ERROR}" HF_TOKEN="${HF_TOKEN:-}" \
|
||||
python3 - "$@" <<'PY'
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
max_retries = int(os.environ["MAX_RETRIES"])
|
||||
base_delay = int(os.environ["RETRY_DELAY"])
|
||||
fail_on_error = os.environ["FAIL_ON_ERROR"] == "1"
|
||||
hf_token = os.environ.get("HF_TOKEN") or None
|
||||
|
||||
repos = sys.argv[1:]
|
||||
failures = []
|
||||
|
||||
for repo in repos:
|
||||
# Try local cache first to avoid unnecessary HF API calls
|
||||
# (which can 429 on busy CI runners even when the model is cached).
|
||||
try:
|
||||
snapshot_download(repo, local_files_only=True, token=hf_token)
|
||||
print(f"CACHED: {repo} (local, no network)")
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
delay = base_delay
|
||||
ok = False
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
print(f"Attempt {attempt + 1}/{max_retries}: {repo}")
|
||||
snapshot_download(repo, token=hf_token)
|
||||
print(f"OK: {repo}")
|
||||
ok = True
|
||||
break
|
||||
except Exception as exc:
|
||||
print(f"Attempt {attempt + 1} failed for {repo}: {exc}")
|
||||
if attempt < max_retries - 1:
|
||||
# Add ±25% jitter to avoid thundering herd
|
||||
jitter = delay * (0.75 + random.random() * 0.5)
|
||||
print(
|
||||
"Waiting %.1fs before retry..." % jitter)
|
||||
time.sleep(jitter)
|
||||
delay *= 2
|
||||
if not ok:
|
||||
failures.append(repo)
|
||||
|
||||
if failures:
|
||||
print("All retry attempts failed for: %s" % ", ".join(failures))
|
||||
sys.exit(1 if fail_on_error else 0)
|
||||
PY
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Download facebook/opt-125m from the LMCache/opt-125m GitHub release
|
||||
# and place it into the HuggingFace hub cache structure so vLLM can
|
||||
# find it via ``facebook/opt-125m`` without any network call to HF.
|
||||
#
|
||||
# Environment:
|
||||
# OPT125M_GH_RELEASE release tag (default: v1.0)
|
||||
# OPT125M_SNAPSHOT HF snapshot hash (default: 27dcfa...)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
GH_RELEASE="${OPT125M_GH_RELEASE:-v1.0}"
|
||||
SNAPSHOT="${OPT125M_SNAPSHOT:-27dcfa74d334bc871f3234de431e71c6eeba5dd6}"
|
||||
|
||||
HF_CACHE="${HF_HOME:-$HOME/.cache/huggingface}/hub"
|
||||
MODEL_DIR="${HF_CACHE}/models--facebook--opt-125m"
|
||||
SNAPSHOT_DIR="${MODEL_DIR}/snapshots/${SNAPSHOT}"
|
||||
|
||||
if [ -f "${SNAPSHOT_DIR}/pytorch_model.bin" ]; then
|
||||
echo "opt-125m already cached in ${SNAPSHOT_DIR}, skipping download"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Downloading opt-125m from GitHub release ${GH_RELEASE}..."
|
||||
mkdir -p "${SNAPSHOT_DIR}"
|
||||
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "${TMP_DIR}"' EXIT
|
||||
|
||||
TARBALL="${TMP_DIR}/opt-125m.tar.gz"
|
||||
DOWNLOAD_URL="https://github.com/LMCache/opt-125m/releases/download/${GH_RELEASE}/opt-125m.tar.gz"
|
||||
|
||||
for i in $(seq 1 5); do
|
||||
if curl -fsSL --retry 3 -o "${TARBALL}" "${DOWNLOAD_URL}"; then
|
||||
break
|
||||
fi
|
||||
if [ "${i}" -eq 5 ]; then
|
||||
echo "!! Failed to download after 5 attempts: ${DOWNLOAD_URL}"
|
||||
exit 1
|
||||
fi
|
||||
sleep $((10 * i))
|
||||
done
|
||||
|
||||
# Extract directly – the tarball contains opt-125m/ at top level.
|
||||
EXTRACT_DIR="${TMP_DIR}/extract"
|
||||
mkdir -p "${EXTRACT_DIR}"
|
||||
tar -xzf "${TARBALL}" -C "${EXTRACT_DIR}"
|
||||
mv "${EXTRACT_DIR}"/opt-125m/* "${SNAPSHOT_DIR}/"
|
||||
|
||||
# Create the refs/main pointer so snapshot_download(local_files_only=True)
|
||||
# can resolve the snapshot hash.
|
||||
mkdir -p "${MODEL_DIR}/refs"
|
||||
echo "${SNAPSHOT}" > "${MODEL_DIR}/refs/main"
|
||||
|
||||
echo "opt-125m cached at ${SNAPSHOT_DIR}"
|
||||
ls -lah "${SNAPSHOT_DIR}/"
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Install lmcache in editable, CPU-only mode (NO_GPU_EXT=1).
|
||||
# Assumes vLLM (and therefore torch) was already installed by
|
||||
# install_vllm_cpu.sh, so we use --no-deps to keep that pinned torch
|
||||
# in place.
|
||||
#
|
||||
# Usage:
|
||||
# install_lmcache_cpu.sh # plain pip
|
||||
# PIP_BIN="uv pip" install_lmcache_cpu.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PIP_BIN="${PIP_BIN:-pip}"
|
||||
|
||||
export NO_GPU_EXT=1
|
||||
export SETUPTOOLS_SCM_PRETEND_VERSION="${SETUPTOOLS_SCM_PRETEND_VERSION:-0.0.0.dev0}"
|
||||
|
||||
${PIP_BIN} install --upgrade pip
|
||||
${PIP_BIN} install -r requirements/build.txt
|
||||
${PIP_BIN} install -r requirements/common.txt
|
||||
${PIP_BIN} install -r requirements/cli.txt
|
||||
${PIP_BIN} install -e . --no-deps --no-build-isolation
|
||||
|
||||
python -c "import lmcache, vllm; \
|
||||
print('lmcache:', lmcache.__version__, 'vllm:', vllm.__version__)"
|
||||
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Install the prebuilt CPU-only vLLM wheel (`vllm-cpu-nightly`) plus a
|
||||
# `vllm-<ver>+cpu.dist-info` alias.
|
||||
#
|
||||
# Why the alias: the wheel installs the `vllm/` package but registers
|
||||
# its dist metadata under `vllm-cpu-nightly`. vLLM's CLI / internal
|
||||
# callers do `importlib.metadata.version("vllm")` (distribution name,
|
||||
# not import name); without the alias that raises PackageNotFoundError
|
||||
# and `vllm serve` won't start. The `+cpu` local label is also needed
|
||||
# so `cpu_platform_plugin()` activates the CPU platform (it greps the
|
||||
# dist metadata for the substring "cpu"); our build strips `+cpu`
|
||||
# before PyPI upload because PyPI rejects local versions.
|
||||
#
|
||||
# Usage:
|
||||
# install_vllm_cpu.sh # use plain `pip`
|
||||
# PIP_BIN="uv pip" install_vllm_cpu.sh
|
||||
# # use `uv pip` and pass extra flags
|
||||
# # via PIP_INSTALL_EXTRA_ARGS
|
||||
#
|
||||
# Idempotent: re-running just rewrites the alias.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PIP_BIN="${PIP_BIN:-pip}"
|
||||
PIP_INSTALL_EXTRA_ARGS="${PIP_INSTALL_EXTRA_ARGS:-}"
|
||||
|
||||
# `--extra-index-url` is required because the wheel pins torch==2.11.0
|
||||
# which only lives on the pytorch CPU index. Harmless on macOS.
|
||||
${PIP_BIN} install "numpy<2"
|
||||
# shellcheck disable=SC2086
|
||||
${PIP_BIN} install vllm-cpu-nightly \
|
||||
--extra-index-url https://download.pytorch.org/whl/cpu \
|
||||
${PIP_INSTALL_EXTRA_ARGS}
|
||||
|
||||
python - <<'PY'
|
||||
import importlib.metadata as md
|
||||
import pathlib
|
||||
import shutil
|
||||
|
||||
dist = md.distribution("vllm-cpu-nightly")
|
||||
ver = dist.version
|
||||
fake_ver = f"{ver}+cpu"
|
||||
site_root = pathlib.Path(dist.locate_file(""))
|
||||
info_name = next(
|
||||
p.parts[0] for p in (dist.files or [])
|
||||
if p.parts and p.parts[0].endswith(".dist-info")
|
||||
)
|
||||
src = site_root / info_name
|
||||
dst = src.with_name(f"vllm-{fake_ver}.dist-info")
|
||||
if dst.exists():
|
||||
shutil.rmtree(dst)
|
||||
shutil.copytree(src, dst)
|
||||
meta = dst / "METADATA"
|
||||
txt = meta.read_text()
|
||||
txt = txt.replace("Name: vllm-cpu-nightly", "Name: vllm", 1)
|
||||
txt = txt.replace(f"Version: {ver}", f"Version: {fake_ver}", 1)
|
||||
meta.write_text(txt)
|
||||
print(f"Aliased {src.name} -> {dst.name}")
|
||||
print("vllm version (via importlib.metadata):", md.version("vllm"))
|
||||
PY
|
||||
|
||||
python -c "import vllm, torch; \
|
||||
print('vllm:', vllm.__version__, 'torch:', torch.__version__, \
|
||||
'cuda:', torch.cuda.is_available())"
|
||||
Executable
+648
@@ -0,0 +1,648 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "Build ID: ${BUILDKITE_BUILD_ID:-local}"
|
||||
echo "Python: $(python3 --version 2>&1 || true)"
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
echo "uv: $(uv --version 2>&1 || true)"
|
||||
else
|
||||
echo "uv: not installed"
|
||||
fi
|
||||
|
||||
BUILD_ID="${BUILDKITE_BUILD_ID:-local_$$}"
|
||||
VENV_DIR=".venv-${BUILD_ID}"
|
||||
SHARED_SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
LMCACHE_LOG="${LMCACHE_LOG_FILE:-/tmp/build_${BUILD_ID}_lmcache_cpu_validation.log}"
|
||||
VLLM_LOG="${VLLM_LOG_FILE:-/tmp/build_${BUILD_ID}_vllm_cpu_validation.log}"
|
||||
LMCACHE_PID=""
|
||||
VLLM_PID=""
|
||||
LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}"
|
||||
LMCACHE_ZMQ_PORT="${LMCACHE_ZMQ_PORT:-5555}"
|
||||
VLLM_PORT="${VLLM_PORT:-8000}"
|
||||
LMCACHE_L1_SIZE_GB="${LMCACHE_L1_SIZE_GB:-1}"
|
||||
LMCACHE_EVICTION_POLICY="${LMCACHE_EVICTION_POLICY:-LRU}"
|
||||
LMCACHE_CHUNK_SIZE="${LMCACHE_CHUNK_SIZE:-128}"
|
||||
VLLM_GPU_MEMORY_UTILIZATION="${VLLM_GPU_MEMORY_UTILIZATION:-0.1}"
|
||||
# CPU-only KV cache pool size in GiB. vLLM defaults to 4 GiB when unset;
|
||||
# 1 GiB is plenty for opt-125m e2e validation.
|
||||
VLLM_CPU_KVCACHE_SPACE="${VLLM_CPU_KVCACHE_SPACE:-1}"
|
||||
# Cap context length and concurrent sequences to shrink scheduler buffers.
|
||||
VLLM_MAX_MODEL_LEN="${VLLM_MAX_MODEL_LEN:-2048}"
|
||||
VLLM_MAX_NUM_SEQS="${VLLM_MAX_NUM_SEQS:-4}"
|
||||
LMCACHE_HEALTHCHECK_TIMEOUT="${LMCACHE_HEALTHCHECK_TIMEOUT:-30}"
|
||||
VLLM_READY_TIMEOUT="${VLLM_READY_TIMEOUT:-120}"
|
||||
# Transport mode selection:
|
||||
# LMCACHE_MP_TRANSFER_MODE=engine_driven -> engine-driven data path,
|
||||
# sub-selected by LMCACHE_SHM_NAME:
|
||||
# LMCACHE_SHM_NAME="" -> pickle transport
|
||||
# LMCACHE_SHM_NAME=__default__ -> shm transport (default)
|
||||
# LMCACHE_MP_TRANSFER_MODE=lmcache_driven -> lmcache-driven IPC handle path
|
||||
LMCACHE_SHM_NAME="${LMCACHE_SHM_NAME-__default__}"
|
||||
LMCACHE_MP_TRANSFER_MODE="${LMCACHE_MP_TRANSFER_MODE:-engine_driven}"
|
||||
# Set SKIP_INSTALL=1 to skip Phase 1 (install) — useful when the caller
|
||||
# has already installed everything (e.g. macOS CI workflow steps).
|
||||
SKIP_INSTALL="${SKIP_INSTALL:-0}"
|
||||
|
||||
# Directory to collect artifacts before workspace is deleted
|
||||
ARTIFACT_DIR="/tmp/build_${BUILD_ID}_artifacts"
|
||||
mkdir -p "${ARTIFACT_DIR}"
|
||||
|
||||
upload_artifacts() {
|
||||
# Copy logs to artifact dir (which survives workspace deletion)
|
||||
cp -f "${LMCACHE_LOG}" "${ARTIFACT_DIR}/lmcache_cpu_validation.log" 2>/dev/null || true
|
||||
cp -f "${VLLM_LOG}" "${ARTIFACT_DIR}/vllm_cpu_validation.log" 2>/dev/null || true
|
||||
|
||||
if [ -n "${BUILDKITE_BUILD_ID:-}" ] && command -v buildkite-agent >/dev/null 2>&1; then
|
||||
buildkite-agent artifact upload "${ARTIFACT_DIR}/*.log" || true
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_workspace() {
|
||||
if [ -n "${BUILDKITE_BUILD_ID:-}" ]; then
|
||||
export TARGET="$PWD"
|
||||
case "$TARGET" in
|
||||
""|"/"|"/usr"|"/var"|"/etc"|"/bin"|"/sbin"|"/opt"|"/home"|"/tmp")
|
||||
echo "❌ Refusing to delete unsafe workspace path: ${TARGET:-<empty>}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
if [ "$TARGET" = "$HOME" ]; then
|
||||
echo "❌ Refusing to delete unsafe workspace path: ${TARGET:-<empty>}"
|
||||
return 1
|
||||
fi
|
||||
if [ ! -d "$TARGET/.git" ] || [ ! -f "$TARGET/pyproject.toml" ]; then
|
||||
echo "❌ Refusing to delete unexpected workspace path: $TARGET"
|
||||
return 1
|
||||
fi
|
||||
echo "Deleting current workspace $TARGET"
|
||||
cd /
|
||||
if command -v sudo >/dev/null 2>&1; then
|
||||
sudo rm -rf "$TARGET"
|
||||
else
|
||||
rm -rf "$TARGET"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
print_failure_logs() {
|
||||
echo "=== LMCache Server Log (${LMCACHE_LOG}) ==="
|
||||
if [ -f "${LMCACHE_LOG}" ]; then
|
||||
tail -n 200 "${LMCACHE_LOG}" || true
|
||||
else
|
||||
echo "Log not found"
|
||||
fi
|
||||
echo "=== vLLM Log (${VLLM_LOG}) ==="
|
||||
if [ -f "${VLLM_LOG}" ]; then
|
||||
tail -n 200 "${VLLM_LOG}" || true
|
||||
else
|
||||
echo "Log not found"
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_processes() {
|
||||
set +e
|
||||
if [ -n "${VLLM_PID}" ] && kill -0 "${VLLM_PID}" 2>/dev/null; then
|
||||
echo "Stopping vLLM (PID=${VLLM_PID})"
|
||||
kill "${VLLM_PID}" 2>/dev/null || true
|
||||
wait "${VLLM_PID}" 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${LMCACHE_PID}" ] && kill -0 "${LMCACHE_PID}" 2>/dev/null; then
|
||||
echo "Stopping LMCache server (PID=${LMCACHE_PID})"
|
||||
kill "${LMCACHE_PID}" 2>/dev/null || true
|
||||
wait "${LMCACHE_PID}" 2>/dev/null || true
|
||||
fi
|
||||
set -e
|
||||
}
|
||||
|
||||
wait_for_endpoint_contains() {
|
||||
local url="$1"
|
||||
local timeout="$2"
|
||||
local expected="$3"
|
||||
local label="$4"
|
||||
local response
|
||||
|
||||
for _ in $(seq 1 "${timeout}"); do
|
||||
if response="$(curl -fsS "${url}" 2>/dev/null)"; then
|
||||
if [ -z "${expected}" ] || echo "${response}" | grep -q "${expected}"; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "❌ ${label} did not become ready within ${timeout}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Scrape a Prometheus counter value from LMCache /metrics endpoint
|
||||
scrape_metric() {
|
||||
local metric_name="$1"
|
||||
python3 - <<EOF
|
||||
import sys, urllib.request
|
||||
url = "http://localhost:${LMCACHE_HTTP_PORT}/metrics"
|
||||
try:
|
||||
body = urllib.request.urlopen(url, timeout=10).read().decode()
|
||||
except Exception as e:
|
||||
print(f"ERROR fetching {url}: {e}", file=sys.stderr)
|
||||
print("0")
|
||||
sys.exit(0)
|
||||
total = 0.0
|
||||
for line in body.splitlines():
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
if not line.startswith("${metric_name}"):
|
||||
continue
|
||||
parts = line.rsplit(" ", 1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
try:
|
||||
total += float(parts[1])
|
||||
except ValueError:
|
||||
continue
|
||||
print(int(total))
|
||||
EOF
|
||||
}
|
||||
|
||||
# Poll a Prometheus counter until its value differs from `baseline`,
|
||||
# or until `timeout` seconds elapse. Returns 0 on change, 1 on timeout
|
||||
# (callers typically `|| true` it because not every probe expects a
|
||||
# change to actually happen).
|
||||
wait_for_metric_change() {
|
||||
local metric_name="$1"
|
||||
local baseline="$2"
|
||||
local timeout="${3:-10}"
|
||||
local current
|
||||
for _ in $(seq 1 "${timeout}"); do
|
||||
current="$(scrape_metric "${metric_name}")"
|
||||
if [ "${current}" != "${baseline}" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Send a completion request and print the text output
|
||||
send_completion() {
|
||||
local prompt_file="$1"
|
||||
local max_tokens="${2:-50}"
|
||||
local body_file
|
||||
body_file="$(mktemp)"
|
||||
# Build the JSON body in a separate process to avoid nested-quote
|
||||
# quoting nightmares with -d "$(python3 -c "...")". Pass the prompt
|
||||
# file path and max_tokens via argv so the python snippet itself
|
||||
# does not need any shell interpolation inside its string body.
|
||||
PROMPT_FILE="${prompt_file}" MAX_TOKENS="${max_tokens}" \
|
||||
python3 - >"${body_file}" <<'PYEOF'
|
||||
import json
|
||||
import os
|
||||
|
||||
prompt = open(os.environ['PROMPT_FILE']).read()
|
||||
print(json.dumps({
|
||||
'model': 'facebook/opt-125m',
|
||||
'prompt': prompt,
|
||||
'max_tokens': int(os.environ['MAX_TOKENS']),
|
||||
'temperature': 0,
|
||||
}))
|
||||
PYEOF
|
||||
local response
|
||||
response="$(curl -fsS "http://localhost:${VLLM_PORT}/v1/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-binary "@${body_file}")"
|
||||
rm -f "${body_file}"
|
||||
echo "${response}" | python3 -c "import json,sys; print(json.load(sys.stdin)['choices'][0]['text'])"
|
||||
}
|
||||
|
||||
start_vllm() {
|
||||
echo "Starting vLLM server..."
|
||||
# Export so multiproc_executor worker children inherit these. Without
|
||||
# VLLM_CPU_KVCACHE_SPACE, CPU backend falls back to
|
||||
# `total_memory * gpu_memory_utilization`, which can request 100s of GiB
|
||||
# on big hosts and OOM (see vllm/v1/worker/cpu_worker.py:determine_available_memory).
|
||||
# VLLM_DEVICE is the modern env var (vLLM 0.8+); VLLM_TARGET_DEVICE is
|
||||
# kept for backwards-compatibility with older vLLM CPU wheels.
|
||||
export VLLM_DEVICE=cpu
|
||||
export VLLM_TARGET_DEVICE=cpu
|
||||
export VLLM_CPU_KVCACHE_SPACE="${VLLM_CPU_KVCACHE_SPACE}"
|
||||
export LMCACHE_MP_TRANSFER_MODE="${LMCACHE_MP_TRANSFER_MODE}"
|
||||
# Pin gloo / vLLM rendezvous to loopback. Otherwise vLLM's
|
||||
# network_utils.get_ip() picks a LAN address (e.g. 192.168.x.x on the
|
||||
# macOS GHA runner) and gloo's init_process_group sits there for ~16
|
||||
# minutes doing slow socket bind/connect retries before the engine
|
||||
# ever loads weights.
|
||||
export VLLM_HOST_IP="${VLLM_HOST_IP:-127.0.0.1}"
|
||||
if [ -z "${GLOO_SOCKET_IFNAME:-}" ]; then
|
||||
case "$(uname -s)" in
|
||||
Darwin) export GLOO_SOCKET_IFNAME=lo0 ;;
|
||||
Linux) export GLOO_SOCKET_IFNAME=lo ;;
|
||||
esac
|
||||
fi
|
||||
local kv_cache_bytes
|
||||
kv_cache_bytes="$(python3 -c "print(int(${VLLM_CPU_KVCACHE_SPACE} * 1024 * 1024 * 1024))")"
|
||||
# Tell LMCacheMPConnector where the lmcache server actually listens.
|
||||
# Without this it falls back to tcp://localhost:5555 and dies with
|
||||
# "Cannot reach the LMCache MP server" whenever we run multiple e2e
|
||||
# steps in parallel/sequence on different ZMQ ports.
|
||||
local kv_transfer_config
|
||||
kv_transfer_config="$(python3 -c "
|
||||
import json
|
||||
print(json.dumps({
|
||||
'kv_connector': 'LMCacheMPConnector',
|
||||
'kv_role': 'kv_both',
|
||||
'kv_connector_module_path': 'lmcache.integration.vllm.lmcache_mp_connector',
|
||||
'kv_connector_extra_config': {
|
||||
'lmcache.mp.host': 'tcp://localhost',
|
||||
'lmcache.mp.port': int('${LMCACHE_ZMQ_PORT}'),
|
||||
},
|
||||
}))")"
|
||||
vllm serve facebook/opt-125m \
|
||||
--port "${VLLM_PORT}" \
|
||||
--dtype bfloat16 \
|
||||
--disable-hybrid-kv-cache-manager \
|
||||
--no-enable-prefix-caching \
|
||||
--gpu-memory-utilization "${VLLM_GPU_MEMORY_UTILIZATION}" \
|
||||
--kv-cache-memory-bytes "${kv_cache_bytes}" \
|
||||
--max-model-len "${VLLM_MAX_MODEL_LEN}" \
|
||||
--max-num-seqs "${VLLM_MAX_NUM_SEQS}" \
|
||||
--kv-transfer-config "${kv_transfer_config}" \
|
||||
--enforce-eager \
|
||||
>"${VLLM_LOG}" 2>&1 &
|
||||
VLLM_PID=$!
|
||||
echo "vLLM server started (PID=${VLLM_PID})"
|
||||
sleep 1
|
||||
if ! kill -0 "${VLLM_PID}" 2>/dev/null; then
|
||||
echo "❌ vLLM server exited immediately after startup. See ${VLLM_LOG} for details"
|
||||
return 1
|
||||
fi
|
||||
echo "Waiting for vLLM readiness at http://localhost:${VLLM_PORT}/v1/models (timeout: ${VLLM_READY_TIMEOUT}s)"
|
||||
if ! wait_for_endpoint_contains "http://localhost:${VLLM_PORT}/v1/models" "${VLLM_READY_TIMEOUT}" "facebook/opt-125m" "vLLM server"; then
|
||||
return 1
|
||||
fi
|
||||
echo "✅ vLLM server is ready"
|
||||
}
|
||||
|
||||
stop_vllm() {
|
||||
if [ -n "${VLLM_PID}" ] && kill -0 "${VLLM_PID}" 2>/dev/null; then
|
||||
echo "Stopping vLLM (PID=${VLLM_PID})"
|
||||
kill "${VLLM_PID}" 2>/dev/null || true
|
||||
wait "${VLLM_PID}" 2>/dev/null || true
|
||||
VLLM_PID=""
|
||||
fi
|
||||
}
|
||||
|
||||
on_error() {
|
||||
local exit_code=$?
|
||||
trap - ERR
|
||||
echo "❌ CPU install validation failed (exit code: ${exit_code})"
|
||||
set +e
|
||||
print_failure_logs
|
||||
cleanup_processes
|
||||
upload_artifacts
|
||||
cleanup_workspace || echo "❌ Workspace cleanup failed"
|
||||
set -e
|
||||
exit "$exit_code"
|
||||
}
|
||||
|
||||
trap on_error ERR
|
||||
|
||||
if [ "${SKIP_INSTALL}" = "1" ]; then
|
||||
echo "=== CPU Install Validation (Phase 1) — SKIPPED (SKIP_INSTALL=1) ==="
|
||||
else
|
||||
echo "=== CPU Install Validation (Phase 1) ==="
|
||||
echo "Creating virtual environment with uv at ${VENV_DIR}"
|
||||
uv venv --python 3.12 "${VENV_DIR}"
|
||||
# shellcheck disable=SC1091
|
||||
source "${VENV_DIR}/bin/activate"
|
||||
|
||||
uv pip install --upgrade pip setuptools wheel
|
||||
|
||||
# `--index-strategy unsafe-best-match` is required because uv's
|
||||
# default `first-index` strategy locks each package to the first
|
||||
# index that lists it; the pytorch CPU index ships an older mirror
|
||||
# of `setuptools` that would block the version vllm-cpu-nightly
|
||||
# pins.
|
||||
uv pip uninstall -y vllm vllm-cpu-nightly 2>/dev/null || true
|
||||
PIP_BIN="uv pip" \
|
||||
PIP_INSTALL_EXTRA_ARGS="--index-strategy unsafe-best-match" \
|
||||
bash "${SHARED_SCRIPTS_DIR}/install_vllm_cpu.sh"
|
||||
|
||||
PIP_BIN="uv pip" \
|
||||
bash "${SHARED_SCRIPTS_DIR}/install_lmcache_cpu.sh"
|
||||
|
||||
echo "Freezing installed package versions"
|
||||
uv pip freeze
|
||||
|
||||
echo "✅ CPU install validation passed"
|
||||
fi
|
||||
|
||||
echo "=== CPU E2E Validation (Phase 2) ==="
|
||||
|
||||
if [ "${SKIP_INSTALL}" = "1" ]; then
|
||||
echo "[Phase 2 / Step 1] numpy<2 install — SKIPPED (SKIP_INSTALL=1)"
|
||||
else
|
||||
echo "[Phase 2 / Step 1] Installing numpy<2 for scipy/vLLM compatibility"
|
||||
uv pip install "numpy<2"
|
||||
echo "✅ numpy<2 installed"
|
||||
fi
|
||||
|
||||
echo "[Phase 2 / Step 2] Downloading facebook/opt-125m model (GitHub release)"
|
||||
bash "${SHARED_SCRIPTS_DIR}/download_opt125m_github.sh"
|
||||
echo "✅ Model download/check complete"
|
||||
|
||||
echo "[Phase 2 / Step 3] Starting LMCache server"
|
||||
echo "LMCache log: ${LMCACHE_LOG}"
|
||||
# Build lmcache server args
|
||||
LMCACHE_ARGS=(
|
||||
--port "${LMCACHE_ZMQ_PORT}"
|
||||
--http-port "${LMCACHE_HTTP_PORT}"
|
||||
--l1-size-gb "${LMCACHE_L1_SIZE_GB}"
|
||||
--eviction-policy "${LMCACHE_EVICTION_POLICY}"
|
||||
--chunk-size "${LMCACHE_CHUNK_SIZE}"
|
||||
)
|
||||
# `engine_driven` / `lmcache_driven` resolve to different worker-side
|
||||
# TransferContexts:
|
||||
# engine_driven -> EngineDrivenTransferContext (worker gathers/scatters data)
|
||||
# sub-mode: SHM (--shm-name __default__) or pickle (--shm-name "")
|
||||
# lmcache_driven -> LMCacheDrivenTransferContext (server-side IPC handle)
|
||||
# Step 5.5 verifies which one the worker actually entered.
|
||||
if [ "${LMCACHE_MP_TRANSFER_MODE}" = "engine_driven" ]; then
|
||||
if [ "${LMCACHE_SHM_NAME}" = "__default__" ]; then
|
||||
echo "Transport mode: engine-driven/shm (shared memory)"
|
||||
EXPECTED_TRANSPORT="shm"
|
||||
else
|
||||
echo "Transport mode: engine-driven/pickle (--shm-name '${LMCACHE_SHM_NAME}')"
|
||||
LMCACHE_ARGS+=(--shm-name "${LMCACHE_SHM_NAME}")
|
||||
EXPECTED_TRANSPORT="pickle"
|
||||
fi
|
||||
elif [ "${LMCACHE_MP_TRANSFER_MODE}" = "lmcache_driven" ]; then
|
||||
echo "Transport mode: lmcache-driven (IPC handle path)"
|
||||
EXPECTED_TRANSPORT="lmcache_driven"
|
||||
else
|
||||
echo "Transport mode: unknown '${LMCACHE_MP_TRANSFER_MODE}',"
|
||||
echo " falling back to LMCACHE_SHM_NAME-based detection"
|
||||
if [ "${LMCACHE_SHM_NAME}" = "__default__" ]; then
|
||||
echo "Transport mode: data/shm (shared memory, fallback)"
|
||||
EXPECTED_TRANSPORT="shm"
|
||||
else
|
||||
echo "Transport mode: data/pickle (--shm-name '${LMCACHE_SHM_NAME}')"
|
||||
LMCACHE_ARGS+=(--shm-name "${LMCACHE_SHM_NAME}")
|
||||
EXPECTED_TRANSPORT="pickle"
|
||||
fi
|
||||
fi
|
||||
|
||||
lmcache server "${LMCACHE_ARGS[@]}" \
|
||||
>"${LMCACHE_LOG}" 2>&1 &
|
||||
LMCACHE_PID=$!
|
||||
echo "LMCache server started (PID=${LMCACHE_PID})"
|
||||
sleep 1
|
||||
if ! kill -0 "${LMCACHE_PID}" 2>/dev/null; then
|
||||
echo "❌ LMCache server exited immediately after startup. See ${LMCACHE_LOG} for details"
|
||||
false
|
||||
fi
|
||||
|
||||
echo "Waiting for LMCache healthcheck at http://localhost:${LMCACHE_HTTP_PORT}/healthcheck (timeout: ${LMCACHE_HEALTHCHECK_TIMEOUT}s)"
|
||||
if ! wait_for_endpoint_contains "http://localhost:${LMCACHE_HTTP_PORT}/healthcheck" "${LMCACHE_HEALTHCHECK_TIMEOUT}" "" "LMCache server"; then
|
||||
false
|
||||
fi
|
||||
echo "✅ LMCache server is healthy"
|
||||
|
||||
echo "[Phase 2 / Step 4] Installing system libs (Linux only) and starting vLLM server"
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
# System libs vLLM's CPU serve path dlopens at import time:
|
||||
# - libnuma1: needed by some vLLM CPU code paths.
|
||||
# - ffmpeg: provides libavutil.so.{56..60} etc. that torchcodec
|
||||
# dlopens. Recent vLLM nightlies import torchcodec
|
||||
# unconditionally, so `vllm serve` aborts with
|
||||
# "libavutil.so.NN: cannot open shared object file"
|
||||
# without FFmpeg — even for text-only models.
|
||||
# Prefer sudo if present (GitHub ubuntu runners need it). Install only
|
||||
# what's missing and never let an apt hiccup fail the step.
|
||||
MISSING_PKGS=()
|
||||
if [ ! -e /usr/lib/x86_64-linux-gnu/libnuma.so.1 ] \
|
||||
&& [ ! -e /lib/x86_64-linux-gnu/libnuma.so.1 ]; then
|
||||
MISSING_PKGS+=(libnuma1)
|
||||
fi
|
||||
# torchcodec supports FFmpeg 4-8; the distro's ffmpeg package provides a
|
||||
# compatible libavutil. Detect via ldconfig so we match whichever
|
||||
# soname (56..60) the runner already ships.
|
||||
if ! ldconfig -p 2>/dev/null | grep -q 'libavutil\.so'; then
|
||||
MISSING_PKGS+=(ffmpeg)
|
||||
fi
|
||||
if [ "${#MISSING_PKGS[@]}" -gt 0 ]; then
|
||||
echo "Installing missing system packages: ${MISSING_PKGS[*]}"
|
||||
if command -v sudo >/dev/null 2>&1; then
|
||||
sudo apt-get update \
|
||||
&& sudo apt-get install -y --no-install-recommends "${MISSING_PKGS[@]}" \
|
||||
|| echo "⚠️ apt-get install (${MISSING_PKGS[*]}) via sudo failed; continuing"
|
||||
else
|
||||
apt-get update \
|
||||
&& apt-get install -y --no-install-recommends "${MISSING_PKGS[@]}" \
|
||||
|| echo "⚠️ apt-get install (${MISSING_PKGS[*]}) failed; continuing"
|
||||
fi
|
||||
else
|
||||
echo "libnuma1 and FFmpeg runtime libs already present, skipping apt install"
|
||||
fi
|
||||
fi
|
||||
# VLLM_DEVICE is the modern env var (vLLM 0.8+)
|
||||
export VLLM_DEVICE=cpu
|
||||
export VLLM_TARGET_DEVICE=cpu
|
||||
start_vllm
|
||||
|
||||
echo "[Phase 2 / Step 5] Sending E2E test request"
|
||||
completion_response="$(curl -fsS "http://localhost:${VLLM_PORT}/v1/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"model":"facebook/opt-125m","prompt":"Hello","max_tokens":5}')"
|
||||
echo "Completion response: ${completion_response}"
|
||||
if ! echo "${completion_response}" | grep -q '"choices"'; then
|
||||
echo "❌ E2E request response failed structural validation"
|
||||
false
|
||||
fi
|
||||
if ! echo "${completion_response}" | grep -q "facebook/opt-125m"; then
|
||||
echo "❌ E2E request response missing expected model"
|
||||
false
|
||||
fi
|
||||
echo "✅ E2E request validation passed"
|
||||
|
||||
# Verify transport mode (logged after vLLM connects to LMCache server)
|
||||
echo "[Phase 2 / Step 5.5] Verifying transport mode: expecting '${EXPECTED_TRANSPORT}'"
|
||||
# Worker logs `Creating transfer context (device_type=<dev>, mode=<m>)`
|
||||
# from worker_transfer.py:create_transfer_context, where <m> is the
|
||||
# resolved MPTransferMode after env-var lookup. This is the single source
|
||||
# of truth for which TransferContext the worker actually entered. Note:
|
||||
# the worker is a child of `vllm serve`, so its LMCache log lines land in
|
||||
# VLLM_LOG (vllm's stdout), not in LMCACHE_LOG (lmcache server's stdout).
|
||||
# The shm/pickle branches still grep LMCACHE_LOG because the strategy
|
||||
# line is emitted by the lmcache server itself.
|
||||
if [ "${EXPECTED_TRANSPORT}" = "lmcache_driven" ]; then
|
||||
if ! grep -q "Creating transfer context.*mode=lmcache_driven" "${VLLM_LOG}" 2>/dev/null; then
|
||||
echo "❌ Expected lmcache-driven worker context but 'mode=lmcache_driven' not found in vLLM log"
|
||||
tail -50 "${VLLM_LOG}"
|
||||
false
|
||||
fi
|
||||
echo "✅ Transport mode confirmed: lmcache-driven (IPC handle path)"
|
||||
elif [ "${EXPECTED_TRANSPORT}" = "engine_driven" ]; then
|
||||
if ! grep -q "Creating transfer context.*mode=engine_driven" "${VLLM_LOG}" 2>/dev/null; then
|
||||
echo "❌ Expected engine-driven worker context but 'mode=engine_driven' not found in vLLM log"
|
||||
tail -50 "${VLLM_LOG}"
|
||||
false
|
||||
fi
|
||||
echo "✅ Transport mode confirmed: engine-driven"
|
||||
elif [ "${EXPECTED_TRANSPORT}" = "shm" ]; then
|
||||
if ! grep -q "Using shm non-GPU transfer strategy" "${LMCACHE_LOG}" 2>/dev/null; then
|
||||
echo "❌ Expected shm transport but server strategy line not found in log"
|
||||
tail -50 "${LMCACHE_LOG}"
|
||||
false
|
||||
fi
|
||||
echo "✅ Transport mode confirmed: shm"
|
||||
elif [ "${EXPECTED_TRANSPORT}" = "pickle" ]; then
|
||||
if ! grep -q "Using pickle non-GPU transfer strategy" "${LMCACHE_LOG}" 2>/dev/null; then
|
||||
echo "❌ Expected pickle transport but server strategy line not found in log"
|
||||
tail -50 "${LMCACHE_LOG}"
|
||||
false
|
||||
fi
|
||||
echo "✅ Transport mode confirmed: pickle"
|
||||
fi
|
||||
|
||||
echo "[Phase 2 / Step 6] Cleaning up Phase 2 vLLM"
|
||||
stop_vllm
|
||||
echo "✅ Phase 2 cleanup completed"
|
||||
|
||||
echo "✅ CPU E2E validation passed"
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# Phase 3: Cache Hit Validation
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
# Scenario:
|
||||
# - LMCache server stays running the entire time
|
||||
# - vLLM instance 1: request A → LMCache store; request A again → LMCache hit
|
||||
# - vLLM restart (instance 2): request A → LMCache hit (cross-instance)
|
||||
# - All three outputs must be identical (bit-exact with temperature=0)
|
||||
# ═══════════════════════════════════════════════════════════════════
|
||||
|
||||
echo "=== Cache Hit Validation (Phase 3) ==="
|
||||
|
||||
# Generate a fixed ~1000 token prompt
|
||||
PROMPT_FILE="/tmp/build_${BUILD_ID}_phase3_prompt.txt"
|
||||
python3 -c "
|
||||
# A diverse, non-repetitive story prompt (~1000 tokens for opt-125m).
|
||||
# The model will continue the narrative; correctness doesn't matter,
|
||||
# only that all three runs produce identical output (cache consistency).
|
||||
story = '''Once upon a time in a small coastal village, there lived an old lighthouse keeper named Thomas. Every evening, he climbed the one hundred and thirty-seven steps to the top of the lighthouse to light the great lamp. The sea was unpredictable in those parts. Ships from distant lands carried spices, silk, and stories of places Thomas had never seen. One stormy night in November, a merchant vessel called the Silver Heron appeared on the horizon, listing dangerously to starboard. Thomas watched through his brass telescope as the waves crashed against its hull. He knew that if the ship did not change course within the next ten minutes, it would strike the jagged rocks known locally as the Devil's Teeth. He grabbed the emergency flare gun from the wooden cabinet and fired three red flares into the sky. The captain of the Silver Heron saw the warning and ordered hard to port. The ship groaned as it turned, barely clearing the outermost rock by twenty meters. The next morning, the captain rowed ashore to thank Thomas personally. He brought a gift: a small wooden box containing a compass that always pointed not north, but toward home. Thomas kept it on his desk for the rest of his days. Years later, when Thomas retired, he passed the compass to his granddaughter Elena, who had inherited his love of the sea. Elena became a marine biologist studying the migration patterns of humpback whales along the Pacific coast. She traveled from Alaska to Mexico following the whale pods, documenting their songs and social behaviors. Her research revealed that whale families maintained bonds across thousands of miles, communicating through low-frequency calls that could travel entire ocean basins. One afternoon while diving near a coral reef off the coast of Baja California, Elena discovered something extraordinary beneath a rocky overhang:'''
|
||||
print(story, end='')
|
||||
" > "${PROMPT_FILE}"
|
||||
echo "Generated prompt file: ${PROMPT_FILE} ($(wc -c < "${PROMPT_FILE}") bytes)"
|
||||
|
||||
# Reset metrics to have a clean baseline
|
||||
echo "[Phase 3 / Step 1] Resetting LMCache metrics"
|
||||
curl -fsS -X POST "http://localhost:${LMCACHE_HTTP_PORT}/metrics/reset" >/dev/null
|
||||
echo "✅ Metrics reset"
|
||||
|
||||
# Start vLLM instance 1
|
||||
echo "[Phase 3 / Step 2] Starting vLLM (instance 1)"
|
||||
start_vllm
|
||||
|
||||
# Request A (first time) → should trigger store
|
||||
echo "[Phase 3 / Step 3] Request A (first) — expecting LMCache store"
|
||||
L1_WRITE_BEFORE=$(scrape_metric "lmcache_mp_l1_write_chunks_total")
|
||||
OUTPUT_1=$(send_completion "${PROMPT_FILE}" 50)
|
||||
echo "Output 1: ${OUTPUT_1}"
|
||||
sleep 2 # allow async store to complete
|
||||
L1_WRITE_AFTER=$(scrape_metric "lmcache_mp_l1_write_chunks_total")
|
||||
STORE_DELTA=$((L1_WRITE_AFTER - L1_WRITE_BEFORE))
|
||||
echo "L1 write chunks delta: ${STORE_DELTA}"
|
||||
if [ "${STORE_DELTA}" -lt 1 ]; then
|
||||
echo "❌ No L1 write activity after first request (expected store)"
|
||||
false
|
||||
fi
|
||||
echo "✅ LMCache store verified (${STORE_DELTA} chunks written)"
|
||||
|
||||
# Request A (second time, same vLLM instance) → should trigger read/hit
|
||||
echo "[Phase 3 / Step 4] Request A (second) — expecting LMCache hit"
|
||||
L1_READ_BEFORE=$(scrape_metric "lmcache_mp_l1_read_chunks_total")
|
||||
OUTPUT_2=$(send_completion "${PROMPT_FILE}" 50)
|
||||
echo "Output 2: ${OUTPUT_2}"
|
||||
sleep 2
|
||||
L1_READ_AFTER=$(scrape_metric "lmcache_mp_l1_read_chunks_total")
|
||||
READ_DELTA=$((L1_READ_AFTER - L1_READ_BEFORE))
|
||||
echo "L1 read chunks delta: ${READ_DELTA}"
|
||||
if [ "${READ_DELTA}" -lt 1 ]; then
|
||||
echo "❌ No L1 read activity on second request (expected cache hit)"
|
||||
false
|
||||
fi
|
||||
echo "✅ LMCache hit verified on same instance (${READ_DELTA} chunks read)"
|
||||
|
||||
# Restart vLLM
|
||||
echo "[Phase 3 / Step 5] Restarting vLLM (instance 2)"
|
||||
stop_vllm
|
||||
sleep 2
|
||||
start_vllm
|
||||
|
||||
# Request A (third time, new vLLM instance) → should trigger read/hit from LMCache
|
||||
echo "[Phase 3 / Step 6] Request A (third) — expecting LMCache hit after vLLM restart"
|
||||
L1_READ_BEFORE=$(scrape_metric "lmcache_mp_l1_read_chunks_total")
|
||||
OUTPUT_3=$(send_completion "${PROMPT_FILE}" 50)
|
||||
echo "Output 3: ${OUTPUT_3}"
|
||||
sleep 2
|
||||
L1_READ_AFTER=$(scrape_metric "lmcache_mp_l1_read_chunks_total")
|
||||
READ_DELTA=$((L1_READ_AFTER - L1_READ_BEFORE))
|
||||
echo "L1 read chunks delta: ${READ_DELTA}"
|
||||
if [ "${READ_DELTA}" -lt 1 ]; then
|
||||
echo "❌ No L1 read activity after vLLM restart (expected cross-instance cache hit)"
|
||||
false
|
||||
fi
|
||||
echo "✅ LMCache cross-instance hit verified (${READ_DELTA} chunks read)"
|
||||
|
||||
# Verify all three outputs are identical
|
||||
echo "[Phase 3 / Step 7] Verifying output consistency"
|
||||
if [ "${OUTPUT_1}" != "${OUTPUT_2}" ]; then
|
||||
echo "❌ Output mismatch between request 1 and request 2"
|
||||
echo " Output 1: ${OUTPUT_1}"
|
||||
echo " Output 2: ${OUTPUT_2}"
|
||||
false
|
||||
fi
|
||||
if [ "${OUTPUT_1}" != "${OUTPUT_3}" ]; then
|
||||
echo "❌ Output mismatch between request 1 and request 3 (after vLLM restart)"
|
||||
echo " Output 1: ${OUTPUT_1}"
|
||||
echo " Output 3: ${OUTPUT_3}"
|
||||
false
|
||||
fi
|
||||
echo "✅ All three outputs are identical — cache does not alter inference results"
|
||||
|
||||
# Negative test: a completely different prompt should NOT hit the cache
|
||||
echo "[Phase 3 / Step 8] Request B (different prompt) — expecting cache MISS"
|
||||
PROMPT_FILE_B="/tmp/build_${BUILD_ID}_phase3_prompt_b.txt"
|
||||
python3 -c "
|
||||
# A completely different prompt that shares no prefix with prompt A
|
||||
story_b = '''In the year 2147, humanity established its first permanent colony on Mars. The settlement, named Arcadia, housed three thousand researchers and engineers working to terraform the red planet. Chief botanist Dr. Yuki Tanaka spent her days in the greenhouse domes, cultivating genetically modified crops that could thrive in Martian soil. The atmospheric processors hummed day and night, slowly converting carbon dioxide into breathable oxygen. It was tedious work measured in decades, but the colonists were patient. Every morning, Dr. Tanaka checked her instruments and recorded the oxygen levels in her logbook. Today the readings showed'''
|
||||
print(story_b, end='')
|
||||
" > "${PROMPT_FILE_B}"
|
||||
L1_READ_BEFORE=$(scrape_metric "lmcache_mp_l1_read_chunks_total")
|
||||
OUTPUT_B=$(send_completion "${PROMPT_FILE_B}" 50)
|
||||
echo "Output B: ${OUTPUT_B}"
|
||||
wait_for_metric_change "lmcache_mp_l1_read_chunks_total" "${L1_READ_BEFORE}" 5 || true
|
||||
L1_READ_AFTER=$(scrape_metric "lmcache_mp_l1_read_chunks_total")
|
||||
READ_DELTA=$((L1_READ_AFTER - L1_READ_BEFORE))
|
||||
echo "L1 read chunks delta for new prompt: ${READ_DELTA}"
|
||||
if [ "${READ_DELTA}" -gt 0 ]; then
|
||||
echo "❌ Unexpected cache hit on a completely different prompt — metrics may be unreliable"
|
||||
false
|
||||
fi
|
||||
echo "✅ Cache miss confirmed for different prompt — metrics are trustworthy"
|
||||
|
||||
echo "[Phase 3 / Step 9] Cleaning up"
|
||||
stop_vllm
|
||||
cleanup_processes
|
||||
echo "✅ Phase 3 cleanup completed"
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "✅ All phases passed (Phase 1 + 2 + 3)"
|
||||
echo "=========================================="
|
||||
|
||||
# Make sure lmcache/vllm processes started in this run are reaped so
|
||||
# the next CI step does not collide on their default ZMQ/HTTP ports.
|
||||
cleanup_processes
|
||||
|
||||
# Upload artifacts BEFORE deleting the workspace
|
||||
upload_artifacts
|
||||
cleanup_workspace
|
||||
Reference in New Issue
Block a user