wehub/lmcache--lmcache
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:24:33 +08:00
commit f213ec8976
2101 changed files with 494002 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.buildkite/README.md
+15
View File
@@ -0,0 +1,15 @@
# Comprehensive Tests
An end-to-end integration suite for LMCache & vLLM latest branch.
## Layout
- **Scripts**: `scripts/vllm-integration-tests.sh`
- **Configs**: `configs/`
- **Pipeline**: `pipelines/comprehensive-tests.yml`
## Prepare
1. Add your YAMLs to `configs/` (e.g. `local_cpu.yaml`, `local_disk.yaml`).
2. Add the filenames to `cases/comprehensive-cases.txt`.
.buildkite/cases/comprehensive-cases.txt
+11
View File
@@ -0,0 +1,11 @@
local_cpu.yaml
local_cpu_mla.yaml
local_disk.yaml
pd.yaml
multi_device.yaml
async.yaml
p2p.yaml
layerwise.yaml
local_cpu_with_v3.yaml
local_disk_with_v3.yaml
p2p_with_v3.yaml
.buildkite/cases/integration-cases.txt
+1
View File
@@ -0,0 +1 @@
dummy.yaml
.buildkite/configs/async.yaml
+32
View File
@@ -0,0 +1,32 @@
workload:
type: long_doc_qa
max-inflight-requests: 20
sleep-time-after-warmup: 20
num-documents: 20
repeat-count: 1
hit-miss-ratio: 2:2
docker:
env:
- "LMCACHE_CHUNK_SIZE=256"
- "LMCACHE_LOCAL_CPU=False"
- "LMCACHE_MAX_LOCAL_CPU_SIZE=70"
- "LMCACHE_MAX_LOCAL_DISK_SIZE=70"
- "LMCACHE_LOCAL_DISK=\"file:///local/end-to-end-tests/local/\""
- "LMCACHE_ENABLE_ASYNC_LOADING=True"
- "LMCACHE_EXTRA_CONFIG={\"lookup_backoff_time\": 0.01, \"use_odirect\": True}"
- "LMCACHE_SAVE_UNFULL_CHUNK=False"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
vllm:
model: "meta-llama/Llama-3.1-8B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
checking-fields:
- query_round_time_per_prompt
.buildkite/configs/dummy.yaml
+22
View File
@@ -0,0 +1,22 @@
workload:
type: dummy
docker:
env:
- "LMCACHE_CHUNK_SIZE=256"
- "LMCACHE_LOCAL_CPU=True"
- "LMCACHE_MAX_LOCAL_CPU_SIZE=5"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
vllm:
model: "meta-llama/Llama-3.2-1B-Instruct"
args:
- "--max-model-len"
- "1024"
- "--gpu-memory-utilization"
- "0.35"
- "--load-format"
- "dummy"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
.buildkite/configs/layerwise.yaml
+26
View File
@@ -0,0 +1,26 @@
workload:
type: long_doc_qa
max-inflight-requests: 20
docker:
env:
- "LMCACHE_CHUNK_SIZE=256"
- "LMCACHE_LOCAL_CPU=True"
- "LMCACHE_MAX_LOCAL_CPU_SIZE=5"
- "LMCACHE_USE_LAYERWISE=true"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
vllm:
model: "meta-llama/Llama-3.2-1B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
- "--compilation-config"
- "{\"cudagraph_mode\":\"PIECEWISE\"}"
checking-fields:
- query_round_time_per_prompt
.buildkite/configs/lmcache_configs/local_cpu_mla.yaml
+15
View File
@@ -0,0 +1,15 @@
# LMCache Configuration File
# This replaces the individual environment variables for cleaner configuration
# Basic configurations
chunk_size: 256
local_cpu: true
max_local_cpu_size: 5
internal_api_server_enabled: true
internal_api_server_socket_path_prefix: "/tmp/lmcache_internal_api_server/socket"
internal_api_server_include_index_list: [1]
extra_config:
save_only_first_rank: true
first_rank_max_local_cpu_size: 5
.buildkite/configs/local_cpu.yaml
+23
View File
@@ -0,0 +1,23 @@
workload:
type: long_doc_qa
max-inflight-requests: 20
docker:
env:
- "LMCACHE_CHUNK_SIZE=256"
- "LMCACHE_LOCAL_CPU=True"
- "LMCACHE_MAX_LOCAL_CPU_SIZE=5"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
vllm:
model: "meta-llama/Llama-3.2-1B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
checking-fields:
- query_round_time_per_prompt
.buildkite/configs/local_cpu_mla.yaml
+30
View File
@@ -0,0 +1,30 @@
workload:
type: long_doc_qa
max-inflight-requests: 20
docker:
env:
- "LMCACHE_CONFIG_FILE=/etc/lmcache/local_cpu_mla.yaml"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
gpu_count: 2
vllm:
model: "deepseek-ai/DeepSeek-V2-Lite-Chat"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- '{"kv_connector":"LMCacheConnectorV1Dynamic","kv_role":"kv_both","kv_connector_module_path":"lmcache.integration.vllm.lmcache_connector_v1","kv_connector_extra_config":{"discard_partial_chunks":false}}'
- "--tensor-parallel-size"
- "2"
- "--block-size"
- "64"
- "--kv-cache-dtype"
- "fp8_e4m3"
- "--quantization"
- "fp8"
checking-fields:
- query_round_time_per_prompt
.buildkite/configs/local_cpu_with_v3.yaml
+25
View File
@@ -0,0 +1,25 @@
workload:
type: long_doc_qa
max-inflight-requests: 20
expected-latency-gain: true
docker:
env:
- "LMCACHE_CHUNK_SIZE=256"
- "LMCACHE_LOCAL_CPU=True"
- "LMCACHE_MAX_LOCAL_CPU_SIZE=5"
- "LMCACHE_USE_GPU_CONNECTOR_V3=True"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
vllm:
model: "meta-llama/Llama-3.2-1B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
checking-fields:
- query_round_time_per_prompt
.buildkite/configs/local_disk.yaml
+26
View File
@@ -0,0 +1,26 @@
workload:
type: long_doc_qa
max-inflight-requests: 20
sleep-time-after-warmup: 20
docker:
env:
- "LMCACHE_CHUNK_SIZE=256"
- "LMCACHE_LOCAL_CPU=False"
- "LMCACHE_MAX_LOCAL_CPU_SIZE=10"
- "LMCACHE_MAX_LOCAL_DISK_SIZE=10"
- "LMCACHE_LOCAL_DISK=\"file:///local/end-to-end-tests/local/\""
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
vllm:
model: "meta-llama/Llama-3.2-1B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
checking-fields:
- query_round_time_per_prompt
.buildkite/configs/local_disk_with_v3.yaml
+27
View File
@@ -0,0 +1,27 @@
workload:
type: long_doc_qa
max-inflight-requests: 20
sleep-time-after-warmup: 20
docker:
env:
- "LMCACHE_CHUNK_SIZE=256"
- "LMCACHE_LOCAL_CPU=False"
- "LMCACHE_MAX_LOCAL_CPU_SIZE=10"
- "LMCACHE_MAX_LOCAL_DISK_SIZE=10"
- "LMCACHE_LOCAL_DISK=\"file:///local/end-to-end-tests/local/\""
- "LMCACHE_USE_GPU_CONNECTOR_V3=True"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
vllm:
model: "meta-llama/Llama-3.2-1B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
checking-fields:
- query_round_time_per_prompt
.buildkite/configs/multi_device.yaml
+26
View File
@@ -0,0 +1,26 @@
workload:
type: long_doc_qa
max-inflight-requests: 20
docker:
env:
- "LMCACHE_CHUNK_SIZE=256"
- "LMCACHE_LOCAL_CPU=True"
- "LMCACHE_MAX_LOCAL_CPU_SIZE=5"
- "LMCACHE_GDS_PATH=/tmp/"
- "LMCACHE_GDS_BUFFER_SIZE=4096"
- "LMCACHE_EXTRA_CONFIG={\"use_direct_io\": true}"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
vllm:
model: "meta-llama/Llama-3.2-1B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
checking-fields:
- query_round_time_per_prompt
.buildkite/configs/p2p.yaml
+69
View File
@@ -0,0 +1,69 @@
workload:
type: long_doc_qa
num-documents: 20
max-inflight-requests: 1
repeat-count: 1
feature:
type: p2p
docker1:
env:
- "LMCACHE_MAX_LOCAL_CPU_SIZE=60"
- "LMCACHE_ENABLE_ASYNC_LOADING=True"
- "LMCACHE_ENABLE_P2P=True"
- "LMCACHE_P2P_HOST=localhost"
- "LMCACHE_P2P_INIT_PORTS=8200"
- "LMCACHE_P2P_LOOKUP_PORTS=8201"
- "LMCACHE_TRANSFER_CHANNEL=nixl"
- "LMCACHE_ENABLE_CONTROLLER=True"
- "LMCACHE_LMCACHE_INSTANCE_ID=lmcache_instance_1"
- "LMCACHE_LMCACHE_WORKER_PORTS=8500"
- "LMCACHE_EXTRA_CONFIG={\"lookup_backoff_time\": 0.001}"
- "LMCACHE_SAVE_UNFULL_CHUNK=False"
- "PYTHONHASHSEED=123"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
pull-port: 8300
reply-port: 8400
docker2:
env:
- "LMCACHE_MAX_LOCAL_CPU_SIZE=60"
- "LMCACHE_ENABLE_ASYNC_LOADING=True"
- "LMCACHE_ENABLE_P2P=True"
- "LMCACHE_P2P_HOST=localhost"
- "LMCACHE_P2P_INIT_PORTS=8202"
- "LMCACHE_P2P_LOOKUP_PORTS=8203"
- "LMCACHE_TRANSFER_CHANNEL=nixl"
- "LMCACHE_ENABLE_CONTROLLER=True"
- "LMCACHE_LMCACHE_INSTANCE_ID=lmcache_instance_2"
- "LMCACHE_LMCACHE_WORKER_PORTS=8501"
- "LMCACHE_EXTRA_CONFIG={\"lookup_backoff_time\": 0.001}"
- "LMCACHE_SAVE_UNFULL_CHUNK=False"
- "PYTHONHASHSEED=123"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
pull-port: 8300
reply-port: 8400
vllm1:
model: "meta-llama/Llama-3.1-8B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
vllm2:
model: "meta-llama/Llama-3.1-8B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
checking-fields:
- warmup_round_time_per_prompt
.buildkite/configs/p2p_with_v3.yaml
+71
View File
@@ -0,0 +1,71 @@
workload:
type: long_doc_qa
num-documents: 20
max-inflight-requests: 1
repeat-count: 1
feature:
type: p2p
docker1:
env:
- "LMCACHE_MAX_LOCAL_CPU_SIZE=60"
- "LMCACHE_ENABLE_ASYNC_LOADING=True"
- "LMCACHE_ENABLE_P2P=True"
- "LMCACHE_P2P_HOST=localhost"
- "LMCACHE_P2P_INIT_PORTS=8200"
- "LMCACHE_P2P_LOOKUP_PORTS=8201"
- "LMCACHE_TRANSFER_CHANNEL=nixl"
- "LMCACHE_ENABLE_CONTROLLER=True"
- "LMCACHE_LMCACHE_INSTANCE_ID=lmcache_instance_1"
- "LMCACHE_LMCACHE_WORKER_PORTS=8500"
- "LMCACHE_EXTRA_CONFIG={\"lookup_backoff_time\": 0.001}"
- "LMCACHE_SAVE_UNFULL_CHUNK=False"
- "PYTHONHASHSEED=123"
- "LMCACHE_USE_GPU_CONNECTOR_V3=True"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
pull-port: 8300
reply-port: 8400
docker2:
env:
- "LMCACHE_MAX_LOCAL_CPU_SIZE=60"
- "LMCACHE_ENABLE_ASYNC_LOADING=True"
- "LMCACHE_ENABLE_P2P=True"
- "LMCACHE_P2P_HOST=localhost"
- "LMCACHE_P2P_INIT_PORTS=8202"
- "LMCACHE_P2P_LOOKUP_PORTS=8203"
- "LMCACHE_TRANSFER_CHANNEL=nixl"
- "LMCACHE_ENABLE_CONTROLLER=True"
- "LMCACHE_LMCACHE_INSTANCE_ID=lmcache_instance_2"
- "LMCACHE_LMCACHE_WORKER_PORTS=8501"
- "LMCACHE_EXTRA_CONFIG={\"lookup_backoff_time\": 0.001}"
- "LMCACHE_SAVE_UNFULL_CHUNK=False"
- "PYTHONHASHSEED=123"
- "LMCACHE_USE_GPU_CONNECTOR_V3=True"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
pull-port: 8300
reply-port: 8400
vllm1:
model: "meta-llama/Llama-3.1-8B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
vllm2:
model: "meta-llama/Llama-3.1-8B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}"
checking-fields:
- warmup_round_time_per_prompt
.buildkite/configs/pd.yaml
+74
View File
@@ -0,0 +1,74 @@
workload:
type: long_doc_qa
max-inflight-requests: 1
repeat-count: 1
completion: true
num-documents: 20
document-length: 16382
feature:
type: pd
docker-prefiller:
env:
# Force vLLM Model Runner V1: MRV2 (enabled by default for Llama/Mistral
# dense models in vLLM nightly, see vllm-project/vllm#43458) is incompatible
# with KV cache connectors and hangs the PD path. Remove once MRV2 supports
# connectors / auto-falls back to MRV1.
- "VLLM_USE_V2_MODEL_RUNNER=0"
- "LMCACHE_LOCAL_CPU=False"
- "LMCACHE_MAX_LOCAL_CPU_SIZE=5"
- "LMCACHE_ENABLE_PD=True"
- "LMCACHE_TRANSFER_CHANNEL=nixl"
- "LMCACHE_PD_ROLE=sender"
- "LMCACHE_PD_PROXY_HOST=localhost"
- "LMCACHE_PD_BUFFER_SIZE=1073741824" # 1GB
- "LMCACHE_PD_BUFFER_DEVICE=cuda"
- "LMCACHE_SAVE_UNFULL_CHUNK=True"
- "PYTHONHASHSEED=0"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
proxy-port: 7500
docker-decoder:
env:
# Force vLLM Model Runner V1: MRV2 (enabled by default for Llama/Mistral
# dense models in vLLM nightly, see vllm-project/vllm#43458) is incompatible
# with KV cache connectors and hangs the PD path. Remove once MRV2 supports
# connectors / auto-falls back to MRV1.
- "VLLM_USE_V2_MODEL_RUNNER=0"
- "LMCACHE_LOCAL_CPU=False"
- "LMCACHE_MAX_LOCAL_CPU_SIZE=0"
- "LMCACHE_ENABLE_PD=True"
- "LMCACHE_TRANSFER_CHANNEL=nixl"
- "LMCACHE_PD_ROLE=receiver"
- "LMCACHE_PD_PEER_HOST=localhost"
- "LMCACHE_PD_BUFFER_SIZE=2147483648" # 2GB
- "LMCACHE_PD_BUFFER_DEVICE=cuda"
- "LMCACHE_SAVE_UNFULL_CHUNK=True"
- "PYTHONHASHSEED=0"
- "LMCACHE_INTERNAL_API_SERVER_ENABLED=True"
- "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket"
init-port: 7300
alloc-port: 7400
vllm-prefiller:
model: "meta-llama/Llama-3.2-1B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_producer\",\"kv_connector_extra_config\": {\"discard_partial_chunks\": false, \"lmcache_rpc_port\": \"producer1\"}}"
vllm-decoder:
model: "meta-llama/Llama-3.2-1B-Instruct"
args:
- "--load-format"
- "dummy"
- "--no-enable-prefix-caching"
- "--kv-transfer-config"
- "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_consumer\",\"kv_connector_extra_config\": {\"discard_partial_chunks\": false, \"lmcache_rpc_port\": \"consumer1\", \"skip_last_n_tokens\": 1}}"
checking-fields:
- query_round_time_per_prompt
.buildkite/correctness/README.md
+52
View File
@@ -0,0 +1,52 @@
# Correctness Test
This is a E2E testing suite for whether having a KV pass through LMCache from vllm (storing and loading) will degrade the accuracy of a transformer.
Unit tests do not suffice in going from request to response and passing through LMCache. We use offline serving
for this testing suite because the API Server of vllm is assumed to be unproblematic.
## Setup
We use the [Measuring Massive Multitask Language Understanding](https://arxiv.org/abs/2009.03300) multiple choice dataset downloaded as a compressed tarball in `data.tar.xz`
The following 3 options are viable:
A. Disable prefix caching on vllm to only use lmcache
B. Use lmcache centralized remote server
C. Use lmcache p2p sharing
We choose option A to keep the CI server lightweight (only 2x L4s needed).
We test one small model for each Attention Architecture. Currently dense/standard attention will use `meta-llama/Llama-3.1-8B-Instruct` and
MLA (multi-head latent attention) will use `deepseek-ai/DeepSeek-V2-Lite` with tensor parallel 2. We do not care about the objective accuracy on the MMLU benchmark but only on any differential that appears from using LMCache.
## CI Agent Pre Set-up
To ensure the speed of the MMLU Correctness tests, please conduct the following set up on your runner beforehand (only need to do once and `setup.sh` will renew your environment afterwards).
1. Create a virtual environment at `~/correctness_venv`
```bash
pip install pandas
```
2. Please run the following commands (these will be run by `setup.sh` every time):
```bash
cd ~
mkdir correctness_repositories
cd correctness_repositories
git clone https://github.com/LMCache/LMCache.git
git clone https://github.com/vllm-project/vllm.git
cd LMCache
pip install -e .
cd ../vllm
pip install -e .
```
Explanation: "pull" upstream code in this CI suite by pulling the latest upstream default branch
and let the editable virtual environment update the latest code. This allows a little bit of pre-setup to greatly optimize every run afterwards.
## Directory Structure
`single-mmlu-test.py`
The MMLU question dataset will be sent to this single endpoint and accuracy will be evaluated (correct answers are included
in the dataset). When LMCache is used, queries are sent twice.
.buildkite/correctness/async_request.py
Executable
+166
View File
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: Apache-2.0
# Standard
import argparse
import asyncio
import copy
import json
import logging
import os
import time
import traceback
# Third Party
from tqdm import tqdm
import aiohttp
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
async def make_request(session, payload: tuple, args):
endpoint = args.endpoint
output_file = args.output_file
obj_name, payload = payload
headers = {
"Authorization": f"Bearer {os.getenv('API_KEY', '')}",
"Content-Type": "application/json",
}
try:
async with session.post(
endpoint, json=payload, headers=headers, ssl=False
) as response:
json_data = await response.json()
content = (
json_data.get("choices", [{}])[0].get("message", {}).get("content", "")
)
logger.info(content)
if json_data.get("id") is not None:
with open(output_file, "a") as f:
f.write(f"{json_data.get('id')}:\n{content}\n")
if json_data.get("choices", [{}])[0].get("finish_reason", "") != "stop":
with open(output_file.replace(".txt", "_length.txt"), "a") as f:
f.write(f"{json_data.get('id')}:\n{content}\n")
return True, json_data
except Exception:
logger.error(f"Request failed (Object name: {obj_name})")
traceback.print_exc()
return False, None
async def process_requests(payloads: dict, args):
semaphore = asyncio.Semaphore(args.max_concurrency)
async def sem_task(session, payload: tuple, args):
async with semaphore:
return await make_request(session, payload, args)
timeout = aiohttp.ClientTimeout(total=6000)
connector = aiohttp.TCPConnector(limit=0)
async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session:
tasks = [sem_task(session, payload, args) for payload in payloads.items()]
generated_token = []
for coro in tqdm(
asyncio.as_completed(tasks), total=len(tasks), desc="Requests Progress"
):
success, content = await coro
if success:
generated_token.append(
content.get("usage", {}).get("completion_tokens", 0)
)
else:
logger.error(f"Error: {content}")
logger.info(f"Total generated tokens: {sum(generated_token)}")
def request_concurrency(requests: dict, args):
same_req = args.same_req
rid = args.rid
counts = args.counts
model = args.model
payloads = dict()
if same_req:
for index, request in requests.items():
if index == rid:
request.update({"model": model})
request.update({"stream": False})
request.update({"temperature": 0})
request.update({"request_id": index})
payloads[index] = request
for i in range(counts):
payloads[i] = copy.deepcopy(payloads[index])
payloads[i]["request_id"] = str(i)
break
else:
for index, request in requests.items():
request.update({"model": model})
request.update({"stream": False})
request.update({"temperature": 0})
request["request_id"] = index
payloads[index] = request
start = time.time()
asyncio.run(process_requests(payloads, args))
end = time.time()
logger.info(f"Total time: {end - start:.2f} seconds")
def main():
parser = argparse.ArgumentParser(description="Concurrent request tester")
parser.add_argument(
"--model", type=str, default="Qwen3/Qwen3-32B", help="Model name"
)
parser.add_argument(
"--endpoint",
type=str,
required=True,
help="API endpoint, e.g. http://host:port/v1/chat/completions",
)
parser.add_argument(
"--request-number",
type=int,
default=500,
help="Number of requests to read from dataset",
)
parser.add_argument(
"--max-concurrency", type=int, default=128, help="Max concurrency"
)
parser.add_argument(
"--dataset_file", type=str, default="data.json", help="Dataset json file"
)
parser.add_argument(
"--output-file",
type=str,
default="result_output.txt",
help="Output result file prefix",
)
parser.add_argument(
"--same-req",
action="store_true",
help="Send the same rid request multiple times",
)
parser.add_argument(
"--rid", type=str, default="", help="The rid to repeat if same_req=True"
)
parser.add_argument(
"--counts", type=int, default=100, help="How many times to send the same rid"
)
args = parser.parse_args()
# --- load dataset ---
with open(args.dataset_file, "r", encoding="utf-8") as f:
data = json.load(f)
data = dict(list(data.items())[: args.request_number])
print(f"Loaded {len(data)} requests.")
# --- call your function ---
request_concurrency(data, args)
if __name__ == "__main__":
main()
.buildkite/correctness/compare_files.py
+86
View File
@@ -0,0 +1,86 @@
# SPDX-License-Identifier: Apache-2.0
# Standard
import argparse
import re
def load_file(path):
data = {}
current_id = None
buffer = []
id_pattern = re.compile(r"chatcmpl-[a-zA-Z0-9\-_]+")
with open(path, "r", encoding="utf-8") as f:
for line in f:
line_strip = line.strip()
if id_pattern.search(line_strip):
if current_id is not None:
data[current_id] = "\n".join(buffer).strip()
current_id = id_pattern.search(line_strip).group(0)
buffer = []
else:
buffer.append(line.rstrip())
if current_id is not None:
data[current_id] = "\n".join(buffer).strip()
return data
def compare_files(file1, file2):
d1 = load_file(file1)
d2 = load_file(file2)
ids1 = set(d1.keys())
ids2 = set(d2.keys())
common = ids1 & ids2
only1 = ids1 - ids2
only2 = ids2 - ids1
same = []
diff = []
for cid in common:
if d1[cid] == d2[cid]:
same.append(cid)
else:
diff.append(cid)
print("====== Statistics ======")
print(f"Common IDs: {len(common)}")
print(f"Identical IDs: {len(same)}")
print(f"Different IDs: {len(diff)}")
print()
print("—— Identical IDs ——")
for cid in same:
print(cid)
print()
print("—— Different IDs ——")
for cid in diff:
print(cid)
print()
print("—— Only in File 1 ——")
for cid in only1:
print(cid)
print()
print("—— Only in File 2 ——")
for cid in only2:
print(cid)
def main():
parser = argparse.ArgumentParser(description="Compare two result files.")
parser.add_argument("--file1", type=str, required=True, help="Path to first file.")
parser.add_argument("--file2", type=str, required=True, help="Path to second file.")
args = parser.parse_args()
file1 = args.file1
file2 = args.file2
compare_files(file1, file2)
if __name__ == "__main__":
main()
.buildkite/correctness/mmlu-test.py
+189
View File
@@ -0,0 +1,189 @@
# SPDX-License-Identifier: Apache-2.0
# Standard
import argparse
import json
import os
import time
# Third Party
from tqdm import tqdm
from transformers import AutoTokenizer, set_seed
# vLLM
from vllm import LLM, SamplingParams
from vllm.config import KVTransferConfig
import numpy as np
import pandas as pd
# setting PYTHONHASHSEED derandomizes token chunking
os.environ["PYTHONHASHSEED"] = "0"
global tokenizer
choices = ["A", "B", "C", "D"]
# grab the idx'th row of the df and generate a prompt string
# format of the MMLU csvs:
# question,option_A,option_B,option_C,option_D,answer
def prompt_string(df, idx, include_answer=True):
prompt = df.iloc[idx, 0]
k = df.shape[1] - 2 # number of columns - 2 (question and answer)
for i in range(k):
prompt += f"\n{choices[i]}. {df.iloc[idx, i + 1]}"
prompt += "\nAnswer:"
if include_answer:
prompt += f" {df.iloc[idx, k]}\n\n"
return prompt
def evaluate(args, llm, subject, dev_df, test_df):
prompts, labels = [], []
shared_multi_shot_prefix = [
f'The following are multiple choice questions (with answers) \
about {subject}. \
You must respond with a single letter only. \
Either "A", "B", "C", or "D". \
Do not include any other text in your response. \n\n'
]
shared_multi_shot_prefix_length = 0
for i in range(dev_df.shape[0]):
# the multi-shot examples should contain answers
shared_multi_shot_prefix.append(prompt_string(dev_df, i))
# Use plain list of token IDs, no torch tensors
token_ids = tokenizer(shared_multi_shot_prefix[-1], add_special_tokens=True)[
"input_ids"
]
shared_multi_shot_prefix_length += len(token_ids)
if shared_multi_shot_prefix_length > 4000:
break
# all already have double newlines at the end
shared_multi_shot_prefix = "".join(shared_multi_shot_prefix)
for i in range(test_df.shape[0]):
# do NOT include the answer for the actual question
# we want the LLM to answer
query_prompt = prompt_string(test_df, i, include_answer=False)
prompt = f"{shared_multi_shot_prefix}\n\n{query_prompt}"
prompts.append(prompt)
label = test_df.iloc[i, test_df.shape[1] - 1]
labels.append(label)
# Create sampling params with deterministic settings
# (temperature=0, seed=42)
sampling_params = SamplingParams(
temperature=0,
max_tokens=2,
seed=42,
n=1,
stop=None,
)
# even though offline serving can batch all the prompts, we do them one at a time
# to keep the vllm scheduler deterministic
outputs = []
for prompt in prompts:
# if we use lmcache, we need to first populate the kv cache
if args.use_lmcache:
llm.generate(prompt, sampling_params)
time.sleep(0.5)
outputs.append(llm.generate(prompt, sampling_params))
time.sleep(0.5)
predictions = []
for output in outputs:
prediction = output.outputs[0].text
prediction_stripped = prediction.strip()
if prediction_stripped and prediction_stripped[0] in choices:
predictions.append(prediction_stripped[0])
else:
# Fallback: look for any A, B, C, D in the response
for char in prediction_stripped:
if char in ["A", "B", "C", "D"]:
predictions.append(char)
break
else:
predictions.append("A") # Default fallback
accuracy = np.mean(np.array(predictions) == np.array(labels))
return accuracy
def main(args):
global tokenizer
tokenizer = AutoTokenizer.from_pretrained(args.model)
ktc = None
if args.use_lmcache:
ktc = KVTransferConfig(
kv_connector="LMCacheConnectorV1",
kv_role="kv_both",
)
llm = LLM(
model=args.model,
max_model_len=8000,
gpu_memory_utilization=0.9,
kv_transfer_config=ktc,
tensor_parallel_size=2,
enable_prefix_caching=False,
enforce_eager=True,
trust_remote_code=True,
)
mmlu_files = os.listdir("data/test")
test_files = [f for f in mmlu_files if f.endswith("_test.csv")]
subjects = sorted([f.split("_test.csv")[0] for f in test_files])
accuracies = []
num_questions = []
output_dict = {}
for subject_raw in tqdm(
subjects[: args.number_of_subjects],
desc="Processing subjects",
):
subject = " ".join(subject_raw.split("_"))
dev_df = pd.read_csv(
os.path.join("data/dev", subject_raw + "_dev.csv"),
header=None,
)
test_df = pd.read_csv(
os.path.join("data/test", subject_raw + "_test.csv"),
header=None,
)
accuracy = evaluate(args, llm, subject, dev_df, test_df)
accuracies.append(accuracy)
num_questions.append(len(test_df))
output_dict[subject_raw] = {
"accuracy": accuracy,
"num_questions": len(test_df),
}
total_accuracy = np.mean(accuracies)
total_num_questions = sum(num_questions)
output_dict["total"] = {
"accuracy": total_accuracy,
"num_questions": total_num_questions,
}
with open(args.result_file, "w") as f:
# output will be a jsonl file
for subject, value in output_dict.items():
f.write(json.dumps({subject: value}) + "\n")
if __name__ == "__main__":
set_seed(42) # some tokenizers may have randomness
parser = argparse.ArgumentParser()
parser.add_argument("--model", type=str, required=True)
parser.add_argument("--number-of-subjects", type=int, default=25)
parser.add_argument("--use-lmcache", action="store_true", default=False)
args = parser.parse_args()
if args.use_lmcache:
args.result_file = args.model.split("/")[-1] + "-lmcache.jsonl"
else:
args.result_file = args.model.split("/")[-1] + "-baseline.jsonl"
main(args)
.buildkite/correctness/pipeline.correctness.yml
+16
View File
@@ -0,0 +1,16 @@
steps:
- label: ":rocket: vLLM correctness"
key: "correctness"
timeout_in_minutes: 60
commands:
- |
set -euo pipefail
echo "[INFO] HOME: $HOME"
echo "[INFO] PWD: $PWD"
chmod +x .buildkite/scripts/vllm-correctness.sh
BUILD_ID="${BUILDKITE_BUILD_ID}" .buildkite/scripts/vllm-correctness.sh
artifact_paths:
- "build_${BUILDKITE_BUILD_ID}.log"
.buildkite/correctness/pipeline.mmlu.yml
+36
View File
@@ -0,0 +1,36 @@
agents:
queue: "shaoting-gcp"
steps:
- label: "Run MMLU KV Transfer Test"
command: |
set -euo pipefail
echo "Starting MMLU test"
echo "Running setup.sh"
bash setup.sh
# buildkite agents start at the root of the Git checkout
cd .buildkite/correctness
echo "Starting Dense KV Transfer Test with meta-llama/Llama-3.1-8B-Instruct"
# produces Llama-3.1-8B-Instruct-baseline.jsonl
python mmlu-test.py --model meta-llama/Llama-3.1-8B-Instruct --number-of-subjects 25
# produces Llama-3.1-8B-Instruct-lmcache.jsonl
python mmlu-test.py --model meta-llama/Llama-3.1-8B-Instruct --number-of-subjects 25 --use-lmcache
echo "Starting MLA KV Transfer Test with deepseek-ai/DeepSeek-V2-Lite"
# produces DeepSeek-V2-Lite-baseline.jsonl
python mmlu-test.py --model deepseek-ai/DeepSeek-V2-Lite --number-of-subjects 25
# produces DeepSeek-V2-Lite-lmcache.jsonl
python mmlu-test.py --model deepseek-ai/DeepSeek-V2-Lite --number-of-subjects 25 --use-lmcache
echo "Summarizing Results"
# the summarize-results.py will look for jsonl files with the same techniqiue as the mmlu-test.py script
# which is with args.model.split("/")[-1]
python summarize-results.py meta-llama/Llama-3.1-8B-Instruct deepseek-ai/DeepSeek-V2-Lite
# this produces correctness-summary.txt
buildkite-agent artifact upload correctness-summary.txt
artifact_paths:
- correctness-summary.txt
.buildkite/correctness/setup.sh
+15
View File
@@ -0,0 +1,15 @@
#!/bin/bash
# buildkite agents start at the root of the Git checkout
cd .buildkite/correctness
unxz data.tar.xz
tar xf data.tar
# see README.md for pre-configuring your CI runner
source ~/correctness_venv/bin/activate
cd ~/correctness_repositories/LMCache
git pull origin dev
cd ~/correctness_repositories/vllm
git pull origin main
.buildkite/correctness/sharegpt2openai.py
+125
View File
@@ -0,0 +1,125 @@
# SPDX-License-Identifier: Apache-2.0
# Standard
import argparse
import json
VLLM_TEMPLATE = {
"model": "Qwen3/Qwen3-32B",
"frequency_penalty": 0.0,
"logit_bias": None,
"logprobs": False,
"top_logprobs": 0,
"max_tokens": 2048,
"max_completion_tokens": 2048,
"n": 1,
"presence_penalty": 0.0,
"response_format": None,
"seed": None,
"stop": [],
"stream": True,
"stream_options": None,
"temperature": 0.01,
"top_p": 1.0,
"tools": None,
"tool_choice": "none",
"reasoning_effort": None,
"include_reasoning": True,
"parallel_tool_calls": False,
"user": None,
"best_of": None,
"use_beam_search": False,
"top_k": -1,
"min_p": None,
"repetition_penalty": 1.0,
"length_penalty": 1.0,
"stop_token_ids": [],
"include_stop_str_in_output": False,
"ignore_eos": False,
"min_tokens": 0,
"skip_special_tokens": True,
"spaces_between_special_tokens": True,
"priority": 0,
"truncate_prompt_tokens": None,
"prompt_logprobs": None,
"allowed_token_ids": None,
"bad_words": [],
"order": "routine",
"lora_type": None,
"reasoning": False,
"echo": False,
"add_generation_prompt": True,
"continue_final_message": False,
"add_special_tokens": False,
"documents": None,
"chat_template": None,
"chat_template_kwargs": {"enable_thinking": False},
"mm_processor_kwargs": None,
"guided_json": None,
"guided_regex": None,
"guided_choice": None,
"guided_grammar": None,
"structural_tag": None,
"guided_decoding_backend": None,
"guided_whitespace_pattern": None,
"kv_transfer_params": None,
"vllm_xargs": None,
}
def convert_sharegpt_to_vllm(input_path, output_path):
with open(input_path, "r", encoding="utf-8") as f:
data = json.load(f)
output = {}
for item in data:
conv = item["conversations"]
# # Delete the gpt's response at the end
if len(conv) > 0 and conv[-1]["from"] == "gpt":
conv = conv[:-1]
messages = []
for c in conv:
role = "user" if c["from"] == "human" else "assistant"
content = c["value"]
messages.append({"role": role, "content": content})
file_key = f"{item['id']}"
out_entry = VLLM_TEMPLATE.copy()
out_entry["messages"] = messages
output[file_key] = out_entry
with open(output_path, "w", encoding="utf-8") as f:
json.dump(output, f, ensure_ascii=False, indent=2)
def main():
parser = argparse.ArgumentParser(
description="Convert ShareGPT JSON dataset to vLLM format."
)
parser.add_argument(
"--input",
"-i",
type=str,
required=True,
help="Path to the input ShareGPT JSON file.",
)
parser.add_argument(
"--output",
"-o",
type=str,
required=True,
help="Path to the output vLLM JSON file.",
)
args = parser.parse_args()
convert_sharegpt_to_vllm(args.input, args.output)
print(f"Converted {args.input} -> {args.output}")
if __name__ == "__main__":
main()
.buildkite/correctness/summarize-results.py
+216
View File
@@ -0,0 +1,216 @@
# SPDX-License-Identifier: Apache-2.0
# Standard
from typing import Dict, List, Tuple
import json
import os
import sys
def load_jsonl_file(filepath: str) -> Dict[str, Dict[str, float]]:
"""Load a JSONL file and return a dictionary of results."""
results: dict[str, dict[str, float]] = {}
if not os.path.exists(filepath):
print(f"Warning: File {filepath} not found")
return results
with open(filepath, "r") as f:
for line in f:
line = line.strip()
if line:
try:
data = json.loads(line)
results.update(data)
except json.JSONDecodeError:
print(f"Error parsing line in {filepath}: {line}")
continue
return results
def compare_results(
baseline_results: Dict, lmcache_results: Dict
) -> Tuple[List[Tuple], float, float]:
"""Compare baseline and lmcache results, return comparison data and totals."""
comparisons = []
baseline_total_correct = 0
baseline_total_questions = 0
lmcache_total_correct = 0
lmcache_total_questions = 0
# Get all subjects (excluding 'total' for now)
all_subjects = set(baseline_results.keys()) | set(lmcache_results.keys())
all_subjects.discard("total") # Handle total separately
# Sort subjects alphabetically
for subject in sorted(all_subjects):
baseline_data = baseline_results.get(
subject, {"accuracy": 0.0, "num_questions": 0}
)
lmcache_data = lmcache_results.get(
subject, {"accuracy": 0.0, "num_questions": 0}
)
baseline_acc = baseline_data["accuracy"]
lmcache_acc = lmcache_data["accuracy"]
difference = lmcache_acc - baseline_acc
# Use the number of questions from whichever dataset has the subject
num_questions = max(
baseline_data["num_questions"], lmcache_data["num_questions"]
)
comparisons.append(
(subject, baseline_acc, lmcache_acc, difference, num_questions)
)
# Accumulate totals
baseline_total_correct += baseline_acc * num_questions
baseline_total_questions += num_questions
lmcache_total_correct += lmcache_acc * num_questions
lmcache_total_questions += num_questions
# Calculate overall accuracies
baseline_total_acc = (
baseline_total_correct / baseline_total_questions
if baseline_total_questions > 0
else 0
)
lmcache_total_acc = (
lmcache_total_correct / lmcache_total_questions
if lmcache_total_questions > 0
else 0
)
return comparisons, baseline_total_acc, lmcache_total_acc
def format_model_report(
model_name: str,
model_short_name: str,
comparisons: List[Tuple],
baseline_total: float,
lmcache_total: float,
) -> str:
"""Format a single model's comparison report."""
report = []
report.append("=" * 80)
report.append(f"MODEL: {model_name}")
report.append("=" * 80)
report.append("")
# Header
report.append(
f"{'Subject':<35} \
{'Baseline':<12} \
{'LMCache':<12} \
{'Difference':<12} \
{'Questions':<10}"
)
report.append("-" * 80)
# Subject comparisons
for subject, baseline_acc, lmcache_acc, difference, num_questions in comparisons:
report.append(
f"{subject:<35} \
{baseline_acc:<12.4f} \
{lmcache_acc:<12.4f} \
{difference:<12.4f} \
{num_questions:<10}"
)
# Totals
total_difference = lmcache_total - baseline_total
report.append("-" * 80)
report.append(
f"{'TOTAL':<35} \
{baseline_total:<12.4f} \
{lmcache_total:<12.4f} \
{total_difference:<12.4f}"
)
report.append("")
# Summary
report.append("SUMMARY:")
report.append(f" Baseline Total Accuracy: {baseline_total:.4f}")
report.append(f" LMCache Total Accuracy: {lmcache_total:.4f}")
report.append(f" Net Accuracy Difference: {total_difference:.4f}")
if total_difference >= 0:
report.append(f" Result: LMCache performs BETTER by {total_difference:.4f}")
else:
report.append(
f" Result: LMCache performs WORSE by {abs(total_difference):.4f}"
)
report.append("")
return "\n".join(report)
def main():
if len(sys.argv) < 2:
print("Usage: python summarize-results.py <model1> <model2> ...")
print(
"Example: python summarize-results.py \
meta-llama/Llama-3.1-8B-Instruct \
deepseek-ai/DeepSeek-V2-Lite"
)
sys.exit(1)
model_names = sys.argv[1:]
all_reports = []
print(f"Processing {len(model_names)} models...")
for model_name in model_names:
print(f"Processing model: {model_name}")
# Extract short name using the same logic as mmlu-test.py
model_short_name = model_name.split("/")[-1]
# Define file paths
baseline_file = f"{model_short_name}-baseline.jsonl"
lmcache_file = f"{model_short_name}-lmcache.jsonl"
print(f" Looking for files: {baseline_file}, {lmcache_file}")
# Load results
baseline_results = load_jsonl_file(baseline_file)
lmcache_results = load_jsonl_file(lmcache_file)
if not baseline_results and not lmcache_results:
print(f" Warning: No data found for {model_name}")
continue
# Compare results
comparisons, baseline_total, lmcache_total = compare_results(
baseline_results, lmcache_results
)
# Generate report for this model
model_report = format_model_report(
model_name,
model_short_name,
comparisons,
baseline_total,
lmcache_total,
)
all_reports.append(model_report)
print(f" Processed {len(comparisons)} subjects")
# Write combined report to file
output_file = "correctness-summary.txt"
with open(output_file, "w") as f:
f.write("MMLU Correctness Comparison Report\n")
f.write("=" * 80 + "\n")
f.write(f"Generated for {len(model_names)} models\n\n")
for report in all_reports:
f.write(report)
f.write("\n")
print(f"\nReport written to {output_file}")
if __name__ == "__main__":
main()
.buildkite/k3_harness/ARCHITECTURE.md
+89
View File
@@ -0,0 +1,89 @@
# K3s CI Harness Architecture
This document outlines the main motivations, design decisions, and architecture of the K3s-based Buildkite CI Harness for LMCache.
## Motivations
1. **Ease of Setup**: Bring up a fully functional CI node on any raw Linux GPU machine with a single script (`setup-cluster.sh`), avoiding manual configuration of dependencies or persistent Buildkite agents.
2. **Machine Agnostic**: Remove reliance on machine-specific scripts (e.g., custom `pick-free-gpu.sh` scripts). The CI runs identically on any machine with an NVIDIA GPU and Docker.
3. **Clean Environments**: Eliminate state bleeding between test runs (no shared pip/uv caches or leftover files). Every job gets a pristine environment.
4. **Automated Resource Management**: Leverage standard Kubernetes primitives for GPU allocation, volume mounting, and cleanup.
## Core Design Decisions
### 1. K3s + GPU Operator
We use **K3s** as a lightweight, single-node Kubernetes cluster. The **NVIDIA GPU Operator** automatically configures the container runtime and exposes GPUs to K8s. This abstracts away the underlying hardware and handles GPU discovery automatically.
### 2. Ephemeral Pod-Based Execution (`agent-stack-k8s`)
Instead of running persistent `buildkite-agent` binaries, we use Buildkite's `agent-stack-k8s`. A controller pod polls the Buildkite queue and dynamically provisions an ephemeral K8s Pod to run each job. When the job finishes, the pod is destroyed.
### 3. Declarative GPU Allocation & Parallelizability
Each pipeline step explicitly requests the number of GPUs it needs in its Kubernetes pod specification:
```yaml
resources:
limits:
nvidia.com/gpu: "2" # Requests 2 GPUs
```
Because Kubernetes handles atomic resource scheduling, **jobs automatically run in parallel on multi-GPU nodes**. If a node has 4 GPUs and three jobs arrive (requesting 1, 2, and 1 GPU respectively), K8s schedules them concurrently. If another job requests 2 GPUs, it waits in the queue until resources free up. This completely eliminates the need for manual GPU locking mechanisms.
### 4. Local Base Image & Ephemeral Setup
To avoid Docker registry dependencies, `setup-cluster.sh` automatically detects the host GPU's compute capability, builds the `lmcache/ci-base:latest` image locally, and imports it directly into K3s. Each job pod uses this base image and dynamically installs vLLM and LMCache (`setup-env.sh`) at runtime.
### 5. Shared Host Volumes
Large, read-heavy directories (like HuggingFace models and datasets) are mounted into pods via `hostPath` to speed up tests without duplicating downloads, while keeping the job environment itself stateless.
---
## Architecture Flow
### Node Initialization (One-Time Setup)
```text
[ Raw Linux Host + NVIDIA GPU ]
|
| (run setup-cluster.sh)
v
+-----------------------------------------------------+
| K3s Cluster |
| |
| 1. Install K3s |
| 2. Install Helm -> Deploy NVIDIA GPU Operator |
| 3. Build CI Base Image (lmcache/ci-base:latest) |
| 4. Import Base Image to K3s containerd |
+-----------------------------------------------------+
|
| (run install-agent-stack.sh)
v
[ agent-stack-k8s Controller Pod (Watches queue) ]
```
### CI Execution Flow
```text
Buildkite Web UI
|
| 1. Job pushed to 'k8s' queue
v
agent-stack-k8s Controller (in K3s)
|
| 2. Reads job, creates ephemeral Pod requesting GPUs
v
+-------------------------------------------------------------+
| Job Pod (e.g., limits: nvidia.com/gpu: "1") |
| |
| - Mounts /data/huggingface from host |
| - Runs setup-env.sh (Installs vLLM, LMCache) |
| - Executes test script (e.g., pytest) |
+-------------------------------------------------------------+
|
| 3. Test finishes (Pass/Fail)
v
agent-stack-k8s Controller
|
| 4. Reports result to Buildkite
| 5. Destroys the Pod (wipes environment)
v
Buildkite Web UI
```
.buildkite/k3_harness/README.md
+143
View File
@@ -0,0 +1,143 @@
# K3s CI Harness
Single-node K3s + NVIDIA GPU Operator for running LMCache CI on any GPU machine.
## Prerequisites
- Linux host with NVIDIA GPU(s) and driver installed
- Docker (for building the CI base image)
- Root access
- A GitHub token (PAT or fine-grained) with read/write access to the repo
## Setup (one-time)
```bash
# 1. Install K3s + GPU Operator + build CI base image
.buildkite/k3_harness/setup-cluster.sh
# 2. Verify GPUs work in pods
.buildkite/k3_harness/smoke-test.sh
# 3. Connect to Buildkite (needs agent token + GitHub token for HTTPS checkout)
.buildkite/k3_harness/install-agent-stack.sh <BUILDKITE_AGENT_TOKEN> <GITHUB_TOKEN>
```
`setup-cluster.sh` does everything: installs K3s, Helm, GPU Operator, builds the CI base image from `ci-base.Dockerfile`, imports it into K3s containerd locally, and creates host volume directories. Safe to re-run.
## How Buildkite integration works
This is different from the bare-metal agent setup where you install a Buildkite agent binary on your own machine, register it with a queue created on the Web UI, and manage it as a systemd service. With agent-stack-k8s, there are **no persistent agents on the machine**.
Instead, agent-stack-k8s runs a controller pod in K8s that polls Buildkite for jobs matching the configured queue. When a job appears, it creates a K8s pod to run it. When the job finishes, the pod is deleted.
**What you need from the Buildkite web UI**:
1. **Create a queue** — Go to **Organization Settings → Default cluster → Queues → New Queue** and create a queue named `k8s` (or your chosen name). The queue needs no configuration — just a name. You do **not** register any agents on it.
2. **Get an agent token** — From the cluster settings page, copy the agent token (or create a new one).
3. **Get a GitHub token** — Create a PAT (or fine-grained token) with read/write access to the repo. This is used for HTTPS checkout and for pushing baselines to `benchmarks-main`.
4. **Run `install-agent-stack.sh`** with the agent token and GitHub token. The script creates a K8s secret and installs agent-stack-k8s.
To use a queue name other than `k8s`, set `BUILDKITE_QUEUE` when running install:
```bash
BUILDKITE_QUEUE=my-queue .buildkite/k3_harness/install-agent-stack.sh <AGENT_TOKEN> <GITHUB_TOKEN>
```
Pipeline steps target the queue with:
```yaml
agents:
queue: "k8s" # must match the queue name
```
## Per-job environment
Every CI job sources `setup-env.sh` to install vLLM + LMCache:
```yaml
command: |
source .buildkite/k3_harness/setup-env.sh
bash .buildkite/scripts/my-test.sh
```
This installs:
1. **vLLM** from nightly wheel
2. **LMCache** from PR source
Each pod has its own ephemeral filesystem that is cleanly erased after being shut down — no shared pip/uv cache, no cross-pod contention.
## Shared volumes
Mounted into every pod via `hostPath`:
| Host | Container | What |
|------|-----------|------|
| `/data/huggingface` | `/root/.cache/huggingface` | Model weights (read-heavy, write-once) |
| `/data/datasets` | `/root/correctness` | Test datasets (read-only after download) |
## GPU allocation
Request GPUs per pipeline step:
```yaml
plugins:
- kubernetes:
podSpec:
containers:
- resources:
limits:
nvidia.com/gpu: "2" # 1 or 2
```
K8s device plugin handles atomic allocation. No `pick-free-gpu.sh`.
## CI base image
`ci-base.Dockerfile` builds an image with CUDA + Python 3.12 + uv + build deps (no vLLM/LMCache). `setup-cluster.sh` builds it automatically, auto-detects your GPU's compute capability for `TORCH_CUDA_ARCH_LIST`, and imports it into K3s containerd — no registry needed.
To rebuild after changing `requirements/*.txt`:
```bash
REBUILD_IMAGE=1 .buildkite/k3_harness/setup-cluster.sh
```
## Files
```
k3_harness/
├── ci-base.Dockerfile # CI base image definition
├── setup-cluster.sh # One-time: K3s + GPU Operator + base image
├── install-agent-stack.sh # One-time: Buildkite agent (needs token + GitHub token)
├── values.yaml # Reference Helm values (documentation only)
├── setup-env.sh # Per-job: install vLLM + LMCache (includes GPU health check)
├── smoke-test.sh # Verify: GPU pod runs in K3s
├── gpu-monitor.sh # Host-level: detect stale non-K8s GPU processes
├── setup-gpu-monitor.sh # Install gpu-monitor.sh as a cron job
├── teardown.sh # Remove everything (preserves /data/*)
└── README.md
```
## GPU monitoring
CI jobs automatically check GPU health at startup (via `check_gpu_health` in `setup-env.sh`). If a GPU assigned to a pod has less than 80% free memory — usually due to a stale host process — the job fails immediately with a clear message instead of wasting time on setup and then getting a cryptic CUDA OOM.
For **host-level monitoring**, install the GPU monitor cron job:
```bash
# Install cron job that checks every 10 minutes for stale non-K8s GPU processes
sudo bash .buildkite/k3_harness/setup-gpu-monitor.sh
```
This runs `gpu-monitor.sh` every 10 minutes. It distinguishes K8s pod processes (in `kubepods` cgroups) from stale host processes and logs warnings to `/var/log/gpu-monitor.log` with PID, command, GPU, memory usage, and process age.
```bash
# View monitor logs
tail -f /var/log/gpu-monitor.log
# Remove the cron job
crontab -l 2>/dev/null | grep -v gpu-monitor.sh | crontab -
```
## Teardown
```bash
.buildkite/k3_harness/teardown.sh
# Removes K3s, GPU Operator, agent-stack. Preserves /data/*
```
.buildkite/k3_harness/ci-base.Dockerfile
+40
View File
@@ -0,0 +1,40 @@
# CI base image: CUDA + Python + uv + build deps.
# No vLLM or LMCache — those are installed per-job by setup-env.sh.
#
# Built automatically by setup-cluster.sh and imported into K3s containerd.
# Rebuild when requirements/*.txt changes.
FROM nvidia/cuda:13.0.2-devel-ubuntu24.04
ENV DEBIAN_FRONTEND=noninteractive
ENV PATH="/opt/venv/bin:${PATH}"
RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
&& echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
&& apt-get update -y \
&& apt-get install -y --no-install-recommends \
ccache software-properties-common git curl sudo jq lsof \
python3 python3-dev python3-venv python3-pip tzdata libxcb1-dev \
libcudart12 \
&& ldconfig \
&& curl -LsSf https://astral.sh/uv/install.sh | sh \
&& mv ~/.local/bin/uv /usr/local/bin/ \
&& mv ~/.local/bin/uvx /usr/local/bin/ \
&& uv venv /opt/venv \
&& . /opt/venv/bin/activate \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /workspace
# Pre-install requirements that rarely change
COPY requirements/common.txt requirements/build.txt requirements/cuda.txt \
requirements/cuda13_core.txt requirements/nixl.txt /tmp/reqs/
RUN . /opt/venv/bin/activate && \
uv pip install -r /tmp/reqs/cuda.txt && \
uv pip install -r /tmp/reqs/build.txt && \
rm -rf /tmp/reqs
# Set at build time to match the CI machine's GPU.
# Query with: nvidia-smi --query-gpu=compute_cap --format=csv,noheader
ARG TORCH_CUDA_ARCH_LIST
ENV TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST}
.buildkite/k3_harness/gpu-monitor.sh
Executable
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env bash
# Host-level GPU monitoring script.
# Detects non-K8s processes holding GPU memory and logs warnings.
#
# Install as a cron job on the CI machine:
# sudo crontab -e
# */10 * * * * /path/to/gpu-monitor.sh >> /var/log/gpu-monitor.log 2>&1
#
# Or install with setup-gpu-monitor.sh for automatic setup.
set -euo pipefail
LOG_PREFIX="[gpu-monitor $(date '+%Y-%m-%d %H:%M:%S')]"
K8S_PIDS_FILE=$(mktemp)
trap 'rm -f "$K8S_PIDS_FILE"' EXIT
if ! command -v nvidia-smi &>/dev/null; then
echo "$LOG_PREFIX ERROR: nvidia-smi not found"
exit 1
fi
# Collect all PIDs using GPU compute
mapfile -t GPU_PIDS < <(
nvidia-smi --query-compute-apps=pid --format=csv,noheader,nounits 2>/dev/null \
| tr -d ' ' | sort -u | grep -v '^$'
)
if [[ ${#GPU_PIDS[@]} -eq 0 ]]; then
# No GPU processes -- all clean
exit 0
fi
# Collect K8s container PIDs (all processes running inside any K8s pod).
# K3s uses containerd; enumerate all container cgroups.
{
shopt -s nullglob
for cgroup_dir in /sys/fs/cgroup/*/kubepods* /sys/fs/cgroup/kubepods*; do
if [[ -d "$cgroup_dir" ]]; then
find "$cgroup_dir" -name "cgroup.procs" -exec cat {} \; 2>/dev/null
fi
done
shopt -u nullglob
} | sort -u > "$K8S_PIDS_FILE" 2>/dev/null || true
# Also check via crictl if available (more reliable)
if command -v crictl &>/dev/null; then
for container_id in $(crictl ps -q 2>/dev/null); do
pid=$(crictl inspect --output go-template --template '{{.info.pid}}' "$container_id" 2>/dev/null || true)
if [[ -n "$pid" && "$pid" != "0" ]]; then
# Include the container PID and all its descendants
echo "$pid" >> "$K8S_PIDS_FILE"
pgrep -P "$pid" --ns "$pid" 2>/dev/null >> "$K8S_PIDS_FILE" || true
fi
done
fi
# Check each GPU process
STALE_FOUND=false
for pid in "${GPU_PIDS[@]}"; do
[[ -z "$pid" ]] && continue
# Is this PID a K8s process?
if grep -qw "$pid" "$K8S_PIDS_FILE" 2>/dev/null; then
continue
fi
# Check if this PID is inside any cgroup containing "kubepods"
if [[ -f "/proc/$pid/cgroup" ]] && grep -q "kubepods" "/proc/$pid/cgroup" 2>/dev/null; then
continue
fi
# This is a non-K8s process using a GPU
STALE_FOUND=true
local_cmdline=$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | head -c 200 || echo "<unknown>")
echo "$LOG_PREFIX STALE GPU PROCESS: PID=$pid cmd='$local_cmdline'"
nvidia-smi --query-compute-apps=pid,gpu_bus_id,used_memory --format=csv,noheader 2>/dev/null \
| grep "^${pid}," | while IFS=, read -r _ bus mem; do
echo "$LOG_PREFIX GPU bus=$bus memory=$mem"
done
# Check process age
if [[ -f "/proc/$pid/stat" ]]; then
local_start=$(awk '{print $22}' "/proc/$pid/stat" 2>/dev/null || echo "0")
local_uptime=$(awk '{print int($1)}' /proc/uptime)
local_clk_tck=$(getconf CLK_TCK)
if [[ "$local_start" -gt 0 && "$local_clk_tck" -gt 0 ]]; then
local_age_secs=$(( local_uptime - local_start / local_clk_tck ))
local_age_hours=$(( local_age_secs / 3600 ))
echo "$LOG_PREFIX Age: ~${local_age_hours}h (${local_age_secs}s)"
fi
fi
done
if [[ "$STALE_FOUND" == "true" ]]; then
echo "$LOG_PREFIX"
echo "$LOG_PREFIX === GPU SUMMARY ==="
nvidia-smi --query-gpu=index,memory.used,memory.total,memory.free --format=csv,noheader \
| while IFS=, read -r idx used total free; do
echo "$LOG_PREFIX GPU $idx: used=$(echo $used | xargs), total=$(echo $total | xargs), free=$(echo $free | xargs)"
done
echo "$LOG_PREFIX WARNING: Stale non-K8s processes are holding GPU memory. CI tests may fail."
echo "$LOG_PREFIX To fix: kill the stale PIDs listed above."
exit 1
fi
.buildkite/k3_harness/install-agent-stack.sh
Executable
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env bash
# Install Buildkite agent-stack-k8s with GitHub token for HTTPS repo access.
#
# Usage:
# install-agent-stack.sh <BUILDKITE_AGENT_TOKEN> <GITHUB_TOKEN>
#
# Arguments:
# BUILDKITE_AGENT_TOKEN — from Buildkite cluster settings
# GITHUB_TOKEN — GitHub PAT or fine-grained token with repo read/write
#
# The queue name defaults to "k8s". Override with BUILDKITE_QUEUE env var.
set -euo pipefail
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
TOKEN="${1:?Usage: $0 <BUILDKITE_AGENT_TOKEN> <GITHUB_TOKEN>}"
GH_TOKEN="${2:?Usage: $0 <BUILDKITE_AGENT_TOKEN> <GITHUB_TOKEN>}"
QUEUE="${BUILDKITE_QUEUE:-k8s}"
# Create (or update) the K8s secret with GitHub credentials.
# Two keys:
# .git-credentials — used by agent-stack-k8s checkout container (HTTPS clone).
# The leading dot matches the file git-credential-store expects.
# GITHUB_TOKEN — injected into job containers for git push (e.g. baselines)
kubectl create secret generic buildkite-git-creds \
--from-literal=.git-credentials="https://x-access-token:${GH_TOKEN}@github.com" \
--from-literal=GITHUB_TOKEN="${GH_TOKEN}" \
-n buildkite --dry-run=client -o yaml | kubectl apply -f -
# Install or upgrade agent-stack-k8s
# - default-checkout-params.gitCredentialsSecret: checkout container uses HTTPS
# instead of SSH. Expects a Secret with a `.git-credentials` key.
# - GITHUB_TOKEN is injected per-step in pipeline.yml (not via global pod-spec-patch)
helm upgrade --install agent-stack-k8s oci://ghcr.io/buildkite/helm/agent-stack-k8s \
--version 0.38.0 \
--namespace buildkite --create-namespace \
--set agentToken="${TOKEN}" \
--set config.queue="${QUEUE}" \
--set-json 'config.default-checkout-params={"gitCredentialsSecret":{"secretName":"buildkite-git-creds"}}' \
--wait --timeout 3m
echo "agent-stack-k8s installed (queue=${QUEUE})"
kubectl get pods -n buildkite
.buildkite/k3_harness/resolve-pinned-vllm.sh
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env bash
# Resolve the vLLM nightly version that this build should install.
#
# Resolution order (first non-empty wins):
# 1. PINNED_VLLM_VERSION env var -- explicit per-build override.
# 2. latest_tested_vllm.txt fetched from
# https://raw.githubusercontent.com/LMCache/LMCache/buildkite_latest_tested_vllm/latest_tested_vllm.txt
# -- the most recent vLLM nightly that the canary build verified.
# 3. Empty string -- caller falls back to "latest nightly".
#
# The pin file's first non-blank, non-comment line is the bare version
# (kept this way so older `head -n1` consumers still work). Subsequent
# key=value lines carry pre-resolved metadata so consumers can install
# the wheel without making any extra API calls:
# short_sha=<short-commit-sha>
# full_sha=<40-char-commit-sha>
# archive_index_url=https://wheels.vllm.ai/<full-sha>/cu130
#
# Toggles:
# USE_PINNED_VLLM=false -- skip step 2 (always probe the latest nightly).
# Used by the canary build itself, since pinning
# to its own previous result would defeat the
# purpose of a freshness check.
#
# Usage:
# source .buildkite/k3_harness/resolve-pinned-vllm.sh
# echo "Resolved: ${PINNED_VLLM_VERSION:-<unpinned, using nightly>}"
# echo "Archive : ${PINNED_VLLM_ARCHIVE_INDEX_URL:-<none>}"
#
# After sourcing the following are set (possibly empty) and exported:
# PINNED_VLLM_VERSION -- e.g. 0.23.1rc1.dev508+gc6dd32a81
# PINNED_VLLM_FULL_SHA -- 40-char commit SHA, if recorded in pin
# PINNED_VLLM_ARCHIVE_INDEX_URL
# -- permanent PEP 503 simple index, if recorded
#
# The script never fails the build: a missing/unreachable pin file just
# falls through to the unpinned path, mirroring the previous behaviour.
# Allow re-sourcing without "unbound variable" complaints under set -u.
PINNED_VLLM_VERSION="${PINNED_VLLM_VERSION:-}"
PINNED_VLLM_ARCHIVE_INDEX_URL="${PINNED_VLLM_ARCHIVE_INDEX_URL:-}"
PINNED_VLLM_FULL_SHA="${PINNED_VLLM_FULL_SHA:-}"
USE_PINNED_VLLM="${USE_PINNED_VLLM:-true}"
# Override URL if you mirror the pin file elsewhere (e.g. an internal
# raw-file proxy for offline CI).
LMCACHE_VLLM_PIN_URL="${LMCACHE_VLLM_PIN_URL:-https://raw.githubusercontent.com/LMCache/LMCache/buildkite_latest_tested_vllm/latest_tested_vllm.txt}"
if [[ -z "${PINNED_VLLM_VERSION}" && "${USE_PINNED_VLLM}" == "true" ]]; then
if command -v curl >/dev/null 2>&1; then
# 5s connect, 10s total -- pin lookup must never dominate setup time.
fetched="$(curl -fsSL --connect-timeout 5 --max-time 10 \
"${LMCACHE_VLLM_PIN_URL}" 2>/dev/null || true)"
# The pin file's first non-blank, non-comment line is the bare
# version (back-compat for older readers); subsequent key=value
# lines carry resolved metadata so consumers can skip the live
# GitHub API lookup. A single awk pass extracts everything.
# Empty / missing keys are tolerated -- we'll just fall back to
# resolving them on demand later in the pipeline.
eval "$(printf '%s\n' "${fetched}" | awk '
BEGIN { ver=""; idx=""; sha="" }
/^[[:space:]]*(#|$)/ { next }
ver == "" {
line=$0
sub(/[[:space:]]+$/, "", line)
ver=line
next
}
/^archive_index_url=/ {
v=$0; sub(/^archive_index_url=/, "", v); idx=v; next
}
/^full_sha=/ {
v=$0; sub(/^full_sha=/, "", v); sha=v; next
}
END {
printf("PINNED_VLLM_VERSION=%s\n", ver)
printf("PINNED_VLLM_ARCHIVE_INDEX_URL=%s\n", idx)
printf("PINNED_VLLM_FULL_SHA=%s\n", sha)
}
')"
fi
fi
export PINNED_VLLM_VERSION
export PINNED_VLLM_ARCHIVE_INDEX_URL
export PINNED_VLLM_FULL_SHA
if [[ -n "${PINNED_VLLM_VERSION}" ]]; then
echo "[resolve-pinned-vllm] Pinned vLLM version: ${PINNED_VLLM_VERSION}" >&2
if [[ -n "${PINNED_VLLM_ARCHIVE_INDEX_URL}" ]]; then
echo "[resolve-pinned-vllm] Archive index:" \
"${PINNED_VLLM_ARCHIVE_INDEX_URL}" >&2
fi
else
echo "[resolve-pinned-vllm] No pinned vLLM; will install latest nightly" >&2
fi
.buildkite/k3_harness/setup-blend-env.sh
Executable
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env bash
# Per-job environment setup for CacheBlend-plugin compatibility tests.
set -euo pipefail
trap 'echo "ERROR: setup-blend-env.sh failed at line $LINENO (exit code $?)" >&2' ERR
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
# 1. Full env: vLLM nightly + LMCache(PR) from source (+ c_ops for this GPU).
# setup-env.sh already runs the GPU health pre-check.
source "${REPO_ROOT}/.buildkite/k3_harness/setup-env.sh"
# setup-env.sh installed its own ERR trap; restore ours for accurate attribution.
trap 'echo "ERROR: setup-blend-env.sh failed at line $LINENO (exit code $?)" >&2' ERR
# 2. Plugin-harness dep not in the LMCache/vLLM dependency closure.
echo "--- :python: Installing cacheblend-plugin harness deps"
uv pip install httpx
# 3. Pull the cacheblend-plugin. Coordinates come from the Buildkite pipeline
# env; CB_PLUGIN_REPO is required (set it in the pipeline env so the repo
# owner/name is not hardcoded here). REF/DIR have sensible fallbacks.
CB_PLUGIN_REPO="${CB_PLUGIN_REPO:?CB_PLUGIN_REPO not set — set the plugin repo (owner/name) in the Buildkite pipeline env}"
CB_PLUGIN_REF="${CB_PLUGIN_REF:-main}"
CB_PLUGIN_DIR="${CB_PLUGIN_DIR:-/tmp/cb-plugin}"
echo "--- :arrow_down: Pulling ${CB_PLUGIN_REPO}@${CB_PLUGIN_REF}"
_cb_tok="${GH_TOKEN:-${GITHUB_TOKEN:-}}"
echo " GH_TOKEN present: $([ -n "${_cb_tok}" ] && echo yes || echo NO)"
if [ -z "${_cb_tok}" ]; then
echo "ERROR: GH_TOKEN/GITHUB_TOKEN not set. ${CB_PLUGIN_REPO} is private — set a" >&2
echo " fine-grained PAT (resource owner = the repo's org, Contents: read)" >&2
echo " as GH_TOKEN in the Buildkite pipeline env." >&2
exit 1
fi
_cb_url="https://x-access-token:${_cb_tok}@github.com/${CB_PLUGIN_REPO}.git"
# Isolate from the Buildkite agent's git config: its checkout credential is a
# GitHub App token scoped to LMCache (injected via the GLOBAL git config —
# insteadOf / extraheader / credential.helper) and 403s on the plugin repo.
# Ignoring global+system config forces git to use ONLY our GH_TOKEN from the URL.
_cb_git=(env GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null
git -c credential.helper=)
rm -rf "${CB_PLUGIN_DIR}"
# --branch handles a branch/tag; a commit SHA needs a full clone + checkout.
if ! "${_cb_git[@]}" clone --depth 1 --branch "${CB_PLUGIN_REF}" "${_cb_url}" "${CB_PLUGIN_DIR}" 2>/dev/null; then
"${_cb_git[@]}" clone "${_cb_url}" "${CB_PLUGIN_DIR}"
git -C "${CB_PLUGIN_DIR}" checkout "${CB_PLUGIN_REF}"
fi
echo " cacheblend-plugin @ $(git -C "${CB_PLUGIN_DIR}" rev-parse --short HEAD) (ref=${CB_PLUGIN_REF})"
# 4. Install the plugin editable; --no-deps keeps the env's torch/vllm/lmcache,
# --no-build-isolation because the plugin is pure-Python (no compile step).
echo "--- :python: Installing cacheblend-plugin (editable)"
git config --global --add safe.directory "${CB_PLUGIN_DIR}" 2>/dev/null || true
uv pip install -e "${CB_PLUGIN_DIR}" --no-deps --no-build-isolation
# 5. Import/registration smoke so a broken install fails here, not 180s into a
# server-boot timeout inside the harness.
python -c "import lmcache_cacheblend; print('cacheblend-plugin imported OK')"
export CB_PLUGIN_DIR CB_PLUGIN_REF
echo "--- :white_check_mark: CacheBlend env ready (plugin @ ${CB_PLUGIN_REF}, dir=${CB_PLUGIN_DIR})"
.buildkite/k3_harness/setup-cluster.sh
Executable
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env bash
# Idempotent K3s + GPU Operator + CI base image setup.
# Safe to re-run — skips components already installed.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
CI_BASE_IMAGE="lmcache/ci-base:latest"
DATA_DIR="/data"
# ── K3s ──────────────────────────────────────────────────────
if command -v k3s &>/dev/null && systemctl is-active --quiet k3s; then
echo "✓ K3s already running"
else
echo "→ Installing K3s..."
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server \
--disable=traefik \
--write-kubeconfig-mode=644" sh -
echo "✓ K3s installed"
fi
# Persist KUBECONFIG for interactive shells
grep -q 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml' ~/.bashrc 2>/dev/null || \
echo 'export KUBECONFIG=/etc/rancher/k3s/k3s.yaml' >> ~/.bashrc
kubectl get nodes
echo ""
# ── Helm ─────────────────────────────────────────────────────
if ! command -v helm &>/dev/null; then
echo "→ Installing Helm..."
curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
fi
echo "✓ Helm $(helm version --short)"
# ── NVIDIA GPU Operator ──────────────────────────────────────
if helm status gpu-operator -n gpu-operator &>/dev/null; then
echo "✓ GPU Operator already installed"
else
echo "→ Installing GPU Operator..."
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo update nvidia
helm install gpu-operator nvidia/gpu-operator \
--namespace gpu-operator --create-namespace \
--set driver.enabled=false \
--set toolkit.enabled=true \
--wait --timeout 5m
echo "✓ GPU Operator installed"
fi
echo "→ Waiting for device plugin..."
kubectl wait --for=condition=ready pod \
-l app=nvidia-device-plugin-daemonset \
-n gpu-operator --timeout=120s
GPU_COUNT=$(kubectl get node -o jsonpath='{.items[0].status.allocatable.nvidia\.com/gpu}')
echo "✓ GPUs available: ${GPU_COUNT}"
# ── CI base image ────────────────────────────────────────────
# Auto-detect GPU compute capability for TORCH_CUDA_ARCH_LIST
COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1)
ARCH_LIST="${COMPUTE_CAP}+PTX"
# Check if image already exists in K3s containerd
if k3s ctr images check | grep -q "${CI_BASE_IMAGE}"; then
echo "✓ CI base image already imported (${CI_BASE_IMAGE})"
echo " To rebuild: run with REBUILD_IMAGE=1"
if [[ "${REBUILD_IMAGE:-}" != "1" ]]; then
SKIP_IMAGE=1
fi
fi
if [[ "${SKIP_IMAGE:-}" != "1" ]]; then
echo "→ Building CI base image (TORCH_CUDA_ARCH_LIST=${ARCH_LIST})..."
docker build \
-f "${SCRIPT_DIR}/ci-base.Dockerfile" \
--build-arg TORCH_CUDA_ARCH_LIST="${ARCH_LIST}" \
-t "${CI_BASE_IMAGE}" \
"${REPO_ROOT}"
echo "→ Importing into K3s containerd..."
docker save "${CI_BASE_IMAGE}" | k3s ctr images import -
echo "✓ CI base image ready"
fi
# ── Shared host volumes ──────────────────────────────────────
mkdir -p "${DATA_DIR}/huggingface" "${DATA_DIR}/datasets"
echo "✓ Host volumes: ${DATA_DIR}/{huggingface,datasets}"
echo ""
echo "=== Cluster ready ==="
echo " K3s: $(k3s --version | head -1)"
echo " GPU Operator: $(helm list -n gpu-operator -o json | python3 -c 'import sys,json;print(json.load(sys.stdin)[0]["app_version"])' 2>/dev/null || echo 'unknown')"
echo " GPUs: ${GPU_COUNT}x $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)"
echo " Arch list: ${ARCH_LIST}"
echo " Base image: ${CI_BASE_IMAGE}"
echo " Data dir: ${DATA_DIR}"
echo ""
echo "Next: run ./install-agent-stack.sh <BUILDKITE_AGENT_TOKEN>"
.buildkite/k3_harness/setup-env.sh
Executable
+298
View File
@@ -0,0 +1,298 @@
#!/usr/bin/env bash
# Per-job environment setup: installs vLLM nightly + LMCache from source.
# Called at the start of every CI job.
set -euo pipefail
# Print the failing command and line number on any error.
trap 'echo "ERROR: setup-env.sh failed at line $LINENO (exit code $?)" >&2' ERR
# ── GPU health pre-check ────────────────────────────────────
# Fail fast if GPUs are occupied by stale host processes.
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
check_gpu_health 80
# Resolve which vLLM nightly to install. Sets PINNED_VLLM_VERSION (empty
# string means "use latest nightly", any other value means
# "install vllm==<that exact version>"). See script header for the full
# resolution order and override knobs.
source "${REPO_ROOT}/.buildkite/k3_harness/resolve-pinned-vllm.sh"
echo "--- :broom: Pre-install bytecode/cache eviction"
# The CI base image pre-installs packages from requirements/*.txt at image
# build time. We've observed k3 jobs (integration + correctness) fail with
# ImportError: cannot import name 'GenerationConfig' from 'transformers'
# after `uv pip install -U vllm ...` upgrades transformers, even though the
# installed .py files clearly expose GenerationConfig (verified both 5.5.0
# and 5.5.4). The same install recipe replayed in a fresh venv outside CI
# always succeeds, so the failure is tied to base-image filesystem state
# (stale __pycache__, partial upgrades via overlayfs, etc.). Evict all
# bytecode caches and uv's download cache up front so later steps operate
# on clean ground; this is cheap (few seconds) and idempotent.
find /opt/venv/lib/python3.12/site-packages -type d -name __pycache__ \
-exec rm -rf {} + 2>/dev/null || true
uv cache clean 2>/dev/null || true
echo "--- :python: Installing vLLM nightly (pinned to cu130 index)"
# The base image is nvidia/cuda:13.0.2-devel-ubuntu24.04 (system nvcc 13).
# vLLM's generic nightly index (wheels.vllm.ai/nightly/vllm/) non-deterministically
# resolves to either a cu128 or a cu130 torch wheel depending on which wheel
# vLLM's nightly CI happened to publish that day. When the resolver picks a
# cu128 torch, torch.utils.cpp_extension._check_cuda_version aborts the
# LMCache editable install with:
# RuntimeError: The detected CUDA version (13.0) mismatches the version
# that was used to compile PyTorch (12.8).
#
# Pin to the cu130 sub-index so torch.version.cuda is always "13.0" and
# matches the base image. This also lets us drop the HTML-scraping + apt
# cuda-compiler alignment dance that lived here before.
# (See https://docs.vllm.ai/ install tips → Nightly → CUDA 13.0.)
#
# --reinstall-package for transformers / tokenizers / huggingface-hub /
# safetensors / vllm forces uv to uninstall-and-reinstall those packages
# even when it thinks the existing install is up to date. That is the
# minimum set to put the full `vllm serve` import chain on a freshly
# extracted wheel, which bypasses whatever filesystem-level mismatch in
# the base image was causing the GenerationConfig ImportError.
#
# When PINNED_VLLM_VERSION is non-empty (resolved by resolve-pinned-vllm.sh
# from the `buildkite_latest_tested_vllm` branch), pin to that exact wheel
# so every CI job matches the version most recently verified by the
# canary build. Empty falls back to "latest nightly".
#
# vLLM's nightly index (wheels.vllm.ai/nightly/<cuda>/) only keeps the
# *latest* wheel; older versions get rolled off within a day or two and
# pinning them there fails with "no version of vllm==X". Historical
# wheels are still served at wheels.vllm.ai/<full-commit-sha>/<cuda>/,
# which is a PEP 503 simple index. The canary records that archive URL
# directly in the pin file, so the common path needs zero extra API
# calls. As a fallback (e.g. an old-format pin file written before the
# canary started recording metadata, or a manual override via
# PINNED_VLLM_VERSION), we still expand the short SHA via the public
# GitHub commits API; GITHUB_TOKEN is honoured (5000 req/h) but optional.
PINNED_VLLM_INDEX_ARGS=()
if [[ -n "${PINNED_VLLM_VERSION:-}" ]]; then
VLLM_INSTALL_SPEC="vllm[runai,tensorizer,flashinfer]==${PINNED_VLLM_VERSION}"
echo "Installing vLLM pinned: ${VLLM_INSTALL_SPEC}"
archive_url="${PINNED_VLLM_ARCHIVE_INDEX_URL:-}"
if [[ -z "${archive_url}" ]]; then
# Old-format pin file or per-build override: resolve on demand.
short_sha="${PINNED_VLLM_VERSION##*+g}"
if [[ "${short_sha}" != "${PINNED_VLLM_VERSION}" \
&& "${short_sha}" =~ ^[0-9a-f]+$ ]]; then
gh_auth_args=()
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
gh_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
fi
full_sha=""
for attempt in 1 2 3; do
full_sha="$(curl -fsSL --connect-timeout 5 --max-time 10 \
-H "Accept: application/vnd.github+json" \
"${gh_auth_args[@]+"${gh_auth_args[@]}"}" \
"https://api.github.com/repos/vllm-project/vllm/commits/${short_sha}" \
2>/dev/null \
| awk -F'"' '/"sha":/ {print $4; exit}')" || true
if [[ "${full_sha}" =~ ^[0-9a-f]{40}$ ]]; then
break
fi
echo "[INFO] GitHub commit lookup attempt ${attempt} for" \
"${short_sha} returned no SHA; retrying..." >&2
sleep 2
done
if [[ "${full_sha}" =~ ^[0-9a-f]{40}$ ]]; then
archive_url="https://wheels.vllm.ai/${full_sha}/cu130"
else
echo "[WARN] could not resolve full SHA for ${short_sha}; pip" \
"may fail if the wheel has rolled off the nightly index" >&2
fi
fi
fi
if [[ -n "${archive_url}" ]]; then
echo "Adding commit-archived index: ${archive_url}"
PINNED_VLLM_INDEX_ARGS+=(--extra-index-url "${archive_url}")
fi
else
VLLM_INSTALL_SPEC="vllm[runai,tensorizer,flashinfer]"
echo "Installing latest vLLM nightly (no pin)"
fi
uv pip install -U "${VLLM_INSTALL_SPEC}" --pre \
--reinstall-package transformers \
--reinstall-package tokenizers \
--reinstall-package huggingface-hub \
--reinstall-package safetensors \
--reinstall-package vllm \
"${PINNED_VLLM_INDEX_ARGS[@]+"${PINNED_VLLM_INDEX_ARGS[@]}"}" \
--extra-index-url https://wheels.vllm.ai/nightly/cu130 \
--extra-index-url https://download.pytorch.org/whl/cu130 \
--index-strategy unsafe-best-match
# Pre-import transformers on the main thread via sitecustomize.py so
# vllm's BG-thread preload can't race ahead of _LazyModule init.
cat > /opt/venv/lib/python3.12/site-packages/sitecustomize.py <<'PY'
try:
import transformers # noqa: F401
except Exception:
pass
PY
# Probe the vLLM CLI by invoking `vllm --help` as a subprocess. This is the
# only probe that exercises the full import chain that `vllm serve` runs:
# vllm.entrypoints.cli.main.main() triggers `import vllm.entrypoints.cli.
# benchmark.main` *inside* the function body, which in turn loads
# vllm.config.model -> vllm.transformers_utils.config -> `from transformers
# import GenerationConfig, PretrainedConfig`. A plain `from vllm.entrypoints.
# cli.main import main` only resolves the `main` symbol; it never executes
# the function, so it silently passes even when the CLI is broken. Shared
# between the pre-install auto-heal loop below and the post-install hard
# probe at the end of this script.
probe_vllm_cli() {
vllm --help 2>&1 >/dev/null
}
# vLLM nightlies periodically add eager imports of packages that aren't in
# their declared deps (e.g. `pandas` from vllm/_aiter_ops.py). Auto-install
# any ModuleNotFoundError modules so the job keeps going. Capped to avoid
# infinite loops; every auto-install is logged so the drift is visible in
# the build output. ImportError with a missing top-level name (e.g. a
# transformers/vLLM API break) bails immediately since reinstalling the
# package wouldn't recover.
dump_transformers_state() {
# Called whenever a CLI probe fails with anything other than a clean
# ModuleNotFoundError. This failure mode has been reproducible only on
# the K3s pods, never on a fresh local venv with identical versions,
# so we need a direct view of the running pod's filesystem + Python
# state to make progress.
echo "=================== DIAGNOSTIC DUMP ===================" >&2
echo "--- uv pip list (relevant packages) ---" >&2
uv pip list 2>/dev/null | grep -iE "^(transformers|tokenizers|huggingface|safetensors|vllm|torch) " >&2 || true
local tf_dir=/opt/venv/lib/python3.12/site-packages/transformers
echo "--- transformers directory listing ---" >&2
ls -la "${tf_dir}/__init__.py" "${tf_dir}/__pycache__/__init__.cpython-312.pyc" 2>&1 >&2 || true
echo "--- dist-info METADATA version ---" >&2
grep -m1 "^Version:" /opt/venv/lib/python3.12/site-packages/transformers-*.dist-info/METADATA 2>/dev/null >&2 || true
echo "--- transformers/__init__.py: __version__ line ---" >&2
grep -n "^__version__" "${tf_dir}/__init__.py" 2>/dev/null >&2 || true
echo "--- transformers/__init__.py: 'generation' key in _import_structure ---" >&2
awk '/"generation":/,/\]/' "${tf_dir}/__init__.py" 2>/dev/null | head -20 >&2 || true
echo "--- Python sees: ---" >&2
python - <<'PY' >&2 2>&1 || true
import sys, importlib
print(f"sys.executable = {sys.executable}")
print(f"sys.path = {sys.path}")
try:
import transformers
print(f"transformers.__file__ = {transformers.__file__}")
print(f"transformers.__version__ = {transformers.__version__}")
print(f"type(transformers) = {type(transformers).__name__}")
print(f"'GenerationConfig' in dir(transformers) = {'GenerationConfig' in dir(transformers)}")
cs2m = getattr(transformers, "_class_to_module", None)
print(f"has _class_to_module = {cs2m is not None}")
if cs2m is not None:
print(f"GenerationConfig in _class_to_module = {'GenerationConfig' in cs2m}")
print(f"first 5 keys = {list(cs2m.keys())[:5]}")
import_struct = getattr(transformers, "_import_structure", None)
print(f"has _import_structure = {import_struct is not None}")
if import_struct is not None:
print(f"'generation' in _import_structure = {'generation' in import_struct}")
print(f"_import_structure['generation'] = {import_struct.get('generation')}")
try:
from transformers import GenerationConfig
print("DIRECT IMPORT of GenerationConfig WORKED")
except Exception as e:
print(f"DIRECT IMPORT failed: {type(e).__name__}: {e}")
except Exception as e:
import traceback
traceback.print_exc()
PY
echo "=================== END DIAGNOSTIC DUMP ===================" >&2
}
MAX_AUTO_INSTALL=5
for i in $(seq 1 "$MAX_AUTO_INSTALL"); do
if err=$(probe_vllm_cli); then
break
fi
mod=$(printf '%s\n' "$err" | sed -n "s/.*No module named '\([^']*\)'.*/\1/p" | head -1)
if [[ -z "$mod" ]]; then
echo "vLLM import failed with a non-ModuleNotFoundError:" >&2
echo "$err" >&2
dump_transformers_state
exit 1
fi
if [[ "$i" == "$MAX_AUTO_INSTALL" ]]; then
echo "Hit $MAX_AUTO_INSTALL auto-install retries; last missing module: $mod" >&2
echo "$err" >&2
exit 1
fi
echo "Auto-installing missing vLLM runtime dep: $mod"
uv pip install "$mod"
done
echo "--- :mag: Verifying torch CUDA matches system nvcc"
# Sanity check: fail fast with a clear message if the cu130 pin above
# somehow didn't produce a cu13x torch. Previously this mismatch surfaced
# deep inside ninja as a cryptic `cusparse.h: No such file or directory`;
# catching it here makes the failure mode obvious.
python - <<'PY'
import subprocess, sys, torch
tc = torch.version.cuda or ""
try:
nv = subprocess.check_output(["nvcc", "--version"], text=True)
sys_major = next(
(line.split("release ")[1].split(",")[0].split(".")[0]
for line in nv.splitlines() if "release " in line),
"",
)
except Exception:
sys_major = ""
torch_major = tc.split(".")[0] if tc else ""
print(f"torch.version.cuda={tc!r}; system nvcc major={sys_major!r}")
if torch_major and sys_major and torch_major != sys_major:
sys.exit(
f"CUDA major mismatch: torch={torch_major} vs nvcc={sys_major}. "
"Check the vLLM nightly cu130 index pin in setup-env.sh."
)
PY
echo "--- :python: Installing LMCache from source"
# Skip setuptools_scm git describe; the repo carries non-PEP-440 tags
# (nightly, nightly-cu13) that crash the newer vcs_versioning backend.
export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE="${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE:-0.0.0+ci}"
uv pip freeze | sort > /tmp/env-before-lmcache.txt
uv pip install -e . --no-build-isolation
uv pip freeze | sort > /tmp/env-after-lmcache.txt
if ! diff -q /tmp/env-before-lmcache.txt /tmp/env-after-lmcache.txt >/dev/null; then
echo "--- :warning: Packages changed during LMCache install"
diff /tmp/env-before-lmcache.txt /tmp/env-after-lmcache.txt || true
fi
echo "--- :broom: Post-install bytecode eviction"
# Belt-and-suspenders companion to the pre-install eviction above: clear
# __pycache__ again after the LMCache editable install, which may have
# triggered fresh imports (setuptools_scm, pyproject build backend) and
# deposited new .pyc files that reference now-downgraded packages.
find /opt/venv/lib/python3.12/site-packages -type d -name __pycache__ \
-exec rm -rf {} + 2>/dev/null || true
echo "--- :mag: Post-install CLI chain probe"
# The LMCache editable install can downgrade transitive deps to honor the
# caps in requirements/common.txt. If that leaves the env in a state where
# `vllm --help` cannot complete its import chain (vllm.entrypoints.cli.
# main.main() -> vllm.entrypoints.cli.benchmark.main -> vllm.config ->
# vllm.transformers_utils.config -> `from transformers import ...`), the
# only other signal is a 180s `wait_for_server` timeout inside each test
# harness. Re-probe the full chain here so broken envs fail fast with the
# actual traceback instead of a generic timeout.
if err=$(probe_vllm_cli); then
echo "vLLM CLI import chain OK post-install."
else
echo "FATAL: vLLM CLI import chain broken after LMCache install." >&2
echo "--- Traceback ---" >&2
echo "$err" >&2
echo "--- Installed packages ---" >&2
uv pip freeze >&2 || true
exit 1
fi
echo "--- :white_check_mark: Environment ready"
python -c "import vllm; import lmcache; print(f'vLLM={vllm.__version__}, LMCache installed from source with no build isolation')"
.buildkite/k3_harness/setup-gpu-monitor.sh
Executable
+44
View File
@@ -0,0 +1,44 @@
#!/usr/bin/env bash
# Install the GPU monitor as a cron job on the CI host machine.
# Run this once during initial cluster setup.
#
# Usage: sudo bash setup-gpu-monitor.sh
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MONITOR_SCRIPT="${SCRIPT_DIR}/gpu-monitor.sh"
LOG_FILE="/var/log/gpu-monitor.log"
CRON_INTERVAL="*/10" # Every 10 minutes
if [[ ! -f "$MONITOR_SCRIPT" ]]; then
echo "ERROR: gpu-monitor.sh not found at $MONITOR_SCRIPT"
exit 1
fi
chmod +x "$MONITOR_SCRIPT"
# Create log file with rotation
touch "$LOG_FILE"
# Set up logrotate
cat > /etc/logrotate.d/gpu-monitor <<EOF
${LOG_FILE} {
daily
rotate 7
compress
missingok
notifempty
}
EOF
# Install cron job (idempotent: removes old entry first)
CRON_CMD="${CRON_INTERVAL} * * * * ${MONITOR_SCRIPT} >> ${LOG_FILE} 2>&1"
(crontab -l 2>/dev/null | grep -v "gpu-monitor.sh" || true; echo "$CRON_CMD") | crontab -
echo "GPU monitor installed:"
echo " Script: $MONITOR_SCRIPT"
echo " Log: $LOG_FILE"
echo " Cron: $CRON_CMD"
echo ""
echo "View logs: tail -f $LOG_FILE"
echo "Remove cron: crontab -l | grep -v gpu-monitor.sh | crontab -"
.buildkite/k3_harness/setup-lmcache-only-env.sh
Executable
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
# Per-job environment setup for jobs that DON'T need vLLM (e.g. unit tests).
# Installs LMCache from source on top of the ci-base image, which already
# has torch + requirements/cuda.txt + build.txt baked in. Much faster than
# setup-env.sh since it skips the vLLM nightly install entirely.
set -euo pipefail
trap 'echo "ERROR: setup-lmcache-only-env.sh failed at line $LINENO (exit code $?)" >&2' ERR
# ── GPU health pre-check ────────────────────────────────────
# Fail fast if GPUs are occupied by stale host processes.
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
check_gpu_health 80
echo "--- :python: Installing LMCache from source (no vLLM)"
# Skip setuptools_scm git describe; the repo carries non-PEP-440 tags
# (nightly, nightly-cu13) that crash the newer vcs_versioning backend.
export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE="${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE:-0.0.0+ci}"
uv pip install -e . --no-build-isolation
echo "--- :white_check_mark: Environment ready (LMCache only, no vLLM)"
python -c "import lmcache; print('LMCache installed from source')"
.buildkite/k3_harness/setup-sglang-env.sh
Executable
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env bash
# Per-job env setup for the SGLang + LMCache MP integration tests.
# Drop the fork install once https://github.com/sgl-project/sglang/pull/24089
# lands.
set -euo pipefail
trap 'echo "ERROR: setup-sglang-env.sh failed at line $LINENO (exit code $?)" >&2' ERR
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
check_gpu_health 80
echo "--- :wrench: System tools (rustup, protoc, libnuma1)"
# rustup: sglang-grpc needs Rust 1.85+ (apt's rustc is too old).
# protoc: sglang-grpc's prost-build shells out to it.
# libnuma1: sgl_kernel's sm100 .so dynamically links to libnuma.so.1.
if ! command -v rustc >/dev/null 2>&1; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \
-y --default-toolchain stable --profile minimal --no-modify-path
fi
# shellcheck disable=SC1091
. "${HOME}/.cargo/env"
rustc --version
APT_NEEDED=()
command -v protoc >/dev/null 2>&1 || APT_NEEDED+=("protobuf-compiler")
[[ -e /usr/lib/x86_64-linux-gnu/libnuma.so.1 || -e /lib/x86_64-linux-gnu/libnuma.so.1 ]] || APT_NEEDED+=("libnuma1")
if [[ ${#APT_NEEDED[@]} -gt 0 ]]; then
apt-get update
apt-get install -y --no-install-recommends "${APT_NEEDED[@]}"
fi
protoc --version
echo "--- :package: SGLang + LMCache install"
SGLANG_URL="git+https://github.com/sgl-project/sglang.git@main#subdirectory=python"
uv pip install "${SGLANG_URL}"
export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE="${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE:-0.0.0+ci}"
uv pip uninstall cupy-cuda12x 2>/dev/null || true
uv pip install -e . --no-build-isolation
python -c "import lmcache, sglang; print(f'sglang={sglang.__version__}; lmcache OK')"
python -c "import cupy; print(f'cupy={cupy.__version__}')"
.buildkite/k3_harness/smoke-test.sh
Executable
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env bash
# Verify K3s + GPU Operator works: schedule a pod that runs nvidia-smi.
set -euo pipefail
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
echo "=== Smoke Test ==="
# 1. Check cluster
echo "→ Cluster"
kubectl get nodes -o wide
echo ""
# 2. Check GPU resources
GPU_COUNT=$(kubectl get node -o jsonpath='{.items[0].status.allocatable.nvidia\.com/gpu}')
echo "→ GPUs allocatable: ${GPU_COUNT}"
if [[ "${GPU_COUNT}" -lt 1 ]]; then
echo "✗ No GPUs found on node"
exit 1
fi
# 3. Run GPU pod
echo "→ Launching GPU test pod..."
kubectl delete pod gpu-smoke-test --ignore-not-found --wait=false &>/dev/null
cat <<'EOF' | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
name: gpu-smoke-test
spec:
restartPolicy: Never
containers:
- name: test
image: nvidia/cuda:12.8.0-base-ubuntu24.04
command: ["nvidia-smi"]
resources:
limits:
nvidia.com/gpu: "1"
EOF
kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/gpu-smoke-test --timeout=120s
echo ""
echo "→ Pod logs:"
kubectl logs gpu-smoke-test
kubectl delete pod gpu-smoke-test
echo ""
echo "=== Smoke test passed ==="
.buildkite/k3_harness/teardown.sh
Executable
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
# Tear down K3s stack. Removes: agent-stack-k8s, GPU Operator, K3s.
# Host data volumes (/data/*) are NOT deleted.
set -euo pipefail
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
echo "=== Teardown ==="
# agent-stack-k8s
if helm status agent-stack-k8s -n buildkite &>/dev/null; then
echo "→ Removing agent-stack-k8s..."
helm uninstall agent-stack-k8s -n buildkite --wait
fi
# GPU Operator
if helm status gpu-operator -n gpu-operator &>/dev/null; then
echo "→ Removing GPU Operator..."
helm uninstall gpu-operator -n gpu-operator --wait
kubectl delete namespace gpu-operator --ignore-not-found
fi
# K3s
if command -v k3s-uninstall.sh &>/dev/null; then
echo "→ Removing K3s..."
/usr/local/bin/k3s-uninstall.sh
fi
echo "✓ Teardown complete"
echo " Note: /data/{huggingface,datasets} preserved"
.buildkite/k3_harness/values.yaml
+25
View File
@@ -0,0 +1,25 @@
# Reference Helm values for agent-stack-k8s.
# install-agent-stack.sh sets these automatically.
# This file is for documentation only.
# Docs: https://github.com/buildkite/agent-stack-k8s
agentToken: "<from install-agent-stack.sh>"
config:
queue: "k8s" # override with BUILDKITE_QUEUE env var
# HTTPS checkout using GitHub token (replaces SSH key)
git-credentials-secret:
name: buildkite-git-creds # created by install-agent-stack.sh
key: git-credentials # contains https://x-access-token:<token>@github.com
# Inject GITHUB_TOKEN into job containers for git push operations
pod-spec-patch:
containers:
- name: container-0
env:
- name: GITHUB_TOKEN
valueFrom:
secretKeyRef:
name: buildkite-git-creds
key: GITHUB_TOKEN
.buildkite/k3_tests/README.md
+154
View File
@@ -0,0 +1,154 @@
# K8s Test Pipelines
Each subdirectory under `k3_tests/` is a self-contained test with these files:
| File | Purpose |
|------|---------|
| `run.sh` | Test script (sources `k3_harness/setup-env.sh`, then runs tests) |
| `pipeline.yml` | K8s pod spec — GPU count, volumes, timeouts |
| `buildkite-pipeline.yml` | What to paste into the Buildkite UI "Steps" editor (runs path filter, then uploads `pipeline.yml`) |
| `BK_WEB_SETUP.md` | Full Buildkite UI setup instructions: env vars, trigger filters, recommendations |
## Buildkite Web UI Setup
### Prerequisites
Before creating pipelines, make sure a queue named `k8s` exists in your Buildkite cluster. Go to **Organization Settings → Default cluster → Queues → New Queue** and create it. The queue needs no configuration and no agents — agent-stack-k8s creates ephemeral pod-based agents automatically when jobs arrive.
### Per-pipeline setup
For each test directory, create a pipeline in the Buildkite UI.
Each directory has a `BK_WEB_SETUP.md` with the exact settings — env vars, GitHub trigger filters, and recommendations for that test. The short version:
1. Go to your org → **New Pipeline**
2. In the **Steps** editor, paste the contents of that test's `buildkite-pipeline.yml`:
```yaml
agents:
queue: "k8s"
env:
HF_TOKEN: "<your HuggingFace token>"
steps:
- label: ":pipeline: Upload pipeline"
command: buildkite-agent pipeline upload .buildkite/k3_tests/<test-name>/pipeline.yml
```
The `agents.queue` must match the queue you created above. This routes the upload step to agent-stack-k8s, which checks out the repo, runs the path filter, and (if the build isn't skipped) uploads the real `pipeline.yml`. Each subsequent step also targets the same queue.
3. `HF_TOKEN` is needed for gated model access (e.g., Llama, Qwen). Set it in the `env` block as shown above, or under **Pipeline Settings → Environment Variables** in the UI — both work
4. Under **GitHub Settings**, configure trigger filters per the test's `BK_WEB_SETUP.md`
5. Save — jobs will run on the K8s queue automatically
### Path-based skip (auto-pass on docs-only changes)
The upload step in `buildkite-pipeline.yml` runs `common_scripts/upload-pipeline.sh`
instead of `buildkite-agent pipeline upload` directly. The wrapper
(`common_scripts/path-filter.sh`) inspects the changed files in the build and:
- **Skips** the build (exits 0 without uploading `pipeline.yml` → build is
green with just the upload step) when *all* changed files match a "trivial"
pattern: `*.md`, `LICENSE*`, `NOTICE*`, `.gitignore`, `.gitattributes`,
`.editorconfig`, `.mailmap`, `CODEOWNERS`, or anything under `docs/` or
`.github/`. (`.github/` is trivial here because k3 tests run on Buildkite,
not GitHub Actions, so workflow / CODEOWNERS / template changes do not
affect them.)
- **Force-runs** the build when any changed file lives under `.buildkite/` —
those PRs are usually fixing the k3 CI itself, so we want them tested on
the PR rather than after merge.
- **Runs** the build whenever there is at least one non-trivial file by
uploading `pipeline.yml`, which contains the real test steps.
Detection:
- PR builds diff against `origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}`
(default `main`) using the merge-base.
- Push builds diff `HEAD~1..HEAD`.
- Scheduled builds (`BUILDKITE_SOURCE=schedule`) are never skipped.
- If the script can't determine the changed files (shallow clone with no
parent, missing base branch, etc.) it falls back to "do not skip".
To bypass the skip and force a full run, add the **`force-ci`** label to the
PR on GitHub. Buildkite picks up PR labels automatically; when the filter
sees `force-ci` it runs the full pipeline regardless of which files changed.
### Trigger strategy
Not all tests should run on every push. The general pattern:
| Test weight | When to trigger | Example filter condition |
|-------------|----------------|------------------------|
| Lightweight (1 GPU, <20 min) | Every push / every PR | *(no filter)* |
| Heavy (multi-GPU, >30 min) | PR label or main branch only | `build.pull_request.labels includes "full" \|\| build.branch == 'dev'` |
Set **"Rebuild on PR label change"** to `Yes` for label-triggered pipelines so adding a label to an existing PR kicks off the build.
## Adding a New Test
1. Create a directory: `.buildkite/k3_tests/<test-name>/`
2. Write a `run.sh` that sources the shared environment setup:
```bash
#!/usr/bin/env bash
set -euo pipefail
cd "$(cd "$(dirname "$0")/../../.." && pwd)"
source .buildkite/k3_harness/setup-env.sh
# ... your test commands ...
```
3. Write a `pipeline.yml`. Set the GPU limit to what your test needs:
```yaml
steps:
- label: ":test_tube: My Test"
command: .buildkite/k3_tests/<test-name>/run.sh
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never # local image, imported into K3s containerd
resources:
limits:
nvidia.com/gpu: "1"
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
```
4. Write a `buildkite-pipeline.yml` (the snippet pasted into the Buildkite UI's Steps
editor). Use `common_scripts/upload-pipeline.sh` so the test gets path-based skip:
```yaml
agents:
queue: "k8s"
steps:
- label: ":pipeline: Upload pipeline"
command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/<test-name>/pipeline.yml
```
5. Write a `BK_WEB_SETUP.md` documenting the Buildkite UI settings for this test (env vars, trigger filters, recommendations). Use an existing test's `BK_WEB_SETUP.md` as a template.
6. `chmod +x` your `run.sh` and create the pipeline in the Buildkite UI.
### Optional: datasets volume
If your test needs pre-downloaded data (e.g., ShareGPT), add the datasets volume:
```yaml
volumeMounts:
- { name: datasets, mountPath: /root/correctness }
volumes:
- { name: datasets, hostPath: { path: /data/datasets, type: DirectoryOrCreate } }
```
### Optional: Docker-in-Docker
If your test runs Docker containers inside the pod:
```yaml
securityContext:
privileged: true
volumeMounts:
- { name: docker-sock, mountPath: /var/run/docker.sock }
volumes:
- { name: docker-sock, hostPath: { path: /var/run/docker.sock } }
```
.buildkite/k3_tests/blend/buildkite-pipeline.yml
+20
View File
@@ -0,0 +1,20 @@
# Paste this into the Buildkite pipeline's "Steps" editor.
# It uploads the real pipeline definition from the repo.
#
# Fill in HF_TOKEN (gated model access), GH_TOKEN (read access to the private
# plugin repo), and CB_PLUGIN_REPO (owner/name of the plugin repo). CB_PLUGIN_REF
# selects which plugin commit to test the PR against; CB_PLUGIN_DIR is the clone
# destination in the pod.
agents:
queue: "k8s"
env:
HF_TOKEN: "" # HuggingFace token — gated model access (e.g. Llama)
GH_TOKEN: "" # GitHub token — read access to the private plugin repo
CB_PLUGIN_REPO: "" # owner/name of the plugin repo (required; set here or in the UI)
CB_PLUGIN_REF: "main" # plugin git ref to test the PR against
CB_PLUGIN_DIR: "/tmp/cb-plugin" # clone destination in the pod
steps:
- label: ":pipeline: Upload pipeline"
command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/blend/pipeline.yml
.buildkite/k3_tests/blend/pipeline.yml
+34
View File
@@ -0,0 +1,34 @@
# CacheBlend <-> LMCache compatibility test.
#
# Builds the PR's LMCache from source (compiling c_ops for the pod's GPU arch),
# pulls the external cacheblend-plugin from GitHub, installs it editable, and
# runs the plugin's vllm_compat_check (--skip-correctness): one blend server +
# vLLM stack, five V3 contract checks ending in a non-prefix blend that must
# engage. Catches LMCache changes that break the external plugin *before* merge
# — the recurring rebase/version-skew class (RPC key renames, handler-signature
# drift, KV-layout changes, etc.).
#
# Needs one GPU. Set HF_TOKEN (gated model) and GH_TOKEN (private plugin repo)
# in the Buildkite UI — see BK_WEB_SETUP.md.
steps:
- label: ":jigsaw: CacheBlend plugin compat (Llama-8B)"
command: .buildkite/k3_tests/blend/run.sh
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never # local image, imported into K3s containerd
resources:
limits:
nvidia.com/gpu: "1"
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
artifact_paths:
- "cb_compat_logs/**/*"
.buildkite/k3_tests/blend/run.sh
Executable
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
cd "${REPO_ROOT}"
# ── Environment setup ────────────────────────────────────────
source .buildkite/k3_harness/setup-blend-env.sh
# ── Ensure all scripts are executable ────────────────────────
chmod +x "${SCRIPT_DIR}"/scripts/*.sh
# ── Run the actual test logic ────────────────────────────────
exec bash "${SCRIPT_DIR}/scripts/run-compat.sh" "$@"
.buildkite/k3_tests/blend/scripts/run-compat.sh
Executable
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env bash
# CacheBlend <-> LMCache compatibility check.
#
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
cd "${REPO_ROOT}"
source .buildkite/k3_tests/common_scripts/helpers.sh
# Fail fast if the GPUs are occupied by stale host processes.
check_gpu_health 80
: "${CB_PLUGIN_DIR:?CB_PLUGIN_DIR not set — source k3_harness/setup-blend-env.sh first}"
MODEL="${CB_MODEL:-meta-llama/Llama-3.1-8B-Instruct}"
MAX_LEN="${CB_MAX_MODEL_LEN:-16384}"
LOG_DIR="${REPO_ROOT}/cb_compat_logs" # workspace-relative so artifact_paths can collect it
mkdir -p "${LOG_DIR}"
# Resolve the venv that owns vllm/lmcache. In the CI pod this is the image's
# /opt/venv (on PATH via setup-env.sh); CB_VENV overrides for local runs.
if [ -n "${CB_VENV:-}" ]; then
VENV="${CB_VENV}"
elif [ -n "${VIRTUAL_ENV:-}" ]; then
VENV="${VIRTUAL_ENV}"
elif command -v vllm >/dev/null 2>&1; then
VENV="$(cd "$(dirname "$(command -v vllm)")/.." && pwd)"
else
VENV="/opt/venv"
fi
echo "+++ :jigsaw: vllm_compat_check model=${MODEL} venv=${VENV} plugin=${CB_PLUGIN_DIR}"
python "${CB_PLUGIN_DIR}/.buildkite/harness/vllm_compat_check.py" \
--model "${MODEL}" \
--gpu-sync 0 \
--skip-correctness \
--venv "${VENV}" \
--max-model-len "${MAX_LEN}" \
--log-dir "${LOG_DIR}"
.buildkite/k3_tests/common_scripts/helpers.sh
Executable
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env bash
# Shared helper functions for K3s test scripts.
# Source this file from every scripts/*.sh.
# Track background PIDs for cleanup
TRACKED_PIDS=()
# Check that all visible GPUs have sufficient free memory.
# Fails fast with a clear message if a host-level process is hogging GPU memory.
# Usage: check_gpu_health [min_free_percent]
# min_free_percent: minimum percentage of GPU memory that must be free (default: 80)
check_gpu_health() {
local min_free_pct="${1:-80}"
echo "--- :mag: GPU health check (require ${min_free_pct}% free memory)"
if ! command -v nvidia-smi &>/dev/null; then
echo " nvidia-smi not found, skipping GPU health check"
return 0
fi
# Parse GPU info into a temp file to avoid subshell variable scoping issues
local gpu_info
gpu_info=$(nvidia-smi --query-gpu=index,memory.total,memory.used,memory.free --format=csv,noheader,nounits 2>/dev/null | sed 's/ //g')
if [[ -z "$gpu_info" ]]; then
echo " No GPUs detected, skipping"
return 0
fi
local has_problem=false
while IFS=, read -r idx total used free; do
if [[ "$total" -eq 0 ]]; then
continue
fi
local free_pct=$((free * 100 / total))
if [[ "$free_pct" -lt "$min_free_pct" ]]; then
echo " WARNING: GPU $idx has only ${free_pct}% free (${free} MiB free / ${total} MiB total, ${used} MiB used)"
# Show which processes are using this GPU
nvidia-smi --query-compute-apps=pid,gpu_uuid,used_memory --format=csv,noheader -i "$idx" 2>/dev/null | while IFS= read -r proc; do
echo " Process: $proc"
done
has_problem=true
else
echo " GPU $idx: OK (${free_pct}% free, ${free} MiB / ${total} MiB)"
fi
done <<< "$gpu_info"
if [[ "$has_problem" == "true" ]]; then
echo ""
echo "FATAL: One or more GPUs have insufficient free memory."
echo "This usually means a stale process on the host is consuming GPU memory."
echo "Check the host with: nvidia-smi"
echo "To fix: kill the offending host processes, then re-run the CI job."
return 1
fi
echo " All GPUs healthy."
return 0
}
# Find an available TCP port starting from a given port number.
# Usage: find_free_port [start_port]
find_free_port() {
local port="${1:-8000}"
while [ "$port" -lt 65536 ]; do
if ! lsof -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1 &&
! timeout 1 bash -c "</dev/tcp/127.0.0.1/${port}" 2>/dev/null; then
echo "$port"
return 0
fi
((port++))
done
echo "ERROR: No available port found starting from ${1:-8000}" >&2
return 1
}
# Wait for a vLLM server to become ready by polling /v1/models.
# Usage: wait_for_server <port> [timeout_secs] [log_file]
# If log_file is provided, its tail is dumped to stderr on timeout so the
# real failure (e.g. an ImportError during startup) is visible inline in the
# job output instead of requiring a trip through build artifacts.
wait_for_server() {
local port="$1"
local timeout="${2:-180}"
local log_file="${3:-}"
echo "Waiting for vLLM on port $port (timeout=${timeout}s)..."
for ((i = 0; i < timeout; i++)); do
if curl -sf "http://localhost:${port}/v1/models" >/dev/null 2>&1; then
echo "vLLM ready on port $port (${i}s)"
return 0
fi
sleep 1
done
echo "vLLM failed to start on port $port within ${timeout}s" >&2
if [[ -n "$log_file" && -f "$log_file" ]]; then
echo "--- :page_facing_up: Last 200 lines of ${log_file}" >&2
tail -n 200 "$log_file" >&2
fi
return 1
}
# Kill all tracked background PIDs and wait for them.
# Call this in a trap handler.
cleanup_pids() {
echo "--- Cleaning up background processes..."
for pid in "${TRACKED_PIDS[@]}"; do
if kill -0 "$pid" 2>/dev/null; then
echo " Killing PID $pid"
kill "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
fi
done
TRACKED_PIDS=()
# Give GPU memory a moment to release
sleep 2
}
.buildkite/k3_tests/common_scripts/path-filter.sh
Executable
+158
View File
@@ -0,0 +1,158 @@
#!/usr/bin/env bash
# Path filter: decide whether a CI build can be skipped based on which files
# changed since the base commit.
#
# Usage:
# source path-filter.sh
# if should_skip_ci; then
# # all changed files are trivial (docs, etc.)
# fi
#
# Rules:
# - If ANY changed file lives under .buildkite/, the build runs.
# (Those PRs are usually fixing the k3 CI itself, so we want to test on
# the PR instead of waiting for it to land on main.)
# - Otherwise, if EVERY changed file matches a "trivial" pattern (markdown,
# LICENSE, anything under docs/ or .github/, etc.), the build can be
# skipped. .github/ is trivial for k3 because k3 tests run on Buildkite,
# not GitHub Actions, so workflow/CODEOWNERS/template changes do not
# affect what the k3 tests do.
# - Anything else → build runs.
#
# Opt-out: add a "force-ci" label to the PR on GitHub. Buildkite exposes
# PR labels via BUILDKITE_PULL_REQUEST_LABELS; if "force-ci" is present
# the filter is bypassed and the full pipeline runs.
#
# Detection of "changed files":
# - PR builds → diff against the merge-base with BUILDKITE_PULL_REQUEST_BASE_BRANCH.
# - Push builds → diff HEAD~1..HEAD.
# - Anything we can't figure out → fall back to "do not skip".
set -uo pipefail
# ── Pattern lists ─────────────────────────────────────────────
# bash `case` patterns: `*` matches any string including `/`, so `docs/*`
# matches `docs/foo/bar.png` as well as `docs/foo`.
_path_filter_is_always_trigger() {
case "$1" in
.buildkite/*) return 0 ;;
esac
return 1
}
_path_filter_is_trivial() {
case "$1" in
*.md) return 0 ;;
LICENSE|LICENSE.*) return 0 ;;
NOTICE|NOTICE.*) return 0 ;;
.gitignore|.gitattributes|.editorconfig|.mailmap) return 0 ;;
CODEOWNERS) return 0 ;;
docs/*) return 0 ;;
.github/*) return 0 ;;
esac
return 1
}
# ── Changed-files detection ───────────────────────────────────
_path_filter_get_changed_files() {
local base_branch base merge_base
# Ephemeral pods may not have GitHub's SSH host key yet.
# Accept new keys automatically so git-fetch doesn't hang on a prompt.
export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new -o LogLevel=ERROR"
if [[ -n "${BUILDKITE_PULL_REQUEST:-}" && "${BUILDKITE_PULL_REQUEST:-}" != "false" ]]; then
base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
# Buildkite checks out shallow; fetch enough history to find the merge-base.
git fetch --no-tags --depth=200 origin "$base_branch" 2>/dev/null || \
git fetch --no-tags origin "$base_branch" 2>/dev/null || true
if base=$(git rev-parse --verify "origin/${base_branch}" 2>/dev/null); then
if merge_base=$(git merge-base HEAD "$base" 2>/dev/null); then
git diff --name-only "$merge_base" HEAD
return 0
fi
# No merge-base (history not deep enough): diff directly.
git diff --name-only "$base" HEAD
return 0
fi
echo "path-filter: could not resolve origin/${base_branch}" >&2
return 1
fi
# Push build (or unknown context): diff against the previous commit.
if git rev-parse --verify HEAD~1 >/dev/null 2>&1; then
git diff --name-only HEAD~1 HEAD
return 0
fi
echo "path-filter: no parent commit available" >&2
return 1
}
# ── Public entry point ────────────────────────────────────────
# Returns 0 if the build can be safely skipped, non-zero otherwise.
# Prints a classification of every changed file to stderr for the build log.
should_skip_ci() {
# PR label opt-out: adding "force-ci" on GitHub forces a full run.
if [[ ",${BUILDKITE_PULL_REQUEST_LABELS:-}," == *",force-ci,"* ]]; then
echo "path-filter: PR has 'force-ci' label → not skipping" >&2
return 1
fi
# Never skip scheduled builds (e.g. nightly baselines with NEED_UPLOAD=true).
if [[ "${BUILDKITE_SOURCE:-}" == "schedule" ]]; then
echo "path-filter: scheduled build (BUILDKITE_SOURCE=schedule) → not skipping" >&2
return 1
fi
local changed_files
if ! changed_files=$(_path_filter_get_changed_files); then
echo "path-filter: could not determine changed files → not skipping" >&2
return 1
fi
if [[ -z "$changed_files" ]]; then
echo "path-filter: no changed files reported → not skipping (safer default)" >&2
return 1
fi
local has_always_trigger=0
local has_non_trivial=0
local trivial_count=0
local total=0
echo "path-filter: classifying changed files:" >&2
while IFS= read -r f; do
[[ -z "$f" ]] && continue
total=$((total + 1))
if _path_filter_is_always_trigger "$f"; then
has_always_trigger=1
echo " [force-trigger] $f" >&2
elif _path_filter_is_trivial "$f"; then
trivial_count=$((trivial_count + 1))
echo " [trivial] $f" >&2
else
has_non_trivial=1
echo " [non-trivial] $f" >&2
fi
done <<< "$changed_files"
echo "path-filter: ${total} files changed (${trivial_count} trivial)" >&2
if [[ "$has_always_trigger" -eq 1 ]]; then
echo "path-filter: at least one file under .buildkite/ → not skipping" >&2
return 1
fi
if [[ "$has_non_trivial" -eq 1 ]]; then
echo "path-filter: non-trivial files changed → not skipping" >&2
return 1
fi
echo "path-filter: all changed files are trivial → SKIP" >&2
return 0
}
.buildkite/k3_tests/common_scripts/upload-pipeline.sh
Executable
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
# Wraps `buildkite-agent pipeline upload` with a path-based skip check.
#
# Usage (called from each test's buildkite-pipeline.yml upload step):
# command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh \
# .buildkite/k3_tests/<test-name>/pipeline.yml
#
# If every changed file in this build is trivial (markdown, LICENSE, .github,
# etc.) and none touch .buildkite/, this script:
# - Annotates the build with a "skipped" note
# - Exits 0 without uploading any further steps → the build is green
# Otherwise it execs `buildkite-agent pipeline upload <pipeline.yml>`, adding
# the real test steps to the build.
#
# Add a "force-ci" label to the PR on GitHub to bypass the check.
set -euo pipefail
PIPELINE_FILE="${1:?Usage: upload-pipeline.sh <path/to/pipeline.yml>}"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
# shellcheck source=path-filter.sh
source "${SCRIPT_DIR}/path-filter.sh"
if should_skip_ci; then
echo "+++ :fast_forward: Skipping CI — only trivial files changed"
if command -v buildkite-agent >/dev/null 2>&1; then
buildkite-agent annotate \
--style success \
--context "path-filter-skip" \
"Skipped: only trivial files (docs, license, etc.) changed. Add a \`force-ci\` label to the PR to force a full run." \
|| true
fi
exit 0
fi
echo "--- :pipeline: Uploading ${PIPELINE_FILE}"
exec buildkite-agent pipeline upload "${PIPELINE_FILE}"
.buildkite/k3_tests/comprehensive/BK_WEB_SETUP.md
+27
View File
@@ -0,0 +1,27 @@
# Buildkite Web UI Setup: Comprehensive Tests
**Steps editor**: paste contents of `buildkite-pipeline.yml` (fill in `HF_TOKEN`).
**GitHub trigger settings**:
- Filter: `build.pull_request.labels includes "full" || build.branch == 'dev'`
- Rebuild on PR label change: Yes
- Skip queued / cancel running branch builds: Yes
Heavy test (10 parallel GPU steps) — run on `"full"` label or dev push, not every PR.
> Builds whose only changes are docs/`*.md`/`LICENSE`/`.github/**` auto-pass
> via the [path filter](../README.md#path-based-skip-auto-pass-on-docs-only-changes).
> Changes under `.buildkite/` always run. Add `force-ci` label to the PR to
> bypass.
## Nightly Scheduled Build (rolling baselines)
Create a **Scheduled Build** on this same pipeline to upload performance baselines:
- **Schedule**: daily (e.g. `0 2 * * *` — 2am UTC)
- **Branch**: `dev`
- **Extra Environment Variables**: `NEED_UPLOAD=true`
Each config step writes a date-stamped baseline (`<feature>-YYYYMMDD.json`) as a Buildkite artifact. The finalize step (`scripts/upload-baselines.sh`) collects them all, prunes files older than 5 days, and pushes a single commit to the `benchmarks-main` branch.
PR builds automatically compare against the rolling 5-day worst-case (max latency) baseline.
.buildkite/k3_tests/comprehensive/buildkite-pipeline.yml
+11
View File
@@ -0,0 +1,11 @@
# Paste this into the Buildkite pipeline's "Steps" editor.
# It uploads the real pipeline definition from the repo.
agents:
queue: "k8s"
env:
HF_TOKEN: "" # set your HuggingFace token here (needed for gated model access)
steps:
- label: ":pipeline: Upload pipeline"
command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/comprehensive/pipeline.yml
.buildkite/k3_tests/comprehensive/pipeline.yml
+143
View File
@@ -0,0 +1,143 @@
# Comprehensive integration tests — one step per config, run in parallel.
# Each step gets its own K8s pod with the exact GPU count it needs.
# 2-GPU group is listed first so these heavier jobs get scheduled before 1-GPU jobs.
x-perf-retry: &perf-retry
automatic:
- exit_status: -1
limit: 2
- exit_status: 1
limit: 2
steps:
- group: ":test_tube: Comprehensive (2-GPU)"
steps:
- label: ":test_tube: pd"
command: .buildkite/k3_tests/comprehensive/run.sh pd.yaml
timeout_in_minutes: 30
retry: *perf-retry
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec: &pod-2gpu
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources: { limits: { "nvidia.com/gpu": "2" } }
volumeMounts: &vol-mounts
- { name: hf-cache, mountPath: /root/.cache/huggingface }
- { name: datasets, mountPath: /root/correctness }
volumes: &vols
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
- { name: datasets, hostPath: { path: /data/datasets, type: DirectoryOrCreate } }
artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"]
- label: ":test_tube: p2p"
command: .buildkite/k3_tests/comprehensive/run.sh p2p.yaml
timeout_in_minutes: 30
retry: *perf-retry
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-2gpu } }]
artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"]
- label: ":test_tube: p2p_with_v3"
command: .buildkite/k3_tests/comprehensive/run.sh p2p_with_v3.yaml
timeout_in_minutes: 30
retry: *perf-retry
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-2gpu } }]
artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"]
- group: ":test_tube: Comprehensive (1-GPU)"
steps:
- label: ":test_tube: local_cpu"
command: .buildkite/k3_tests/comprehensive/run.sh local_cpu.yaml
timeout_in_minutes: 30
retry: *perf-retry
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec: &pod-1gpu
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources: { limits: { "nvidia.com/gpu": "1" } }
volumeMounts: *vol-mounts
volumes: *vols
artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"]
- label: ":test_tube: local_disk"
command: .buildkite/k3_tests/comprehensive/run.sh local_disk.yaml
timeout_in_minutes: 30
retry: *perf-retry
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"]
- label: ":test_tube: async"
command: .buildkite/k3_tests/comprehensive/run.sh async.yaml
timeout_in_minutes: 30
retry: *perf-retry
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"]
- label: ":test_tube: local_cpu_with_v3"
command: .buildkite/k3_tests/comprehensive/run.sh local_cpu_with_v3.yaml
timeout_in_minutes: 30
retry: *perf-retry
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"]
- label: ":test_tube: local_disk_with_v3"
command: .buildkite/k3_tests/comprehensive/run.sh local_disk_with_v3.yaml
timeout_in_minutes: 30
retry: *perf-retry
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"]
- label: ":test_tube: multi_device"
command: .buildkite/k3_tests/comprehensive/run.sh multi_device.yaml
timeout_in_minutes: 30
retry: *perf-retry
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"]
- label: ":test_tube: layerwise"
command: .buildkite/k3_tests/comprehensive/run.sh layerwise.yaml
timeout_in_minutes: 30
retry: *perf-retry
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"]
# Nightly baseline upload: only runs when NEED_UPLOAD=true (scheduled build).
# Collects date-stamped baseline artifacts from all config steps,
# prunes files older than 5 days, and pushes a single commit to benchmarks-main.
- wait: ~
if: build.env("NEED_UPLOAD") == "true"
- label: ":arrow_up: Upload rolling baselines"
if: build.env("NEED_UPLOAD") == "true"
command: .buildkite/k3_tests/comprehensive/scripts/upload-baselines.sh
timeout_in_minutes: 5
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources: { limits: { cpu: "1", memory: "2Gi" } }
env:
- name: GITHUB_TOKEN
valueFrom:
secretKeyRef:
name: buildkite-git-creds
key: GITHUB_TOKEN
.buildkite/k3_tests/comprehensive/run.sh
Executable
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
# Comprehensive integration test entrypoint for K8s pods.
# Usage: run.sh <config.yaml>
# Thin wrapper: sets up environment, then delegates to scripts/.
set -euo pipefail
CFG_NAME="${1:?Usage: $0 <config.yaml>}"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
cd "${REPO_ROOT}"
# ── Environment setup ────────────────────────────────────────
source .buildkite/k3_harness/setup-env.sh
# Install test utilities (yq for YAML parsing, jq for JSON, openai/pandas/matplotlib for benchmarks)
uv pip install yq jq openai pandas matplotlib 2>/dev/null || true
# ── Ensure all scripts are executable ────────────────────────
chmod +x "${SCRIPT_DIR}"/scripts/*.sh
# ── Run the actual test logic ────────────────────────────────
exec bash "${SCRIPT_DIR}/scripts/run-single-config.sh" "$CFG_NAME"
.buildkite/k3_tests/comprehensive/scripts/run-single-config.sh
Executable
+531
View File
@@ -0,0 +1,531 @@
#!/usr/bin/env bash
# Comprehensive integration test: run a single config natively in a K8s pod.
# Replaces the old Docker-based vllm-integration-tests.sh.
#
# Usage: run-single-config.sh <config.yaml>
set -euo pipefail
CFG_NAME="${1:?Usage: $0 <config.yaml>}"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
cd "${REPO_ROOT}"
source .buildkite/k3_tests/common_scripts/helpers.sh
CONFIG_DIR="${REPO_ROOT}/.buildkite/configs"
LOGFILE="${REPO_ROOT}/${CFG_NAME%.yaml}.log"
BUILD_ID="${BUILDKITE_BUILD_ID:-local_$$}"
NEED_UPLOAD="${NEED_UPLOAD:-false}"
SERVER_WAIT_TIMEOUT="${SERVER_WAIT_TIMEOUT:-240}"
# Tee all output so Buildkite can collect it as an artifact
exec > >(tee -a "$LOGFILE") 2>&1
# ── Validate config ──────────────────────────────────────────
cfg_file="${CONFIG_DIR}/${CFG_NAME}"
if [[ ! -f "$cfg_file" ]]; then
echo "Config not found: ${cfg_file}"
exit 1
fi
feature_type=$(yq -r '.feature.type // ""' "$cfg_file")
echo -e "\033[1;33m===== Testing LMCache with ${CFG_NAME} (type=${feature_type:-standard}) =====\033[0m"
# Prevent git from hanging on prompts
export GIT_TERMINAL_PROMPT=0
# PID tracking for cleanup
PIDS=()
on_exit() {
local rc=$?
echo "--- Cleaning up (exit code: $rc)..."
for p in "${PIDS[@]}"; do
kill "$p" 2>/dev/null || true
wait "$p" 2>/dev/null || true
done
# Copy vLLM server logs to repo root so Buildkite can collect them as artifacts
cp /tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}*.log "${REPO_ROOT}/" 2>/dev/null || true
sleep 5
}
trap on_exit EXIT INT TERM
###############
# UTILITIES #
###############
# Start a vLLM process natively (no Docker).
# Reads env vars from a yq-extracted docker section and vllm args from a vllm section.
start_single_server() {
local docker_section="$1"
local vllm_section="$2"
local port="$3"
local gpu="${4:-}"
local logfile="${5:-/tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}.log}"
# Collect env vars from docker.env[]
local -a env_cmd=()
while IFS= read -r e; do
[[ -n "$e" ]] && env_cmd+=("$e")
done < <(yq -r '.env[]?' <<<"$docker_section")
# Standard env vars
env_cmd+=("VLLM_USE_FLASHINFER_SAMPLER=0")
[[ -n "${HF_TOKEN:-}" ]] && env_cmd+=("HF_TOKEN=${HF_TOKEN}")
[[ -n "$gpu" ]] && env_cmd+=("CUDA_VISIBLE_DEVICES=${gpu}")
# Override socket prefix so each server gets its own socket directory.
# In the old Docker setup, a volume mount mapped /tmp/lmcache_internal_api_server
# inside the container to /tmp/lmcache_internal_api_server/${port} on the host.
# Without Docker, we replicate this by setting the prefix to include the port.
mkdir -p "/tmp/lmcache_internal_api_server/${port}"
env_cmd+=("LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/${port}/socket")
# Parse vllm model and args
local vllm_model
vllm_model="$(yq -r '.model' <<<"$vllm_section")"
local -a vllm_cli_args=()
mapfile -t vllm_cli_args < <(yq -r '.args // [] | .[]' <<<"$vllm_section")
echo "Starting vLLM: model=$vllm_model port=$port gpu=${gpu:-all}"
env "${env_cmd[@]}" \
vllm serve "$vllm_model" "${vllm_cli_args[@]}" --port "$port" \
>"$logfile" 2>&1 &
local pid=$!
PIDS+=("$pid")
echo " PID=$pid log=$logfile"
# Wait for readiness
if ! wait_for_server "$port" "$SERVER_WAIT_TIMEOUT"; then
echo "Server failed to start. Last 50 lines of log:"
tail -50 "$logfile" || true
return 1
fi
}
###############
# LAUNCH #
###############
PORT=$(find_free_port 8000)
PORT1=""
PORT2=""
model=""
if [[ "$feature_type" == "pd" ]]; then
# ── Prefiller-Decoder mode (2 GPUs) ─────────────────────
PORT1=$(find_free_port 8100)
PORT2=$(find_free_port $((PORT1 + 100)))
prefiller_docker="$(yq '.["docker-prefiller"]' "$cfg_file")"
prefiller_vllm="$(yq '.["vllm-prefiller"]' "$cfg_file")"
decoder_docker="$(yq '.["docker-decoder"]' "$cfg_file")"
decoder_vllm="$(yq '.["vllm-decoder"]' "$cfg_file")"
model="$(yq -r '.["vllm-prefiller"].model' "$cfg_file")"
# Extra PD env vars
proxy=$(yq -er '.["docker-prefiller"]["proxy-port"]' "$cfg_file" 2>/dev/null || echo "7500")
init=$(yq -er '.["docker-decoder"]["init-port"]' "$cfg_file" 2>/dev/null || echo "7300")
alloc=$(yq -er '.["docker-decoder"]["alloc-port"]' "$cfg_file" 2>/dev/null || echo "7400")
# Inject PD-specific env vars into docker sections
prefiller_docker=$(echo "$prefiller_docker" | yq -y --arg proxy "$proxy" '. + {"env": (.env + ["LMCACHE_PD_PROXY_PORT=" + $proxy])}')
decoder_docker=$(echo "$decoder_docker" | yq -y --arg init "$init" --arg alloc "$alloc" '. + {"env": (.env + ["LMCACHE_PD_PEER_INIT_PORT=" + $init, "LMCACHE_PD_PEER_ALLOC_PORT=" + $alloc])}')
echo "--- Starting prefiller on port $PORT1 (GPU 0)"
# Prefiller needs UCX_TLS for NIXL transport
prefiller_docker=$(echo "$prefiller_docker" | yq -y ". + {\"env\": (.env + [\"UCX_TLS=cuda_ipc,cuda_copy,tcp\"])}")
start_single_server "$prefiller_docker" "$prefiller_vllm" "$PORT1" "0" \
"${REPO_ROOT}/${CFG_NAME%.yaml}-prefiller.log"
echo "--- Starting decoder on port $PORT2 (GPU 1)"
decoder_docker=$(echo "$decoder_docker" | yq -y ". + {\"env\": (.env + [\"UCX_TLS=cuda_ipc,cuda_copy,tcp\"])}")
start_single_server "$decoder_docker" "$decoder_vllm" "$PORT2" "1" \
"${REPO_ROOT}/${CFG_NAME%.yaml}-decoder.log"
# Start disagg proxy
echo "--- Starting PD proxy on port $PORT"
python3 "${REPO_ROOT}/examples/disagg_prefill/disagg_proxy_server.py" \
--port "$PORT" \
--prefiller-port "$PORT1" \
--decoder-port "$PORT2" \
--decoder-init-port "$init" \
--decoder-alloc-port "$alloc" \
--proxy-port "$proxy" \
> "${REPO_ROOT}/${CFG_NAME%.yaml}-proxy.log" 2>&1 &
PIDS+=($!)
sleep 10
elif [[ "$feature_type" == "p2p" ]]; then
# ── Peer-to-Peer mode (2 GPUs) ──────────────────────────
PORT1=$(find_free_port 8177)
PORT2=$(find_free_port $((PORT1 + 100)))
docker1="$(yq '.docker1' "$cfg_file")"
vllm1="$(yq '.vllm1' "$cfg_file")"
docker2="$(yq '.docker2' "$cfg_file")"
vllm2="$(yq '.vllm2' "$cfg_file")"
model="$(yq -r '.vllm1.model' "$cfg_file")"
# Controller ports
pull=$(yq -er '.docker1["pull-port"]' "$cfg_file" 2>/dev/null || echo "8300")
reply=$(yq -er '.docker1["reply-port"]' "$cfg_file" 2>/dev/null || echo "8400")
# Inject controller URLs
docker1=$(echo "$docker1" | yq -y --arg pull "$pull" --arg reply "$reply" '. + {"env": (.env + ["LMCACHE_CONTROLLER_PULL_URL=localhost:" + $pull, "LMCACHE_CONTROLLER_REPLY_URL=localhost:" + $reply, "UCX_TLS=tcp"])}')
pull2=$(yq -er '.docker2["pull-port"]' "$cfg_file" 2>/dev/null || echo "$pull")
reply2=$(yq -er '.docker2["reply-port"]' "$cfg_file" 2>/dev/null || echo "$reply")
docker2=$(echo "$docker2" | yq -y --arg pull2 "$pull2" --arg reply2 "$reply2" '. + {"env": (.env + ["LMCACHE_CONTROLLER_PULL_URL=localhost:" + $pull2, "LMCACHE_CONTROLLER_REPLY_URL=localhost:" + $reply2, "UCX_TLS=tcp"])}')
# Start controller
echo "--- Starting P2P controller on port $PORT"
PYTHONHASHSEED=123 lmcache_controller \
--host localhost \
--port "$PORT" \
--monitor-ports "{\"pull\": ${pull}, \"reply\": ${reply}}" \
> "/tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}_controller.log" 2>&1 &
PIDS+=($!)
sleep 10
echo "--- Starting instance 1 on port $PORT1 (GPU 0)"
start_single_server "$docker1" "$vllm1" "$PORT1" "0" \
"/tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}1.log"
echo "--- Starting instance 2 on port $PORT2 (GPU 1)"
start_single_server "$docker2" "$vllm2" "$PORT2" "1" \
"/tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}2.log"
else
# ── Single server mode (1 GPU) ──────────────────────────
docker_section="$(yq '.docker' "$cfg_file")"
vllm_section="$(yq '.vllm' "$cfg_file")"
model="$(yq -r '.vllm.model' "$cfg_file")"
echo "--- Starting single server on port $PORT"
start_single_server "$docker_section" "$vllm_section" "$PORT" "" \
"/tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}.log"
fi
###############
# WORKLOAD #
###############
test_mode="$(yq -r '.workload.type' "$cfg_file")"
if [[ "$test_mode" == "dummy" ]]; then
echo "--- Running dummy test (simple HTTP request)"
http_status_code=$(
curl --max-time 60 "http://localhost:${PORT}/v1/completions" \
-w "%{http_code}" -o response-file.txt \
-H "Content-Type: application/json" \
-d '{
"model": "'"$model"'",
"prompt": "<|begin_of_text|><|system|>\nYou are a helpful AI assistant.\n<|user|>\nWhat is the capital of France?\n<|assistant|>",
"max_tokens": 100,
"temperature": 0.7
}'
)
if [[ "$http_status_code" -ne 200 ]]; then
echo "FAIL: Dummy test HTTP status ${http_status_code}"
cat response-file.txt
exit 1
fi
echo "PASS: Dummy test succeeded"
cat response-file.txt
elif [[ "$test_mode" == "long_doc_qa" ]]; then
echo "--- Running long_doc_qa workload"
# Build workload JSON (merge workload section with model, strip non-CLI fields)
# Fields like expected-latency-gain are used by the checking logic, not long_doc_qa.py.
# "completion" -> "completions" rename to match the argparse flag.
workload_yaml="$(yq --arg model "$model" '(.workload * {"model": $model}) | del(.type) | del(.["expected-latency-gain"]) | if .completion then .completions = .completion | del(.completion) else . end' "$cfg_file")"
cfg_json="$(yq '.' "$cfg_file")"
# Determine which checking fields are requested
check_warmup_round_time_per_prompt=$(
jq -e '(.["checking-fields"] // []) | index("warmup_round_time_per_prompt") != null' \
<<< "$cfg_json" >/dev/null && echo true || echo false
)
check_query_round_time_per_prompt=$(
jq -e '(.["checking-fields"] // []) | index("query_round_time_per_prompt") != null' \
<<< "$cfg_json" >/dev/null && echo true || echo false
)
check_query_ttft_per_prompt=$(
jq -e '(.["checking-fields"] // []) | index("warmup_ttft_per_prompt") != null' \
<<< "$cfg_json" >/dev/null && echo true || echo false
)
run_long_doc_qa_workload() {
local workload_config="$1"
local port="$2"
local check_warmup="${3:-false}"
local check_ttft="${4:-false}"
local check_round="${5:-false}"
local feat_type="${6:-dummy}"
local need_upload="${7:-false}"
echo "Running long_doc_qa on port $port"
# Build args array from JSON config
local -a workload_args=()
mapfile -d '' -t workload_args < <(
jq -j '
to_entries[]
| select(.value != null and (.value|tostring) != "")
| (
if (.value|type) == "boolean" then
if .value then ["--\(.key)", "\u0000"] else [] end
elif (.value|type) == "array" then
.value[]
| ["--\(.key)", "\u0000",
(if (type=="string") then . else tostring end),
"\u0000"]
else
["--\(.key)", "\u0000",
(if ((.value|type)=="string") then .value else (.value|tostring) end),
"\u0000"]
end
)
| .[]
' <<<"$workload_config"
)
local json
json=$(
python3 "${REPO_ROOT}/benchmarks/long_doc_qa/long_doc_qa.py" \
"${workload_args[@]}" \
--port="$port" \
--output="response.txt" \
--json-output \
2>>response.txt | tail -n 1
)
local query_ttft_per_prompt query_round_time_per_prompt warmup_round_time_per_prompt
query_ttft_per_prompt=$(echo "$json" | jq -r '.query_ttft_per_prompt')
query_round_time_per_prompt=$(echo "$json" | jq -r '.query_round_time_per_prompt')
warmup_round_time_per_prompt=$(echo "$json" | jq -r '.warmup_round_time_per_prompt')
if [[ "$need_upload" == "true" ]]; then
# Nightly: write date-stamped file and upload as Buildkite artifact.
# A finalize step (upload-baselines.sh) will collect all artifacts,
# prune old files, and make a single commit to benchmarks-main.
local datestamp
datestamp="$(date +%Y%m%d)"
local artifact_name="${feat_type}-${datestamp}.json"
local artifact_path="${REPO_ROOT}/benchmarks/long_doc_qa/${artifact_name}"
echo "$json"
mkdir -p "$(dirname "$artifact_path")"
printf '%s\n' "$json" > "$artifact_path"
echo "[INFO] Uploading baseline artifact: ${artifact_name}"
buildkite-agent artifact upload "$artifact_path" 2>/dev/null || {
echo "[WARN] buildkite-agent not available; wrote $artifact_path locally"
}
return 0
fi
# ── PR comparison: rolling 5-day worst-case baseline ──────
# Fetch all date-stamped baselines for this feature from benchmarks-main.
# Use max() of each metric (= worst/slowest) as the threshold baseline.
# This makes the gate more tolerant of nightly hardware noise.
timeout 30 git fetch origin benchmarks-main >/dev/null 2>&1 || true
# List all baseline files for this feature (date-stamped and legacy).
# Note: git ls-tree does NOT expand shell globs in pathspecs, so we
# list the directory and filter with grep instead.
local -a baseline_files=()
mapfile -t baseline_files < <(
git ls-tree --name-only origin/benchmarks-main -- \
benchmarks/long_doc_qa/ 2>/dev/null \
| grep -E "^benchmarks/long_doc_qa/${feat_type}(-[0-9]{8})?\.json$" \
|| true
)
if [[ ${#baseline_files[@]} -eq 0 ]]; then
if [[ "$feat_type" != "dummy" ]]; then
echo "No baselines found for ${feat_type} -- skipping performance comparisons"
echo "Current metrics: TTFT=$query_ttft_per_prompt, Latency=$query_round_time_per_prompt, Warmup=$warmup_round_time_per_prompt"
fi
return 0
fi
echo "[INFO] Found ${#baseline_files[@]} baseline file(s) for ${feat_type}:"
printf " %s\n" "${baseline_files[@]}"
# Compute rolling max (worst-case) across all baselines
local expected_query_ttft=0 expected_query_round=0 expected_warmup=0
for bf in "${baseline_files[@]}"; do
local bj
bj=$(git show "origin/benchmarks-main:${bf}" 2>/dev/null || echo "")
if [[ -z "$bj" ]] || ! echo "$bj" | jq -e . >/dev/null 2>&1; then
echo " Skipping invalid baseline: $bf"
continue
fi
# Take max of each metric (higher time = worse perf = more permissive gate)
expected_query_ttft=$(awk "BEGIN { a=$expected_query_ttft; b=$(echo "$bj" | jq -r '.query_ttft_per_prompt // 0'); print (a>b?a:b) }")
expected_query_round=$(awk "BEGIN { a=$expected_query_round; b=$(echo "$bj" | jq -r '.query_round_time_per_prompt // 0'); print (a>b?a:b) }")
expected_warmup=$(awk "BEGIN { a=$expected_warmup; b=$(echo "$bj" | jq -r '.warmup_round_time_per_prompt // 0'); print (a>b?a:b) }")
done
echo "[INFO] Rolling worst-case baseline: TTFT=$expected_query_ttft, Latency=$expected_query_round, Warmup=$expected_warmup"
# Sanity check: if all baselines were invalid, skip comparison
if awk "BEGIN { exit ($expected_query_ttft == 0 && $expected_query_round == 0 && $expected_warmup == 0) ? 0 : 1 }"; then
echo "All baselines were invalid or empty -- skipping performance comparisons"
return 0
fi
# Adaptive threshold: with fewer baselines the worst-case max is less
# representative, so we allow more headroom.
# 5+ baselines → 1.2x (20% tolerance, full confidence)
# 4 baselines → 1.3x
# 3 baselines → 1.4x
# 2 baselines → 1.5x
# 1 baseline → 1.6x (60% tolerance, low confidence)
local num_baselines=${#baseline_files[@]}
local threshold
threshold=$(awk -v n="$num_baselines" 'BEGIN {
gap = (5 - n); if (gap < 0) gap = 0
printf "%.1f", 1.2 + gap * 0.1
}')
local pct
pct=$(awk -v t="$threshold" 'BEGIN { printf "%d", (t - 1) * 100 }')
echo "[INFO] Baselines: ${num_baselines}/5 — using ${pct}% tolerance (threshold=${threshold}x)"
if [[ "$check_ttft" == "true" ]]; then
echo "Worst-case baseline query ttft per prompt: $expected_query_ttft"
echo "Actual query ttft per prompt: $query_ttft_per_prompt"
awk -v expected="$expected_query_ttft" -v actual="$query_ttft_per_prompt" -v t="$threshold" -v pct="$pct" 'BEGIN {
if (actual > expected * t) { printf "FAIL: Query ttft per prompt >%d%% worse than rolling baseline\n", pct; exit 1 }
else { print "PASS: Query ttft per prompt within threshold" }
}'
fi
if [[ "$check_round" == "true" ]]; then
echo "Worst-case baseline query round time per prompt: $expected_query_round"
echo "Actual query round time per prompt: $query_round_time_per_prompt"
awk -v expected="$expected_query_round" -v actual="$query_round_time_per_prompt" -v t="$threshold" -v pct="$pct" 'BEGIN {
if (actual > expected * t) { printf "FAIL: Query round time per prompt >%d%% worse than rolling baseline\n", pct; exit 1 }
else { print "PASS: Query round time per prompt within threshold" }
}'
fi
if [[ "$check_warmup" == "true" ]]; then
echo "Worst-case baseline warmup round time per prompt: $expected_warmup"
echo "Actual warmup round time per prompt: $warmup_round_time_per_prompt"
awk -v expected="$expected_warmup" -v actual="$warmup_round_time_per_prompt" -v t="$threshold" -v pct="$pct" 'BEGIN {
if (actual > expected * t) { printf "FAIL: Warmup round time per prompt >%d%% worse than rolling baseline\n", pct; exit 1 }
else { print "PASS: Warmup round time per prompt within threshold" }
}'
fi
}
if [[ "$feature_type" == "p2p" ]]; then
run_long_doc_qa_workload "$workload_yaml" "$PORT1"
run_long_doc_qa_workload "$workload_yaml" "$PORT2" \
"$check_warmup_round_time_per_prompt" "$check_query_ttft_per_prompt" \
"$check_query_round_time_per_prompt" "${CFG_NAME%.yaml}" "$NEED_UPLOAD"
else
run_long_doc_qa_workload "$workload_yaml" "$PORT" \
"$check_warmup_round_time_per_prompt" "$check_query_ttft_per_prompt" \
"$check_query_round_time_per_prompt" "${CFG_NAME%.yaml}" "$NEED_UPLOAD"
fi
fi
###############
# MEMORY LEAK #
###############
check_memory_leak() {
local port="$1"
local socket_path="/tmp/lmcache_internal_api_server/${port}/socket_7000"
local use_hot
use_hot=$(curl -s --unix-socket "$socket_path" "http://localhost/conf" 2>/dev/null | jq -r '.local_cpu // false')
if [[ -z "$use_hot" || "$use_hot" == "null" ]]; then
use_hot="false"
fi
echo "Checking memory leak on socket_path $socket_path (use_hot=$use_hot)..."
local metrics
metrics=$(curl -s --unix-socket "$socket_path" "http://localhost/metrics" 2>/dev/null)
if [[ -z "$metrics" ]]; then
echo "ERROR: Failed to fetch metrics from $socket_path"
return 1
fi
local local_cpu_hot_cache_count active_memory_objs_count pinned_memory_objs_count pin_monitor_pinned_objects_count
local_cpu_hot_cache_count=$(echo "$metrics" | grep -E '^lmcache:local_cpu_hot_cache_count\b' | awk '{print $2}' | head -n 1)
active_memory_objs_count=$(echo "$metrics" | grep -E '^lmcache:active_memory_objs_count\b' | awk '{print $2}' | head -n 1)
pinned_memory_objs_count=$(echo "$metrics" | grep -E '^lmcache:pinned_memory_objs_count\b' | awk '{print $2}' | head -n 1)
pin_monitor_pinned_objects_count=$(echo "$metrics" | grep -E '^lmcache:pin_monitor_pinned_objects_count\b' | awk '{print $2}' | head -n 1)
local_cpu_hot_cache_count=$(printf "%.0f" "${local_cpu_hot_cache_count:-0}")
active_memory_objs_count=$(printf "%.0f" "${active_memory_objs_count:-0}")
pinned_memory_objs_count=$(printf "%.0f" "${pinned_memory_objs_count:-0}")
pin_monitor_pinned_objects_count=$(printf "%.0f" "${pin_monitor_pinned_objects_count:-0}")
echo " local_cpu_hot_cache_count: $local_cpu_hot_cache_count"
echo " active_memory_objs_count: $active_memory_objs_count"
echo " pinned_memory_objs_count: $pinned_memory_objs_count"
echo " pin_monitor_pinned_objects_count: $pin_monitor_pinned_objects_count"
local has_leak=false
if [[ "$pinned_memory_objs_count" -ne 0 ]]; then
echo "ERROR: Memory leak detected - pinned_memory_objs_count ($pinned_memory_objs_count) should be 0"
has_leak=true
fi
if [[ "$use_hot" == "false" || "$use_hot" == "False" ]]; then
if [[ "$local_cpu_hot_cache_count" -ne 0 ]]; then
echo "ERROR: Memory leak - local_cpu_hot_cache_count ($local_cpu_hot_cache_count) should be 0 when use_hot=false"
has_leak=true
fi
if [[ "$active_memory_objs_count" -ne 0 ]]; then
echo "ERROR: Memory leak - active_memory_objs_count ($active_memory_objs_count) should be 0 when use_hot=false"
has_leak=true
fi
else
if [[ "$active_memory_objs_count" -ne "$local_cpu_hot_cache_count" ]]; then
echo "ERROR: Memory leak - active_memory_objs_count ($active_memory_objs_count) should equal local_cpu_hot_cache_count ($local_cpu_hot_cache_count) when use_hot=true"
has_leak=true
fi
fi
if [[ "$has_leak" == "true" ]]; then
echo "$metrics"
return 1
fi
echo " Memory leak check passed!"
return 0
}
# Wait for metrics to settle
sleep 30
echo "--- Checking for memory leaks..."
if [[ "$feature_type" == "p2p" ]]; then
echo "Skipping memory leak check for p2p (known issue with pinned_memory_objs not clearing)."
elif [[ "$feature_type" == "pd" ]]; then
if ! check_memory_leak "$PORT1"; then
echo "Memory leak check failed for instance 1 (port $PORT1)"
exit 1
fi
if ! check_memory_leak "$PORT2"; then
echo "Memory leak check failed for instance 2 (port $PORT2)"
exit 1
fi
elif [[ "$CFG_NAME" == "multi_device.yaml" || "$CFG_NAME" == "layerwise.yaml" ]]; then
echo "Skipping memory leak check for $CFG_NAME case as it's a flaky test while run check_memory_leak check."
else
if ! check_memory_leak "$PORT"; then
echo "Memory leak check failed for $CFG_NAME"
exit 1
fi
fi
echo "===== PASS: ${CFG_NAME} ====="
.buildkite/k3_tests/comprehensive/scripts/upload-baselines.sh
Executable
+129
View File
@@ -0,0 +1,129 @@
#!/usr/bin/env bash
# Finalize nightly baseline upload.
# Runs AFTER all comprehensive config steps complete (via `wait`).
#
# 1. Downloads all *-YYYYMMDD.json artifacts from this build
# 2. Checks out benchmarks-main
# 3. Copies new date-stamped files, prunes entries older than 5 days
# 4. Makes a single commit and pushes to benchmarks-main
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
BASELINE_DIR="${REPO_ROOT}/benchmarks/long_doc_qa"
KEEP_DAYS=5
cd "${REPO_ROOT}"
# Prevent git from hanging on prompts
export GIT_TERMINAL_PROMPT=0
###############
# DOWNLOAD #
###############
ARTIFACT_DIR="/tmp/baseline_artifacts"
mkdir -p "$ARTIFACT_DIR"
echo "--- Downloading baseline artifacts from this build"
buildkite-agent artifact download "benchmarks/long_doc_qa/*-*.json" "$ARTIFACT_DIR" 2>/dev/null || {
echo "No baseline artifacts found. This is expected if no long_doc_qa configs ran."
exit 0
}
NEW_FILES=("$ARTIFACT_DIR"/benchmarks/long_doc_qa/*-*.json)
if [[ ! -e "${NEW_FILES[0]}" ]]; then
echo "No date-stamped baseline files found in artifacts."
exit 0
fi
echo "Downloaded ${#NEW_FILES[@]} baseline file(s):"
printf " %s\n" "${NEW_FILES[@]}"
###############
# CHECKOUT #
###############
# Work in a detached worktree to avoid touching the main checkout
WORK_DIR="/tmp/baselines_push_$$"
trap 'rm -rf "$WORK_DIR"' EXIT
# Push baselines to a dedicated CI repo instead of the main LMCache repo.
CI_REPO="LMCache/LMCache"
CI_BRANCH="benchmarks-main"
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
CI_REPO_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${CI_REPO}.git"
else
echo "[WARN] GITHUB_TOKEN not set — push may fail without credentials"
CI_REPO_URL="https://github.com/${CI_REPO}.git"
fi
echo "--- Preparing ${CI_BRANCH} branch from ${CI_REPO}"
git clone --depth=1 --branch "${CI_BRANCH}" "${CI_REPO_URL}" "$WORK_DIR" 2>/dev/null || {
# Branch doesn't exist yet — create an orphan
mkdir -p "$WORK_DIR"
git -C "$WORK_DIR" init
git -C "$WORK_DIR" remote add origin "${CI_REPO_URL}"
git -C "$WORK_DIR" checkout --orphan "${CI_BRANCH}"
}
###############
# COPY + PRUNE#
###############
mkdir -p "$WORK_DIR/benchmarks/long_doc_qa"
# Copy new files
for f in "${NEW_FILES[@]}"; do
cp "$f" "$WORK_DIR/benchmarks/long_doc_qa/"
done
# Prune date-stamped files older than KEEP_DAYS
echo "--- Pruning baselines older than ${KEEP_DAYS} days"
CUTOFF_DATE="$(date -d "${KEEP_DAYS} days ago" +%Y%m%d 2>/dev/null || date -v-${KEEP_DAYS}d +%Y%m%d)"
for f in "$WORK_DIR"/benchmarks/long_doc_qa/*-*.json; do
[[ ! -e "$f" ]] && continue
fname="$(basename "$f")"
# Extract YYYYMMDD from filename like "local_cpu-20260301.json"
file_date="${fname##*-}"
file_date="${file_date%.json}"
if [[ "$file_date" =~ ^[0-9]{8}$ ]] && [[ "$file_date" < "$CUTOFF_DATE" ]]; then
echo " Removing old baseline: $fname"
rm "$f"
fi
done
# Also remove legacy single-point files (feature.json without date) since we now use rolling
# Keep them if they exist for backward compat during transition, but don't create new ones.
###############
# COMMIT+PUSH #
###############
cd "$WORK_DIR"
git add benchmarks/long_doc_qa/
if git diff --cached --quiet 2>/dev/null; then
echo "No changes to commit."
exit 0
fi
TODAY="$(date +%Y-%m-%d)"
git -c user.email="ci@lmcache.ai" -c user.name="LMCache CI" \
commit -m "Update rolling baselines: ${TODAY}" || true
echo "--- Pushing to ${CI_REPO} ${CI_BRANCH}"
# Try normal push first; fall back to force-push if history diverged
if ! git push origin "HEAD:${CI_BRANCH}" 2>/dev/null; then
echo "[WARN] Normal push failed, force-pushing..."
git push origin "+HEAD:${CI_BRANCH}" 2>/dev/null || {
echo "[ERROR] Failed to push baselines to ${CI_REPO} ${CI_BRANCH}"
exit 1
}
fi
echo "--- Baselines uploaded successfully"
git log --oneline -1
ls -la benchmarks/long_doc_qa/
.buildkite/k3_tests/correctness/BK_WEB_SETUP.md
+14
View File
@@ -0,0 +1,14 @@
# Buildkite Web UI Setup: Correctness Tests
**Steps editor**: paste contents of `buildkite-pipeline.yml` (fill in `HF_TOKEN`).
**GitHub trigger settings**:
- Filter: *(none — runs on every push/PR)*
- Skip queued / cancel running branch builds: Yes
Lightweight (1 GPU) — good candidate for a required GitHub status check.
> Builds whose only changes are docs/`*.md`/`LICENSE`/`.github/**` auto-pass
> via the [path filter](../README.md#path-based-skip-auto-pass-on-docs-only-changes).
> Changes under `.buildkite/` always run. Add `force-ci` label to the PR to
> bypass.
.buildkite/k3_tests/correctness/buildkite-pipeline.yml
+11
View File
@@ -0,0 +1,11 @@
# Paste this into the Buildkite pipeline's "Steps" editor.
# It uploads the real pipeline definition from the repo.
agents:
queue: "k8s"
env:
HF_TOKEN: "" # set your HuggingFace token here (needed for gated model access)
steps:
- label: ":pipeline: Upload pipeline"
command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/correctness/pipeline.yml
.buildkite/k3_tests/correctness/pipeline.yml
+27
View File
@@ -0,0 +1,27 @@
# Correctness test — verifies LMCache produces identical output to base vLLM.
# Runs: base vLLM → LMCache vLLM → man bash prefix test → ShareGPT diff.
# Requires pre-downloaded model weights and ShareGPT dataset.
steps:
- label: ":white_check_mark: Correctness"
command: .buildkite/k3_tests/correctness/run.sh
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources:
limits:
nvidia.com/gpu: "1"
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
- { name: datasets, mountPath: /root/correctness }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
- { name: datasets, hostPath: { path: /data/datasets, type: DirectoryOrCreate } }
artifact_paths:
- "build_*.log"
.buildkite/k3_tests/correctness/run.sh
Executable
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env bash
# Correctness test entrypoint for K8s pods.
# Thin wrapper: sets up environment, then delegates to scripts/.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
cd "${REPO_ROOT}"
# ── Environment setup ────────────────────────────────────────
source .buildkite/k3_harness/setup-env.sh
uv pip install aiohttp tqdm pandas huggingface_hub
# ── Ensure all scripts are executable ────────────────────────
chmod +x "${SCRIPT_DIR}"/scripts/*.sh
# ── Run the actual test logic ────────────────────────────────
# Retry up to 3 times. vLLM's `VLLM_BATCH_INVARIANT=1` guarantee
# currently does not hold across process restarts in the vllm nightlies
# we pin to (reproduced locally: two back-to-back vanilla vllm-only runs
# produce ~55 different outputs out of 100). Phase 4's bitwise
# comparison between the base-vllm run and the vllm+LMCache run is
# bimodal (0 diff or ~50 diff) depending on which cuBLAS/kernel plans
# the two separate processes happen to pick. Retrying lets a lucky
# scheduling pass; a real LMCache regression will persistently fail.
MAX_ATTEMPTS="${CORRECTNESS_MAX_ATTEMPTS:-3}"
for attempt in $(seq 1 "${MAX_ATTEMPTS}"); do
echo "=== Correctness attempt ${attempt}/${MAX_ATTEMPTS} ==="
if bash "${SCRIPT_DIR}/scripts/run-correctness.sh"; then
exit 0
fi
echo "[INFO] Attempt ${attempt} failed."
done
echo "[FAIL] Correctness persistently failed after ${MAX_ATTEMPTS} attempts." >&2
exit 1
.buildkite/k3_tests/correctness/scripts/run-correctness.sh
Executable
+221
View File
@@ -0,0 +1,221 @@
#!/usr/bin/env bash
# Self-contained correctness test for K8s pods.
# Replaces the old vllm-correctness.sh which assumed a pre-existing venv
# and pick-free-gpu.sh. Here everything runs natively in the pod.
#
# Tests that LMCache produces identical output to base vLLM.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
cd "${REPO_ROOT}"
source .buildkite/k3_tests/common_scripts/helpers.sh
###############
# CONFIG #
###############
BUILD_ID="${BUILDKITE_BUILD_ID:-local_$$}"
MODEL="Qwen/Qwen2.5-14B-Instruct"
WORK_LOG="/tmp/build_${BUILD_ID}_correctness.log"
VLLM_LOG="/tmp/build_${BUILD_ID}_vllm.log"
ARTIFACT="build_${BUILD_ID}.log"
SERVER_WAIT_TIMEOUT=180
CORRECTNESS_DIR=".buildkite/correctness"
REQUEST_NUMBER=100
MAX_CONCURRENCY=40
# K8s assigns GPUs via device plugin
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}"
###############
# DATA SETUP #
###############
SHAREGPT_PATH="/root/correctness/.ShareGPT_V3_unfiltered_cleaned_split.json"
if [[ ! -f "$SHAREGPT_PATH" ]]; then
echo "[INFO] ShareGPT dataset not found, downloading..."
mkdir -p /root/correctness
wget -q \
"https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" \
-O "$SHAREGPT_PATH"
fi
if [[ ! -s "$SHAREGPT_PATH" ]]; then
echo "[ERROR] ShareGPT dataset file is empty: $SHAREGPT_PATH"
exit 1
fi
###############
# HELPERS #
###############
VLLM_PID=""
CI_CACHE_DIR="$PWD/.vllm_cache_${BUILD_ID}"
mkdir -p "$CI_CACHE_DIR"
export FLASHINFER_WORKSPACE_DIR="$CI_CACHE_DIR/flashinfer"
collect_artifact() {
echo "[INFO] Collecting logs into ${ARTIFACT}"
cat "${WORK_LOG}" "${VLLM_LOG}" > "${ARTIFACT}" 2>/dev/null || true
}
stop_vllm() {
if [[ -n "${VLLM_PID:-}" ]]; then
echo "[INFO] Stopping vLLM process (PID: ${VLLM_PID})"
kill "${VLLM_PID}" >/dev/null 2>&1 || true
wait "${VLLM_PID}" 2>/dev/null || true
VLLM_PID=""
sleep 5
fi
}
trap 'rc=$?; stop_vllm; collect_artifact; exit $rc' EXIT INT TERM
exec > >(tee -a "${WORK_LOG}") 2>&1
echo "=== DIAGNOSTICS: GPU STATE before CI ==="
nvidia-smi
echo "[INFO] Using GPU(s): ${CUDA_VISIBLE_DEVICES}"
echo "[INFO] Converting ShareGPT dataset to OpenAI format..."
python "${CORRECTNESS_DIR}/sharegpt2openai.py" -i "$SHAREGPT_PATH" -o "./shareGPT_dataset.json"
###############
# PHASE 1 #
###############
# Base Server (Baseline) -- vLLM WITHOUT LMCache
PORT=$(find_free_port 8000)
echo "[INFO] Starting BASE vLLM server on port ${PORT}..."
VLLM_SERVER_DEV_MODE=1 \
VLLM_BATCH_INVARIANT=1 \
vllm serve "${MODEL}" \
--port "${PORT}" \
--trust-remote-code \
--enforce-eager \
--attention-backend FLASH_ATTN \
--gpu-memory-utilization 0.8 \
>"${VLLM_LOG}" 2>&1 &
VLLM_PID=$!
echo "[INFO] Waiting for Base server readiness..."
if ! wait_for_server "$PORT" "$SERVER_WAIT_TIMEOUT"; then
echo "[ERROR] Base vLLM failed to start"
exit 1
fi
echo "[TEST] Running ShareGPT Batch (Base)..."
python "${CORRECTNESS_DIR}/async_request.py" \
--model "${MODEL}" \
--endpoint "http://localhost:${PORT}/v1/chat/completions" \
--output-file "sharegpt_base.txt" \
--dataset_file "./shareGPT_dataset.json" \
--request-number "${REQUEST_NUMBER}" \
--max-concurrency "${MAX_CONCURRENCY}"
stop_vllm
###############
# PHASE 2 #
###############
# LMCache Server -- vLLM WITH LMCache
echo "[INFO] Preparing LMCache config (cpu.yaml)..."
cat <<EOF > cpu.yaml
chunk_size: 256
local_cpu: true
max_local_cpu_size: 50
EOF
PORT=$(find_free_port 8000)
echo "[INFO] Starting LMCACHE vLLM server on port ${PORT}..."
LMCACHE_CONFIG_FILE=cpu.yaml \
VLLM_SERVER_DEV_MODE=1 \
VLLM_BATCH_INVARIANT=1 \
vllm serve "${MODEL}" \
--port "${PORT}" \
--trust-remote-code \
--enforce-eager \
--attention-backend FLASH_ATTN \
--gpu-memory-utilization 0.8 \
--kv-transfer-config '{"kv_connector":"LMCacheConnectorV1","kv_role":"kv_both"}' \
>>"${VLLM_LOG}" 2>&1 &
VLLM_PID=$!
echo "[INFO] Waiting for LMCache server readiness..."
if ! wait_for_server "$PORT" "$SERVER_WAIT_TIMEOUT"; then
echo "[ERROR] LMCache vLLM failed to start"
exit 1
fi
echo "[TEST] Running ShareGPT Batch (LMCache)..."
python "${CORRECTNESS_DIR}/async_request.py" \
--model "${MODEL}" \
--endpoint "http://localhost:${PORT}/v1/chat/completions" \
--output-file "sharegpt_lmcache.txt" \
--dataset_file "./shareGPT_dataset.json" \
--request-number "${REQUEST_NUMBER}" \
--max-concurrency "${MAX_CONCURRENCY}"
###############
# PHASE 3 #
###############
# Man bash correctness test (on the LMCache server)
echo "[TEST] Running technical man bash correctness test..."
CONTEXT="$(man bash | sed 's/.\x08//g' | tr -s '[:space:]' ' ' | awk '{for(i=1;i<=NF;i++){printf "%s ",$i; if(++c==5000) exit}}')"
HALF_CONTEXT="$(man bash | sed 's/.\x08//g' | tr -s '[:space:]' ' ' | awk '{for(i=1;i<=NF;i++){printf "%s ",$i; if(++c==2500) exit}}')"
send_completion() {
curl -s "http://localhost:${PORT}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "$(jq -n --arg model "${MODEL}" --arg content "$1" '{model: $model, temperature: 0, max_tokens: 100, messages: [{role:"user",content:$content}]}')" |
jq -r '.choices[0].message.content'
}
echo "[STEP 1] Full context (LMCache)"
OUT1="$(send_completion "${CONTEXT}")"
echo "[STEP 2] Reset prefix cache"
curl -s -X POST "http://localhost:${PORT}/reset_prefix_cache" >/dev/null
echo "[STEP 3] Half context"
send_completion "${HALF_CONTEXT}" >/dev/null
echo "[STEP 4] Full context again"
OUT2="$(send_completion "${CONTEXT}")"
if [[ "${OUT1}" != "${OUT2}" ]]; then
echo "[FAIL] man bash output mismatch!"
exit 1
fi
###############
# PHASE 4 #
###############
# ShareGPT File Comparison
echo "[INFO] Comparing ShareGPT results..."
COMPARE_OUT=$(python "${CORRECTNESS_DIR}/compare_files.py" --file1 "sharegpt_base.txt" --file2 "sharegpt_lmcache.txt")
echo "--- COMPARISON RESULTS ---"
echo "${COMPARE_OUT}"
echo "--------------------------"
DIFFERENT_COUNT=$(echo "${COMPARE_OUT}" | grep "^Different IDs:" | grep -oE '[0-9]+' | tail -1 || echo "0")
ONLY_FILE1_COUNT=$(echo "${COMPARE_OUT}" | awk '/^—— Only in File 1 ——$/,/^—— Only in File 2 ——$/ {print}' | grep -cE "^chatcmpl-" || echo "0")
ONLY_FILE2_COUNT=$(echo "${COMPARE_OUT}" | awk '/^—— Only in File 2 ——$/,/^$/ {print}' | grep -cE "^chatcmpl-" || echo "0")
echo "[INFO] Analysis: Different=${DIFFERENT_COUNT}, Only in Base=${ONLY_FILE1_COUNT}, Only in LMCache=${ONLY_FILE2_COUNT}"
if [[ "${DIFFERENT_COUNT}" -gt 0 ]] || [[ "${ONLY_FILE1_COUNT}" -gt 0 ]] || [[ "${ONLY_FILE2_COUNT}" -gt 0 ]]; then
echo "[FAIL] Inconsistency detected between Base and LMCache outputs."
exit 1
fi
echo "[PASS] All correctness tests passed -- identical output verified."
.buildkite/k3_tests/integration/BK_WEB_SETUP.md
+14
View File
@@ -0,0 +1,14 @@
# Buildkite Web UI Setup: Integration Tests
**Steps editor**: paste contents of `buildkite-pipeline.yml` (fill in `HF_TOKEN`).
**GitHub trigger settings**:
- Filter: *(none — runs on every push/PR)*
- Skip queued / cancel running branch builds: Yes
Lightweight (1 GPU) — good candidate for a required GitHub status check.
> Builds whose only changes are docs/`*.md`/`LICENSE`/`.github/**` auto-pass
> via the [path filter](../README.md#path-based-skip-auto-pass-on-docs-only-changes).
> Changes under `.buildkite/` always run. Add `force-ci` label to the PR to
> bypass.
.buildkite/k3_tests/integration/buildkite-pipeline.yml
+11
View File
@@ -0,0 +1,11 @@
# Paste this into the Buildkite pipeline's "Steps" editor.
# It uploads the real pipeline definition from the repo.
agents:
queue: "k8s"
env:
HF_TOKEN: "" # set your HuggingFace token here (needed for gated model access)
steps:
- label: ":pipeline: Upload pipeline"
command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/integration/pipeline.yml
.buildkite/k3_tests/integration/pipeline.yml
+49
View File
@@ -0,0 +1,49 @@
# Integration tests — starts vLLM with LMCache directly (no Docker),
# tests CPU and disk backends via the OpenAI API.
steps:
- label: ":test_tube: Integration — CPU backend"
command: .buildkite/k3_tests/integration/run.sh cpu
timeout_in_minutes: 20
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources:
limits:
nvidia.com/gpu: "1"
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
artifact_paths:
- "*.csv"
- "*.pdf"
- "*.log"
- label: ":test_tube: Integration — Disk backend"
command: .buildkite/k3_tests/integration/run.sh disk
timeout_in_minutes: 20
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources:
limits:
nvidia.com/gpu: "1"
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
artifact_paths:
- "*.csv"
- "*.pdf"
- "*.log"
.buildkite/k3_tests/integration/run.sh
Executable
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
# Integration test entrypoint for K8s pods.
# Thin wrapper: sets up environment, then delegates to scripts/.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
cd "${REPO_ROOT}"
# ── Environment setup ────────────────────────────────────────
source .buildkite/k3_harness/setup-env.sh
uv pip install aiohttp
# ── Ensure all scripts are executable ────────────────────────
chmod +x "${SCRIPT_DIR}"/scripts/*.sh
# ── Run the actual test logic ────────────────────────────────
exec bash "${SCRIPT_DIR}/scripts/run-integration.sh" "$@"
.buildkite/k3_tests/integration/scripts/run-integration.sh
Executable
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Integration test logic: starts vLLM with LMCache (CPU and disk backends),
# sends requests via the OpenAI API, and verifies responses.
# Runs directly in the K8s pod -- no Docker.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
cd "${REPO_ROOT}"
source .buildkite/k3_tests/common_scripts/helpers.sh
TEST_NAME="${1:-all}" # "cpu", "disk", or "all"
MODEL="meta-llama/Llama-3.2-1B-Instruct"
PORT=8000
VLLM_PID=""
cleanup() {
if [[ -n "$VLLM_PID" ]]; then
echo "--- Shutting down vLLM (PID=$VLLM_PID)"
kill "$VLLM_PID" 2>/dev/null || true
wait "$VLLM_PID" 2>/dev/null || true
fi
}
trap cleanup EXIT
send_request() {
local port=$1 prompt=$2
curl -sf "http://localhost:${port}/v1/completions" \
-H "Content-Type: application/json" \
-d "{
\"model\": \"${MODEL}\",
\"prompt\": \"${prompt}\",
\"max_tokens\": 64,
\"temperature\": 0
}"
}
run_test() {
local test_name=$1
shift
local env_vars=("$@")
echo "--- :test_tube: ${test_name}"
# Start vLLM with LMCache
env "${env_vars[@]}" \
vllm serve "$MODEL" \
--port "$PORT" \
--load-format dummy \
--enforce-eager \
--no-enable-prefix-caching \
--gpu-memory-utilization 0.8 \
--kv-transfer-config '{"kv_connector":"LMCacheConnectorV1","kv_role":"kv_both"}' \
> "${test_name}-vllm.log" 2>&1 &
VLLM_PID=$!
wait_for_server "$PORT" 300 "${test_name}-vllm.log"
# Send requests -- same prompt twice to exercise the cache (second hit should be cached)
local prompt="The quick brown fox jumps over the lazy dog. Explain this sentence in detail."
echo "Sending request 1 (cold)..."
local resp1
resp1=$(send_request "$PORT" "$prompt")
echo "Response 1: $(echo "$resp1" | python3 -c 'import sys,json; r=json.load(sys.stdin); print(r["choices"][0]["text"][:100])' 2>/dev/null || echo "PARSE_ERROR")"
echo "Sending request 2 (should hit cache)..."
local resp2
resp2=$(send_request "$PORT" "$prompt")
echo "Response 2: $(echo "$resp2" | python3 -c 'import sys,json; r=json.load(sys.stdin); print(r["choices"][0]["text"][:100])' 2>/dev/null || echo "PARSE_ERROR")"
# Verify we got valid responses
for resp_name in resp1 resp2; do
local resp_val="${!resp_name}"
if ! echo "$resp_val" | python3 -c 'import sys,json; r=json.load(sys.stdin); assert r["choices"][0]["text"]' 2>/dev/null; then
echo "FAIL: ${test_name} -- invalid response in ${resp_name}"
echo "$resp_val"
kill "$VLLM_PID" 2>/dev/null; wait "$VLLM_PID" 2>/dev/null || true; VLLM_PID=""
return 1
fi
done
echo "PASS: ${test_name}"
# Shut down for next test
kill "$VLLM_PID" 2>/dev/null; wait "$VLLM_PID" 2>/dev/null || true; VLLM_PID=""
sleep 2
}
# ── Test 1: CPU backend ───────────────────────────────────────
if [[ "$TEST_NAME" == "cpu" || "$TEST_NAME" == "all" ]]; then
run_test "local_cpu" \
"LMCACHE_CHUNK_SIZE=256" \
"LMCACHE_LOCAL_CPU=True" \
"LMCACHE_MAX_LOCAL_CPU_SIZE=5"
fi
# ── Test 2: Disk backend ─────────────────────────────────────
if [[ "$TEST_NAME" == "disk" || "$TEST_NAME" == "all" ]]; then
DISK_DIR=$(mktemp -d)
trap "cleanup; rm -rf $DISK_DIR" EXIT
run_test "local_disk" \
"LMCACHE_CHUNK_SIZE=256" \
"LMCACHE_LOCAL_CPU=True" \
"LMCACHE_MAX_LOCAL_CPU_SIZE=5" \
"LMCACHE_LOCAL_DISK=file://${DISK_DIR}"
fi
echo "--- :white_check_mark: Integration tests passed (${TEST_NAME})"
.buildkite/k3_tests/multiprocess/BK_WEB_SETUP.md
+15
View File
@@ -0,0 +1,15 @@
# Buildkite Web UI Setup: Multiprocess Tests
**Steps editor**: paste contents of `buildkite-pipeline.yml` (fill in `HF_TOKEN`).
**GitHub trigger settings**:
- Filter: `build.pull_request.labels includes "mp" || build.pull_request.labels includes "full" || build.branch == 'dev'`
- Rebuild on PR label change: Yes
- Skip queued / cancel running branch builds: Yes
Heavy test (2 GPUs, Docker-in-Docker, ~45 min) — run on `"mp"`/`"full"` label or dev push, not every PR.
> Builds whose only changes are docs/`*.md`/`LICENSE`/`.github/**` auto-pass
> via the [path filter](../README.md#path-based-skip-auto-pass-on-docs-only-changes).
> Changes under `.buildkite/` always run. Add `force-ci` label to the PR to
> bypass.
.buildkite/k3_tests/multiprocess/buildkite-pipeline.yml
+11
View File
@@ -0,0 +1,11 @@
# Paste this into the Buildkite pipeline's "Steps" editor.
# It uploads the real pipeline definition from the repo.
agents:
queue: "k8s"
env:
HF_TOKEN: "" # set your HuggingFace token here (needed for gated model access)
steps:
- label: ":pipeline: Upload pipeline"
command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/multiprocess/pipeline.yml
.buildkite/k3_tests/multiprocess/pipeline.yml
+279
View File
@@ -0,0 +1,279 @@
# Multiprocess tests — one step per test, run in parallel.
# Each step gets its own K8s pod with the exact GPU count it needs.
# Tests are grouped by GPU requirement; the 2-GPU group is listed first so
# these heavier jobs get scheduled before 1-GPU jobs.
# 2-GPU: LMCache+vLLM on GPU 0, baseline on GPU 1 (or TP=2 across both).
# 1-GPU: LMCache+vLLM on GPU 0 only (no baseline server).
# Retry policy for known-flaky long-running multiprocess steps.
# Note: exit_status "*" matches 1-255 only; -1 (agent lost) needs a separate rule.
x-flaky-retry: &flaky-retry
automatic:
- exit_status: -1
limit: 2
- exit_status: "*"
limit: 2
steps:
- group: ":compression: Multiprocess (2-GPU)"
if: build.env("VERIFY_AND_PIN_VLLM") !~ /true/
steps:
- label: ":compression: vllm_bench"
command: .buildkite/k3_tests/multiprocess/run.sh vllm_bench
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec: &pod-2gpu
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources: { limits: { "nvidia.com/gpu": "2" } }
volumeMounts: &vol-mounts
- { name: hf-cache, mountPath: /root/.cache/huggingface }
volumes: &vols
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
artifact_paths: ["*.log"]
- label: ":compression: long_doc_qa"
command: .buildkite/k3_tests/multiprocess/run.sh long_doc_qa
timeout_in_minutes: 30
retry: *flaky-retry
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-2gpu } }]
artifact_paths: ["*.log"]
- label: ":compression: long_doc_qa_l2"
command: .buildkite/k3_tests/multiprocess/run.sh long_doc_qa_l2
timeout_in_minutes: 30
retry: *flaky-retry
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-2gpu } }]
artifact_paths: ["*.log"]
- label: ":compression: deadlock"
command: .buildkite/k3_tests/multiprocess/run.sh deadlock
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-2gpu } }]
artifact_paths: ["*.log"]
# Two LMCache MP instances + two vLLM instances + one coordinator on a
# single 2-GPU pod (localhost). Verifies engine B serves a long prompt's
# KV from engine A over P2P.
- label: ":compression: p2p"
command: .buildkite/k3_tests/multiprocess/run.sh p2p
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-2gpu } }]
artifact_paths: ["*.log"]
- group: ":compression: Multiprocess (1-GPU)"
if: build.env("VERIFY_AND_PIN_VLLM") !~ /true/
steps:
- label: ":compression: lm_eval"
command: .buildkite/k3_tests/multiprocess/run.sh lm_eval
timeout_in_minutes: 30
retry: *flaky-retry
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec: &pod-1gpu
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources: { limits: { "nvidia.com/gpu": "1" } }
volumeMounts: *vol-mounts
volumes: *vols
artifact_paths: ["*.log"]
# Engine-driven transfer mode correctness on GPU (SHM transport).
# Default CUDA dispatch is lmcache_driven (server-side IPC handle);
# this step forces engine_driven (worker-side gather/scatter via SHM)
# and verifies gsm8k outputs are still bit-exact across cache-population
# and cache-hit runs.
#
# To enable SHM transport:
# - L1_USE_LAZY=false disables lazy allocation so L1 is backed by a
# POSIX SHM segment that the worker can mmap.
# - CPU_BUFFER_SIZE=20 caps L1 at 20GB (needs /dev/shm >= 20GB).
# - dshm volume provides a 24Gi tmpfs mount at /dev/shm.
- label: ":compression: lm_eval_engine_driven"
command: .buildkite/k3_tests/multiprocess/run.sh lm_eval
timeout_in_minutes: 30
env:
LMCACHE_MP_TRANSFER_MODE: "engine_driven"
L1_USE_LAZY: "false"
CPU_BUFFER_SIZE: "80"
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources: { limits: { "nvidia.com/gpu": "1" } }
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
- { name: dshm, mountPath: /dev/shm }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
- { name: dshm, emptyDir: { medium: Memory, sizeLimit: 128Gi } }
artifact_paths: ["*.log"]
# HMA correctness check on google/gemma-4-31B-it (a hybrid model whose KV
# cache groups get different block sizes). Runs gsm8k, resets vLLM's prefix
# cache (LMCache preserved), reruns served by LMCache, and asserts the two
# runs' scores match. Single GPU, no baseline.
- label: ":compression: hma_lm_eval_gemma4"
command: .buildkite/k3_tests/multiprocess/run.sh hma_lm_eval_gemma4
timeout_in_minutes: 60
retry: *flaky-retry
env:
MODEL: "google/gemma-4-31B-it"
# Allow a small score difference between the two runs.
SCORE_TOLERANCE: "0.03"
ATTENTION_BACKEND: "auto"
GPU_MEMORY_UTILIZATION: "0.85"
# 31B load + CUDA-graph capture is slow; raise the readiness timeout
# above the 300s default.
MAX_WAIT_SECONDS: "600"
# Cap samples and enlarge the CPU pool so the retrieve run stays
# cache-served (31B's per-token KV is large).
LIMIT: "100"
CPU_BUFFER_SIZE: "200"
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log"]
- label: ":compression: fault_tolerance"
command: .buildkite/k3_tests/multiprocess/run.sh fault_tolerance
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log"]
- label: ":compression: restart_recovery"
command: .buildkite/k3_tests/multiprocess/run.sh restart_recovery
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log"]
- label: ":compression: cache_stats"
command: .buildkite/k3_tests/multiprocess/run.sh cache_stats
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log"]
- label: ":compression: http_api"
command: .buildkite/k3_tests/multiprocess/run.sh http_api
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins: [{ kubernetes: { podSpec: *pod-1gpu } }]
artifact_paths: ["*.log"]
- label: ":compression: gds_smoke_test"
command: .buildkite/k3_tests/multiprocess/run.sh gds_smoke_test
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources: { limits: { "nvidia.com/gpu": "1" } }
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
- { name: scratch, mountPath: /scratch }
- { name: udev, mountPath: /run/udev, readOnly: true }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
- { name: scratch, hostPath: { path: /data/gds-scratch, type: DirectoryOrCreate } }
- { name: udev, hostPath: { path: /run/udev, type: Directory } }
artifact_paths: ["*.log"]
- group: ":compression: Multiprocess (CPU-only)"
if: build.env("VERIFY_AND_PIN_VLLM") !~ /true/
steps:
# Note: LMCACHE_SHM_NAME uses `${VAR-default}` (not `:-`) in the
# runner script, so an empty string ("") is meaningful and differs
# from unset. `__default__` is the sentinel meaning "shm mode".
- label: ":compression: cpu_e2e_validation ({{matrix.mode}})"
# matrix.adjustments does not support `env:` in Buildkite. Instead
# we branch on {{matrix.mode}} inside the command to set the
# per-mode env before invoking the shared runner script.
command: |
case "{{matrix.mode}}" in
shm) ;; # rely on script defaults (engine-driven + shm transport)
pickle) export LMCACHE_SHM_NAME="" ;;
server_copy) export LMCACHE_MP_TRANSFER_MODE="lmcache_driven" ;;
esac
bash .github/scripts/run-cpu-e2e-validation.sh
matrix:
setup:
mode:
- "shm"
- "pickle"
- "server_copy"
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources:
requests:
cpu: "8"
memory: "256Gi"
limits:
cpu: "8"
memory: "256Gi"
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
- { name: dshm, mountPath: /dev/shm }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
- { name: dshm, emptyDir: { medium: Memory, sizeLimit: 4Gi } }
# Canary path for VERIFY_AND_PIN_VLLM=true: skips every real test group
# above (each group's `if` evaluates to false) and instead runs a single
# vllm_bench step that proves the freshly-installed vLLM nightly works
# with LMCache. On success, the verified vllm wheel version is pushed to
# the `buildkite_latest_tested_vllm` branch so downstream builds can pin
# to a known-good nightly instead of resolving "latest" again.
- label: ":pushpin: Verify & pin vLLM nightly"
if: build.env("VERIFY_AND_PIN_VLLM") =~ /true/
command: |
.buildkite/k3_tests/multiprocess/run.sh vllm_bench
bash .buildkite/k3_tests/multiprocess/scripts/pin-tested-vllm.sh
timeout_in_minutes: 30
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources: { limits: { "nvidia.com/gpu": "2" } }
env:
- name: GITHUB_TOKEN
valueFrom:
secretKeyRef:
name: buildkite-git-creds
key: GITHUB_TOKEN
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
artifact_paths: ["*.log"]
.buildkite/k3_tests/multiprocess/run.sh
Executable
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
# Multiprocess test entrypoint for K8s pods.
# Usage: run.sh <test_name>
# test_name: lm_eval | hma_lm_eval_gemma4 | vllm_bench | long_doc_qa
# | long_doc_qa_l2 | fault_tolerance | deadlock | restart_recovery
# | gds_smoke_test | p2p
# Thin wrapper: sets up environment, then delegates to scripts/.
# No Docker -- all processes run natively in the pod.
set -euo pipefail
TEST_NAME="${1:?Usage: $0 <test_name> (lm_eval|hma_lm_eval_gemma4|vllm_bench|long_doc_qa|long_doc_qa_l2|fault_tolerance|deadlock|restart_recovery|cache_stats|http_api|p2p)}"
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
cd "${REPO_ROOT}"
# ── Environment setup ────────────────────────────────────────
source .buildkite/k3_harness/setup-env.sh
# Install test extras (lm-eval for eval workload, openai/pandas/matplotlib for benchmarks)
uv pip install 'lm-eval[api]' openai pandas matplotlib
# ── Ensure all scripts are executable ────────────────────────
chmod +x "${SCRIPT_DIR}"/scripts/*.sh
# ── Run the actual test logic ────────────────────────────────
exec bash "${SCRIPT_DIR}/scripts/run-single-test.sh" "$TEST_NAME"
.buildkite/k3_tests/multiprocess/scripts/cleanup.sh
Executable
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env bash
# Cleanup background processes launched for multiprocessing tests.
# This script should always be called, even on failure.
BUILD_ID="${BUILD_ID:-local_$$}"
PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}"
echo "=== Cleaning up background processes ==="
if [[ -f "$PID_FILE" ]]; then
while IFS= read -r pid; do
if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
echo "Killing PID $pid"
# Capture a snippet of the process log before killing
kill "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
echo "PID $pid stopped"
fi
done < "$PID_FILE"
rm -f "$PID_FILE"
else
echo "No PID file found at $PID_FILE"
fi
# Also kill any stray vllm/lmcache processes from this build
# (safety net in case PIDs weren't recorded)
for port in "${VLLM_PORT:-8000}" "${VLLM_BASELINE_PORT:-9000}" "${LMCACHE_PORT:-6555}"; do
fuser -k "${port}/tcp" 2>/dev/null || true
done
# Remove the GDS slab scratch dir (only set for gds_* tests). It lives on the
# /scratch hostPath (host-local NVMe), so it persists past the pod and the
# preallocated slab is large -- drop it now that the server is stopped.
if [[ -n "${GDS_L1_PATH:-}" ]]; then
echo "Removing GDS slab dir: $GDS_L1_PATH"
rm -rf "${GDS_L1_PATH}" 2>/dev/null || true
fi
echo "=== Cleanup complete ==="
# Copy server logs to the workspace so Buildkite can collect them as artifacts
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
cp /tmp/build_${BUILD_ID}_*.log "${REPO_ROOT}/" 2>/dev/null || true
# Wait for GPU memory to be fully released
echo "Waiting 5 seconds for GPU memory to be released..."
sleep 5
.buildkite/k3_tests/multiprocess/scripts/launch-processes.sh
Executable
+199
View File
@@ -0,0 +1,199 @@
#!/usr/bin/env bash
# Launch LMCache MP server, vLLM with LMCache, and vLLM baseline
# as native background processes (no Docker).
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# Configuration (inherited from run-mp-test.sh)
LMCACHE_PORT="${LMCACHE_PORT:-6555}"
vllm_port="${VLLM_PORT:-8000}"
vllm_baseline_port="${VLLM_BASELINE_PORT:-9000}"
CPU_BUFFER_SIZE="${CPU_BUFFER_SIZE:-80}"
MAX_WORKERS="${MAX_WORKERS:-4}"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
BUILD_ID="${BUILD_ID:-local_$$}"
# K8s assigns exactly 2 GPUs as devices 0 and 1 (overridable for local runs).
GPU_FOR_VLLM="${GPU_FOR_VLLM:-0}"
GPU_FOR_BASELINE="${GPU_FOR_BASELINE:-1}"
echo "Using GPU $GPU_FOR_VLLM for vLLM with LMCache"
echo "Using GPU $GPU_FOR_BASELINE for vLLM baseline"
# Check GPU memory and set gpu-memory-utilization for very large GPUs.
# Without this, vLLM allocates so much KV cache that APC covers all prefixes
# and LMCache's cache path is never exercised, making the test pass vacuously.
GPU_MEMORY_UTIL_ARG=""
GPU_MEMORY_MB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits -i "${GPU_FOR_VLLM}" | tr -d ' ')
GPU_MEMORY_GB=$((GPU_MEMORY_MB / 1024))
echo "Detected GPU memory: ${GPU_MEMORY_GB}GB (${GPU_MEMORY_MB}MB)"
if [ -n "${GPU_MEMORY_UTILIZATION:-}" ]; then
# Explicit override (e.g. large models like gemma-4-31B whose ~63GB of
# weights alone exceed the default 0.5 fraction and would fail to load).
echo "Using configured --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION}"
GPU_MEMORY_UTIL_ARG="--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION}"
elif [ "$GPU_MEMORY_GB" -gt 90 ]; then
echo "GPU memory > 90GB, adding --gpu-memory-utilization 0.5"
GPU_MEMORY_UTIL_ARG="--gpu-memory-utilization 0.5"
fi
# Attention backend for both vLLM servers. Defaults to FLASH_ATTN (what the
# batch-invariant lm_eval needs). Models with heterogeneous head dimensions
# (e.g. gemma-4) must NOT pin FLASH_ATTN -- set ATTENTION_BACKEND=auto so vLLM
# selects the backend itself (gemma-4 auto-forces TRITON_ATTN).
ATTENTION_BACKEND="${ATTENTION_BACKEND:-FLASH_ATTN}"
ATTENTION_BACKEND_ARG=""
if [ -n "$ATTENTION_BACKEND" ] && [ "$ATTENTION_BACKEND" != "auto" ]; then
ATTENTION_BACKEND_ARG="--attention-backend $ATTENTION_BACKEND"
fi
# Optionally run vLLM in eager mode (skip CUDA graph capture) for both servers.
# Off by default: verified to break the bit-exact run1 == run2 check in the
# determinism tests (lm_eval) -- eager changes the kernel path enough to diverge
# across the cold/warm batch difference even under VLLM_BATCH_INVARIANT. Enable
# (ENFORCE_EAGER=1) only for large models whose CUDA-graph capture would
# otherwise time out at launch (those tests use a tolerance, not bit-exactness).
ENFORCE_EAGER_ARG=""
if [ "${ENFORCE_EAGER:-0}" = "1" ] || [ "${ENFORCE_EAGER:-0}" = "true" ]; then
ENFORCE_EAGER_ARG="--enforce-eager"
fi
# Pin max model length for both servers; defaults to "auto" (vLLM derives the
# largest length that fits KV memory). Verified not to affect the bit-exact
# determinism tests. Override via MAX_MODEL_LEN if a model needs a fixed length.
MAX_MODEL_LEN="${MAX_MODEL_LEN:-auto}"
MAX_MODEL_LEN_ARG="--max-model-len ${MAX_MODEL_LEN}"
# LMCache server chunk size in tokens. Empty -> server default.
CHUNK_SIZE_ARG=""
if [ -n "${CHUNK_SIZE:-}" ]; then
CHUNK_SIZE_ARG="--chunk-size ${CHUNK_SIZE}"
fi
# vLLM batch-invariant mode. On by default; GDN/Mamba backends do not support it.
BATCH_INVARIANT="${BATCH_INVARIANT:-1}"
# Mamba KV cache mode + prefix caching, set only for hybrid Mamba models.
MAMBA_ARGS=""
if [ -n "${MAMBA_CACHE_MODE:-}" ]; then
MAMBA_ARGS="--mamba-cache-mode ${MAMBA_CACHE_MODE} --enable-prefix-caching"
fi
# Max tokens per scheduler step. Empty -> vLLM default.
MAX_NUM_BATCHED_TOKENS_ARG=""
if [ -n "${MAX_NUM_BATCHED_TOKENS:-}" ]; then
MAX_NUM_BATCHED_TOKENS_ARG="--max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS}"
fi
# L1 lazy allocation mode. Default is lazy (--l1-use-lazy). Set L1_USE_LAZY=false
# to disable lazy allocation, which enables POSIX SHM-backed L1 pool for the
# engine_driven SHM transfer path. When lazy is enabled (default), the SHM pool
# is disabled and engine_driven falls back to pickle transport.
L1_LAZY_ARG=""
if [ "${L1_USE_LAZY:-true}" = "false" ]; then
L1_LAZY_ARG="--no-l1-use-lazy"
echo "L1 lazy allocation disabled (SHM transport enabled)"
fi
# Store PIDs in a file so cleanup.sh can find them
PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}"
> "$PID_FILE"
# ── 1. LMCache Multiprocess Server ──────────────────────────
echo "=== Launching LMCache MP server ==="
echo "Port: $LMCACHE_PORT"
# Optional GDS L1 slab tier (gds_* tests). When GDS_L1_PATH is set, the L1
# medium becomes an NVMe slab accessed via cuFile DMA instead of pinned DRAM;
# --l1-size-gb then sizes the slab. The path must be on a GDS-capable
# filesystem (local NVMe), provided by the /scratch hostPath mount.
GDS_L1_ARG=""
if [ -n "${GDS_L1_PATH:-}" ]; then
echo "GDS L1 tier enabled; slab directory: $GDS_L1_PATH"
GDS_L1_ARG="--gds-l1-path ${GDS_L1_PATH}"
fi
CUDA_VISIBLE_DEVICES="${GPU_FOR_VLLM}" \
lmcache server \
--l1-size-gb "$CPU_BUFFER_SIZE" \
--eviction-policy LRU \
--max-workers "$MAX_WORKERS" \
$CHUNK_SIZE_ARG \
--port "$LMCACHE_PORT" \
${GDS_L1_ARG} \
${L1_LAZY_ARG} \
> "/tmp/build_${BUILD_ID}_lmcache.log" 2>&1 &
LMCACHE_PID=$!
echo "$LMCACHE_PID" >> "$PID_FILE"
echo "LMCache MP server started (PID=$LMCACHE_PID)"
# Wait for LMCache to initialize
echo "Waiting for LMCache to initialize..."
sleep 10
# Unset VLLM_PORT so vLLM's internal get_open_port() picks a random
# ephemeral port for torch.distributed instead of trying serving_port+1.
# Without this, both instances fight over the same internal port.
unset VLLM_PORT
# ── 2. vLLM with LMCache ────────────────────────────────────
echo "=== Launching vLLM with LMCache ==="
echo "Model: $MODEL"
echo "Port: $vllm_port"
CUDA_VISIBLE_DEVICES="${GPU_FOR_VLLM}" \
VLLM_ENABLE_V1_MULTIPROCESSING=0 \
VLLM_SERVER_DEV_MODE=1 \
VLLM_BATCH_INVARIANT=${BATCH_INVARIANT} \
PYTHONHASHSEED=0 \
vllm serve "$MODEL" \
--kv-transfer-config "{\"kv_connector\":\"LMCacheMPConnector\", \"kv_role\":\"kv_both\", \"kv_load_failure_policy\": \"recompute\", \"kv_connector_extra_config\": {\"lmcache.mp.port\": $LMCACHE_PORT, \"lmcache.mp.mq_timeout\": 10}}" \
$ATTENTION_BACKEND_ARG \
--port "$vllm_port" \
--no-async-scheduling \
$MAX_MODEL_LEN_ARG \
$ENFORCE_EAGER_ARG \
$GPU_MEMORY_UTIL_ARG \
$MAMBA_ARGS \
$MAX_NUM_BATCHED_TOKENS_ARG \
> "/tmp/build_${BUILD_ID}_vllm.log" 2>&1 &
VLLM_PID=$!
echo "$VLLM_PID" >> "$PID_FILE"
echo "vLLM with LMCache started (PID=$VLLM_PID)"
# ── 3. vLLM Baseline (without LMCache) ──────────────────────
# Only launched for tests that compare against a baseline (2-GPU pods).
# Single-GPU tests set LAUNCH_BASELINE=false and skip this entirely.
if [[ "${LAUNCH_BASELINE:-true}" == "true" ]]; then
echo "=== Launching vLLM baseline ==="
echo "Port: $vllm_baseline_port"
CUDA_VISIBLE_DEVICES="${GPU_FOR_BASELINE}" \
VLLM_ENABLE_V1_MULTIPROCESSING=0 \
VLLM_SERVER_DEV_MODE=1 \
VLLM_BATCH_INVARIANT=1 \
PYTHONHASHSEED=0 \
vllm serve "$MODEL" \
$ATTENTION_BACKEND_ARG \
--port "$vllm_baseline_port" \
--no-async-scheduling \
$MAX_MODEL_LEN_ARG \
$ENFORCE_EAGER_ARG \
$GPU_MEMORY_UTIL_ARG \
> "/tmp/build_${BUILD_ID}_vllm_baseline.log" 2>&1 &
VLLM_BASELINE_PID=$!
echo "$VLLM_BASELINE_PID" >> "$PID_FILE"
echo "vLLM baseline started (PID=$VLLM_BASELINE_PID)"
else
echo "=== Skipping vLLM baseline (LAUNCH_BASELINE=false, 1-GPU test) ==="
fi
echo "=== All processes launched ==="
echo "PIDs: LMCache=$LMCACHE_PID, vLLM=$VLLM_PID, Baseline=${VLLM_BASELINE_PID:-skipped}"
.buildkite/k3_tests/multiprocess/scripts/pin-tested-vllm.sh
+175
View File
@@ -0,0 +1,175 @@
#!/usr/bin/env bash
# Pin the currently-installed vLLM nightly to the
# `buildkite_latest_tested_vllm` branch.
#
# Runs ONLY when the canary vllm_bench step has succeeded and
# VERIFY_AND_PIN_VLLM=true. Records the just-verified vllm wheel version so
# downstream builds can install the same version deterministically instead
# of resolving "latest nightly" again.
#
# Two files are maintained on the dedicated branch:
# tested_vllm_versions.csv -- JSON Lines, append-only history. Each line
# is one self-contained record.
# latest_tested_vllm.txt -- Plain text, single line: the most recent
# verified version. Overwritten every run.
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)"
cd "${REPO_ROOT}"
export GIT_TERMINAL_PROMPT=0
# ── Resolve the version that's actually installed in this pod ───────────
VLLM_VERSION="$(python -c 'import vllm; print(vllm.__version__)')"
if [[ -z "${VLLM_VERSION}" ]]; then
echo "[ERROR] could not read vllm.__version__ from the live env" >&2
exit 1
fi
echo "Verified vLLM version: ${VLLM_VERSION}"
# ── Resolve commit SHAs so consumers don't need to call any external API ─
# The PEP 440 local version after `+g` is the short commit SHA, e.g.
# 0.23.1rc1.dev508+gc6dd32a81 -> c6dd32a81. Expand it to the full 40-char
# SHA via the public GitHub commits API; we already have GITHUB_TOKEN in
# the env (5000 req/h). The full SHA gives us the permanent
# wheels.vllm.ai/<full-sha>/<cuda>/ archive URL, which keeps working even
# after the rolling nightly index has dropped the wheel.
VLLM_SHORT_SHA="${VLLM_VERSION##*+g}"
if [[ "${VLLM_SHORT_SHA}" == "${VLLM_VERSION}" \
|| ! "${VLLM_SHORT_SHA}" =~ ^[0-9a-f]+$ ]]; then
VLLM_SHORT_SHA=""
fi
VLLM_FULL_SHA=""
if [[ -n "${VLLM_SHORT_SHA}" ]]; then
gh_auth_args=()
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
gh_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
fi
for attempt in 1 2 3; do
VLLM_FULL_SHA="$(curl -fsSL --connect-timeout 5 --max-time 10 \
-H "Accept: application/vnd.github+json" \
"${gh_auth_args[@]+"${gh_auth_args[@]}"}" \
"https://api.github.com/repos/vllm-project/vllm/commits/${VLLM_SHORT_SHA}" \
2>/dev/null \
| jq -r '.sha // empty')" || true
if [[ "${VLLM_FULL_SHA}" =~ ^[0-9a-f]{40}$ ]]; then
break
fi
VLLM_FULL_SHA=""
echo "[INFO] GitHub commit lookup attempt ${attempt} for" \
"${VLLM_SHORT_SHA} returned no SHA; retrying..." >&2
sleep 2
done
fi
if [[ -n "${VLLM_FULL_SHA}" ]]; then
VLLM_ARCHIVE_INDEX="https://wheels.vllm.ai/${VLLM_FULL_SHA}/cu130"
echo "Resolved full SHA: ${VLLM_FULL_SHA}"
echo "Archive index: ${VLLM_ARCHIVE_INDEX}"
else
VLLM_ARCHIVE_INDEX=""
echo "[WARN] could not resolve full SHA for short SHA" \
"'${VLLM_SHORT_SHA:-<none>}'; archive_index_url will be empty" \
"and consumers will fall back to live API lookup" >&2
fi
CI_REPO="LMCache/LMCache"
CI_BRANCH="buildkite_latest_tested_vllm"
if [[ -n "${GITHUB_TOKEN:-}" ]]; then
CI_REPO_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${CI_REPO}.git"
else
echo "[WARN] GITHUB_TOKEN not set — push will likely fail" >&2
CI_REPO_URL="https://github.com/${CI_REPO}.git"
fi
WORK_DIR="/tmp/pin_vllm_$$"
trap 'rm -rf "${WORK_DIR}"' EXIT
echo "--- Preparing ${CI_BRANCH} branch from ${CI_REPO}"
if ! git clone --depth=1 --branch "${CI_BRANCH}" "${CI_REPO_URL}" \
"${WORK_DIR}" 2>/dev/null; then
# Branch does not exist yet -- create an orphan with no parent history.
rm -rf "${WORK_DIR}"
mkdir -p "${WORK_DIR}"
git -C "${WORK_DIR}" init -q
git -C "${WORK_DIR}" remote add origin "${CI_REPO_URL}"
git -C "${WORK_DIR}" checkout --orphan "${CI_BRANCH}"
# Drop anything that the orphan checkout might have staged from HEAD.
git -C "${WORK_DIR}" rm -rf --cached . >/dev/null 2>&1 || true
find "${WORK_DIR}" -mindepth 1 -maxdepth 1 ! -name ".git" \
-exec rm -rf {} +
fi
# ── Update files ────────────────────────────────────────────────────────
HISTORY_FILE="${WORK_DIR}/tested_vllm_versions.csv"
LATEST_FILE="${WORK_DIR}/latest_tested_vllm.txt"
TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
BUILD_URL="${BUILDKITE_BUILD_URL:-}"
BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-}"
COMMIT_SHA="${BUILDKITE_COMMIT:-}"
# Append-only history (JSON Lines). Built via python so quoting is safe.
# Use a quoted heredoc (<<'PY') to disable Bash expansion inside the script,
# and pass variables via the environment to avoid syntax errors from special
# characters in values like BUILD_URL.
TIMESTAMP="${TIMESTAMP}" \
VLLM_VERSION="${VLLM_VERSION}" \
VLLM_SHORT_SHA="${VLLM_SHORT_SHA}" \
VLLM_FULL_SHA="${VLLM_FULL_SHA}" \
VLLM_ARCHIVE_INDEX="${VLLM_ARCHIVE_INDEX}" \
BUILD_NUMBER="${BUILD_NUMBER}" \
BUILD_URL="${BUILD_URL}" \
COMMIT_SHA="${COMMIT_SHA}" \
python - "$HISTORY_FILE" <<'PY'
import json, os, sys
path = sys.argv[1]
record = {
"timestamp": os.environ.get("TIMESTAMP", ""),
"vllm_version": os.environ.get("VLLM_VERSION", ""),
"vllm_short_sha": os.environ.get("VLLM_SHORT_SHA", ""),
"vllm_full_sha": os.environ.get("VLLM_FULL_SHA", ""),
"archive_index_url": os.environ.get("VLLM_ARCHIVE_INDEX", ""),
"build_number": os.environ.get("BUILD_NUMBER", ""),
"build_url": os.environ.get("BUILD_URL", ""),
"commit": os.environ.get("COMMIT_SHA", ""),
}
with open(path, "a", encoding="utf-8") as f:
f.write(json.dumps(record) + "\n")
PY
# Latest pointer. The first line is the bare version so older consumers
# that just `head -n1` keep working; the trailing key=value lines let new
# consumers skip the live GitHub API lookup entirely.
{
printf '%s\n' "${VLLM_VERSION}"
printf 'short_sha=%s\n' "${VLLM_SHORT_SHA}"
printf 'full_sha=%s\n' "${VLLM_FULL_SHA}"
printf 'archive_index_url=%s\n' "${VLLM_ARCHIVE_INDEX}"
} > "${LATEST_FILE}"
# ── Commit + push ───────────────────────────────────────────────────────
cd "${WORK_DIR}"
git add tested_vllm_versions.csv latest_tested_vllm.txt
if git diff --cached --quiet 2>/dev/null; then
echo "No changes to commit (version unchanged?)."
exit 0
fi
git -c user.email="ci@lmcache.ai" -c user.name="LMCache CI" \
commit -m "Pin verified vLLM nightly: ${VLLM_VERSION}"
echo "--- Pushing to ${CI_REPO} ${CI_BRANCH}"
if ! git push origin "HEAD:${CI_BRANCH}" 2>/dev/null; then
echo "[WARN] Normal push failed, force-pushing..." >&2
git push origin "+HEAD:${CI_BRANCH}" 2>/dev/null || {
echo "[ERROR] Failed to push to ${CI_REPO} ${CI_BRANCH}" >&2
exit 1
}
fi
echo "--- Pinned vLLM ${VLLM_VERSION} successfully"
git log --oneline -1
.buildkite/k3_tests/multiprocess/scripts/run-cache-stats.sh
Executable
+214
View File
@@ -0,0 +1,214 @@
#!/usr/bin/env bash
# Test that kv_transfer_params / cached_token_stats flows end-to-end
# through the OpenAI-compatible API when LMCache MP mode is active.
#
# Flow:
# 1. Send a long prompt (cold — populates LMCache, no cache hit)
# 2. Send the same prompt again (warm — should hit LMCache)
# 3. Verify the response contains cached_token_stats with expected values
set -e
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# Configuration (inherited from run-single-test.sh)
VLLM_PORT="${VLLM_PORT:-8000}"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
BUILD_ID="${BUILD_ID:-local_$$}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
STATS_DIR="$RESULTS_DIR/cache_stats"
mkdir -p "$STATS_DIR"
echo "=== Cache Stats Reporting Test ==="
echo "Model: $MODEL"
echo "vLLM Port: $VLLM_PORT"
echo "Results dir: $STATS_DIR"
echo ""
# Build a prompt long enough to span multiple LMCache chunks (default
# chunk_size=256 tokens). Repeating a sentence gives us ~600+ tokens.
LONG_CONTENT="Explain the history of computer science in great detail. $(printf 'The Turing machine is a fundamental concept in theoretical computer science that defines an abstract machine capable of manipulating symbols on a strip of tape according to a table of rules. %.0s' {1..20})"
send_request() {
local label="$1"
local output_file="$2"
echo "--- Sending request: $label ---"
local http_code
http_code=$(curl -s -o "$output_file" -w "%{http_code}" \
-X POST "http://localhost:${VLLM_PORT}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "{
\"model\": \"${MODEL}\",
\"messages\": [{\"role\": \"user\", \"content\": $(python3 -c "import json; print(json.dumps('$LONG_CONTENT'))")}],
\"max_tokens\": 1,
\"kv_transfer_params\": {\"cached_token_stats\": true}
}")
if [ "$http_code" -ne 200 ]; then
echo "FAIL: $label returned HTTP $http_code"
cat "$output_file"
return 1
fi
echo "$label: HTTP 200 OK"
}
validate_stats_present() {
local label="$1"
local response_file="$2"
python3 -c "
import json, sys
with open('$response_file') as f:
data = json.load(f)
kv_params = data.get('kv_transfer_params')
if kv_params is None:
print('FAIL: $label — kv_transfer_params is missing from response')
sys.exit(1)
stats = kv_params.get('cached_token_stats')
if stats is None:
print('FAIL: $label — cached_token_stats is missing from kv_transfer_params')
print(f' kv_transfer_params = {kv_params}')
sys.exit(1)
required_keys = [
'num_vllm_cached_tokens',
'num_lmcache_cached_tokens',
'num_lmcache_extra_cached_tokens',
]
missing = [k for k in required_keys if k not in stats]
if missing:
print(f'FAIL: $label — missing keys in cached_token_stats: {missing}')
print(f' cached_token_stats = {stats}')
sys.exit(1)
for k in required_keys:
v = stats[k]
if not isinstance(v, int) or v < 0:
print(f'FAIL: $label — {k} should be a non-negative integer, got {v!r}')
sys.exit(1)
print(f'PASS: $label — cached_token_stats present with all required keys')
print(f' num_vllm_cached_tokens: {stats[\"num_vllm_cached_tokens\"]}')
print(f' num_lmcache_cached_tokens: {stats[\"num_lmcache_cached_tokens\"]}')
print(f' num_lmcache_extra_cached_tokens: {stats[\"num_lmcache_extra_cached_tokens\"]}')
"
}
validate_warm_hit() {
local cold_file="$1"
local warm_file="$2"
python3 -c "
import json, sys
with open('$cold_file') as f:
cold = json.load(f)
with open('$warm_file') as f:
warm = json.load(f)
cold_stats = cold['kv_transfer_params']['cached_token_stats']
warm_stats = warm['kv_transfer_params']['cached_token_stats']
cold_lmcache = cold_stats['num_lmcache_cached_tokens']
warm_lmcache = warm_stats['num_lmcache_cached_tokens']
print(f'Cold request — num_lmcache_cached_tokens: {cold_lmcache}')
print(f'Warm request — num_lmcache_cached_tokens: {warm_lmcache}')
if warm_lmcache <= cold_lmcache:
print(f'FAIL: warm request should have more LMCache hits than cold request')
print(f' cold={cold_lmcache}, warm={warm_lmcache}')
sys.exit(1)
if warm_lmcache == 0:
print('FAIL: warm request has 0 LMCache cached tokens (cache not populated?)')
sys.exit(1)
print(f'PASS: warm request has more LMCache hits ({warm_lmcache} > {cold_lmcache})')
"
}
# ── Step 1: Cold request (populates LMCache) ──────────────────
echo "============================================"
echo "=== Step 1: Cold request ==="
echo "============================================"
if ! send_request "Cold" "$STATS_DIR/cold_response.json"; then
exit 1
fi
if ! validate_stats_present "Cold" "$STATS_DIR/cold_response.json"; then
exit 1
fi
echo ""
# Small delay to let the store operation complete in LMCache
sleep 2
# ── Step 2: Warm request (same prompt, should hit cache) ──────
echo "============================================"
echo "=== Step 2: Warm request ==="
echo "============================================"
if ! send_request "Warm" "$STATS_DIR/warm_response.json"; then
exit 1
fi
if ! validate_stats_present "Warm" "$STATS_DIR/warm_response.json"; then
exit 1
fi
echo ""
# ── Step 3: Validate cache hit improvement ────────────────────
echo "============================================"
echo "=== Step 3: Validate cache hit ==="
echo "============================================"
if ! validate_warm_hit "$STATS_DIR/cold_response.json" "$STATS_DIR/warm_response.json"; then
exit 1
fi
echo ""
# ── Step 4: Verify opt-in behavior ────────────────────────────
# Request WITHOUT kv_transfer_params should NOT have stats in response.
echo "============================================"
echo "=== Step 4: Verify opt-in (no stats without opt-in) ==="
echo "============================================"
echo "--- Sending request without kv_transfer_params ---"
http_code=$(curl -s -o "$STATS_DIR/no_opt_in_response.json" -w "%{http_code}" \
-X POST "http://localhost:${VLLM_PORT}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "{
\"model\": \"${MODEL}\",
\"messages\": [{\"role\": \"user\", \"content\": \"Hello, how are you?\"}],
\"max_tokens\": 1
}")
if [ "$http_code" -ne 200 ]; then
echo "FAIL: no-opt-in request returned HTTP $http_code"
exit 1
fi
python3 -c "
import json, sys
with open('$STATS_DIR/no_opt_in_response.json') as f:
data = json.load(f)
kv_params = data.get('kv_transfer_params')
if kv_params is not None:
print(f'FAIL: kv_transfer_params should be absent without opt-in, got {kv_params}')
sys.exit(1)
print('PASS: kv_transfer_params correctly absent when not opted in')
"
echo ""
# ── Summary ───────────────────────────────────────────────────
echo "============================================"
echo "=== Cache Stats Reporting Test PASSED ==="
echo "============================================"
.buildkite/k3_tests/multiprocess/scripts/run-deadlock.sh
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env bash
# Self-contained deadlock regression test.
#
# Launches DeepSeek-V2-Lite-Chat with TP=2 (both GPUs) + LMCache server,
# sends 50 requests with ~30K token prefixes, and verifies they all
# complete within 3 minutes. A CUDA-driver/GIL deadlock would cause
# requests to hang indefinitely, failing the timeout.
#
# This test is self-contained: it handles its own server lifecycle
# instead of using the standard launch-processes.sh / wait-for-servers.sh.
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# ── Configuration ───────────────────────────────────────────
MODEL="deepseek-ai/DeepSeek-V2-Lite-Chat"
LMCACHE_PORT="${LMCACHE_PORT:-15554}"
VLLM_PORT="${VLLM_PORT:-8000}"
BUILD_ID="${BUILD_ID:-local_$$}"
PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}"
TIMEOUT_SECONDS=180 # 3 minutes
# ── Install py-spy for deadlock diagnosis ──────────────────
echo "=== Installing py-spy ==="
uv pip install py-spy
PY_SPY="$(which py-spy)"
echo "py-spy installed at: $PY_SPY"
PYSPY_LOG="/tmp/build_${BUILD_ID}_pyspy.log"
# ── Helper: dump stacks of server processes via py-spy ─────
dump_stacks() {
echo "" | tee -a "$PYSPY_LOG"
echo "=== py-spy stack dump (native + Python) ===" | tee -a "$PYSPY_LOG"
if kill -0 "$LMCACHE_PID" 2>/dev/null; then
echo "" | tee -a "$PYSPY_LOG"
echo "--- LMCache server (PID=$LMCACHE_PID) ---" | tee -a "$PYSPY_LOG"
sudo "$PY_SPY" dump --pid "$LMCACHE_PID" --native 2>&1 | tee -a "$PYSPY_LOG" || true
fi
# Copy to repo root so cleanup.sh collects it as a Buildkite artifact
cp "$PYSPY_LOG" "${REPO_ROOT}/build_${BUILD_ID}_pyspy.log" 2>/dev/null || true
}
# ── 1. Launch LMCache server ───────────────────────────────
echo "=== Launching LMCache server ==="
echo "Port: $LMCACHE_PORT"
lmcache server \
--host localhost \
--port "$LMCACHE_PORT" \
--chunk-size 256 \
--l1-size-gb 50 \
--eviction-policy LRU \
--max-workers 2 \
> "/tmp/build_${BUILD_ID}_lmcache.log" 2>&1 &
LMCACHE_PID=$!
echo "$LMCACHE_PID" >> "$PID_FILE"
echo "LMCache server started (PID=$LMCACHE_PID)"
sleep 10
# ── 2. Launch vLLM with DeepSeek TP=2 ─────────────────────
echo "=== Launching vLLM (DeepSeek TP=2) ==="
echo "Model: $MODEL"
echo "Port: $VLLM_PORT"
# Save VLLM_PORT before unsetting — vLLM's internal get_open_port()
# would otherwise collide with the serving port for torch.distributed.
SAVED_VLLM_PORT="$VLLM_PORT"
unset VLLM_PORT
FLASHINFER_DISABLE_VERSION_CHECK=1 \
VLLM_SERVER_DEV_MODE=1 \
vllm serve "$MODEL" \
--tensor-parallel-size 2 \
--distributed-executor-backend mp \
--block-size 64 \
--trust-remote-code \
--load-format dummy \
--enable-prefix-caching \
--enable-chunked-prefill \
--gpu-memory-utilization 0.8 \
--max-model-len 65536 \
--hf-overrides '{"max_position_embeddings":65536}' \
--max-num-seqs 32 \
--max-num-batched-tokens 16000 \
--scheduling-policy fcfs \
--port "$SAVED_VLLM_PORT" \
--enforce-eager \
--kv-transfer-config "{\"kv_connector\":\"LMCacheMPConnector\", \"kv_role\":\"kv_both\", \"kv_load_failure_policy\": \"recompute\", \"kv_connector_extra_config\": {\"lmcache.mp.port\": $LMCACHE_PORT, \"lmcache.mp.mq_timeout\": 60}}" \
> "/tmp/build_${BUILD_ID}_vllm.log" 2>&1 &
VLLM_PID=$!
echo "$VLLM_PID" >> "$PID_FILE"
echo "vLLM started (PID=$VLLM_PID)"
VLLM_PORT="$SAVED_VLLM_PORT"
# ── 3. Wait for vLLM to be ready ──────────────────────────
echo "=== Waiting for vLLM to be ready ==="
if ! wait_for_server "$VLLM_PORT" 600; then
echo "vLLM failed to start. Last 100 lines of log:"
tail -100 "/tmp/build_${BUILD_ID}_vllm.log" 2>/dev/null || true
exit 1
fi
# ── 4. Run benchmark with timeout ─────────────────────────
echo "=== Running lmcache bench engine (random-prefill, 50 reqs, ~30K tokens) ==="
echo "Timeout: ${TIMEOUT_SECONDS}s"
if ! timeout "$TIMEOUT_SECONDS" lmcache bench engine \
--engine-url "http://localhost:${VLLM_PORT}" \
--workload random-prefill \
--tokens-per-gb-kvcache 6000 \
--rp-request-length 30000 \
--rp-num-requests 50 \
--no-interactive \
--no-csv \
-q; then
echo "FAIL: Benchmark failed or timed out (possible deadlock)"
echo ""
echo "=== LMCache log (last 50 lines) ==="
tail -50 "/tmp/build_${BUILD_ID}_lmcache.log" 2>/dev/null || true
echo ""
echo "=== vLLM log (last 50 lines) ==="
tail -50 "/tmp/build_${BUILD_ID}_vllm.log" 2>/dev/null || true
exit 1
fi
echo ""
echo "=== Benchmark completed within ${TIMEOUT_SECONDS}s ==="
echo "PASS: No deadlock detected"
.buildkite/k3_tests/multiprocess/scripts/run-fault-tolerance.sh
Executable
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env bash
# Test LMCache fault tolerance: verify vLLM requests complete after
# the LMCache MP server is killed mid-flight.
#
# Flow:
# 1. Run a warmup bench (measures baseline timing)
# 2. Run bench again, killing LMCache server mid-flight
# 3. Run bench fully without LMCache server
# 4. Verify all prompts completed in every phase
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# Configuration (inherited from run-mp-test.sh)
VLLM_PORT="${VLLM_PORT:-8000}"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
BUILD_ID="${BUILD_ID:-local_$$}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
LMCACHE_PORT="${LMCACHE_PORT:-6555}"
# Bench parameters
NUM_PROMPTS="${NUM_PROMPTS:-50}"
RANDOM_INPUT_LEN="${RANDOM_INPUT_LEN:-10000}"
RANDOM_OUTPUT_LEN="${RANDOM_OUTPUT_LEN:-1}"
RANDOM_SEED="${RANDOM_SEED:-42}"
CPU_BUFFER_SIZE="${CPU_BUFFER_SIZE:-80}"
MAX_WORKERS="${MAX_WORKERS:-4}"
# Output directory
FT_DIR="$RESULTS_DIR/fault_tolerance"
mkdir -p "$FT_DIR"
echo "=== Fault Tolerance Test ==="
echo "Model: $MODEL"
echo "vLLM Port: $VLLM_PORT"
echo "LMCache Port: $LMCACHE_PORT"
echo "Bench: $NUM_PROMPTS prompts, input_len=$RANDOM_INPUT_LEN, output_len=$RANDOM_OUTPUT_LEN"
echo "Results dir: $FT_DIR"
echo ""
# ── Step 0: Restart LMCache + vLLM with fresh state ─────────
# Previous steps may have left processes in an unknown state.
# Restart both so vLLM registers its GPU context with the new server.
echo "============================================"
echo "=== Restarting LMCache + vLLM ==="
echo "============================================"
PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}"
GPU_DEVICE="${GPU_FOR_VLLM:-0}"
# Kill existing LMCache + vLLM (keep baseline on line 3)
if [ -f "$PID_FILE" ]; then
OLD_LMCACHE_PID=$(sed -n '1p' "$PID_FILE")
OLD_VLLM_PID=$(sed -n '2p' "$PID_FILE")
for pid in $OLD_LMCACHE_PID $OLD_VLLM_PID; do
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
echo "Killing PID $pid"
kill "$pid" 2>/dev/null || true
wait "$pid" 2>/dev/null || true
fi
done
sleep 2
fi
# Launch LMCache with L1 config
CUDA_VISIBLE_DEVICES="${GPU_DEVICE}" \
lmcache server \
--l1-size-gb "$CPU_BUFFER_SIZE" \
--eviction-policy LRU \
--max-workers "$MAX_WORKERS" \
--port "$LMCACHE_PORT" \
> "/tmp/build_${BUILD_ID}_lmcache_ft.log" 2>&1 &
NEW_LMCACHE_PID=$!
echo "LMCache server started (PID=$NEW_LMCACHE_PID)"
sleep 10
# Launch vLLM with LMCache
GPU_MEMORY_UTIL_ARG=""
GPU_MEMORY_MB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits -i "${GPU_DEVICE}" | tr -d ' ')
GPU_MEMORY_GB=$((GPU_MEMORY_MB / 1024))
if [ "$GPU_MEMORY_GB" -gt 90 ]; then
GPU_MEMORY_UTIL_ARG="--gpu-memory-utilization 0.5"
fi
env -u VLLM_PORT \
CUDA_VISIBLE_DEVICES="${GPU_DEVICE}" \
VLLM_ENABLE_V1_MULTIPROCESSING=0 \
VLLM_SERVER_DEV_MODE=1 \
VLLM_BATCH_INVARIANT=1 \
PYTHONHASHSEED=0 \
vllm serve "$MODEL" \
--kv-transfer-config "{\"kv_connector\":\"LMCacheMPConnector\", \"kv_role\":\"kv_both\", \"kv_load_failure_policy\": \"recompute\", \"kv_connector_extra_config\": {\"lmcache.mp.port\": $LMCACHE_PORT, \"lmcache.mp.mq_timeout\": 10}}" \
--attention-backend FLASH_ATTN \
--port "$VLLM_PORT" \
--no-async-scheduling \
$GPU_MEMORY_UTIL_ARG \
> "/tmp/build_${BUILD_ID}_vllm_ft.log" 2>&1 &
NEW_VLLM_PID=$!
echo "vLLM started (PID=$NEW_VLLM_PID)"
# Update PID file
if [ -f "$PID_FILE" ]; then
sed -i "1s/.*/$NEW_LMCACHE_PID/" "$PID_FILE"
sed -i "2s/.*/$NEW_VLLM_PID/" "$PID_FILE"
else
echo "$NEW_LMCACHE_PID" > "$PID_FILE"
echo "$NEW_VLLM_PID" >> "$PID_FILE"
fi
if ! wait_for_server "$VLLM_PORT" 300; then
echo "vLLM failed to start"
tail -50 "/tmp/build_${BUILD_ID}_lmcache_ft.log" || true
tail -50 "/tmp/build_${BUILD_ID}_vllm_ft.log" || true
exit 1
fi
echo ""
# ── Helpers ──────────────────────────────────────────────────
run_bench() {
local description="$1"
local result_file="$2"
echo ""
echo "--- $description ---"
vllm bench serve \
--seed "$RANDOM_SEED" \
--port "$VLLM_PORT" \
--model "$MODEL" \
--dataset-name random \
--random-input-len "$RANDOM_INPUT_LEN" \
--random-output-len "$RANDOM_OUTPUT_LEN" \
--num-prompts "$NUM_PROMPTS" \
--ignore-eos \
--backend openai-chat \
--endpoint /v1/chat/completions \
--result-dir "$FT_DIR" \
--result-filename "$result_file" \
--save-result
local completed
completed=$(python3 -c "
import json
with open('$FT_DIR/$result_file') as f:
data = json.load(f)
print(data.get('completed', 0))
")
echo "$description: $completed / $NUM_PROMPTS completed"
if [ "$completed" -ne "$NUM_PROMPTS" ]; then
echo "FAIL: Expected $NUM_PROMPTS completed, got $completed"
return 1
fi
echo "PASS: All $NUM_PROMPTS prompts completed"
return 0
}
get_lmcache_pid() {
local pid_file="/tmp/lmcache_mp_pids_${BUILD_ID}"
if [[ -f "$pid_file" ]]; then
head -1 "$pid_file"
fi
}
# ── Step 1: Warmup bench ─────────────────────────────────────
echo "============================================"
echo "=== Fault Tolerance Step 1: Warmup bench ==="
echo "============================================"
if ! run_bench "Warmup (with LMCache)" "ft_warmup.json"; then
echo "FAIL: Warmup bench failed"
exit 1
fi
# Extract duration to calibrate kill timing
WARMUP_DURATION=$(python3 -c "import json; print(json.load(open('$FT_DIR/ft_warmup.json'))['duration'])")
KILL_DELAY=$(python3 -c "print(max(3, int($WARMUP_DURATION * 0.4)))")
echo "Warmup took ${WARMUP_DURATION}s. Will kill LMCache after ${KILL_DELAY}s in next run."
# ── Step 2: Bench with mid-flight LMCache kill ───────────────
echo ""
echo "============================================"
echo "=== Fault Tolerance Step 2: Mid-flight kill ==="
echo "============================================"
LMCACHE_PID=$(get_lmcache_pid)
if [ -z "$LMCACHE_PID" ] || ! kill -0 "$LMCACHE_PID" 2>/dev/null; then
echo "FAIL: LMCache server not running (PID=$LMCACHE_PID)"
exit 1
fi
echo "LMCache server PID: $LMCACHE_PID"
echo "Will kill after ${KILL_DELAY}s into bench."
# Start bench in background
run_bench "Mid-flight kill" "ft_midflight.json" &
BENCH_PID=$!
# Wait, then kill LMCache
sleep "$KILL_DELAY"
echo "Killing LMCache server (PID: $LMCACHE_PID)..."
kill "$LMCACHE_PID" 2>/dev/null
wait "$LMCACHE_PID" 2>/dev/null || true
echo "LMCache server killed at +${KILL_DELAY}s."
# Wait for bench to finish
echo "Waiting for bench to complete..."
if ! wait "$BENCH_PID"; then
echo "FAIL: Bench did not complete after mid-flight LMCache kill."
echo "--- vLLM log (last 50 lines) ---"
tail -50 "/tmp/build_${BUILD_ID}_vllm.log" 2>/dev/null || true
exit 1
fi
# ── Step 3: Bench fully without LMCache server ───────────────
echo ""
echo "============================================"
echo "=== Fault Tolerance Step 3: Without LMCache ==="
echo "============================================"
if ! run_bench "Without LMCache" "ft_without_lmcache.json"; then
echo "FAIL: Bench failed without LMCache server."
echo "--- vLLM log (last 50 lines) ---"
tail -50 "/tmp/build_${BUILD_ID}_vllm.log" 2>/dev/null || true
exit 1
fi
# ── Summary ──────────────────────────────────────────────────
echo ""
echo "============================================"
echo "=== Fault Tolerance Test PASSED ==="
echo "============================================"
warmup_completed=$(python3 -c "import json; print(json.load(open('$FT_DIR/ft_warmup.json'))['completed'])")
warmup_duration=$(python3 -c "import json; print(f\"{json.load(open('$FT_DIR/ft_warmup.json'))['duration']:.1f}\")")
midflight_completed=$(python3 -c "import json; print(json.load(open('$FT_DIR/ft_midflight.json'))['completed'])")
midflight_duration=$(python3 -c "import json; print(f\"{json.load(open('$FT_DIR/ft_midflight.json'))['duration']:.1f}\")")
without_completed=$(python3 -c "import json; print(json.load(open('$FT_DIR/ft_without_lmcache.json'))['completed'])")
without_duration=$(python3 -c "import json; print(f\"{json.load(open('$FT_DIR/ft_without_lmcache.json'))['duration']:.1f}\")")
echo " Warmup (with LMCache): $warmup_completed/$NUM_PROMPTS in ${warmup_duration}s"
echo " Mid-flight kill: $midflight_completed/$NUM_PROMPTS in ${midflight_duration}s (killed at +${KILL_DELAY}s)"
echo " Without LMCache: $without_completed/$NUM_PROMPTS in ${without_duration}s"
echo ""
.buildkite/k3_tests/multiprocess/scripts/run-gds-smoke.sh
Executable
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env bash
# GDS L1 smoke test. Sends a few completions (cold) to store KV to the slab,
# resets vLLM's prefix cache, then re-sends them (warm) to read the KV back from
# LMCache/GDS. Passes if every request returns HTTP 200, a real LMCache retrieve
# happened, and the warm (GDS-retrieved) outputs match the cold (recomputed)
# ones -- i.e. the GDS store/retrieve path works and is correct.
#
# Expects the GDS-enabled LMCache server + vLLM to already be running, with
# VLLM_SERVER_DEV_MODE=1 (for /reset_prefix_cache).
set -e
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
VLLM_PORT="${VLLM_PORT:-8000}"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
BUILD_ID="${BUILD_ID:-local_$$}"
LMCACHE_LOG="/tmp/build_${BUILD_ID}_lmcache.log"
N_PROMPTS="${GDS_SMOKE_PROMPTS:-4}"
OUT_DIR="$(mktemp -d)"
trap 'rm -rf "$OUT_DIR"' EXIT
# A long-ish prompt so each request stores at least one LMCache chunk.
build_prompt() { # $1 = unique id
local filler="The key-value cache stores attention keys and values across transformer layers. "
local body="" i
for i in $(seq 1 80); do body="${body}${filler}"; done
printf 'Document %s. %s' "$1" "$body"
}
# Send N_PROMPTS completions; capture each generated text to
# $OUT_DIR/<label>_<i>.txt and require every request to return HTTP 200.
send_batch() { # $1 = phase label (cold|warm)
local label="$1" ok=0 i prompt payload resp http body
for i in $(seq 1 "$N_PROMPTS"); do
prompt="$(build_prompt "$i")"
payload=$(python3 -c 'import json,sys; print(json.dumps({"model":sys.argv[1],"prompt":sys.argv[2],"max_tokens":16,"temperature":0}))' "$MODEL" "$prompt")
resp=$(curl -s -w $'\n%{http_code}' \
"http://127.0.0.1:${VLLM_PORT}/v1/completions" \
-H "Content-Type: application/json" -d "$payload")
http="${resp##*$'\n'}"
body="${resp%$'\n'*}"
printf '%s' "$body" \
| python3 -c 'import json,sys; print(json.load(sys.stdin)["choices"][0]["text"])' \
> "${OUT_DIR}/${label}_${i}.txt" 2>/dev/null \
|| echo "<no-output>" > "${OUT_DIR}/${label}_${i}.txt"
echo " [$label] req $i -> HTTP $http"
[ "$http" = "200" ] && ok=$((ok + 1))
done
[ "$ok" -eq "$N_PROMPTS" ] || { echo "[$label] only $ok/$N_PROMPTS returned HTTP 200"; return 1; }
}
# Count completed LMCache retrieves recorded in the server log (0 if no log yet).
count_retrieves() {
[ -f "$LMCACHE_LOG" ] || { echo 0; return; }
grep -c "Retrieved" "$LMCACHE_LOG" 2>/dev/null || true
}
echo "============================================"
echo "=== GDS smoke: phase 1 (cold -> store KV to the GDS slab) ==="
echo "============================================"
send_batch cold
echo "Waiting for async stores to drain to the LMCache server..."
sleep 3
retrieves_before=$(count_retrieves)
echo "============================================"
echo "=== Reset vLLM prefix cache (force warm requests through LMCache/GDS) ==="
echo "============================================"
reset_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST \
"http://127.0.0.1:${VLLM_PORT}/reset_prefix_cache")
if [ "$reset_code" != "200" ]; then
echo "reset_prefix_cache failed (HTTP $reset_code); is VLLM_SERVER_DEV_MODE=1?"
exit 1
fi
sleep 2
echo "============================================"
echo "=== GDS smoke: phase 2 (warm -> retrieve KV from the GDS slab) ==="
echo "============================================"
send_batch warm
retrieves_after=$(count_retrieves)
# 1. A real GDS retrieve must have happened (else warm recomputed / hit the APC).
echo ""
echo "LMCache retrieves logged: before=${retrieves_before} after=${retrieves_after}"
if [ "$retrieves_after" -le "$retrieves_before" ]; then
echo "GDS smoke FAILED: no LMCache retrieve recorded -- the GDS read path was"
echo "not exercised (warm requests recomputed or hit vLLM's prefix cache)."
exit 1
fi
# 2. The KV retrieved from the GDS slab must produce the same output as the
# cold recompute (deterministic decoding -> byte-identical completions).
echo "=== Verifying warm (GDS-retrieved) outputs match cold (recomputed) ==="
mismatch=0
for i in $(seq 1 "$N_PROMPTS"); do
if diff -q "${OUT_DIR}/cold_${i}.txt" "${OUT_DIR}/warm_${i}.txt" >/dev/null 2>&1; then
echo " prompt $i: match"
else
echo " prompt $i: MISMATCH"
mismatch=$((mismatch + 1))
fi
done
if [ "$mismatch" -ne 0 ]; then
echo "GDS smoke FAILED: ${mismatch}/${N_PROMPTS} warm outputs differ from cold"
echo "-- the KV retrieved from the GDS slab is incorrect."
exit 1
fi
echo "=== GDS smoke test passed: GDS store + retrieve path works and is correct ==="
.buildkite/k3_tests/multiprocess/scripts/run-hma-lm-eval.sh
Executable
+253
View File
@@ -0,0 +1,253 @@
#!/usr/bin/env bash
# HMA (hybrid memory allocator) correctness test using a real hybrid model.
#
# Models (selected by run-single-test.sh):
# - google/gemma-4-31B-it: sliding-window + full-attention hybrid whose full
# layers have a larger head_dim, so vLLM gives the KV cache groups
# different block sizes -- exercising per-group HMA store/retrieve.
# - Qwen/Qwen3.5-0.8B: Mamba/GDN + full-attention hybrid, exercising the
# registration-time cache re-views (kv_cache_group_edits.py).
#
# Flow (single GPU, no baseline server):
# 1. vLLM run: lm_eval (gsm8k) against vLLM+LMCache, populating LMCache.
# 2. Reset vLLM's *local* prefix cache (APC) only, leaving LMCache intact, via
# the dev-mode endpoint POST /reset_prefix_cache (reset_external defaults to
# false, so the LMCache-managed cache is preserved).
# 3. LMCache retrieve run: re-run lm_eval; vLLM's APC misses, so the prefix KV
# is served by LMCache.
# 4. Assert the two runs' gsm8k scores match -- a broken LMCache would skew the
# retrieved KV and make them diverge.
# 5. Assert LMCache actually served retrieves in the retrieve run (non-vacuous).
#
# The reset endpoint requires VLLM_SERVER_DEV_MODE=1 (set by launch-processes.sh).
set -e
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# Configuration
VLLM_PORT="${VLLM_PORT:-8000}"
MODEL="${MODEL:-google/gemma-4-31B-it}"
NUM_CONCURRENT="${NUM_CONCURRENT:-50}"
# 31B has a large per-token KV footprint; cap the sample count so the working
# set fits the CPU pool (a too-large set thrashes and the retrieve run misses).
LIMIT="${LIMIT:-100}"
# Max abs difference allowed between the two runs' gsm8k scores; 0 requires an
# exact match. For non-bit-exact backends, raise LIMIT to shrink run-to-run
# drift (~1/sqrt(LIMIT)) rather than loosening this.
SCORE_TOLERANCE="${SCORE_TOLERANCE:-0}"
# Seconds to let async LMCache stores drain before the retrieve run.
STORE_DRAIN_SECONDS="${STORE_DRAIN_SECONDS:-20}"
BUILD_ID="${BUILD_ID:-local_$$}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
# LMCache MP server log, scanned to confirm the retrieve run hit LMCache.
LMCACHE_LOG="${LMCACHE_LOG:-/tmp/build_${BUILD_ID}_lmcache.log}"
HMA_DIR="$RESULTS_DIR/hma_lm_eval"
VLLM_RUN_DIR="$HMA_DIR/vllm_run"
RETRIEVE_RUN_DIR="$HMA_DIR/retrieve_run"
echo "=== HMA lm_eval correctness test ==="
echo "Model: $MODEL"
echo "vLLM (LMCache) port: $VLLM_PORT"
echo "Concurrent requests: $NUM_CONCURRENT"
echo "Limit: $LIMIT"
echo "Score tolerance: $SCORE_TOLERANCE"
echo "Results dir: $HMA_DIR"
echo ""
mkdir -p "$VLLM_RUN_DIR" "$RETRIEVE_RUN_DIR"
# Run one lm_eval gsm8k pass against a vLLM OpenAI-compatible server.
#
# Globals (read):
# MODEL - HuggingFace model id, echoed to lm_eval's model_args.
# NUM_CONCURRENT - number of in-flight requests lm_eval issues.
# LIMIT - number of gsm8k samples to evaluate.
# Arguments:
# $1 port - TCP port of the vLLM /v1/completions endpoint to evaluate.
# $2 output_dir - directory lm_eval writes results_*.json / samples_*.jsonl to.
# $3 run_name - human-readable label used only in progress log lines.
# Outputs:
# Writes lm_eval result and per-sample files under output_dir; prints progress
# to stdout.
# Returns:
# lm_eval's exit status (non-zero if the evaluation run fails). Propagated to
# the caller via ``set -e``.
run_lm_eval() {
local port="$1"
local output_dir="$2"
local run_name="$3"
echo "=== Running lm_eval ($run_name) on port $port ==="
lm_eval --model local-completions --tasks gsm8k \
--model_args "model=${MODEL},base_url=http://127.0.0.1:${port}/v1/completions,num_concurrent=${NUM_CONCURRENT},max_retries=3,tokenized_requests=False" \
--limit "$LIMIT" \
--seed 0 \
-s --output_path "$output_dir" \
--gen_kwargs '{"temperature": 0.0}'
echo "$run_name completed"
echo ""
}
# Reset a vLLM server's local prefix cache (APC) while preserving LMCache.
#
# POSTs to the dev-mode /reset_prefix_cache endpoint without reset_external
# (which defaults to false), so only vLLM's GPU-side automatic prefix cache is
# cleared and the LMCache-managed cache is left intact.
# Arguments:
# $1 port - TCP port of the vLLM server whose local APC should be reset.
# Outputs:
# Progress / failure detail to stdout.
# Returns:
# 0 if the server acknowledged with HTTP 200; 1 otherwise (e.g. the endpoint
# is absent because VLLM_SERVER_DEV_MODE was not set when launching vLLM).
reset_vllm_prefix_cache() {
local port="$1"
echo "=== Resetting vLLM local prefix cache on port $port (LMCache preserved) ==="
# reset_external defaults to false -> only vLLM's APC is cleared.
local code
code=$(curl -s -o /dev/null -w "%{http_code}" -X POST \
"http://127.0.0.1:${port}/reset_prefix_cache")
if [ "$code" != "200" ]; then
echo "Failed to reset prefix cache (HTTP $code). Is VLLM_SERVER_DEV_MODE=1?"
return 1
fi
echo "vLLM prefix cache reset."
echo ""
}
# Count completed LMCache retrieves recorded in the server log so far.
#
# Used to prove run 2 was actually served by LMCache (the delta around run 2
# must be > 0), so the correctness comparison cannot pass vacuously by silently
# recomputing.
# Globals (read):
# LMCACHE_LOG - path to the LMCache MP server log file.
# Arguments:
# none.
# Outputs:
# The integer count of "Retrieved" log lines to stdout (0 if the log file does
# not exist yet).
count_retrieves() {
# NB: ``grep -c`` prints 0 *and* exits 1 on no match, so guard the file
# existence and use ``|| true`` (not ``|| echo 0``) to avoid emitting "0\n0".
[ -f "$LMCACHE_LOG" ] || { echo 0; return; }
grep -c "Retrieved" "$LMCACHE_LOG" 2>/dev/null || true
}
# ── 1. vLLM run: compute from scratch, populating LMCache ───
run_lm_eval "$VLLM_PORT" "$VLLM_RUN_DIR" "vLLM run"
# Let async stores drain to the LMCache server before invalidating the APC.
echo "Waiting ${STORE_DRAIN_SECONDS}s for LMCache stores to drain..."
sleep "$STORE_DRAIN_SECONDS"
retrieves_before=$(count_retrieves)
# ── 2. Invalidate vLLM's local prefix cache (keep LMCache) ──
reset_vllm_prefix_cache "$VLLM_PORT"
# ── 3. Retrieve run: vLLM APC misses -> LMCache serves the KV ─
run_lm_eval "$VLLM_PORT" "$RETRIEVE_RUN_DIR" "LMCache retrieve run"
retrieves_after=$(count_retrieves)
# ── 4. Compare scores and verify LMCache was actually used ──
echo "============================================"
echo "=== Verifying HMA store/retrieve correctness ==="
echo "============================================"
echo "LMCache retrieves logged: before=${retrieves_before}, after=${retrieves_after}"
python3 - "$VLLM_RUN_DIR" "$RETRIEVE_RUN_DIR" \
"$SCORE_TOLERANCE" "$retrieves_before" "$retrieves_after" <<'PYEOF'
import glob
import json
import os
import sys
vllm_run_dir, retrieve_run_dir, tol_s, before_s, after_s = sys.argv[1:6]
tol = float(tol_s)
retrieves_before = int(before_s)
retrieves_after = int(after_s)
def gsm8k_score_and_stderr(results_dir: str) -> tuple[float, float]:
"""Return the gsm8k (exact_match, stderr) from an lm_eval results directory.
Prefers the strict-match variant; falls back to any non-stderr
``exact_match`` metric key (paired with its ``exact_match_stderr`` twin).
Args:
results_dir: Directory passed to ``lm_eval --output_path``. Searched
recursively for the newest ``results_*.json`` (lm_eval nests it
under a per-model subdirectory and stamps the filename with a
timestamp).
Returns:
``(score, stderr)``: the gsm8k ``exact_match`` accuracy in
``[0.0, 1.0]`` and its reported sampling stderr (0.0 if absent).
Raises:
SystemExit: If no ``results_*.json`` exists under ``results_dir`` or the
newest one contains no ``exact_match`` metric for the gsm8k task.
"""
files = glob.glob(os.path.join(results_dir, "**", "results_*.json"), recursive=True)
if not files:
raise SystemExit(f"No results_*.json under {results_dir}")
latest = max(files, key=os.path.getmtime)
with open(latest) as f:
data = json.load(f)
metrics = data["results"]["gsm8k"]
preferred = "exact_match,strict-match"
if preferred in metrics:
stderr = float(metrics.get("exact_match_stderr,strict-match", 0.0))
return float(metrics[preferred]), stderr
for key, value in metrics.items():
if key.startswith("exact_match,") and "stderr" not in key:
variant = key.split(",", 1)[1]
stderr = float(metrics.get(f"exact_match_stderr,{variant}", 0.0))
return float(value), stderr
raise SystemExit(f"No exact_match metric in {latest}: {sorted(metrics)}")
s_vllm, e_vllm = gsm8k_score_and_stderr(vllm_run_dir)
s_retrieve, e_retrieve = gsm8k_score_and_stderr(retrieve_run_dir)
print(f" vLLM run gsm8k exact_match = {s_vllm:.4f} +/- {e_vllm:.4f}")
print(f" LMCache retrieve run gsm8k exact_match = {s_retrieve:.4f} +/- {e_retrieve:.4f}")
print(f" tolerance = {tol}")
failures = []
# The two runs must match -- a broken LMCache would skew the retrieved KV.
if abs(s_vllm - s_retrieve) > tol:
failures.append(
f"score drift between runs: |{s_vllm:.4f} - {s_retrieve:.4f}| = "
f"{abs(s_vllm - s_retrieve):.4f} > {tol}"
)
# Non-vacuous: the retrieve run must have been served by LMCache, not recompute.
if retrieves_after <= retrieves_before:
failures.append(
"LMCache served no retrieves during the retrieve run "
f"(before={retrieves_before}, after={retrieves_after})"
)
if failures:
print("\nFAILED:")
for f in failures:
print(f" - {f}")
sys.exit(1)
print(
f"\nPASS: vLLM and LMCache-retrieve gsm8k scores match (tol={tol}); "
f"LMCache served {retrieves_after - retrieves_before} retrieves."
)
PYEOF
echo ""
echo "============================================"
echo "=== HMA lm_eval correctness test passed ==="
echo "============================================"
.buildkite/k3_tests/multiprocess/scripts/run-http-api.sh
Executable
+446
View File
@@ -0,0 +1,446 @@
#!/usr/bin/env bash
# End-to-end test for LMCache HTTP API endpoints and CLI commands.
#
# Part 1 — HTTP API endpoints:
# Systematically exercises every HTTP endpoint on the LMCache
# multiprocess HTTP server (port 8080) with a live engine + vLLM.
#
# Part 2 — CLI commands:
# Tests `lmcache describe` and `lmcache kvcache clear`
# against the running server.
#
# Requires: LMCache MP server + vLLM launched by launch-processes.sh.
set -e
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# Configuration (inherited from run-single-test.sh)
VLLM_PORT="${VLLM_PORT:-8000}"
LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
BUILD_ID="${BUILD_ID:-local_$$}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
HTTP_DIR="$RESULTS_DIR/http_api"
mkdir -p "$HTTP_DIR"
BASE_URL="http://localhost:${LMCACHE_HTTP_PORT}"
echo "=== HTTP API & CLI Test ==="
echo "Model: $MODEL"
echo "vLLM port: $VLLM_PORT"
echo "LMCache HTTP: $BASE_URL"
echo "Results dir: $HTTP_DIR"
echo ""
# ── Counters ────────────────────────────────────────────────
TESTS_RUN=0
TESTS_PASSED=0
TESTS_FAILED=0
# ── Helpers ─────────────────────────────────────────────────
pass() {
local label="$1"
TESTS_RUN=$((TESTS_RUN + 1))
TESTS_PASSED=$((TESTS_PASSED + 1))
echo "PASS: $label"
}
fail() {
local label="$1"
shift
TESTS_RUN=$((TESTS_RUN + 1))
TESTS_FAILED=$((TESTS_FAILED + 1))
echo "FAIL: $label — $*"
}
check_http() {
local label="$1"
local method="$2"
local path="$3"
local output="$4"
local expected_code="${5:-200}"
local body="${6:-}"
local curl_args=(-s -o "$output" -w "%{http_code}")
case "$method" in
GET) curl_args+=("${BASE_URL}${path}") ;;
POST)
curl_args+=(-X POST -H "Content-Type: application/json")
[ -n "$body" ] && curl_args+=(-d "$body")
curl_args+=("${BASE_URL}${path}")
;;
PUT)
curl_args+=(-X PUT -H "Content-Type: application/json")
[ -n "$body" ] && curl_args+=(-d "$body")
curl_args+=("${BASE_URL}${path}")
;;
DELETE)
curl_args+=(-X DELETE "${BASE_URL}${path}")
;;
esac
local http_code
http_code=$(curl "${curl_args[@]}")
if [ "$http_code" -ne "$expected_code" ]; then
fail "$label" "expected HTTP $expected_code, got $http_code"
echo " Response body:"
cat "$output" 2>/dev/null || true
echo ""
return 1
fi
pass "$label (HTTP $http_code)"
return 0
}
assert_json_key() {
local label="$1"
local file="$2"
local key="$3"
local expected_value="${4:-}"
python3 -c "
import json, sys
with open('$file') as f:
data = json.load(f)
keys = '$key'.split('.')
cur = data
for k in keys:
if not isinstance(cur, dict) or k not in cur:
print('FAIL: $label — key \"$key\" not found')
sys.exit(1)
cur = cur[k]
if '$expected_value' and str(cur) != '$expected_value':
print('FAIL: $label — expected \"$expected_value\", got ' + repr(cur))
sys.exit(1)
" || {
fail "$label" "JSON key assertion failed for key '$key'"
return 1
}
pass "$label (key '$key')"
}
assert_contains() {
local label="$1"
local file="$2"
local pattern="$3"
if grep -q "$pattern" "$file" 2>/dev/null; then
pass "$label (contains '$pattern')"
else
fail "$label" "output does not contain '$pattern'"
return 1
fi
}
# ================================================================
# PART 1: HTTP API ENDPOINT TESTS
# ================================================================
echo ""
echo "========================================================"
echo "=== Part 1: HTTP API Endpoint Tests ==="
echo "========================================================"
# ── Step 0: Warm up cache via vLLM ─────────────────────────
echo ""
echo "============================================"
echo "=== Step 0: Warm up cache ==="
echo "============================================"
warmup_code=$(curl -s -o "$HTTP_DIR/warmup.json" -w "%{http_code}" \
-X POST "http://localhost:${VLLM_PORT}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "{
\"model\": \"${MODEL}\",
\"messages\": [{\"role\": \"user\", \"content\": \"Explain the concept of caching in computer science.\"}],
\"max_tokens\": 1
}")
if [ "$warmup_code" -ne 200 ]; then
echo "FATAL: warmup request failed with HTTP $warmup_code"
cat "$HTTP_DIR/warmup.json" 2>/dev/null || true
exit 1
fi
echo "Warmup request successful (HTTP 200)"
sleep 2
# ── Step 1: Root + Healthcheck ─────────────────────────────
echo ""
echo "============================================"
echo "=== Step 1: Root + Healthcheck ==="
echo "============================================"
check_http "GET /" GET "/" "$HTTP_DIR/root.json" 200
assert_json_key "GET / — status" "$HTTP_DIR/root.json" "status" "ok"
assert_json_key "GET / — service" "$HTTP_DIR/root.json" "service" "LMCache HTTP API"
check_http "GET /healthcheck" GET "/healthcheck" "$HTTP_DIR/healthcheck.json" 200
assert_json_key "GET /healthcheck — status" "$HTTP_DIR/healthcheck.json" "status" "healthy"
# ── Step 2: Configuration ──────────────────────────────────
echo ""
echo "============================================"
echo "=== Step 2: Configuration (/config) ==="
echo "============================================"
check_http "GET /config" GET "/config" "$HTTP_DIR/config.json" 200
python3 -c "
import json, sys
with open('$HTTP_DIR/config.json') as f:
data = json.load(f)
required = ['mp', 'storage_manager', 'observability', 'http']
missing = [k for k in required if k not in data]
if missing:
print(f'FAIL: /config missing keys: {missing}')
sys.exit(1)
print('All config sections present: ' + ', '.join(required))
" && pass "GET /config — required keys present" \
|| fail "GET /config — required keys present" "missing config sections"
# ── Step 3: Status ─────────────────────────────────────────
echo ""
echo "============================================"
echo "=== Step 3: Status (/status) ==="
echo "============================================"
check_http "GET /status" GET "/status" "$HTTP_DIR/status.json" 200
python3 -c "
import json, sys
with open('$HTTP_DIR/status.json') as f:
data = json.load(f)
if not isinstance(data, dict):
print('FAIL: /status did not return a JSON dict')
sys.exit(1)
print('Status response is a valid dict with ' + str(len(data)) + ' keys')
" && pass "GET /status — valid JSON dict" \
|| fail "GET /status — valid JSON dict" "response is not a dict"
# ── Step 4: Metrics ────────────────────────────────────────
echo ""
echo "============================================"
echo "=== Step 4: Metrics (/metrics) ==="
echo "============================================"
check_http "GET /metrics" GET "/metrics" "$HTTP_DIR/metrics.txt" 200
assert_contains "GET /metrics — has lmcache metrics" "$HTTP_DIR/metrics.txt" "lmcache_"
assert_contains "GET /metrics — Prometheus format" "$HTTP_DIR/metrics.txt" "# "
# ── Step 5: Environment ────────────────────────────────────
echo ""
echo "============================================"
echo "=== Step 5: Environment (/env) ==="
echo "============================================"
check_http "GET /env" GET "/env" "$HTTP_DIR/env.json" 200
python3 -c "
import json, sys
with open('$HTTP_DIR/env.json') as f:
data = json.load(f)
if not isinstance(data, dict):
print('FAIL: /env did not return a JSON dict')
sys.exit(1)
if 'PATH' not in data:
print('FAIL: /env missing PATH key')
sys.exit(1)
print('/env returned ' + str(len(data)) + ' environment variables')
" && pass "GET /env — valid JSON with PATH" \
|| fail "GET /env — valid JSON with PATH" "validation failed"
# ── Step 6: Log Level ──────────────────────────────────────
echo ""
echo "============================================"
echo "=== Step 6: Log Level (/loglevel) ==="
echo "============================================"
check_http "GET /loglevel (list)" GET "/loglevel" "$HTTP_DIR/loglevel_list.txt" 200
assert_contains "GET /loglevel — header" "$HTTP_DIR/loglevel_list.txt" "Loggers and Levels"
check_http "GET /loglevel (set DEBUG)" GET "/loglevel?logger_name=lmcache&level=DEBUG" "$HTTP_DIR/loglevel_set.txt" 200
assert_contains "GET /loglevel (set DEBUG) — confirmation" "$HTTP_DIR/loglevel_set.txt" "Set lmcache level to DEBUG"
check_http "GET /loglevel (invalid level)" GET "/loglevel?logger_name=lmcache&level=INVALID_LEVEL" "$HTTP_DIR/loglevel_bad.txt" 400
# ── Step 7: Threads ────────────────────────────────────────
echo ""
echo "============================================"
echo "=== Step 7: Threads ==="
echo "============================================"
check_http "GET /threads" GET "/threads" "$HTTP_DIR/threads.txt" 200
assert_contains "GET /threads — header" "$HTTP_DIR/threads.txt" "Thread Summary"
check_http "GET /periodic-threads" GET "/periodic-threads" "$HTTP_DIR/periodic_threads.json" 200
python3 -c "
import json, sys
with open('$HTTP_DIR/periodic_threads.json') as f:
data = json.load(f)
if 'summary' not in data or 'threads' not in data:
print('FAIL: /periodic-threads missing summary or threads key')
sys.exit(1)
if 'total_count' not in data['summary']:
print('FAIL: /periodic-threads summary missing total_count')
sys.exit(1)
print('/periodic-threads: ' + str(data['summary']['total_count']) + ' threads registered')
" && pass "GET /periodic-threads — structure" \
|| fail "GET /periodic-threads — structure" "validation failed"
check_http "GET /periodic-threads-health" GET "/periodic-threads-health" "$HTTP_DIR/periodic_health.json" 200
assert_json_key "GET /periodic-threads-health — healthy key" "$HTTP_DIR/periodic_health.json" "healthy"
# ── Step 8: Clear Cache ────────────────────────────────────
echo ""
echo "============================================"
echo "=== Step 8: Clear Cache (/cache/clear) ==="
echo "============================================"
check_http "POST /cache/clear" POST "/cache/clear" "$HTTP_DIR/clear_cache.json" 200
assert_json_key "POST /cache/clear — status" "$HTTP_DIR/clear_cache.json" "status" "ok"
# ── Step 9: Quota CRUD ─────────────────────────────────────
echo ""
echo "============================================"
echo "=== Step 9: Quota CRUD ==="
echo "============================================"
QUOTA_SALT="test_salt_${BUILD_ID}"
# 9a: Create quota
check_http "PUT /quota/${QUOTA_SALT}" PUT "/quota/${QUOTA_SALT}" "$HTTP_DIR/quota_put.json" 200 '{"limit_gb": 1.0}'
assert_json_key "PUT quota — status" "$HTTP_DIR/quota_put.json" "status" "ok"
# 9b: Read quota
check_http "GET /quota/${QUOTA_SALT}" GET "/quota/${QUOTA_SALT}" "$HTTP_DIR/quota_get.json" 200
python3 -c "
import json, sys
with open('$HTTP_DIR/quota_get.json') as f:
data = json.load(f)
if not data.get('exists'):
print('FAIL: quota should exist after PUT')
sys.exit(1)
limit = data.get('limit_gb', 0)
if abs(limit - 1.0) > 0.01:
print(f'FAIL: expected limit_gb ~1.0, got {limit}')
sys.exit(1)
print(f'Quota exists with limit_gb={limit}')
" && pass "GET /quota — exists with correct limit" \
|| fail "GET /quota — exists with correct limit" "validation failed"
# 9c: List quotas
check_http "GET /quota (list)" GET "/quota" "$HTTP_DIR/quota_list.json" 200
assert_json_key "GET /quota (list) — users key" "$HTTP_DIR/quota_list.json" "users"
# 9d: Delete quota
check_http "DELETE /quota/${QUOTA_SALT}" DELETE "/quota/${QUOTA_SALT}" "$HTTP_DIR/quota_delete.json" 200
assert_json_key "DELETE quota — status" "$HTTP_DIR/quota_delete.json" "status" "removed"
# 9e: Verify deletion
check_http "GET /quota/${QUOTA_SALT} (after delete)" GET "/quota/${QUOTA_SALT}" "$HTTP_DIR/quota_gone.json" 200
python3 -c "
import json, sys
with open('$HTTP_DIR/quota_gone.json') as f:
data = json.load(f)
if data.get('exists'):
print('FAIL: quota should not exist after DELETE')
sys.exit(1)
print('Quota correctly gone after DELETE')
" && pass "GET /quota after DELETE — not exists" \
|| fail "GET /quota after DELETE — not exists" "quota still exists"
# 9f: Delete non-existent
check_http "DELETE /quota/nonexistent_salt" DELETE "/quota/nonexistent_salt" "$HTTP_DIR/quota_notfound.json" 200
assert_json_key "DELETE nonexistent — status" "$HTTP_DIR/quota_notfound.json" "status" "not_found"
# 9g: Error — negative limit
check_http "PUT /quota (negative limit)" PUT "/quota/err_salt" "$HTTP_DIR/quota_neg.json" 400 '{"limit_gb": -1.0}'
# 9h: Error — missing key
check_http "PUT /quota (missing limit_gb)" PUT "/quota/err_salt" "$HTTP_DIR/quota_badkey.json" 400 '{"bad_key": 1}'
# ── Step 10: Version ───────────────────────────────────────
echo ""
echo "============================================"
echo "=== Step 10: Version ==="
echo "============================================"
check_http "GET /version" GET "/version" "$HTTP_DIR/version.txt" 200
check_http "GET /lmc_version" GET "/lmc_version" "$HTTP_DIR/lmc_version.txt" 200
check_http "GET /commit_id" GET "/commit_id" "$HTTP_DIR/commit_id.txt" 200
# ================================================================
# PART 2: CLI COMMAND TESTS
# ================================================================
echo ""
echo "========================================================"
echo "=== Part 2: CLI Command Tests ==="
echo "========================================================"
# ── Step 11: lmcache describe kvcache ──────────────────────
echo ""
echo "============================================"
echo "=== Step 11: lmcache describe kvcache ==="
echo "============================================"
if lmcache describe kvcache --url "${BASE_URL}" > "$HTTP_DIR/describe.txt" 2>&1; then
pass "lmcache describe kvcache — exit code 0"
else
fail "lmcache describe kvcache — exit code 0" "command returned non-zero"
echo " Output:"
cat "$HTTP_DIR/describe.txt" 2>/dev/null || true
fi
assert_contains "describe — Health section" "$HTTP_DIR/describe.txt" "Health"
assert_contains "describe — L1 capacity section" "$HTTP_DIR/describe.txt" "L1 capacity"
# ── Step 12: lmcache kvcache clear ─────────────────────────
echo ""
echo "============================================"
echo "=== Step 12: lmcache kvcache clear ==="
echo "============================================"
if lmcache kvcache clear --url "${BASE_URL}" > "$HTTP_DIR/kvcache_clear.txt" 2>&1; then
pass "lmcache kvcache clear — exit code 0"
else
fail "lmcache kvcache clear — exit code 0" "command returned non-zero"
echo " Output:"
cat "$HTTP_DIR/kvcache_clear.txt" 2>/dev/null || true
fi
assert_contains "kvcache clear — OK" "$HTTP_DIR/kvcache_clear.txt" "OK"
# ================================================================
# SUMMARY
# ================================================================
echo ""
echo "========================================================"
echo "=== Summary ==="
echo "========================================================"
echo "Tests run: $TESTS_RUN"
echo "Tests passed: $TESTS_PASSED"
echo "Tests failed: $TESTS_FAILED"
echo ""
if [ "$TESTS_FAILED" -gt 0 ]; then
echo "============================================"
echo "=== HTTP API & CLI Test FAILED ==="
echo "============================================"
exit 1
fi
echo "============================================"
echo "=== HTTP API & CLI Test PASSED ==="
echo "============================================"
.buildkite/k3_tests/multiprocess/scripts/run-lm-eval.sh
Executable
+120
View File
@@ -0,0 +1,120 @@
#!/usr/bin/env bash
# Run lm_eval workload test against vLLM server.
# Sends the same requests twice to test LMCache caching behavior.
# Adapted from the old Docker-based run-lm-eval.sh -- no venv setup needed
# (setup-env.sh + extras already installed by run.sh).
set -e
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# Configuration
VLLM_PORT="${VLLM_PORT:-8000}"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
NUM_CONCURRENT="${NUM_CONCURRENT:-50}"
LIMIT="${LIMIT:-300}"
BUILD_ID="${BUILD_ID:-local_$$}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
# Output directories
LM_EVAL_DIR="$RESULTS_DIR/lm_eval"
FIRST_RUN_DIR="$LM_EVAL_DIR/first_run"
SECOND_RUN_DIR="$LM_EVAL_DIR/second_run"
echo "=== LM-Eval Workload Test ==="
echo "Model: $MODEL"
echo "vLLM Port: $VLLM_PORT"
echo "Concurrent requests: $NUM_CONCURRENT"
echo "Limit: $LIMIT"
echo "Results dir: $LM_EVAL_DIR"
echo ""
mkdir -p "$FIRST_RUN_DIR" "$SECOND_RUN_DIR"
run_lm_eval() {
local run_name="$1"
local output_dir="$2"
echo "=== Running lm_eval ($run_name) ==="
lm_eval --model local-completions --tasks gsm8k \
--model_args "model=${MODEL},base_url=http://127.0.0.1:${VLLM_PORT}/v1/completions,num_concurrent=${NUM_CONCURRENT},max_retries=3,tokenized_requests=False" \
--limit "$LIMIT" \
--seed 0 \
-s --output_path "$output_dir" \
--gen_kwargs '{"temperature": 0.0}'
echo "$run_name completed"
echo ""
}
verify_samples_match() {
local first_dir="$1"
local second_dir="$2"
echo "=== Verifying samples files match ==="
first_samples=$(find "$first_dir" -name "samples_gsm8k_*.jsonl" -type f 2>/dev/null | head -1)
second_samples=$(find "$second_dir" -name "samples_gsm8k_*.jsonl" -type f 2>/dev/null | head -1)
if [ -z "$first_samples" ]; then
echo "Could not find samples_gsm8k_*.jsonl in first run directory: $first_dir"
find "$first_dir" -type f -name "*.jsonl" || true
return 1
fi
if [ -z "$second_samples" ]; then
echo "Could not find samples_gsm8k_*.jsonl in second run directory: $second_dir"
find "$second_dir" -type f -name "*.jsonl" || true
return 1
fi
echo "First run samples: $first_samples"
echo "Second run samples: $second_samples"
first_sorted=$(mktemp)
second_sorted=$(mktemp)
sort "$first_samples" > "$first_sorted"
sort "$second_samples" > "$second_sorted"
if diff -q "$first_sorted" "$second_sorted" > /dev/null 2>&1; then
echo "Samples files are identical!"
rm -f "$first_sorted" "$second_sorted"
return 0
else
echo "Samples files differ!"
echo ""
echo "=== Diff (first 50 lines) ==="
diff "$first_sorted" "$second_sorted" | head -50 || true
rm -f "$first_sorted" "$second_sorted"
return 1
fi
}
# First run -- populates cache
echo "============================================"
echo "=== First lm_eval run (cache population) ==="
echo "============================================"
run_lm_eval "first_run" "$FIRST_RUN_DIR"
# Second run -- should use cached results
echo "============================================"
echo "=== Second lm_eval run (cache hit) ==="
echo "============================================"
run_lm_eval "second_run" "$SECOND_RUN_DIR"
# Verify consistency
echo "============================================"
echo "=== Verifying output consistency ==="
echo "============================================"
if ! verify_samples_match "$FIRST_RUN_DIR" "$SECOND_RUN_DIR"; then
echo "Verification failed: samples files do not match"
exit 1
fi
echo "============================================"
echo "=== LM-Eval workload test completed ==="
echo "============================================"
.buildkite/k3_tests/multiprocess/scripts/run-long-doc-qa-l2.sh
Executable
+575
View File
@@ -0,0 +1,575 @@
#!/usr/bin/env bash
# Run long_doc_qa for L2 skip_l1 mode with mock L2 adapter.
#
# This script:
# 1. Kills the existing LMCache MP server
# 2. Relaunches it with L2 config (skip_l1 + mock L2 at 2 GB/s)
# 3. Waits for vLLM to reconnect
# 4. Runs long_doc_qa against baseline (vLLM only) and L2-enabled vLLM
# 5. Verifies L2 query is faster than baseline and warmup overhead is bounded
#
# Expects the following env vars from run-mp-test.sh:
# VLLM_PORT, VLLM_BASELINE_PORT, MODEL, BUILD_ID, RESULTS_DIR, LMCACHE_DIR,
# LMCACHE_PORT, CPU_BUFFER_SIZE, MAX_WORKERS, GPU_FOR_VLLM (optional)
set -e
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# Configuration
VLLM_PORT="${VLLM_PORT:-8000}"
VLLM_BASELINE_PORT="${VLLM_BASELINE_PORT:-9000}"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
BUILD_ID="${BUILD_ID:-local_$$}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
LMCACHE_DIR="${LMCACHE_DIR:-$REPO_ROOT}"
LMCACHE_PORT="${LMCACHE_PORT:-6555}"
CPU_BUFFER_SIZE="${CPU_BUFFER_SIZE:-80}"
MAX_WORKERS="${MAX_WORKERS:-4}"
DOCUMENT_LENGTH="${DOCUMENT_LENGTH:-10000}"
NUM_DOCUMENTS="${NUM_DOCUMENTS:-30}"
OUTPUT_LEN="${OUTPUT_LEN:-200}"
REPEAT_COUNT="${REPEAT_COUNT:-2}"
REPEAT_MODE="${REPEAT_MODE:-tile}"
SHUFFLE_SEED="${SHUFFLE_SEED:-0}"
MAX_INFLIGHT_REQUESTS="${MAX_INFLIGHT_REQUESTS:-5}"
# Mock L2 config
L2_MAX_SIZE_GB="${L2_MAX_SIZE_GB:-80}"
L2_BANDWIDTH_GB="${L2_BANDWIDTH_GB:-4}"
# L2 performance thresholds
# Recent CI runs show ~1.51-1.67x query speedup, ~1.77-2.02x TTFT speedup,
# and ~0.87-0.99x warmup overhead. Tighten from the previous pass-anything
# thresholds (1.0x/1.0x/2.0x) while leaving headroom for variance.
MIN_L2_SPEEDUP="${MIN_L2_SPEEDUP:-1.3}"
MIN_L2_TTFT_SPEEDUP="${MIN_L2_TTFT_SPEEDUP:-1.5}"
MAX_WARMUP_OVERHEAD="${MAX_WARMUP_OVERHEAD:-1.2}"
L2_RESULTS_DIR="$RESULTS_DIR/long_doc_qa_l2"
PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}"
# /metrics is now served by the LMCache FastAPI HTTP server (port 8080
# by default) — the legacy ``--prometheus-port`` standalone server was
# disabled for the ``lmcache server`` entrypoint by #3164. Defined here
# (not just in Step 4) so the relaunch and the curl scrape agree.
METRICS_HTTP_PORT="${METRICS_HTTP_PORT:-8080}"
echo "=== Long Doc QA L2 Performance Test ==="
echo "Model: $MODEL"
echo "L2 adapter: mock (${L2_MAX_SIZE_GB}GB, ${L2_BANDWIDTH_GB}GB/s)"
echo "Store policy: skip_l1 | Eviction: noop"
echo "Thresholds: speedup>=${MIN_L2_SPEEDUP}x, TTFT speedup>=${MIN_L2_TTFT_SPEEDUP}x, overhead<=${MAX_WARMUP_OVERHEAD}x"
echo "Results: $L2_RESULTS_DIR"
echo ""
mkdir -p "$L2_RESULTS_DIR"
# ---------------------------------------------------------------------------
# Step 1: Kill existing LMCache + vLLM, relaunch both with L2 config
# ---------------------------------------------------------------------------
echo "--- Stopping existing LMCache MP server and vLLM ---"
# PID file layout: line1=LMCache, line2=vLLM w/ LMCache, line3=vLLM baseline.
# These processes were launched by an earlier script (launch-processes.sh)
# and are not children of this shell, so ``wait $pid`` is a no-op here.
# We instead poll until each PID actually exits, then poll until the
# Prometheus port is free, otherwise the LMCache relaunch below would
# fail to bind /metrics and the metrics check would fail spuriously.
if [ -f "$PID_FILE" ]; then
LMCACHE_PID=$(sed -n '1p' "$PID_FILE")
VLLM_PID=$(sed -n '2p' "$PID_FILE")
for pid in $LMCACHE_PID $VLLM_PID; do
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
echo "Killing PID $pid"
kill "$pid" 2>/dev/null || true
for _ in $(seq 1 60); do
kill -0 "$pid" 2>/dev/null || break
sleep 0.5
done
# Last resort: SIGKILL if SIGTERM didn't take after 30s.
if kill -0 "$pid" 2>/dev/null; then
echo "PID $pid still alive after SIGTERM; sending SIGKILL"
kill -9 "$pid" 2>/dev/null || true
fi
fi
done
# Poll until the Prometheus port is fully released so the new server
# below can bind it cleanly.
for _ in $(seq 1 30); do
if ! (ss -ltn 2>/dev/null || netstat -ltn 2>/dev/null) \
| awk '{print $4}' | grep -qE ":${METRICS_HTTP_PORT}$"; then
break
fi
sleep 0.5
done
fi
echo "--- Launching LMCache MP server with L2 config ---"
L2_ADAPTER_JSON="{\"type\":\"mock\",\"max_size_gb\":${L2_MAX_SIZE_GB},\"mock_bandwidth_gb\":${L2_BANDWIDTH_GB}}"
# Determine GPU to use
GPU_DEVICE="${GPU_FOR_VLLM:-0}"
CUDA_VISIBLE_DEVICES="${GPU_DEVICE}" \
lmcache server \
--l1-size-gb "$CPU_BUFFER_SIZE" \
--eviction-policy noop \
--l2-store-policy skip_l1 \
--l2-prefetch-policy default \
--l2-adapter "$L2_ADAPTER_JSON" \
--max-workers "$MAX_WORKERS" \
--metrics-sample-rate 1.0 \
--http-port "$METRICS_HTTP_PORT" \
--port "$LMCACHE_PORT" \
> "/tmp/build_${BUILD_ID}_lmcache_l2.log" 2>&1 &
NEW_LMCACHE_PID=$!
echo "LMCache L2 server started (PID=$NEW_LMCACHE_PID)"
echo "Waiting for LMCache L2 to initialize..."
sleep 10
echo "--- Launching vLLM with LMCache ---"
# Compute GPU memory utilization for large GPUs
GPU_MEMORY_UTIL_ARG=""
GPU_MEMORY_MB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits -i "${GPU_DEVICE}" | tr -d ' ')
GPU_MEMORY_GB=$((GPU_MEMORY_MB / 1024))
if [ "$GPU_MEMORY_GB" -gt 90 ]; then
GPU_MEMORY_UTIL_ARG="--gpu-memory-utilization 0.5"
fi
# Unset VLLM_PORT in child env so vLLM's torch.distributed picks a free port
env -u VLLM_PORT \
CUDA_VISIBLE_DEVICES="${GPU_DEVICE}" \
VLLM_ENABLE_V1_MULTIPROCESSING=0 \
VLLM_SERVER_DEV_MODE=1 \
VLLM_BATCH_INVARIANT=1 \
PYTHONHASHSEED=0 \
vllm serve "$MODEL" \
--kv-transfer-config "{\"kv_connector\":\"LMCacheMPConnector\", \"kv_role\":\"kv_both\", \"kv_load_failure_policy\": \"recompute\", \"kv_connector_extra_config\": {\"lmcache.mp.port\": $LMCACHE_PORT, \"lmcache.mp.mq_timeout\": 10}}" \
--attention-backend FLASH_ATTN \
--port "$VLLM_PORT" \
--no-async-scheduling \
$GPU_MEMORY_UTIL_ARG \
> "/tmp/build_${BUILD_ID}_vllm_l2.log" 2>&1 &
NEW_VLLM_PID=$!
echo "vLLM started (PID=$NEW_VLLM_PID)"
# Update PID file (replace lines 1 and 2, keep baseline on line 3)
if [ -f "$PID_FILE" ]; then
sed -i "1s/.*/$NEW_LMCACHE_PID/" "$PID_FILE"
sed -i "2s/.*/$NEW_VLLM_PID/" "$PID_FILE"
else
echo "$NEW_LMCACHE_PID" > "$PID_FILE"
echo "$NEW_VLLM_PID" >> "$PID_FILE"
fi
# Wait for vLLM to be ready (needs time to load model)
echo "--- Waiting for vLLM to be ready ---"
if ! wait_for_server "$VLLM_PORT" 300; then
echo "vLLM failed to start after restart"
echo "LMCache L2 log (last 50 lines):"
tail -50 "/tmp/build_${BUILD_ID}_lmcache_l2.log" || true
echo "vLLM log (last 50 lines):"
tail -50 "/tmp/build_${BUILD_ID}_vllm_l2.log" || true
exit 1
fi
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
run_long_doc_qa() {
local port="$1"
local result_file="$2"
local description="$3"
echo "--- Running long_doc_qa ($description) on port $port ---"
local output_file="$L2_RESULTS_DIR/${description}_output.txt"
python3 "$LMCACHE_DIR/benchmarks/long_doc_qa/long_doc_qa.py" \
--port "$port" \
--model "$MODEL" \
--document-length "$DOCUMENT_LENGTH" \
--num-documents "$NUM_DOCUMENTS" \
--output-len "$OUTPUT_LEN" \
--repeat-count "$REPEAT_COUNT" \
--repeat-mode "$REPEAT_MODE" \
--shuffle-seed "$SHUFFLE_SEED" \
--max-inflight-requests "$MAX_INFLIGHT_REQUESTS" \
--output "$output_file" \
--json-output \
2>>"$output_file" | tee "$result_file"
echo "Completed: $description"
echo ""
}
extract_json_field() {
local json_file="$1"
local field="$2"
tail -n 1 "$json_file" | python3 -c "
import json, sys
try:
data = json.loads(sys.stdin.read())
v = data.get('$field')
print(v if v is not None else 'null')
except Exception:
print('null')
"
}
# ---------------------------------------------------------------------------
# Step 2: Run benchmarks
# ---------------------------------------------------------------------------
# Phase 1: Baseline -- reuse results from step 5 (same port, same params)
STEP5_BASELINE="$RESULTS_DIR/long_doc_qa/baseline_result.json"
if [ -f "$STEP5_BASELINE" ]; then
echo "============================================"
echo "=== Phase 1: Reusing baseline from step 5 ==="
echo "============================================"
cp "$STEP5_BASELINE" "$L2_RESULTS_DIR/baseline_result.json"
echo "Copied baseline results from $STEP5_BASELINE"
echo ""
else
echo "============================================"
echo "=== Phase 1: Baseline vLLM (no LMCache) ==="
echo "============================================"
run_long_doc_qa "$VLLM_BASELINE_PORT" "$L2_RESULTS_DIR/baseline_result.json" "baseline"
fi
# Phase 2+3: L2 warmup + query (repeat_count=2, tile mode)
# Round 1 (warmup): prompts -> L1 write buffer -> L2 store -> L1 delete
# Round 2 (query): prompts -> L1 miss -> L2 prefetch -> L1 load -> serve
echo "============================================"
echo "=== Phase 2+3: vLLM + LMCache L2 ==="
echo "============================================"
run_long_doc_qa "$VLLM_PORT" "$L2_RESULTS_DIR/l2_result.json" "l2"
# ---------------------------------------------------------------------------
# Step 3: Verify thresholds
# ---------------------------------------------------------------------------
echo "============================================"
echo "=== Verifying L2 Performance ==="
echo "============================================"
baseline_query_ttft=$(extract_json_field "$L2_RESULTS_DIR/baseline_result.json" "query_ttft_per_prompt")
baseline_query_round_time=$(extract_json_field "$L2_RESULTS_DIR/baseline_result.json" "query_round_time_per_prompt")
baseline_warmup_round_time=$(extract_json_field "$L2_RESULTS_DIR/baseline_result.json" "warmup_round_time_per_prompt")
l2_query_ttft=$(extract_json_field "$L2_RESULTS_DIR/l2_result.json" "query_ttft_per_prompt")
l2_query_round_time=$(extract_json_field "$L2_RESULTS_DIR/l2_result.json" "query_round_time_per_prompt")
l2_warmup_round_time=$(extract_json_field "$L2_RESULTS_DIR/l2_result.json" "warmup_round_time_per_prompt")
python3 << EOF
import sys
def sf(val):
try: return float(val)
except: return None
bqt = sf("$baseline_query_ttft")
bqrt = sf("$baseline_query_round_time")
bwrt = sf("$baseline_warmup_round_time")
lqt = sf("$l2_query_ttft")
lqrt = sf("$l2_query_round_time")
lwrt = sf("$l2_warmup_round_time")
min_spd = float("$MIN_L2_SPEEDUP")
min_ttft = float("$MIN_L2_TTFT_SPEEDUP")
max_oh = float("$MAX_WARMUP_OVERHEAD")
failed = False
print("=" * 60)
print("L2 Performance Summary")
print("=" * 60)
print(f"{'Metric':<35} {'Baseline':>12} {'L2':>12}")
print("-" * 60)
for name, bv, lv in [
("query_ttft_per_prompt (s)", bqt, lqt),
("query_round_time_per_prompt (s)", bqrt, lqrt),
("warmup_round_time_per_prompt (s)", bwrt, lwrt),
]:
bs = f"{bv:.4f}" if bv else "N/A"
ls = f"{lv:.4f}" if lv else "N/A"
print(f"{name:<35} {bs:>12} {ls:>12}")
print()
print("=" * 60)
print("Threshold Verification")
print("=" * 60)
# 1. L2 query round-time speedup
if lqrt and bqrt and lqrt > 0:
s = bqrt / lqrt
ok = s >= min_spd
print(f"[{'PASS' if ok else 'FAIL'}] L2 query speedup: {s:.2f}x (need >= {min_spd}x)")
if not ok: failed = True
else:
print("[FAIL] Cannot compute L2 query speedup"); failed = True
# 2. L2 TTFT speedup
if lqt and bqt and lqt > 0:
s = bqt / lqt
ok = s >= min_ttft
print(f"[{'PASS' if ok else 'FAIL'}] L2 TTFT speedup: {s:.2f}x (need >= {min_ttft}x)")
if not ok: failed = True
else:
print("[FAIL] Cannot compute L2 TTFT speedup"); failed = True
# 3. Warmup overhead
if lwrt and bwrt and bwrt > 0:
o = lwrt / bwrt
ok = o <= max_oh
print(f"[{'PASS' if ok else 'FAIL'}] Warmup overhead: {o:.2f}x (need <= {max_oh}x)")
if not ok: failed = True
else:
print("[FAIL] Cannot compute warmup overhead"); failed = True
print()
if failed:
print("[FAIL] L2 performance verification FAILED")
sys.exit(1)
else:
print("[PASS] All L2 performance thresholds passed")
EOF
# ---------------------------------------------------------------------------
# Step 4: Verify L2 data flow via Prometheus metrics
# ---------------------------------------------------------------------------
echo "============================================"
echo "=== Verifying L2 Data Flow (Metrics) ==="
echo "============================================"
L2_METRICS_FILE="$L2_RESULTS_DIR/prometheus_metrics.txt"
# Retry briefly: when LMCache is relaunched on the same port as the
# previous instance, the Prometheus socket can take a moment to come
# back up, and a single-shot curl loses the metrics check silently.
> "$L2_METRICS_FILE"
for i in 1 2 3 4 5; do
if curl -sf "http://localhost:${METRICS_HTTP_PORT}/metrics" \
> "$L2_METRICS_FILE" 2>/dev/null && [ -s "$L2_METRICS_FILE" ]; then
break
fi
sleep 2
done
if [ ! -s "$L2_METRICS_FILE" ]; then
echo "FAIL: could not fetch /metrics from LMCache HTTP server (port $METRICS_HTTP_PORT)."
echo " /metrics being unreachable means we cannot verify the L2"
echo " data flow or the observability surface; failing the test"
echo " rather than silently skipping."
echo ""
echo "--- LMCache L2 server log (last 50 lines) ---"
tail -50 "/tmp/build_${BUILD_ID}_lmcache_l2.log" 2>&1 || true
echo ""
echo "--- Listening sockets on port ${METRICS_HTTP_PORT} ---"
(ss -ltnp 2>/dev/null || netstat -ltnp 2>/dev/null || true) \
| awk -v p=":${METRICS_HTTP_PORT}" '$0 ~ p'
exit 1
fi
python3 -c "
import sys
with open('$L2_METRICS_FILE') as f:
metrics_text = f.read()
def get_counter(name):
for line in metrics_text.splitlines():
if line.startswith(name + ' ') or line.startswith(name + '{'):
return float(line.rsplit(' ', 1)[-1])
return 0.0
# L1 metrics
l1_write_keys = get_counter('lmcache_mp_l1_write_chunks_total')
# L2 metrics
store_keys = get_counter('lmcache_mp_l2_store_submitted_objects_chunks_total')
store_succeeded = get_counter('lmcache_mp_l2_store_completed_objects_chunks_total')
prefetch_lookups = get_counter('lmcache_mp_l2_prefetch_lookup_requests_total')
prefetch_hits = get_counter('lmcache_mp_l2_prefetch_hit_chunks_total')
prefetch_loaded = get_counter('lmcache_mp_l2_prefetch_load_completed_chunks_total')
print('=' * 60)
print('Data Flow Metrics')
print('=' * 60)
print(f' L1 write keys: {l1_write_keys:.0f}')
print(f' L2 store keys submitted: {store_keys:.0f}')
print(f' L2 store keys succeeded: {store_succeeded:.0f}')
print(f' L2 prefetch lookups: {prefetch_lookups:.0f}')
print(f' L2 prefetch prefix hits: {prefetch_hits:.0f}')
print(f' L2 prefetch keys loaded: {prefetch_loaded:.0f}')
print()
failed = False
def check(cond, pass_msg, fail_msg):
global failed
if cond:
print(f'[PASS] {pass_msg}')
else:
print(f'[FAIL] {fail_msg}')
failed = True
# 1. L1 store activity (warmup writes KV to L1 before L2 store)
check(l1_write_keys > 0,
f'L1 store: {l1_write_keys:.0f} keys written',
'No keys written to L1 (expected > 0 from warmup)')
# 2. L2 store submitted and completed
check(store_keys > 0,
f'L2 store: {store_keys:.0f} keys submitted',
'No keys submitted to L2 store')
check(store_succeeded > 0,
f'L2 store: {store_succeeded:.0f} keys succeeded',
'No keys successfully stored to L2')
# 3. L2 prefetch submitted and completed (query round: L1 cold, L2 has data)
check(prefetch_lookups > 0,
f'L2 prefetch: {prefetch_lookups:.0f} lookup requests',
'No prefetch lookups (expected > 0 from query round)')
check(prefetch_hits > 0,
f'L2 prefetch: {prefetch_hits:.0f} prefix hits',
'No prefix hits from L2 lookup')
check(prefetch_loaded > 0,
f'L2 prefetch: {prefetch_loaded:.0f} keys loaded',
'No keys loaded from L2')
print()
if failed:
print('[FAIL] Data flow verification FAILED')
sys.exit(1)
else:
print('[PASS] All data flow checks passed')
"
# ---------------------------------------------------------------------------
# Step 5: Verify the rest of the MP observability surface
# ---------------------------------------------------------------------------
# The data-flow block above is L2-focused. This block goes wider — it
# asserts that every metric we publish from MP mode actually advances
# during the run. ``--metrics-sample-rate 1.0`` was set on the relaunch
# above so the histograms record on every event (the default 0.01 would
# leave them empty in this short workload and flake the assertions).
echo ""
echo "============================================"
echo "=== Verifying full MP observability surface ==="
echo "============================================"
python3 - "$L2_METRICS_FILE" <<'PYEOF'
import re
import sys
with open(sys.argv[1]) as f:
text = f.read()
def counter_total(name: str) -> float:
"""Sum a counter across all label combinations."""
total = 0.0
pat = re.compile(rf"^{re.escape(name)}(\{{[^}}]*\}})?\s+([0-9eE+\-.]+)\s*$", re.M)
for _, value in pat.findall(text):
try:
total += float(value)
except ValueError:
pass
return total
def histogram_count(base_name: str) -> float:
"""Sum the ``_count`` series across all label combinations.
Non-zero means the histogram observed at least one sample.
The OTel→Prometheus bridge appends the OTel ``unit`` to the metric
name (e.g. unit ``GB/s`` → ``GB_per_second``), so the actual series
looks like ``<base>_GB_per_second_count``. Match that as a suffix
so this works whether or not the unit is present.
"""
pat = re.compile(
rf"^{re.escape(base_name)}(?:_[A-Za-z_]+)?_count(?:\{{[^}}]*\}})?\s+"
rf"([0-9eE+\-.]+)\s*$",
re.M,
)
return sum(float(v) for v in pat.findall(text))
def has_label(base_name: str, label: str) -> bool:
"""Check that at least one sample of `base_name` carries the named label.
Tolerates the OTel unit suffix that Prometheus appends to histograms.
"""
pat = re.compile(
rf"^{re.escape(base_name)}(?:_[A-Za-z_]+)?(?:_count|_sum|_bucket)?"
rf"\{{[^}}]*\b{re.escape(label)}=",
re.M,
)
return bool(pat.search(text))
# (kind, metric_name, optional_label_to_assert_present_or_None)
checks = [
# ── Newer counters (with label dimensions) ─────────────────────
("counter", "lmcache_mp_l2_store_completed_requests_total", "l2_name"),
("counter", "lmcache_mp_l2_load_completed_requests_total", "l2_name"),
("counter", "lmcache_mp_lookup_requested_tokens_total", "model_name"),
("counter", "lmcache_mp_lookup_hit_tokens_total", "model_name"),
("counter", "lmcache_mp_num_chunks_loaded_total", "worker_id"),
# ── Histograms. The OTel→Prometheus bridge appends the OTel
# ``unit`` to the series name, so a histogram declared with
# ``unit="GB/s"`` actually reports as
# ``<name>_GB_per_second_count`` / ``..._sum`` / ``..._bucket``.
# Match by base name and let the helper tolerate the unit suffix.
("hist", "lmcache_mp_l0_l1_store_throughput", None),
("hist", "lmcache_mp_l0_l1_load_throughput", None),
("hist", "lmcache_mp_l2_store_throughput", "l2_name"),
("hist", "lmcache_mp_l2_load_throughput", "l2_name"),
]
failed = False
for kind, name, label in checks:
if kind == "counter":
value = counter_total(name)
ok = value > 0
detail = f"total={value:.0f}"
else:
value = histogram_count(name)
ok = value > 0
detail = f"_count={value:.0f}"
status = "PASS" if ok else "FAIL"
print(f"[{status}] {name}: {detail}")
if not ok:
failed = True
continue
if label is not None:
if has_label(name, label):
print(f" └─ label '{label}' present")
else:
print(f"[FAIL] {name}: expected label '{label}' is missing")
failed = True
print()
if failed:
print("[FAIL] Observability metric verification FAILED")
print(" (some metric did not advance, or its label dimension is missing)")
sys.exit(1)
print("[PASS] All observability metrics populated.")
PYEOF
echo "============================================"
echo "=== L2 Long Doc QA test completed ==="
echo "============================================"
echo "Results: $L2_RESULTS_DIR"
.buildkite/k3_tests/multiprocess/scripts/run-long-doc-qa.sh
Executable
+264
View File
@@ -0,0 +1,264 @@
#!/usr/bin/env bash
# Run long_doc_qa workload test against both vLLM servers.
# Compares performance between LMCache-enabled and baseline vLLM.
# Adapted from the old Docker-based run-long-doc-qa.sh.
set -e
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# Configuration
VLLM_PORT="${VLLM_PORT:-8000}"
VLLM_BASELINE_PORT="${VLLM_BASELINE_PORT:-9000}"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
BUILD_ID="${BUILD_ID:-local_$$}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
LMCACHE_DIR="${LMCACHE_DIR:-$REPO_ROOT}"
DOCUMENT_LENGTH="${DOCUMENT_LENGTH:-10000}"
NUM_DOCUMENTS="${NUM_DOCUMENTS:-30}"
OUTPUT_LEN="${OUTPUT_LEN:-200}"
REPEAT_COUNT="${REPEAT_COUNT:-2}"
REPEAT_MODE="${REPEAT_MODE:-tile}"
SHUFFLE_SEED="${SHUFFLE_SEED:-0}"
MAX_INFLIGHT_REQUESTS="${MAX_INFLIGHT_REQUESTS:-5}"
# Relative performance thresholds (compared against baseline run in same job)
# Negative values mean LMCache must be *faster* than baseline by at least that %.
# Recent CI runs show ~77-84% TTFT improvement and ~27-40% round-time improvement,
# so requiring 60% and 15% respectively leaves comfortable headroom.
MAX_TTFT_SLOWDOWN_PCT="${MAX_TTFT_SLOWDOWN_PCT:--60}"
MAX_ROUND_TIME_SLOWDOWN_PCT="${MAX_ROUND_TIME_SLOWDOWN_PCT:--15}"
# Output directory
LONG_DOC_QA_DIR="$RESULTS_DIR/long_doc_qa"
echo "=== Long Doc QA Test ==="
echo "Model: $MODEL"
echo "vLLM Port (with LMCache): $VLLM_PORT"
echo "vLLM Baseline Port (without LMCache): $VLLM_BASELINE_PORT"
echo "Document length: $DOCUMENT_LENGTH"
echo "Number of documents: $NUM_DOCUMENTS"
echo "Output length: $OUTPUT_LEN"
echo "Results dir: $LONG_DOC_QA_DIR"
echo ""
echo "Performance thresholds (relative to baseline, negative = must be faster):"
echo " Max TTFT slowdown: ${MAX_TTFT_SLOWDOWN_PCT}% (LMCache must be >= $(echo "$MAX_TTFT_SLOWDOWN_PCT" | tr -d '-')% faster)"
echo " Max round time slowdown: ${MAX_ROUND_TIME_SLOWDOWN_PCT}% (LMCache must be >= $(echo "$MAX_ROUND_TIME_SLOWDOWN_PCT" | tr -d '-')% faster)"
echo ""
mkdir -p "$LONG_DOC_QA_DIR"
run_long_doc_qa() {
local port="$1"
local result_file="$2"
local description="$3"
echo "=== Running long_doc_qa ($description) ==="
local output_file="$LONG_DOC_QA_DIR/${description}_output.txt"
python3 "$LMCACHE_DIR/benchmarks/long_doc_qa/long_doc_qa.py" \
--port "$port" \
--model "$MODEL" \
--document-length "$DOCUMENT_LENGTH" \
--num-documents "$NUM_DOCUMENTS" \
--output-len "$OUTPUT_LEN" \
--repeat-count "$REPEAT_COUNT" \
--repeat-mode "$REPEAT_MODE" \
--shuffle-seed "$SHUFFLE_SEED" \
--max-inflight-requests "$MAX_INFLIGHT_REQUESTS" \
--output "$output_file" \
--json-output \
2>>"$output_file" | tee "$result_file"
echo "$description benchmark completed"
echo ""
}
extract_json_field() {
local json_file="$1"
local field="$2"
local json_line
json_line=$(tail -n 1 "$json_file")
python3 -c "
import json, sys
try:
data = json.loads('''$json_line''')
value = data.get('$field', 'null')
print(value if value is not None else 'null')
except json.JSONDecodeError:
print('null')
"
}
compare_results() {
local lmcache_result="$LONG_DOC_QA_DIR/lmcache_result.json"
local baseline_result="$LONG_DOC_QA_DIR/baseline_result.json"
echo "=== Comparing benchmark results ==="
if [ ! -f "$lmcache_result" ] || [ ! -f "$baseline_result" ]; then
echo "Result files not found"
return 1
fi
lmcache_query_ttft=$(extract_json_field "$lmcache_result" "query_ttft_per_prompt")
lmcache_query_round_time=$(extract_json_field "$lmcache_result" "query_round_time_per_prompt")
lmcache_warmup_round_time=$(extract_json_field "$lmcache_result" "warmup_round_time_per_prompt")
baseline_query_ttft=$(extract_json_field "$baseline_result" "query_ttft_per_prompt")
baseline_query_round_time=$(extract_json_field "$baseline_result" "query_round_time_per_prompt")
baseline_warmup_round_time=$(extract_json_field "$baseline_result" "warmup_round_time_per_prompt")
echo ""
echo "============================================"
echo "=== Performance Summary ==="
echo "============================================"
python3 << EOF
import sys
def safe_float(val):
try:
return float(val)
except (ValueError, TypeError):
return None
def format_comparison(name, lmcache_val, baseline_val):
lmcache = safe_float(lmcache_val)
baseline = safe_float(baseline_val)
if lmcache is None or baseline is None:
return f"{name}: Unable to compare (invalid values)"
if baseline > 0:
diff_pct = ((lmcache - baseline) / baseline) * 100
diff_str = f"{abs(diff_pct):.2f}% {'faster' if diff_pct < 0 else 'slower'}"
else:
diff_str = "N/A"
return f"{name}:\n Baseline: {baseline:.4f}s\n LMCache: {lmcache:.4f}s\n Diff: {diff_str}"
print(format_comparison("query_ttft_per_prompt", "$lmcache_query_ttft", "$baseline_query_ttft"))
print()
print(format_comparison("query_round_time_per_prompt", "$lmcache_query_round_time", "$baseline_query_round_time"))
print()
print(format_comparison("warmup_round_time_per_prompt", "$lmcache_warmup_round_time", "$baseline_warmup_round_time"))
print()
# Summary table
print(f"{'Metric':<35} {'Baseline':>12} {'LMCache':>12} {'Diff':>10}")
print("-" * 70)
metrics = [
("query_ttft_per_prompt", "$baseline_query_ttft", "$lmcache_query_ttft"),
("query_round_time_per_prompt", "$baseline_query_round_time", "$lmcache_query_round_time"),
("warmup_round_time_per_prompt", "$baseline_warmup_round_time", "$lmcache_warmup_round_time"),
]
for name, bval, lval in metrics:
b = safe_float(bval)
l = safe_float(lval)
if b is not None and l is not None:
diff_pct = ((l - b) / b) * 100 if b > 0 else 0
print(f"{name:<35} {b:>12.4f} {l:>12.4f} {diff_pct:>+9.1f}%")
else:
print(f"{name:<35} {'N/A':>12} {'N/A':>12} {'N/A':>10}")
print()
EOF
return 0
}
verify_thresholds() {
local lmcache_result="$LONG_DOC_QA_DIR/lmcache_result.json"
local baseline_result="$LONG_DOC_QA_DIR/baseline_result.json"
echo "=== Verifying LMCache performance vs baseline ==="
echo "Max allowed TTFT slowdown: ${MAX_TTFT_SLOWDOWN_PCT}%"
echo "Max allowed query round time slowdown: ${MAX_ROUND_TIME_SLOWDOWN_PCT}%"
echo ""
lmcache_query_ttft=$(extract_json_field "$lmcache_result" "query_ttft_per_prompt")
lmcache_query_round_time=$(extract_json_field "$lmcache_result" "query_round_time_per_prompt")
baseline_query_ttft=$(extract_json_field "$baseline_result" "query_ttft_per_prompt")
baseline_query_round_time=$(extract_json_field "$baseline_result" "query_round_time_per_prompt")
python3 << EOF
import sys
def safe_float(val):
try:
return float(val)
except (ValueError, TypeError):
return None
def check_metric(name, lmcache_val, baseline_val, max_slowdown_pct):
lmc = safe_float(lmcache_val)
base = safe_float(baseline_val)
if lmc is None or base is None or base <= 0:
print(f"{name}: unable to compare (lmcache={lmcache_val}, baseline={baseline_val}) -- FAIL")
return False
pct = ((lmc - base) / base) * 100
label = f"{abs(pct):.1f}% faster" if pct < 0 else f"{pct:.1f}% slower"
if max_slowdown_pct < 0:
threshold_label = f"need >= {abs(max_slowdown_pct):.0f}% faster"
else:
threshold_label = f"max {max_slowdown_pct}% slower"
if pct <= max_slowdown_pct:
print(f"{name}: {lmc:.4f}s vs baseline {base:.4f}s ({label}, {threshold_label}) -- PASS")
return True
else:
print(f"{name}: {lmc:.4f}s vs baseline {base:.4f}s ({label}, {threshold_label}) -- FAIL")
return False
failed = False
if not check_metric("query_ttft_per_prompt",
"$lmcache_query_ttft", "$baseline_query_ttft",
float("$MAX_TTFT_SLOWDOWN_PCT")):
failed = True
if not check_metric("query_round_time_per_prompt",
"$lmcache_query_round_time", "$baseline_query_round_time",
float("$MAX_ROUND_TIME_SLOWDOWN_PCT")):
failed = True
if failed:
print("\nThreshold verification FAILED")
sys.exit(1)
else:
print("\nAll thresholds passed")
sys.exit(0)
EOF
}
# Run benchmark against baseline
echo "============================================"
echo "=== Benchmark: Baseline vLLM (without LMCache) ==="
echo "============================================"
run_long_doc_qa "$VLLM_BASELINE_PORT" "$LONG_DOC_QA_DIR/baseline_result.json" "baseline"
# Run benchmark against vLLM with LMCache
echo "============================================"
echo "=== Benchmark: vLLM with LMCache ==="
echo "============================================"
run_long_doc_qa "$VLLM_PORT" "$LONG_DOC_QA_DIR/lmcache_result.json" "lmcache"
# Compare
echo "============================================"
echo "=== Comparing Results ==="
echo "============================================"
if ! compare_results; then
echo "Comparison failed"
exit 1
fi
# Verify thresholds
echo "============================================"
echo "=== Verifying Performance Thresholds ==="
echo "============================================"
if ! verify_thresholds; then
echo "Threshold verification failed"
exit 1
fi
echo "============================================"
echo "=== Long Doc QA test completed ==="
echo "============================================"
.buildkite/k3_tests/multiprocess/scripts/run-mp-test.sh
Executable
+122
View File
@@ -0,0 +1,122 @@
#!/usr/bin/env bash
# Main orchestrator for multiprocessing tests (native, no Docker).
# Launches LMCache MP server + vLLM with LMCache + vLLM baseline,
# then runs workloads (lm_eval, vllm bench, long_doc_qa).
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
cd "${REPO_ROOT}"
source .buildkite/k3_tests/common_scripts/helpers.sh
# ── Configuration ────────────────────────────────────────────
export LMCACHE_PORT="${LMCACHE_PORT:-6555}"
export VLLM_PORT="${VLLM_PORT:-8000}"
export VLLM_BASELINE_PORT="${VLLM_BASELINE_PORT:-9000}"
export MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-300}"
export BUILD_ID="${BUILDKITE_BUILD_ID:-local_$$}"
export MODEL="${MODEL:-Qwen/Qwen3-14B}"
export CPU_BUFFER_SIZE="${CPU_BUFFER_SIZE:-80}"
export MAX_WORKERS="${MAX_WORKERS:-4}"
export LMCACHE_DIR="$REPO_ROOT"
export RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
mkdir -p "$RESULTS_DIR"
TEST_RESULT=0
# Cleanup: always kill background processes on exit
trap '"${SCRIPT_DIR}/cleanup.sh"' EXIT
echo "============================================"
echo "=== LMCache Multiprocessing Test ==="
echo "============================================"
echo "Build ID: $BUILD_ID"
echo "Model: $MODEL"
echo "LMCache port: $LMCACHE_PORT"
echo "vLLM port: $VLLM_PORT"
echo "vLLM baseline port: $VLLM_BASELINE_PORT"
echo "Results dir: $RESULTS_DIR"
echo ""
# Step 1: Launch native processes (replaces Docker containers)
echo "============================================"
echo "=== Step 1: Launching native processes ==="
echo "============================================"
if ! "${SCRIPT_DIR}/launch-processes.sh"; then
echo "Failed to launch processes"
TEST_RESULT=1
exit 1
fi
echo ""
# Step 2: Wait for vLLM to be ready
echo "============================================"
echo "=== Step 2: Waiting for vLLM to be ready ==="
echo "============================================"
if ! "${SCRIPT_DIR}/wait-for-servers.sh"; then
echo "vLLM failed to become ready"
TEST_RESULT=1
exit 1
fi
echo ""
# Step 3: Run lm_eval workload test
echo "============================================"
echo "=== Step 3: Running lm_eval workload ==="
echo "============================================"
if ! "${SCRIPT_DIR}/run-lm-eval.sh"; then
echo "lm_eval workload test failed"
TEST_RESULT=1
exit 1
fi
echo ""
# Step 4: Run vllm bench serve test
echo "============================================"
echo "=== Step 4: Running vllm bench serve ==="
echo "============================================"
if ! "${SCRIPT_DIR}/run-vllm-bench.sh"; then
echo "vllm bench serve test failed"
TEST_RESULT=1
exit 1
fi
echo ""
# Step 5: Run long doc QA test
echo "============================================"
echo "=== Step 5: Running long doc QA ==="
echo "============================================"
if ! "${SCRIPT_DIR}/run-long-doc-qa.sh"; then
echo "long doc QA test failed"
TEST_RESULT=1
exit 1
fi
echo ""
# Step 6: L2 long doc QA test (restarts LMCache with L2 config)
echo "============================================"
echo "=== Step 6: Running long doc QA (L2) ==="
echo "============================================"
if ! "${SCRIPT_DIR}/run-long-doc-qa-l2.sh"; then
echo "long doc QA L2 test failed"
TEST_RESULT=1
exit 1
fi
echo ""
# Step 7: Fault tolerance test (kills LMCache server -- must be last)
echo "============================================"
echo "=== Step 7: Running fault tolerance test ==="
echo "============================================"
if ! "${SCRIPT_DIR}/run-fault-tolerance.sh"; then
echo "fault tolerance test failed"
TEST_RESULT=1
exit 1
fi
echo ""
echo "============================================"
echo "=== All tests passed! ==="
echo "============================================"
.buildkite/k3_tests/multiprocess/scripts/run-p2p.sh
+247
View File
@@ -0,0 +1,247 @@
#!/usr/bin/env bash
# P2P end-to-end test: two LMCache MP instances + two vLLM instances + one
# coordinator, all in a single 2-GPU pod over localhost.
#
# Topology:
# coordinator : http://127.0.0.1:12300
# lmcache A (GPU 0) : zmq 6555, http 7555, p2p-advertise 127.0.0.1:8555
# lmcache B (GPU 1) : zmq 6556, http 7556, p2p-advertise 127.0.0.1:8556
# vLLM A (GPU 0) : port 8000, connector -> lmcache A (mp.port 6555)
# vLLM B (GPU 1) : port 8001, connector -> lmcache B (mp.port 6556)
#
# Flow:
# 1. Both lmcache servers register with the coordinator and discover each
# other; each creates a P2P L2 adapter for its peer.
# 2. Send a long prompt to vLLM A (cold) -> stored in lmcache A's L1.
# 3. Send the same prompt to vLLM B. B's L1 is empty and its only L2 adapter
# is the P2P adapter pointing at A, so any LMCache hit on B must have been
# fetched from A over P2P.
# 4. Assert B's num_lmcache_cached_tokens > 0.
#
# Self-contained: launches every process itself and records PIDs in the shared
# PID file so cleanup.sh tears them down on exit.
set -e
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
BUILD_ID="${BUILD_ID:-local_$$}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-300}"
CPU_BUFFER_SIZE="${CPU_BUFFER_SIZE:-80}"
MAX_WORKERS="${MAX_WORKERS:-4}"
# Ports (localhost, single pod).
COORDINATOR_PORT="${COORDINATOR_PORT:-12300}"
COORDINATOR_URL="http://127.0.0.1:${COORDINATOR_PORT}"
LMCACHE_A_PORT=6555; LMCACHE_A_HTTP=7555; LMCACHE_A_P2P="127.0.0.1:8555"
LMCACHE_B_PORT=6556; LMCACHE_B_HTTP=7556; LMCACHE_B_P2P="127.0.0.1:8556"
VLLM_A_PORT=8000
VLLM_B_PORT=8001
# Tell cleanup.sh which ports to free (PID-file kill covers the rest).
export VLLM_PORT="$VLLM_A_PORT"
export VLLM_BASELINE_PORT="$VLLM_B_PORT"
export LMCACHE_PORT="$LMCACHE_A_PORT"
# GPUs: K8s assigns devices 0 and 1.
GPU_A="${GPU_FOR_A:-0}"
GPU_B="${GPU_FOR_B:-1}"
P2P_DIR="$RESULTS_DIR/p2p"
mkdir -p "$P2P_DIR"
PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}"
> "$PID_FILE"
# vLLM GPU-memory fraction: clamp on very large GPUs so APC does not cover the
# whole prefix and hide the LMCache path (mirrors launch-processes.sh).
GPU_MEMORY_UTIL_ARG=""
GPU_MEMORY_MB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits -i "${GPU_A}" | tr -d ' ')
GPU_MEMORY_GB=$((GPU_MEMORY_MB / 1024))
echo "Detected GPU memory: ${GPU_MEMORY_GB}GB"
if [ -n "${GPU_MEMORY_UTILIZATION:-}" ]; then
GPU_MEMORY_UTIL_ARG="--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION}"
elif [ "$GPU_MEMORY_GB" -gt 90 ]; then
GPU_MEMORY_UTIL_ARG="--gpu-memory-utilization 0.5"
fi
CONNECTOR_CONFIG() {
# $1 = lmcache mp port
echo "{\"kv_connector\":\"LMCacheMPConnector\", \"kv_role\":\"kv_both\", \"kv_load_failure_policy\": \"recompute\", \"kv_connector_extra_config\": {\"lmcache.mp.port\": $1, \"lmcache.mp.mq_timeout\": 10}}"
}
launch_lmcache() {
# $1=gpu $2=zmq_port $3=http_port $4=p2p_advertise $5=instance_id $6=logname
CUDA_VISIBLE_DEVICES="$1" \
lmcache server \
--l1-size-gb "$CPU_BUFFER_SIZE" \
--eviction-policy LRU \
--max-workers "$MAX_WORKERS" \
--port "$2" \
--http-port "$3" \
--host 127.0.0.1 \
--instance-id "$5" \
--coordinator-url "$COORDINATOR_URL" \
--coordinator-advertise-ip 127.0.0.1 \
--p2p-advertise-url "$4" \
> "/tmp/build_${BUILD_ID}_${6}.log" 2>&1 &
local pid=$!
echo "$pid" >> "$PID_FILE"
echo "$pid"
}
launch_vllm() {
# $1=gpu $2=serving_port $3=lmcache_mp_port $4=logname
# Unset VLLM_PORT so vLLM's get_open_port() picks a random internal port
# for torch.distributed instead of colliding on serving_port+1.
env -u VLLM_PORT \
CUDA_VISIBLE_DEVICES="$1" \
VLLM_ENABLE_V1_MULTIPROCESSING=0 \
VLLM_SERVER_DEV_MODE=1 \
VLLM_BATCH_INVARIANT=1 \
PYTHONHASHSEED=0 \
vllm serve "$MODEL" \
--kv-transfer-config "$(CONNECTOR_CONFIG "$3")" \
--attention-backend FLASH_ATTN \
--port "$2" \
--no-async-scheduling \
--max-model-len auto \
$GPU_MEMORY_UTIL_ARG \
> "/tmp/build_${BUILD_ID}_${4}.log" 2>&1 &
local pid=$!
echo "$pid" >> "$PID_FILE"
echo "$pid"
}
# ── Step 1: coordinator ──────────────────────────────────────
echo "=== Launching coordinator on ${COORDINATOR_URL} ==="
lmcache coordinator --host 0.0.0.0 --port "$COORDINATOR_PORT" \
> "/tmp/build_${BUILD_ID}_coordinator.log" 2>&1 &
echo "$!" >> "$PID_FILE"
echo "Waiting for coordinator health..."
for _ in $(seq 1 30); do
if curl -sf "${COORDINATOR_URL}/healthz" > /dev/null 2>&1; then break; fi
sleep 1
done
# ── Step 2: two LMCache MP servers ───────────────────────────
echo "=== Launching LMCache servers ==="
launch_lmcache "$GPU_A" "$LMCACHE_A_PORT" "$LMCACHE_A_HTTP" "$LMCACHE_A_P2P" \
"p2p-a-${BUILD_ID}" "lmcache_a" > /dev/null
launch_lmcache "$GPU_B" "$LMCACHE_B_PORT" "$LMCACHE_B_HTTP" "$LMCACHE_B_P2P" \
"p2p-b-${BUILD_ID}" "lmcache_b" > /dev/null
echo "Waiting for LMCache servers to initialize..."
sleep 10
# ── Step 3: wait for P2P discovery (A must see B as a peer) ──
echo "=== Waiting for P2P peer discovery ==="
discovered=false
for _ in $(seq 1 "$MAX_WAIT_SECONDS"); do
peers=$(curl -sf "http://127.0.0.1:${LMCACHE_A_HTTP}/status" 2>/dev/null \
| python3 -c "import json,sys; print(json.load(sys.stdin).get('p2p_peer_count', 0))" \
2>/dev/null || echo 0)
if [ "${peers:-0}" -ge 1 ]; then
echo "lmcache A discovered ${peers} P2P peer(s)."
discovered=true
break
fi
sleep 2
done
if [ "$discovered" != "true" ]; then
echo "FAIL: lmcache A did not discover its P2P peer in time"
echo "--- lmcache A log (tail) ---"; tail -50 "/tmp/build_${BUILD_ID}_lmcache_a.log" || true
echo "--- lmcache B log (tail) ---"; tail -50 "/tmp/build_${BUILD_ID}_lmcache_b.log" || true
echo "--- coordinator instances ---"; curl -s "${COORDINATOR_URL}/instances" || true
exit 1
fi
# ── Step 4: two vLLM servers ─────────────────────────────────
echo "=== Launching vLLM servers ==="
launch_vllm "$GPU_A" "$VLLM_A_PORT" "$LMCACHE_A_PORT" "vllm_a" > /dev/null
launch_vllm "$GPU_B" "$VLLM_B_PORT" "$LMCACHE_B_PORT" "vllm_b" > /dev/null
wait_for_vllm() {
local port="$1" name="$2" logfile="$3"
echo "Waiting for $name on port $port..."
local deadline=$(( $(date +%s) + MAX_WAIT_SECONDS ))
while [ "$(date +%s)" -lt "$deadline" ]; do
if curl -sf "http://localhost:${port}/health" > /dev/null 2>&1 \
|| curl -sf "http://localhost:${port}/v1/models" > /dev/null 2>&1; then
echo "$name is ready."
return 0
fi
sleep 5
done
echo "FAIL: $name did not become ready in ${MAX_WAIT_SECONDS}s"
tail -100 "$logfile" 2>/dev/null || true
return 1
}
wait_for_vllm "$VLLM_A_PORT" "vLLM A" "/tmp/build_${BUILD_ID}_vllm_a.log"
wait_for_vllm "$VLLM_B_PORT" "vLLM B" "/tmp/build_${BUILD_ID}_vllm_b.log"
# ── Step 5: long prompt to A (cold), then B (P2P hit) ────────
LONG_CONTENT="Explain the history of computer science in great detail. $(printf 'The Turing machine is a fundamental concept in theoretical computer science that defines an abstract machine capable of manipulating symbols on a strip of tape according to a table of rules. %.0s' {1..20})"
send_request() {
# $1=port $2=label $3=output_file
local http_code
http_code=$(curl -s -o "$3" -w "%{http_code}" \
-X POST "http://localhost:${1}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "{
\"model\": \"${MODEL}\",
\"messages\": [{\"role\": \"user\", \"content\": $(python3 -c "import json;print(json.dumps('$LONG_CONTENT'))")}],
\"max_tokens\": 1,
\"kv_transfer_params\": {\"cached_token_stats\": true}
}")
if [ "$http_code" -ne 200 ]; then
echo "FAIL: $2 returned HTTP $http_code"; cat "$3"; return 1
fi
echo "$2: HTTP 200 OK"
}
echo "=== Step 5a: cold request to vLLM A (populates lmcache A) ==="
send_request "$VLLM_A_PORT" "engine-A-cold" "$P2P_DIR/a_cold.json"
# Let A's store complete and become lockable before B looks it up over P2P.
sleep 8
echo "=== Step 5b: same request to vLLM B (should hit A over P2P) ==="
send_request "$VLLM_B_PORT" "engine-B-p2p" "$P2P_DIR/b_p2p.json"
# ── Step 6: assert B got a P2P cache hit ─────────────────────
echo "=== Step 6: verify P2P cache hit on engine B ==="
python3 -c "
import json, sys
with open('$P2P_DIR/b_p2p.json') as f:
data = json.load(f)
stats = (data.get('kv_transfer_params') or {}).get('cached_token_stats')
if stats is None:
print('FAIL: cached_token_stats missing from engine B response')
print(data)
sys.exit(1)
lmcache_hits = stats['num_lmcache_cached_tokens']
vllm_hits = stats['num_vllm_cached_tokens']
print(f'engine B num_vllm_cached_tokens: {vllm_hits}')
print(f'engine B num_lmcache_cached_tokens: {lmcache_hits}')
# B never saw this prompt and its only L2 adapter is the P2P adapter to A, so a
# positive LMCache hit can only have come from A over P2P.
if lmcache_hits <= 0:
print('FAIL: engine B had no LMCache hits; P2P fetch did not happen')
sys.exit(1)
print(f'PASS: engine B served {lmcache_hits} tokens from peer A over P2P')
"
echo "============================================"
echo "=== P2P test PASSED ==="
echo "============================================"
.buildkite/k3_tests/multiprocess/scripts/run-restart-recovery.sh
Executable
+287
View File
@@ -0,0 +1,287 @@
#!/usr/bin/env bash
# Test LMCache server restart recovery: vLLM workers should re-register
# their KV caches with the new LMCache server (driven by the heartbeat
# thread's recover callback) and resume successful stores.
#
# Flow:
# 1. Run a `lmcache bench engine --workload random-prefill` round.
# 2. Snapshot lmcache_mp_l1_write_chunks_total via /metrics.
# 3. Kill the LMCache server, relaunch on the same port.
# 4. Wait for the new server to be ready and for the worker to
# re-register (poll /status until cache_context_meta is non-empty).
# 5. Run the same bench round again.
# 6. Snapshot the metric again.
# 7. Assert run2 > 0 and run2 >= 0.8 * run1.
#
# The metric counter is reset to 0 by the new server process, so
# run2 is the absolute count for the post-restart benchmark only.
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# ── Configuration (inherited from run-single-test.sh) ─────────
VLLM_PORT="${VLLM_PORT:-8000}"
LMCACHE_PORT="${LMCACHE_PORT:-6555}"
LMCACHE_HTTP_PORT="${LMCACHE_HTTP_PORT:-8080}"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
BUILD_ID="${BUILD_ID:-local_$$}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
CPU_BUFFER_SIZE="${CPU_BUFFER_SIZE:-80}"
MAX_WORKERS="${MAX_WORKERS:-4}"
# Bench parameters — small enough for CI but big enough to register
# a non-trivial l1_write_keys count per round.
NUM_REQUESTS="${RR_NUM_REQUESTS:-100}"
REQUEST_LEN="${RR_REQUEST_LEN:-5000}"
KV_CACHE_VOLUME="${RR_KV_CACHE_VOLUME:-5}"
# Recovery timing
RECOVER_TIMEOUT="${RR_RECOVER_TIMEOUT:-150}"
# Output
OUT_DIR="$RESULTS_DIR/restart_recovery"
mkdir -p "$OUT_DIR"
PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}"
echo "=== Restart Recovery Test ==="
echo "Model: $MODEL"
echo "vLLM URL: http://localhost:${VLLM_PORT}"
echo "LMCache HTTP URL: http://localhost:${LMCACHE_HTTP_PORT}"
echo "Bench: ${NUM_REQUESTS} requests x ${REQUEST_LEN} tokens"
echo "Recovery timeout: ${RECOVER_TIMEOUT}s"
echo ""
# ── Helpers ──────────────────────────────────────────────────
run_bench_round() {
local label="$1"
local seed=$2
local out_subdir="$OUT_DIR/$label"
mkdir -p "$out_subdir"
echo "--- bench round: $label ---"
if ! lmcache bench engine \
--engine-url "http://localhost:${VLLM_PORT}" \
--lmcache-url "http://localhost:${LMCACHE_HTTP_PORT}" \
--workload random-prefill \
--rp-num-requests "$NUM_REQUESTS" \
--rp-request-length "$REQUEST_LEN" \
--kv-cache-volume "$KV_CACHE_VOLUME" \
--no-interactive \
--no-csv \
--json \
--quiet \
--seed $seed \
--output-dir "$out_subdir" \
2>&1 | tee "$out_subdir/bench.log"; then
echo "FAIL: bench round '$label' returned non-zero (failed requests)"
return 1
fi
# Quick sanity from JSON summary
if [ -f "$out_subdir/bench_summary.json" ]; then
local successful failed
successful=$(python3 -c "import json; print(json.load(open('$out_subdir/bench_summary.json')).get('successful_requests', 0))")
failed=$(python3 -c "import json; print(json.load(open('$out_subdir/bench_summary.json')).get('failed_requests', 0))")
echo "$label: successful=$successful failed=$failed"
if [ "$failed" -ne 0 ]; then
echo "FAIL: $failed requests failed in '$label'"
return 1
fi
fi
return 0
}
scrape_l1_write_keys() {
# Print the latest lmcache_mp_l1_write_chunks_total value (single sample,
# unlabeled) from the LMCache HTTP server's /metrics endpoint.
python3 - <<EOF
import sys, urllib.request
url = "http://localhost:${LMCACHE_HTTP_PORT}/metrics"
try:
body = urllib.request.urlopen(url, timeout=10).read().decode()
except Exception as e:
print(f"ERROR fetching {url}: {e}", file=sys.stderr)
sys.exit(1)
total = None
for line in body.splitlines():
if line.startswith("#"):
continue
if not line.startswith("lmcache_mp_l1_write_chunks_total"):
continue
parts = line.rsplit(" ", 1)
if len(parts) != 2:
continue
try:
val = float(parts[1])
except ValueError:
continue
total = (total or 0.0) + val
if total is None:
print(f"ERROR: lmcache_mp_l1_write_chunks_total not found at {url}", file=sys.stderr)
sys.exit(1)
print(int(total))
EOF
}
wait_for_lmcache_http() {
local deadline=$(( $(date +%s) + 60 ))
while [ "$(date +%s)" -lt "$deadline" ]; do
if curl -sf "http://localhost:${LMCACHE_HTTP_PORT}/healthcheck" > /dev/null 2>&1; then
echo "LMCache HTTP healthy"
return 0
fi
sleep 2
done
echo "FAIL: LMCache HTTP did not come back within 60s"
return 1
}
wait_for_worker_reregister() {
# Poll /status until cache_context_meta has at least one entry,
# which proves the vLLM worker re-registered with the new server.
local deadline=$(( $(date +%s) + RECOVER_TIMEOUT ))
while [ "$(date +%s)" -lt "$deadline" ]; do
local count
count=$(python3 - <<EOF 2>/dev/null
import json, urllib.request
try:
body = urllib.request.urlopen("http://localhost:${LMCACHE_HTTP_PORT}/status", timeout=5).read()
data = json.loads(body)
print(len(data.get("cache_context_meta", {})))
except Exception:
print(0)
EOF
)
if [ "$count" -gt 0 ]; then
echo "Worker re-registered (cache_context_meta entries: $count)"
return 0
fi
sleep 2
done
echo "FAIL: worker did not re-register within ${RECOVER_TIMEOUT}s"
return 1
}
restart_lmcache() {
local old_pid
old_pid=$(sed -n '1p' "$PID_FILE" 2>/dev/null || true)
if [ -z "$old_pid" ]; then
echo "FAIL: no LMCache PID in $PID_FILE"
return 1
fi
echo "Killing LMCache PID $old_pid..."
if kill -0 "$old_pid" 2>/dev/null; then
kill "$old_pid" 2>/dev/null || true
wait "$old_pid" 2>/dev/null || true
fi
sleep 60
echo "Relaunching LMCache on port ${LMCACHE_PORT} / HTTP ${LMCACHE_HTTP_PORT}..."
lmcache server \
--l1-size-gb "$CPU_BUFFER_SIZE" \
--eviction-policy LRU \
--max-workers "$MAX_WORKERS" \
--port "$LMCACHE_PORT" \
--http-port "$LMCACHE_HTTP_PORT" \
> "/tmp/build_${BUILD_ID}_lmcache_restart.log" 2>&1 &
local new_pid=$!
echo "$new_pid" > /tmp/.new_lmcache_pid
# Replace line 1 of the PID file so cleanup.sh kills the new process.
sed -i "1s/.*/$new_pid/" "$PID_FILE"
echo "LMCache restarted (PID=$new_pid)"
return 0
}
# ── Step 1: Bench round 1 ────────────────────────────────────
echo "============================================"
echo "=== Round 1: bench against original server ==="
echo "============================================"
if ! run_bench_round "round1" "41"; then
echo "FAIL: round 1 bench failed"
exit 1
fi
ROUND1_WRITES=$(scrape_l1_write_keys) || {
echo "FAIL: could not scrape /metrics after round 1"
exit 1
}
echo "Round 1 lmcache_mp_l1_write_chunks_total = $ROUND1_WRITES"
if [ "$ROUND1_WRITES" -le 0 ]; then
echo "FAIL: round 1 produced no L1 writes; benchmark setup is broken"
exit 1
fi
# ── Step 2: Kill + relaunch LMCache ──────────────────────────
echo ""
echo "============================================"
echo "=== Restarting LMCache server ==="
echo "============================================"
if ! restart_lmcache; then
exit 1
fi
if ! wait_for_lmcache_http; then
echo "--- new lmcache log (last 80 lines) ---"
tail -80 "/tmp/build_${BUILD_ID}_lmcache_restart.log" 2>/dev/null || true
exit 1
fi
if ! wait_for_worker_reregister; then
echo "--- new lmcache log (last 80 lines) ---"
tail -80 "/tmp/build_${BUILD_ID}_lmcache_restart.log" 2>/dev/null || true
echo "--- vllm log (last 80 lines) ---"
tail -80 "/tmp/build_${BUILD_ID}_vllm.log" 2>/dev/null || true
exit 1
fi
# ── Step 3: Bench round 2 ────────────────────────────────────
echo ""
echo "============================================"
echo "=== Round 2: bench against restarted server ==="
echo "============================================"
if ! run_bench_round "round2" "42"; then
echo "FAIL: round 2 bench failed"
echo "--- vllm log (last 80 lines) ---"
tail -80 "/tmp/build_${BUILD_ID}_vllm.log" 2>/dev/null || true
exit 1
fi
ROUND2_WRITES=$(scrape_l1_write_keys) || {
echo "FAIL: could not scrape /metrics after round 2"
exit 1
}
echo "Round 2 lmcache_mp_l1_write_chunks_total = $ROUND2_WRITES"
# ── Step 4: Compare metrics ──────────────────────────────────
echo ""
echo "============================================"
echo "=== Verification ==="
echo "============================================"
echo "Round 1 writes: $ROUND1_WRITES"
echo "Round 2 writes: $ROUND2_WRITES"
if [ "$ROUND2_WRITES" -le 0 ]; then
echo "FAIL: round 2 produced no L1 writes — re-registration likely failed"
exit 1
fi
# Floor-divide: round2 >= 0.8 * round1 <=> round2 * 10 >= round1 * 8
if [ $((ROUND2_WRITES * 10)) -lt $((ROUND1_WRITES * 8)) ]; then
pct=$(python3 -c "print(f'{$ROUND2_WRITES / $ROUND1_WRITES * 100:.1f}')")
echo "FAIL: round 2 only ${pct}% of round 1 writes (threshold: 80%)"
exit 1
fi
pct=$(python3 -c "print(f'{$ROUND2_WRITES / $ROUND1_WRITES * 100:.1f}')")
echo "PASS: round 2 = ${pct}% of round 1 (>= 80%)"
echo ""
echo "============================================"
echo "=== Restart Recovery Test PASSED ==="
echo "============================================"
.buildkite/k3_tests/multiprocess/scripts/run-single-test.sh
Executable
+181
View File
@@ -0,0 +1,181 @@
#!/usr/bin/env bash
# Orchestrator for a single multiprocessing test (native, no Docker).
# Usage: run-single-test.sh <test_name>
# test_name: lm_eval | hma_lm_eval_gemma4 | vllm_bench | long_doc_qa
# | long_doc_qa_l2 | fault_tolerance | deadlock | restart_recovery
# | gds_smoke_test
#
# Each invocation is self-contained: launches servers, runs one test, cleans up.
# This mirrors the comprehensive tests' run-single-config.sh pattern.
set -o pipefail
TEST_NAME="${1:?Usage: $0 <test_name>}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
cd "${REPO_ROOT}"
source .buildkite/k3_tests/common_scripts/helpers.sh
# ── Configuration ────────────────────────────────────────────
export LMCACHE_PORT="${LMCACHE_PORT:-6555}"
export VLLM_PORT="${VLLM_PORT:-8000}"
export VLLM_BASELINE_PORT="${VLLM_BASELINE_PORT:-9000}"
export MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-300}"
export BUILD_ID="${BUILDKITE_BUILD_ID:-local_$$}"
# gds_smoke_test enables the GDS L1 NVMe-slab tier
GDS_SCRATCH="${GDS_SCRATCH:-/scratch}"
if [ "$TEST_NAME" = "gds_smoke_test" ]; then
export GDS_L1_PATH="${GDS_SCRATCH}/lmcache-gds-${BUILD_ID}-${TEST_NAME}"
echo "GDS L1 tier enabled (slab dir: $GDS_L1_PATH)"
fi
# Per-test default model (overridable via the MODEL env var). The HMA test needs
# a hybrid model whose KV cache groups have different block sizes, so the
# connector exercises the per-group hybrid-memory-allocator path.
if [ "$TEST_NAME" = "hma_lm_eval_gemma4" ]; then
# gemma-4-31B-it is public (no gating, so no HF token check) and has
# heterogeneous head dims (head_dim 256 / global_head_dim 512), so vLLM
# gives its KV cache groups different block sizes -- this is what exercises
# LMCache's per-group block-size handling. It forces TRITON_ATTN, so the
# pipeline sets ATTENTION_BACKEND=auto; its ~63GB of weights also need a
# higher GPU_MEMORY_UTILIZATION than the default (all set in pipeline.yml).
export MODEL="${MODEL:-google/gemma-4-31B-it}"
elif [ "$TEST_NAME" = "hma_lm_eval_qwen3_5" ]; then
# Qwen3.5-0.8B is a Mamba/GDN + full-attention hybrid (caches re-viewed at
# registration; see lmcache/integration/vllm/kv_cache_group_edits.py).
export MODEL="${MODEL:-Qwen/Qwen3.5-0.8B}"
export ATTENTION_BACKEND="${ATTENTION_BACKEND:-auto}"
# LMCache chunk size must be a multiple of the unified vLLM block size (544).
export CHUNK_SIZE="${CHUNK_SIZE:-544}"
# GDN supports only the 'align' Mamba cache mode.
export MAMBA_CACHE_MODE="${MAMBA_CACHE_MODE:-align}"
# 'align' snapshots the Mamba state only at scheduler-step boundaries; cap
# the step at the chunk size for one reusable snapshot per chunk.
export MAX_NUM_BATCHED_TOKENS="${MAX_NUM_BATCHED_TOKENS:-544}"
# GDN has no batch-invariant mode, so runs are not bit-exact; compare within
# a score tolerance and use enough samples to shrink run-to-run drift
# (~1/sqrt(LIMIT)) well inside it.
export BATCH_INVARIANT="${BATCH_INVARIANT:-0}"
export SCORE_TOLERANCE="${SCORE_TOLERANCE:-0.05}"
export LIMIT="${LIMIT:-300}"
else
export MODEL="${MODEL:-Qwen/Qwen3-14B}"
fi
export CPU_BUFFER_SIZE="${CPU_BUFFER_SIZE:-80}"
export MAX_WORKERS="${MAX_WORKERS:-4}"
export LMCACHE_DIR="$REPO_ROOT"
export RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
mkdir -p "$RESULTS_DIR"
# Cleanup: always kill background processes on exit
trap '"${SCRIPT_DIR}/cleanup.sh"' EXIT
echo "============================================"
echo "=== LMCache Multiprocessing Test: ${TEST_NAME} ==="
echo "============================================"
echo "Build ID: $BUILD_ID"
echo "Model: $MODEL"
echo "LMCache port: $LMCACHE_PORT"
echo "vLLM port: $VLLM_PORT"
echo "vLLM baseline port: $VLLM_BASELINE_PORT"
echo "Results dir: $RESULTS_DIR"
echo ""
# Tests that handle their own server lifecycle (different GPU/model config)
SELF_CONTAINED_TESTS=" deadlock p2p "
# Tests that compare against a baseline vLLM (no LMCache) on a second GPU.
# Only these need the baseline server (and thus a 2-GPU pod); everything
# else runs on GPU 0 alone, so launch-processes.sh skips the baseline.
BASELINE_TESTS=" vllm_bench long_doc_qa long_doc_qa_l2 "
if [[ "$BASELINE_TESTS" == *" $TEST_NAME "* ]]; then
export LAUNCH_BASELINE=true
else
export LAUNCH_BASELINE=false
fi
if [[ "$SELF_CONTAINED_TESTS" != *" $TEST_NAME "* ]]; then
# ── Step 1: Launch native processes ──────────────────────────
echo "============================================"
echo "=== Launching native processes ==="
echo "============================================"
if ! "${SCRIPT_DIR}/launch-processes.sh"; then
echo "Failed to launch processes"
exit 1
fi
echo ""
# ── Step 2: Wait for vLLM to be ready ───────────────────────
echo "============================================"
echo "=== Waiting for vLLM to be ready ==="
echo "============================================"
if ! "${SCRIPT_DIR}/wait-for-servers.sh"; then
echo "vLLM failed to become ready"
exit 1
fi
echo ""
fi
# ── Step 3: Run the requested test ──────────────────────────
echo "============================================"
echo "=== Running test: ${TEST_NAME} ==="
echo "============================================"
case "$TEST_NAME" in
lm_eval)
exec_script="${SCRIPT_DIR}/run-lm-eval.sh"
;;
hma_lm_eval_gemma4)
exec_script="${SCRIPT_DIR}/run-hma-lm-eval.sh"
;;
hma_lm_eval_qwen3_5)
exec_script="${SCRIPT_DIR}/run-hma-lm-eval.sh"
;;
vllm_bench)
exec_script="${SCRIPT_DIR}/run-vllm-bench.sh"
;;
long_doc_qa)
exec_script="${SCRIPT_DIR}/run-long-doc-qa.sh"
;;
long_doc_qa_l2)
exec_script="${SCRIPT_DIR}/run-long-doc-qa-l2.sh"
;;
fault_tolerance)
exec_script="${SCRIPT_DIR}/run-fault-tolerance.sh"
;;
deadlock)
exec_script="${SCRIPT_DIR}/run-deadlock.sh"
;;
restart_recovery)
exec_script="${SCRIPT_DIR}/run-restart-recovery.sh"
;;
cache_stats)
exec_script="${SCRIPT_DIR}/run-cache-stats.sh"
;;
p2p)
exec_script="${SCRIPT_DIR}/run-p2p.sh"
;;
http_api)
exec_script="${SCRIPT_DIR}/run-http-api.sh"
;;
gds_smoke_test)
exec_script="${SCRIPT_DIR}/run-gds-smoke.sh"
;;
*)
echo "Unknown test: $TEST_NAME"
echo "Valid tests: lm_eval, hma_lm_eval_gemma4, vllm_bench, long_doc_qa, long_doc_qa_l2, fault_tolerance, deadlock, restart_recovery, cache_stats, http_api, gds_smoke_test, p2p"
exit 1
;;
esac
if ! "$exec_script"; then
echo "${TEST_NAME} test failed"
exit 1
fi
echo ""
echo "============================================"
echo "=== Test ${TEST_NAME} passed! ==="
echo "============================================"
.buildkite/k3_tests/multiprocess/scripts/run-vllm-bench.sh
Executable
+276
View File
@@ -0,0 +1,276 @@
#!/usr/bin/env bash
# Run vllm bench serve test against both vLLM servers.
# Compares performance between LMCache-enabled and baseline vLLM.
# Adapted from the old Docker-based run-vllm-bench.sh.
set -e
set -o pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
# Configuration
VLLM_PORT="${VLLM_PORT:-8000}"
VLLM_BASELINE_PORT="${VLLM_BASELINE_PORT:-9000}"
MODEL="${MODEL:-Qwen/Qwen3-14B}"
NUM_PROMPTS="${NUM_PROMPTS:-50}"
RANDOM_INPUT_LEN="${RANDOM_INPUT_LEN:-10000}"
RANDOM_OUTPUT_LEN="${RANDOM_OUTPUT_LEN:-1}"
BUILD_ID="${BUILD_ID:-local_$$}"
RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}"
# Expected values
EXPECTED_TOTAL_INPUT_TOKENS=$((NUM_PROMPTS * RANDOM_INPUT_LEN))
EXPECTED_COMPLETED=$NUM_PROMPTS
MAX_SLOWDOWN_PERCENT=5
# Reproducible seed
RANDOM_SEED="${RANDOM_SEED:-$(date +%s)}"
# Output directory
VLLM_BENCH_DIR="$RESULTS_DIR/vllm_bench"
echo "=== vLLM Bench Serve Test ==="
echo "Model: $MODEL"
echo "vLLM Port (with LMCache): $VLLM_PORT"
echo "vLLM Baseline Port (without LMCache): $VLLM_BASELINE_PORT"
echo "Number of prompts: $NUM_PROMPTS"
echo "Random input length: $RANDOM_INPUT_LEN"
echo "Random output length: $RANDOM_OUTPUT_LEN"
echo "Results dir: $VLLM_BENCH_DIR"
echo ""
mkdir -p "$VLLM_BENCH_DIR"
run_vllm_bench() {
local port="$1"
local result_filename="$2"
local description="$3"
local seed="$4"
echo "=== Running vllm bench serve ($description) ==="
echo "Port: $port, Seed: $seed"
vllm bench serve \
--seed "$seed" \
--port "$port" \
--model "$MODEL" \
--dataset-name random \
--random-input-len "$RANDOM_INPUT_LEN" \
--random-output-len "$RANDOM_OUTPUT_LEN" \
--num-prompts "$NUM_PROMPTS" \
--ignore-eos \
--backend openai-chat \
--endpoint /v1/chat/completions \
--result-dir "$VLLM_BENCH_DIR" \
--result-filename "$result_filename" \
--save-result
echo "$description benchmark completed"
echo ""
}
extract_json_field() {
local json_file="$1"
local field="$2"
python3 -c "
import json
with open('$json_file', 'r') as f:
data = json.load(f)
print(data.get('$field', 'null'))
"
}
verify_results() {
local lmcache_result="$VLLM_BENCH_DIR/lmcache.json"
local baseline_result="$VLLM_BENCH_DIR/baseline.json"
echo "=== Verifying benchmark results ==="
if [ ! -f "$lmcache_result" ]; then
echo "LMCache result file not found: $lmcache_result"
return 1
fi
if [ ! -f "$baseline_result" ]; then
echo "Baseline result file not found: $baseline_result"
return 1
fi
# Extract values
lmcache_total_input_tokens=$(extract_json_field "$lmcache_result" "total_input_tokens")
lmcache_completed=$(extract_json_field "$lmcache_result" "completed")
lmcache_throughput=$(extract_json_field "$lmcache_result" "total_token_throughput")
baseline_total_input_tokens=$(extract_json_field "$baseline_result" "total_input_tokens")
baseline_completed=$(extract_json_field "$baseline_result" "completed")
baseline_throughput=$(extract_json_field "$baseline_result" "total_token_throughput")
echo "=== LMCache Results ==="
echo " total_input_tokens: $lmcache_total_input_tokens"
echo " completed: $lmcache_completed"
echo " total_token_throughput: $lmcache_throughput"
echo ""
echo "=== Baseline Results ==="
echo " total_input_tokens: $baseline_total_input_tokens"
echo " completed: $baseline_completed"
echo " total_token_throughput: $baseline_throughput"
echo ""
local failed=0
echo "=== Verification ==="
# vLLM's random dataset decodes and re-encodes token sequences, which can
# drift slightly from the requested length (see RandomDataset in
# vllm/benchmarks/datasets.py). Allow 1% tolerance.
local token_tolerance=$((EXPECTED_TOTAL_INPUT_TOKENS / 100))
check_input_tokens() {
local label="$1"
local actual="$2"
local diff=$((actual - EXPECTED_TOTAL_INPUT_TOKENS))
local abs_diff=${diff#-}
if [ "$abs_diff" -le "$token_tolerance" ] 2>/dev/null; then
echo "$label total_input_tokens: $actual (expected: $EXPECTED_TOTAL_INPUT_TOKENS ±$token_tolerance) PASS"
else
echo "$label total_input_tokens: $actual (expected: $EXPECTED_TOTAL_INPUT_TOKENS ±$token_tolerance) FAIL"
failed=1
fi
}
check_input_tokens "LMCache" "$lmcache_total_input_tokens"
check_input_tokens "Baseline" "$baseline_total_input_tokens"
if [ "$lmcache_completed" -eq "$EXPECTED_COMPLETED" ] 2>/dev/null; then
echo "LMCache completed: $lmcache_completed (expected: $EXPECTED_COMPLETED) PASS"
else
echo "LMCache completed: $lmcache_completed (expected: $EXPECTED_COMPLETED) FAIL"
failed=1
fi
if [ "$baseline_completed" -eq "$EXPECTED_COMPLETED" ] 2>/dev/null; then
echo "Baseline completed: $baseline_completed (expected: $EXPECTED_COMPLETED) PASS"
else
echo "Baseline completed: $baseline_completed (expected: $EXPECTED_COMPLETED) FAIL"
failed=1
fi
# Throughput comparison
throughput_check=$(python3 -c "
lmcache_tp = $lmcache_throughput
baseline_tp = $baseline_throughput
max_slowdown = $MAX_SLOWDOWN_PERCENT
min_acceptable = baseline_tp * (1 - max_slowdown / 100.0)
if baseline_tp > 0:
slowdown_pct = ((baseline_tp - lmcache_tp) / baseline_tp) * 100
else:
slowdown_pct = 0
if lmcache_tp >= min_acceptable:
print(f'PASS|{slowdown_pct:.2f}')
else:
print(f'FAIL|{slowdown_pct:.2f}')
")
throughput_status=$(echo "$throughput_check" | cut -d'|' -f1)
slowdown_pct=$(echo "$throughput_check" | cut -d'|' -f2)
if [ "$throughput_status" = "PASS" ]; then
echo "Throughput: LMCache is ${slowdown_pct}% slower (max allowed: ${MAX_SLOWDOWN_PERCENT}%) PASS"
else
echo "Throughput: LMCache is ${slowdown_pct}% slower (max allowed: ${MAX_SLOWDOWN_PERCENT}%) FAIL"
failed=1
fi
# Sanity check: on a random (no-reuse) workload, LMCache should NOT be
# significantly faster than baseline. If it is, the benchmark setup is
# asymmetric and the results are unreliable as a regression test.
local max_speedup_pct=10
speedup_check=$(python3 -c "
lmcache_tp = $lmcache_throughput
baseline_tp = $baseline_throughput
if baseline_tp > 0:
speedup_pct = ((lmcache_tp - baseline_tp) / baseline_tp) * 100
else:
speedup_pct = 0
if speedup_pct > $max_speedup_pct:
print(f'WARN|{speedup_pct:.2f}')
else:
print(f'OK|{speedup_pct:.2f}')
")
local speedup_status speedup_pct
speedup_status=$(echo "$speedup_check" | cut -d'|' -f1)
speedup_pct=$(echo "$speedup_check" | cut -d'|' -f2)
if [ "$speedup_status" = "WARN" ]; then
echo "WARNING: LMCache is ${speedup_pct}% faster than baseline on random workload (max expected: ${max_speedup_pct}%)"
echo "This suggests a measurement asymmetry, not a real cache benefit."
failed=1
else
echo "Speedup sanity check: LMCache is ${speedup_pct}% faster (max expected: ${max_speedup_pct}%) OK"
fi
echo ""
return "$failed"
}
warmup_server() {
local port="$1"
local description="$2"
local num_warmup="${3:-3}"
echo "=== Warming up $description (port $port) ==="
# Send a few chat completion requests to warm up the tokenizer,
# chat template (Jinja2), and engine pipeline. Without this, the
# first-ever batch of requests incurs ~25s of cold-start overhead
# (BPE compilation, template compilation, etc.) which skews the
# benchmark since lm-eval (Step 3) only warms the LMCache server.
for i in $(seq 1 "$num_warmup"); do
curl -s -X POST "http://localhost:${port}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "{
\"model\": \"${MODEL}\",
\"messages\": [{\"role\": \"user\", \"content\": \"Warmup request ${i}. The quick brown fox jumps over the lazy dog.\"}],
\"max_tokens\": 1
}" > /dev/null 2>&1
done
echo "$description warmup complete"
}
echo "Using random seed: $RANDOM_SEED"
echo ""
# Warm up both servers so tokenizer/template compilation does not
# skew the benchmark. See THROUGHPUT_FIX_PROPOSAL.md for details.
echo "============================================"
echo "=== Warming up both servers ==="
echo "============================================"
warmup_server "$VLLM_PORT" "vLLM with LMCache"
warmup_server "$VLLM_BASELINE_PORT" "vLLM baseline"
echo ""
# Baseline first
echo "============================================"
echo "=== Benchmark: Baseline vLLM (without LMCache) ==="
echo "============================================"
run_vllm_bench "$VLLM_BASELINE_PORT" "baseline.json" "Baseline vLLM" "$RANDOM_SEED"
# LMCache
echo "============================================"
echo "=== Benchmark: vLLM with LMCache ==="
echo "============================================"
run_vllm_bench "$VLLM_PORT" "lmcache.json" "vLLM with LMCache" "$RANDOM_SEED"
# Verify
echo "============================================"
echo "=== Verifying benchmark results ==="
echo "============================================"
if ! verify_results; then
echo "Verification failed"
exit 1
fi
echo "============================================"
echo "=== vLLM Bench test completed ==="
echo "============================================"
.buildkite/k3_tests/multiprocess/scripts/wait-for-servers.sh
Executable
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env bash
# Wait for vLLM servers to be ready (native processes, no Docker).
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)"
source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh"
VLLM_PORT="${VLLM_PORT:-8000}"
VLLM_BASELINE_PORT="${VLLM_BASELINE_PORT:-9000}"
MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-600}"
BUILD_ID="${BUILD_ID:-local_$$}"
# Wait for a vLLM server with health check
wait_for_vllm_server() {
local port="$1"
local description="$2"
local logfile="$3"
echo "=== Waiting for $description to be ready ==="
echo "Port: $port, Max wait: ${MAX_WAIT_SECONDS}s"
local start_time end_time
start_time=$(date +%s)
end_time=$((start_time + MAX_WAIT_SECONDS))
while true; do
local current_time elapsed
current_time=$(date +%s)
elapsed=$((current_time - start_time))
if [ "$current_time" -ge "$end_time" ]; then
echo "Timeout: $description did not become ready within ${MAX_WAIT_SECONDS}s"
echo ""
echo "=== $description log (last 100 lines) ==="
tail -100 "$logfile" 2>/dev/null || true
return 1
fi
if curl -sf "http://localhost:${port}/health" > /dev/null 2>&1; then
echo "$description is ready! (took ${elapsed}s)"
return 0
fi
if curl -sf "http://localhost:${port}/v1/models" > /dev/null 2>&1; then
echo "$description is ready! (took ${elapsed}s)"
return 0
fi
echo "Waiting for $description... (${elapsed}s elapsed)"
sleep 5
done
}
# Wait for both servers (they start simultaneously)
if ! wait_for_vllm_server "$VLLM_PORT" "vLLM with LMCache" \
"/tmp/build_${BUILD_ID}_vllm.log"; then
exit 1
fi
# The baseline server only exists for 2-GPU tests; 1-GPU tests set
# LAUNCH_BASELINE=false in launch-processes.sh and never start it.
if [[ "${LAUNCH_BASELINE:-true}" == "true" ]]; then
if ! wait_for_vllm_server "$VLLM_BASELINE_PORT" "vLLM baseline (without LMCache)" \
"/tmp/build_${BUILD_ID}_vllm_baseline.log"; then
exit 1
fi
fi
echo ""
echo "=== All vLLM servers are ready ==="
.buildkite/k3_tests/sglang/BK_WEB_SETUP.md
+16
View File
@@ -0,0 +1,16 @@
# Buildkite Web UI Setup: SGLang + LMCache MP
**Steps editor**: paste contents of `buildkite-pipeline.yml` (fill in `HF_TOKEN`).
**GitHub trigger settings**:
- Filter (the SGLang MP adapter's real dependency surface spans most of `lmcache/`):
- `lmcache/**`
- `.buildkite/k3_tests/sglang/**`
- Skip queued / cancel running branch builds: Yes
Two GPU jobs (correctness + performance), ~5 min each — lightweight enough for a required PR status check on changes that touch the MP integration.
> Builds whose only changes are docs/`*.md`/`LICENSE`/`.github/**` auto-pass
> via the [path filter](../README.md#path-based-skip-auto-pass-on-docs-only-changes).
> Changes under `.buildkite/` always run. Add the `force-ci` label to a PR to
> bypass.
.buildkite/k3_tests/sglang/buildkite-pipeline.yml
+12
View File
@@ -0,0 +1,12 @@
# Paste this into the Buildkite pipeline's "Steps" editor for the
# SGLang + LMCache MP integration test. Uploads the real pipeline
# definition from the repo.
agents:
queue: "k8s"
env:
HF_TOKEN: "" # set your HuggingFace token here (needed for gated model access)
steps:
- label: ":pipeline: Upload pipeline"
command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/sglang/pipeline.yml
.buildkite/k3_tests/sglang/pipeline.yml
+54
View File
@@ -0,0 +1,54 @@
# SGLang + LMCache MP integration tests.
# 1) Correctness: LMCache does not change output, and is actually exercised.
# 2) Performance: LMCache makes TTFT smaller.
steps:
- label: ":test_tube: SGLang+LMCache — Correctness"
command: .buildkite/k3_tests/sglang/run.sh correctness
timeout_in_minutes: 15
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources:
limits:
nvidia.com/gpu: "1"
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
artifact_paths:
- "*.log"
- "*.json"
- label: ":zap: SGLang+LMCache — Performance"
command: .buildkite/k3_tests/sglang/run.sh perf
timeout_in_minutes: 15
retry:
automatic:
- exit_status: -1
limit: 2
- exit_status: 1
limit: 2
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
resources:
limits:
nvidia.com/gpu: "1"
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
artifact_paths:
- "*.log"
- "*.json"
.buildkite/k3_tests/sglang/run.sh
Executable
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env bash
# SGLang + LMCache MP integration test entrypoint for K8s pods.
# Sources the SGLang-specific env setup, then dispatches to scripts/.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
cd "${REPO_ROOT}"
source .buildkite/k3_harness/setup-sglang-env.sh
chmod +x "${SCRIPT_DIR}"/scripts/*.sh
case "${1:-}" in
correctness)
exec bash "${SCRIPT_DIR}/scripts/run-correctness.sh"
;;
perf)
exec bash "${SCRIPT_DIR}/scripts/run-perf.sh"
;;
*)
echo "Usage: $0 {correctness|perf}" >&2
exit 2
;;
esac
.buildkite/k3_tests/sglang/scripts/common.sh
Executable
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env bash
# Shared helpers for SGLang + LMCache MP CI tests.
MODEL="${MODEL:-Qwen/Qwen2.5-7B-Instruct}"
DAEMON_PORT="${DAEMON_PORT:-6200}"
DAEMON_HTTP_PORT="${DAEMON_HTTP_PORT:-7200}"
LMC_CONFIG_FILE="${LMC_CONFIG_FILE:-/tmp/lmcache_mp_ci.yaml}"
DAEMON_PID=""
SGLANG_PID=""
cleanup_all() {
if [[ -n "${SGLANG_PID}" ]]; then
kill -9 "${SGLANG_PID}" 2>/dev/null || true
pkill -9 -P "${SGLANG_PID}" 2>/dev/null || true
pkill -9 -f "sglang::scheduler" 2>/dev/null || true
wait "${SGLANG_PID}" 2>/dev/null || true
SGLANG_PID=""
fi
if [[ -n "${DAEMON_PID}" ]]; then
kill -9 "${DAEMON_PID}" 2>/dev/null || true
wait "${DAEMON_PID}" 2>/dev/null || true
DAEMON_PID=""
fi
sleep 2
}
launch_daemon() {
local log_file="$1"
cat > "${LMC_CONFIG_FILE}" <<EOF
mp_host: 127.0.0.1
mp_port: ${DAEMON_PORT}
EOF
lmcache server \
--host 127.0.0.1 --port "${DAEMON_PORT}" --http-port "${DAEMON_HTTP_PORT}" \
--chunk-size 256 --l1-size-gb 20 --eviction-policy LRU \
> "${log_file}" 2>&1 &
DAEMON_PID=$!
for ((i = 0; i < 60; i++)); do
if grep -q "ZMQ cache server is running" "${log_file}" 2>/dev/null; then
echo " daemon ready (${i}s, log=${log_file})"
return 0
fi
sleep 1
done
echo "FAIL: daemon did not start within 60s" >&2
tail -n 200 "${log_file}" >&2
return 1
}
# Launch SGLang. $1=port, $2=log file, $3="lmcache" or "no-lmcache".
launch_sglang() {
local port="$1" log_file="$2" mode="$3"
local lmcache_args=()
if [[ "${mode}" == "lmcache" ]]; then
lmcache_args=(--enable-lmcache --lmcache-config-file "${LMC_CONFIG_FILE}")
fi
python -m sglang.launch_server \
--model-path "${MODEL}" \
--host 127.0.0.1 --port "${port}" \
"${lmcache_args[@]}" \
> "${log_file}" 2>&1 &
SGLANG_PID=$!
wait_sglang_ready "${port}" "${log_file}"
}
wait_sglang_ready() {
local port="$1" log_file="$2" timeout="${3:-240}"
for ((i = 0; i < timeout; i++)); do
if curl -sf "http://127.0.0.1:${port}/health" >/dev/null 2>&1; then
echo " sglang ready on port ${port} (${i}s, mode=${mode:-?}, log=${log_file})"
return 0
fi
if grep -q "SIGQUIT received\|FATAL\|RuntimeError" "${log_file}" 2>/dev/null; then
echo "FAIL: SGLang crashed during startup" >&2
tail -n 200 "${log_file}" >&2
return 1
fi
sleep 1
done
echo "FAIL: SGLang did not become ready on port ${port} within ${timeout}s" >&2
tail -n 200 "${log_file}" >&2
return 1
}
# Non-streaming chat completion. Args: port, prompt, max_tokens.
chat_completion() {
local port="$1" prompt="$2" max_tokens="${3:-64}"
curl -sf "http://127.0.0.1:${port}/v1/chat/completions" \
-H "Content-Type: application/json" \
-d "$(python3 -c "
import json, sys
print(json.dumps({
'model': '${MODEL}',
'messages': [{'role': 'user', 'content': sys.argv[1]}],
'max_tokens': int(sys.argv[2]),
'temperature': 0.0,
'stream': False,
}))" "${prompt}" "${max_tokens}")" \
| python3 -c "
import json, sys
resp = json.load(sys.stdin)
print(resp['choices'][0]['message']['content'], end='')
"
}
# Sum LMCache L1 chunk reads from the daemon's /metrics; 0 if unreachable.
count_retrievals() {
local val
val=$( { curl -sf --max-time 3 "http://127.0.0.1:${DAEMON_HTTP_PORT}/metrics" 2>/dev/null || true; } \
| awk '/^lmcache_mp_l1_read_chunks_total[ {]/ { sum += $NF } END { printf "%d", sum + 0 }')
echo "${val:-0}"
}
# Two distinct ~2500-token prompts for run-correctness.sh.
generate_prompt() {
local variant="${1:-a}"
python3 -c "
import sys
sentences = {
'a': 'The quick brown fox jumps over the lazy dog while a curious cat watches from the windowsill on a quiet afternoon in early autumn. ',
'b': 'An old man walks slowly along a foggy riverside path at dawn carrying a fishing rod and dreaming of his grandchildren far away. ',
}
print(sentences[sys.argv[1]] * 80 + 'Summarize the scene above in one short sentence.')
" "${variant}"
}
# Run lmcache bench engine + long-doc-qa; echo mean TTFT in seconds. Pool > HBM forces LMCache hits on each doc's 2nd query.
run_long_doc_qa_ttft() {
local port="$1"
local outdir
outdir="$(mktemp -d -t lmc-bench-XXXXXX)"
lmcache bench engine \
--engine-url "http://127.0.0.1:${port}" \
--model "${MODEL}" \
--workload long-doc-qa \
--ldqa-document-length 10000 \
--ldqa-query-per-document 2 \
--ldqa-shuffle-policy tile \
--tokens-per-gb-kvcache 17500 --kv-cache-volume 15 \
--json --no-csv --no-interactive --quiet \
--output-dir "${outdir}" > "perf-bench-${port}.log" 2>&1
python3 -c "
import json
print(json.load(open('${outdir}/bench_summary.json'))['results']['mean_ttft_ms'] / 1000)
"
}
.buildkite/k3_tests/sglang/scripts/run-correctness.sh
Executable
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env bash
# Correctness: LMCache must not change the output, and must be exercised. A → B → flush_cache → A on the LMCache server: A populates LMCache, B is a separate request, /flush_cache clears SGLang's radix (but not LMCache), so the second A is a radix miss / LMCache hit. The cache-hit output is diffed against a no-LMCache reference run. Servers are launched sequentially (kill between phases) so two SGLang processes don't fight for GPU memory.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/common.sh"
DAEMON_LOG="correctness-daemon.log"
LMC_SGL_LOG="correctness-sgl-lmc.log"
NO_SGL_LOG="correctness-sgl-no.log"
PORT_LMC=30200
PORT_NO=30201
trap cleanup_all EXIT
echo "--- :rocket: Start daemon"
launch_daemon "${DAEMON_LOG}"
PROMPT_A="$(generate_prompt a)"
PROMPT_B="$(generate_prompt b)"
echo "--- :test_tube: Launch SGLang+LMCache; drive A → B → flush_cache → A"
launch_sglang "${PORT_LMC}" "${LMC_SGL_LOG}" "lmcache"
chat_completion "${PORT_LMC}" "${PROMPT_A}" 64 > /tmp/lmc_run1_A.txt
chat_completion "${PORT_LMC}" "${PROMPT_B}" 64 > /tmp/lmc_run2_B.txt
curl -sf -X POST "http://127.0.0.1:${PORT_LMC}/flush_cache" >/dev/null
RETRIEVALS_BEFORE=$(count_retrievals)
chat_completion "${PORT_LMC}" "${PROMPT_A}" 64 > /tmp/lmc_out.txt
RETRIEVALS_AFTER=$(count_retrievals)
RETRIEVAL_DELTA=$((RETRIEVALS_AFTER - RETRIEVALS_BEFORE))
echo " daemon retrieval delta on the cache-hit call: ${RETRIEVAL_DELTA}"
echo "--- :recycle: Kill LMCache SGLang, relaunch without LMCache"
kill -9 "${SGLANG_PID}" 2>/dev/null || true
pkill -9 -f "sglang::scheduler" 2>/dev/null || true
wait "${SGLANG_PID}" 2>/dev/null || true
SGLANG_PID=""
sleep 3
launch_sglang "${PORT_NO}" "${NO_SGL_LOG}" "no-lmcache"
echo "--- :mag: Reference call on the no-LMCache server"
chat_completion "${PORT_NO}" "${PROMPT_A}" 64 > /tmp/no_out.txt
echo "+++ :scales: Verdict"
if [[ "${RETRIEVAL_DELTA}" -lt 1 ]]; then
echo "FAIL: LMCache was not exercised (retrieval delta ${RETRIEVAL_DELTA} < 1)" >&2
tail -n 60 "${DAEMON_LOG}" >&2
exit 1
fi
if ! diff -u /tmp/no_out.txt /tmp/lmc_out.txt; then
echo "FAIL: LMCache changed the output" >&2
exit 1
fi
echo "PASS — outputs match AND LMCache served ${RETRIEVAL_DELTA} retrieval(s) on the cache-hit call."
.buildkite/k3_tests/sglang/scripts/run-perf.sh
Executable
+55
View File
@@ -0,0 +1,55 @@
#!/usr/bin/env bash
# Performance: query-round TTFT with LMCache must be lower than without.
# Phase A: launch SGLang + LMCache → bench → record TTFT and verify
# the daemon log shows RETRIEVE traffic. Phase B: relaunch SGLang
# without LMCache → same bench → record TTFT.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "${SCRIPT_DIR}/common.sh"
trap cleanup_all EXIT
DAEMON_LOG="perf-daemon.log"
LMC_LOG="perf-sgl-lmc.log"
NO_LOG="perf-sgl-no.log"
PORT_LMC=30200
PORT_NO=30201
echo "--- :rocket: Phase A — start daemon + LMCache-enabled SGLang"
launch_daemon "${DAEMON_LOG}"
launch_sglang "${PORT_LMC}" "${LMC_LOG}" "lmcache"
echo "--- :stopwatch: Phase A — bench, record mean TTFT"
RETRIEVALS_BEFORE=$(count_retrievals)
TTFT_LMC=$(run_long_doc_qa_ttft "${PORT_LMC}")
RETRIEVALS_AFTER=$(count_retrievals)
RETRIEVAL_DELTA=$((RETRIEVALS_AFTER - RETRIEVALS_BEFORE))
echo " ttft_with_lmcache = ${TTFT_LMC}s, retrieval delta = ${RETRIEVAL_DELTA}"
echo "--- :recycle: Phase B — relaunch SGLang without LMCache"
kill -9 "${SGLANG_PID}" 2>/dev/null || true
pkill -9 -f "sglang::scheduler" 2>/dev/null || true
wait "${SGLANG_PID}" 2>/dev/null || true
SGLANG_PID=""
sleep 3
launch_sglang "${PORT_NO}" "${NO_LOG}" "no-lmcache"
echo "--- :stopwatch: Phase B — bench, record mean TTFT"
TTFT_NO=$(run_long_doc_qa_ttft "${PORT_NO}")
echo " ttft_without_lmcache = ${TTFT_NO}s"
echo "+++ :scales: Verdict"
echo " ttft_with_lmcache = ${TTFT_LMC}s"
echo " ttft_without_lmcache = ${TTFT_NO}s"
echo " retrieval delta = ${RETRIEVAL_DELTA}"
if [[ "${RETRIEVAL_DELTA}" -lt 1 ]]; then
echo "FAIL: LMCache was not exercised (retrieval delta ${RETRIEVAL_DELTA} < 1)" >&2
tail -n 60 "${DAEMON_LOG}" >&2
exit 1
fi
if ! python3 -c "import sys; sys.exit(0 if ${TTFT_LMC} < ${TTFT_NO} else 1)"; then
echo "FAIL: LMCache did not improve query-round TTFT (${TTFT_LMC}s >= ${TTFT_NO}s)" >&2
exit 1
fi
echo "PASS — ttft_with_lmcache (${TTFT_LMC}s) < ttft_without_lmcache (${TTFT_NO}s); ${RETRIEVAL_DELTA} retrieval(s)."
.buildkite/k3_tests/unit/buildkite-pipeline.yml
+13
View File
@@ -0,0 +1,13 @@
# Paste this into the Buildkite pipeline's "Steps" editor.
# It uploads the real pipeline definition from the repo.
agents:
queue: "k3-h200-local"
env:
# Optional: unit tests run without it. Setting it unlocks one extra test
# (test_segment_token_database) that's gated on HF credentials.
HF_TOKEN: ""
steps:
- label: ":pipeline: Upload pipeline"
command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/unit/pipeline.yml
.buildkite/k3_tests/unit/pipeline.yml
+44
View File
@@ -0,0 +1,44 @@
# Unit tests — runs pytest with coverage against LMCache source.
# No vLLM required; uses setup-lmcache-only-env.sh for a lighter install.
steps:
- label: ":test_tube: Unit Tests"
command: .buildkite/k3_tests/unit/run.sh
timeout_in_minutes: 60
agents: { queue: "k8s" }
plugins:
- kubernetes:
podSpec:
containers:
- name: container-0
image: lmcache/ci-base:latest
imagePullPolicy: Never
env:
- { name: CUDA_LAUNCH_BLOCKING, value: "1" }
resources:
requests:
cpu: "32"
memory: "96Gi"
limits:
nvidia.com/gpu: "1"
cpu: "64"
memory: "200Gi"
volumeMounts:
- { name: hf-cache, mountPath: /root/.cache/huggingface }
# K8s defaults /dev/shm to 64 MiB, which causes SIGBUS in
# tests/v1/shm_allocator (allocates 5 GiB via SharedMemory).
# Mount a 16 GiB tmpfs to give shm tests headroom.
- { name: dshm, mountPath: /dev/shm }
# GDS-capable scratch dir on the host's local NVMe. Direct
# hostPath (no subPath) — K8s' subPath bind-mount breaks
# cuFile's fs-type detection. run.sh creates a per-build
# subdir here and exposes it via LMCACHE_TEST_TMPDIR.
- { name: scratch, mountPath: /scratch }
volumes:
- { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } }
- { name: dshm, emptyDir: { medium: Memory, sizeLimit: 16Gi } }
- { name: scratch, hostPath: { path: /data/gds-scratch, type: DirectoryOrCreate } }
artifact_paths:
- "durations/test.html"
- "coverage-test.xml"
- "coverage-test/**/*"
.buildkite/k3_tests/unit/run.sh
Executable
+43
View File
@@ -0,0 +1,43 @@
#!/usr/bin/env bash
# Unit test entrypoint for K8s pods.
# Installs LMCache (no vLLM) + test deps, then runs pytest with coverage.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
cd "${REPO_ROOT}"
# ── Per-build scratch dir ────────────────────────────────────
# /scratch is a shared hostPath mount (see pipeline.yml) backed by ext4/xfs
# on local NVMe — the only place GDS (cuFile) works in this pod (overlayfs
# /tmp fails with CU_FILE_IO_NOT_SUPPORTED). Give this build its own
# subdirectory so concurrent pods don't collide, and clean it up on exit.
# Direct subdir rather than K8s subPathExpr since the latter's bind mount
# breaks cuFile's fs-type detection.
#
# LMCACHE_TEST_TMPDIR is consumed by the handful of test files that need
# GDS-capable scratch (test_gds_backend, test_cache_engine, the xpu
# benchmarks). Leaving TMPDIR unset means /tmp stays pod-internal overlay
# for pip/uv/build caches — those don't need direct I/O.
BUILD_TAG="${BUILDKITE_BUILD_ID:-manual-$$}"
export LMCACHE_TEST_TMPDIR="/scratch/bk-${BUILD_TAG}"
mkdir -p "${LMCACHE_TEST_TMPDIR}"
trap 'rm -rf "${LMCACHE_TEST_TMPDIR}" 2>/dev/null || true' EXIT
# ── Environment setup ────────────────────────────────────────
source .buildkite/k3_harness/setup-lmcache-only-env.sh
uv pip install -r requirements/test.txt
# ── Run unit tests with coverage ─────────────────────────────
LMCACHE_TRACK_USAGE="false" \
pytest --maxfail=1 --cov=lmcache \
--cov-report term --cov-report=html:coverage-test \
--cov-report=xml:coverage-test.xml --html=durations/test.html \
--ignore=tests/disagg --ignore=tests/v1/test_pos_kernels.py \
--ignore=tests/skipped \
--ignore=tests/v1/storage_backend/test_eic.py
cat << EOF | buildkite-agent annotate --style "info"
Read the <a href="artifact://coverage-test/index.html">uploaded coverage report</a>
EOF
.buildkite/operator/integration/pipeline.yaml
+28
View File
@@ -0,0 +1,28 @@
steps:
- label: "Detect operator changes"
key: "operator_detect_changes"
command: |
set -eu
BASE_REF="origin/dev"
if ! git fetch --quiet origin dev 2>/dev/null; then
echo "WARN: could not fetch $$BASE_REF; running tests defensively."
buildkite-agent pipeline upload .buildkite/operator/integration/tests.yaml
exit 0
fi
if ! CHANGED=$(git diff --name-only "$$BASE_REF...HEAD" -- operator/ .buildkite/operator/ 2>/dev/null); then
echo "WARN: git diff failed; running tests defensively."
buildkite-agent pipeline upload .buildkite/operator/integration/tests.yaml
exit 0
fi
if [ -z "$$CHANGED" ]; then
echo "No operator or operator-pipeline changes vs $$BASE_REF; skipping integration tests."
exit 0
fi
echo "Operator changes detected:"
echo "$$CHANGED"
buildkite-agent pipeline upload .buildkite/operator/integration/tests.yaml
.buildkite/operator/integration/tests.yaml
+38
View File
@@ -0,0 +1,38 @@
# The CacheBlend integration spec (cacheblend_integration_smoke_test.go) pulls a
# PRIVATE payload image. Set these as pipeline secrets in the Buildkite UI (same
# mechanism as HF_TOKEN); they propagate through `make test-e2e-gpu-kind` ->
# `go test` -> the spec via os.Getenv. Do NOT hardcode the token here.
# CACHEBLEND_REGISTRY_USER docker registry username (e.g. Docker Hub user)
# CACHEBLEND_REGISTRY_TOKEN docker registry read-only PAT
# CACHEBLEND_REGISTRY_SERVER optional, default https://index.docker.io/v1/
# When USER/TOKEN are unset the spec Skips, so this build stays green without them.
env:
GPU_KIND_CLUSTER: "operator-test-e2e-gpu-${BUILDKITE_BUILD_NUMBER}"
steps:
- label: "Operator Integration: test-e2e-gpu-kind"
key: "operator_test_e2e_gpu_kind"
timeout_in_minutes: 90
command: |
echo $$PWD # for debugging
cd operator
make test-e2e-gpu-kind
- label: "Cleanup Kind cluster"
key: "operator_cleanup_kind"
depends_on: "operator_test_e2e_gpu_kind"
allow_dependency_failure: true
timeout_in_minutes: 10
command: |
cd operator
make cleanup-test-e2e-gpu-kind
- label: "Cleanup workspace"
depends_on: "operator_cleanup_kind"
allow_dependency_failure: true
command: |
export TARGET="$$PWD"
echo "Deleting current workspace $$TARGET"
cd /
rm -rf "$$TARGET" || true
.buildkite/pipeline.yml
+60
View File
@@ -0,0 +1,60 @@
env:
PATH: "$HOME/.local/bin:$PATH"
steps:
- label: "Unit Tests"
key: "unit_tests"
timeout_in_minutes: 60
command: |
echo $PWD # for debugging
uv venv --python 3.12 .venv-$BUILDKITE_BUILD_ID
source .venv-$BUILDKITE_BUILD_ID/bin/activate
uv pip install --upgrade pip setuptools wheel
if command -v nvidia-smi >/dev/null 2>&1; then
source .buildkite/scripts/pick-free-gpu.sh 18000
export CUDA_LAUNCH_BLOCKING=1
# Pin torch to a version compatible with the CI machine's CUDA 12.x driver
uv pip install "torch>=2.6.0,<2.11.0"
else
source .buildkite/scripts/pick-free-gpu-amd.sh 18000
export AMD_SERIALIZE_KERNEL=1
export PYTORCH_ROCM_ARCH="gfx942"
export TORCH_DONT_CHECK_COMPILER_ABI=1
export CXX=hipcc
export BUILD_WITH_HIP=1
uv pip install torch torchvision --index-url https://download.pytorch.org/whl/rocm7.0
uv pip install -r requirements/rocm_core.txt
fi
uv pip install -r requirements/common.txt
uv pip install -r requirements/test.txt
uv pip install -e . --no-build-isolation
uv pip freeze
LMCACHE_TRACK_USAGE="false" \
pytest --maxfail=1 --cov=lmcache \
--cov-report term --cov-report=html:coverage-test \
--cov-report=xml:coverage-test.xml --html=durations/test.html \
--ignore=tests/disagg --ignore=tests/v1/test_pos_kernels.py \
--ignore=tests/v1/test_nixl_batched_contains.py \
--ignore=tests/v1/test_device_id_race.py \
--ignore=tests/v1/test_nixl_multipath.py \
--ignore=tests/skipped \
--ignore=tests/v1/storage_backend/test_eic.py
cat << EOF | buildkite-agent annotate --style "info"
Read the <a href="artifact://coverage-test/index.html">uploaded coverage report</a>
EOF
export TARGET="$PWD"
echo "Deleting current workspace $TARGET"
cd /
sudo rm -rf "$TARGET"
artifact_paths:
- "durations/test.html"
- "coverage-test.xml"
- "coverage-test/**/*"
.buildkite/pipelines/clean.yml
+6
View File
@@ -0,0 +1,6 @@
steps:
- label: ":wastebasket: Kill all"
key: "clean"
command: |
bash .buildkite/scripts/clean.sh
sudo rm -rf /var/cache/buildkite/*.lock

Some files were not shown because too many files have changed in this diff Show More