From f213ec89765936f8d36a71674c92a605834810fc Mon Sep 17 00:00:00 2001 From: wehub-resource-sync Date: Mon, 13 Jul 2026 12:24:33 +0800 Subject: [PATCH] chore: import upstream snapshot with attribution --- .buildkite/README.md | 15 + .buildkite/cases/comprehensive-cases.txt | 11 + .buildkite/cases/integration-cases.txt | 1 + .buildkite/configs/async.yaml | 32 + .buildkite/configs/dummy.yaml | 22 + .buildkite/configs/layerwise.yaml | 26 + .../lmcache_configs/local_cpu_mla.yaml | 15 + .buildkite/configs/local_cpu.yaml | 23 + .buildkite/configs/local_cpu_mla.yaml | 30 + .buildkite/configs/local_cpu_with_v3.yaml | 25 + .buildkite/configs/local_disk.yaml | 26 + .buildkite/configs/local_disk_with_v3.yaml | 27 + .buildkite/configs/multi_device.yaml | 26 + .buildkite/configs/p2p.yaml | 69 + .buildkite/configs/p2p_with_v3.yaml | 71 + .buildkite/configs/pd.yaml | 74 + .buildkite/correctness/README.md | 52 + .buildkite/correctness/async_request.py | 166 + .buildkite/correctness/compare_files.py | 86 + .buildkite/correctness/mmlu-test.py | 189 + .../correctness/pipeline.correctness.yml | 16 + .buildkite/correctness/pipeline.mmlu.yml | 36 + .buildkite/correctness/setup.sh | 15 + .buildkite/correctness/sharegpt2openai.py | 125 + .buildkite/correctness/summarize-results.py | 216 + .buildkite/k3_harness/ARCHITECTURE.md | 89 + .buildkite/k3_harness/README.md | 143 + .buildkite/k3_harness/ci-base.Dockerfile | 40 + .buildkite/k3_harness/gpu-monitor.sh | 105 + .buildkite/k3_harness/install-agent-stack.sh | 43 + .buildkite/k3_harness/resolve-pinned-vllm.sh | 96 + .buildkite/k3_harness/setup-blend-env.sh | 63 + .buildkite/k3_harness/setup-cluster.sh | 101 + .buildkite/k3_harness/setup-env.sh | 298 ++ .buildkite/k3_harness/setup-gpu-monitor.sh | 44 + .../k3_harness/setup-lmcache-only-env.sh | 23 + .buildkite/k3_harness/setup-sglang-env.sh | 42 + .buildkite/k3_harness/smoke-test.sh | 49 + .buildkite/k3_harness/teardown.sh | 30 + .buildkite/k3_harness/values.yaml | 25 + .buildkite/k3_tests/README.md | 154 + .../k3_tests/blend/buildkite-pipeline.yml | 20 + .buildkite/k3_tests/blend/pipeline.yml | 34 + .buildkite/k3_tests/blend/run.sh | 17 + .../k3_tests/blend/scripts/run-compat.sh | 42 + .buildkite/k3_tests/common_scripts/helpers.sh | 119 + .../k3_tests/common_scripts/path-filter.sh | 158 + .../common_scripts/upload-pipeline.sh | 38 + .../k3_tests/comprehensive/BK_WEB_SETUP.md | 27 + .../comprehensive/buildkite-pipeline.yml | 11 + .../k3_tests/comprehensive/pipeline.yml | 143 + .buildkite/k3_tests/comprehensive/run.sh | 23 + .../scripts/run-single-config.sh | 531 +++ .../comprehensive/scripts/upload-baselines.sh | 129 + .../k3_tests/correctness/BK_WEB_SETUP.md | 14 + .../correctness/buildkite-pipeline.yml | 11 + .buildkite/k3_tests/correctness/pipeline.yml | 27 + .buildkite/k3_tests/correctness/run.sh | 36 + .../correctness/scripts/run-correctness.sh | 221 + .../k3_tests/integration/BK_WEB_SETUP.md | 14 + .../integration/buildkite-pipeline.yml | 11 + .buildkite/k3_tests/integration/pipeline.yml | 49 + .buildkite/k3_tests/integration/run.sh | 19 + .../integration/scripts/run-integration.sh | 112 + .../k3_tests/multiprocess/BK_WEB_SETUP.md | 15 + .../multiprocess/buildkite-pipeline.yml | 11 + .buildkite/k3_tests/multiprocess/pipeline.yml | 279 ++ .buildkite/k3_tests/multiprocess/run.sh | 27 + .../k3_tests/multiprocess/scripts/cleanup.sh | 48 + .../multiprocess/scripts/launch-processes.sh | 199 + .../multiprocess/scripts/pin-tested-vllm.sh | 175 + .../multiprocess/scripts/run-cache-stats.sh | 214 + .../multiprocess/scripts/run-deadlock.sh | 137 + .../scripts/run-fault-tolerance.sh | 253 ++ .../multiprocess/scripts/run-gds-smoke.sh | 113 + .../multiprocess/scripts/run-hma-lm-eval.sh | 253 ++ .../multiprocess/scripts/run-http-api.sh | 446 ++ .../multiprocess/scripts/run-lm-eval.sh | 120 + .../scripts/run-long-doc-qa-l2.sh | 575 +++ .../multiprocess/scripts/run-long-doc-qa.sh | 264 ++ .../multiprocess/scripts/run-mp-test.sh | 122 + .../k3_tests/multiprocess/scripts/run-p2p.sh | 247 ++ .../scripts/run-restart-recovery.sh | 287 ++ .../multiprocess/scripts/run-single-test.sh | 181 + .../multiprocess/scripts/run-vllm-bench.sh | 276 ++ .../multiprocess/scripts/wait-for-servers.sh | 72 + .buildkite/k3_tests/sglang/BK_WEB_SETUP.md | 16 + .../k3_tests/sglang/buildkite-pipeline.yml | 12 + .buildkite/k3_tests/sglang/pipeline.yml | 54 + .buildkite/k3_tests/sglang/run.sh | 25 + .buildkite/k3_tests/sglang/scripts/common.sh | 147 + .../sglang/scripts/run-correctness.sh | 54 + .../k3_tests/sglang/scripts/run-perf.sh | 55 + .../k3_tests/unit/buildkite-pipeline.yml | 13 + .buildkite/k3_tests/unit/pipeline.yml | 44 + .buildkite/k3_tests/unit/run.sh | 43 + .buildkite/operator/integration/pipeline.yaml | 28 + .buildkite/operator/integration/tests.yaml | 38 + .buildkite/pipeline.yml | 60 + .buildkite/pipelines/clean.yml | 6 + .buildkite/pipelines/comprehensive-tests.yml | 31 + .buildkite/pipelines/end-to-end-tests.yml | 50 + .buildkite/pipelines/multiprocessing-test.yml | 11 + .buildkite/scripts/bare-machine-cleanup.sh | 25 + .buildkite/scripts/clean.sh | 31 + .buildkite/scripts/end-to-end-test.sh | 90 + .buildkite/scripts/gpu_zombie_killer.sh | 57 + .buildkite/scripts/multi-round-qa.sh | 45 + .../scripts/multiprocessing-test/README.md | 339 ++ .../build-mp-docker-image.sh | 24 + .../scripts/multiprocessing-test/cleanup.sh | 52 + .../scripts/multiprocessing-test/common.sh | 59 + .../multiprocessing-test/launch-containers.sh | 126 + .../run-fault-tolerance.sh | 152 + .../multiprocessing-test/run-lm-eval.sh | 140 + .../multiprocessing-test/run-long-doc-qa.sh | 345 ++ .../multiprocessing-test/run-mp-test.sh | 128 + .../multiprocessing-test/run-vllm-bench.sh | 253 ++ .../multiprocessing-test/test-launch.sh | 106 + .../multiprocessing-test/wait-for-vllm.sh | 77 + .buildkite/scripts/pick-free-gpu-amd.sh | 105 + .buildkite/scripts/pick-free-gpu.sh | 69 + .../scripts/should-run-comprehensive.sh | 81 + .buildkite/scripts/vllm-correctness.sh | 273 ++ .buildkite/scripts/vllm-integration-tests.sh | 876 ++++ .buildkite/vllm-integration-tests.yml | 30 + .clang-format | 27 + .claude/skills/create-pr/SKILL.md | 106 + .claude/skills/hybrid-benchmarking/SKILL.md | 176 + .claude/skills/pr-review/SKILL.md | 205 + .claude/skills/pre-pr-check/SKILL.md | 256 ++ .cursor/BUGBOT.md | 74 + .gemini/styleguide.md | 75 + .github/CODEOWNERS | 109 + .github/ISSUE_TEMPLATE/blank_issue.md | 30 + .github/ISSUE_TEMPLATE/bug_report.md | 40 + .github/ISSUE_TEMPLATE/feature_request.md | 22 + .github/PULL_REQUEST_TEMPLATE.md | 12 + .github/actions/free-disk-space/action.yml | 29 + .github/dependabot.yml | 25 + .github/scripts/cpu_device_test.sh | 156 + .github/scripts/cpu_server_bench_test.sh | 139 + .github/scripts/cpu_vllm_e2e_test.sh | 84 + .github/scripts/download_model.sh | 87 + .github/scripts/download_opt125m_github.sh | 58 + .github/scripts/install_lmcache_cpu.sh | 27 + .github/scripts/install_vllm_cpu.sh | 66 + .github/scripts/run-cpu-e2e-validation.sh | 648 +++ .github/workflows/actionlint.dockerfile | 3 + .github/workflows/actionlint.yml | 52 + .github/workflows/aerospike_integration.yml | 142 + .github/workflows/automerge-labeler.yml | 17 + .github/workflows/build_cli_artifacts.yml | 106 + .github/workflows/build_cpu_artifacts.yml | 122 + .github/workflows/build_cu129_artifacts.yml | 105 + .github/workflows/build_doc.yml | 106 + .github/workflows/build_main_artifacts.yml | 82 + .github/workflows/code_quality_checks.yml | 68 + .github/workflows/codeql.yml | 112 + .github/workflows/cpu_device.yml | 174 + .github/workflows/matchers/actionlint.json | 17 + .github/workflows/matchers/mypy.json | 16 + .github/workflows/nightly_build.yml | 221 + .github/workflows/operator_ci.yml | 76 + .github/workflows/operator_nightly.yml | 80 + .github/workflows/operator_release.yml | 100 + .github/workflows/pr_full_build.yml | 52 + .github/workflows/publish.yml | 531 +++ .github/workflows/scorecard.yml | 84 + .github/workflows/stale_bot.yml | 51 + .github/workflows/sync_torch_version.yml | 147 + .github/workflows/test.yml | 170 + .github/workflows/translate_doc_zh.yml | 109 + .gitignore | 153 + .isort.cfg | 11 + .pre-commit-config.yaml | 133 + AGENTS.md | 219 + CLAUDE.md | 58 + CMakeLists.txt | 2 + CODE_OF_CONDUCT.md | 128 + CONTRIBUTING.md | 135 + DCO | 34 + LICENSE | 201 + MAINTAINERS.md | 14 + MANIFEST.in | 10 + README.md | 153 + README.wehub.md | 7 + SECURITY.md | 9 + asset/deployment_modes_dark.png | Bin 0 -> 69078 bytes asset/deployment_modes_light.png | Bin 0 -> 73311 bytes asset/ecosystem.png | Bin 0 -> 248858 bytes asset/logo.png | Bin 0 -> 32418 bytes asset/partner_dark.png | Bin 0 -> 378055 bytes asset/partner_light.png | Bin 0 -> 309221 bytes benchmarks/long_doc_qa/long_doc_qa.py | 778 ++++ .../long_doc_qa/long_doc_qa_recommender.py | 195 + .../microbenchmark/bitmap_ops_benchmark.py | 94 + .../microbenchmark/ttl_lock_benchmark.py | 541 +++ benchmarks/multi_doc_qa/README.md | 30 + benchmarks/multi_doc_qa/lmcache.yaml | 1 + benchmarks/multi_doc_qa/lmcache_blend.yaml | 6 + benchmarks/multi_doc_qa/multi_doc_qa.py | 495 +++ benchmarks/multi_doc_qa/shuffle_doc_qa.py | 289 ++ benchmarks/multi_round_qa/README.md | 127 + .../multi_round_qa/data_preprocessing.py | 143 + benchmarks/multi_round_qa/multi-round-qa.py | 764 ++++ .../multi_round_qa/prepare_sharegpt_data.sh | 4 + benchmarks/multi_round_qa/requirements.txt | 3 + benchmarks/multi_round_qa/utils.py | 125 + benchmarks/musa/README.md | 18 + benchmarks/musa/bench_inprocess_transfer.py | 286 ++ benchmarks/rag/README.md | 111 + benchmarks/rag/launch_lmcache.sh | 30 + benchmarks/rag/launch_vllm.sh | 15 + benchmarks/rag/precompute.py | 250 ++ benchmarks/rag/rag.py | 433 ++ benchmarks/rag/requirements.txt | 4 + benchmarks/rag/utils.py | 233 + benchmarks/storage_backend_io/README.md | 226 + .../storage_backend_io_benchmark.py | 1257 ++++++ .../ttft-estimator/llama-h100-example.png | Bin 0 -> 26401 bytes benchmarks/ttft-estimator/ttft-estimator.py | 129 + csrc/ac_dec.cu | 538 +++ csrc/ac_enc.cu | 477 +++ csrc/cachegen_kernels.cuh | 45 + csrc/cal_cdf.cu | 129 + csrc/completion_recorder.cpp | 59 + csrc/completion_recorder.h | 51 + csrc/cuda_compat.h | 13 + csrc/dispatch_utils.h | 39 + csrc/engine_kv_format.h | 143 + csrc/event_recorder.cpp | 102 + csrc/event_recorder.h | 78 + csrc/kv_transfer_types.h | 13 + csrc/mem_alloc.cpp | 235 + csrc/mem_alloc.h | 27 + csrc/mem_kernels.cu | 1091 +++++ csrc/mem_kernels.cuh | 54 + csrc/mp_mem_kernels.cu | 472 ++ csrc/mp_mem_kernels.cuh | 160 + csrc/pos_kernels.cu | 148 + csrc/pos_kernels.cuh | 11 + csrc/pybind.cpp | 181 + csrc/storage_backends/README.md | 311 ++ csrc/storage_backends/aerospike/connector.cpp | 533 +++ csrc/storage_backends/aerospike/connector.h | 103 + csrc/storage_backends/aerospike/pybind.cpp | 25 + csrc/storage_backends/connector_base.h | 657 +++ csrc/storage_backends/connector_interface.h | 130 + .../storage_backends/connector_pybind_utils.h | 179 + csrc/storage_backends/connector_types.h | 104 + csrc/storage_backends/event_notifier.h | 180 + csrc/storage_backends/fs/connector.cpp | 298 ++ csrc/storage_backends/fs/connector.h | 80 + csrc/storage_backends/fs/pybind.cpp | 15 + csrc/storage_backends/mooncake/connector.cpp | 323 ++ csrc/storage_backends/mooncake/connector.h | 91 + csrc/storage_backends/mooncake/pybind.cpp | 36 + csrc/storage_backends/redis/connector.cpp | 377 ++ csrc/storage_backends/redis/connector.h | 119 + csrc/storage_backends/redis/pybind.cpp | 14 + csrc/storage_manager/bitmap.cpp | 243 ++ csrc/storage_manager/bitmap.h | 166 + csrc/storage_manager/fold.cpp | 78 + csrc/storage_manager/fold.h | 64 + .../periodic_event_notifier.cpp | 133 + .../storage_manager/periodic_event_notifier.h | 54 + csrc/storage_manager/pybind.cpp | 137 + csrc/storage_manager/ttl_lock.cpp | 95 + csrc/storage_manager/ttl_lock.h | 90 + csrc/storage_manager/utils.cpp | 111 + csrc/storage_manager/utils.h | 83 + csrc/sycl/ac_dec_sycl.cpp | 551 +++ csrc/sycl/ac_enc_sycl.cpp | 379 ++ csrc/sycl/cachegen_kernels_sycl.h | 41 + csrc/sycl/cal_cdf_sycl.cpp | 216 + csrc/sycl/mem_kernels_sycl.cpp | 1023 +++++ csrc/sycl/mem_kernels_sycl.h | 56 + csrc/sycl/pos_kernels_sycl.cpp | 199 + csrc/sycl/pybind_sycl.cpp | 77 + csrc/utils.cpp | 13 + csrc/utils.h | 3 + docker/Dockerfile | 155 + docker/Dockerfile.lightweight | 21 + docker/Dockerfile.payload | 62 + docker/Dockerfile.rocm | 126 + docker/Dockerfile.rocm-lightweight | 43 + docker/Dockerfile.standalone | 151 + docker/Dockerfile.xpu | 124 + docker/README.md | 225 + docker/example_build.sh | 25 + docker/example_run.sh | 21 + docs/Makefile | 20 + docs/README.md | 1 + docs/coding_standards.md | 376 ++ docs/design/ARCHITECTURE_MULTI_HARDWARE.md | 136 + docs/design/README.md | 33 + docs/design/cli/commands.md | 403 ++ .../bench/engine_bench/bench-engine.md | 691 +++ docs/design/cli/commands/describe.md | 476 +++ docs/design/cli/commands/kvcache-command.md | 254 ++ docs/design/cli/commands/ping.md | 221 + docs/design/cli/commands/query-command.md | 193 + docs/design/cli/framework-and-metrics.md | 381 ++ .../design/integration/tensorrt_llm/README.md | 81 + .../vllm/hybrid-kv-cache-groups.md | 187 + .../integration/vllm/kv_cache_group_edits.md | 150 + .../observability/request-event-span.md | 375 ++ docs/design/sdk/batch.md | 53 + docs/design/sdk/kvcache.md | 116 + docs/design/sdk/stream.md | 75 + .../tools/controller_benchmark/README.md | 1 + .../transfer_channel_benchmark/README.md | 1 + docs/design/usage_telemetry/README.md | 1 + .../v1/distributed/bitmap_ops/README.md | 57 + docs/design/v1/distributed/l2_adapters/dax.md | 139 + .../distributed/l2_adapters/fault_inject.md | 100 + .../v1/distributed/l2_adapters/l2_eviction.md | 273 ++ .../l2_adapters/l2_per_user_quota.md | 850 ++++ .../v1/distributed/l2_adapters/nixl_store.md | 296 ++ .../v1/distributed/l2_adapters/overall.md | 599 +++ .../distributed/l2_adapters/p2p_l2_adapter.md | 108 + .../v1/distributed/l2_adapters/plugin.md | 359 ++ .../v1/distributed/l2_adapters/raw_block.md | 155 + .../distributed/l2_adapters/serde_wrapper.md | 301 ++ docs/design/v1/distributed/serde/README.md | 269 ++ .../design/v1/distributed/serde/turboquant.md | 336 ++ .../distributed/transfer_channel/overall.md | 221 + docs/design/v1/encoder-cache.md | 240 ++ .../v1/gpu_connector/layout-invariant.md | 270 ++ docs/design/v1/hidden_state_store.md | 109 + docs/design/v1/mp_coordinator/README.md | 163 + docs/design/v1/mp_coordinator/blend_lookup.md | 262 ++ docs/design/v1/mp_coordinator/l2_prefetch.md | 105 + .../mp_coordinator/l2_usage_and_eviction.md | 210 + docs/design/v1/mp_observability/DEBUG.md | 93 + docs/design/v1/mp_observability/EVENTS.md | 268 ++ docs/design/v1/mp_observability/METRICS.md | 543 +++ docs/design/v1/mp_observability/README.md | 1 + .../blend_v3_observability.md | 209 + docs/design/v1/mp_observability/errors.md | 70 + docs/design/v1/mp_observability/event-bus.md | 203 + docs/design/v1/mp_observability/trace.md | 546 +++ .../multiprocess/device_ipc_wrapper_design.md | 79 + .../engine_driven_transfer_design.md | 290 ++ .../v1/multiprocess/http_api_extension.md | 166 + .../v1/multiprocess/mp_runtime_plugin.md | 422 ++ .../v1/multiprocess/protocols/README.md | 1 + docs/design/v1/multiprocess/raw_cuda_ipc.md | 75 + .../design/v1/multiprocess/worker_liveness.md | 184 + docs/design/v1/pd_async_reservation_design.md | 171 + docs/design/v1/platform/event_notifier.md | 206 + docs/make.bat | 35 + docs/source/.nojekyll | 0 docs/source/_static/basic_codepath.svg | 107 + .../source/_static/bench_interactive_demo.gif | Bin 0 -> 608407 bytes docs/source/_static/custom.css | 143 + docs/source/_static/custom.js | 283 ++ .../source/_static/full_layerwise_diagram.svg | 256 ++ docs/source/_static/kv_cache_calculator.html | 874 ++++ docs/source/_static/modelconfig.json | 292 ++ docs/source/_static/scroll.css | 32 + docs/source/_templates/custom.html | 0 docs/source/_templates/versioning.html | 17 + docs/source/api_reference/configurations.rst | 557 +++ .../api_reference/dynamic_connector.rst | 72 + docs/source/api_reference/multimodality.rst | 464 ++ .../source/api_reference/storage_backends.rst | 9 + docs/source/assets/InfiniStore-usage.png | Bin 0 -> 110926 bytes docs/source/assets/deployment_modes_dark.png | Bin 0 -> 69078 bytes docs/source/assets/deployment_modes_light.png | Bin 0 -> 73311 bytes docs/source/assets/ecosystem.png | Bin 0 -> 248858 bytes docs/source/assets/lmcache-controller.png | Bin 0 -> 118850 bytes docs/source/assets/lmcache-logo.png | Bin 0 -> 17357 bytes docs/source/assets/lmcache-logo_crop.png | Bin 0 -> 17439 bytes docs/source/assets/maru-kvcache.png | Bin 0 -> 50599 bytes docs/source/assets/mooncake-store-preview.png | Bin 0 -> 262647 bytes docs/source/cli/bench.rst | 1382 ++++++ docs/source/cli/coordinator.rst | 97 + docs/source/cli/describe.rst | 218 + docs/source/cli/index.rst | 108 + docs/source/cli/kvcache.rst | 126 + docs/source/cli/ping.rst | 75 + docs/source/cli/query.rst | 90 + docs/source/cli/quota.rst | 301 ++ docs/source/cli/server.rst | 103 + docs/source/cli/tool.rst | 53 + docs/source/cli/trace.rst | 102 + docs/source/community/blogs.rst | 7 + docs/source/community/index.rst | 11 + docs/source/community/meetings.rst | 49 + docs/source/conf.py | 188 + docs/source/controller/freeze_mode.rst | 100 + docs/source/controller/index.rst | 59 + docs/source/developer_guide/architecture.rst | 112 + docs/source/developer_guide/cli.rst | 503 +++ docs/source/developer_guide/contributing.rst | 204 + .../developer_guide/extending_http_api.rst | 109 + .../extending_lmcache/native_connectors.rst | 686 +++ .../remote_storage_plugins.rst | 294 ++ .../extending_lmcache/runtime_plugins.rst | 89 + .../extending_lmcache/storage_plugins.rst | 248 ++ docs/source/developer_guide/index.rst | 13 + docs/source/developer_guide/integration.rst | 48 + .../developer_guide/usage/basic_check.rst | 144 + docs/source/developer_guide/usage/index.rst | 7 + .../usage/usage_stats_collection.rst | 155 + .../disaggregated_prefill/nixl/1p1d.rst | 275 ++ .../disaggregated_prefill/nixl/index.rst | 23 + .../disaggregated_prefill/nixl/xpyd.rst | 361 ++ .../disaggregated_prefill/shared_storage.rst | 4 + docs/source/distributed_kv_cache.rst | 14 + docs/source/getting_started/benchmarking.rst | 157 + docs/source/getting_started/index.rst | 16 + docs/source/getting_started/installation.rst | 265 ++ .../getting_started/kv_cache_calculator.rst | 18 + docs/source/getting_started/quickstart.rst | 480 +++ .../quickstart/disaggregated_prefill.rst | 213 + .../getting_started/quickstart/index.rst | 72 + .../quickstart/multimodality.rst | 225 + .../quickstart/offload_kv_cache.rst | 480 +++ .../quickstart/share_kv_cache.rst | 270 ++ .../quickstart/standalone_starter.rst | 303 ++ docs/source/index.rst | 203 + docs/source/interacting_with_server.rst | 13 + .../internal_api_server/common_apis.rst | 355 ++ .../internal_api_server/controller_apis.rst | 201 + .../dynamic_backend_management.rst | 181 + .../internal_api_server.rst | 85 + docs/source/internal_api_server/vllm_apis.rst | 1112 +++++ docs/source/kv_cache/async_loading.rst | 132 + docs/source/kv_cache/caching_policies.rst | 13 + docs/source/kv_cache/multiprocess_mode.rst | 12 + docs/source/kv_cache/p2p_sharing.rst | 237 ++ docs/source/kv_cache/storage_backends/3fs.rst | 174 + .../kv_cache/storage_backends/azure.rst | 78 + .../kv_cache/storage_backends/bigtable.rst | 152 + .../kv_cache/storage_backends/cpu_ram.rst | 330 ++ .../storage_backends/custom_backend.rst | 6 + docs/source/kv_cache/storage_backends/dax.rst | 254 ++ docs/source/kv_cache/storage_backends/eic.rst | 45 + docs/source/kv_cache/storage_backends/fs.rst | 194 + docs/source/kv_cache/storage_backends/gds.rst | 275 ++ .../kv_cache/storage_backends/hfbucket.rst | 183 + .../kv_cache/storage_backends/index.rst | 41 + .../kv_cache/storage_backends/infinistore.rst | 208 + .../storage_backends/local_storage.rst | 464 ++ .../source/kv_cache/storage_backends/maru.rst | 118 + .../source/kv_cache/storage_backends/mock.rst | 70 + .../kv_cache/storage_backends/mooncake.rst | 255 ++ .../source/kv_cache/storage_backends/nixl.rst | 275 ++ .../kv_cache/storage_backends/redis.rst | 455 ++ .../source/kv_cache/storage_backends/resp.rst | 456 ++ docs/source/kv_cache/storage_backends/s3.rst | 155 + .../storage_backends/sagemaker_hyperpod.rst | 76 + .../kv_cache/storage_backends/valkey.rst | 205 + .../source/kv_cache/storage_backends/weka.rst | 137 + .../kv_cache_management/check_finish.rst | 11 + docs/source/kv_cache_management/clear.rst | 97 + docs/source/kv_cache_management/compress.rst | 124 + docs/source/kv_cache_management/health.rst | 45 + docs/source/kv_cache_management/index.rst | 122 + docs/source/kv_cache_management/lookup.rst | 98 + docs/source/kv_cache_management/move.rst | 135 + docs/source/kv_cache_management/pin.rst | 98 + .../kv_cache_management/query_worker_info.rst | 73 + .../kv_cache_optimizations/blending.rst | 88 + .../kv_cache_optimizations/cacheblend.rst | 26 + .../compression/cachegen.rst | 32 + .../compression/index.rst | 13 + docs/source/kv_cache_optimizations/index.rst | 11 + .../kv_cache_optimizations/layerwise.rst | 116 + .../segmented_prefill.rst | 6 + docs/source/legacy/index.rst | 58 + docs/source/locale/README.md | 27 + .../api_reference/configurations.po | 1698 ++++++++ .../api_reference/dynamic_connector.po | 108 + .../api_reference/multimodality.po | 193 + .../api_reference/storage_backends.po | 37 + .../locale/zh_CN/LC_MESSAGES/cli/bench.po | 1958 +++++++++ .../zh_CN/LC_MESSAGES/cli/coordinator.po | 208 + .../locale/zh_CN/LC_MESSAGES/cli/describe.po | 244 ++ .../locale/zh_CN/LC_MESSAGES/cli/index.po | 233 + .../locale/zh_CN/LC_MESSAGES/cli/kvcache.po | 157 + .../locale/zh_CN/LC_MESSAGES/cli/ping.po | 123 + .../locale/zh_CN/LC_MESSAGES/cli/query.po | 178 + .../locale/zh_CN/LC_MESSAGES/cli/quota.po | 239 ++ .../locale/zh_CN/LC_MESSAGES/cli/server.po | 258 ++ .../locale/zh_CN/LC_MESSAGES/cli/tool.po | 95 + .../locale/zh_CN/LC_MESSAGES/cli/trace.po | 170 + .../zh_CN/LC_MESSAGES/community/blogs.po | 36 + .../zh_CN/LC_MESSAGES/community/index.po | 31 + .../zh_CN/LC_MESSAGES/community/meetings.po | 143 + .../LC_MESSAGES/controller/freeze_mode.po | 203 + .../zh_CN/LC_MESSAGES/controller/index.po | 126 + .../developer_guide/architecture.po | 260 ++ .../zh_CN/LC_MESSAGES/developer_guide/cli.po | 517 +++ .../developer_guide/contributing.po | 497 +++ .../developer_guide/extending_http_api.po | 211 + .../extending_lmcache/native_connectors.po | 786 ++++ .../remote_storage_plugins.po | 291 ++ .../extending_lmcache/runtime_plugins.po | 185 + .../extending_lmcache/storage_plugins.po | 379 ++ .../LC_MESSAGES/developer_guide/index.po | 31 + .../developer_guide/integration.po | 114 + .../developer_guide/usage/basic_check.po | 218 + .../developer_guide/usage/index.po | 25 + .../usage/usage_stats_collection.po | 123 + .../disaggregated_prefill/nixl/1p1d.po | 362 ++ .../disaggregated_prefill/nixl/index.po | 64 + .../disaggregated_prefill/nixl/xpyd.po | 424 ++ .../disaggregated_prefill/shared_storage.po | 29 + .../zh_CN/LC_MESSAGES/distributed_kv_cache.po | 32 + .../getting_started/benchmarking.po | 267 ++ .../LC_MESSAGES/getting_started/index.po | 32 + .../getting_started/installation.po | 240 ++ .../getting_started/kv_cache_calculator.po | 45 + .../LC_MESSAGES/getting_started/quickstart.po | 492 +++ .../quickstart/disaggregated_prefill.po | 293 ++ .../getting_started/quickstart/index.po | 168 + .../quickstart/multimodality.po | 80 + .../quickstart/offload_kv_cache.po | 362 ++ .../quickstart/share_kv_cache.po | 255 ++ .../quickstart/standalone_starter.po | 383 ++ docs/source/locale/zh_CN/LC_MESSAGES/index.po | 437 ++ .../LC_MESSAGES/interacting_with_server.po | 31 + .../internal_api_server/common_apis.po | 415 ++ .../internal_api_server/controller_apis.po | 317 ++ .../dynamic_backend_management.po | 174 + .../internal_api_server.po | 128 + .../internal_api_server/vllm_apis.po | 1046 +++++ .../LC_MESSAGES/kv_cache/async_loading.po | 232 + .../LC_MESSAGES/kv_cache/caching_policies.po | 54 + .../LC_MESSAGES/kv_cache/multiprocess_mode.po | 38 + .../zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po | 208 + .../kv_cache/storage_backends/3fs.po | 174 + .../kv_cache/storage_backends/bigtable.po | 374 ++ .../kv_cache/storage_backends/cpu_ram.po | 319 ++ .../storage_backends/custom_backend.po | 39 + .../kv_cache/storage_backends/dax.po | 383 ++ .../kv_cache/storage_backends/eic.po | 85 + .../kv_cache/storage_backends/fs.po | 317 ++ .../kv_cache/storage_backends/gds.po | 369 ++ .../kv_cache/storage_backends/hfbucket.po | 297 ++ .../kv_cache/storage_backends/index.po | 43 + .../kv_cache/storage_backends/infinistore.po | 257 ++ .../storage_backends/local_storage.po | 398 ++ .../kv_cache/storage_backends/maru.po | 213 + .../kv_cache/storage_backends/mock.po | 116 + .../kv_cache/storage_backends/mooncake.po | 567 +++ .../kv_cache/storage_backends/nixl.po | 435 ++ .../kv_cache/storage_backends/redis.po | 375 ++ .../kv_cache/storage_backends/resp.po | 716 ++++ .../kv_cache/storage_backends/s3.po | 231 + .../storage_backends/sagemaker_hyperpod.po | 185 + .../kv_cache/storage_backends/valkey.po | 304 ++ .../kv_cache/storage_backends/weka.po | 146 + .../kv_cache_management/check_finish.po | 36 + .../LC_MESSAGES/kv_cache_management/clear.po | 88 + .../kv_cache_management/compress.po | 103 + .../LC_MESSAGES/kv_cache_management/health.po | 69 + .../LC_MESSAGES/kv_cache_management/index.po | 229 + .../LC_MESSAGES/kv_cache_management/lookup.po | 97 + .../LC_MESSAGES/kv_cache_management/move.po | 95 + .../LC_MESSAGES/kv_cache_management/pin.po | 85 + .../kv_cache_management/query_worker_info.po | 77 + .../kv_cache_optimizations/blending.po | 89 + .../kv_cache_optimizations/cacheblend.po | 57 + .../compression/cachegen.po | 63 + .../compression/index.po | 39 + .../kv_cache_optimizations/index.po | 31 + .../kv_cache_optimizations/layerwise.po | 308 ++ .../segmented_prefill.po | 29 + .../locale/zh_CN/LC_MESSAGES/legacy/index.po | 56 + .../locale/zh_CN/LC_MESSAGES/mp/cachegen.po | 32 + .../zh_CN/LC_MESSAGES/mp/configuration.po | 1377 ++++++ .../zh_CN/LC_MESSAGES/mp/coordinator.po | 1180 +++++ .../locale/zh_CN/LC_MESSAGES/mp/deployment.po | 358 ++ .../LC_MESSAGES/mp/disaggregated_prefill.po | 38 + .../LC_MESSAGES/mp/frontend_dashboard.po | 354 ++ .../locale/zh_CN/LC_MESSAGES/mp/http_api.po | 2411 +++++++++++ .../zh_CN/LC_MESSAGES/mp/hybrid_models.po | 546 +++ .../locale/zh_CN/LC_MESSAGES/mp/index.po | 1292 ++++++ .../LC_MESSAGES/mp/kv_cache_management.po | 38 + .../LC_MESSAGES/mp/l2_storage/aerospike.po | 134 + .../zh_CN/LC_MESSAGES/mp/l2_storage/dax.po | 161 + .../LC_MESSAGES/mp/l2_storage/fault_inject.po | 127 + .../mp/l2_storage/file_and_block.po | 32 + .../zh_CN/LC_MESSAGES/mp/l2_storage/fs.po | 67 + .../LC_MESSAGES/mp/l2_storage/fs_native.po | 125 + .../LC_MESSAGES/mp/l2_storage/hfbucket.po | 117 + .../zh_CN/LC_MESSAGES/mp/l2_storage/index.po | 547 +++ .../zh_CN/LC_MESSAGES/mp/l2_storage/mock.po | 43 + .../mp/l2_storage/mooncake_store.po | 169 + .../zh_CN/LC_MESSAGES/mp/l2_storage/nixl.po | 248 ++ .../zh_CN/LC_MESSAGES/mp/l2_storage/plugin.po | 154 + .../LC_MESSAGES/mp/l2_storage/raw_block.po | 185 + .../mp/l2_storage/remote_and_distributed.po | 31 + .../zh_CN/LC_MESSAGES/mp/l2_storage/resp.po | 78 + .../zh_CN/LC_MESSAGES/mp/l2_storage/s3.po | 88 + .../mp/l2_storage/supported_storages.po | 182 + .../LC_MESSAGES/mp/observability/index.po | 455 ++ .../LC_MESSAGES/mp/observability/logs.po | 83 + .../LC_MESSAGES/mp/observability/metrics.po | 1124 +++++ .../LC_MESSAGES/mp/observability/traces.po | 279 ++ .../locale/zh_CN/LC_MESSAGES/mp/operator.po | 1830 ++++++++ .../source/locale/zh_CN/LC_MESSAGES/mp/p2p.po | 373 ++ .../source/locale/zh_CN/LC_MESSAGES/mp/sdk.po | 221 + .../locale/zh_CN/LC_MESSAGES/mp/serde.po | 275 ++ .../LC_MESSAGES/mp/tracing_and_debugging.po | 489 +++ .../LC_MESSAGES/non_kv_cache/encoder_cache.po | 332 ++ .../LC_MESSAGES/non_kv_cache/encodings.po | 33 + .../LC_MESSAGES/non_kv_cache/hidden_states.po | 175 + .../zh_CN/LC_MESSAGES/non_kv_cache/index.po | 29 + .../production/docker_deployment.po | 107 + .../production/dynamo_coordination.po | 31 + .../zh_CN/LC_MESSAGES/production/index.po | 29 + .../production/kubernetes_deployment.po | 58 + .../LC_MESSAGES/production/kv_cache_events.po | 212 + .../observability/chunk_statistics.po | 589 +++ .../production/observability/frontend.po | 234 + .../observability/health_monitor.po | 495 +++ .../production/observability/index.po | 33 + .../observability/internal_api_server.po | 145 + .../production/observability/metrics.po | 899 ++++ .../observability/periodic_thread_api.po | 241 ++ .../production/observability/vllm_endpoint.po | 469 ++ .../production/performance_tuning.po | 140 + .../LC_MESSAGES/recipes/deepseek_v4_flash.po | 166 + .../zh_CN/LC_MESSAGES/recipes/devstral.po | 139 + .../zh_CN/LC_MESSAGES/recipes/gemma3.po | 163 + .../zh_CN/LC_MESSAGES/recipes/gemma4.po | 179 + .../zh_CN/LC_MESSAGES/recipes/glm5_2.po | 150 + .../zh_CN/LC_MESSAGES/recipes/gpt_oss.po | 145 + .../locale/zh_CN/LC_MESSAGES/recipes/index.po | 257 ++ .../locale/zh_CN/LC_MESSAGES/recipes/llama.po | 188 + .../zh_CN/LC_MESSAGES/recipes/minimax_m2.po | 169 + .../zh_CN/LC_MESSAGES/recipes/minimax_m3.po | 140 + .../zh_CN/LC_MESSAGES/recipes/mixtral.po | 148 + .../locale/zh_CN/LC_MESSAGES/recipes/phi3.po | 148 + .../locale/zh_CN/LC_MESSAGES/recipes/qwen3.po | 169 + .../zh_CN/LC_MESSAGES/recipes/qwen3_5.po | 262 ++ .../recipes/uniform_attention_models.po | 268 ++ docs/source/mp/architecture.png | Bin 0 -> 269876 bytes docs/source/mp/cachegen.rst | 8 + docs/source/mp/configuration.rst | 614 +++ docs/source/mp/coordinator.rst | 925 ++++ docs/source/mp/deployment.rst | 270 ++ docs/source/mp/disaggregated_prefill.rst | 225 + docs/source/mp/frontend_dashboard.rst | 223 + docs/source/mp/http_api.rst | 1676 ++++++++ docs/source/mp/hybrid_models.rst | 281 ++ docs/source/mp/index.rst | 642 +++ docs/source/mp/kv_cache_management.rst | 13 + docs/source/mp/l2_storage/aerospike.rst | 63 + docs/source/mp/l2_storage/dax.rst | 101 + docs/source/mp/l2_storage/fault_inject.rst | 76 + docs/source/mp/l2_storage/file_and_block.rst | 13 + docs/source/mp/l2_storage/fs.rst | 32 + docs/source/mp/l2_storage/fs_native.rst | 82 + docs/source/mp/l2_storage/hfbucket.rst | 52 + docs/source/mp/l2_storage/index.rst | 343 ++ docs/source/mp/l2_storage/mock.rst | 14 + docs/source/mp/l2_storage/mooncake_store.rst | 111 + docs/source/mp/l2_storage/nixl.rst | 135 + docs/source/mp/l2_storage/plugin.rst | 88 + docs/source/mp/l2_storage/raw_block.rst | 76 + .../mp/l2_storage/remote_and_distributed.rst | 14 + docs/source/mp/l2_storage/resp.rst | 34 + docs/source/mp/l2_storage/s3.rst | 40 + .../mp/l2_storage/supported_storages.rst | 65 + docs/source/mp/observability/index.rst | 284 ++ docs/source/mp/observability/logs.rst | 34 + docs/source/mp/observability/metrics.rst | 585 +++ docs/source/mp/observability/traces.rst | 192 + docs/source/mp/operator.rst | 1336 ++++++ docs/source/mp/p2p.rst | 250 ++ docs/source/mp/sdk.rst | 142 + docs/source/mp/serde.rst | 191 + docs/source/mp/tracing_and_debugging.rst | 330 ++ docs/source/non_kv_cache/encoder_cache.rst | 177 + docs/source/non_kv_cache/encodings.rst | 9 + docs/source/non_kv_cache/hidden_states.rst | 116 + docs/source/non_kv_cache/index.rst | 10 + docs/source/production/docker_deployment.rst | 76 + .../source/production/dynamo_coordination.rst | 7 + docs/source/production/index.rst | 13 + .../production/kubernetes_deployment.rst | 37 + docs/source/production/kv_cache_events.rst | 160 + .../observability/chunk_statistics.rst | 381 ++ .../production/observability/frontend.rst | 168 + .../observability/health_monitor.rst | 188 + .../source/production/observability/index.rst | 20 + .../observability/internal_api_server.rst | 70 + .../production/observability/metrics.rst | 394 ++ .../observability/periodic_thread_api.rst | 180 + .../observability/vllm_endpoint.rst | 199 + docs/source/production/performance_tuning.rst | 121 + docs/source/recipes/deepseek_v4_flash.rst | 107 + docs/source/recipes/devstral.rst | 80 + docs/source/recipes/gemma3.rst | 95 + docs/source/recipes/gemma4.rst | 100 + docs/source/recipes/glm5_2.rst | 105 + docs/source/recipes/gpt_oss.rst | 93 + docs/source/recipes/index.rst | 22 + docs/source/recipes/llama.rst | 132 + docs/source/recipes/minimax_m2.rst | 119 + docs/source/recipes/minimax_m3.rst | 96 + docs/source/recipes/mixtral.rst | 92 + docs/source/recipes/phi3.rst | 90 + docs/source/recipes/qwen3.rst | 124 + docs/source/recipes/qwen3_5.rst | 137 + .../recipes/uniform_attention_models.rst | 117 + examples/README.md | 94 + examples/agents/prefix_analysis.py | 471 ++ examples/agents/requirements.txt | 4 + examples/basic_check/README.md | 47 + examples/basic_check/example_config.yaml | 13 + examples/blend_kv/README.md | 38 + examples/blend_kv/batched_kv.py | 51 + examples/blend_kv/batched_tp_kv.py | 56 + examples/blend_kv/blend_kv.py | 49 + examples/blend_kv/chunk1.txt | 62 + examples/blend_kv/chunk2.txt | 114 + examples/blend_kv/example_blending.yaml | 5 + examples/blend_kv/online_kv.py | 129 + examples/blend_kv/tp_kv.py | 52 + examples/blend_kv_v1/README.md | 19 + examples/blend_kv_v1/blend.py | 217 + examples/cache_controller/README.md | 15 + examples/cache_controller/clear/README.md | 46 + examples/cache_controller/clear/example.yaml | 13 + examples/cache_controller/compress/README.md | 86 + .../cache_controller/compress/example.yaml | 13 + examples/cache_controller/health/README.md | 26 + examples/cache_controller/health/example.yaml | 13 + examples/cache_controller/lookup/README.md | 64 + examples/cache_controller/lookup/example.yaml | 13 + examples/cache_controller/move/README.md | 68 + examples/cache_controller/move/instance1.yaml | 17 + examples/cache_controller/move/instance2.yaml | 17 + examples/cache_controller/pin/README.md | 62 + examples/cache_controller/pin/example.yaml | 13 + examples/cache_interface/README.md | 89 + examples/cache_interface/example.yaml | 3 + examples/cache_with_configs/README.md | 42 + examples/cache_with_configs/example.yaml | 4 + examples/chunk_statistics/README.md | 258 ++ .../chunk_statistics/analyze_chunk_hashes.py | 423 ++ examples/chunk_statistics/auto_stop.yaml | 20 + examples/chunk_statistics/file_hash.yaml | 15 + .../chunk_statistics/memory_bloom_filter.yaml | 16 + examples/disagg_prefill/1p1d/README.md | 89 + .../1p1d/configs/lmcache-decoder-config.yaml | 49 + ...lmcache-decoder-pd-with-remote-config.yaml | 58 + .../configs/lmcache-prefiller-config.yaml | 47 + ...cache-prefiller-pd-with-remote-config.yaml | 55 + .../1p1d/disagg_example_1p1d.sh | 176 + .../1p1d/disagg_vllm_launcher.sh | 62 + examples/disagg_prefill/README.md | 81 + .../disagg_prefill/disagg_proxy_server.py | 785 ++++ examples/disagg_prefill/xpyd/README.md | 157 + .../configs/lmcache-decoder-1-config.yaml | 49 + ...cache-decoder-1-pd-with-remote-config.yaml | 58 + .../configs/lmcache-decoder-2-config.yaml | 49 + ...cache-decoder-2-pd-with-remote-config.yaml | 58 + .../configs/lmcache-prefiller-config.yaml | 50 + ...cache-prefiller-pd-with-remote-config.yaml | 55 + .../xpyd/disagg_example_xpyd.sh | 181 + .../xpyd/disagg_vllm_launcher.sh | 96 + examples/disagg_prefill_mp/README.md | 65 + examples/disagg_prefill_mp/architecture.png | Bin 0 -> 134277 bytes .../disagg_prefill_mp/disagg_proxy_server.py | 461 ++ examples/dynamo_integration/README.md | 65 + .../deploy/agg_lmcache_mp.yaml | 56 + .../deploy/disagg_lmcache_mp.yaml | 97 + .../deploy/lmcache_engine.yaml | 13 + .../launch/agg_lmcache_mp.sh | 62 + .../launch/disagg_lmcache_mp.sh | 80 + examples/frontend/README.md | 19 + examples/frontend/chat_session.py | 69 + examples/frontend/example.yaml | 4 + examples/frontend/ffmpeg.txt | 2069 +++++++++ examples/frontend/frontend.py | 54 + examples/kubernetes/health_probe.py | 84 + examples/kv_cache_calculator/README.md | 109 + .../kv_cache_calculator/generate_config.py | 83 + .../kv_cache_calculator.html | 526 +++ examples/kv_cache_calculator/modelconfig.json | 201 + examples/kv_cache_calculator/requirement.txt | 1 + examples/kv_cache_reuse/README.md | 8 + .../kv_cache_reuse/local_backends/README.md | 14 + .../kv_cache_reuse/local_backends/offload.py | 152 + .../local_backends/rust_backend_offload.py | 179 + .../kv_cache_reuse/remote_backends/README.md | 8 + .../remote_backends/external/README.md | 41 + .../external/backend_type.yaml | 11 + .../remote_backends/fs/example.yaml | 9 + .../remote_backends/infinistore/README.md | 45 + .../infinistore/backend_type.yaml | 2 + .../remote_backends/mock/README.md | 25 + .../remote_backends/mock/example.yaml | 4 + .../remote_backends/mooncakestore/README.md | 29 + .../mooncakestore/mooncake.yaml | 15 + .../remote_backends/resp/README.md | 137 + .../resp/benchmark_resp_client.py | 154 + .../remote_backends/resp/resp.yaml | 16 + .../remote_backends/s3/README.md | 34 + .../remote_backends/s3/example.yaml | 18 + .../valkey/VALKEY_CONNECTOR_BENCHMARKING.md | 293 ++ .../remote_backends/valkey/benchmark_l2.py | 304 ++ .../remote_backends/valkey/valkey.yaml | 9 + .../share_across_instances/README.md | 7 + .../centralized_sharing/README.md | 51 + .../centralized_sharing/example.yaml | 4 + .../p2p_sharing/README.md | 55 + .../p2p_sharing/example1.yaml | 24 + .../p2p_sharing/example2.yaml | 21 + examples/kvcache_sdk/README.md | 56 + examples/kvcache_sdk/e2e_kv_edit.ipynb | 423 ++ .../lmc_external_l2_adapter/pyproject.toml | 23 + .../scripts/install_and_test.sh | 46 + .../src/lmc_external_l2_adapter/__init__.py | 17 + .../src/lmc_external_l2_adapter/adapter.py | 351 ++ .../tests/test_plugin.py | 415 ++ .../csrc/connector.cpp | 249 ++ .../csrc/connector.h | 81 + .../csrc/pybind.cpp | 21 + .../pyproject.toml | 24 + .../lmc_external_native_connector/setup.py | 58 + .../lmc_external_native_connector/__init__.py | 28 + .../connector.py | 66 + examples/mp_runtime_plugins/README.md | 93 + examples/mp_runtime_plugins/mp_heartbeat.sh | 37 + examples/mp_runtime_plugins/mp_plugin.py | 94 + examples/multi_process/lmcache-daemonset.yaml | 52 + examples/multi_process/vllm-deployment.yaml | 63 + examples/observability/README.md | 152 + examples/observability/docker-compose.yml | 70 + .../provisioning/dashboards/dashboards.yml | 9 + .../provisioning/dashboards/lmcache.json | 2791 ++++++++++++ .../provisioning/datasources/datasources.yml | 20 + examples/observability/otel-collector.yml | 25 + examples/observability/prometheus.yml | 8 + examples/observability/start-server.sh | 78 + examples/observability/tempo.yml | 42 + examples/online_session/README.md | 157 + examples/online_session/bench_ttft_sweep.sh | 24 + examples/online_session/example.yaml | 4 + examples/online_session/ffmpeg.txt | 2069 +++++++++ .../openai_chat_completion_client.py | 280 ++ examples/p2p/README.md | 210 + examples/p2p/send_request.py | 63 + examples/redis_lookup/README.md | 152 + examples/remote_config_server/README.md | 123 + .../remote_config_server/config_server.py | 155 + examples/remote_config_server/example.yaml | 15 + examples/runtime_plugins/README.md | 73 + examples/runtime_plugins/all_plugin.sh | 21 + .../runtime_plugins/scheduler_foo_plugin.py | 46 + examples/runtime_plugins/worker_0_test.sh | 21 + examples/serde/fp8/README.md | 94 + examples/serde/fp8/run_serde_fp8_example.sh | 238 ++ examples/serde/turboquant/README.md | 99 + .../serde/turboquant/bench_serde_baselines.py | 248 ++ examples/serde/turboquant/bench_turboquant.py | 213 + examples/sgl_integration/README.md | 35 + examples/token_dropping/token_dropping.ipynb | 595 +++ examples/token_dropping/utils.py | 240 ++ format.sh | 7 + lmcache/__init__.py | 36 + lmcache/banner.py | 123 + lmcache/cli/__init__.py | 1 + lmcache/cli/commands/__init__.py | 40 + lmcache/cli/commands/base.py | 254 ++ lmcache/cli/commands/bench/__init__.py | 21 + .../commands/bench/engine_bench/__init__.py | 49 + .../commands/bench/engine_bench/command.py | 617 +++ .../cli/commands/bench/engine_bench/config.py | 251 ++ .../engine_bench/interactive/__init__.py | 381 ++ .../engine_bench/interactive/config.json | 10 + .../bench/engine_bench/interactive/schema.py | 418 ++ .../bench/engine_bench/interactive/state.py | 331 ++ .../engine_bench/interactive/terminal.py | 349 ++ .../commands/bench/engine_bench/progress.py | 145 + .../bench/engine_bench/request_sender.py | 256 ++ .../cli/commands/bench/engine_bench/stats.py | 275 ++ .../bench/engine_bench/workloads/__init__.py | 206 + .../bench/engine_bench/workloads/base.py | 132 + .../workloads/long_doc_permutator.py | 415 ++ .../engine_bench/workloads/long_doc_qa.py | 290 ++ .../workloads/multi_round_chat.py | 337 ++ .../workloads/prefix_suffix_tuner.py | 637 +++ .../engine_bench/workloads/random_prefill.py | 178 + .../bench/l2_adapter_bench/__init__.py | 38 + .../bench/l2_adapter_bench/command.py | 711 ++++ .../commands/bench/l2_adapter_bench/data.py | 195 + .../bench/l2_adapter_bench/profiling.py | 359 ++ .../commands/bench/l2_adapter_bench/result.py | 174 + .../commands/bench/l2_adapter_bench/runner.py | 332 ++ .../commands/bench/server_bench/__init__.py | 38 + .../commands/bench/server_bench/command.py | 700 +++ .../commands/bench/server_bench/helpers.py | 1158 +++++ lmcache/cli/commands/coordinator.py | 203 + lmcache/cli/commands/describe.py | 489 +++ lmcache/cli/commands/kvcache.py | 133 + lmcache/cli/commands/mock.py | 75 + lmcache/cli/commands/ping.py | 113 + lmcache/cli/commands/query/__init__.py | 20 + lmcache/cli/commands/query/_prompt.py | 134 + lmcache/cli/commands/query/_request.py | 368 ++ lmcache/cli/commands/query/engine_command.py | 109 + lmcache/cli/commands/query/kvcache_command.py | 25 + lmcache/cli/commands/quota/__init__.py | 20 + lmcache/cli/commands/quota/_helpers.py | 78 + lmcache/cli/commands/quota/delete_command.py | 50 + lmcache/cli/commands/quota/get_command.py | 54 + lmcache/cli/commands/quota/list_command.py | 52 + lmcache/cli/commands/quota/set_command.py | 63 + lmcache/cli/commands/server.py | 109 + lmcache/cli/commands/tool/__init__.py | 20 + .../commands/tool/cache_simulator/__init__.py | 21 + .../cache_simulator/gen_dataset_command.py | 48 + .../tool/cache_simulator/simulate_command.py | 46 + .../tool/cache_simulator/sweep_command.py | 45 + .../tool/transfer_channel_benchmark.py | 76 + lmcache/cli/commands/trace/__init__.py | 24 + lmcache/cli/commands/trace/_dispatch.py | 249 ++ lmcache/cli/commands/trace/_driver.py | 372 ++ lmcache/cli/commands/trace/_stats.py | 289 ++ lmcache/cli/commands/trace/info_command.py | 56 + lmcache/cli/commands/trace/replay_command.py | 326 ++ lmcache/cli/documents/lmcache.txt | 11 + lmcache/cli/main.py | 44 + lmcache/cli/metrics/__init__.py | 29 + lmcache/cli/metrics/formatter.py | 171 + lmcache/cli/metrics/handler.py | 94 + lmcache/cli/metrics/metrics.py | 161 + lmcache/cli/metrics/section.py | 77 + lmcache/connections.py | 173 + lmcache/integration/__init__.py | 2 + lmcache/integration/base_service_factory.py | 169 + .../integration/request_telemetry/__init__.py | 1 + lmcache/integration/request_telemetry/base.py | 51 + .../integration/request_telemetry/factory.py | 113 + .../integration/request_telemetry/fastapi.py | 109 + lmcache/integration/request_telemetry/noop.py | 35 + lmcache/integration/sglang/__init__.py | 2 + .../sglang/multi_process_adapter.py | 555 +++ lmcache/integration/sglang/sglang_adapter.py | 376 ++ lmcache/integration/sglang/utils.py | 31 + lmcache/integration/tensorrt_llm/__init__.py | 78 + .../tensorrt_llm/tensorrt_adapter.py | 377 ++ .../tensorrt_llm/tensorrt_mp_adapter.py | 488 +++ lmcache/integration/tensorrt_llm/utils.py | 94 + lmcache/integration/vllm/__init__.py | 1 + .../integration/vllm/kv_cache_group_edits.py | 402 ++ lmcache/integration/vllm/kv_cache_groups.py | 192 + .../integration/vllm/lmcache_connector_v1.py | 208 + .../vllm/lmcache_connector_v1_085.py | 145 + .../integration/vllm/lmcache_mp_connector.py | 1436 +++++++ .../vllm/lmcache_mp_connector_0180.py | 1080 +++++ .../vllm/lmcache_mp_connector_0201.py | 1201 ++++++ .../vllm/tests/test_mm_hash_utils.py | 112 + lmcache/integration/vllm/utils.py | 419 ++ lmcache/integration/vllm/vllm_ec_adapter.py | 194 + .../vllm/vllm_multi_process_adapter.py | 1802 ++++++++ .../integration/vllm/vllm_service_factory.py | 339 ++ lmcache/integration/vllm/vllm_v1_adapter.py | 1942 +++++++++ lmcache/lmcache_frontend/__init__.py | 2 + lmcache/lmcache_frontend/app.py | 1036 +++++ lmcache/lmcache_frontend/config.json | 1 + lmcache/lmcache_frontend/heartbeat.py | 233 + .../lmcache_mp_plugin/__init__.py | 1 + .../lmcache_mp_frontend_plugin.py | 176 + .../run_mp_server_with_frontend.sh | 21 + lmcache/lmcache_frontend/static/css/style.css | 190 + lmcache/lmcache_frontend/static/img/logo.png | Bin 0 -> 39745 bytes lmcache/lmcache_frontend/static/index.html | 239 ++ lmcache/lmcache_frontend/static/js/app.js | 1189 ++++++ lmcache/logging.py | 107 + lmcache/native_storage_ops.pyi | 349 ++ lmcache/observability.py | 2020 +++++++++ lmcache/python_ops_fallback.py | 2764 ++++++++++++ lmcache/sdk/__init__.py | 10 + lmcache/sdk/batch.py | 321 ++ lmcache/sdk/kvcache.py | 539 +++ lmcache/sdk/stream.py | 326 ++ lmcache/sdk/wrapper/__init__.py | 9 + lmcache/sdk/wrapper/contiguous.py | 75 + lmcache/storage_backend/serde/__init__.py | 1 + .../storage_backend/serde/cachegen_basics.py | 210 + .../storage_backend/serde/cachegen_decoder.py | 212 + .../storage_backend/serde/cachegen_encoder.py | 397 ++ lmcache/storage_backend/serde/serde.py | 75 + lmcache/tools/__init__.py | 1 + lmcache/tools/cache_simulator/README.md | 392 ++ lmcache/tools/cache_simulator/__init__.py | 1 + .../cache_simulator/docs/simulate_example.png | Bin 0 -> 286842 bytes .../cache_simulator/docs/sweep_example.png | Bin 0 -> 78797 bytes .../cache_simulator/gen_bench_dataset.py | 360 ++ lmcache/tools/cache_simulator/lru_cache.py | 124 + .../tools/cache_simulator/plot_hit_rate.py | 231 + lmcache/tools/cache_simulator/simulator.py | 795 ++++ lmcache/tools/controller_benchmark/README.md | 161 + .../tools/controller_benchmark/__init__.py | 1 + .../tools/controller_benchmark/__main__.py | 331 ++ .../tools/controller_benchmark/benchmark.py | 660 +++ lmcache/tools/controller_benchmark/config.py | 44 + .../tools/controller_benchmark/constants.py | 10 + .../controller_benchmark/handlers/__init__.py | 31 + .../controller_benchmark/handlers/admit.py | 52 + .../controller_benchmark/handlers/base.py | 47 + .../handlers/deregister.py | 49 + .../controller_benchmark/handlers/evict.py | 52 + .../handlers/heartbeat.py | 56 + .../handlers/p2p_lookup.py | 47 + .../controller_benchmark/handlers/register.py | 56 + lmcache/tools/mp_status_viewer/__init__.py | 1 + lmcache/tools/mp_status_viewer/__main__.py | 95 + lmcache/tools/simple_discover_service.py | 118 + .../transfer_channel_benchmark/README.md | 128 + .../transfer_channel_benchmark/__init__.py | 1 + .../transfer_channel_benchmark/__main__.py | 39 + .../transfer_channel_benchmark/benchmark.py | 385 ++ .../transfer_channel_benchmark/config.py | 252 ++ lmcache/usage_context.py | 24 + lmcache/usage_telemetry/README.md | 137 + lmcache/usage_telemetry/__init__.py | 72 + lmcache/usage_telemetry/context.py | 176 + lmcache/usage_telemetry/continuous.py | 203 + lmcache/usage_telemetry/env_probe.py | 139 + lmcache/usage_telemetry/guard.py | 49 + lmcache/usage_telemetry/identity.py | 107 + lmcache/usage_telemetry/messages.py | 286 ++ lmcache/usage_telemetry/mp.py | 98 + lmcache/usage_telemetry/transport.py | 97 + lmcache/utils.py | 781 ++++ lmcache/v1/__init__.py | 2 + lmcache/v1/api_server/__init__.py | 2 + lmcache/v1/api_server/__main__.py | 537 +++ lmcache/v1/basic_check.py | 112 + lmcache/v1/cache_controller/__init__.py | 9 + .../v1/cache_controller/commands/__init__.py | 15 + lmcache/v1/cache_controller/commands/base.py | 35 + .../v1/cache_controller/commands/full_sync.py | 49 + lmcache/v1/cache_controller/config.py | 176 + .../v1/cache_controller/controller_manager.py | 535 +++ .../cache_controller/controllers/__init__.py | 11 + .../controllers/full_sync_tracker.py | 473 +++ .../controllers/kv_controller.py | 439 ++ .../controllers/registration_controller.py | 282 ++ lmcache/v1/cache_controller/executor.py | 463 ++ .../frontend/static/css/style.css | 201 + .../frontend/static/img/logo.png | Bin 0 -> 39745 bytes .../frontend/static/index.html | 234 + .../frontend/static/js/controller_app.js | 660 +++ .../v1/cache_controller/full_sync_sender.py | 475 +++ lmcache/v1/cache_controller/locks.py | 149 + lmcache/v1/cache_controller/message.py | 828 ++++ lmcache/v1/cache_controller/observability.py | 208 + lmcache/v1/cache_controller/utils.py | 679 +++ lmcache/v1/cache_controller/worker.py | 665 +++ lmcache/v1/cache_engine.py | 2193 ++++++++++ lmcache/v1/cache_interface.py | 19 + lmcache/v1/check/__init__.py | 74 + lmcache/v1/check/check_mode_gen.py | 86 + .../v1/check/check_mode_test_l2_adapter.py | 327 ++ lmcache/v1/check/check_mode_test_remote.py | 155 + .../check/check_mode_test_storage_manager.py | 142 + lmcache/v1/check/utils.py | 566 +++ lmcache/v1/compute/__init__.py | 2 + lmcache/v1/compute/attention/__init__.py | 1 + lmcache/v1/compute/attention/abstract.py | 39 + lmcache/v1/compute/attention/flash_attn.py | 139 + .../compute/attention/flash_infer_sparse.py | 285 ++ lmcache/v1/compute/attention/metadata.py | 176 + .../attention/triton_kernels/__init__.py | 13 + .../triton_kernels/block_sparse_attention.py | 559 +++ lmcache/v1/compute/attention/triton_sparse.py | 145 + lmcache/v1/compute/attention/utils.py | 101 + lmcache/v1/compute/blend/__init__.py | 7 + lmcache/v1/compute/blend/blender.py | 169 + lmcache/v1/compute/blend/metadata.py | 34 + lmcache/v1/compute/blend/utils.py | 63 + lmcache/v1/compute/models/__init__.py | 0 lmcache/v1/compute/models/base.py | 156 + lmcache/v1/compute/models/llama.py | 9 + lmcache/v1/compute/models/qwen3.py | 24 + lmcache/v1/compute/models/utils.py | 68 + lmcache/v1/compute/positional_encoding.py | 204 + lmcache/v1/config.py | 1234 ++++++ lmcache/v1/config_base.py | 848 ++++ lmcache/v1/distributed/api.py | 404 ++ lmcache/v1/distributed/bitmap_ops/README.md | 1 + lmcache/v1/distributed/bitmap_ops/__init__.py | 31 + lmcache/v1/distributed/bitmap_ops/fold.py | 339 ++ lmcache/v1/distributed/config.py | 622 +++ lmcache/v1/distributed/error.py | 64 + lmcache/v1/distributed/eviction.py | 192 + .../distributed/eviction_policy/__init__.py | 25 + .../v1/distributed/eviction_policy/factory.py | 32 + .../eviction_policy/isolated_lru.py | 191 + lmcache/v1/distributed/eviction_policy/lru.py | 244 ++ .../v1/distributed/eviction_policy/noop.py | 50 + lmcache/v1/distributed/internal_api.py | 209 + lmcache/v1/distributed/l1_manager.py | 894 ++++ .../v1/distributed/l2_adapters/__init__.py | 68 + .../l2_adapters/aerospike_l2_adapter.py | 239 ++ lmcache/v1/distributed/l2_adapters/base.py | 552 +++ lmcache/v1/distributed/l2_adapters/config.py | 512 +++ .../distributed/l2_adapters/dax_l2_adapter.py | 1581 +++++++ lmcache/v1/distributed/l2_adapters/factory.py | 205 + .../l2_adapters/fault_inject_l2_adapter.py | 464 ++ .../distributed/l2_adapters/fs_l2_adapter.py | 764 ++++ .../l2_adapters/fs_native_l2_adapter.py | 175 + .../l2_adapters/hfbucket_l2_adapter.py | 905 ++++ .../l2_adapters/mock_l2_adapter.py | 525 +++ .../l2_adapters/mooncake_store_l2_adapter.py | 215 + .../native_connector_l2_adapter.py | 536 +++ .../l2_adapters/native_plugin_l2_adapter.py | 199 + .../nixl_store_dynamic_l2_adapter.py | 886 ++++ .../l2_adapters/nixl_store_l2_adapter.py | 1011 +++++ .../distributed/l2_adapters/p2p_l2_adapter.py | 406 ++ .../l2_adapters/plugin_l2_adapter.py | 214 + .../l2_adapters/raw_block_l2_adapter.py | 803 ++++ .../l2_adapters/reconfiguration.py | 64 + .../l2_adapters/resp_l2_adapter.py | 176 + .../distributed/l2_adapters/s3_l2_adapter.py | 1296 ++++++ .../distributed/l2_adapters/serde_wrapper.py | 654 +++ .../v1/distributed/memory_manager/__init__.py | 30 + .../devdax_l1_memory_manager.py | 76 + .../memory_manager/gds_l1_memory_manager.py | 158 + .../memory_manager/l1_manager_protocol.py | 50 + .../memory_manager/l1_memory_manager.py | 216 + lmcache/v1/distributed/quota_manager.py | 149 + lmcache/v1/distributed/serde/__init__.py | 63 + lmcache/v1/distributed/serde/asym_k16_v8.py | 537 +++ .../v1/distributed/serde/async_processor.py | 233 + lmcache/v1/distributed/serde/base.py | 215 + lmcache/v1/distributed/serde/factory.py | 69 + lmcache/v1/distributed/serde/fp8.py | 103 + lmcache/v1/distributed/serde/multi.py | 293 ++ .../distributed/serde/turboquant/__init__.py | 15 + .../serde/turboquant/decode_kernel.py | 173 + .../serde/turboquant/store_kernel.py | 445 ++ .../serde/turboquant/turboquant.py | 896 ++++ lmcache/v1/distributed/serde/utils.py | 55 + lmcache/v1/distributed/storage_controller.py | 39 + .../storage_controllers/__init__.py | 43 + .../storage_controllers/adapter_lifecycle.py | 45 + .../eviction_controller.py | 410 ++ .../prefetch_controller.py | 1263 ++++++ .../storage_controllers/prefetch_policy.py | 193 + .../storage_controllers/store_controller.py | 797 ++++ .../storage_controllers/store_policy.py | 213 + lmcache/v1/distributed/storage_manager.py | 1136 +++++ lmcache/v1/distributed/tiers.py | 23 + .../distributed/transfer_channel/__init__.py | 105 + .../distributed/transfer_channel/abstract.py | 166 + .../v1/distributed/transfer_channel/api.py | 47 + .../distributed/transfer_channel/factory.py | 100 + .../transfer_channel/impl/__init__.py | 9 + .../transfer_channel/impl/nixl_impl.py | 626 +++ lmcache/v1/ec_engine.py | 202 + lmcache/v1/event_manager.py | 145 + lmcache/v1/exceptions/__init__.py | 16 + lmcache/v1/gpu_connector/__init__.py | 247 ++ lmcache/v1/gpu_connector/_cufile_async.py | 401 ++ lmcache/v1/gpu_connector/_gds_async.py | 46 + lmcache/v1/gpu_connector/_hipfile_async.py | 532 +++ lmcache/v1/gpu_connector/gds_context.py | 410 ++ lmcache/v1/gpu_connector/gpu_connectors.py | 2222 ++++++++++ lmcache/v1/gpu_connector/gpu_ops.py | 146 + lmcache/v1/gpu_connector/hpu_connector.py | 337 ++ .../v1/gpu_connector/kv_format/__init__.py | 30 + .../v1/gpu_connector/kv_format/contiguity.py | 144 + .../v1/gpu_connector/kv_format/detection.py | 47 + .../kv_format/detectors/__init__.py | 11 + .../gpu_connector/kv_format/detectors/base.py | 52 + .../kv_format/detectors/registry.py | 43 + .../kv_format/detectors/sglang.py | 67 + .../kv_format/detectors/trtllm.py | 61 + .../gpu_connector/kv_format/detectors/vllm.py | 72 + .../gpu_connector/kv_format/specs/__init__.py | 22 + .../v1/gpu_connector/kv_format/specs/base.py | 178 + .../kv_format/specs/nb_nl_two_bs_nh_hs.py | 58 + .../kv_format/specs/nb_nl_two_nh_bs_hs.py | 59 + .../kv_format/specs/nl_x_nb_bs_hs.py | 57 + .../kv_format/specs/nl_x_nb_nh_bs_two_hs.py | 62 + .../kv_format/specs/nl_x_nb_two_bs_nh_hs.py | 60 + .../kv_format/specs/nl_x_nb_two_nh_bs_hs.py | 61 + .../kv_format/specs/nl_x_nbbs_one_hs.py | 62 + .../kv_format/specs/nl_x_two_nb_bs_nh_hs.py | 60 + .../kv_format/specs/nl_x_two_nb_nh_bs_hs.py | 60 + .../gpu_connector/kv_format/specs/registry.py | 56 + .../kv_format/specs/two_x_nl_x_nb_bs_nh_hs.py | 61 + .../kv_format/specs/two_x_nl_x_nbbs_nh_hs.py | 65 + lmcache/v1/gpu_connector/kv_format/types.py | 52 + .../v1/gpu_connector/mock_gpu_connector.py | 67 + lmcache/v1/gpu_connector/musa_connectors.py | 834 ++++ lmcache/v1/gpu_connector/utils.py | 767 ++++ lmcache/v1/gpu_connector/xpu_connectors.py | 1796 ++++++++ lmcache/v1/health_monitor/__init__.py | 1 + lmcache/v1/health_monitor/base.py | 568 +++ lmcache/v1/health_monitor/checks/__init__.py | 1 + .../checks/remote_backend_check.py | 308 ++ lmcache/v1/health_monitor/constants.py | 36 + lmcache/v1/hidden_state_store.py | 364 ++ lmcache/v1/internal_api_server/__init__.py | 0 .../v1/internal_api_server/api_registry.py | 52 + lmcache/v1/internal_api_server/api_server.py | 134 + .../v1/internal_api_server/common/__init__.py | 1 + .../v1/internal_api_server/common/env_api.py | 22 + .../common/loglevel_api.py | 57 + .../internal_api_server/common/metrics_api.py | 29 + .../common/periodic_thread_api.py | 138 + .../common/run_script_api.py | 83 + .../internal_api_server/common/thread_api.py | 63 + .../controller/__init__.py | 1 + .../controller/key_stats_api.py | 81 + .../controller/worker_info_api.py | 136 + lmcache/v1/internal_api_server/utils.py | 43 + .../v1/internal_api_server/vllm/__init__.py | 1 + .../internal_api_server/vllm/backend_api.py | 221 + .../v1/internal_api_server/vllm/bypass_api.py | 204 + .../v1/internal_api_server/vllm/cache_api.py | 897 ++++ .../vllm/chunk_statistics_api.py | 141 + .../v1/internal_api_server/vllm/conf_api.py | 147 + .../v1/internal_api_server/vllm/freeze_api.py | 172 + .../internal_api_server/vllm/hot_cache_api.py | 184 + .../internal_api_server/vllm/inference_api.py | 65 + .../vllm/load_fs_chunks_api.py | 320 ++ .../v1/internal_api_server/vllm/lookup_api.py | 145 + .../internal_api_server/vllm/version_api.py | 25 + lmcache/v1/kv_codec/__init__.py | 63 + lmcache/v1/kv_codec/asym_k16_v8.py | 473 +++ lmcache/v1/kv_codec/encoded_kv.py | 403 ++ lmcache/v1/kv_codec/errors.py | 39 + lmcache/v1/kv_layer_groups.py | 872 ++++ lmcache/v1/lookup_client/__init__.py | 25 + lmcache/v1/lookup_client/abstract_client.py | 77 + .../v1/lookup_client/async_lookup_message.py | 50 + .../chunk_statistics_lookup_client.py | 209 + lmcache/v1/lookup_client/factory.py | 247 ++ .../lookup_client/hit_limit_lookup_client.py | 86 + .../lmcache_async_lookup_client.py | 407 ++ .../v1/lookup_client/lmcache_lookup_client.py | 293 ++ .../lmcache_lookup_client_bypass.py | 99 + .../lookup_client/mooncake_lookup_client.py | 96 + .../record_strategies/__init__.py | 69 + .../lookup_client/record_strategies/base.py | 327 ++ .../record_strategies/file_hash.py | 130 + .../record_strategies/memory_bloom_filter.py | 81 + lmcache/v1/manager.py | 539 +++ lmcache/v1/memory_allocators/__init__.py | 45 + .../ad_hoc_memory_allocator.py | 152 + .../v1/memory_allocators/buffer_allocator.py | 112 + .../cu_file_memory_allocator.py | 38 + .../devdax_memory_allocator.py | 501 +++ .../memory_allocators/gpu_memory_allocator.py | 148 + .../hip_file_memory_allocator.py | 34 + .../host_memory_allocator.py | 135 + .../lazy_memory_allocator.py | 334 ++ .../mixed_memory_allocator.py | 276 ++ .../paged_cpu_gpu_memory_allocator.py | 205 + .../paged_tensor_memory_allocator.py | 345 ++ .../memory_allocators/pin_memory_allocator.py | 146 + .../tensor_memory_allocator.py | 328 ++ lmcache/v1/memory_management.py | 1626 +++++++ lmcache/v1/metadata.py | 118 + lmcache/v1/mp_coordinator/__init__.py | 17 + lmcache/v1/mp_coordinator/__main__.py | 34 + lmcache/v1/mp_coordinator/app.py | 191 + lmcache/v1/mp_coordinator/blend_client.py | 287 ++ lmcache/v1/mp_coordinator/blend_directory.py | 392 ++ .../mp_coordinator/cache_control/__init__.py | 1 + .../cache_control/event_listener.py | 127 + .../cache_control/eviction_manager.py | 243 ++ .../cache_control/prefetch_manager.py | 98 + .../cache_control/resync_manager.py | 149 + .../cache_control/usage_manager.py | 108 + lmcache/v1/mp_coordinator/config.py | 171 + .../v1/mp_coordinator/http_apis/__init__.py | 6 + .../http_apis/blend_directory_api.py | 107 + .../v1/mp_coordinator/http_apis/cache_api.py | 207 + .../mp_coordinator/http_apis/dependencies.py | 88 + .../v1/mp_coordinator/http_apis/health_api.py | 20 + .../mp_coordinator/http_apis/instances_api.py | 120 + .../v1/mp_coordinator/http_apis/quota_api.py | 246 ++ lmcache/v1/mp_coordinator/registrar.py | 148 + lmcache/v1/mp_coordinator/registry.py | 175 + lmcache/v1/mp_coordinator/schemas.py | 447 ++ lmcache/v1/mp_coordinator/utils/__init__.py | 1 + .../v1/mp_coordinator/utils/cache_utils.py | 66 + .../v1/mp_observability/AGENTS.override.md | 21 + lmcache/v1/mp_observability/README.md | 208 + lmcache/v1/mp_observability/config.py | 404 ++ lmcache/v1/mp_observability/errors.py | 66 + lmcache/v1/mp_observability/event.py | 163 + lmcache/v1/mp_observability/event_bus.py | 387 ++ lmcache/v1/mp_observability/otel_init.py | 219 + .../mp_observability/subscribers/__init__.py | 26 + .../subscribers/logging/__init__.py | 27 + .../subscribers/logging/cb_server.py | 119 + .../subscribers/logging/l1.py | 56 + .../subscribers/logging/l2.py | 80 + .../subscribers/logging/lookup_hash.py | 209 + .../subscribers/logging/mp_server.py | 90 + .../subscribers/logging/sm.py | 59 + .../subscribers/logging/timeout.py | 28 + .../subscribers/metrics/__init__.py | 62 + .../subscribers/metrics/cb_server.py | 211 + .../subscribers/metrics/engine.py | 69 + .../subscribers/metrics/event_bus.py | 61 + .../subscribers/metrics/l0_l1_throughput.py | 148 + .../subscribers/metrics/l0_lifecycle.py | 290 ++ .../subscribers/metrics/l1.py | 62 + .../subscribers/metrics/l1_eviction_loop.py | 48 + .../subscribers/metrics/l1_failures.py | 88 + .../subscribers/metrics/l1_lifecycle.py | 166 + .../subscribers/metrics/l2.py | 185 + .../subscribers/metrics/l2_failures.py | 60 + .../subscribers/metrics/l2_throughput.py | 191 + .../subscribers/metrics/lookup.py | 93 + .../subscribers/metrics/sm_lifecycle.py | 166 + .../subscribers/metrics/timeout.py | 31 + .../subscribers/metrics/utils.py | 28 + .../subscribers/tracing/__init__.py | 24 + .../subscribers/tracing/cb_server.py | 429 ++ .../subscribers/tracing/mp_server.py | 346 ++ .../subscribers/tracing/span_registry.py | 148 + .../subscribers/tracing/timeout.py | 70 + lmcache/v1/mp_observability/trace/__init__.py | 50 + lmcache/v1/mp_observability/trace/codecs.py | 316 ++ .../v1/mp_observability/trace/decorator.py | 147 + lmcache/v1/mp_observability/trace/format.py | 132 + .../v1/mp_observability/trace/lifecycle.py | 83 + lmcache/v1/mp_observability/trace/reader.py | 167 + lmcache/v1/mp_observability/trace/recorder.py | 300 ++ lmcache/v1/multiprocess/__init__.py | 0 lmcache/v1/multiprocess/affinity_pool.py | 154 + .../v1/multiprocess/cache_control/__init__.py | 1 + .../v1/multiprocess/cache_control/errors.py | 22 + .../cache_control/key_resolver.py | 91 + .../cache_control/object_service.py | 180 + .../cache_control/prefetch_service.py | 108 + lmcache/v1/multiprocess/config.py | 680 +++ lmcache/v1/multiprocess/custom_types.py | 255 ++ lmcache/v1/multiprocess/engine_context.py | 299 ++ lmcache/v1/multiprocess/engine_module.py | 130 + lmcache/v1/multiprocess/futures.py | 203 + lmcache/v1/multiprocess/group_view.py | 264 ++ lmcache/v1/multiprocess/http_api_registry.py | 44 + lmcache/v1/multiprocess/http_apis/__init__.py | 1 + .../v1/multiprocess/http_apis/cache_api.py | 351 ++ .../v1/multiprocess/http_apis/common_api.py | 46 + .../v1/multiprocess/http_apis/config_api.py | 87 + .../v1/multiprocess/http_apis/dependencies.py | 78 + .../multiprocess/http_apis/error_handlers.py | 55 + lmcache/v1/multiprocess/http_apis/info_api.py | 76 + .../v1/multiprocess/http_apis/quota_api.py | 173 + .../multiprocess/http_apis/reconfigure_api.py | 337 ++ lmcache/v1/multiprocess/http_apis/schemas.py | 73 + lmcache/v1/multiprocess/http_server.py | 292 ++ lmcache/v1/multiprocess/modules/__init__.py | 1 + lmcache/v1/multiprocess/modules/blend.py | 1185 ++++++ lmcache/v1/multiprocess/modules/blend_v3.py | 1868 ++++++++ .../modules/engine_driven_transfer.py | 535 +++ .../modules/lmcache_driven_transfer.py | 1288 ++++++ lmcache/v1/multiprocess/modules/lookup.py | 605 +++ lmcache/v1/multiprocess/modules/management.py | 209 + .../v1/multiprocess/modules/p2p_controller.py | 573 +++ .../multiprocess/modules/server_transfer.py | 440 ++ .../mp_runtime_plugin_launcher.py | 109 + lmcache/v1/multiprocess/mq.py | 875 ++++ lmcache/v1/multiprocess/native_completion.py | 150 + lmcache/v1/multiprocess/posix_shm.py | 273 ++ lmcache/v1/multiprocess/protocol.py | 86 + lmcache/v1/multiprocess/protocols/README.md | 213 + lmcache/v1/multiprocess/protocols/__init__.py | 131 + lmcache/v1/multiprocess/protocols/base.py | 107 + lmcache/v1/multiprocess/protocols/blend.py | 109 + lmcache/v1/multiprocess/protocols/blend_v2.py | 57 + lmcache/v1/multiprocess/protocols/blend_v3.py | 72 + .../v1/multiprocess/protocols/controller.py | 57 + lmcache/v1/multiprocess/protocols/debug.py | 34 + lmcache/v1/multiprocess/protocols/engine.py | 251 ++ .../multiprocess/protocols/observability.py | 39 + lmcache/v1/multiprocess/protocols/p2p.py | 61 + lmcache/v1/multiprocess/server.py | 447 ++ lmcache/v1/multiprocess/session.py | 234 + lmcache/v1/multiprocess/token_hasher.py | 450 ++ .../multiprocess/transfer_context/__init__.py | 45 + .../transfer_context/async_engine_driven.py | 369 ++ .../v1/multiprocess/transfer_context/base.py | 714 ++++ .../multiprocess/transfer_context/pickle.py | 110 + .../v1/multiprocess/transfer_context/shm.py | 276 ++ .../transfer_context/worker_transfer.py | 653 +++ lmcache/v1/multiprocess/warm_prefetch.py | 140 + lmcache/v1/offload_server/__init__.py | 17 + lmcache/v1/offload_server/abstract_server.py | 37 + lmcache/v1/offload_server/message.py | 30 + lmcache/v1/offload_server/zmq_server.py | 122 + lmcache/v1/periodic_thread.py | 584 +++ lmcache/v1/pin_monitor.py | 260 ++ lmcache/v1/platform/__init__.py | 225 + lmcache/v1/platform/_registry.py | 223 + lmcache/v1/platform/base_cache_context.py | 345 ++ lmcache/v1/platform/base_device_spec.py | 133 + lmcache/v1/platform/base_ipc_wrapper.py | 160 + lmcache/v1/platform/base_pin_memory.py | 49 + lmcache/v1/platform/cache_context.py | 216 + lmcache/v1/platform/cpu/__init__.py | 8 + lmcache/v1/platform/cpu/cache_context.py | 439 ++ lmcache/v1/platform/cpu/shm.py | 321 ++ lmcache/v1/platform/cpu/stub_cpu_device.py | 338 ++ lmcache/v1/platform/cuda/__init__.py | 41 + lmcache/v1/platform/cuda/cache_context.py | 628 +++ lmcache/v1/platform/cuda/ipc_wrapper.py | 189 + lmcache/v1/platform/cuda/pin_memory.py | 171 + lmcache/v1/platform/event_notifier.py | 259 ++ lmcache/v1/platform/hpu/__init__.py | 35 + lmcache/v1/platform/musa/__init__.py | 41 + lmcache/v1/platform/musa/ipc.py | 318 ++ .../v1/platform/musa/native_kv_transfer.py | 486 +++ lmcache/v1/platform/musa/ops.py | 91 + lmcache/v1/platform/xpu/__init__.py | 35 + lmcache/v1/plugin/__init__.py | 0 lmcache/v1/plugin/runtime_plugin_launcher.py | 211 + lmcache/v1/protocol.py | 317 ++ lmcache/v1/rpc/__init__.py | 17 + lmcache/v1/rpc/transport.py | 105 + lmcache/v1/rpc/zmq_transport.py | 213 + lmcache/v1/rpc_utils.py | 165 + lmcache/v1/server/__init__.py | 2 + lmcache/v1/server/__main__.py | 170 + lmcache/v1/server/storage_backend/__init__.py | 21 + .../storage_backend/abstract_backend.py | 80 + .../server/storage_backend/local_backend.py | 75 + lmcache/v1/server/utils.py | 21 + lmcache/v1/standalone/__init__.py | 1 + lmcache/v1/standalone/__main__.py | 584 +++ lmcache/v1/standalone/manager.py | 80 + .../standalone/standalone_service_factory.py | 86 + lmcache/v1/storage_backend/__init__.py | 336 ++ .../v1/storage_backend/abstract_backend.py | 460 ++ lmcache/v1/storage_backend/audit_backend.py | 250 ++ .../storage_backend/batched_message_sender.py | 226 + .../storage_backend/cache_policy/__init__.py | 45 + .../cache_policy/base_policy.py | 87 + .../v1/storage_backend/cache_policy/fifo.py | 58 + .../v1/storage_backend/cache_policy/lfu.py | 105 + .../v1/storage_backend/cache_policy/lru.py | 81 + .../v1/storage_backend/cache_policy/mru.py | 61 + .../v1/storage_backend/connector/__init__.py | 423 ++ .../connector/audit_adapter.py | 77 + .../connector/audit_connector.py | 320 ++ .../connector/azure_adapter.py | 71 + .../connector/azure_connector.py | 316 ++ .../connector/base_connector.py | 379 ++ .../connector/bigtable_adapter.py | 30 + .../connector/bigtable_config.py | 136 + .../connector/bigtable_connector.py | 929 ++++ .../connector/bigtable_schema.py | 58 + .../connector/blackhole_adapter.py | 21 + .../connector/blackhole_connector.py | 37 + .../storage_backend/connector/eic_adapter.py | 31 + .../connector/eic_connector.py | 758 ++++ .../connector/external_adapter.py | 79 + .../storage_backend/connector/fs_adapter.py | 51 + .../storage_backend/connector/fs_connector.py | 409 ++ .../connector/hf3fs_adapter.py | 275 ++ .../connector/hf3fs_connector.py | 622 +++ .../connector/hfbucket_adapter.py | 55 + .../connector/hfbucket_connector.py | 965 +++++ .../connector/infinistore_adapter.py | 56 + .../connector/infinistore_connector.py | 177 + .../connector/instrumented_connector.py | 219 + .../storage_backend/connector/lm_adapter.py | 31 + .../storage_backend/connector/lm_connector.py | 176 + .../storage_backend/connector/mock_adapter.py | 57 + .../connector/mock_connector.py | 349 ++ .../connector/mooncakestore_adapter.py | 43 + .../connector/mooncakestore_connector.py | 830 ++++ .../connector/redis_adapter.py | 197 + .../connector/redis_connector.py | 828 ++++ .../storage_backend/connector/s3_adapter.py | 59 + .../storage_backend/connector/s3_connector.py | 699 +++ .../connector/sagemaker_hyperpod_adapter.py | 233 + .../connector/sagemaker_hyperpod_connector.py | 987 +++++ .../connector/valkey_adapter.py | 176 + .../connector/valkey_connector.py | 724 ++++ lmcache/v1/storage_backend/dax/__init__.py | 9 + lmcache/v1/storage_backend/dax/core.py | 1094 +++++ lmcache/v1/storage_backend/gds_backend.py | 1272 ++++++ lmcache/v1/storage_backend/hipfile_shim.py | 167 + .../storage_backend/job_executor/__init__.py | 0 .../job_executor/base_executor.py | 34 + .../job_executor/pq_executor.py | 235 + .../v1/storage_backend/local_cpu_backend.py | 966 +++++ .../v1/storage_backend/local_disk_backend.py | 774 ++++ lmcache/v1/storage_backend/maru_backend.py | 735 ++++ .../storage_backend/naive_serde/__init__.py | 50 + .../naive_serde/cachegen_basics.py | 133 + .../naive_serde/cachegen_decoder.py | 140 + .../naive_serde/cachegen_encoder.py | 84 + .../storage_backend/naive_serde/kivi_serde.py | 22 + .../naive_serde/naive_serde.py | 18 + .../v1/storage_backend/naive_serde/serde.py | 37 + .../native_clients/aerospike_client.py | 74 + .../native_clients/connector_client_base.py | 165 + .../native_clients/resp_client.py | 35 + .../storage_backend/nixl_storage_backend.py | 2183 ++++++++++ lmcache/v1/storage_backend/p2p_backend.py | 790 ++++ lmcache/v1/storage_backend/path_sharder.py | 115 + lmcache/v1/storage_backend/pd_backend.py | 1087 +++++ .../v1/storage_backend/pd_backend_async.py | 1705 ++++++++ .../v1/storage_backend/plugins/dax_backend.py | 917 ++++ .../plugins/rust_raw_block_backend.py | 699 +++ .../v1/storage_backend/raw_block/__init__.py | 42 + lmcache/v1/storage_backend/raw_block/core.py | 1946 +++++++++ .../v1/storage_backend/raw_block/key_codec.py | 172 + lmcache/v1/storage_backend/remote_backend.py | 625 +++ lmcache/v1/storage_backend/resp_client.py | 227 + .../storage_backend_listener.py | 19 + lmcache/v1/storage_backend/storage_manager.py | 1425 +++++++ lmcache/v1/system_detection.py | 110 + lmcache/v1/token_database.py | 580 +++ lmcache/v1/transfer_channel/__init__.py | 83 + lmcache/v1/transfer_channel/abstract.py | 285 ++ .../transfer_channel/mock_memory_channel.py | 156 + lmcache/v1/transfer_channel/nixl_channel.py | 688 +++ .../v1/transfer_channel/py_socket_channel.py | 260 ++ lmcache/v1/transfer_channel/transfer_utils.py | 61 + lmcache/v1/utils/__init__.py | 1 + lmcache/v1/utils/bloom_filter.py | 109 + lmcache/v1/utils/cache_utils.py | 125 + lmcache/v1/utils/json_utils.py | 56 + lmcache/v1/utils/router_discovery.py | 54 + lmcache/v1/utils/subclass_discovery.py | 219 + operator/.custom-gcl.yml | 11 + operator/.devcontainer/devcontainer.json | 35 + operator/.devcontainer/post-install.sh | 153 + operator/.dockerignore | 11 + operator/.golangci.yml | 61 + operator/.pre-commit-config.yaml | 32 + operator/AGENTS.md | 628 +++ operator/DESIGN.md | 596 +++ operator/Dockerfile | 31 + operator/Makefile | 55 + operator/PROJECT | 39 + operator/README.md | 356 ++ .../api/v1alpha1/cacheblendengine_defaults.go | 85 + .../api/v1alpha1/cacheblendengine_test.go | 559 +++ .../api/v1alpha1/cacheblendengine_types.go | 280 ++ .../v1alpha1/cacheblendengine_validation.go | 92 + operator/api/v1alpha1/groupversion_info.go | 36 + .../v1alpha1/lmcachecoordinator_defaults.go | 38 + .../api/v1alpha1/lmcachecoordinator_types.go | 238 ++ .../v1alpha1/lmcachecoordinator_validation.go | 78 + .../api/v1alpha1/lmcacheengine_defaults.go | 47 + operator/api/v1alpha1/lmcacheengine_test.go | 378 ++ operator/api/v1alpha1/lmcacheengine_types.go | 538 +++ .../api/v1alpha1/lmcacheengine_validation.go | 158 + .../api/v1alpha1/zz_generated.deepcopy.go | 1144 +++++ operator/cmd/main.go | 237 ++ operator/config/certmanager/certificate.yaml | 26 + operator/config/certmanager/issuer.yaml | 12 + .../config/certmanager/kustomization.yaml | 6 + .../config/certmanager/kustomizeconfig.yaml | 8 + .../lmcache.lmcache.ai_cacheblendengines.yaml | 3782 +++++++++++++++++ ...mcache.lmcache.ai_lmcachecoordinators.yaml | 3481 +++++++++++++++ .../lmcache.lmcache.ai_lmcacheengines.yaml | 3754 ++++++++++++++++ operator/config/crd/kustomization.yaml | 18 + operator/config/crd/kustomizeconfig.yaml | 12 + .../default/cert_metrics_manager_patch.yaml | 30 + operator/config/default/kustomization.yaml | 234 + .../config/default/manager_metrics_patch.yaml | 4 + .../config/default/manager_webhook_patch.yaml | 20 + operator/config/default/metrics_service.yaml | 18 + operator/config/manager/kustomization.yaml | 8 + operator/config/manager/manager.yaml | 99 + .../network-policy/allow-metrics-traffic.yaml | 27 + .../config/network-policy/kustomization.yaml | 2 + operator/config/prometheus/kustomization.yaml | 11 + operator/config/prometheus/monitor.yaml | 27 + .../config/prometheus/monitor_tls_patch.yaml | 19 + operator/config/rbac/kustomization.yaml | 28 + .../config/rbac/leader_election_role.yaml | 40 + .../rbac/leader_election_role_binding.yaml | 15 + .../config/rbac/lmcacheengine_admin_role.yaml | 27 + .../rbac/lmcacheengine_editor_role.yaml | 33 + .../rbac/lmcacheengine_viewer_role.yaml | 29 + operator/config/rbac/metrics_auth_role.yaml | 17 + .../rbac/metrics_auth_role_binding.yaml | 12 + operator/config/rbac/metrics_reader_role.yaml | 9 + operator/config/rbac/role.yaml | 85 + operator/config/rbac/role_binding.yaml | 15 + operator/config/rbac/service_account.yaml | 8 + operator/config/samples/kustomization.yaml | 6 + .../lmcache_v1alpha1_cacheblendengine.yaml | 94 + .../lmcache_v1alpha1_lmcachecoordinator.yaml | 67 + .../lmcache_v1alpha1_lmcacheengine.yaml | 126 + .../lmcache_v1alpha1_lmcacheengine_amd.yaml | 16 + ...ache_v1alpha1_lmcacheengine_injection.yaml | 41 + ...cache_v1alpha1_lmcacheengine_l2_redis.yaml | 36 + ...che_v1alpha1_lmcacheengine_production.yaml | 40 + .../samples/vllm_cacheblend_deployment.yaml | 86 + .../samples/vllm_lmcache_deployment.yaml | 80 + .../vllm_lmcache_injection_deployment.yaml | 90 + operator/config/webhook/kustomization.yaml | 15 + operator/config/webhook/kustomizeconfig.yaml | 25 + operator/config/webhook/manifests.yaml | 46 + .../mutating_webhook_selectors_patch.yaml | 40 + operator/config/webhook/service.yaml | 16 + operator/go.mod | 101 + operator/go.sum | 257 ++ operator/hack/boilerplate.go.txt | 15 + operator/hack/pre-commit | 35 + .../cacheblend_reconcile_helpers.go | 443 ++ .../controller/cacheblendengine_controller.go | 128 + .../cacheblendengine_controller_test.go | 212 + .../coordinator_reconcile_helpers.go | 291 ++ .../lmcachecoordinator_controller.go | 107 + .../lmcachecoordinator_controller_test.go | 183 + .../controller/lmcacheengine_controller.go | 136 + .../lmcacheengine_controller_test.go | 86 + .../internal/controller/reconcile_helpers.go | 508 +++ .../controller/reconcile_helpers_test.go | 741 ++++ operator/internal/controller/suite_test.go | 122 + .../internal/resources/cacheblend_engine.go | 187 + .../resources/cacheblend_engine_test.go | 392 ++ operator/internal/resources/compute.go | 222 + operator/internal/resources/configmap.go | 102 + .../resources/coordinator_deployment.go | 238 ++ .../internal/resources/coordinator_resolve.go | 63 + .../internal/resources/coordinator_service.go | 157 + .../internal/resources/coordinator_test.go | 175 + operator/internal/resources/daemonset.go | 300 ++ operator/internal/resources/helpers.go | 55 + operator/internal/resources/labels.go | 44 + operator/internal/resources/resources_test.go | 1187 ++++++ operator/internal/resources/service.go | 99 + operator/internal/resources/servicemonitor.go | 68 + .../webhook/cacheblend_inject_builders.go | 250 ++ .../webhook/cacheblend_pod_injector.go | 297 ++ .../cacheblend_pod_injector_envtest_test.go | 178 + .../webhook/cacheblend_pod_injector_test.go | 575 +++ .../webhook/lmcache_inject_builders.go | 118 + .../internal/webhook/lmcache_pod_injector.go | 209 + .../webhook/lmcache_pod_injector_test.go | 474 +++ operator/internal/webhook/payload_staging.go | 134 + .../internal/webhook/pod_inject_common.go | 245 ++ .../internal/webhook/webhook_suite_test.go | 173 + operator/make/build.mk | 43 + operator/make/deploy.mk | 24 + operator/make/dev.mk | 17 + operator/make/e2e-gpu.mk | 175 + operator/make/e2e.mk | 64 + operator/make/lint.mk | 22 + operator/make/tools.mk | 83 + operator/make/unit.mk | 47 + operator/test/e2e/auth_smoke_test.go | 155 + .../e2e/cacheblend_integration_smoke_test.go | 252 ++ .../coordinator_registration_smoke_test.go | 134 + operator/test/e2e/crd_smoke_test.go | 252 ++ operator/test/e2e/e2e_suite_test.go | 182 + operator/test/e2e/e2e_test.go | 269 ++ .../test/e2e/field_coverage_smoke_test.go | 133 + operator/test/e2e/lifecycle_smoke_test.go | 146 + .../test/e2e/lmcache_injection_smoke_test.go | 220 + operator/test/e2e/runtime_smoke_test.go | 117 + operator/test/e2e/smoke_helpers_test.go | 353 ++ .../test/e2e/vllm_integration_smoke_test.go | 293 ++ operator/test/utils/cbe.go | 107 + operator/test/utils/fixtures.go | 82 + .../test/utils/fixtures/cacheblendengine.yaml | 50 + .../golden/kv_transfer_config_minimal.json | 9 + .../test/utils/fixtures/lmc_custom_port.yaml | 18 + operator/test/utils/fixtures/lmc_minimal.yaml | 19 + operator/test/utils/fixtures/lmc_runtime.yaml | 22 + .../utils/fixtures/lmc_servicemonitor.yaml | 20 + .../lmc_with_redis_l2_authsecret.yaml | 22 + .../fixtures/vllm_cacheblend_deployment.yaml | 92 + .../test/utils/fixtures/vllm_deployment.yaml | 122 + .../vllm_lmcache_injection_deployment.yaml | 49 + operator/test/utils/http.go | 157 + operator/test/utils/lmc.go | 210 + operator/test/utils/portforward.go | 173 + operator/test/utils/runner.go | 65 + operator/test/utils/secret.go | 92 + operator/test/utils/utils.go | 212 + operator/test/utils/wait.go | 176 + pyproject.toml | 187 + pyproject_cli.toml | 138 + pytest.ini | 5 + requirements/bench.txt | 8 + requirements/build.txt | 11 + requirements/cli.txt | 5 + requirements/common.txt | 49 + requirements/cuda.txt | 21 + requirements/cuda12_core.txt | 4 + requirements/cuda13_core.txt | 5 + requirements/docs.txt | 17 + requirements/lint.txt | 2 + requirements/nixl.txt | 9 + requirements/rocm_core.txt | 9 + requirements/test.txt | 10 + rust/raw_block/.gitignore | 2 + rust/raw_block/Cargo.toml | 15 + rust/raw_block/README.md | 181 + rust/raw_block/pyproject.toml | 18 + rust/raw_block/src/lib.rs | 3104 ++++++++++++++ setup.py | 62 + setup_extensions/__init__.py | 24 + setup_extensions/build_profiles/__init__.py | 163 + setup_extensions/build_profiles/cuda.py | 114 + setup_extensions/build_profiles/musa.py | 36 + setup_extensions/build_profiles/rocm.py | 125 + setup_extensions/build_profiles/sycl.py | 91 + setup_extensions/common_cpp.py | 98 + setup_extensions/policy.py | 249 ++ .../storage_backend_profiles/__init__.py | 59 + .../storage_backend_profiles/aerospike.py | 95 + .../storage_backend_profiles/mooncake.py | 65 + tests/__init__.py | 0 tests/aerospike_ce.conf.template | 53 + tests/benchmarks/test_benchmark.py | 459 ++ tests/benchmarks/test_cachegen.py | 84 + .../test_musa_inprocess_benchmark.py | 37 + .../test_xpu_connector_benchmark.py | 477 +++ .../benchmarks/test_xpu_kernels_microbench.py | 176 + .../test_xpu_layerwise_connector_benchmark.py | 602 +++ tests/cli/__init__.py | 1 + tests/cli/commands/__init__.py | 1 + tests/cli/commands/bench/__init__.py | 1 + .../commands/bench/engine_bench/__init__.py | 1 + .../engine_bench/interactive/__init__.py | 1 + .../engine_bench/interactive/test_back.py | 92 + .../engine_bench/interactive/test_schema.py | 112 + .../engine_bench/interactive/test_state.py | 368 ++ .../bench/engine_bench/test_config.py | 340 ++ .../bench/engine_bench/test_progress.py | 204 + .../bench/engine_bench/test_request_sender.py | 513 +++ .../commands/bench/engine_bench/test_stats.py | 407 ++ .../bench/engine_bench/workloads/__init__.py | 1 + .../workloads/test_base_workload.py | 153 + .../workloads/test_create_workload.py | 283 ++ .../workloads/test_long_doc_permutator.py | 529 +++ .../workloads/test_long_doc_qa.py | 385 ++ .../workloads/test_multi_round_chat.py | 488 +++ .../workloads/test_prefix_suffix_tuner.py | 766 ++++ .../workloads/test_random_prefill.py | 256 ++ .../bench/l2_adapter_bench/test_data.py | 61 + .../bench/l2_adapter_bench/test_profiling.py | 57 + .../cli/commands/bench/test_bench_command.py | 714 ++++ tests/cli/commands/bench/test_server_bench.py | 625 +++ tests/cli/commands/test_coordinator.py | 120 + tests/cli/commands/test_query.py | 244 ++ tests/cli/commands/test_quota.py | 158 + tests/cli/commands/test_server.py | 151 + tests/cli/commands/trace/__init__.py | 0 tests/cli/commands/trace/test_dispatch.py | 162 + tests/cli/commands/trace/test_driver.py | 319 ++ tests/cli/commands/trace/test_stats.py | 126 + tests/cli/conftest.py | 48 + tests/cli/test_describe.py | 616 +++ tests/cli/test_ping.py | 233 + tests/conftest.py | 920 ++++ tests/data/test_creation_from_file/disk.yaml | 4 + tests/data/test_creation_from_file/fail.yaml | 1 + .../data/test_creation_from_file/hybrid.yaml | 4 + tests/data/test_creation_from_file/local.yaml | 4 + .../data/test_creation_from_file/remote.yaml | 4 + tests/disagg/README.md | 136 + tests/disagg/test_nixl_cache_engine.py | 337 ++ tests/disagg/test_nixl_channel.py | 164 + tests/disagg/test_nixl_storage_backend.py | 523 +++ tests/lmcache_frontend/__init__.py | 1 + .../lmcache_mp_plugin/__init__.py | 1 + .../test_lmcache_mp_frontend_plugin.py | 116 + tests/lmcache_frontend/test_heartbeat.py | 143 + tests/test_banner.py | 81 + tests/test_observability.py | 466 ++ tests/test_serde.py | 135 + tests/test_usage_telemetry.py | 520 +++ tests/test_utils.py | 687 +++ tests/tools/test_cache_simulator.py | 332 ++ tests/tools/test_controller_zmq_benchmark.py | 562 +++ tests/v1/__init__.py | 0 tests/v1/cache_controller/__init__.py | 1 + tests/v1/cache_controller/conftest.py | 468 ++ tests/v1/cache_controller/pytest.ini | 26 + tests/v1/cache_controller/test_config.py | 311 ++ tests/v1/cache_controller/test_full_sync.py | 447 ++ .../cache_controller/test_full_sync_sender.py | 272 ++ .../v1/cache_controller/test_kv_controller.py | 721 ++++ tests/v1/cache_controller/test_locks.py | 347 ++ tests/v1/cache_controller/test_messages.py | 739 ++++ .../test_registration_controller.py | 628 +++ .../v1/cache_controller/test_registry_tree.py | 1163 +++++ .../compute/attention/test_triton_kernels.py | 297 ++ .../attention/test_triton_sparse_e2e.py | 190 + tests/v1/data/gds.yaml | 6 + tests/v1/data/hipfile.yaml | 7 + tests/v1/data/hipfile_gds.yaml | 9 + tests/v1/data/nixl.yaml | 11 + tests/v1/data/nixl_multipath.yaml | 14 + tests/v1/data/test_config.yaml | 15 + tests/v1/distributed/__init__.py | 1 + tests/v1/distributed/conftest.py | 91 + .../l2_adapters/test_p2p_l2_adapter.py | 422 ++ .../test_p2p_l2_adapter_integration.py | 222 + .../v1/distributed/memory_manager/__init__.py | 1 + .../test_gds_l1_memory_manager.py | 85 + tests/v1/distributed/serde/__init__.py | 1 + .../serde/test_asym_k16_v8_multi.py | 195 + .../serde/test_asym_k16_v8_v_only.py | 237 ++ .../distributed/serde/test_async_processor.py | 256 ++ tests/v1/distributed/serde/test_factory.py | 178 + tests/v1/distributed/serde/test_fp8.py | 104 + tests/v1/distributed/serde/test_multi.py | 449 ++ tests/v1/distributed/serde/test_serde_e2e.py | 560 +++ .../v1/distributed/serde/test_serde_fs_e2e.py | 217 + tests/v1/distributed/serde/test_turboquant.py | 623 +++ tests/v1/distributed/serde/test_utils.py | 50 + .../test_aerospike_l2_adapter_config.py | 92 + .../test_aerospike_l2_integration.py | 151 + tests/v1/distributed/test_bitmap_ops.py | 567 +++ .../test_cache_salt_l2_eviction.py | 286 ++ tests/v1/distributed/test_dax_l2_adapter.py | 921 ++++ .../distributed/test_devdax_l1_allocator.py | 607 +++ .../test_distributed_storage_manager.py | 904 ++++ .../v1/distributed/test_encoded_object_key.py | 110 + .../test_fault_inject_l2_adapter.py | 256 ++ .../v1/distributed/test_fs_l2_adapter_keys.py | 323 ++ .../distributed/test_hfbucket_l2_adapter.py | 581 +++ .../test_isolated_lru_eviction_policy.py | 137 + .../distributed/test_l1_l2_state_metrics.py | 379 ++ tests/v1/distributed/test_l1_manager.py | 1938 +++++++++ .../v1/distributed/test_l1_memory_manager.py | 556 +++ tests/v1/distributed/test_l1_single_region.py | 56 + tests/v1/distributed/test_l2_adapter_base.py | 323 ++ .../v1/distributed/test_l2_adapter_factory.py | 1138 +++++ .../distributed/test_lru_eviction_policy.py | 403 ++ tests/v1/distributed/test_mock_l2_adapter.py | 888 ++++ .../test_mooncake_store_l2_adapter.py | 991 +++++ tests/v1/distributed/test_multimodel.py | 144 + .../test_native_connector_l2_adapter.py | 1360 ++++++ .../test_nixl_store_dynamic_l2_adapter.py | 869 ++++ .../distributed/test_nixl_store_l2_adapter.py | 1068 +++++ .../distributed/test_object_key_parallel.py | 734 ++++ .../distributed/test_prefetch_controller.py | 1500 +++++++ tests/v1/distributed/test_prefetch_policy.py | 308 ++ tests/v1/distributed/test_quota_manager.py | 247 ++ .../distributed/test_raw_block_l2_adapter.py | 629 +++ tests/v1/distributed/test_report_status.py | 222 + tests/v1/distributed/test_resp_env_vars.py | 173 + .../test_resp_l2_adapter_integration.py | 292 ++ tests/v1/distributed/test_runtime_adapters.py | 392 ++ tests/v1/distributed/test_s3_l2_adapter.py | 902 ++++ .../test_storage_manager_l2_keys.py | 116 + tests/v1/distributed/test_store_controller.py | 724 ++++ tests/v1/distributed/test_store_policy.py | 119 + .../distributed/transfer_channel/__init__.py | 1 + .../distributed/transfer_channel/test_api.py | 70 + .../test_context_lifecycle.py | 144 + .../transfer_channel/test_factory.py | 106 + .../transfer_channel/test_nixl_impl.py | 236 + .../test_blocks_first_fused_kv_format.py | 181 + tests/v1/gpu_connector/test_concrete_shape.py | 68 + tests/v1/gpu_connector/test_gds_context.py | 323 ++ tests/v1/gpu_connector/test_hipfile_async.py | 227 + .../test_kv_format_classification.py | 61 + .../gpu_connector/test_kv_format_detection.py | 112 + .../v1/gpu_connector/test_kv_format_specs.py | 292 ++ .../test_normalize_per_layer_formats.py | 64 + .../v1/gpu_connector/test_utils_shape_desc.py | 238 ++ .../internal_api_server/test_backend_api.py | 254 ++ .../internal_api_server/test_cache_clear.py | 136 + .../internal_api_server/test_hot_cache_api.py | 113 + .../test_kvcache_check_api.py | 266 ++ .../test_load_fs_chunks.py | 294 ++ .../v1/internal_api_server/test_lookup_api.py | 334 ++ .../test_per_instance_app.py | 115 + .../v1/internal_api_server/test_run_script.py | 139 + .../test_scripts/allowed_imports.py | 11 + .../internal_api_server/test_scripts/basic.py | 3 + .../test_scripts/builtin_functions.py | 7 + .../test_scripts/default_result.py | 7 + .../test_scripts/empty_allowed_imports.py | 7 + .../test_scripts/exception.py | 7 + .../test_scripts/invalid_module.py | 7 + .../test_scripts/math_module.py | 10 + .../test_scripts/multiple_imports.py | 11 + .../test_scripts/os_module.py | 11 + .../test_scripts/without_allowed_imports.py | 10 + .../test_worker_info_api.py | 307 ++ tests/v1/lookup_client/__init__.py | 0 .../test_chunk_statistics_lookup_client.py | 357 ++ .../test_lmcache_lookup_client.py | 457 ++ tests/v1/mp_coordinator/__init__.py | 1 + tests/v1/mp_coordinator/test_api.py | 158 + tests/v1/mp_coordinator/test_blend_client.py | 213 + .../v1/mp_coordinator/test_blend_directory.py | 259 ++ .../test_blend_directory_api.py | 101 + tests/v1/mp_coordinator/test_cache_api.py | 183 + tests/v1/mp_coordinator/test_config.py | 39 + .../mp_coordinator/test_eviction_manager.py | 526 +++ tests/v1/mp_coordinator/test_health.py | 39 + tests/v1/mp_coordinator/test_integration.py | 104 + .../mp_coordinator/test_prefetch_manager.py | 124 + tests/v1/mp_coordinator/test_quota_api.py | 368 ++ tests/v1/mp_coordinator/test_registrar.py | 226 + tests/v1/mp_coordinator/test_registry.py | 108 + .../v1/mp_coordinator/test_resync_manager.py | 250 ++ tests/v1/mp_coordinator/test_usage_manager.py | 175 + .../mp_observability/subscribers/__init__.py | 1 + .../subscribers/logging/__init__.py | 1 + .../subscribers/logging/test_cb_server.py | 213 + .../logging/test_lookup_hash_logger.py | 321 ++ .../subscribers/logging/test_mp_server.py | 130 + .../subscribers/logging/test_timeout.py | 77 + .../subscribers/metrics/counter_helpers.py | 74 + .../subscribers/metrics/otel_setup.py | 88 + .../subscribers/metrics/test_cb_server.py | 401 ++ .../subscribers/metrics/test_engine.py | 229 + .../metrics/test_event_bus_self_metrics.py | 96 + .../metrics/test_l0_l1_throughput.py | 390 ++ .../subscribers/metrics/test_l0_lifecycle.py | 477 +++ .../subscribers/metrics/test_l1.py | 184 + .../metrics/test_l1_eviction_loop.py | 107 + .../subscribers/metrics/test_l1_failures.py | 305 ++ .../subscribers/metrics/test_l1_lifecycle.py | 331 ++ .../subscribers/metrics/test_l2.py | 766 ++++ .../subscribers/metrics/test_l2_failures.py | 210 + .../subscribers/metrics/test_l2_throughput.py | 436 ++ .../subscribers/metrics/test_lookup.py | 360 ++ .../subscribers/metrics/test_sm_lifecycle.py | 362 ++ .../subscribers/metrics/test_timeout.py | 119 + .../subscribers/tracing/__init__.py | 1 + .../subscribers/tracing/test_cb_server.py | 1076 +++++ .../subscribers/tracing/test_mp_server.py | 781 ++++ .../subscribers/tracing/test_timeout.py | 120 + tests/v1/mp_observability/test_errors.py | 93 + tests/v1/mp_observability/test_event_bus.py | 539 +++ .../mp_observability/test_event_recorder.py | 200 + tests/v1/mp_observability/test_otel_init.py | 70 + .../mp_observability/test_register_gauge.py | 132 + tests/v1/mp_observability/trace/__init__.py | 0 .../v1/mp_observability/trace/test_codecs.py | 171 + .../mp_observability/trace/test_decorator.py | 177 + .../mp_observability/trace/test_recorder.py | 295 ++ tests/v1/multiprocess/__init__.py | 2 + tests/v1/multiprocess/http_apis/__init__.py | 0 .../multiprocess/http_apis/test_cache_api.py | 582 +++ .../multiprocess/http_apis/test_common_api.py | 156 + .../multiprocess/http_apis/test_config_api.py | 201 + .../multiprocess/http_apis/test_info_api.py | 73 + .../http_apis/test_reconfigure_api.py | 336 ++ tests/v1/multiprocess/test_affinity_pool.py | 248 ++ ...st_async_engine_driven_transfer_context.py | 460 ++ .../test_batched_iteration_with_skip.py | 112 + .../test_blend_v3_load_store_opts.py | 568 +++ tests/v1/multiprocess/test_cache_server.py | 753 ++++ .../multiprocess/test_completion_recorder.py | 179 + .../multiprocess/test_compute_mp_checksums.py | 250 ++ tests/v1/multiprocess/test_config.py | 129 + tests/v1/multiprocess/test_custom_types.py | 367 ++ .../test_engine_driven_transfer.py | 1573 +++++++ .../multiprocess/test_engine_passthroughs.py | 56 + tests/v1/multiprocess/test_free_locks.py | 273 ++ tests/v1/multiprocess/test_futures.py | 586 +++ .../v1/multiprocess/test_http_api_registry.py | 195 + .../multiprocess/test_http_quota_endpoints.py | 256 ++ tests/v1/multiprocess/test_http_server.py | 379 ++ .../test_lmcache_driven_layout_registry.py | 203 + .../multiprocess/test_lookup_wait_prefetch.py | 81 + .../test_mp_runtime_plugin_launcher.py | 147 + tests/v1/multiprocess/test_mq.py | 872 ++++ .../multiprocess/test_mq_handler_helpers.py | 258 ++ .../multiprocess/test_optimized_lookup_v3.py | 799 ++++ tests/v1/multiprocess/test_p2p_controller.py | 485 +++ tests/v1/multiprocess/test_posix_shm.py | 78 + .../v1/multiprocess/test_query_lookup_hits.py | 305 ++ .../multiprocess/test_raw_block_l2_adapter.py | 120 + tests/v1/multiprocess/test_session.py | 359 ++ .../multiprocess/test_skip_first_n_tokens.py | 228 + tests/v1/multiprocess/test_token_hasher.py | 375 ++ tests/v1/multiprocess/test_unified_touch.py | 335 ++ .../test_usage_telemetry_server.py | 119 + tests/v1/multiprocess/test_warm_prefetch.py | 105 + tests/v1/multiprocess/test_worker_liveness.py | 311 ++ tests/v1/native_storage_ops/test_bitmap.py | 680 +++ .../test_pattern_matcher.py | 701 +++ .../test_periodic_event_notifier.py | 231 + tests/v1/native_storage_ops/test_ttl_lock.py | 464 ++ tests/v1/platform/__init__.py | 1 + .../platform/test_cache_context_dispatch.py | 196 + tests/v1/platform/test_cpu_shm.py | 273 ++ tests/v1/platform/test_gpu_cache_context.py | 550 +++ .../v1/plugin/test_runtime_plugin_launcher.py | 363 ++ tests/v1/shm_allocator/__init__.py | 0 tests/v1/shm_allocator/csrc/.gitignore | 1 + tests/v1/shm_allocator/csrc/CMakeLists.txt | 9 + .../v1/shm_allocator/csrc/shm_file_worker.cpp | 345 ++ .../v1/shm_allocator/shmfile_basic_check.yaml | 33 + tests/v1/shm_allocator/shmfile_connector.py | 311 ++ tests/v1/shm_allocator/test_shm_allocator.py | 463 ++ tests/v1/shm_allocator/verify_shmfile_tcp.py | 261 ++ tests/v1/storage_backend/__init__.py | 1 + .../storage_backend/raw_block_test_utils.py | 265 ++ .../storage_backend/test_audit_connector.py | 654 +++ .../storage_backend/test_azure_connector.py | 327 ++ .../test_batched_message_sender.py | 421 ++ .../test_bigtable_connector.py | 981 +++++ .../test_bigtable_integration.py | 281 ++ tests/v1/storage_backend/test_dax_backend.py | 2090 +++++++++ tests/v1/storage_backend/test_eic.py | 132 + tests/v1/storage_backend/test_fs_connector.py | 410 ++ tests/v1/storage_backend/test_gds_backend.py | 1057 +++++ .../storage_backend/test_hf3fs_connector.py | 656 +++ .../test_hfbucket_connector.py | 518 +++ .../storage_backend/test_local_cpu_backend.py | 732 ++++ .../test_local_cpu_backend_nixl_paged.py | 245 ++ .../test_local_disk_backend.py | 611 +++ tests/v1/storage_backend/test_maru_backend.py | 788 ++++ .../storage_backend/test_nixl_shared_pool.py | 688 +++ .../test_p2p_backend_with_controller.py | 609 +++ tests/v1/storage_backend/test_path_sharder.py | 209 + .../storage_backend/test_pd_backend_async.py | 1297 ++++++ .../test_pd_backend_bidirectional.py | 439 ++ .../test_pd_backend_buffer_alignment.py | 65 + .../v1/storage_backend/test_raw_block_core.py | 125 + .../storage_backend/test_raw_block_device.py | 152 + .../test_raw_block_key_codec.py | 63 + .../test_raw_block_uring_cmd.py | 320 ++ .../test_remote_storage_plugin.py | 534 +++ .../v1/storage_backend/test_resp_connector.py | 477 +++ .../test_rust_raw_block_backend.py | 2448 +++++++++++ .../storage_backend/test_storage_manager.py | 370 ++ .../v1/storage_backend/test_storage_plugin.py | 484 +++ .../storage_backend/test_valkey_connector.py | 1305 ++++++ tests/v1/test_address_manager.py | 1207 ++++++ tests/v1/test_basic_check.py | 266 ++ tests/v1/test_bloom_filter.py | 190 + tests/v1/test_c_ops_fallback_parity.py | 396 ++ tests/v1/test_cache_engine.py | 2118 +++++++++ tests/v1/test_cache_engine_cleanup.py | 76 + tests/v1/test_cache_interface.py | 2 + tests/v1/test_cache_policy.py | 207 + tests/v1/test_config.py | 1053 +++++ tests/v1/test_connector.py | 452 ++ tests/v1/test_connector_discovery.py | 82 + tests/v1/test_decode_save_and_preemption.py | 402 ++ tests/v1/test_device_id_race.py | 178 + tests/v1/test_ec_connector.py | 121 + tests/v1/test_event_notifier.py | 246 ++ tests/v1/test_freeze_mode_integration.py | 332 ++ tests/v1/test_gds.py | 132 + tests/v1/test_gpu_connector.py | 963 +++++ tests/v1/test_health_monitor.py | 361 ++ .../test_health_monitor_fallback_recovery.py | 641 +++ tests/v1/test_hidden_state_store.py | 434 ++ tests/v1/test_impl_completeness.py | 96 + tests/v1/test_json_utils.py | 108 + tests/v1/test_kv_cache_groups.py | 182 + tests/v1/test_kv_layer_groups_manager.py | 677 +++ tests/v1/test_manager.py | 559 +++ tests/v1/test_mem_kernels.py | 971 +++++ .../test_memory_allocator_import_surface.py | 173 + tests/v1/test_memory_management.py | 1179 +++++ tests/v1/test_mp_mem_kernels.py | 471 ++ tests/v1/test_musa_connector.py | 1023 +++++ tests/v1/test_musa_mp_handle.py | 244 ++ tests/v1/test_musa_native.py | 457 ++ tests/v1/test_musa_support.py | 291 ++ tests/v1/test_nixl_batched_contains.py | 391 ++ tests/v1/test_nixl_doca_memos.py | 221 + tests/v1/test_nixl_multipath.py | 65 + tests/v1/test_nixl_storage.py | 483 +++ tests/v1/test_pos_kernels.py | 64 + tests/v1/test_python_ops_fallback.py | 2867 +++++++++++++ tests/v1/test_remote_metadata.py | 101 + tests/v1/test_remote_mla_worker_id_as0.py | 186 + tests/v1/test_staging_buffer.py | 192 + tests/v1/test_subclass_discovery.py | 612 +++ tests/v1/test_token_database.py | 157 + tests/v1/test_trtllm_integration.py | 377 ++ ...adapter_request_finished_scheduler_role.py | 251 ++ tests/v1/test_v1_adapter_state_desync.py | 155 + tests/v1/test_vllm_integration.py | 2 + tests/v1/test_vllm_kv_cache_groups.py | 346 ++ tests/v1/test_vllm_layerwise_wait_for_save.py | 150 + tests/v1/test_vllm_mp_adapter.py | 750 ++++ tests/v1/test_xpu_connector.py | 838 ++++ tests/v1/test_xpu_sglang_connector.py | 899 ++++ tests/v1/utils.py | 861 ++++ tools/check_spdx_header.py | 55 + tools/translate_docs_zh.py | 420 ++ 2101 files changed, 494002 insertions(+) create mode 100644 .buildkite/README.md create mode 100644 .buildkite/cases/comprehensive-cases.txt create mode 100644 .buildkite/cases/integration-cases.txt create mode 100644 .buildkite/configs/async.yaml create mode 100644 .buildkite/configs/dummy.yaml create mode 100644 .buildkite/configs/layerwise.yaml create mode 100644 .buildkite/configs/lmcache_configs/local_cpu_mla.yaml create mode 100644 .buildkite/configs/local_cpu.yaml create mode 100644 .buildkite/configs/local_cpu_mla.yaml create mode 100644 .buildkite/configs/local_cpu_with_v3.yaml create mode 100644 .buildkite/configs/local_disk.yaml create mode 100644 .buildkite/configs/local_disk_with_v3.yaml create mode 100644 .buildkite/configs/multi_device.yaml create mode 100644 .buildkite/configs/p2p.yaml create mode 100644 .buildkite/configs/p2p_with_v3.yaml create mode 100644 .buildkite/configs/pd.yaml create mode 100644 .buildkite/correctness/README.md create mode 100755 .buildkite/correctness/async_request.py create mode 100644 .buildkite/correctness/compare_files.py create mode 100644 .buildkite/correctness/mmlu-test.py create mode 100644 .buildkite/correctness/pipeline.correctness.yml create mode 100644 .buildkite/correctness/pipeline.mmlu.yml create mode 100644 .buildkite/correctness/setup.sh create mode 100644 .buildkite/correctness/sharegpt2openai.py create mode 100644 .buildkite/correctness/summarize-results.py create mode 100644 .buildkite/k3_harness/ARCHITECTURE.md create mode 100644 .buildkite/k3_harness/README.md create mode 100644 .buildkite/k3_harness/ci-base.Dockerfile create mode 100755 .buildkite/k3_harness/gpu-monitor.sh create mode 100755 .buildkite/k3_harness/install-agent-stack.sh create mode 100644 .buildkite/k3_harness/resolve-pinned-vllm.sh create mode 100755 .buildkite/k3_harness/setup-blend-env.sh create mode 100755 .buildkite/k3_harness/setup-cluster.sh create mode 100755 .buildkite/k3_harness/setup-env.sh create mode 100755 .buildkite/k3_harness/setup-gpu-monitor.sh create mode 100755 .buildkite/k3_harness/setup-lmcache-only-env.sh create mode 100755 .buildkite/k3_harness/setup-sglang-env.sh create mode 100755 .buildkite/k3_harness/smoke-test.sh create mode 100755 .buildkite/k3_harness/teardown.sh create mode 100644 .buildkite/k3_harness/values.yaml create mode 100644 .buildkite/k3_tests/README.md create mode 100644 .buildkite/k3_tests/blend/buildkite-pipeline.yml create mode 100644 .buildkite/k3_tests/blend/pipeline.yml create mode 100755 .buildkite/k3_tests/blend/run.sh create mode 100755 .buildkite/k3_tests/blend/scripts/run-compat.sh create mode 100755 .buildkite/k3_tests/common_scripts/helpers.sh create mode 100755 .buildkite/k3_tests/common_scripts/path-filter.sh create mode 100755 .buildkite/k3_tests/common_scripts/upload-pipeline.sh create mode 100644 .buildkite/k3_tests/comprehensive/BK_WEB_SETUP.md create mode 100644 .buildkite/k3_tests/comprehensive/buildkite-pipeline.yml create mode 100644 .buildkite/k3_tests/comprehensive/pipeline.yml create mode 100755 .buildkite/k3_tests/comprehensive/run.sh create mode 100755 .buildkite/k3_tests/comprehensive/scripts/run-single-config.sh create mode 100755 .buildkite/k3_tests/comprehensive/scripts/upload-baselines.sh create mode 100644 .buildkite/k3_tests/correctness/BK_WEB_SETUP.md create mode 100644 .buildkite/k3_tests/correctness/buildkite-pipeline.yml create mode 100644 .buildkite/k3_tests/correctness/pipeline.yml create mode 100755 .buildkite/k3_tests/correctness/run.sh create mode 100755 .buildkite/k3_tests/correctness/scripts/run-correctness.sh create mode 100644 .buildkite/k3_tests/integration/BK_WEB_SETUP.md create mode 100644 .buildkite/k3_tests/integration/buildkite-pipeline.yml create mode 100644 .buildkite/k3_tests/integration/pipeline.yml create mode 100755 .buildkite/k3_tests/integration/run.sh create mode 100755 .buildkite/k3_tests/integration/scripts/run-integration.sh create mode 100644 .buildkite/k3_tests/multiprocess/BK_WEB_SETUP.md create mode 100644 .buildkite/k3_tests/multiprocess/buildkite-pipeline.yml create mode 100644 .buildkite/k3_tests/multiprocess/pipeline.yml create mode 100755 .buildkite/k3_tests/multiprocess/run.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/cleanup.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/launch-processes.sh create mode 100644 .buildkite/k3_tests/multiprocess/scripts/pin-tested-vllm.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-cache-stats.sh create mode 100644 .buildkite/k3_tests/multiprocess/scripts/run-deadlock.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-fault-tolerance.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-gds-smoke.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-hma-lm-eval.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-http-api.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-lm-eval.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-long-doc-qa-l2.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-long-doc-qa.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-mp-test.sh create mode 100644 .buildkite/k3_tests/multiprocess/scripts/run-p2p.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-restart-recovery.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-single-test.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/run-vllm-bench.sh create mode 100755 .buildkite/k3_tests/multiprocess/scripts/wait-for-servers.sh create mode 100644 .buildkite/k3_tests/sglang/BK_WEB_SETUP.md create mode 100644 .buildkite/k3_tests/sglang/buildkite-pipeline.yml create mode 100644 .buildkite/k3_tests/sglang/pipeline.yml create mode 100755 .buildkite/k3_tests/sglang/run.sh create mode 100755 .buildkite/k3_tests/sglang/scripts/common.sh create mode 100755 .buildkite/k3_tests/sglang/scripts/run-correctness.sh create mode 100755 .buildkite/k3_tests/sglang/scripts/run-perf.sh create mode 100644 .buildkite/k3_tests/unit/buildkite-pipeline.yml create mode 100644 .buildkite/k3_tests/unit/pipeline.yml create mode 100755 .buildkite/k3_tests/unit/run.sh create mode 100644 .buildkite/operator/integration/pipeline.yaml create mode 100644 .buildkite/operator/integration/tests.yaml create mode 100644 .buildkite/pipeline.yml create mode 100644 .buildkite/pipelines/clean.yml create mode 100644 .buildkite/pipelines/comprehensive-tests.yml create mode 100644 .buildkite/pipelines/end-to-end-tests.yml create mode 100644 .buildkite/pipelines/multiprocessing-test.yml create mode 100644 .buildkite/scripts/bare-machine-cleanup.sh create mode 100644 .buildkite/scripts/clean.sh create mode 100644 .buildkite/scripts/end-to-end-test.sh create mode 100644 .buildkite/scripts/gpu_zombie_killer.sh create mode 100644 .buildkite/scripts/multi-round-qa.sh create mode 100644 .buildkite/scripts/multiprocessing-test/README.md create mode 100755 .buildkite/scripts/multiprocessing-test/build-mp-docker-image.sh create mode 100755 .buildkite/scripts/multiprocessing-test/cleanup.sh create mode 100644 .buildkite/scripts/multiprocessing-test/common.sh create mode 100755 .buildkite/scripts/multiprocessing-test/launch-containers.sh create mode 100755 .buildkite/scripts/multiprocessing-test/run-fault-tolerance.sh create mode 100755 .buildkite/scripts/multiprocessing-test/run-lm-eval.sh create mode 100755 .buildkite/scripts/multiprocessing-test/run-long-doc-qa.sh create mode 100755 .buildkite/scripts/multiprocessing-test/run-mp-test.sh create mode 100755 .buildkite/scripts/multiprocessing-test/run-vllm-bench.sh create mode 100755 .buildkite/scripts/multiprocessing-test/test-launch.sh create mode 100755 .buildkite/scripts/multiprocessing-test/wait-for-vllm.sh create mode 100644 .buildkite/scripts/pick-free-gpu-amd.sh create mode 100644 .buildkite/scripts/pick-free-gpu.sh create mode 100755 .buildkite/scripts/should-run-comprehensive.sh create mode 100644 .buildkite/scripts/vllm-correctness.sh create mode 100755 .buildkite/scripts/vllm-integration-tests.sh create mode 100644 .buildkite/vllm-integration-tests.yml create mode 100644 .clang-format create mode 100644 .claude/skills/create-pr/SKILL.md create mode 100644 .claude/skills/hybrid-benchmarking/SKILL.md create mode 100644 .claude/skills/pr-review/SKILL.md create mode 100644 .claude/skills/pre-pr-check/SKILL.md create mode 100644 .cursor/BUGBOT.md create mode 100644 .gemini/styleguide.md create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/blank_issue.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/actions/free-disk-space/action.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/scripts/cpu_device_test.sh create mode 100644 .github/scripts/cpu_server_bench_test.sh create mode 100644 .github/scripts/cpu_vllm_e2e_test.sh create mode 100644 .github/scripts/download_model.sh create mode 100644 .github/scripts/download_opt125m_github.sh create mode 100644 .github/scripts/install_lmcache_cpu.sh create mode 100644 .github/scripts/install_vllm_cpu.sh create mode 100755 .github/scripts/run-cpu-e2e-validation.sh create mode 100644 .github/workflows/actionlint.dockerfile create mode 100644 .github/workflows/actionlint.yml create mode 100644 .github/workflows/aerospike_integration.yml create mode 100644 .github/workflows/automerge-labeler.yml create mode 100644 .github/workflows/build_cli_artifacts.yml create mode 100644 .github/workflows/build_cpu_artifacts.yml create mode 100644 .github/workflows/build_cu129_artifacts.yml create mode 100644 .github/workflows/build_doc.yml create mode 100644 .github/workflows/build_main_artifacts.yml create mode 100644 .github/workflows/code_quality_checks.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/cpu_device.yml create mode 100644 .github/workflows/matchers/actionlint.json create mode 100644 .github/workflows/matchers/mypy.json create mode 100644 .github/workflows/nightly_build.yml create mode 100644 .github/workflows/operator_ci.yml create mode 100644 .github/workflows/operator_nightly.yml create mode 100644 .github/workflows/operator_release.yml create mode 100644 .github/workflows/pr_full_build.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/scorecard.yml create mode 100644 .github/workflows/stale_bot.yml create mode 100644 .github/workflows/sync_torch_version.yml create mode 100644 .github/workflows/test.yml create mode 100644 .github/workflows/translate_doc_zh.yml create mode 100644 .gitignore create mode 100644 .isort.cfg create mode 100644 .pre-commit-config.yaml create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 CMakeLists.txt create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 DCO create mode 100644 LICENSE create mode 100644 MAINTAINERS.md create mode 100644 MANIFEST.in create mode 100644 README.md create mode 100644 README.wehub.md create mode 100644 SECURITY.md create mode 100644 asset/deployment_modes_dark.png create mode 100644 asset/deployment_modes_light.png create mode 100644 asset/ecosystem.png create mode 100644 asset/logo.png create mode 100644 asset/partner_dark.png create mode 100644 asset/partner_light.png create mode 100644 benchmarks/long_doc_qa/long_doc_qa.py create mode 100644 benchmarks/long_doc_qa/long_doc_qa_recommender.py create mode 100644 benchmarks/microbenchmark/bitmap_ops_benchmark.py create mode 100644 benchmarks/microbenchmark/ttl_lock_benchmark.py create mode 100644 benchmarks/multi_doc_qa/README.md create mode 100644 benchmarks/multi_doc_qa/lmcache.yaml create mode 100644 benchmarks/multi_doc_qa/lmcache_blend.yaml create mode 100644 benchmarks/multi_doc_qa/multi_doc_qa.py create mode 100644 benchmarks/multi_doc_qa/shuffle_doc_qa.py create mode 100644 benchmarks/multi_round_qa/README.md create mode 100644 benchmarks/multi_round_qa/data_preprocessing.py create mode 100644 benchmarks/multi_round_qa/multi-round-qa.py create mode 100644 benchmarks/multi_round_qa/prepare_sharegpt_data.sh create mode 100644 benchmarks/multi_round_qa/requirements.txt create mode 100644 benchmarks/multi_round_qa/utils.py create mode 100644 benchmarks/musa/README.md create mode 100644 benchmarks/musa/bench_inprocess_transfer.py create mode 100644 benchmarks/rag/README.md create mode 100644 benchmarks/rag/launch_lmcache.sh create mode 100644 benchmarks/rag/launch_vllm.sh create mode 100644 benchmarks/rag/precompute.py create mode 100644 benchmarks/rag/rag.py create mode 100644 benchmarks/rag/requirements.txt create mode 100644 benchmarks/rag/utils.py create mode 100644 benchmarks/storage_backend_io/README.md create mode 100644 benchmarks/storage_backend_io/storage_backend_io_benchmark.py create mode 100644 benchmarks/ttft-estimator/llama-h100-example.png create mode 100644 benchmarks/ttft-estimator/ttft-estimator.py create mode 100644 csrc/ac_dec.cu create mode 100644 csrc/ac_enc.cu create mode 100644 csrc/cachegen_kernels.cuh create mode 100644 csrc/cal_cdf.cu create mode 100644 csrc/completion_recorder.cpp create mode 100644 csrc/completion_recorder.h create mode 100644 csrc/cuda_compat.h create mode 100644 csrc/dispatch_utils.h create mode 100644 csrc/engine_kv_format.h create mode 100644 csrc/event_recorder.cpp create mode 100644 csrc/event_recorder.h create mode 100644 csrc/kv_transfer_types.h create mode 100644 csrc/mem_alloc.cpp create mode 100644 csrc/mem_alloc.h create mode 100644 csrc/mem_kernels.cu create mode 100644 csrc/mem_kernels.cuh create mode 100644 csrc/mp_mem_kernels.cu create mode 100644 csrc/mp_mem_kernels.cuh create mode 100644 csrc/pos_kernels.cu create mode 100644 csrc/pos_kernels.cuh create mode 100644 csrc/pybind.cpp create mode 100644 csrc/storage_backends/README.md create mode 100644 csrc/storage_backends/aerospike/connector.cpp create mode 100644 csrc/storage_backends/aerospike/connector.h create mode 100644 csrc/storage_backends/aerospike/pybind.cpp create mode 100644 csrc/storage_backends/connector_base.h create mode 100644 csrc/storage_backends/connector_interface.h create mode 100644 csrc/storage_backends/connector_pybind_utils.h create mode 100644 csrc/storage_backends/connector_types.h create mode 100644 csrc/storage_backends/event_notifier.h create mode 100644 csrc/storage_backends/fs/connector.cpp create mode 100644 csrc/storage_backends/fs/connector.h create mode 100644 csrc/storage_backends/fs/pybind.cpp create mode 100644 csrc/storage_backends/mooncake/connector.cpp create mode 100644 csrc/storage_backends/mooncake/connector.h create mode 100644 csrc/storage_backends/mooncake/pybind.cpp create mode 100644 csrc/storage_backends/redis/connector.cpp create mode 100644 csrc/storage_backends/redis/connector.h create mode 100644 csrc/storage_backends/redis/pybind.cpp create mode 100644 csrc/storage_manager/bitmap.cpp create mode 100644 csrc/storage_manager/bitmap.h create mode 100644 csrc/storage_manager/fold.cpp create mode 100644 csrc/storage_manager/fold.h create mode 100644 csrc/storage_manager/periodic_event_notifier.cpp create mode 100644 csrc/storage_manager/periodic_event_notifier.h create mode 100644 csrc/storage_manager/pybind.cpp create mode 100644 csrc/storage_manager/ttl_lock.cpp create mode 100644 csrc/storage_manager/ttl_lock.h create mode 100644 csrc/storage_manager/utils.cpp create mode 100644 csrc/storage_manager/utils.h create mode 100644 csrc/sycl/ac_dec_sycl.cpp create mode 100644 csrc/sycl/ac_enc_sycl.cpp create mode 100644 csrc/sycl/cachegen_kernels_sycl.h create mode 100644 csrc/sycl/cal_cdf_sycl.cpp create mode 100644 csrc/sycl/mem_kernels_sycl.cpp create mode 100644 csrc/sycl/mem_kernels_sycl.h create mode 100644 csrc/sycl/pos_kernels_sycl.cpp create mode 100644 csrc/sycl/pybind_sycl.cpp create mode 100644 csrc/utils.cpp create mode 100644 csrc/utils.h create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.lightweight create mode 100644 docker/Dockerfile.payload create mode 100644 docker/Dockerfile.rocm create mode 100644 docker/Dockerfile.rocm-lightweight create mode 100644 docker/Dockerfile.standalone create mode 100644 docker/Dockerfile.xpu create mode 100644 docker/README.md create mode 100644 docker/example_build.sh create mode 100644 docker/example_run.sh create mode 100644 docs/Makefile create mode 100644 docs/README.md create mode 100644 docs/coding_standards.md create mode 100644 docs/design/ARCHITECTURE_MULTI_HARDWARE.md create mode 100644 docs/design/README.md create mode 100644 docs/design/cli/commands.md create mode 100644 docs/design/cli/commands/bench/engine_bench/bench-engine.md create mode 100644 docs/design/cli/commands/describe.md create mode 100644 docs/design/cli/commands/kvcache-command.md create mode 100644 docs/design/cli/commands/ping.md create mode 100644 docs/design/cli/commands/query-command.md create mode 100644 docs/design/cli/framework-and-metrics.md create mode 100644 docs/design/integration/tensorrt_llm/README.md create mode 100644 docs/design/integration/vllm/hybrid-kv-cache-groups.md create mode 100644 docs/design/integration/vllm/kv_cache_group_edits.md create mode 100644 docs/design/observability/request-event-span.md create mode 100644 docs/design/sdk/batch.md create mode 100644 docs/design/sdk/kvcache.md create mode 100644 docs/design/sdk/stream.md create mode 120000 docs/design/tools/controller_benchmark/README.md create mode 120000 docs/design/tools/transfer_channel_benchmark/README.md create mode 120000 docs/design/usage_telemetry/README.md create mode 100644 docs/design/v1/distributed/bitmap_ops/README.md create mode 100644 docs/design/v1/distributed/l2_adapters/dax.md create mode 100644 docs/design/v1/distributed/l2_adapters/fault_inject.md create mode 100644 docs/design/v1/distributed/l2_adapters/l2_eviction.md create mode 100644 docs/design/v1/distributed/l2_adapters/l2_per_user_quota.md create mode 100644 docs/design/v1/distributed/l2_adapters/nixl_store.md create mode 100644 docs/design/v1/distributed/l2_adapters/overall.md create mode 100644 docs/design/v1/distributed/l2_adapters/p2p_l2_adapter.md create mode 100644 docs/design/v1/distributed/l2_adapters/plugin.md create mode 100644 docs/design/v1/distributed/l2_adapters/raw_block.md create mode 100644 docs/design/v1/distributed/l2_adapters/serde_wrapper.md create mode 100644 docs/design/v1/distributed/serde/README.md create mode 100644 docs/design/v1/distributed/serde/turboquant.md create mode 100644 docs/design/v1/distributed/transfer_channel/overall.md create mode 100644 docs/design/v1/encoder-cache.md create mode 100644 docs/design/v1/gpu_connector/layout-invariant.md create mode 100644 docs/design/v1/hidden_state_store.md create mode 100644 docs/design/v1/mp_coordinator/README.md create mode 100644 docs/design/v1/mp_coordinator/blend_lookup.md create mode 100644 docs/design/v1/mp_coordinator/l2_prefetch.md create mode 100644 docs/design/v1/mp_coordinator/l2_usage_and_eviction.md create mode 100644 docs/design/v1/mp_observability/DEBUG.md create mode 100644 docs/design/v1/mp_observability/EVENTS.md create mode 100644 docs/design/v1/mp_observability/METRICS.md create mode 120000 docs/design/v1/mp_observability/README.md create mode 100644 docs/design/v1/mp_observability/blend_v3_observability.md create mode 100644 docs/design/v1/mp_observability/errors.md create mode 100644 docs/design/v1/mp_observability/event-bus.md create mode 100644 docs/design/v1/mp_observability/trace.md create mode 100644 docs/design/v1/multiprocess/device_ipc_wrapper_design.md create mode 100644 docs/design/v1/multiprocess/engine_driven_transfer_design.md create mode 100644 docs/design/v1/multiprocess/http_api_extension.md create mode 100644 docs/design/v1/multiprocess/mp_runtime_plugin.md create mode 120000 docs/design/v1/multiprocess/protocols/README.md create mode 100644 docs/design/v1/multiprocess/raw_cuda_ipc.md create mode 100644 docs/design/v1/multiprocess/worker_liveness.md create mode 100644 docs/design/v1/pd_async_reservation_design.md create mode 100644 docs/design/v1/platform/event_notifier.md create mode 100644 docs/make.bat create mode 100644 docs/source/.nojekyll create mode 100644 docs/source/_static/basic_codepath.svg create mode 100644 docs/source/_static/bench_interactive_demo.gif create mode 100644 docs/source/_static/custom.css create mode 100644 docs/source/_static/custom.js create mode 100644 docs/source/_static/full_layerwise_diagram.svg create mode 100644 docs/source/_static/kv_cache_calculator.html create mode 100644 docs/source/_static/modelconfig.json create mode 100644 docs/source/_static/scroll.css create mode 100644 docs/source/_templates/custom.html create mode 100644 docs/source/_templates/versioning.html create mode 100644 docs/source/api_reference/configurations.rst create mode 100644 docs/source/api_reference/dynamic_connector.rst create mode 100644 docs/source/api_reference/multimodality.rst create mode 100644 docs/source/api_reference/storage_backends.rst create mode 100644 docs/source/assets/InfiniStore-usage.png create mode 100644 docs/source/assets/deployment_modes_dark.png create mode 100644 docs/source/assets/deployment_modes_light.png create mode 100644 docs/source/assets/ecosystem.png create mode 100644 docs/source/assets/lmcache-controller.png create mode 100644 docs/source/assets/lmcache-logo.png create mode 100644 docs/source/assets/lmcache-logo_crop.png create mode 100644 docs/source/assets/maru-kvcache.png create mode 100644 docs/source/assets/mooncake-store-preview.png create mode 100644 docs/source/cli/bench.rst create mode 100644 docs/source/cli/coordinator.rst create mode 100644 docs/source/cli/describe.rst create mode 100644 docs/source/cli/index.rst create mode 100644 docs/source/cli/kvcache.rst create mode 100644 docs/source/cli/ping.rst create mode 100644 docs/source/cli/query.rst create mode 100644 docs/source/cli/quota.rst create mode 100644 docs/source/cli/server.rst create mode 100644 docs/source/cli/tool.rst create mode 100644 docs/source/cli/trace.rst create mode 100644 docs/source/community/blogs.rst create mode 100644 docs/source/community/index.rst create mode 100644 docs/source/community/meetings.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/controller/freeze_mode.rst create mode 100644 docs/source/controller/index.rst create mode 100644 docs/source/developer_guide/architecture.rst create mode 100644 docs/source/developer_guide/cli.rst create mode 100644 docs/source/developer_guide/contributing.rst create mode 100644 docs/source/developer_guide/extending_http_api.rst create mode 100644 docs/source/developer_guide/extending_lmcache/native_connectors.rst create mode 100644 docs/source/developer_guide/extending_lmcache/remote_storage_plugins.rst create mode 100644 docs/source/developer_guide/extending_lmcache/runtime_plugins.rst create mode 100644 docs/source/developer_guide/extending_lmcache/storage_plugins.rst create mode 100644 docs/source/developer_guide/index.rst create mode 100644 docs/source/developer_guide/integration.rst create mode 100644 docs/source/developer_guide/usage/basic_check.rst create mode 100644 docs/source/developer_guide/usage/index.rst create mode 100644 docs/source/developer_guide/usage/usage_stats_collection.rst create mode 100644 docs/source/disaggregated_prefill/nixl/1p1d.rst create mode 100644 docs/source/disaggregated_prefill/nixl/index.rst create mode 100644 docs/source/disaggregated_prefill/nixl/xpyd.rst create mode 100644 docs/source/disaggregated_prefill/shared_storage.rst create mode 100644 docs/source/distributed_kv_cache.rst create mode 100644 docs/source/getting_started/benchmarking.rst create mode 100644 docs/source/getting_started/index.rst create mode 100644 docs/source/getting_started/installation.rst create mode 100644 docs/source/getting_started/kv_cache_calculator.rst create mode 100644 docs/source/getting_started/quickstart.rst create mode 100644 docs/source/getting_started/quickstart/disaggregated_prefill.rst create mode 100644 docs/source/getting_started/quickstart/index.rst create mode 100644 docs/source/getting_started/quickstart/multimodality.rst create mode 100644 docs/source/getting_started/quickstart/offload_kv_cache.rst create mode 100644 docs/source/getting_started/quickstart/share_kv_cache.rst create mode 100644 docs/source/getting_started/quickstart/standalone_starter.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/interacting_with_server.rst create mode 100644 docs/source/internal_api_server/common_apis.rst create mode 100644 docs/source/internal_api_server/controller_apis.rst create mode 100644 docs/source/internal_api_server/dynamic_backend_management.rst create mode 100644 docs/source/internal_api_server/internal_api_server.rst create mode 100644 docs/source/internal_api_server/vllm_apis.rst create mode 100644 docs/source/kv_cache/async_loading.rst create mode 100644 docs/source/kv_cache/caching_policies.rst create mode 100644 docs/source/kv_cache/multiprocess_mode.rst create mode 100644 docs/source/kv_cache/p2p_sharing.rst create mode 100644 docs/source/kv_cache/storage_backends/3fs.rst create mode 100644 docs/source/kv_cache/storage_backends/azure.rst create mode 100644 docs/source/kv_cache/storage_backends/bigtable.rst create mode 100644 docs/source/kv_cache/storage_backends/cpu_ram.rst create mode 100644 docs/source/kv_cache/storage_backends/custom_backend.rst create mode 100644 docs/source/kv_cache/storage_backends/dax.rst create mode 100644 docs/source/kv_cache/storage_backends/eic.rst create mode 100644 docs/source/kv_cache/storage_backends/fs.rst create mode 100644 docs/source/kv_cache/storage_backends/gds.rst create mode 100644 docs/source/kv_cache/storage_backends/hfbucket.rst create mode 100644 docs/source/kv_cache/storage_backends/index.rst create mode 100644 docs/source/kv_cache/storage_backends/infinistore.rst create mode 100644 docs/source/kv_cache/storage_backends/local_storage.rst create mode 100644 docs/source/kv_cache/storage_backends/maru.rst create mode 100644 docs/source/kv_cache/storage_backends/mock.rst create mode 100644 docs/source/kv_cache/storage_backends/mooncake.rst create mode 100644 docs/source/kv_cache/storage_backends/nixl.rst create mode 100644 docs/source/kv_cache/storage_backends/redis.rst create mode 100644 docs/source/kv_cache/storage_backends/resp.rst create mode 100644 docs/source/kv_cache/storage_backends/s3.rst create mode 100644 docs/source/kv_cache/storage_backends/sagemaker_hyperpod.rst create mode 100644 docs/source/kv_cache/storage_backends/valkey.rst create mode 100644 docs/source/kv_cache/storage_backends/weka.rst create mode 100644 docs/source/kv_cache_management/check_finish.rst create mode 100644 docs/source/kv_cache_management/clear.rst create mode 100644 docs/source/kv_cache_management/compress.rst create mode 100644 docs/source/kv_cache_management/health.rst create mode 100644 docs/source/kv_cache_management/index.rst create mode 100644 docs/source/kv_cache_management/lookup.rst create mode 100644 docs/source/kv_cache_management/move.rst create mode 100644 docs/source/kv_cache_management/pin.rst create mode 100644 docs/source/kv_cache_management/query_worker_info.rst create mode 100644 docs/source/kv_cache_optimizations/blending.rst create mode 100644 docs/source/kv_cache_optimizations/cacheblend.rst create mode 100644 docs/source/kv_cache_optimizations/compression/cachegen.rst create mode 100644 docs/source/kv_cache_optimizations/compression/index.rst create mode 100644 docs/source/kv_cache_optimizations/index.rst create mode 100644 docs/source/kv_cache_optimizations/layerwise.rst create mode 100644 docs/source/kv_cache_optimizations/segmented_prefill.rst create mode 100644 docs/source/legacy/index.rst create mode 100644 docs/source/locale/README.md create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/api_reference/configurations.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/api_reference/dynamic_connector.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/api_reference/multimodality.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/api_reference/storage_backends.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/bench.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/coordinator.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/describe.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/kvcache.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/ping.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/query.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/quota.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/server.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/tool.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/cli/trace.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/community/blogs.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/community/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/community/meetings.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/controller/freeze_mode.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/controller/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/architecture.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/cli.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/contributing.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_http_api.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/native_connectors.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/remote_storage_plugins.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/runtime_plugins.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/extending_lmcache/storage_plugins.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/integration.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/basic_check.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/developer_guide/usage/usage_stats_collection.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/1p1d.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/nixl/xpyd.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/disaggregated_prefill/shared_storage.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/distributed_kv_cache.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/benchmarking.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/installation.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/kv_cache_calculator.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/disaggregated_prefill.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/multimodality.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/offload_kv_cache.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/share_kv_cache.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/getting_started/quickstart/standalone_starter.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/interacting_with_server.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/common_apis.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/controller_apis.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/dynamic_backend_management.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/internal_api_server.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/internal_api_server/vllm_apis.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/async_loading.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/caching_policies.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/multiprocess_mode.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/p2p_sharing.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/3fs.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/bigtable.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/cpu_ram.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/custom_backend.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/dax.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/eic.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/fs.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/gds.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/hfbucket.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/infinistore.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/local_storage.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/maru.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mock.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/mooncake.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/nixl.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/redis.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/resp.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/s3.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/sagemaker_hyperpod.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/valkey.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache/storage_backends/weka.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/check_finish.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/clear.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/compress.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/health.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/lookup.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/move.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/pin.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_management/query_worker_info.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/blending.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/cacheblend.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/cachegen.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/compression/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/layerwise.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/kv_cache_optimizations/segmented_prefill.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/legacy/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/cachegen.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/configuration.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/coordinator.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/deployment.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/disaggregated_prefill.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/frontend_dashboard.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/http_api.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/hybrid_models.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/kv_cache_management.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/aerospike.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/dax.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/fault_inject.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/file_and_block.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/fs.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/fs_native.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/hfbucket.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/mock.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/mooncake_store.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/nixl.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/plugin.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/raw_block.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/remote_and_distributed.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/resp.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/s3.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/l2_storage/supported_storages.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/observability/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/observability/logs.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/observability/metrics.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/observability/traces.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/operator.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/p2p.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/sdk.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/serde.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/mp/tracing_and_debugging.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/encoder_cache.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/encodings.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/hidden_states.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/non_kv_cache/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/docker_deployment.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/dynamo_coordination.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/kubernetes_deployment.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/kv_cache_events.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/chunk_statistics.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/frontend.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/health_monitor.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/internal_api_server.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/metrics.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/periodic_thread_api.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/observability/vllm_endpoint.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/production/performance_tuning.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/deepseek_v4_flash.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/devstral.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/gemma3.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/gemma4.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/glm5_2.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/gpt_oss.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/index.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/llama.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/minimax_m2.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/minimax_m3.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/mixtral.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/phi3.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/qwen3.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/qwen3_5.po create mode 100644 docs/source/locale/zh_CN/LC_MESSAGES/recipes/uniform_attention_models.po create mode 100644 docs/source/mp/architecture.png create mode 100644 docs/source/mp/cachegen.rst create mode 100644 docs/source/mp/configuration.rst create mode 100644 docs/source/mp/coordinator.rst create mode 100644 docs/source/mp/deployment.rst create mode 100644 docs/source/mp/disaggregated_prefill.rst create mode 100644 docs/source/mp/frontend_dashboard.rst create mode 100644 docs/source/mp/http_api.rst create mode 100644 docs/source/mp/hybrid_models.rst create mode 100644 docs/source/mp/index.rst create mode 100644 docs/source/mp/kv_cache_management.rst create mode 100644 docs/source/mp/l2_storage/aerospike.rst create mode 100644 docs/source/mp/l2_storage/dax.rst create mode 100644 docs/source/mp/l2_storage/fault_inject.rst create mode 100644 docs/source/mp/l2_storage/file_and_block.rst create mode 100644 docs/source/mp/l2_storage/fs.rst create mode 100644 docs/source/mp/l2_storage/fs_native.rst create mode 100644 docs/source/mp/l2_storage/hfbucket.rst create mode 100644 docs/source/mp/l2_storage/index.rst create mode 100644 docs/source/mp/l2_storage/mock.rst create mode 100644 docs/source/mp/l2_storage/mooncake_store.rst create mode 100644 docs/source/mp/l2_storage/nixl.rst create mode 100644 docs/source/mp/l2_storage/plugin.rst create mode 100644 docs/source/mp/l2_storage/raw_block.rst create mode 100644 docs/source/mp/l2_storage/remote_and_distributed.rst create mode 100644 docs/source/mp/l2_storage/resp.rst create mode 100644 docs/source/mp/l2_storage/s3.rst create mode 100644 docs/source/mp/l2_storage/supported_storages.rst create mode 100644 docs/source/mp/observability/index.rst create mode 100644 docs/source/mp/observability/logs.rst create mode 100644 docs/source/mp/observability/metrics.rst create mode 100644 docs/source/mp/observability/traces.rst create mode 100644 docs/source/mp/operator.rst create mode 100644 docs/source/mp/p2p.rst create mode 100644 docs/source/mp/sdk.rst create mode 100644 docs/source/mp/serde.rst create mode 100644 docs/source/mp/tracing_and_debugging.rst create mode 100644 docs/source/non_kv_cache/encoder_cache.rst create mode 100644 docs/source/non_kv_cache/encodings.rst create mode 100644 docs/source/non_kv_cache/hidden_states.rst create mode 100644 docs/source/non_kv_cache/index.rst create mode 100644 docs/source/production/docker_deployment.rst create mode 100644 docs/source/production/dynamo_coordination.rst create mode 100644 docs/source/production/index.rst create mode 100644 docs/source/production/kubernetes_deployment.rst create mode 100644 docs/source/production/kv_cache_events.rst create mode 100644 docs/source/production/observability/chunk_statistics.rst create mode 100644 docs/source/production/observability/frontend.rst create mode 100644 docs/source/production/observability/health_monitor.rst create mode 100644 docs/source/production/observability/index.rst create mode 100644 docs/source/production/observability/internal_api_server.rst create mode 100644 docs/source/production/observability/metrics.rst create mode 100644 docs/source/production/observability/periodic_thread_api.rst create mode 100644 docs/source/production/observability/vllm_endpoint.rst create mode 100644 docs/source/production/performance_tuning.rst create mode 100644 docs/source/recipes/deepseek_v4_flash.rst create mode 100644 docs/source/recipes/devstral.rst create mode 100644 docs/source/recipes/gemma3.rst create mode 100644 docs/source/recipes/gemma4.rst create mode 100644 docs/source/recipes/glm5_2.rst create mode 100644 docs/source/recipes/gpt_oss.rst create mode 100644 docs/source/recipes/index.rst create mode 100644 docs/source/recipes/llama.rst create mode 100644 docs/source/recipes/minimax_m2.rst create mode 100644 docs/source/recipes/minimax_m3.rst create mode 100644 docs/source/recipes/mixtral.rst create mode 100644 docs/source/recipes/phi3.rst create mode 100644 docs/source/recipes/qwen3.rst create mode 100644 docs/source/recipes/qwen3_5.rst create mode 100644 docs/source/recipes/uniform_attention_models.rst create mode 100644 examples/README.md create mode 100644 examples/agents/prefix_analysis.py create mode 100644 examples/agents/requirements.txt create mode 100644 examples/basic_check/README.md create mode 100644 examples/basic_check/example_config.yaml create mode 100644 examples/blend_kv/README.md create mode 100644 examples/blend_kv/batched_kv.py create mode 100644 examples/blend_kv/batched_tp_kv.py create mode 100644 examples/blend_kv/blend_kv.py create mode 100644 examples/blend_kv/chunk1.txt create mode 100644 examples/blend_kv/chunk2.txt create mode 100644 examples/blend_kv/example_blending.yaml create mode 100644 examples/blend_kv/online_kv.py create mode 100644 examples/blend_kv/tp_kv.py create mode 100644 examples/blend_kv_v1/README.md create mode 100644 examples/blend_kv_v1/blend.py create mode 100644 examples/cache_controller/README.md create mode 100644 examples/cache_controller/clear/README.md create mode 100644 examples/cache_controller/clear/example.yaml create mode 100644 examples/cache_controller/compress/README.md create mode 100644 examples/cache_controller/compress/example.yaml create mode 100644 examples/cache_controller/health/README.md create mode 100644 examples/cache_controller/health/example.yaml create mode 100644 examples/cache_controller/lookup/README.md create mode 100644 examples/cache_controller/lookup/example.yaml create mode 100644 examples/cache_controller/move/README.md create mode 100644 examples/cache_controller/move/instance1.yaml create mode 100644 examples/cache_controller/move/instance2.yaml create mode 100644 examples/cache_controller/pin/README.md create mode 100644 examples/cache_controller/pin/example.yaml create mode 100644 examples/cache_interface/README.md create mode 100644 examples/cache_interface/example.yaml create mode 100644 examples/cache_with_configs/README.md create mode 100644 examples/cache_with_configs/example.yaml create mode 100644 examples/chunk_statistics/README.md create mode 100644 examples/chunk_statistics/analyze_chunk_hashes.py create mode 100644 examples/chunk_statistics/auto_stop.yaml create mode 100644 examples/chunk_statistics/file_hash.yaml create mode 100644 examples/chunk_statistics/memory_bloom_filter.yaml create mode 100644 examples/disagg_prefill/1p1d/README.md create mode 100644 examples/disagg_prefill/1p1d/configs/lmcache-decoder-config.yaml create mode 100644 examples/disagg_prefill/1p1d/configs/lmcache-decoder-pd-with-remote-config.yaml create mode 100644 examples/disagg_prefill/1p1d/configs/lmcache-prefiller-config.yaml create mode 100644 examples/disagg_prefill/1p1d/configs/lmcache-prefiller-pd-with-remote-config.yaml create mode 100644 examples/disagg_prefill/1p1d/disagg_example_1p1d.sh create mode 100644 examples/disagg_prefill/1p1d/disagg_vllm_launcher.sh create mode 100644 examples/disagg_prefill/README.md create mode 100644 examples/disagg_prefill/disagg_proxy_server.py create mode 100644 examples/disagg_prefill/xpyd/README.md create mode 100644 examples/disagg_prefill/xpyd/configs/lmcache-decoder-1-config.yaml create mode 100644 examples/disagg_prefill/xpyd/configs/lmcache-decoder-1-pd-with-remote-config.yaml create mode 100644 examples/disagg_prefill/xpyd/configs/lmcache-decoder-2-config.yaml create mode 100644 examples/disagg_prefill/xpyd/configs/lmcache-decoder-2-pd-with-remote-config.yaml create mode 100644 examples/disagg_prefill/xpyd/configs/lmcache-prefiller-config.yaml create mode 100644 examples/disagg_prefill/xpyd/configs/lmcache-prefiller-pd-with-remote-config.yaml create mode 100644 examples/disagg_prefill/xpyd/disagg_example_xpyd.sh create mode 100644 examples/disagg_prefill/xpyd/disagg_vllm_launcher.sh create mode 100644 examples/disagg_prefill_mp/README.md create mode 100644 examples/disagg_prefill_mp/architecture.png create mode 100644 examples/disagg_prefill_mp/disagg_proxy_server.py create mode 100644 examples/dynamo_integration/README.md create mode 100644 examples/dynamo_integration/deploy/agg_lmcache_mp.yaml create mode 100644 examples/dynamo_integration/deploy/disagg_lmcache_mp.yaml create mode 100644 examples/dynamo_integration/deploy/lmcache_engine.yaml create mode 100755 examples/dynamo_integration/launch/agg_lmcache_mp.sh create mode 100755 examples/dynamo_integration/launch/disagg_lmcache_mp.sh create mode 100644 examples/frontend/README.md create mode 100644 examples/frontend/chat_session.py create mode 100644 examples/frontend/example.yaml create mode 100644 examples/frontend/ffmpeg.txt create mode 100644 examples/frontend/frontend.py create mode 100644 examples/kubernetes/health_probe.py create mode 100644 examples/kv_cache_calculator/README.md create mode 100644 examples/kv_cache_calculator/generate_config.py create mode 100644 examples/kv_cache_calculator/kv_cache_calculator.html create mode 100644 examples/kv_cache_calculator/modelconfig.json create mode 100644 examples/kv_cache_calculator/requirement.txt create mode 100644 examples/kv_cache_reuse/README.md create mode 100644 examples/kv_cache_reuse/local_backends/README.md create mode 100644 examples/kv_cache_reuse/local_backends/offload.py create mode 100644 examples/kv_cache_reuse/local_backends/rust_backend_offload.py create mode 100644 examples/kv_cache_reuse/remote_backends/README.md create mode 100644 examples/kv_cache_reuse/remote_backends/external/README.md create mode 100644 examples/kv_cache_reuse/remote_backends/external/backend_type.yaml create mode 100644 examples/kv_cache_reuse/remote_backends/fs/example.yaml create mode 100644 examples/kv_cache_reuse/remote_backends/infinistore/README.md create mode 100644 examples/kv_cache_reuse/remote_backends/infinistore/backend_type.yaml create mode 100644 examples/kv_cache_reuse/remote_backends/mock/README.md create mode 100644 examples/kv_cache_reuse/remote_backends/mock/example.yaml create mode 100644 examples/kv_cache_reuse/remote_backends/mooncakestore/README.md create mode 100644 examples/kv_cache_reuse/remote_backends/mooncakestore/mooncake.yaml create mode 100644 examples/kv_cache_reuse/remote_backends/resp/README.md create mode 100644 examples/kv_cache_reuse/remote_backends/resp/benchmark_resp_client.py create mode 100644 examples/kv_cache_reuse/remote_backends/resp/resp.yaml create mode 100644 examples/kv_cache_reuse/remote_backends/s3/README.md create mode 100644 examples/kv_cache_reuse/remote_backends/s3/example.yaml create mode 100644 examples/kv_cache_reuse/remote_backends/valkey/VALKEY_CONNECTOR_BENCHMARKING.md create mode 100644 examples/kv_cache_reuse/remote_backends/valkey/benchmark_l2.py create mode 100644 examples/kv_cache_reuse/remote_backends/valkey/valkey.yaml create mode 100644 examples/kv_cache_reuse/share_across_instances/README.md create mode 100644 examples/kv_cache_reuse/share_across_instances/centralized_sharing/README.md create mode 100644 examples/kv_cache_reuse/share_across_instances/centralized_sharing/example.yaml create mode 100644 examples/kv_cache_reuse/share_across_instances/p2p_sharing/README.md create mode 100644 examples/kv_cache_reuse/share_across_instances/p2p_sharing/example1.yaml create mode 100644 examples/kv_cache_reuse/share_across_instances/p2p_sharing/example2.yaml create mode 100644 examples/kvcache_sdk/README.md create mode 100644 examples/kvcache_sdk/e2e_kv_edit.ipynb create mode 100644 examples/lmc_external_l2_adapter/pyproject.toml create mode 100755 examples/lmc_external_l2_adapter/scripts/install_and_test.sh create mode 100644 examples/lmc_external_l2_adapter/src/lmc_external_l2_adapter/__init__.py create mode 100644 examples/lmc_external_l2_adapter/src/lmc_external_l2_adapter/adapter.py create mode 100644 examples/lmc_external_l2_adapter/tests/test_plugin.py create mode 100644 examples/lmc_external_native_connector/csrc/connector.cpp create mode 100644 examples/lmc_external_native_connector/csrc/connector.h create mode 100644 examples/lmc_external_native_connector/csrc/pybind.cpp create mode 100644 examples/lmc_external_native_connector/pyproject.toml create mode 100644 examples/lmc_external_native_connector/setup.py create mode 100644 examples/lmc_external_native_connector/src/lmc_external_native_connector/__init__.py create mode 100644 examples/lmc_external_native_connector/src/lmc_external_native_connector/connector.py create mode 100644 examples/mp_runtime_plugins/README.md create mode 100755 examples/mp_runtime_plugins/mp_heartbeat.sh create mode 100644 examples/mp_runtime_plugins/mp_plugin.py create mode 100644 examples/multi_process/lmcache-daemonset.yaml create mode 100644 examples/multi_process/vllm-deployment.yaml create mode 100644 examples/observability/README.md create mode 100644 examples/observability/docker-compose.yml create mode 100644 examples/observability/grafana/provisioning/dashboards/dashboards.yml create mode 100644 examples/observability/grafana/provisioning/dashboards/lmcache.json create mode 100644 examples/observability/grafana/provisioning/datasources/datasources.yml create mode 100644 examples/observability/otel-collector.yml create mode 100644 examples/observability/prometheus.yml create mode 100644 examples/observability/start-server.sh create mode 100644 examples/observability/tempo.yml create mode 100644 examples/online_session/README.md create mode 100644 examples/online_session/bench_ttft_sweep.sh create mode 100644 examples/online_session/example.yaml create mode 100644 examples/online_session/ffmpeg.txt create mode 100644 examples/online_session/openai_chat_completion_client.py create mode 100644 examples/p2p/README.md create mode 100644 examples/p2p/send_request.py create mode 100644 examples/redis_lookup/README.md create mode 100644 examples/remote_config_server/README.md create mode 100644 examples/remote_config_server/config_server.py create mode 100644 examples/remote_config_server/example.yaml create mode 100644 examples/runtime_plugins/README.md create mode 100644 examples/runtime_plugins/all_plugin.sh create mode 100644 examples/runtime_plugins/scheduler_foo_plugin.py create mode 100644 examples/runtime_plugins/worker_0_test.sh create mode 100644 examples/serde/fp8/README.md create mode 100755 examples/serde/fp8/run_serde_fp8_example.sh create mode 100644 examples/serde/turboquant/README.md create mode 100644 examples/serde/turboquant/bench_serde_baselines.py create mode 100644 examples/serde/turboquant/bench_turboquant.py create mode 100644 examples/sgl_integration/README.md create mode 100644 examples/token_dropping/token_dropping.ipynb create mode 100644 examples/token_dropping/utils.py create mode 100755 format.sh create mode 100644 lmcache/__init__.py create mode 100644 lmcache/banner.py create mode 100644 lmcache/cli/__init__.py create mode 100644 lmcache/cli/commands/__init__.py create mode 100644 lmcache/cli/commands/base.py create mode 100644 lmcache/cli/commands/bench/__init__.py create mode 100644 lmcache/cli/commands/bench/engine_bench/__init__.py create mode 100644 lmcache/cli/commands/bench/engine_bench/command.py create mode 100644 lmcache/cli/commands/bench/engine_bench/config.py create mode 100644 lmcache/cli/commands/bench/engine_bench/interactive/__init__.py create mode 100644 lmcache/cli/commands/bench/engine_bench/interactive/config.json create mode 100644 lmcache/cli/commands/bench/engine_bench/interactive/schema.py create mode 100644 lmcache/cli/commands/bench/engine_bench/interactive/state.py create mode 100644 lmcache/cli/commands/bench/engine_bench/interactive/terminal.py create mode 100644 lmcache/cli/commands/bench/engine_bench/progress.py create mode 100644 lmcache/cli/commands/bench/engine_bench/request_sender.py create mode 100644 lmcache/cli/commands/bench/engine_bench/stats.py create mode 100644 lmcache/cli/commands/bench/engine_bench/workloads/__init__.py create mode 100644 lmcache/cli/commands/bench/engine_bench/workloads/base.py create mode 100644 lmcache/cli/commands/bench/engine_bench/workloads/long_doc_permutator.py create mode 100644 lmcache/cli/commands/bench/engine_bench/workloads/long_doc_qa.py create mode 100644 lmcache/cli/commands/bench/engine_bench/workloads/multi_round_chat.py create mode 100644 lmcache/cli/commands/bench/engine_bench/workloads/prefix_suffix_tuner.py create mode 100644 lmcache/cli/commands/bench/engine_bench/workloads/random_prefill.py create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/__init__.py create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/command.py create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/data.py create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/profiling.py create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/result.py create mode 100644 lmcache/cli/commands/bench/l2_adapter_bench/runner.py create mode 100644 lmcache/cli/commands/bench/server_bench/__init__.py create mode 100644 lmcache/cli/commands/bench/server_bench/command.py create mode 100644 lmcache/cli/commands/bench/server_bench/helpers.py create mode 100644 lmcache/cli/commands/coordinator.py create mode 100644 lmcache/cli/commands/describe.py create mode 100644 lmcache/cli/commands/kvcache.py create mode 100644 lmcache/cli/commands/mock.py create mode 100644 lmcache/cli/commands/ping.py create mode 100644 lmcache/cli/commands/query/__init__.py create mode 100644 lmcache/cli/commands/query/_prompt.py create mode 100644 lmcache/cli/commands/query/_request.py create mode 100644 lmcache/cli/commands/query/engine_command.py create mode 100644 lmcache/cli/commands/query/kvcache_command.py create mode 100644 lmcache/cli/commands/quota/__init__.py create mode 100644 lmcache/cli/commands/quota/_helpers.py create mode 100644 lmcache/cli/commands/quota/delete_command.py create mode 100644 lmcache/cli/commands/quota/get_command.py create mode 100644 lmcache/cli/commands/quota/list_command.py create mode 100644 lmcache/cli/commands/quota/set_command.py create mode 100644 lmcache/cli/commands/server.py create mode 100644 lmcache/cli/commands/tool/__init__.py create mode 100644 lmcache/cli/commands/tool/cache_simulator/__init__.py create mode 100644 lmcache/cli/commands/tool/cache_simulator/gen_dataset_command.py create mode 100644 lmcache/cli/commands/tool/cache_simulator/simulate_command.py create mode 100644 lmcache/cli/commands/tool/cache_simulator/sweep_command.py create mode 100644 lmcache/cli/commands/tool/transfer_channel_benchmark.py create mode 100644 lmcache/cli/commands/trace/__init__.py create mode 100644 lmcache/cli/commands/trace/_dispatch.py create mode 100644 lmcache/cli/commands/trace/_driver.py create mode 100644 lmcache/cli/commands/trace/_stats.py create mode 100644 lmcache/cli/commands/trace/info_command.py create mode 100644 lmcache/cli/commands/trace/replay_command.py create mode 100644 lmcache/cli/documents/lmcache.txt create mode 100644 lmcache/cli/main.py create mode 100644 lmcache/cli/metrics/__init__.py create mode 100644 lmcache/cli/metrics/formatter.py create mode 100644 lmcache/cli/metrics/handler.py create mode 100644 lmcache/cli/metrics/metrics.py create mode 100644 lmcache/cli/metrics/section.py create mode 100644 lmcache/connections.py create mode 100644 lmcache/integration/__init__.py create mode 100644 lmcache/integration/base_service_factory.py create mode 100644 lmcache/integration/request_telemetry/__init__.py create mode 100644 lmcache/integration/request_telemetry/base.py create mode 100644 lmcache/integration/request_telemetry/factory.py create mode 100644 lmcache/integration/request_telemetry/fastapi.py create mode 100644 lmcache/integration/request_telemetry/noop.py create mode 100644 lmcache/integration/sglang/__init__.py create mode 100644 lmcache/integration/sglang/multi_process_adapter.py create mode 100644 lmcache/integration/sglang/sglang_adapter.py create mode 100644 lmcache/integration/sglang/utils.py create mode 100644 lmcache/integration/tensorrt_llm/__init__.py create mode 100644 lmcache/integration/tensorrt_llm/tensorrt_adapter.py create mode 100644 lmcache/integration/tensorrt_llm/tensorrt_mp_adapter.py create mode 100644 lmcache/integration/tensorrt_llm/utils.py create mode 100644 lmcache/integration/vllm/__init__.py create mode 100644 lmcache/integration/vllm/kv_cache_group_edits.py create mode 100644 lmcache/integration/vllm/kv_cache_groups.py create mode 100644 lmcache/integration/vllm/lmcache_connector_v1.py create mode 100644 lmcache/integration/vllm/lmcache_connector_v1_085.py create mode 100644 lmcache/integration/vllm/lmcache_mp_connector.py create mode 100644 lmcache/integration/vllm/lmcache_mp_connector_0180.py create mode 100644 lmcache/integration/vllm/lmcache_mp_connector_0201.py create mode 100644 lmcache/integration/vllm/tests/test_mm_hash_utils.py create mode 100644 lmcache/integration/vllm/utils.py create mode 100644 lmcache/integration/vllm/vllm_ec_adapter.py create mode 100644 lmcache/integration/vllm/vllm_multi_process_adapter.py create mode 100644 lmcache/integration/vllm/vllm_service_factory.py create mode 100644 lmcache/integration/vllm/vllm_v1_adapter.py create mode 100644 lmcache/lmcache_frontend/__init__.py create mode 100644 lmcache/lmcache_frontend/app.py create mode 100644 lmcache/lmcache_frontend/config.json create mode 100644 lmcache/lmcache_frontend/heartbeat.py create mode 100644 lmcache/lmcache_frontend/lmcache_mp_plugin/__init__.py create mode 100644 lmcache/lmcache_frontend/lmcache_mp_plugin/lmcache_mp_frontend_plugin.py create mode 100755 lmcache/lmcache_frontend/run_mp_server_with_frontend.sh create mode 100644 lmcache/lmcache_frontend/static/css/style.css create mode 100644 lmcache/lmcache_frontend/static/img/logo.png create mode 100644 lmcache/lmcache_frontend/static/index.html create mode 100644 lmcache/lmcache_frontend/static/js/app.js create mode 100644 lmcache/logging.py create mode 100644 lmcache/native_storage_ops.pyi create mode 100644 lmcache/observability.py create mode 100644 lmcache/python_ops_fallback.py create mode 100644 lmcache/sdk/__init__.py create mode 100644 lmcache/sdk/batch.py create mode 100644 lmcache/sdk/kvcache.py create mode 100644 lmcache/sdk/stream.py create mode 100644 lmcache/sdk/wrapper/__init__.py create mode 100644 lmcache/sdk/wrapper/contiguous.py create mode 100644 lmcache/storage_backend/serde/__init__.py create mode 100644 lmcache/storage_backend/serde/cachegen_basics.py create mode 100644 lmcache/storage_backend/serde/cachegen_decoder.py create mode 100644 lmcache/storage_backend/serde/cachegen_encoder.py create mode 100644 lmcache/storage_backend/serde/serde.py create mode 100644 lmcache/tools/__init__.py create mode 100644 lmcache/tools/cache_simulator/README.md create mode 100644 lmcache/tools/cache_simulator/__init__.py create mode 100644 lmcache/tools/cache_simulator/docs/simulate_example.png create mode 100644 lmcache/tools/cache_simulator/docs/sweep_example.png create mode 100644 lmcache/tools/cache_simulator/gen_bench_dataset.py create mode 100644 lmcache/tools/cache_simulator/lru_cache.py create mode 100644 lmcache/tools/cache_simulator/plot_hit_rate.py create mode 100644 lmcache/tools/cache_simulator/simulator.py create mode 100644 lmcache/tools/controller_benchmark/README.md create mode 100644 lmcache/tools/controller_benchmark/__init__.py create mode 100644 lmcache/tools/controller_benchmark/__main__.py create mode 100644 lmcache/tools/controller_benchmark/benchmark.py create mode 100644 lmcache/tools/controller_benchmark/config.py create mode 100644 lmcache/tools/controller_benchmark/constants.py create mode 100644 lmcache/tools/controller_benchmark/handlers/__init__.py create mode 100644 lmcache/tools/controller_benchmark/handlers/admit.py create mode 100644 lmcache/tools/controller_benchmark/handlers/base.py create mode 100644 lmcache/tools/controller_benchmark/handlers/deregister.py create mode 100644 lmcache/tools/controller_benchmark/handlers/evict.py create mode 100644 lmcache/tools/controller_benchmark/handlers/heartbeat.py create mode 100644 lmcache/tools/controller_benchmark/handlers/p2p_lookup.py create mode 100644 lmcache/tools/controller_benchmark/handlers/register.py create mode 100644 lmcache/tools/mp_status_viewer/__init__.py create mode 100644 lmcache/tools/mp_status_viewer/__main__.py create mode 100644 lmcache/tools/simple_discover_service.py create mode 100644 lmcache/tools/transfer_channel_benchmark/README.md create mode 100644 lmcache/tools/transfer_channel_benchmark/__init__.py create mode 100644 lmcache/tools/transfer_channel_benchmark/__main__.py create mode 100644 lmcache/tools/transfer_channel_benchmark/benchmark.py create mode 100644 lmcache/tools/transfer_channel_benchmark/config.py create mode 100644 lmcache/usage_context.py create mode 100644 lmcache/usage_telemetry/README.md create mode 100644 lmcache/usage_telemetry/__init__.py create mode 100644 lmcache/usage_telemetry/context.py create mode 100644 lmcache/usage_telemetry/continuous.py create mode 100644 lmcache/usage_telemetry/env_probe.py create mode 100644 lmcache/usage_telemetry/guard.py create mode 100644 lmcache/usage_telemetry/identity.py create mode 100644 lmcache/usage_telemetry/messages.py create mode 100644 lmcache/usage_telemetry/mp.py create mode 100644 lmcache/usage_telemetry/transport.py create mode 100644 lmcache/utils.py create mode 100644 lmcache/v1/__init__.py create mode 100644 lmcache/v1/api_server/__init__.py create mode 100644 lmcache/v1/api_server/__main__.py create mode 100644 lmcache/v1/basic_check.py create mode 100644 lmcache/v1/cache_controller/__init__.py create mode 100644 lmcache/v1/cache_controller/commands/__init__.py create mode 100644 lmcache/v1/cache_controller/commands/base.py create mode 100644 lmcache/v1/cache_controller/commands/full_sync.py create mode 100644 lmcache/v1/cache_controller/config.py create mode 100644 lmcache/v1/cache_controller/controller_manager.py create mode 100644 lmcache/v1/cache_controller/controllers/__init__.py create mode 100644 lmcache/v1/cache_controller/controllers/full_sync_tracker.py create mode 100644 lmcache/v1/cache_controller/controllers/kv_controller.py create mode 100644 lmcache/v1/cache_controller/controllers/registration_controller.py create mode 100644 lmcache/v1/cache_controller/executor.py create mode 100644 lmcache/v1/cache_controller/frontend/static/css/style.css create mode 100644 lmcache/v1/cache_controller/frontend/static/img/logo.png create mode 100644 lmcache/v1/cache_controller/frontend/static/index.html create mode 100644 lmcache/v1/cache_controller/frontend/static/js/controller_app.js create mode 100644 lmcache/v1/cache_controller/full_sync_sender.py create mode 100644 lmcache/v1/cache_controller/locks.py create mode 100644 lmcache/v1/cache_controller/message.py create mode 100644 lmcache/v1/cache_controller/observability.py create mode 100644 lmcache/v1/cache_controller/utils.py create mode 100644 lmcache/v1/cache_controller/worker.py create mode 100644 lmcache/v1/cache_engine.py create mode 100644 lmcache/v1/cache_interface.py create mode 100644 lmcache/v1/check/__init__.py create mode 100644 lmcache/v1/check/check_mode_gen.py create mode 100644 lmcache/v1/check/check_mode_test_l2_adapter.py create mode 100644 lmcache/v1/check/check_mode_test_remote.py create mode 100644 lmcache/v1/check/check_mode_test_storage_manager.py create mode 100644 lmcache/v1/check/utils.py create mode 100644 lmcache/v1/compute/__init__.py create mode 100644 lmcache/v1/compute/attention/__init__.py create mode 100644 lmcache/v1/compute/attention/abstract.py create mode 100644 lmcache/v1/compute/attention/flash_attn.py create mode 100644 lmcache/v1/compute/attention/flash_infer_sparse.py create mode 100644 lmcache/v1/compute/attention/metadata.py create mode 100644 lmcache/v1/compute/attention/triton_kernels/__init__.py create mode 100644 lmcache/v1/compute/attention/triton_kernels/block_sparse_attention.py create mode 100644 lmcache/v1/compute/attention/triton_sparse.py create mode 100644 lmcache/v1/compute/attention/utils.py create mode 100644 lmcache/v1/compute/blend/__init__.py create mode 100644 lmcache/v1/compute/blend/blender.py create mode 100644 lmcache/v1/compute/blend/metadata.py create mode 100644 lmcache/v1/compute/blend/utils.py create mode 100644 lmcache/v1/compute/models/__init__.py create mode 100644 lmcache/v1/compute/models/base.py create mode 100644 lmcache/v1/compute/models/llama.py create mode 100644 lmcache/v1/compute/models/qwen3.py create mode 100644 lmcache/v1/compute/models/utils.py create mode 100644 lmcache/v1/compute/positional_encoding.py create mode 100644 lmcache/v1/config.py create mode 100644 lmcache/v1/config_base.py create mode 100644 lmcache/v1/distributed/api.py create mode 120000 lmcache/v1/distributed/bitmap_ops/README.md create mode 100644 lmcache/v1/distributed/bitmap_ops/__init__.py create mode 100644 lmcache/v1/distributed/bitmap_ops/fold.py create mode 100644 lmcache/v1/distributed/config.py create mode 100644 lmcache/v1/distributed/error.py create mode 100644 lmcache/v1/distributed/eviction.py create mode 100644 lmcache/v1/distributed/eviction_policy/__init__.py create mode 100644 lmcache/v1/distributed/eviction_policy/factory.py create mode 100644 lmcache/v1/distributed/eviction_policy/isolated_lru.py create mode 100644 lmcache/v1/distributed/eviction_policy/lru.py create mode 100644 lmcache/v1/distributed/eviction_policy/noop.py create mode 100644 lmcache/v1/distributed/internal_api.py create mode 100644 lmcache/v1/distributed/l1_manager.py create mode 100644 lmcache/v1/distributed/l2_adapters/__init__.py create mode 100644 lmcache/v1/distributed/l2_adapters/aerospike_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/base.py create mode 100644 lmcache/v1/distributed/l2_adapters/config.py create mode 100644 lmcache/v1/distributed/l2_adapters/dax_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/factory.py create mode 100644 lmcache/v1/distributed/l2_adapters/fault_inject_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/fs_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/fs_native_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/hfbucket_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/mock_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/mooncake_store_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/native_connector_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/native_plugin_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/nixl_store_dynamic_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/nixl_store_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/p2p_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/plugin_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/raw_block_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/reconfiguration.py create mode 100644 lmcache/v1/distributed/l2_adapters/resp_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/s3_l2_adapter.py create mode 100644 lmcache/v1/distributed/l2_adapters/serde_wrapper.py create mode 100644 lmcache/v1/distributed/memory_manager/__init__.py create mode 100644 lmcache/v1/distributed/memory_manager/devdax_l1_memory_manager.py create mode 100644 lmcache/v1/distributed/memory_manager/gds_l1_memory_manager.py create mode 100644 lmcache/v1/distributed/memory_manager/l1_manager_protocol.py create mode 100644 lmcache/v1/distributed/memory_manager/l1_memory_manager.py create mode 100644 lmcache/v1/distributed/quota_manager.py create mode 100644 lmcache/v1/distributed/serde/__init__.py create mode 100644 lmcache/v1/distributed/serde/asym_k16_v8.py create mode 100644 lmcache/v1/distributed/serde/async_processor.py create mode 100644 lmcache/v1/distributed/serde/base.py create mode 100644 lmcache/v1/distributed/serde/factory.py create mode 100644 lmcache/v1/distributed/serde/fp8.py create mode 100644 lmcache/v1/distributed/serde/multi.py create mode 100644 lmcache/v1/distributed/serde/turboquant/__init__.py create mode 100644 lmcache/v1/distributed/serde/turboquant/decode_kernel.py create mode 100644 lmcache/v1/distributed/serde/turboquant/store_kernel.py create mode 100644 lmcache/v1/distributed/serde/turboquant/turboquant.py create mode 100644 lmcache/v1/distributed/serde/utils.py create mode 100644 lmcache/v1/distributed/storage_controller.py create mode 100644 lmcache/v1/distributed/storage_controllers/__init__.py create mode 100644 lmcache/v1/distributed/storage_controllers/adapter_lifecycle.py create mode 100644 lmcache/v1/distributed/storage_controllers/eviction_controller.py create mode 100644 lmcache/v1/distributed/storage_controllers/prefetch_controller.py create mode 100644 lmcache/v1/distributed/storage_controllers/prefetch_policy.py create mode 100644 lmcache/v1/distributed/storage_controllers/store_controller.py create mode 100644 lmcache/v1/distributed/storage_controllers/store_policy.py create mode 100644 lmcache/v1/distributed/storage_manager.py create mode 100644 lmcache/v1/distributed/tiers.py create mode 100644 lmcache/v1/distributed/transfer_channel/__init__.py create mode 100644 lmcache/v1/distributed/transfer_channel/abstract.py create mode 100644 lmcache/v1/distributed/transfer_channel/api.py create mode 100644 lmcache/v1/distributed/transfer_channel/factory.py create mode 100644 lmcache/v1/distributed/transfer_channel/impl/__init__.py create mode 100644 lmcache/v1/distributed/transfer_channel/impl/nixl_impl.py create mode 100644 lmcache/v1/ec_engine.py create mode 100644 lmcache/v1/event_manager.py create mode 100644 lmcache/v1/exceptions/__init__.py create mode 100644 lmcache/v1/gpu_connector/__init__.py create mode 100644 lmcache/v1/gpu_connector/_cufile_async.py create mode 100644 lmcache/v1/gpu_connector/_gds_async.py create mode 100644 lmcache/v1/gpu_connector/_hipfile_async.py create mode 100644 lmcache/v1/gpu_connector/gds_context.py create mode 100644 lmcache/v1/gpu_connector/gpu_connectors.py create mode 100644 lmcache/v1/gpu_connector/gpu_ops.py create mode 100644 lmcache/v1/gpu_connector/hpu_connector.py create mode 100644 lmcache/v1/gpu_connector/kv_format/__init__.py create mode 100644 lmcache/v1/gpu_connector/kv_format/contiguity.py create mode 100644 lmcache/v1/gpu_connector/kv_format/detection.py create mode 100644 lmcache/v1/gpu_connector/kv_format/detectors/__init__.py create mode 100644 lmcache/v1/gpu_connector/kv_format/detectors/base.py create mode 100644 lmcache/v1/gpu_connector/kv_format/detectors/registry.py create mode 100644 lmcache/v1/gpu_connector/kv_format/detectors/sglang.py create mode 100644 lmcache/v1/gpu_connector/kv_format/detectors/trtllm.py create mode 100644 lmcache/v1/gpu_connector/kv_format/detectors/vllm.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/__init__.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/base.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/nb_nl_two_bs_nh_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/nb_nl_two_nh_bs_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/nl_x_nb_bs_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/nl_x_nb_nh_bs_two_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/nl_x_nb_two_bs_nh_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/nl_x_nb_two_nh_bs_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/nl_x_nbbs_one_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/nl_x_two_nb_bs_nh_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/nl_x_two_nb_nh_bs_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/registry.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/two_x_nl_x_nb_bs_nh_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/specs/two_x_nl_x_nbbs_nh_hs.py create mode 100644 lmcache/v1/gpu_connector/kv_format/types.py create mode 100644 lmcache/v1/gpu_connector/mock_gpu_connector.py create mode 100644 lmcache/v1/gpu_connector/musa_connectors.py create mode 100644 lmcache/v1/gpu_connector/utils.py create mode 100644 lmcache/v1/gpu_connector/xpu_connectors.py create mode 100644 lmcache/v1/health_monitor/__init__.py create mode 100644 lmcache/v1/health_monitor/base.py create mode 100644 lmcache/v1/health_monitor/checks/__init__.py create mode 100644 lmcache/v1/health_monitor/checks/remote_backend_check.py create mode 100644 lmcache/v1/health_monitor/constants.py create mode 100644 lmcache/v1/hidden_state_store.py create mode 100644 lmcache/v1/internal_api_server/__init__.py create mode 100644 lmcache/v1/internal_api_server/api_registry.py create mode 100644 lmcache/v1/internal_api_server/api_server.py create mode 100644 lmcache/v1/internal_api_server/common/__init__.py create mode 100644 lmcache/v1/internal_api_server/common/env_api.py create mode 100644 lmcache/v1/internal_api_server/common/loglevel_api.py create mode 100644 lmcache/v1/internal_api_server/common/metrics_api.py create mode 100644 lmcache/v1/internal_api_server/common/periodic_thread_api.py create mode 100644 lmcache/v1/internal_api_server/common/run_script_api.py create mode 100644 lmcache/v1/internal_api_server/common/thread_api.py create mode 100644 lmcache/v1/internal_api_server/controller/__init__.py create mode 100644 lmcache/v1/internal_api_server/controller/key_stats_api.py create mode 100644 lmcache/v1/internal_api_server/controller/worker_info_api.py create mode 100644 lmcache/v1/internal_api_server/utils.py create mode 100644 lmcache/v1/internal_api_server/vllm/__init__.py create mode 100644 lmcache/v1/internal_api_server/vllm/backend_api.py create mode 100644 lmcache/v1/internal_api_server/vllm/bypass_api.py create mode 100644 lmcache/v1/internal_api_server/vllm/cache_api.py create mode 100644 lmcache/v1/internal_api_server/vllm/chunk_statistics_api.py create mode 100644 lmcache/v1/internal_api_server/vllm/conf_api.py create mode 100644 lmcache/v1/internal_api_server/vllm/freeze_api.py create mode 100644 lmcache/v1/internal_api_server/vllm/hot_cache_api.py create mode 100644 lmcache/v1/internal_api_server/vllm/inference_api.py create mode 100644 lmcache/v1/internal_api_server/vllm/load_fs_chunks_api.py create mode 100644 lmcache/v1/internal_api_server/vllm/lookup_api.py create mode 100644 lmcache/v1/internal_api_server/vllm/version_api.py create mode 100644 lmcache/v1/kv_codec/__init__.py create mode 100644 lmcache/v1/kv_codec/asym_k16_v8.py create mode 100644 lmcache/v1/kv_codec/encoded_kv.py create mode 100644 lmcache/v1/kv_codec/errors.py create mode 100644 lmcache/v1/kv_layer_groups.py create mode 100644 lmcache/v1/lookup_client/__init__.py create mode 100644 lmcache/v1/lookup_client/abstract_client.py create mode 100644 lmcache/v1/lookup_client/async_lookup_message.py create mode 100644 lmcache/v1/lookup_client/chunk_statistics_lookup_client.py create mode 100644 lmcache/v1/lookup_client/factory.py create mode 100644 lmcache/v1/lookup_client/hit_limit_lookup_client.py create mode 100644 lmcache/v1/lookup_client/lmcache_async_lookup_client.py create mode 100644 lmcache/v1/lookup_client/lmcache_lookup_client.py create mode 100644 lmcache/v1/lookup_client/lmcache_lookup_client_bypass.py create mode 100644 lmcache/v1/lookup_client/mooncake_lookup_client.py create mode 100644 lmcache/v1/lookup_client/record_strategies/__init__.py create mode 100644 lmcache/v1/lookup_client/record_strategies/base.py create mode 100644 lmcache/v1/lookup_client/record_strategies/file_hash.py create mode 100644 lmcache/v1/lookup_client/record_strategies/memory_bloom_filter.py create mode 100644 lmcache/v1/manager.py create mode 100644 lmcache/v1/memory_allocators/__init__.py create mode 100644 lmcache/v1/memory_allocators/ad_hoc_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/buffer_allocator.py create mode 100644 lmcache/v1/memory_allocators/cu_file_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/devdax_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/gpu_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/hip_file_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/host_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/lazy_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/mixed_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/paged_cpu_gpu_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/paged_tensor_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/pin_memory_allocator.py create mode 100644 lmcache/v1/memory_allocators/tensor_memory_allocator.py create mode 100644 lmcache/v1/memory_management.py create mode 100644 lmcache/v1/metadata.py create mode 100644 lmcache/v1/mp_coordinator/__init__.py create mode 100644 lmcache/v1/mp_coordinator/__main__.py create mode 100644 lmcache/v1/mp_coordinator/app.py create mode 100644 lmcache/v1/mp_coordinator/blend_client.py create mode 100644 lmcache/v1/mp_coordinator/blend_directory.py create mode 100644 lmcache/v1/mp_coordinator/cache_control/__init__.py create mode 100644 lmcache/v1/mp_coordinator/cache_control/event_listener.py create mode 100644 lmcache/v1/mp_coordinator/cache_control/eviction_manager.py create mode 100644 lmcache/v1/mp_coordinator/cache_control/prefetch_manager.py create mode 100644 lmcache/v1/mp_coordinator/cache_control/resync_manager.py create mode 100644 lmcache/v1/mp_coordinator/cache_control/usage_manager.py create mode 100644 lmcache/v1/mp_coordinator/config.py create mode 100644 lmcache/v1/mp_coordinator/http_apis/__init__.py create mode 100644 lmcache/v1/mp_coordinator/http_apis/blend_directory_api.py create mode 100644 lmcache/v1/mp_coordinator/http_apis/cache_api.py create mode 100644 lmcache/v1/mp_coordinator/http_apis/dependencies.py create mode 100644 lmcache/v1/mp_coordinator/http_apis/health_api.py create mode 100644 lmcache/v1/mp_coordinator/http_apis/instances_api.py create mode 100644 lmcache/v1/mp_coordinator/http_apis/quota_api.py create mode 100644 lmcache/v1/mp_coordinator/registrar.py create mode 100644 lmcache/v1/mp_coordinator/registry.py create mode 100644 lmcache/v1/mp_coordinator/schemas.py create mode 100644 lmcache/v1/mp_coordinator/utils/__init__.py create mode 100644 lmcache/v1/mp_coordinator/utils/cache_utils.py create mode 100644 lmcache/v1/mp_observability/AGENTS.override.md create mode 100644 lmcache/v1/mp_observability/README.md create mode 100644 lmcache/v1/mp_observability/config.py create mode 100644 lmcache/v1/mp_observability/errors.py create mode 100644 lmcache/v1/mp_observability/event.py create mode 100644 lmcache/v1/mp_observability/event_bus.py create mode 100644 lmcache/v1/mp_observability/otel_init.py create mode 100644 lmcache/v1/mp_observability/subscribers/__init__.py create mode 100644 lmcache/v1/mp_observability/subscribers/logging/__init__.py create mode 100644 lmcache/v1/mp_observability/subscribers/logging/cb_server.py create mode 100644 lmcache/v1/mp_observability/subscribers/logging/l1.py create mode 100644 lmcache/v1/mp_observability/subscribers/logging/l2.py create mode 100644 lmcache/v1/mp_observability/subscribers/logging/lookup_hash.py create mode 100644 lmcache/v1/mp_observability/subscribers/logging/mp_server.py create mode 100644 lmcache/v1/mp_observability/subscribers/logging/sm.py create mode 100644 lmcache/v1/mp_observability/subscribers/logging/timeout.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/__init__.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/cb_server.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/engine.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/event_bus.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/l0_l1_throughput.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/l0_lifecycle.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/l1.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/l1_eviction_loop.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/l1_failures.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/l1_lifecycle.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/l2.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/l2_failures.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/l2_throughput.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/lookup.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/sm_lifecycle.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/timeout.py create mode 100644 lmcache/v1/mp_observability/subscribers/metrics/utils.py create mode 100644 lmcache/v1/mp_observability/subscribers/tracing/__init__.py create mode 100644 lmcache/v1/mp_observability/subscribers/tracing/cb_server.py create mode 100644 lmcache/v1/mp_observability/subscribers/tracing/mp_server.py create mode 100644 lmcache/v1/mp_observability/subscribers/tracing/span_registry.py create mode 100644 lmcache/v1/mp_observability/subscribers/tracing/timeout.py create mode 100644 lmcache/v1/mp_observability/trace/__init__.py create mode 100644 lmcache/v1/mp_observability/trace/codecs.py create mode 100644 lmcache/v1/mp_observability/trace/decorator.py create mode 100644 lmcache/v1/mp_observability/trace/format.py create mode 100644 lmcache/v1/mp_observability/trace/lifecycle.py create mode 100644 lmcache/v1/mp_observability/trace/reader.py create mode 100644 lmcache/v1/mp_observability/trace/recorder.py create mode 100644 lmcache/v1/multiprocess/__init__.py create mode 100644 lmcache/v1/multiprocess/affinity_pool.py create mode 100644 lmcache/v1/multiprocess/cache_control/__init__.py create mode 100644 lmcache/v1/multiprocess/cache_control/errors.py create mode 100644 lmcache/v1/multiprocess/cache_control/key_resolver.py create mode 100644 lmcache/v1/multiprocess/cache_control/object_service.py create mode 100644 lmcache/v1/multiprocess/cache_control/prefetch_service.py create mode 100644 lmcache/v1/multiprocess/config.py create mode 100644 lmcache/v1/multiprocess/custom_types.py create mode 100644 lmcache/v1/multiprocess/engine_context.py create mode 100644 lmcache/v1/multiprocess/engine_module.py create mode 100644 lmcache/v1/multiprocess/futures.py create mode 100644 lmcache/v1/multiprocess/group_view.py create mode 100644 lmcache/v1/multiprocess/http_api_registry.py create mode 100644 lmcache/v1/multiprocess/http_apis/__init__.py create mode 100644 lmcache/v1/multiprocess/http_apis/cache_api.py create mode 100644 lmcache/v1/multiprocess/http_apis/common_api.py create mode 100644 lmcache/v1/multiprocess/http_apis/config_api.py create mode 100644 lmcache/v1/multiprocess/http_apis/dependencies.py create mode 100644 lmcache/v1/multiprocess/http_apis/error_handlers.py create mode 100644 lmcache/v1/multiprocess/http_apis/info_api.py create mode 100644 lmcache/v1/multiprocess/http_apis/quota_api.py create mode 100644 lmcache/v1/multiprocess/http_apis/reconfigure_api.py create mode 100644 lmcache/v1/multiprocess/http_apis/schemas.py create mode 100644 lmcache/v1/multiprocess/http_server.py create mode 100644 lmcache/v1/multiprocess/modules/__init__.py create mode 100644 lmcache/v1/multiprocess/modules/blend.py create mode 100644 lmcache/v1/multiprocess/modules/blend_v3.py create mode 100644 lmcache/v1/multiprocess/modules/engine_driven_transfer.py create mode 100644 lmcache/v1/multiprocess/modules/lmcache_driven_transfer.py create mode 100644 lmcache/v1/multiprocess/modules/lookup.py create mode 100644 lmcache/v1/multiprocess/modules/management.py create mode 100644 lmcache/v1/multiprocess/modules/p2p_controller.py create mode 100644 lmcache/v1/multiprocess/modules/server_transfer.py create mode 100644 lmcache/v1/multiprocess/mp_runtime_plugin_launcher.py create mode 100644 lmcache/v1/multiprocess/mq.py create mode 100644 lmcache/v1/multiprocess/native_completion.py create mode 100644 lmcache/v1/multiprocess/posix_shm.py create mode 100644 lmcache/v1/multiprocess/protocol.py create mode 100644 lmcache/v1/multiprocess/protocols/README.md create mode 100644 lmcache/v1/multiprocess/protocols/__init__.py create mode 100644 lmcache/v1/multiprocess/protocols/base.py create mode 100644 lmcache/v1/multiprocess/protocols/blend.py create mode 100644 lmcache/v1/multiprocess/protocols/blend_v2.py create mode 100644 lmcache/v1/multiprocess/protocols/blend_v3.py create mode 100644 lmcache/v1/multiprocess/protocols/controller.py create mode 100644 lmcache/v1/multiprocess/protocols/debug.py create mode 100644 lmcache/v1/multiprocess/protocols/engine.py create mode 100644 lmcache/v1/multiprocess/protocols/observability.py create mode 100644 lmcache/v1/multiprocess/protocols/p2p.py create mode 100644 lmcache/v1/multiprocess/server.py create mode 100644 lmcache/v1/multiprocess/session.py create mode 100644 lmcache/v1/multiprocess/token_hasher.py create mode 100644 lmcache/v1/multiprocess/transfer_context/__init__.py create mode 100644 lmcache/v1/multiprocess/transfer_context/async_engine_driven.py create mode 100644 lmcache/v1/multiprocess/transfer_context/base.py create mode 100644 lmcache/v1/multiprocess/transfer_context/pickle.py create mode 100644 lmcache/v1/multiprocess/transfer_context/shm.py create mode 100644 lmcache/v1/multiprocess/transfer_context/worker_transfer.py create mode 100644 lmcache/v1/multiprocess/warm_prefetch.py create mode 100644 lmcache/v1/offload_server/__init__.py create mode 100644 lmcache/v1/offload_server/abstract_server.py create mode 100644 lmcache/v1/offload_server/message.py create mode 100644 lmcache/v1/offload_server/zmq_server.py create mode 100644 lmcache/v1/periodic_thread.py create mode 100644 lmcache/v1/pin_monitor.py create mode 100644 lmcache/v1/platform/__init__.py create mode 100644 lmcache/v1/platform/_registry.py create mode 100644 lmcache/v1/platform/base_cache_context.py create mode 100644 lmcache/v1/platform/base_device_spec.py create mode 100644 lmcache/v1/platform/base_ipc_wrapper.py create mode 100644 lmcache/v1/platform/base_pin_memory.py create mode 100644 lmcache/v1/platform/cache_context.py create mode 100644 lmcache/v1/platform/cpu/__init__.py create mode 100644 lmcache/v1/platform/cpu/cache_context.py create mode 100644 lmcache/v1/platform/cpu/shm.py create mode 100644 lmcache/v1/platform/cpu/stub_cpu_device.py create mode 100644 lmcache/v1/platform/cuda/__init__.py create mode 100644 lmcache/v1/platform/cuda/cache_context.py create mode 100644 lmcache/v1/platform/cuda/ipc_wrapper.py create mode 100644 lmcache/v1/platform/cuda/pin_memory.py create mode 100644 lmcache/v1/platform/event_notifier.py create mode 100644 lmcache/v1/platform/hpu/__init__.py create mode 100644 lmcache/v1/platform/musa/__init__.py create mode 100644 lmcache/v1/platform/musa/ipc.py create mode 100644 lmcache/v1/platform/musa/native_kv_transfer.py create mode 100644 lmcache/v1/platform/musa/ops.py create mode 100644 lmcache/v1/platform/xpu/__init__.py create mode 100644 lmcache/v1/plugin/__init__.py create mode 100644 lmcache/v1/plugin/runtime_plugin_launcher.py create mode 100644 lmcache/v1/protocol.py create mode 100644 lmcache/v1/rpc/__init__.py create mode 100644 lmcache/v1/rpc/transport.py create mode 100644 lmcache/v1/rpc/zmq_transport.py create mode 100644 lmcache/v1/rpc_utils.py create mode 100644 lmcache/v1/server/__init__.py create mode 100644 lmcache/v1/server/__main__.py create mode 100644 lmcache/v1/server/storage_backend/__init__.py create mode 100644 lmcache/v1/server/storage_backend/abstract_backend.py create mode 100644 lmcache/v1/server/storage_backend/local_backend.py create mode 100644 lmcache/v1/server/utils.py create mode 100644 lmcache/v1/standalone/__init__.py create mode 100644 lmcache/v1/standalone/__main__.py create mode 100644 lmcache/v1/standalone/manager.py create mode 100644 lmcache/v1/standalone/standalone_service_factory.py create mode 100644 lmcache/v1/storage_backend/__init__.py create mode 100644 lmcache/v1/storage_backend/abstract_backend.py create mode 100644 lmcache/v1/storage_backend/audit_backend.py create mode 100644 lmcache/v1/storage_backend/batched_message_sender.py create mode 100644 lmcache/v1/storage_backend/cache_policy/__init__.py create mode 100644 lmcache/v1/storage_backend/cache_policy/base_policy.py create mode 100644 lmcache/v1/storage_backend/cache_policy/fifo.py create mode 100644 lmcache/v1/storage_backend/cache_policy/lfu.py create mode 100644 lmcache/v1/storage_backend/cache_policy/lru.py create mode 100644 lmcache/v1/storage_backend/cache_policy/mru.py create mode 100644 lmcache/v1/storage_backend/connector/__init__.py create mode 100644 lmcache/v1/storage_backend/connector/audit_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/audit_connector.py create mode 100644 lmcache/v1/storage_backend/connector/azure_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/azure_connector.py create mode 100644 lmcache/v1/storage_backend/connector/base_connector.py create mode 100644 lmcache/v1/storage_backend/connector/bigtable_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/bigtable_config.py create mode 100644 lmcache/v1/storage_backend/connector/bigtable_connector.py create mode 100644 lmcache/v1/storage_backend/connector/bigtable_schema.py create mode 100644 lmcache/v1/storage_backend/connector/blackhole_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/blackhole_connector.py create mode 100644 lmcache/v1/storage_backend/connector/eic_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/eic_connector.py create mode 100644 lmcache/v1/storage_backend/connector/external_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/fs_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/fs_connector.py create mode 100644 lmcache/v1/storage_backend/connector/hf3fs_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/hf3fs_connector.py create mode 100644 lmcache/v1/storage_backend/connector/hfbucket_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/hfbucket_connector.py create mode 100644 lmcache/v1/storage_backend/connector/infinistore_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/infinistore_connector.py create mode 100644 lmcache/v1/storage_backend/connector/instrumented_connector.py create mode 100644 lmcache/v1/storage_backend/connector/lm_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/lm_connector.py create mode 100644 lmcache/v1/storage_backend/connector/mock_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/mock_connector.py create mode 100644 lmcache/v1/storage_backend/connector/mooncakestore_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/mooncakestore_connector.py create mode 100644 lmcache/v1/storage_backend/connector/redis_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/redis_connector.py create mode 100644 lmcache/v1/storage_backend/connector/s3_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/s3_connector.py create mode 100644 lmcache/v1/storage_backend/connector/sagemaker_hyperpod_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/sagemaker_hyperpod_connector.py create mode 100644 lmcache/v1/storage_backend/connector/valkey_adapter.py create mode 100644 lmcache/v1/storage_backend/connector/valkey_connector.py create mode 100644 lmcache/v1/storage_backend/dax/__init__.py create mode 100644 lmcache/v1/storage_backend/dax/core.py create mode 100644 lmcache/v1/storage_backend/gds_backend.py create mode 100644 lmcache/v1/storage_backend/hipfile_shim.py create mode 100644 lmcache/v1/storage_backend/job_executor/__init__.py create mode 100644 lmcache/v1/storage_backend/job_executor/base_executor.py create mode 100644 lmcache/v1/storage_backend/job_executor/pq_executor.py create mode 100644 lmcache/v1/storage_backend/local_cpu_backend.py create mode 100644 lmcache/v1/storage_backend/local_disk_backend.py create mode 100644 lmcache/v1/storage_backend/maru_backend.py create mode 100644 lmcache/v1/storage_backend/naive_serde/__init__.py create mode 100644 lmcache/v1/storage_backend/naive_serde/cachegen_basics.py create mode 100644 lmcache/v1/storage_backend/naive_serde/cachegen_decoder.py create mode 100644 lmcache/v1/storage_backend/naive_serde/cachegen_encoder.py create mode 100644 lmcache/v1/storage_backend/naive_serde/kivi_serde.py create mode 100644 lmcache/v1/storage_backend/naive_serde/naive_serde.py create mode 100644 lmcache/v1/storage_backend/naive_serde/serde.py create mode 100644 lmcache/v1/storage_backend/native_clients/aerospike_client.py create mode 100644 lmcache/v1/storage_backend/native_clients/connector_client_base.py create mode 100644 lmcache/v1/storage_backend/native_clients/resp_client.py create mode 100644 lmcache/v1/storage_backend/nixl_storage_backend.py create mode 100644 lmcache/v1/storage_backend/p2p_backend.py create mode 100644 lmcache/v1/storage_backend/path_sharder.py create mode 100644 lmcache/v1/storage_backend/pd_backend.py create mode 100644 lmcache/v1/storage_backend/pd_backend_async.py create mode 100644 lmcache/v1/storage_backend/plugins/dax_backend.py create mode 100644 lmcache/v1/storage_backend/plugins/rust_raw_block_backend.py create mode 100644 lmcache/v1/storage_backend/raw_block/__init__.py create mode 100644 lmcache/v1/storage_backend/raw_block/core.py create mode 100644 lmcache/v1/storage_backend/raw_block/key_codec.py create mode 100644 lmcache/v1/storage_backend/remote_backend.py create mode 100644 lmcache/v1/storage_backend/resp_client.py create mode 100644 lmcache/v1/storage_backend/storage_backend_listener.py create mode 100644 lmcache/v1/storage_backend/storage_manager.py create mode 100644 lmcache/v1/system_detection.py create mode 100644 lmcache/v1/token_database.py create mode 100644 lmcache/v1/transfer_channel/__init__.py create mode 100644 lmcache/v1/transfer_channel/abstract.py create mode 100644 lmcache/v1/transfer_channel/mock_memory_channel.py create mode 100644 lmcache/v1/transfer_channel/nixl_channel.py create mode 100644 lmcache/v1/transfer_channel/py_socket_channel.py create mode 100644 lmcache/v1/transfer_channel/transfer_utils.py create mode 100644 lmcache/v1/utils/__init__.py create mode 100644 lmcache/v1/utils/bloom_filter.py create mode 100644 lmcache/v1/utils/cache_utils.py create mode 100644 lmcache/v1/utils/json_utils.py create mode 100644 lmcache/v1/utils/router_discovery.py create mode 100644 lmcache/v1/utils/subclass_discovery.py create mode 100644 operator/.custom-gcl.yml create mode 100644 operator/.devcontainer/devcontainer.json create mode 100644 operator/.devcontainer/post-install.sh create mode 100644 operator/.dockerignore create mode 100644 operator/.golangci.yml create mode 100644 operator/.pre-commit-config.yaml create mode 100644 operator/AGENTS.md create mode 100644 operator/DESIGN.md create mode 100644 operator/Dockerfile create mode 100644 operator/Makefile create mode 100644 operator/PROJECT create mode 100644 operator/README.md create mode 100644 operator/api/v1alpha1/cacheblendengine_defaults.go create mode 100644 operator/api/v1alpha1/cacheblendengine_test.go create mode 100644 operator/api/v1alpha1/cacheblendengine_types.go create mode 100644 operator/api/v1alpha1/cacheblendengine_validation.go create mode 100644 operator/api/v1alpha1/groupversion_info.go create mode 100644 operator/api/v1alpha1/lmcachecoordinator_defaults.go create mode 100644 operator/api/v1alpha1/lmcachecoordinator_types.go create mode 100644 operator/api/v1alpha1/lmcachecoordinator_validation.go create mode 100644 operator/api/v1alpha1/lmcacheengine_defaults.go create mode 100644 operator/api/v1alpha1/lmcacheengine_test.go create mode 100644 operator/api/v1alpha1/lmcacheengine_types.go create mode 100644 operator/api/v1alpha1/lmcacheengine_validation.go create mode 100644 operator/api/v1alpha1/zz_generated.deepcopy.go create mode 100644 operator/cmd/main.go create mode 100644 operator/config/certmanager/certificate.yaml create mode 100644 operator/config/certmanager/issuer.yaml create mode 100644 operator/config/certmanager/kustomization.yaml create mode 100644 operator/config/certmanager/kustomizeconfig.yaml create mode 100644 operator/config/crd/bases/lmcache.lmcache.ai_cacheblendengines.yaml create mode 100644 operator/config/crd/bases/lmcache.lmcache.ai_lmcachecoordinators.yaml create mode 100644 operator/config/crd/bases/lmcache.lmcache.ai_lmcacheengines.yaml create mode 100644 operator/config/crd/kustomization.yaml create mode 100644 operator/config/crd/kustomizeconfig.yaml create mode 100644 operator/config/default/cert_metrics_manager_patch.yaml create mode 100644 operator/config/default/kustomization.yaml create mode 100644 operator/config/default/manager_metrics_patch.yaml create mode 100644 operator/config/default/manager_webhook_patch.yaml create mode 100644 operator/config/default/metrics_service.yaml create mode 100644 operator/config/manager/kustomization.yaml create mode 100644 operator/config/manager/manager.yaml create mode 100644 operator/config/network-policy/allow-metrics-traffic.yaml create mode 100644 operator/config/network-policy/kustomization.yaml create mode 100644 operator/config/prometheus/kustomization.yaml create mode 100644 operator/config/prometheus/monitor.yaml create mode 100644 operator/config/prometheus/monitor_tls_patch.yaml create mode 100644 operator/config/rbac/kustomization.yaml create mode 100644 operator/config/rbac/leader_election_role.yaml create mode 100644 operator/config/rbac/leader_election_role_binding.yaml create mode 100644 operator/config/rbac/lmcacheengine_admin_role.yaml create mode 100644 operator/config/rbac/lmcacheengine_editor_role.yaml create mode 100644 operator/config/rbac/lmcacheengine_viewer_role.yaml create mode 100644 operator/config/rbac/metrics_auth_role.yaml create mode 100644 operator/config/rbac/metrics_auth_role_binding.yaml create mode 100644 operator/config/rbac/metrics_reader_role.yaml create mode 100644 operator/config/rbac/role.yaml create mode 100644 operator/config/rbac/role_binding.yaml create mode 100644 operator/config/rbac/service_account.yaml create mode 100644 operator/config/samples/kustomization.yaml create mode 100644 operator/config/samples/lmcache_v1alpha1_cacheblendengine.yaml create mode 100644 operator/config/samples/lmcache_v1alpha1_lmcachecoordinator.yaml create mode 100644 operator/config/samples/lmcache_v1alpha1_lmcacheengine.yaml create mode 100644 operator/config/samples/lmcache_v1alpha1_lmcacheengine_amd.yaml create mode 100644 operator/config/samples/lmcache_v1alpha1_lmcacheengine_injection.yaml create mode 100644 operator/config/samples/lmcache_v1alpha1_lmcacheengine_l2_redis.yaml create mode 100644 operator/config/samples/lmcache_v1alpha1_lmcacheengine_production.yaml create mode 100644 operator/config/samples/vllm_cacheblend_deployment.yaml create mode 100644 operator/config/samples/vllm_lmcache_deployment.yaml create mode 100644 operator/config/samples/vllm_lmcache_injection_deployment.yaml create mode 100644 operator/config/webhook/kustomization.yaml create mode 100644 operator/config/webhook/kustomizeconfig.yaml create mode 100644 operator/config/webhook/manifests.yaml create mode 100644 operator/config/webhook/mutating_webhook_selectors_patch.yaml create mode 100644 operator/config/webhook/service.yaml create mode 100644 operator/go.mod create mode 100644 operator/go.sum create mode 100644 operator/hack/boilerplate.go.txt create mode 100755 operator/hack/pre-commit create mode 100644 operator/internal/controller/cacheblend_reconcile_helpers.go create mode 100644 operator/internal/controller/cacheblendengine_controller.go create mode 100644 operator/internal/controller/cacheblendengine_controller_test.go create mode 100644 operator/internal/controller/coordinator_reconcile_helpers.go create mode 100644 operator/internal/controller/lmcachecoordinator_controller.go create mode 100644 operator/internal/controller/lmcachecoordinator_controller_test.go create mode 100644 operator/internal/controller/lmcacheengine_controller.go create mode 100644 operator/internal/controller/lmcacheengine_controller_test.go create mode 100644 operator/internal/controller/reconcile_helpers.go create mode 100644 operator/internal/controller/reconcile_helpers_test.go create mode 100644 operator/internal/controller/suite_test.go create mode 100644 operator/internal/resources/cacheblend_engine.go create mode 100644 operator/internal/resources/cacheblend_engine_test.go create mode 100644 operator/internal/resources/compute.go create mode 100644 operator/internal/resources/configmap.go create mode 100644 operator/internal/resources/coordinator_deployment.go create mode 100644 operator/internal/resources/coordinator_resolve.go create mode 100644 operator/internal/resources/coordinator_service.go create mode 100644 operator/internal/resources/coordinator_test.go create mode 100644 operator/internal/resources/daemonset.go create mode 100644 operator/internal/resources/helpers.go create mode 100644 operator/internal/resources/labels.go create mode 100644 operator/internal/resources/resources_test.go create mode 100644 operator/internal/resources/service.go create mode 100644 operator/internal/resources/servicemonitor.go create mode 100644 operator/internal/webhook/cacheblend_inject_builders.go create mode 100644 operator/internal/webhook/cacheblend_pod_injector.go create mode 100644 operator/internal/webhook/cacheblend_pod_injector_envtest_test.go create mode 100644 operator/internal/webhook/cacheblend_pod_injector_test.go create mode 100644 operator/internal/webhook/lmcache_inject_builders.go create mode 100644 operator/internal/webhook/lmcache_pod_injector.go create mode 100644 operator/internal/webhook/lmcache_pod_injector_test.go create mode 100644 operator/internal/webhook/payload_staging.go create mode 100644 operator/internal/webhook/pod_inject_common.go create mode 100644 operator/internal/webhook/webhook_suite_test.go create mode 100644 operator/make/build.mk create mode 100644 operator/make/deploy.mk create mode 100644 operator/make/dev.mk create mode 100644 operator/make/e2e-gpu.mk create mode 100644 operator/make/e2e.mk create mode 100644 operator/make/lint.mk create mode 100644 operator/make/tools.mk create mode 100644 operator/make/unit.mk create mode 100644 operator/test/e2e/auth_smoke_test.go create mode 100644 operator/test/e2e/cacheblend_integration_smoke_test.go create mode 100644 operator/test/e2e/coordinator_registration_smoke_test.go create mode 100644 operator/test/e2e/crd_smoke_test.go create mode 100644 operator/test/e2e/e2e_suite_test.go create mode 100644 operator/test/e2e/e2e_test.go create mode 100644 operator/test/e2e/field_coverage_smoke_test.go create mode 100644 operator/test/e2e/lifecycle_smoke_test.go create mode 100644 operator/test/e2e/lmcache_injection_smoke_test.go create mode 100644 operator/test/e2e/runtime_smoke_test.go create mode 100644 operator/test/e2e/smoke_helpers_test.go create mode 100644 operator/test/e2e/vllm_integration_smoke_test.go create mode 100644 operator/test/utils/cbe.go create mode 100644 operator/test/utils/fixtures.go create mode 100644 operator/test/utils/fixtures/cacheblendengine.yaml create mode 100644 operator/test/utils/fixtures/golden/kv_transfer_config_minimal.json create mode 100644 operator/test/utils/fixtures/lmc_custom_port.yaml create mode 100644 operator/test/utils/fixtures/lmc_minimal.yaml create mode 100644 operator/test/utils/fixtures/lmc_runtime.yaml create mode 100644 operator/test/utils/fixtures/lmc_servicemonitor.yaml create mode 100644 operator/test/utils/fixtures/lmc_with_redis_l2_authsecret.yaml create mode 100644 operator/test/utils/fixtures/vllm_cacheblend_deployment.yaml create mode 100644 operator/test/utils/fixtures/vllm_deployment.yaml create mode 100644 operator/test/utils/fixtures/vllm_lmcache_injection_deployment.yaml create mode 100644 operator/test/utils/http.go create mode 100644 operator/test/utils/lmc.go create mode 100644 operator/test/utils/portforward.go create mode 100644 operator/test/utils/runner.go create mode 100644 operator/test/utils/secret.go create mode 100644 operator/test/utils/utils.go create mode 100644 operator/test/utils/wait.go create mode 100644 pyproject.toml create mode 100644 pyproject_cli.toml create mode 100644 pytest.ini create mode 100644 requirements/bench.txt create mode 100644 requirements/build.txt create mode 100644 requirements/cli.txt create mode 100644 requirements/common.txt create mode 100644 requirements/cuda.txt create mode 100644 requirements/cuda12_core.txt create mode 100644 requirements/cuda13_core.txt create mode 100644 requirements/docs.txt create mode 100644 requirements/lint.txt create mode 100644 requirements/nixl.txt create mode 100644 requirements/rocm_core.txt create mode 100644 requirements/test.txt create mode 100644 rust/raw_block/.gitignore create mode 100644 rust/raw_block/Cargo.toml create mode 100644 rust/raw_block/README.md create mode 100644 rust/raw_block/pyproject.toml create mode 100644 rust/raw_block/src/lib.rs create mode 100644 setup.py create mode 100644 setup_extensions/__init__.py create mode 100644 setup_extensions/build_profiles/__init__.py create mode 100644 setup_extensions/build_profiles/cuda.py create mode 100644 setup_extensions/build_profiles/musa.py create mode 100644 setup_extensions/build_profiles/rocm.py create mode 100644 setup_extensions/build_profiles/sycl.py create mode 100644 setup_extensions/common_cpp.py create mode 100644 setup_extensions/policy.py create mode 100644 setup_extensions/storage_backend_profiles/__init__.py create mode 100644 setup_extensions/storage_backend_profiles/aerospike.py create mode 100644 setup_extensions/storage_backend_profiles/mooncake.py create mode 100644 tests/__init__.py create mode 100644 tests/aerospike_ce.conf.template create mode 100644 tests/benchmarks/test_benchmark.py create mode 100644 tests/benchmarks/test_cachegen.py create mode 100644 tests/benchmarks/test_musa_inprocess_benchmark.py create mode 100644 tests/benchmarks/test_xpu_connector_benchmark.py create mode 100644 tests/benchmarks/test_xpu_kernels_microbench.py create mode 100644 tests/benchmarks/test_xpu_layerwise_connector_benchmark.py create mode 100644 tests/cli/__init__.py create mode 100644 tests/cli/commands/__init__.py create mode 100644 tests/cli/commands/bench/__init__.py create mode 100644 tests/cli/commands/bench/engine_bench/__init__.py create mode 100644 tests/cli/commands/bench/engine_bench/interactive/__init__.py create mode 100644 tests/cli/commands/bench/engine_bench/interactive/test_back.py create mode 100644 tests/cli/commands/bench/engine_bench/interactive/test_schema.py create mode 100644 tests/cli/commands/bench/engine_bench/interactive/test_state.py create mode 100644 tests/cli/commands/bench/engine_bench/test_config.py create mode 100644 tests/cli/commands/bench/engine_bench/test_progress.py create mode 100644 tests/cli/commands/bench/engine_bench/test_request_sender.py create mode 100644 tests/cli/commands/bench/engine_bench/test_stats.py create mode 100644 tests/cli/commands/bench/engine_bench/workloads/__init__.py create mode 100644 tests/cli/commands/bench/engine_bench/workloads/test_base_workload.py create mode 100644 tests/cli/commands/bench/engine_bench/workloads/test_create_workload.py create mode 100644 tests/cli/commands/bench/engine_bench/workloads/test_long_doc_permutator.py create mode 100644 tests/cli/commands/bench/engine_bench/workloads/test_long_doc_qa.py create mode 100644 tests/cli/commands/bench/engine_bench/workloads/test_multi_round_chat.py create mode 100644 tests/cli/commands/bench/engine_bench/workloads/test_prefix_suffix_tuner.py create mode 100644 tests/cli/commands/bench/engine_bench/workloads/test_random_prefill.py create mode 100644 tests/cli/commands/bench/l2_adapter_bench/test_data.py create mode 100644 tests/cli/commands/bench/l2_adapter_bench/test_profiling.py create mode 100644 tests/cli/commands/bench/test_bench_command.py create mode 100644 tests/cli/commands/bench/test_server_bench.py create mode 100644 tests/cli/commands/test_coordinator.py create mode 100644 tests/cli/commands/test_query.py create mode 100644 tests/cli/commands/test_quota.py create mode 100644 tests/cli/commands/test_server.py create mode 100644 tests/cli/commands/trace/__init__.py create mode 100644 tests/cli/commands/trace/test_dispatch.py create mode 100644 tests/cli/commands/trace/test_driver.py create mode 100644 tests/cli/commands/trace/test_stats.py create mode 100644 tests/cli/conftest.py create mode 100644 tests/cli/test_describe.py create mode 100644 tests/cli/test_ping.py create mode 100644 tests/conftest.py create mode 100644 tests/data/test_creation_from_file/disk.yaml create mode 100644 tests/data/test_creation_from_file/fail.yaml create mode 100644 tests/data/test_creation_from_file/hybrid.yaml create mode 100644 tests/data/test_creation_from_file/local.yaml create mode 100644 tests/data/test_creation_from_file/remote.yaml create mode 100644 tests/disagg/README.md create mode 100644 tests/disagg/test_nixl_cache_engine.py create mode 100644 tests/disagg/test_nixl_channel.py create mode 100644 tests/disagg/test_nixl_storage_backend.py create mode 100644 tests/lmcache_frontend/__init__.py create mode 100644 tests/lmcache_frontend/lmcache_mp_plugin/__init__.py create mode 100644 tests/lmcache_frontend/lmcache_mp_plugin/test_lmcache_mp_frontend_plugin.py create mode 100644 tests/lmcache_frontend/test_heartbeat.py create mode 100644 tests/test_banner.py create mode 100644 tests/test_observability.py create mode 100644 tests/test_serde.py create mode 100644 tests/test_usage_telemetry.py create mode 100644 tests/test_utils.py create mode 100644 tests/tools/test_cache_simulator.py create mode 100644 tests/tools/test_controller_zmq_benchmark.py create mode 100644 tests/v1/__init__.py create mode 100644 tests/v1/cache_controller/__init__.py create mode 100644 tests/v1/cache_controller/conftest.py create mode 100644 tests/v1/cache_controller/pytest.ini create mode 100644 tests/v1/cache_controller/test_config.py create mode 100644 tests/v1/cache_controller/test_full_sync.py create mode 100644 tests/v1/cache_controller/test_full_sync_sender.py create mode 100644 tests/v1/cache_controller/test_kv_controller.py create mode 100644 tests/v1/cache_controller/test_locks.py create mode 100644 tests/v1/cache_controller/test_messages.py create mode 100644 tests/v1/cache_controller/test_registration_controller.py create mode 100644 tests/v1/cache_controller/test_registry_tree.py create mode 100644 tests/v1/compute/attention/test_triton_kernels.py create mode 100644 tests/v1/compute/attention/test_triton_sparse_e2e.py create mode 100644 tests/v1/data/gds.yaml create mode 100644 tests/v1/data/hipfile.yaml create mode 100644 tests/v1/data/hipfile_gds.yaml create mode 100644 tests/v1/data/nixl.yaml create mode 100644 tests/v1/data/nixl_multipath.yaml create mode 100644 tests/v1/data/test_config.yaml create mode 100644 tests/v1/distributed/__init__.py create mode 100644 tests/v1/distributed/conftest.py create mode 100644 tests/v1/distributed/l2_adapters/test_p2p_l2_adapter.py create mode 100644 tests/v1/distributed/l2_adapters/test_p2p_l2_adapter_integration.py create mode 100644 tests/v1/distributed/memory_manager/__init__.py create mode 100644 tests/v1/distributed/memory_manager/test_gds_l1_memory_manager.py create mode 100644 tests/v1/distributed/serde/__init__.py create mode 100644 tests/v1/distributed/serde/test_asym_k16_v8_multi.py create mode 100644 tests/v1/distributed/serde/test_asym_k16_v8_v_only.py create mode 100644 tests/v1/distributed/serde/test_async_processor.py create mode 100644 tests/v1/distributed/serde/test_factory.py create mode 100644 tests/v1/distributed/serde/test_fp8.py create mode 100644 tests/v1/distributed/serde/test_multi.py create mode 100644 tests/v1/distributed/serde/test_serde_e2e.py create mode 100644 tests/v1/distributed/serde/test_serde_fs_e2e.py create mode 100644 tests/v1/distributed/serde/test_turboquant.py create mode 100644 tests/v1/distributed/serde/test_utils.py create mode 100644 tests/v1/distributed/test_aerospike_l2_adapter_config.py create mode 100644 tests/v1/distributed/test_aerospike_l2_integration.py create mode 100644 tests/v1/distributed/test_bitmap_ops.py create mode 100644 tests/v1/distributed/test_cache_salt_l2_eviction.py create mode 100644 tests/v1/distributed/test_dax_l2_adapter.py create mode 100644 tests/v1/distributed/test_devdax_l1_allocator.py create mode 100644 tests/v1/distributed/test_distributed_storage_manager.py create mode 100644 tests/v1/distributed/test_encoded_object_key.py create mode 100644 tests/v1/distributed/test_fault_inject_l2_adapter.py create mode 100644 tests/v1/distributed/test_fs_l2_adapter_keys.py create mode 100644 tests/v1/distributed/test_hfbucket_l2_adapter.py create mode 100644 tests/v1/distributed/test_isolated_lru_eviction_policy.py create mode 100644 tests/v1/distributed/test_l1_l2_state_metrics.py create mode 100644 tests/v1/distributed/test_l1_manager.py create mode 100644 tests/v1/distributed/test_l1_memory_manager.py create mode 100644 tests/v1/distributed/test_l1_single_region.py create mode 100644 tests/v1/distributed/test_l2_adapter_base.py create mode 100644 tests/v1/distributed/test_l2_adapter_factory.py create mode 100644 tests/v1/distributed/test_lru_eviction_policy.py create mode 100644 tests/v1/distributed/test_mock_l2_adapter.py create mode 100644 tests/v1/distributed/test_mooncake_store_l2_adapter.py create mode 100644 tests/v1/distributed/test_multimodel.py create mode 100644 tests/v1/distributed/test_native_connector_l2_adapter.py create mode 100644 tests/v1/distributed/test_nixl_store_dynamic_l2_adapter.py create mode 100644 tests/v1/distributed/test_nixl_store_l2_adapter.py create mode 100644 tests/v1/distributed/test_object_key_parallel.py create mode 100644 tests/v1/distributed/test_prefetch_controller.py create mode 100644 tests/v1/distributed/test_prefetch_policy.py create mode 100644 tests/v1/distributed/test_quota_manager.py create mode 100644 tests/v1/distributed/test_raw_block_l2_adapter.py create mode 100644 tests/v1/distributed/test_report_status.py create mode 100644 tests/v1/distributed/test_resp_env_vars.py create mode 100644 tests/v1/distributed/test_resp_l2_adapter_integration.py create mode 100644 tests/v1/distributed/test_runtime_adapters.py create mode 100644 tests/v1/distributed/test_s3_l2_adapter.py create mode 100644 tests/v1/distributed/test_storage_manager_l2_keys.py create mode 100644 tests/v1/distributed/test_store_controller.py create mode 100644 tests/v1/distributed/test_store_policy.py create mode 100644 tests/v1/distributed/transfer_channel/__init__.py create mode 100644 tests/v1/distributed/transfer_channel/test_api.py create mode 100644 tests/v1/distributed/transfer_channel/test_context_lifecycle.py create mode 100644 tests/v1/distributed/transfer_channel/test_factory.py create mode 100644 tests/v1/distributed/transfer_channel/test_nixl_impl.py create mode 100644 tests/v1/gpu_connector/test_blocks_first_fused_kv_format.py create mode 100644 tests/v1/gpu_connector/test_concrete_shape.py create mode 100644 tests/v1/gpu_connector/test_gds_context.py create mode 100644 tests/v1/gpu_connector/test_hipfile_async.py create mode 100644 tests/v1/gpu_connector/test_kv_format_classification.py create mode 100644 tests/v1/gpu_connector/test_kv_format_detection.py create mode 100644 tests/v1/gpu_connector/test_kv_format_specs.py create mode 100644 tests/v1/gpu_connector/test_normalize_per_layer_formats.py create mode 100644 tests/v1/gpu_connector/test_utils_shape_desc.py create mode 100644 tests/v1/internal_api_server/test_backend_api.py create mode 100644 tests/v1/internal_api_server/test_cache_clear.py create mode 100644 tests/v1/internal_api_server/test_hot_cache_api.py create mode 100644 tests/v1/internal_api_server/test_kvcache_check_api.py create mode 100644 tests/v1/internal_api_server/test_load_fs_chunks.py create mode 100644 tests/v1/internal_api_server/test_lookup_api.py create mode 100644 tests/v1/internal_api_server/test_per_instance_app.py create mode 100644 tests/v1/internal_api_server/test_run_script.py create mode 100644 tests/v1/internal_api_server/test_scripts/allowed_imports.py create mode 100644 tests/v1/internal_api_server/test_scripts/basic.py create mode 100644 tests/v1/internal_api_server/test_scripts/builtin_functions.py create mode 100644 tests/v1/internal_api_server/test_scripts/default_result.py create mode 100644 tests/v1/internal_api_server/test_scripts/empty_allowed_imports.py create mode 100644 tests/v1/internal_api_server/test_scripts/exception.py create mode 100644 tests/v1/internal_api_server/test_scripts/invalid_module.py create mode 100644 tests/v1/internal_api_server/test_scripts/math_module.py create mode 100644 tests/v1/internal_api_server/test_scripts/multiple_imports.py create mode 100644 tests/v1/internal_api_server/test_scripts/os_module.py create mode 100644 tests/v1/internal_api_server/test_scripts/without_allowed_imports.py create mode 100644 tests/v1/internal_api_server/test_worker_info_api.py create mode 100644 tests/v1/lookup_client/__init__.py create mode 100644 tests/v1/lookup_client/test_chunk_statistics_lookup_client.py create mode 100644 tests/v1/lookup_client/test_lmcache_lookup_client.py create mode 100644 tests/v1/mp_coordinator/__init__.py create mode 100644 tests/v1/mp_coordinator/test_api.py create mode 100644 tests/v1/mp_coordinator/test_blend_client.py create mode 100644 tests/v1/mp_coordinator/test_blend_directory.py create mode 100644 tests/v1/mp_coordinator/test_blend_directory_api.py create mode 100644 tests/v1/mp_coordinator/test_cache_api.py create mode 100644 tests/v1/mp_coordinator/test_config.py create mode 100644 tests/v1/mp_coordinator/test_eviction_manager.py create mode 100644 tests/v1/mp_coordinator/test_health.py create mode 100644 tests/v1/mp_coordinator/test_integration.py create mode 100644 tests/v1/mp_coordinator/test_prefetch_manager.py create mode 100644 tests/v1/mp_coordinator/test_quota_api.py create mode 100644 tests/v1/mp_coordinator/test_registrar.py create mode 100644 tests/v1/mp_coordinator/test_registry.py create mode 100644 tests/v1/mp_coordinator/test_resync_manager.py create mode 100644 tests/v1/mp_coordinator/test_usage_manager.py create mode 100644 tests/v1/mp_observability/subscribers/__init__.py create mode 100644 tests/v1/mp_observability/subscribers/logging/__init__.py create mode 100644 tests/v1/mp_observability/subscribers/logging/test_cb_server.py create mode 100644 tests/v1/mp_observability/subscribers/logging/test_lookup_hash_logger.py create mode 100644 tests/v1/mp_observability/subscribers/logging/test_mp_server.py create mode 100644 tests/v1/mp_observability/subscribers/logging/test_timeout.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/counter_helpers.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/otel_setup.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_cb_server.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_engine.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_event_bus_self_metrics.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_l0_l1_throughput.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_l0_lifecycle.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_l1.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_l1_eviction_loop.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_l1_failures.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_l1_lifecycle.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_l2.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_l2_failures.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_l2_throughput.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_lookup.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_sm_lifecycle.py create mode 100644 tests/v1/mp_observability/subscribers/metrics/test_timeout.py create mode 100644 tests/v1/mp_observability/subscribers/tracing/__init__.py create mode 100644 tests/v1/mp_observability/subscribers/tracing/test_cb_server.py create mode 100644 tests/v1/mp_observability/subscribers/tracing/test_mp_server.py create mode 100644 tests/v1/mp_observability/subscribers/tracing/test_timeout.py create mode 100644 tests/v1/mp_observability/test_errors.py create mode 100644 tests/v1/mp_observability/test_event_bus.py create mode 100644 tests/v1/mp_observability/test_event_recorder.py create mode 100644 tests/v1/mp_observability/test_otel_init.py create mode 100644 tests/v1/mp_observability/test_register_gauge.py create mode 100644 tests/v1/mp_observability/trace/__init__.py create mode 100644 tests/v1/mp_observability/trace/test_codecs.py create mode 100644 tests/v1/mp_observability/trace/test_decorator.py create mode 100644 tests/v1/mp_observability/trace/test_recorder.py create mode 100644 tests/v1/multiprocess/__init__.py create mode 100644 tests/v1/multiprocess/http_apis/__init__.py create mode 100644 tests/v1/multiprocess/http_apis/test_cache_api.py create mode 100644 tests/v1/multiprocess/http_apis/test_common_api.py create mode 100644 tests/v1/multiprocess/http_apis/test_config_api.py create mode 100644 tests/v1/multiprocess/http_apis/test_info_api.py create mode 100644 tests/v1/multiprocess/http_apis/test_reconfigure_api.py create mode 100644 tests/v1/multiprocess/test_affinity_pool.py create mode 100644 tests/v1/multiprocess/test_async_engine_driven_transfer_context.py create mode 100644 tests/v1/multiprocess/test_batched_iteration_with_skip.py create mode 100644 tests/v1/multiprocess/test_blend_v3_load_store_opts.py create mode 100644 tests/v1/multiprocess/test_cache_server.py create mode 100644 tests/v1/multiprocess/test_completion_recorder.py create mode 100644 tests/v1/multiprocess/test_compute_mp_checksums.py create mode 100644 tests/v1/multiprocess/test_config.py create mode 100644 tests/v1/multiprocess/test_custom_types.py create mode 100644 tests/v1/multiprocess/test_engine_driven_transfer.py create mode 100644 tests/v1/multiprocess/test_engine_passthroughs.py create mode 100644 tests/v1/multiprocess/test_free_locks.py create mode 100644 tests/v1/multiprocess/test_futures.py create mode 100644 tests/v1/multiprocess/test_http_api_registry.py create mode 100644 tests/v1/multiprocess/test_http_quota_endpoints.py create mode 100644 tests/v1/multiprocess/test_http_server.py create mode 100644 tests/v1/multiprocess/test_lmcache_driven_layout_registry.py create mode 100644 tests/v1/multiprocess/test_lookup_wait_prefetch.py create mode 100644 tests/v1/multiprocess/test_mp_runtime_plugin_launcher.py create mode 100644 tests/v1/multiprocess/test_mq.py create mode 100644 tests/v1/multiprocess/test_mq_handler_helpers.py create mode 100644 tests/v1/multiprocess/test_optimized_lookup_v3.py create mode 100644 tests/v1/multiprocess/test_p2p_controller.py create mode 100644 tests/v1/multiprocess/test_posix_shm.py create mode 100644 tests/v1/multiprocess/test_query_lookup_hits.py create mode 100644 tests/v1/multiprocess/test_raw_block_l2_adapter.py create mode 100644 tests/v1/multiprocess/test_session.py create mode 100644 tests/v1/multiprocess/test_skip_first_n_tokens.py create mode 100644 tests/v1/multiprocess/test_token_hasher.py create mode 100644 tests/v1/multiprocess/test_unified_touch.py create mode 100644 tests/v1/multiprocess/test_usage_telemetry_server.py create mode 100644 tests/v1/multiprocess/test_warm_prefetch.py create mode 100644 tests/v1/multiprocess/test_worker_liveness.py create mode 100644 tests/v1/native_storage_ops/test_bitmap.py create mode 100644 tests/v1/native_storage_ops/test_pattern_matcher.py create mode 100644 tests/v1/native_storage_ops/test_periodic_event_notifier.py create mode 100644 tests/v1/native_storage_ops/test_ttl_lock.py create mode 100644 tests/v1/platform/__init__.py create mode 100644 tests/v1/platform/test_cache_context_dispatch.py create mode 100644 tests/v1/platform/test_cpu_shm.py create mode 100644 tests/v1/platform/test_gpu_cache_context.py create mode 100644 tests/v1/plugin/test_runtime_plugin_launcher.py create mode 100644 tests/v1/shm_allocator/__init__.py create mode 100644 tests/v1/shm_allocator/csrc/.gitignore create mode 100644 tests/v1/shm_allocator/csrc/CMakeLists.txt create mode 100644 tests/v1/shm_allocator/csrc/shm_file_worker.cpp create mode 100644 tests/v1/shm_allocator/shmfile_basic_check.yaml create mode 100644 tests/v1/shm_allocator/shmfile_connector.py create mode 100644 tests/v1/shm_allocator/test_shm_allocator.py create mode 100644 tests/v1/shm_allocator/verify_shmfile_tcp.py create mode 100644 tests/v1/storage_backend/__init__.py create mode 100644 tests/v1/storage_backend/raw_block_test_utils.py create mode 100644 tests/v1/storage_backend/test_audit_connector.py create mode 100644 tests/v1/storage_backend/test_azure_connector.py create mode 100644 tests/v1/storage_backend/test_batched_message_sender.py create mode 100644 tests/v1/storage_backend/test_bigtable_connector.py create mode 100644 tests/v1/storage_backend/test_bigtable_integration.py create mode 100644 tests/v1/storage_backend/test_dax_backend.py create mode 100644 tests/v1/storage_backend/test_eic.py create mode 100644 tests/v1/storage_backend/test_fs_connector.py create mode 100644 tests/v1/storage_backend/test_gds_backend.py create mode 100644 tests/v1/storage_backend/test_hf3fs_connector.py create mode 100644 tests/v1/storage_backend/test_hfbucket_connector.py create mode 100644 tests/v1/storage_backend/test_local_cpu_backend.py create mode 100644 tests/v1/storage_backend/test_local_cpu_backend_nixl_paged.py create mode 100644 tests/v1/storage_backend/test_local_disk_backend.py create mode 100644 tests/v1/storage_backend/test_maru_backend.py create mode 100644 tests/v1/storage_backend/test_nixl_shared_pool.py create mode 100644 tests/v1/storage_backend/test_p2p_backend_with_controller.py create mode 100644 tests/v1/storage_backend/test_path_sharder.py create mode 100644 tests/v1/storage_backend/test_pd_backend_async.py create mode 100644 tests/v1/storage_backend/test_pd_backend_bidirectional.py create mode 100644 tests/v1/storage_backend/test_pd_backend_buffer_alignment.py create mode 100644 tests/v1/storage_backend/test_raw_block_core.py create mode 100644 tests/v1/storage_backend/test_raw_block_device.py create mode 100644 tests/v1/storage_backend/test_raw_block_key_codec.py create mode 100644 tests/v1/storage_backend/test_raw_block_uring_cmd.py create mode 100644 tests/v1/storage_backend/test_remote_storage_plugin.py create mode 100644 tests/v1/storage_backend/test_resp_connector.py create mode 100644 tests/v1/storage_backend/test_rust_raw_block_backend.py create mode 100644 tests/v1/storage_backend/test_storage_manager.py create mode 100644 tests/v1/storage_backend/test_storage_plugin.py create mode 100644 tests/v1/storage_backend/test_valkey_connector.py create mode 100644 tests/v1/test_address_manager.py create mode 100644 tests/v1/test_basic_check.py create mode 100644 tests/v1/test_bloom_filter.py create mode 100644 tests/v1/test_c_ops_fallback_parity.py create mode 100644 tests/v1/test_cache_engine.py create mode 100644 tests/v1/test_cache_engine_cleanup.py create mode 100644 tests/v1/test_cache_interface.py create mode 100644 tests/v1/test_cache_policy.py create mode 100644 tests/v1/test_config.py create mode 100644 tests/v1/test_connector.py create mode 100644 tests/v1/test_connector_discovery.py create mode 100644 tests/v1/test_decode_save_and_preemption.py create mode 100644 tests/v1/test_device_id_race.py create mode 100644 tests/v1/test_ec_connector.py create mode 100644 tests/v1/test_event_notifier.py create mode 100644 tests/v1/test_freeze_mode_integration.py create mode 100644 tests/v1/test_gds.py create mode 100644 tests/v1/test_gpu_connector.py create mode 100644 tests/v1/test_health_monitor.py create mode 100644 tests/v1/test_health_monitor_fallback_recovery.py create mode 100644 tests/v1/test_hidden_state_store.py create mode 100644 tests/v1/test_impl_completeness.py create mode 100644 tests/v1/test_json_utils.py create mode 100644 tests/v1/test_kv_cache_groups.py create mode 100644 tests/v1/test_kv_layer_groups_manager.py create mode 100644 tests/v1/test_manager.py create mode 100644 tests/v1/test_mem_kernels.py create mode 100644 tests/v1/test_memory_allocator_import_surface.py create mode 100644 tests/v1/test_memory_management.py create mode 100644 tests/v1/test_mp_mem_kernels.py create mode 100644 tests/v1/test_musa_connector.py create mode 100644 tests/v1/test_musa_mp_handle.py create mode 100644 tests/v1/test_musa_native.py create mode 100644 tests/v1/test_musa_support.py create mode 100644 tests/v1/test_nixl_batched_contains.py create mode 100644 tests/v1/test_nixl_doca_memos.py create mode 100644 tests/v1/test_nixl_multipath.py create mode 100644 tests/v1/test_nixl_storage.py create mode 100644 tests/v1/test_pos_kernels.py create mode 100644 tests/v1/test_python_ops_fallback.py create mode 100644 tests/v1/test_remote_metadata.py create mode 100644 tests/v1/test_remote_mla_worker_id_as0.py create mode 100644 tests/v1/test_staging_buffer.py create mode 100644 tests/v1/test_subclass_discovery.py create mode 100644 tests/v1/test_token_database.py create mode 100644 tests/v1/test_trtllm_integration.py create mode 100644 tests/v1/test_v1_adapter_request_finished_scheduler_role.py create mode 100644 tests/v1/test_v1_adapter_state_desync.py create mode 100644 tests/v1/test_vllm_integration.py create mode 100644 tests/v1/test_vllm_kv_cache_groups.py create mode 100644 tests/v1/test_vllm_layerwise_wait_for_save.py create mode 100644 tests/v1/test_vllm_mp_adapter.py create mode 100644 tests/v1/test_xpu_connector.py create mode 100644 tests/v1/test_xpu_sglang_connector.py create mode 100644 tests/v1/utils.py create mode 100644 tools/check_spdx_header.py create mode 100644 tools/translate_docs_zh.py diff --git a/.buildkite/README.md b/.buildkite/README.md new file mode 100644 index 0000000..73ccfde --- /dev/null +++ b/.buildkite/README.md @@ -0,0 +1,15 @@ +# Comprehensive Tests + +An end-to-end integration suite for LMCache & vLLM latest branch. + +## Layout + +- **Scripts**: `scripts/vllm-integration-tests.sh` +- **Configs**: `configs/` +- **Pipeline**: `pipelines/comprehensive-tests.yml` + +## Prepare + +1. Add your YAMLs to `configs/` (e.g. `local_cpu.yaml`, `local_disk.yaml`). + +2. Add the filenames to `cases/comprehensive-cases.txt`. diff --git a/.buildkite/cases/comprehensive-cases.txt b/.buildkite/cases/comprehensive-cases.txt new file mode 100644 index 0000000..8ca9964 --- /dev/null +++ b/.buildkite/cases/comprehensive-cases.txt @@ -0,0 +1,11 @@ +local_cpu.yaml +local_cpu_mla.yaml +local_disk.yaml +pd.yaml +multi_device.yaml +async.yaml +p2p.yaml +layerwise.yaml +local_cpu_with_v3.yaml +local_disk_with_v3.yaml +p2p_with_v3.yaml \ No newline at end of file diff --git a/.buildkite/cases/integration-cases.txt b/.buildkite/cases/integration-cases.txt new file mode 100644 index 0000000..1cc79cb --- /dev/null +++ b/.buildkite/cases/integration-cases.txt @@ -0,0 +1 @@ +dummy.yaml diff --git a/.buildkite/configs/async.yaml b/.buildkite/configs/async.yaml new file mode 100644 index 0000000..df244dd --- /dev/null +++ b/.buildkite/configs/async.yaml @@ -0,0 +1,32 @@ +workload: + type: long_doc_qa + max-inflight-requests: 20 + sleep-time-after-warmup: 20 + num-documents: 20 + repeat-count: 1 + hit-miss-ratio: 2:2 + +docker: + env: + - "LMCACHE_CHUNK_SIZE=256" + - "LMCACHE_LOCAL_CPU=False" + - "LMCACHE_MAX_LOCAL_CPU_SIZE=70" + - "LMCACHE_MAX_LOCAL_DISK_SIZE=70" + - "LMCACHE_LOCAL_DISK=\"file:///local/end-to-end-tests/local/\"" + - "LMCACHE_ENABLE_ASYNC_LOADING=True" + - "LMCACHE_EXTRA_CONFIG={\"lookup_backoff_time\": 0.01, \"use_odirect\": True}" + - "LMCACHE_SAVE_UNFULL_CHUNK=False" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + +vllm: + model: "meta-llama/Llama-3.1-8B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + +checking-fields: + - query_round_time_per_prompt diff --git a/.buildkite/configs/dummy.yaml b/.buildkite/configs/dummy.yaml new file mode 100644 index 0000000..c266453 --- /dev/null +++ b/.buildkite/configs/dummy.yaml @@ -0,0 +1,22 @@ +workload: + type: dummy + +docker: + env: + - "LMCACHE_CHUNK_SIZE=256" + - "LMCACHE_LOCAL_CPU=True" + - "LMCACHE_MAX_LOCAL_CPU_SIZE=5" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + +vllm: + model: "meta-llama/Llama-3.2-1B-Instruct" + args: + - "--max-model-len" + - "1024" + - "--gpu-memory-utilization" + - "0.35" + - "--load-format" + - "dummy" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" \ No newline at end of file diff --git a/.buildkite/configs/layerwise.yaml b/.buildkite/configs/layerwise.yaml new file mode 100644 index 0000000..ed41144 --- /dev/null +++ b/.buildkite/configs/layerwise.yaml @@ -0,0 +1,26 @@ +workload: + type: long_doc_qa + max-inflight-requests: 20 + +docker: + env: + - "LMCACHE_CHUNK_SIZE=256" + - "LMCACHE_LOCAL_CPU=True" + - "LMCACHE_MAX_LOCAL_CPU_SIZE=5" + - "LMCACHE_USE_LAYERWISE=true" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + +vllm: + model: "meta-llama/Llama-3.2-1B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + - "--compilation-config" + - "{\"cudagraph_mode\":\"PIECEWISE\"}" + +checking-fields: + - query_round_time_per_prompt diff --git a/.buildkite/configs/lmcache_configs/local_cpu_mla.yaml b/.buildkite/configs/lmcache_configs/local_cpu_mla.yaml new file mode 100644 index 0000000..8acd238 --- /dev/null +++ b/.buildkite/configs/lmcache_configs/local_cpu_mla.yaml @@ -0,0 +1,15 @@ +# LMCache Configuration File +# This replaces the individual environment variables for cleaner configuration + +# Basic configurations +chunk_size: 256 +local_cpu: true +max_local_cpu_size: 5 + +internal_api_server_enabled: true +internal_api_server_socket_path_prefix: "/tmp/lmcache_internal_api_server/socket" +internal_api_server_include_index_list: [1] + +extra_config: + save_only_first_rank: true + first_rank_max_local_cpu_size: 5 diff --git a/.buildkite/configs/local_cpu.yaml b/.buildkite/configs/local_cpu.yaml new file mode 100644 index 0000000..6727480 --- /dev/null +++ b/.buildkite/configs/local_cpu.yaml @@ -0,0 +1,23 @@ +workload: + type: long_doc_qa + max-inflight-requests: 20 + +docker: + env: + - "LMCACHE_CHUNK_SIZE=256" + - "LMCACHE_LOCAL_CPU=True" + - "LMCACHE_MAX_LOCAL_CPU_SIZE=5" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + +vllm: + model: "meta-llama/Llama-3.2-1B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + +checking-fields: + - query_round_time_per_prompt diff --git a/.buildkite/configs/local_cpu_mla.yaml b/.buildkite/configs/local_cpu_mla.yaml new file mode 100644 index 0000000..a48f287 --- /dev/null +++ b/.buildkite/configs/local_cpu_mla.yaml @@ -0,0 +1,30 @@ +workload: + type: long_doc_qa + max-inflight-requests: 20 + +docker: + env: + - "LMCACHE_CONFIG_FILE=/etc/lmcache/local_cpu_mla.yaml" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + gpu_count: 2 + +vllm: + model: "deepseek-ai/DeepSeek-V2-Lite-Chat" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - '{"kv_connector":"LMCacheConnectorV1Dynamic","kv_role":"kv_both","kv_connector_module_path":"lmcache.integration.vllm.lmcache_connector_v1","kv_connector_extra_config":{"discard_partial_chunks":false}}' + - "--tensor-parallel-size" + - "2" + - "--block-size" + - "64" + - "--kv-cache-dtype" + - "fp8_e4m3" + - "--quantization" + - "fp8" + +checking-fields: + - query_round_time_per_prompt diff --git a/.buildkite/configs/local_cpu_with_v3.yaml b/.buildkite/configs/local_cpu_with_v3.yaml new file mode 100644 index 0000000..d5592ee --- /dev/null +++ b/.buildkite/configs/local_cpu_with_v3.yaml @@ -0,0 +1,25 @@ +workload: + type: long_doc_qa + max-inflight-requests: 20 + expected-latency-gain: true + +docker: + env: + - "LMCACHE_CHUNK_SIZE=256" + - "LMCACHE_LOCAL_CPU=True" + - "LMCACHE_MAX_LOCAL_CPU_SIZE=5" + - "LMCACHE_USE_GPU_CONNECTOR_V3=True" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + +vllm: + model: "meta-llama/Llama-3.2-1B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + +checking-fields: + - query_round_time_per_prompt diff --git a/.buildkite/configs/local_disk.yaml b/.buildkite/configs/local_disk.yaml new file mode 100644 index 0000000..0e5162c --- /dev/null +++ b/.buildkite/configs/local_disk.yaml @@ -0,0 +1,26 @@ +workload: + type: long_doc_qa + max-inflight-requests: 20 + sleep-time-after-warmup: 20 + +docker: + env: + - "LMCACHE_CHUNK_SIZE=256" + - "LMCACHE_LOCAL_CPU=False" + - "LMCACHE_MAX_LOCAL_CPU_SIZE=10" + - "LMCACHE_MAX_LOCAL_DISK_SIZE=10" + - "LMCACHE_LOCAL_DISK=\"file:///local/end-to-end-tests/local/\"" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + +vllm: + model: "meta-llama/Llama-3.2-1B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + +checking-fields: + - query_round_time_per_prompt diff --git a/.buildkite/configs/local_disk_with_v3.yaml b/.buildkite/configs/local_disk_with_v3.yaml new file mode 100644 index 0000000..0bc380a --- /dev/null +++ b/.buildkite/configs/local_disk_with_v3.yaml @@ -0,0 +1,27 @@ +workload: + type: long_doc_qa + max-inflight-requests: 20 + sleep-time-after-warmup: 20 + +docker: + env: + - "LMCACHE_CHUNK_SIZE=256" + - "LMCACHE_LOCAL_CPU=False" + - "LMCACHE_MAX_LOCAL_CPU_SIZE=10" + - "LMCACHE_MAX_LOCAL_DISK_SIZE=10" + - "LMCACHE_LOCAL_DISK=\"file:///local/end-to-end-tests/local/\"" + - "LMCACHE_USE_GPU_CONNECTOR_V3=True" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + +vllm: + model: "meta-llama/Llama-3.2-1B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + +checking-fields: + - query_round_time_per_prompt diff --git a/.buildkite/configs/multi_device.yaml b/.buildkite/configs/multi_device.yaml new file mode 100644 index 0000000..9bb5bf9 --- /dev/null +++ b/.buildkite/configs/multi_device.yaml @@ -0,0 +1,26 @@ +workload: + type: long_doc_qa + max-inflight-requests: 20 + +docker: + env: + - "LMCACHE_CHUNK_SIZE=256" + - "LMCACHE_LOCAL_CPU=True" + - "LMCACHE_MAX_LOCAL_CPU_SIZE=5" + - "LMCACHE_GDS_PATH=/tmp/" + - "LMCACHE_GDS_BUFFER_SIZE=4096" + - "LMCACHE_EXTRA_CONFIG={\"use_direct_io\": true}" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + +vllm: + model: "meta-llama/Llama-3.2-1B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + +checking-fields: + - query_round_time_per_prompt diff --git a/.buildkite/configs/p2p.yaml b/.buildkite/configs/p2p.yaml new file mode 100644 index 0000000..f66010f --- /dev/null +++ b/.buildkite/configs/p2p.yaml @@ -0,0 +1,69 @@ +workload: + type: long_doc_qa + num-documents: 20 + max-inflight-requests: 1 + repeat-count: 1 + +feature: + type: p2p + +docker1: + env: + - "LMCACHE_MAX_LOCAL_CPU_SIZE=60" + - "LMCACHE_ENABLE_ASYNC_LOADING=True" + - "LMCACHE_ENABLE_P2P=True" + - "LMCACHE_P2P_HOST=localhost" + - "LMCACHE_P2P_INIT_PORTS=8200" + - "LMCACHE_P2P_LOOKUP_PORTS=8201" + - "LMCACHE_TRANSFER_CHANNEL=nixl" + - "LMCACHE_ENABLE_CONTROLLER=True" + - "LMCACHE_LMCACHE_INSTANCE_ID=lmcache_instance_1" + - "LMCACHE_LMCACHE_WORKER_PORTS=8500" + - "LMCACHE_EXTRA_CONFIG={\"lookup_backoff_time\": 0.001}" + - "LMCACHE_SAVE_UNFULL_CHUNK=False" + - "PYTHONHASHSEED=123" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + pull-port: 8300 + reply-port: 8400 + +docker2: + env: + - "LMCACHE_MAX_LOCAL_CPU_SIZE=60" + - "LMCACHE_ENABLE_ASYNC_LOADING=True" + - "LMCACHE_ENABLE_P2P=True" + - "LMCACHE_P2P_HOST=localhost" + - "LMCACHE_P2P_INIT_PORTS=8202" + - "LMCACHE_P2P_LOOKUP_PORTS=8203" + - "LMCACHE_TRANSFER_CHANNEL=nixl" + - "LMCACHE_ENABLE_CONTROLLER=True" + - "LMCACHE_LMCACHE_INSTANCE_ID=lmcache_instance_2" + - "LMCACHE_LMCACHE_WORKER_PORTS=8501" + - "LMCACHE_EXTRA_CONFIG={\"lookup_backoff_time\": 0.001}" + - "LMCACHE_SAVE_UNFULL_CHUNK=False" + - "PYTHONHASHSEED=123" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + pull-port: 8300 + reply-port: 8400 + +vllm1: + model: "meta-llama/Llama-3.1-8B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + +vllm2: + model: "meta-llama/Llama-3.1-8B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + +checking-fields: + - warmup_round_time_per_prompt diff --git a/.buildkite/configs/p2p_with_v3.yaml b/.buildkite/configs/p2p_with_v3.yaml new file mode 100644 index 0000000..708e2ad --- /dev/null +++ b/.buildkite/configs/p2p_with_v3.yaml @@ -0,0 +1,71 @@ +workload: + type: long_doc_qa + num-documents: 20 + max-inflight-requests: 1 + repeat-count: 1 + +feature: + type: p2p + +docker1: + env: + - "LMCACHE_MAX_LOCAL_CPU_SIZE=60" + - "LMCACHE_ENABLE_ASYNC_LOADING=True" + - "LMCACHE_ENABLE_P2P=True" + - "LMCACHE_P2P_HOST=localhost" + - "LMCACHE_P2P_INIT_PORTS=8200" + - "LMCACHE_P2P_LOOKUP_PORTS=8201" + - "LMCACHE_TRANSFER_CHANNEL=nixl" + - "LMCACHE_ENABLE_CONTROLLER=True" + - "LMCACHE_LMCACHE_INSTANCE_ID=lmcache_instance_1" + - "LMCACHE_LMCACHE_WORKER_PORTS=8500" + - "LMCACHE_EXTRA_CONFIG={\"lookup_backoff_time\": 0.001}" + - "LMCACHE_SAVE_UNFULL_CHUNK=False" + - "PYTHONHASHSEED=123" + - "LMCACHE_USE_GPU_CONNECTOR_V3=True" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + pull-port: 8300 + reply-port: 8400 + +docker2: + env: + - "LMCACHE_MAX_LOCAL_CPU_SIZE=60" + - "LMCACHE_ENABLE_ASYNC_LOADING=True" + - "LMCACHE_ENABLE_P2P=True" + - "LMCACHE_P2P_HOST=localhost" + - "LMCACHE_P2P_INIT_PORTS=8202" + - "LMCACHE_P2P_LOOKUP_PORTS=8203" + - "LMCACHE_TRANSFER_CHANNEL=nixl" + - "LMCACHE_ENABLE_CONTROLLER=True" + - "LMCACHE_LMCACHE_INSTANCE_ID=lmcache_instance_2" + - "LMCACHE_LMCACHE_WORKER_PORTS=8501" + - "LMCACHE_EXTRA_CONFIG={\"lookup_backoff_time\": 0.001}" + - "LMCACHE_SAVE_UNFULL_CHUNK=False" + - "PYTHONHASHSEED=123" + - "LMCACHE_USE_GPU_CONNECTOR_V3=True" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + pull-port: 8300 + reply-port: 8400 + +vllm1: + model: "meta-llama/Llama-3.1-8B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + +vllm2: + model: "meta-llama/Llama-3.1-8B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_both\"}" + +checking-fields: + - warmup_round_time_per_prompt diff --git a/.buildkite/configs/pd.yaml b/.buildkite/configs/pd.yaml new file mode 100644 index 0000000..31e2244 --- /dev/null +++ b/.buildkite/configs/pd.yaml @@ -0,0 +1,74 @@ +workload: + type: long_doc_qa + max-inflight-requests: 1 + repeat-count: 1 + completion: true + num-documents: 20 + document-length: 16382 + +feature: + type: pd + +docker-prefiller: + env: + # Force vLLM Model Runner V1: MRV2 (enabled by default for Llama/Mistral + # dense models in vLLM nightly, see vllm-project/vllm#43458) is incompatible + # with KV cache connectors and hangs the PD path. Remove once MRV2 supports + # connectors / auto-falls back to MRV1. + - "VLLM_USE_V2_MODEL_RUNNER=0" + - "LMCACHE_LOCAL_CPU=False" + - "LMCACHE_MAX_LOCAL_CPU_SIZE=5" + - "LMCACHE_ENABLE_PD=True" + - "LMCACHE_TRANSFER_CHANNEL=nixl" + - "LMCACHE_PD_ROLE=sender" + - "LMCACHE_PD_PROXY_HOST=localhost" + - "LMCACHE_PD_BUFFER_SIZE=1073741824" # 1GB + - "LMCACHE_PD_BUFFER_DEVICE=cuda" + - "LMCACHE_SAVE_UNFULL_CHUNK=True" + - "PYTHONHASHSEED=0" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + proxy-port: 7500 + +docker-decoder: + env: + # Force vLLM Model Runner V1: MRV2 (enabled by default for Llama/Mistral + # dense models in vLLM nightly, see vllm-project/vllm#43458) is incompatible + # with KV cache connectors and hangs the PD path. Remove once MRV2 supports + # connectors / auto-falls back to MRV1. + - "VLLM_USE_V2_MODEL_RUNNER=0" + - "LMCACHE_LOCAL_CPU=False" + - "LMCACHE_MAX_LOCAL_CPU_SIZE=0" + - "LMCACHE_ENABLE_PD=True" + - "LMCACHE_TRANSFER_CHANNEL=nixl" + - "LMCACHE_PD_ROLE=receiver" + - "LMCACHE_PD_PEER_HOST=localhost" + - "LMCACHE_PD_BUFFER_SIZE=2147483648" # 2GB + - "LMCACHE_PD_BUFFER_DEVICE=cuda" + - "LMCACHE_SAVE_UNFULL_CHUNK=True" + - "PYTHONHASHSEED=0" + - "LMCACHE_INTERNAL_API_SERVER_ENABLED=True" + - "LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/socket" + init-port: 7300 + alloc-port: 7400 + +vllm-prefiller: + model: "meta-llama/Llama-3.2-1B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_producer\",\"kv_connector_extra_config\": {\"discard_partial_chunks\": false, \"lmcache_rpc_port\": \"producer1\"}}" + +vllm-decoder: + model: "meta-llama/Llama-3.2-1B-Instruct" + args: + - "--load-format" + - "dummy" + - "--no-enable-prefix-caching" + - "--kv-transfer-config" + - "{\"kv_connector\":\"LMCacheConnectorV1\",\"kv_role\":\"kv_consumer\",\"kv_connector_extra_config\": {\"discard_partial_chunks\": false, \"lmcache_rpc_port\": \"consumer1\", \"skip_last_n_tokens\": 1}}" + +checking-fields: + - query_round_time_per_prompt diff --git a/.buildkite/correctness/README.md b/.buildkite/correctness/README.md new file mode 100644 index 0000000..dfa14ac --- /dev/null +++ b/.buildkite/correctness/README.md @@ -0,0 +1,52 @@ +# Correctness Test + +This is a E2E testing suite for whether having a KV pass through LMCache from vllm (storing and loading) will degrade the accuracy of a transformer. + +Unit tests do not suffice in going from request to response and passing through LMCache. We use offline serving +for this testing suite because the API Server of vllm is assumed to be unproblematic. + +## Setup + +We use the [Measuring Massive Multitask Language Understanding](https://arxiv.org/abs/2009.03300) multiple choice dataset downloaded as a compressed tarball in `data.tar.xz` + +The following 3 options are viable: +A. Disable prefix caching on vllm to only use lmcache +B. Use lmcache centralized remote server +C. Use lmcache p2p sharing + +We choose option A to keep the CI server lightweight (only 2x L4s needed). + +We test one small model for each Attention Architecture. Currently dense/standard attention will use `meta-llama/Llama-3.1-8B-Instruct` and +MLA (multi-head latent attention) will use `deepseek-ai/DeepSeek-V2-Lite` with tensor parallel 2. We do not care about the objective accuracy on the MMLU benchmark but only on any differential that appears from using LMCache. + +## CI Agent Pre Set-up + +To ensure the speed of the MMLU Correctness tests, please conduct the following set up on your runner beforehand (only need to do once and `setup.sh` will renew your environment afterwards). + +1. Create a virtual environment at `~/correctness_venv` + +```bash +pip install pandas +``` + +2. Please run the following commands (these will be run by `setup.sh` every time): +```bash +cd ~ +mkdir correctness_repositories +cd correctness_repositories +git clone https://github.com/LMCache/LMCache.git +git clone https://github.com/vllm-project/vllm.git +cd LMCache +pip install -e . +cd ../vllm +pip install -e . +``` + +Explanation: "pull" upstream code in this CI suite by pulling the latest upstream default branch +and let the editable virtual environment update the latest code. This allows a little bit of pre-setup to greatly optimize every run afterwards. + +## Directory Structure +`single-mmlu-test.py` + +The MMLU question dataset will be sent to this single endpoint and accuracy will be evaluated (correct answers are included +in the dataset). When LMCache is used, queries are sent twice. \ No newline at end of file diff --git a/.buildkite/correctness/async_request.py b/.buildkite/correctness/async_request.py new file mode 100755 index 0000000..e1bfa1c --- /dev/null +++ b/.buildkite/correctness/async_request.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# Standard +import argparse +import asyncio +import copy +import json +import logging +import os +import time +import traceback + +# Third Party +from tqdm import tqdm +import aiohttp + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + +async def make_request(session, payload: tuple, args): + endpoint = args.endpoint + output_file = args.output_file + obj_name, payload = payload + headers = { + "Authorization": f"Bearer {os.getenv('API_KEY', '')}", + "Content-Type": "application/json", + } + try: + async with session.post( + endpoint, json=payload, headers=headers, ssl=False + ) as response: + json_data = await response.json() + content = ( + json_data.get("choices", [{}])[0].get("message", {}).get("content", "") + ) + logger.info(content) + if json_data.get("id") is not None: + with open(output_file, "a") as f: + f.write(f"{json_data.get('id')}:\n{content}\n") + if json_data.get("choices", [{}])[0].get("finish_reason", "") != "stop": + with open(output_file.replace(".txt", "_length.txt"), "a") as f: + f.write(f"{json_data.get('id')}:\n{content}\n") + return True, json_data + except Exception: + logger.error(f"Request failed (Object name: {obj_name})") + traceback.print_exc() + return False, None + + +async def process_requests(payloads: dict, args): + semaphore = asyncio.Semaphore(args.max_concurrency) + + async def sem_task(session, payload: tuple, args): + async with semaphore: + return await make_request(session, payload, args) + + timeout = aiohttp.ClientTimeout(total=6000) + connector = aiohttp.TCPConnector(limit=0) + async with aiohttp.ClientSession(timeout=timeout, connector=connector) as session: + tasks = [sem_task(session, payload, args) for payload in payloads.items()] + + generated_token = [] + for coro in tqdm( + asyncio.as_completed(tasks), total=len(tasks), desc="Requests Progress" + ): + success, content = await coro + if success: + generated_token.append( + content.get("usage", {}).get("completion_tokens", 0) + ) + else: + logger.error(f"Error: {content}") + logger.info(f"Total generated tokens: {sum(generated_token)}") + + +def request_concurrency(requests: dict, args): + same_req = args.same_req + rid = args.rid + counts = args.counts + model = args.model + payloads = dict() + if same_req: + for index, request in requests.items(): + if index == rid: + request.update({"model": model}) + request.update({"stream": False}) + request.update({"temperature": 0}) + request.update({"request_id": index}) + payloads[index] = request + for i in range(counts): + payloads[i] = copy.deepcopy(payloads[index]) + payloads[i]["request_id"] = str(i) + break + + else: + for index, request in requests.items(): + request.update({"model": model}) + request.update({"stream": False}) + request.update({"temperature": 0}) + request["request_id"] = index + payloads[index] = request + + start = time.time() + asyncio.run(process_requests(payloads, args)) + end = time.time() + logger.info(f"Total time: {end - start:.2f} seconds") + + +def main(): + parser = argparse.ArgumentParser(description="Concurrent request tester") + + parser.add_argument( + "--model", type=str, default="Qwen3/Qwen3-32B", help="Model name" + ) + parser.add_argument( + "--endpoint", + type=str, + required=True, + help="API endpoint, e.g. http://host:port/v1/chat/completions", + ) + parser.add_argument( + "--request-number", + type=int, + default=500, + help="Number of requests to read from dataset", + ) + parser.add_argument( + "--max-concurrency", type=int, default=128, help="Max concurrency" + ) + parser.add_argument( + "--dataset_file", type=str, default="data.json", help="Dataset json file" + ) + parser.add_argument( + "--output-file", + type=str, + default="result_output.txt", + help="Output result file prefix", + ) + parser.add_argument( + "--same-req", + action="store_true", + help="Send the same rid request multiple times", + ) + parser.add_argument( + "--rid", type=str, default="", help="The rid to repeat if same_req=True" + ) + parser.add_argument( + "--counts", type=int, default=100, help="How many times to send the same rid" + ) + + args = parser.parse_args() + + # --- load dataset --- + with open(args.dataset_file, "r", encoding="utf-8") as f: + data = json.load(f) + + data = dict(list(data.items())[: args.request_number]) + + print(f"Loaded {len(data)} requests.") + + # --- call your function --- + request_concurrency(data, args) + + +if __name__ == "__main__": + main() diff --git a/.buildkite/correctness/compare_files.py b/.buildkite/correctness/compare_files.py new file mode 100644 index 0000000..2c8635a --- /dev/null +++ b/.buildkite/correctness/compare_files.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: Apache-2.0 +# Standard +import argparse +import re + + +def load_file(path): + data = {} + current_id = None + buffer = [] + + id_pattern = re.compile(r"chatcmpl-[a-zA-Z0-9\-_]+") + + with open(path, "r", encoding="utf-8") as f: + for line in f: + line_strip = line.strip() + if id_pattern.search(line_strip): + if current_id is not None: + data[current_id] = "\n".join(buffer).strip() + current_id = id_pattern.search(line_strip).group(0) + buffer = [] + else: + buffer.append(line.rstrip()) + + if current_id is not None: + data[current_id] = "\n".join(buffer).strip() + + return data + + +def compare_files(file1, file2): + d1 = load_file(file1) + d2 = load_file(file2) + + ids1 = set(d1.keys()) + ids2 = set(d2.keys()) + + common = ids1 & ids2 + only1 = ids1 - ids2 + only2 = ids2 - ids1 + + same = [] + diff = [] + + for cid in common: + if d1[cid] == d2[cid]: + same.append(cid) + else: + diff.append(cid) + print("====== Statistics ======") + print(f"Common IDs: {len(common)}") + print(f"Identical IDs: {len(same)}") + print(f"Different IDs: {len(diff)}") + print() + print("—— Identical IDs ——") + for cid in same: + print(cid) + print() + print("—— Different IDs ——") + for cid in diff: + print(cid) + print() + print("—— Only in File 1 ——") + for cid in only1: + print(cid) + print() + print("—— Only in File 2 ——") + for cid in only2: + print(cid) + + +def main(): + parser = argparse.ArgumentParser(description="Compare two result files.") + parser.add_argument("--file1", type=str, required=True, help="Path to first file.") + parser.add_argument("--file2", type=str, required=True, help="Path to second file.") + + args = parser.parse_args() + + file1 = args.file1 + file2 = args.file2 + + compare_files(file1, file2) + + +if __name__ == "__main__": + main() diff --git a/.buildkite/correctness/mmlu-test.py b/.buildkite/correctness/mmlu-test.py new file mode 100644 index 0000000..fb4e939 --- /dev/null +++ b/.buildkite/correctness/mmlu-test.py @@ -0,0 +1,189 @@ +# SPDX-License-Identifier: Apache-2.0 +# Standard +import argparse +import json +import os +import time + +# Third Party +from tqdm import tqdm +from transformers import AutoTokenizer, set_seed + +# vLLM +from vllm import LLM, SamplingParams +from vllm.config import KVTransferConfig +import numpy as np +import pandas as pd + +# setting PYTHONHASHSEED derandomizes token chunking +os.environ["PYTHONHASHSEED"] = "0" + +global tokenizer +choices = ["A", "B", "C", "D"] + + +# grab the idx'th row of the df and generate a prompt string +# format of the MMLU csvs: +# question,option_A,option_B,option_C,option_D,answer +def prompt_string(df, idx, include_answer=True): + prompt = df.iloc[idx, 0] + k = df.shape[1] - 2 # number of columns - 2 (question and answer) + for i in range(k): + prompt += f"\n{choices[i]}. {df.iloc[idx, i + 1]}" + prompt += "\nAnswer:" + if include_answer: + prompt += f" {df.iloc[idx, k]}\n\n" + return prompt + + +def evaluate(args, llm, subject, dev_df, test_df): + prompts, labels = [], [] + + shared_multi_shot_prefix = [ + f'The following are multiple choice questions (with answers) \ + about {subject}. \ + You must respond with a single letter only. \ + Either "A", "B", "C", or "D". \ + Do not include any other text in your response. \n\n' + ] + shared_multi_shot_prefix_length = 0 + for i in range(dev_df.shape[0]): + # the multi-shot examples should contain answers + shared_multi_shot_prefix.append(prompt_string(dev_df, i)) + + # Use plain list of token IDs, no torch tensors + token_ids = tokenizer(shared_multi_shot_prefix[-1], add_special_tokens=True)[ + "input_ids" + ] + shared_multi_shot_prefix_length += len(token_ids) + + if shared_multi_shot_prefix_length > 4000: + break + + # all already have double newlines at the end + shared_multi_shot_prefix = "".join(shared_multi_shot_prefix) + + for i in range(test_df.shape[0]): + # do NOT include the answer for the actual question + # we want the LLM to answer + query_prompt = prompt_string(test_df, i, include_answer=False) + prompt = f"{shared_multi_shot_prefix}\n\n{query_prompt}" + prompts.append(prompt) + label = test_df.iloc[i, test_df.shape[1] - 1] + labels.append(label) + + # Create sampling params with deterministic settings + # (temperature=0, seed=42) + sampling_params = SamplingParams( + temperature=0, + max_tokens=2, + seed=42, + n=1, + stop=None, + ) + + # even though offline serving can batch all the prompts, we do them one at a time + # to keep the vllm scheduler deterministic + outputs = [] + for prompt in prompts: + # if we use lmcache, we need to first populate the kv cache + if args.use_lmcache: + llm.generate(prompt, sampling_params) + time.sleep(0.5) + outputs.append(llm.generate(prompt, sampling_params)) + time.sleep(0.5) + + predictions = [] + for output in outputs: + prediction = output.outputs[0].text + prediction_stripped = prediction.strip() + if prediction_stripped and prediction_stripped[0] in choices: + predictions.append(prediction_stripped[0]) + else: + # Fallback: look for any A, B, C, D in the response + for char in prediction_stripped: + if char in ["A", "B", "C", "D"]: + predictions.append(char) + break + else: + predictions.append("A") # Default fallback + + accuracy = np.mean(np.array(predictions) == np.array(labels)) + return accuracy + + +def main(args): + global tokenizer + tokenizer = AutoTokenizer.from_pretrained(args.model) + + ktc = None + if args.use_lmcache: + ktc = KVTransferConfig( + kv_connector="LMCacheConnectorV1", + kv_role="kv_both", + ) + llm = LLM( + model=args.model, + max_model_len=8000, + gpu_memory_utilization=0.9, + kv_transfer_config=ktc, + tensor_parallel_size=2, + enable_prefix_caching=False, + enforce_eager=True, + trust_remote_code=True, + ) + + mmlu_files = os.listdir("data/test") + test_files = [f for f in mmlu_files if f.endswith("_test.csv")] + subjects = sorted([f.split("_test.csv")[0] for f in test_files]) + + accuracies = [] + num_questions = [] + output_dict = {} + + for subject_raw in tqdm( + subjects[: args.number_of_subjects], + desc="Processing subjects", + ): + subject = " ".join(subject_raw.split("_")) + dev_df = pd.read_csv( + os.path.join("data/dev", subject_raw + "_dev.csv"), + header=None, + ) + test_df = pd.read_csv( + os.path.join("data/test", subject_raw + "_test.csv"), + header=None, + ) + accuracy = evaluate(args, llm, subject, dev_df, test_df) + accuracies.append(accuracy) + num_questions.append(len(test_df)) + output_dict[subject_raw] = { + "accuracy": accuracy, + "num_questions": len(test_df), + } + + total_accuracy = np.mean(accuracies) + total_num_questions = sum(num_questions) + output_dict["total"] = { + "accuracy": total_accuracy, + "num_questions": total_num_questions, + } + + with open(args.result_file, "w") as f: + # output will be a jsonl file + for subject, value in output_dict.items(): + f.write(json.dumps({subject: value}) + "\n") + + +if __name__ == "__main__": + set_seed(42) # some tokenizers may have randomness + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=str, required=True) + parser.add_argument("--number-of-subjects", type=int, default=25) + parser.add_argument("--use-lmcache", action="store_true", default=False) + args = parser.parse_args() + if args.use_lmcache: + args.result_file = args.model.split("/")[-1] + "-lmcache.jsonl" + else: + args.result_file = args.model.split("/")[-1] + "-baseline.jsonl" + main(args) diff --git a/.buildkite/correctness/pipeline.correctness.yml b/.buildkite/correctness/pipeline.correctness.yml new file mode 100644 index 0000000..24af1cb --- /dev/null +++ b/.buildkite/correctness/pipeline.correctness.yml @@ -0,0 +1,16 @@ +steps: + - label: ":rocket: vLLM correctness" + key: "correctness" + timeout_in_minutes: 60 + commands: + - | + set -euo pipefail + + echo "[INFO] HOME: $HOME" + echo "[INFO] PWD: $PWD" + + chmod +x .buildkite/scripts/vllm-correctness.sh + BUILD_ID="${BUILDKITE_BUILD_ID}" .buildkite/scripts/vllm-correctness.sh + + artifact_paths: + - "build_${BUILDKITE_BUILD_ID}.log" \ No newline at end of file diff --git a/.buildkite/correctness/pipeline.mmlu.yml b/.buildkite/correctness/pipeline.mmlu.yml new file mode 100644 index 0000000..392f296 --- /dev/null +++ b/.buildkite/correctness/pipeline.mmlu.yml @@ -0,0 +1,36 @@ +agents: + queue: "shaoting-gcp" + +steps: + - label: "Run MMLU KV Transfer Test" + command: | + set -euo pipefail + echo "Starting MMLU test" + echo "Running setup.sh" + bash setup.sh + + # buildkite agents start at the root of the Git checkout + cd .buildkite/correctness + + echo "Starting Dense KV Transfer Test with meta-llama/Llama-3.1-8B-Instruct" + # produces Llama-3.1-8B-Instruct-baseline.jsonl + python mmlu-test.py --model meta-llama/Llama-3.1-8B-Instruct --number-of-subjects 25 + # produces Llama-3.1-8B-Instruct-lmcache.jsonl + python mmlu-test.py --model meta-llama/Llama-3.1-8B-Instruct --number-of-subjects 25 --use-lmcache + + echo "Starting MLA KV Transfer Test with deepseek-ai/DeepSeek-V2-Lite" + # produces DeepSeek-V2-Lite-baseline.jsonl + python mmlu-test.py --model deepseek-ai/DeepSeek-V2-Lite --number-of-subjects 25 + # produces DeepSeek-V2-Lite-lmcache.jsonl + python mmlu-test.py --model deepseek-ai/DeepSeek-V2-Lite --number-of-subjects 25 --use-lmcache + + echo "Summarizing Results" + # the summarize-results.py will look for jsonl files with the same techniqiue as the mmlu-test.py script + # which is with args.model.split("/")[-1] + python summarize-results.py meta-llama/Llama-3.1-8B-Instruct deepseek-ai/DeepSeek-V2-Lite + + # this produces correctness-summary.txt + buildkite-agent artifact upload correctness-summary.txt + + artifact_paths: + - correctness-summary.txt diff --git a/.buildkite/correctness/setup.sh b/.buildkite/correctness/setup.sh new file mode 100644 index 0000000..c328e98 --- /dev/null +++ b/.buildkite/correctness/setup.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# buildkite agents start at the root of the Git checkout + +cd .buildkite/correctness + +unxz data.tar.xz +tar xf data.tar + +# see README.md for pre-configuring your CI runner +source ~/correctness_venv/bin/activate +cd ~/correctness_repositories/LMCache +git pull origin dev +cd ~/correctness_repositories/vllm +git pull origin main diff --git a/.buildkite/correctness/sharegpt2openai.py b/.buildkite/correctness/sharegpt2openai.py new file mode 100644 index 0000000..90a498a --- /dev/null +++ b/.buildkite/correctness/sharegpt2openai.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: Apache-2.0 +# Standard +import argparse +import json + +VLLM_TEMPLATE = { + "model": "Qwen3/Qwen3-32B", + "frequency_penalty": 0.0, + "logit_bias": None, + "logprobs": False, + "top_logprobs": 0, + "max_tokens": 2048, + "max_completion_tokens": 2048, + "n": 1, + "presence_penalty": 0.0, + "response_format": None, + "seed": None, + "stop": [], + "stream": True, + "stream_options": None, + "temperature": 0.01, + "top_p": 1.0, + "tools": None, + "tool_choice": "none", + "reasoning_effort": None, + "include_reasoning": True, + "parallel_tool_calls": False, + "user": None, + "best_of": None, + "use_beam_search": False, + "top_k": -1, + "min_p": None, + "repetition_penalty": 1.0, + "length_penalty": 1.0, + "stop_token_ids": [], + "include_stop_str_in_output": False, + "ignore_eos": False, + "min_tokens": 0, + "skip_special_tokens": True, + "spaces_between_special_tokens": True, + "priority": 0, + "truncate_prompt_tokens": None, + "prompt_logprobs": None, + "allowed_token_ids": None, + "bad_words": [], + "order": "routine", + "lora_type": None, + "reasoning": False, + "echo": False, + "add_generation_prompt": True, + "continue_final_message": False, + "add_special_tokens": False, + "documents": None, + "chat_template": None, + "chat_template_kwargs": {"enable_thinking": False}, + "mm_processor_kwargs": None, + "guided_json": None, + "guided_regex": None, + "guided_choice": None, + "guided_grammar": None, + "structural_tag": None, + "guided_decoding_backend": None, + "guided_whitespace_pattern": None, + "kv_transfer_params": None, + "vllm_xargs": None, +} + + +def convert_sharegpt_to_vllm(input_path, output_path): + with open(input_path, "r", encoding="utf-8") as f: + data = json.load(f) + + output = {} + + for item in data: + conv = item["conversations"] + + # # Delete the gpt's response at the end + if len(conv) > 0 and conv[-1]["from"] == "gpt": + conv = conv[:-1] + + messages = [] + for c in conv: + role = "user" if c["from"] == "human" else "assistant" + content = c["value"] + messages.append({"role": role, "content": content}) + + file_key = f"{item['id']}" + + out_entry = VLLM_TEMPLATE.copy() + out_entry["messages"] = messages + + output[file_key] = out_entry + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(output, f, ensure_ascii=False, indent=2) + + +def main(): + parser = argparse.ArgumentParser( + description="Convert ShareGPT JSON dataset to vLLM format." + ) + parser.add_argument( + "--input", + "-i", + type=str, + required=True, + help="Path to the input ShareGPT JSON file.", + ) + parser.add_argument( + "--output", + "-o", + type=str, + required=True, + help="Path to the output vLLM JSON file.", + ) + + args = parser.parse_args() + + convert_sharegpt_to_vllm(args.input, args.output) + print(f"Converted {args.input} -> {args.output}") + + +if __name__ == "__main__": + main() diff --git a/.buildkite/correctness/summarize-results.py b/.buildkite/correctness/summarize-results.py new file mode 100644 index 0000000..46539e5 --- /dev/null +++ b/.buildkite/correctness/summarize-results.py @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: Apache-2.0 +# Standard +from typing import Dict, List, Tuple +import json +import os +import sys + + +def load_jsonl_file(filepath: str) -> Dict[str, Dict[str, float]]: + """Load a JSONL file and return a dictionary of results.""" + results: dict[str, dict[str, float]] = {} + + if not os.path.exists(filepath): + print(f"Warning: File {filepath} not found") + return results + + with open(filepath, "r") as f: + for line in f: + line = line.strip() + if line: + try: + data = json.loads(line) + results.update(data) + except json.JSONDecodeError: + print(f"Error parsing line in {filepath}: {line}") + continue + + return results + + +def compare_results( + baseline_results: Dict, lmcache_results: Dict +) -> Tuple[List[Tuple], float, float]: + """Compare baseline and lmcache results, return comparison data and totals.""" + comparisons = [] + baseline_total_correct = 0 + baseline_total_questions = 0 + lmcache_total_correct = 0 + lmcache_total_questions = 0 + + # Get all subjects (excluding 'total' for now) + all_subjects = set(baseline_results.keys()) | set(lmcache_results.keys()) + all_subjects.discard("total") # Handle total separately + + # Sort subjects alphabetically + for subject in sorted(all_subjects): + baseline_data = baseline_results.get( + subject, {"accuracy": 0.0, "num_questions": 0} + ) + lmcache_data = lmcache_results.get( + subject, {"accuracy": 0.0, "num_questions": 0} + ) + + baseline_acc = baseline_data["accuracy"] + lmcache_acc = lmcache_data["accuracy"] + difference = lmcache_acc - baseline_acc + + # Use the number of questions from whichever dataset has the subject + num_questions = max( + baseline_data["num_questions"], lmcache_data["num_questions"] + ) + + comparisons.append( + (subject, baseline_acc, lmcache_acc, difference, num_questions) + ) + + # Accumulate totals + baseline_total_correct += baseline_acc * num_questions + baseline_total_questions += num_questions + lmcache_total_correct += lmcache_acc * num_questions + lmcache_total_questions += num_questions + + # Calculate overall accuracies + baseline_total_acc = ( + baseline_total_correct / baseline_total_questions + if baseline_total_questions > 0 + else 0 + ) + lmcache_total_acc = ( + lmcache_total_correct / lmcache_total_questions + if lmcache_total_questions > 0 + else 0 + ) + + return comparisons, baseline_total_acc, lmcache_total_acc + + +def format_model_report( + model_name: str, + model_short_name: str, + comparisons: List[Tuple], + baseline_total: float, + lmcache_total: float, +) -> str: + """Format a single model's comparison report.""" + report = [] + report.append("=" * 80) + report.append(f"MODEL: {model_name}") + report.append("=" * 80) + report.append("") + + # Header + report.append( + f"{'Subject':<35} \ + {'Baseline':<12} \ + {'LMCache':<12} \ + {'Difference':<12} \ + {'Questions':<10}" + ) + report.append("-" * 80) + + # Subject comparisons + for subject, baseline_acc, lmcache_acc, difference, num_questions in comparisons: + report.append( + f"{subject:<35} \ + {baseline_acc:<12.4f} \ + {lmcache_acc:<12.4f} \ + {difference:<12.4f} \ + {num_questions:<10}" + ) + + # Totals + total_difference = lmcache_total - baseline_total + report.append("-" * 80) + report.append( + f"{'TOTAL':<35} \ + {baseline_total:<12.4f} \ + {lmcache_total:<12.4f} \ + {total_difference:<12.4f}" + ) + report.append("") + + # Summary + report.append("SUMMARY:") + report.append(f" Baseline Total Accuracy: {baseline_total:.4f}") + report.append(f" LMCache Total Accuracy: {lmcache_total:.4f}") + report.append(f" Net Accuracy Difference: {total_difference:.4f}") + if total_difference >= 0: + report.append(f" Result: LMCache performs BETTER by {total_difference:.4f}") + else: + report.append( + f" Result: LMCache performs WORSE by {abs(total_difference):.4f}" + ) + report.append("") + + return "\n".join(report) + + +def main(): + if len(sys.argv) < 2: + print("Usage: python summarize-results.py ...") + print( + "Example: python summarize-results.py \ + meta-llama/Llama-3.1-8B-Instruct \ + deepseek-ai/DeepSeek-V2-Lite" + ) + sys.exit(1) + + model_names = sys.argv[1:] + all_reports = [] + + print(f"Processing {len(model_names)} models...") + + for model_name in model_names: + print(f"Processing model: {model_name}") + + # Extract short name using the same logic as mmlu-test.py + model_short_name = model_name.split("/")[-1] + + # Define file paths + baseline_file = f"{model_short_name}-baseline.jsonl" + lmcache_file = f"{model_short_name}-lmcache.jsonl" + + print(f" Looking for files: {baseline_file}, {lmcache_file}") + + # Load results + baseline_results = load_jsonl_file(baseline_file) + lmcache_results = load_jsonl_file(lmcache_file) + + if not baseline_results and not lmcache_results: + print(f" Warning: No data found for {model_name}") + continue + + # Compare results + comparisons, baseline_total, lmcache_total = compare_results( + baseline_results, lmcache_results + ) + + # Generate report for this model + model_report = format_model_report( + model_name, + model_short_name, + comparisons, + baseline_total, + lmcache_total, + ) + all_reports.append(model_report) + + print(f" Processed {len(comparisons)} subjects") + + # Write combined report to file + output_file = "correctness-summary.txt" + with open(output_file, "w") as f: + f.write("MMLU Correctness Comparison Report\n") + f.write("=" * 80 + "\n") + f.write(f"Generated for {len(model_names)} models\n\n") + + for report in all_reports: + f.write(report) + f.write("\n") + + print(f"\nReport written to {output_file}") + + +if __name__ == "__main__": + main() diff --git a/.buildkite/k3_harness/ARCHITECTURE.md b/.buildkite/k3_harness/ARCHITECTURE.md new file mode 100644 index 0000000..73807f4 --- /dev/null +++ b/.buildkite/k3_harness/ARCHITECTURE.md @@ -0,0 +1,89 @@ +# K3s CI Harness Architecture + +This document outlines the main motivations, design decisions, and architecture of the K3s-based Buildkite CI Harness for LMCache. + +## Motivations + +1. **Ease of Setup**: Bring up a fully functional CI node on any raw Linux GPU machine with a single script (`setup-cluster.sh`), avoiding manual configuration of dependencies or persistent Buildkite agents. +2. **Machine Agnostic**: Remove reliance on machine-specific scripts (e.g., custom `pick-free-gpu.sh` scripts). The CI runs identically on any machine with an NVIDIA GPU and Docker. +3. **Clean Environments**: Eliminate state bleeding between test runs (no shared pip/uv caches or leftover files). Every job gets a pristine environment. +4. **Automated Resource Management**: Leverage standard Kubernetes primitives for GPU allocation, volume mounting, and cleanup. + +## Core Design Decisions + +### 1. K3s + GPU Operator +We use **K3s** as a lightweight, single-node Kubernetes cluster. The **NVIDIA GPU Operator** automatically configures the container runtime and exposes GPUs to K8s. This abstracts away the underlying hardware and handles GPU discovery automatically. + +### 2. Ephemeral Pod-Based Execution (`agent-stack-k8s`) +Instead of running persistent `buildkite-agent` binaries, we use Buildkite's `agent-stack-k8s`. A controller pod polls the Buildkite queue and dynamically provisions an ephemeral K8s Pod to run each job. When the job finishes, the pod is destroyed. + +### 3. Declarative GPU Allocation & Parallelizability +Each pipeline step explicitly requests the number of GPUs it needs in its Kubernetes pod specification: + +```yaml +resources: + limits: + nvidia.com/gpu: "2" # Requests 2 GPUs +``` + +Because Kubernetes handles atomic resource scheduling, **jobs automatically run in parallel on multi-GPU nodes**. If a node has 4 GPUs and three jobs arrive (requesting 1, 2, and 1 GPU respectively), K8s schedules them concurrently. If another job requests 2 GPUs, it waits in the queue until resources free up. This completely eliminates the need for manual GPU locking mechanisms. + +### 4. Local Base Image & Ephemeral Setup +To avoid Docker registry dependencies, `setup-cluster.sh` automatically detects the host GPU's compute capability, builds the `lmcache/ci-base:latest` image locally, and imports it directly into K3s. Each job pod uses this base image and dynamically installs vLLM and LMCache (`setup-env.sh`) at runtime. + +### 5. Shared Host Volumes +Large, read-heavy directories (like HuggingFace models and datasets) are mounted into pods via `hostPath` to speed up tests without duplicating downloads, while keeping the job environment itself stateless. + +--- + +## Architecture Flow + +### Node Initialization (One-Time Setup) + +```text +[ Raw Linux Host + NVIDIA GPU ] + | + | (run setup-cluster.sh) + v ++-----------------------------------------------------+ +| K3s Cluster | +| | +| 1. Install K3s | +| 2. Install Helm -> Deploy NVIDIA GPU Operator | +| 3. Build CI Base Image (lmcache/ci-base:latest) | +| 4. Import Base Image to K3s containerd | ++-----------------------------------------------------+ + | + | (run install-agent-stack.sh) + v +[ agent-stack-k8s Controller Pod (Watches queue) ] +``` + +### CI Execution Flow + +```text +Buildkite Web UI + | + | 1. Job pushed to 'k8s' queue + v +agent-stack-k8s Controller (in K3s) + | + | 2. Reads job, creates ephemeral Pod requesting GPUs + v ++-------------------------------------------------------------+ +| Job Pod (e.g., limits: nvidia.com/gpu: "1") | +| | +| - Mounts /data/huggingface from host | +| - Runs setup-env.sh (Installs vLLM, LMCache) | +| - Executes test script (e.g., pytest) | ++-------------------------------------------------------------+ + | + | 3. Test finishes (Pass/Fail) + v +agent-stack-k8s Controller + | + | 4. Reports result to Buildkite + | 5. Destroys the Pod (wipes environment) + v +Buildkite Web UI +``` \ No newline at end of file diff --git a/.buildkite/k3_harness/README.md b/.buildkite/k3_harness/README.md new file mode 100644 index 0000000..9cf7fb2 --- /dev/null +++ b/.buildkite/k3_harness/README.md @@ -0,0 +1,143 @@ +# K3s CI Harness + +Single-node K3s + NVIDIA GPU Operator for running LMCache CI on any GPU machine. + +## Prerequisites + +- Linux host with NVIDIA GPU(s) and driver installed +- Docker (for building the CI base image) +- Root access +- A GitHub token (PAT or fine-grained) with read/write access to the repo + +## Setup (one-time) + +```bash +# 1. Install K3s + GPU Operator + build CI base image +.buildkite/k3_harness/setup-cluster.sh + +# 2. Verify GPUs work in pods +.buildkite/k3_harness/smoke-test.sh + +# 3. Connect to Buildkite (needs agent token + GitHub token for HTTPS checkout) +.buildkite/k3_harness/install-agent-stack.sh +``` + +`setup-cluster.sh` does everything: installs K3s, Helm, GPU Operator, builds the CI base image from `ci-base.Dockerfile`, imports it into K3s containerd locally, and creates host volume directories. Safe to re-run. + +## How Buildkite integration works + +This is different from the bare-metal agent setup where you install a Buildkite agent binary on your own machine, register it with a queue created on the Web UI, and manage it as a systemd service. With agent-stack-k8s, there are **no persistent agents on the machine**. + +Instead, agent-stack-k8s runs a controller pod in K8s that polls Buildkite for jobs matching the configured queue. When a job appears, it creates a K8s pod to run it. When the job finishes, the pod is deleted. + +**What you need from the Buildkite web UI**: + +1. **Create a queue** — Go to **Organization Settings → Default cluster → Queues → New Queue** and create a queue named `k8s` (or your chosen name). The queue needs no configuration — just a name. You do **not** register any agents on it. +2. **Get an agent token** — From the cluster settings page, copy the agent token (or create a new one). +3. **Get a GitHub token** — Create a PAT (or fine-grained token) with read/write access to the repo. This is used for HTTPS checkout and for pushing baselines to `benchmarks-main`. +4. **Run `install-agent-stack.sh`** with the agent token and GitHub token. The script creates a K8s secret and installs agent-stack-k8s. + +To use a queue name other than `k8s`, set `BUILDKITE_QUEUE` when running install: +```bash +BUILDKITE_QUEUE=my-queue .buildkite/k3_harness/install-agent-stack.sh +``` + +Pipeline steps target the queue with: +```yaml +agents: + queue: "k8s" # must match the queue name +``` + +## Per-job environment + +Every CI job sources `setup-env.sh` to install vLLM + LMCache: + +```yaml +command: | + source .buildkite/k3_harness/setup-env.sh + bash .buildkite/scripts/my-test.sh +``` + +This installs: +1. **vLLM** from nightly wheel +2. **LMCache** from PR source + +Each pod has its own ephemeral filesystem that is cleanly erased after being shut down — no shared pip/uv cache, no cross-pod contention. + +## Shared volumes + +Mounted into every pod via `hostPath`: + +| Host | Container | What | +|------|-----------|------| +| `/data/huggingface` | `/root/.cache/huggingface` | Model weights (read-heavy, write-once) | +| `/data/datasets` | `/root/correctness` | Test datasets (read-only after download) | + +## GPU allocation + +Request GPUs per pipeline step: + +```yaml +plugins: + - kubernetes: + podSpec: + containers: + - resources: + limits: + nvidia.com/gpu: "2" # 1 or 2 +``` + +K8s device plugin handles atomic allocation. No `pick-free-gpu.sh`. + +## CI base image + +`ci-base.Dockerfile` builds an image with CUDA + Python 3.12 + uv + build deps (no vLLM/LMCache). `setup-cluster.sh` builds it automatically, auto-detects your GPU's compute capability for `TORCH_CUDA_ARCH_LIST`, and imports it into K3s containerd — no registry needed. + +To rebuild after changing `requirements/*.txt`: +```bash +REBUILD_IMAGE=1 .buildkite/k3_harness/setup-cluster.sh +``` + +## Files + +``` +k3_harness/ +├── ci-base.Dockerfile # CI base image definition +├── setup-cluster.sh # One-time: K3s + GPU Operator + base image +├── install-agent-stack.sh # One-time: Buildkite agent (needs token + GitHub token) +├── values.yaml # Reference Helm values (documentation only) +├── setup-env.sh # Per-job: install vLLM + LMCache (includes GPU health check) +├── smoke-test.sh # Verify: GPU pod runs in K3s +├── gpu-monitor.sh # Host-level: detect stale non-K8s GPU processes +├── setup-gpu-monitor.sh # Install gpu-monitor.sh as a cron job +├── teardown.sh # Remove everything (preserves /data/*) +└── README.md +``` + +## GPU monitoring + +CI jobs automatically check GPU health at startup (via `check_gpu_health` in `setup-env.sh`). If a GPU assigned to a pod has less than 80% free memory — usually due to a stale host process — the job fails immediately with a clear message instead of wasting time on setup and then getting a cryptic CUDA OOM. + +For **host-level monitoring**, install the GPU monitor cron job: + +```bash +# Install cron job that checks every 10 minutes for stale non-K8s GPU processes +sudo bash .buildkite/k3_harness/setup-gpu-monitor.sh +``` + +This runs `gpu-monitor.sh` every 10 minutes. It distinguishes K8s pod processes (in `kubepods` cgroups) from stale host processes and logs warnings to `/var/log/gpu-monitor.log` with PID, command, GPU, memory usage, and process age. + +```bash +# View monitor logs +tail -f /var/log/gpu-monitor.log + +# Remove the cron job +crontab -l 2>/dev/null | grep -v gpu-monitor.sh | crontab - +``` + +## Teardown + +```bash +.buildkite/k3_harness/teardown.sh +# Removes K3s, GPU Operator, agent-stack. Preserves /data/* +``` diff --git a/.buildkite/k3_harness/ci-base.Dockerfile b/.buildkite/k3_harness/ci-base.Dockerfile new file mode 100644 index 0000000..47f1ac5 --- /dev/null +++ b/.buildkite/k3_harness/ci-base.Dockerfile @@ -0,0 +1,40 @@ +# CI base image: CUDA + Python + uv + build deps. +# No vLLM or LMCache — those are installed per-job by setup-env.sh. +# +# Built automatically by setup-cluster.sh and imported into K3s containerd. +# Rebuild when requirements/*.txt changes. + +FROM nvidia/cuda:13.0.2-devel-ubuntu24.04 + +ENV DEBIAN_FRONTEND=noninteractive +ENV PATH="/opt/venv/bin:${PATH}" + +RUN echo 'tzdata tzdata/Areas select America' | debconf-set-selections \ + && echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \ + && apt-get update -y \ + && apt-get install -y --no-install-recommends \ + ccache software-properties-common git curl sudo jq lsof \ + python3 python3-dev python3-venv python3-pip tzdata libxcb1-dev \ + libcudart12 \ + && ldconfig \ + && curl -LsSf https://astral.sh/uv/install.sh | sh \ + && mv ~/.local/bin/uv /usr/local/bin/ \ + && mv ~/.local/bin/uvx /usr/local/bin/ \ + && uv venv /opt/venv \ + && . /opt/venv/bin/activate \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace + +# Pre-install requirements that rarely change +COPY requirements/common.txt requirements/build.txt requirements/cuda.txt \ + requirements/cuda13_core.txt requirements/nixl.txt /tmp/reqs/ +RUN . /opt/venv/bin/activate && \ + uv pip install -r /tmp/reqs/cuda.txt && \ + uv pip install -r /tmp/reqs/build.txt && \ + rm -rf /tmp/reqs + +# Set at build time to match the CI machine's GPU. +# Query with: nvidia-smi --query-gpu=compute_cap --format=csv,noheader +ARG TORCH_CUDA_ARCH_LIST +ENV TORCH_CUDA_ARCH_LIST=${TORCH_CUDA_ARCH_LIST} diff --git a/.buildkite/k3_harness/gpu-monitor.sh b/.buildkite/k3_harness/gpu-monitor.sh new file mode 100755 index 0000000..f4cd757 --- /dev/null +++ b/.buildkite/k3_harness/gpu-monitor.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Host-level GPU monitoring script. +# Detects non-K8s processes holding GPU memory and logs warnings. +# +# Install as a cron job on the CI machine: +# sudo crontab -e +# */10 * * * * /path/to/gpu-monitor.sh >> /var/log/gpu-monitor.log 2>&1 +# +# Or install with setup-gpu-monitor.sh for automatic setup. + +set -euo pipefail + +LOG_PREFIX="[gpu-monitor $(date '+%Y-%m-%d %H:%M:%S')]" +K8S_PIDS_FILE=$(mktemp) +trap 'rm -f "$K8S_PIDS_FILE"' EXIT + +if ! command -v nvidia-smi &>/dev/null; then + echo "$LOG_PREFIX ERROR: nvidia-smi not found" + exit 1 +fi + +# Collect all PIDs using GPU compute +mapfile -t GPU_PIDS < <( + nvidia-smi --query-compute-apps=pid --format=csv,noheader,nounits 2>/dev/null \ + | tr -d ' ' | sort -u | grep -v '^$' +) + +if [[ ${#GPU_PIDS[@]} -eq 0 ]]; then + # No GPU processes -- all clean + exit 0 +fi + +# Collect K8s container PIDs (all processes running inside any K8s pod). +# K3s uses containerd; enumerate all container cgroups. +{ + shopt -s nullglob + for cgroup_dir in /sys/fs/cgroup/*/kubepods* /sys/fs/cgroup/kubepods*; do + if [[ -d "$cgroup_dir" ]]; then + find "$cgroup_dir" -name "cgroup.procs" -exec cat {} \; 2>/dev/null + fi + done + shopt -u nullglob +} | sort -u > "$K8S_PIDS_FILE" 2>/dev/null || true + +# Also check via crictl if available (more reliable) +if command -v crictl &>/dev/null; then + for container_id in $(crictl ps -q 2>/dev/null); do + pid=$(crictl inspect --output go-template --template '{{.info.pid}}' "$container_id" 2>/dev/null || true) + if [[ -n "$pid" && "$pid" != "0" ]]; then + # Include the container PID and all its descendants + echo "$pid" >> "$K8S_PIDS_FILE" + pgrep -P "$pid" --ns "$pid" 2>/dev/null >> "$K8S_PIDS_FILE" || true + fi + done +fi + +# Check each GPU process +STALE_FOUND=false +for pid in "${GPU_PIDS[@]}"; do + [[ -z "$pid" ]] && continue + + # Is this PID a K8s process? + if grep -qw "$pid" "$K8S_PIDS_FILE" 2>/dev/null; then + continue + fi + + # Check if this PID is inside any cgroup containing "kubepods" + if [[ -f "/proc/$pid/cgroup" ]] && grep -q "kubepods" "/proc/$pid/cgroup" 2>/dev/null; then + continue + fi + + # This is a non-K8s process using a GPU + STALE_FOUND=true + local_cmdline=$(tr '\0' ' ' < "/proc/$pid/cmdline" 2>/dev/null | head -c 200 || echo "") + + echo "$LOG_PREFIX STALE GPU PROCESS: PID=$pid cmd='$local_cmdline'" + nvidia-smi --query-compute-apps=pid,gpu_bus_id,used_memory --format=csv,noheader 2>/dev/null \ + | grep "^${pid}," | while IFS=, read -r _ bus mem; do + echo "$LOG_PREFIX GPU bus=$bus memory=$mem" + done + + # Check process age + if [[ -f "/proc/$pid/stat" ]]; then + local_start=$(awk '{print $22}' "/proc/$pid/stat" 2>/dev/null || echo "0") + local_uptime=$(awk '{print int($1)}' /proc/uptime) + local_clk_tck=$(getconf CLK_TCK) + if [[ "$local_start" -gt 0 && "$local_clk_tck" -gt 0 ]]; then + local_age_secs=$(( local_uptime - local_start / local_clk_tck )) + local_age_hours=$(( local_age_secs / 3600 )) + echo "$LOG_PREFIX Age: ~${local_age_hours}h (${local_age_secs}s)" + fi + fi +done + +if [[ "$STALE_FOUND" == "true" ]]; then + echo "$LOG_PREFIX" + echo "$LOG_PREFIX === GPU SUMMARY ===" + nvidia-smi --query-gpu=index,memory.used,memory.total,memory.free --format=csv,noheader \ + | while IFS=, read -r idx used total free; do + echo "$LOG_PREFIX GPU $idx: used=$(echo $used | xargs), total=$(echo $total | xargs), free=$(echo $free | xargs)" + done + echo "$LOG_PREFIX WARNING: Stale non-K8s processes are holding GPU memory. CI tests may fail." + echo "$LOG_PREFIX To fix: kill the stale PIDs listed above." + exit 1 +fi diff --git a/.buildkite/k3_harness/install-agent-stack.sh b/.buildkite/k3_harness/install-agent-stack.sh new file mode 100755 index 0000000..dbdb949 --- /dev/null +++ b/.buildkite/k3_harness/install-agent-stack.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Install Buildkite agent-stack-k8s with GitHub token for HTTPS repo access. +# +# Usage: +# install-agent-stack.sh +# +# Arguments: +# BUILDKITE_AGENT_TOKEN — from Buildkite cluster settings +# GITHUB_TOKEN — GitHub PAT or fine-grained token with repo read/write +# +# The queue name defaults to "k8s". Override with BUILDKITE_QUEUE env var. +set -euo pipefail + +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + +TOKEN="${1:?Usage: $0 }" +GH_TOKEN="${2:?Usage: $0 }" +QUEUE="${BUILDKITE_QUEUE:-k8s}" + +# Create (or update) the K8s secret with GitHub credentials. +# Two keys: +# .git-credentials — used by agent-stack-k8s checkout container (HTTPS clone). +# The leading dot matches the file git-credential-store expects. +# GITHUB_TOKEN — injected into job containers for git push (e.g. baselines) +kubectl create secret generic buildkite-git-creds \ + --from-literal=.git-credentials="https://x-access-token:${GH_TOKEN}@github.com" \ + --from-literal=GITHUB_TOKEN="${GH_TOKEN}" \ + -n buildkite --dry-run=client -o yaml | kubectl apply -f - + +# Install or upgrade agent-stack-k8s +# - default-checkout-params.gitCredentialsSecret: checkout container uses HTTPS +# instead of SSH. Expects a Secret with a `.git-credentials` key. +# - GITHUB_TOKEN is injected per-step in pipeline.yml (not via global pod-spec-patch) +helm upgrade --install agent-stack-k8s oci://ghcr.io/buildkite/helm/agent-stack-k8s \ + --version 0.38.0 \ + --namespace buildkite --create-namespace \ + --set agentToken="${TOKEN}" \ + --set config.queue="${QUEUE}" \ + --set-json 'config.default-checkout-params={"gitCredentialsSecret":{"secretName":"buildkite-git-creds"}}' \ + --wait --timeout 3m + +echo "agent-stack-k8s installed (queue=${QUEUE})" +kubectl get pods -n buildkite diff --git a/.buildkite/k3_harness/resolve-pinned-vllm.sh b/.buildkite/k3_harness/resolve-pinned-vllm.sh new file mode 100644 index 0000000..b48289b --- /dev/null +++ b/.buildkite/k3_harness/resolve-pinned-vllm.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Resolve the vLLM nightly version that this build should install. +# +# Resolution order (first non-empty wins): +# 1. PINNED_VLLM_VERSION env var -- explicit per-build override. +# 2. latest_tested_vllm.txt fetched from +# https://raw.githubusercontent.com/LMCache/LMCache/buildkite_latest_tested_vllm/latest_tested_vllm.txt +# -- the most recent vLLM nightly that the canary build verified. +# 3. Empty string -- caller falls back to "latest nightly". +# +# The pin file's first non-blank, non-comment line is the bare version +# (kept this way so older `head -n1` consumers still work). Subsequent +# key=value lines carry pre-resolved metadata so consumers can install +# the wheel without making any extra API calls: +# short_sha= +# full_sha=<40-char-commit-sha> +# archive_index_url=https://wheels.vllm.ai//cu130 +# +# Toggles: +# USE_PINNED_VLLM=false -- skip step 2 (always probe the latest nightly). +# Used by the canary build itself, since pinning +# to its own previous result would defeat the +# purpose of a freshness check. +# +# Usage: +# source .buildkite/k3_harness/resolve-pinned-vllm.sh +# echo "Resolved: ${PINNED_VLLM_VERSION:-}" +# echo "Archive : ${PINNED_VLLM_ARCHIVE_INDEX_URL:-}" +# +# After sourcing the following are set (possibly empty) and exported: +# PINNED_VLLM_VERSION -- e.g. 0.23.1rc1.dev508+gc6dd32a81 +# PINNED_VLLM_FULL_SHA -- 40-char commit SHA, if recorded in pin +# PINNED_VLLM_ARCHIVE_INDEX_URL +# -- permanent PEP 503 simple index, if recorded +# +# The script never fails the build: a missing/unreachable pin file just +# falls through to the unpinned path, mirroring the previous behaviour. + +# Allow re-sourcing without "unbound variable" complaints under set -u. +PINNED_VLLM_VERSION="${PINNED_VLLM_VERSION:-}" +PINNED_VLLM_ARCHIVE_INDEX_URL="${PINNED_VLLM_ARCHIVE_INDEX_URL:-}" +PINNED_VLLM_FULL_SHA="${PINNED_VLLM_FULL_SHA:-}" +USE_PINNED_VLLM="${USE_PINNED_VLLM:-true}" + +# Override URL if you mirror the pin file elsewhere (e.g. an internal +# raw-file proxy for offline CI). +LMCACHE_VLLM_PIN_URL="${LMCACHE_VLLM_PIN_URL:-https://raw.githubusercontent.com/LMCache/LMCache/buildkite_latest_tested_vllm/latest_tested_vllm.txt}" + +if [[ -z "${PINNED_VLLM_VERSION}" && "${USE_PINNED_VLLM}" == "true" ]]; then + if command -v curl >/dev/null 2>&1; then + # 5s connect, 10s total -- pin lookup must never dominate setup time. + fetched="$(curl -fsSL --connect-timeout 5 --max-time 10 \ + "${LMCACHE_VLLM_PIN_URL}" 2>/dev/null || true)" + # The pin file's first non-blank, non-comment line is the bare + # version (back-compat for older readers); subsequent key=value + # lines carry resolved metadata so consumers can skip the live + # GitHub API lookup. A single awk pass extracts everything. + # Empty / missing keys are tolerated -- we'll just fall back to + # resolving them on demand later in the pipeline. + eval "$(printf '%s\n' "${fetched}" | awk ' + BEGIN { ver=""; idx=""; sha="" } + /^[[:space:]]*(#|$)/ { next } + ver == "" { + line=$0 + sub(/[[:space:]]+$/, "", line) + ver=line + next + } + /^archive_index_url=/ { + v=$0; sub(/^archive_index_url=/, "", v); idx=v; next + } + /^full_sha=/ { + v=$0; sub(/^full_sha=/, "", v); sha=v; next + } + END { + printf("PINNED_VLLM_VERSION=%s\n", ver) + printf("PINNED_VLLM_ARCHIVE_INDEX_URL=%s\n", idx) + printf("PINNED_VLLM_FULL_SHA=%s\n", sha) + } + ')" + fi +fi + +export PINNED_VLLM_VERSION +export PINNED_VLLM_ARCHIVE_INDEX_URL +export PINNED_VLLM_FULL_SHA + +if [[ -n "${PINNED_VLLM_VERSION}" ]]; then + echo "[resolve-pinned-vllm] Pinned vLLM version: ${PINNED_VLLM_VERSION}" >&2 + if [[ -n "${PINNED_VLLM_ARCHIVE_INDEX_URL}" ]]; then + echo "[resolve-pinned-vllm] Archive index:" \ + "${PINNED_VLLM_ARCHIVE_INDEX_URL}" >&2 + fi +else + echo "[resolve-pinned-vllm] No pinned vLLM; will install latest nightly" >&2 +fi diff --git a/.buildkite/k3_harness/setup-blend-env.sh b/.buildkite/k3_harness/setup-blend-env.sh new file mode 100755 index 0000000..df07689 --- /dev/null +++ b/.buildkite/k3_harness/setup-blend-env.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Per-job environment setup for CacheBlend-plugin compatibility tests. + +set -euo pipefail + +trap 'echo "ERROR: setup-blend-env.sh failed at line $LINENO (exit code $?)" >&2' ERR + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# 1. Full env: vLLM nightly + LMCache(PR) from source (+ c_ops for this GPU). +# setup-env.sh already runs the GPU health pre-check. +source "${REPO_ROOT}/.buildkite/k3_harness/setup-env.sh" + +# setup-env.sh installed its own ERR trap; restore ours for accurate attribution. +trap 'echo "ERROR: setup-blend-env.sh failed at line $LINENO (exit code $?)" >&2' ERR + +# 2. Plugin-harness dep not in the LMCache/vLLM dependency closure. +echo "--- :python: Installing cacheblend-plugin harness deps" +uv pip install httpx + +# 3. Pull the cacheblend-plugin. Coordinates come from the Buildkite pipeline +# env; CB_PLUGIN_REPO is required (set it in the pipeline env so the repo +# owner/name is not hardcoded here). REF/DIR have sensible fallbacks. +CB_PLUGIN_REPO="${CB_PLUGIN_REPO:?CB_PLUGIN_REPO not set — set the plugin repo (owner/name) in the Buildkite pipeline env}" +CB_PLUGIN_REF="${CB_PLUGIN_REF:-main}" +CB_PLUGIN_DIR="${CB_PLUGIN_DIR:-/tmp/cb-plugin}" + +echo "--- :arrow_down: Pulling ${CB_PLUGIN_REPO}@${CB_PLUGIN_REF}" +_cb_tok="${GH_TOKEN:-${GITHUB_TOKEN:-}}" +echo " GH_TOKEN present: $([ -n "${_cb_tok}" ] && echo yes || echo NO)" +if [ -z "${_cb_tok}" ]; then + echo "ERROR: GH_TOKEN/GITHUB_TOKEN not set. ${CB_PLUGIN_REPO} is private — set a" >&2 + echo " fine-grained PAT (resource owner = the repo's org, Contents: read)" >&2 + echo " as GH_TOKEN in the Buildkite pipeline env." >&2 + exit 1 +fi +_cb_url="https://x-access-token:${_cb_tok}@github.com/${CB_PLUGIN_REPO}.git" +# Isolate from the Buildkite agent's git config: its checkout credential is a +# GitHub App token scoped to LMCache (injected via the GLOBAL git config — +# insteadOf / extraheader / credential.helper) and 403s on the plugin repo. +# Ignoring global+system config forces git to use ONLY our GH_TOKEN from the URL. +_cb_git=(env GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null + git -c credential.helper=) +rm -rf "${CB_PLUGIN_DIR}" +# --branch handles a branch/tag; a commit SHA needs a full clone + checkout. +if ! "${_cb_git[@]}" clone --depth 1 --branch "${CB_PLUGIN_REF}" "${_cb_url}" "${CB_PLUGIN_DIR}" 2>/dev/null; then + "${_cb_git[@]}" clone "${_cb_url}" "${CB_PLUGIN_DIR}" + git -C "${CB_PLUGIN_DIR}" checkout "${CB_PLUGIN_REF}" +fi +echo " cacheblend-plugin @ $(git -C "${CB_PLUGIN_DIR}" rev-parse --short HEAD) (ref=${CB_PLUGIN_REF})" + +# 4. Install the plugin editable; --no-deps keeps the env's torch/vllm/lmcache, +# --no-build-isolation because the plugin is pure-Python (no compile step). +echo "--- :python: Installing cacheblend-plugin (editable)" +git config --global --add safe.directory "${CB_PLUGIN_DIR}" 2>/dev/null || true +uv pip install -e "${CB_PLUGIN_DIR}" --no-deps --no-build-isolation + +# 5. Import/registration smoke so a broken install fails here, not 180s into a +# server-boot timeout inside the harness. +python -c "import lmcache_cacheblend; print('cacheblend-plugin imported OK')" + +export CB_PLUGIN_DIR CB_PLUGIN_REF +echo "--- :white_check_mark: CacheBlend env ready (plugin @ ${CB_PLUGIN_REF}, dir=${CB_PLUGIN_DIR})" diff --git a/.buildkite/k3_harness/setup-cluster.sh b/.buildkite/k3_harness/setup-cluster.sh new file mode 100755 index 0000000..969e20b --- /dev/null +++ b/.buildkite/k3_harness/setup-cluster.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Idempotent K3s + GPU Operator + CI base image setup. +# Safe to re-run — skips components already installed. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + +CI_BASE_IMAGE="lmcache/ci-base:latest" +DATA_DIR="/data" + +# ── K3s ────────────────────────────────────────────────────── +if command -v k3s &>/dev/null && systemctl is-active --quiet k3s; then + echo "✓ K3s already running" +else + echo "→ Installing K3s..." + curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server \ + --disable=traefik \ + --write-kubeconfig-mode=644" sh - + echo "✓ K3s installed" +fi + +# Persist KUBECONFIG for interactive shells +grep -q 'KUBECONFIG=/etc/rancher/k3s/k3s.yaml' ~/.bashrc 2>/dev/null || \ + echo 'export KUBECONFIG=/etc/rancher/k3s/k3s.yaml' >> ~/.bashrc + +kubectl get nodes +echo "" + +# ── Helm ───────────────────────────────────────────────────── +if ! command -v helm &>/dev/null; then + echo "→ Installing Helm..." + curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash +fi +echo "✓ Helm $(helm version --short)" + +# ── NVIDIA GPU Operator ────────────────────────────────────── +if helm status gpu-operator -n gpu-operator &>/dev/null; then + echo "✓ GPU Operator already installed" +else + echo "→ Installing GPU Operator..." + helm repo add nvidia https://helm.ngc.nvidia.com/nvidia + helm repo update nvidia + helm install gpu-operator nvidia/gpu-operator \ + --namespace gpu-operator --create-namespace \ + --set driver.enabled=false \ + --set toolkit.enabled=true \ + --wait --timeout 5m + echo "✓ GPU Operator installed" +fi + +echo "→ Waiting for device plugin..." +kubectl wait --for=condition=ready pod \ + -l app=nvidia-device-plugin-daemonset \ + -n gpu-operator --timeout=120s + +GPU_COUNT=$(kubectl get node -o jsonpath='{.items[0].status.allocatable.nvidia\.com/gpu}') +echo "✓ GPUs available: ${GPU_COUNT}" + +# ── CI base image ──────────────────────────────────────────── +# Auto-detect GPU compute capability for TORCH_CUDA_ARCH_LIST +COMPUTE_CAP=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader | head -1) +ARCH_LIST="${COMPUTE_CAP}+PTX" + +# Check if image already exists in K3s containerd +if k3s ctr images check | grep -q "${CI_BASE_IMAGE}"; then + echo "✓ CI base image already imported (${CI_BASE_IMAGE})" + echo " To rebuild: run with REBUILD_IMAGE=1" + if [[ "${REBUILD_IMAGE:-}" != "1" ]]; then + SKIP_IMAGE=1 + fi +fi + +if [[ "${SKIP_IMAGE:-}" != "1" ]]; then + echo "→ Building CI base image (TORCH_CUDA_ARCH_LIST=${ARCH_LIST})..." + docker build \ + -f "${SCRIPT_DIR}/ci-base.Dockerfile" \ + --build-arg TORCH_CUDA_ARCH_LIST="${ARCH_LIST}" \ + -t "${CI_BASE_IMAGE}" \ + "${REPO_ROOT}" + + echo "→ Importing into K3s containerd..." + docker save "${CI_BASE_IMAGE}" | k3s ctr images import - + echo "✓ CI base image ready" +fi + +# ── Shared host volumes ────────────────────────────────────── +mkdir -p "${DATA_DIR}/huggingface" "${DATA_DIR}/datasets" +echo "✓ Host volumes: ${DATA_DIR}/{huggingface,datasets}" + +echo "" +echo "=== Cluster ready ===" +echo " K3s: $(k3s --version | head -1)" +echo " GPU Operator: $(helm list -n gpu-operator -o json | python3 -c 'import sys,json;print(json.load(sys.stdin)[0]["app_version"])' 2>/dev/null || echo 'unknown')" +echo " GPUs: ${GPU_COUNT}x $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1)" +echo " Arch list: ${ARCH_LIST}" +echo " Base image: ${CI_BASE_IMAGE}" +echo " Data dir: ${DATA_DIR}" +echo "" +echo "Next: run ./install-agent-stack.sh " diff --git a/.buildkite/k3_harness/setup-env.sh b/.buildkite/k3_harness/setup-env.sh new file mode 100755 index 0000000..a5c5642 --- /dev/null +++ b/.buildkite/k3_harness/setup-env.sh @@ -0,0 +1,298 @@ +#!/usr/bin/env bash +# Per-job environment setup: installs vLLM nightly + LMCache from source. +# Called at the start of every CI job. +set -euo pipefail + +# Print the failing command and line number on any error. +trap 'echo "ERROR: setup-env.sh failed at line $LINENO (exit code $?)" >&2' ERR + +# ── GPU health pre-check ──────────────────────────────────── +# Fail fast if GPUs are occupied by stale host processes. +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh" +check_gpu_health 80 + +# Resolve which vLLM nightly to install. Sets PINNED_VLLM_VERSION (empty +# string means "use latest nightly", any other value means +# "install vllm=="). See script header for the full +# resolution order and override knobs. +source "${REPO_ROOT}/.buildkite/k3_harness/resolve-pinned-vllm.sh" + +echo "--- :broom: Pre-install bytecode/cache eviction" +# The CI base image pre-installs packages from requirements/*.txt at image +# build time. We've observed k3 jobs (integration + correctness) fail with +# ImportError: cannot import name 'GenerationConfig' from 'transformers' +# after `uv pip install -U vllm ...` upgrades transformers, even though the +# installed .py files clearly expose GenerationConfig (verified both 5.5.0 +# and 5.5.4). The same install recipe replayed in a fresh venv outside CI +# always succeeds, so the failure is tied to base-image filesystem state +# (stale __pycache__, partial upgrades via overlayfs, etc.). Evict all +# bytecode caches and uv's download cache up front so later steps operate +# on clean ground; this is cheap (few seconds) and idempotent. +find /opt/venv/lib/python3.12/site-packages -type d -name __pycache__ \ + -exec rm -rf {} + 2>/dev/null || true +uv cache clean 2>/dev/null || true + +echo "--- :python: Installing vLLM nightly (pinned to cu130 index)" +# The base image is nvidia/cuda:13.0.2-devel-ubuntu24.04 (system nvcc 13). +# vLLM's generic nightly index (wheels.vllm.ai/nightly/vllm/) non-deterministically +# resolves to either a cu128 or a cu130 torch wheel depending on which wheel +# vLLM's nightly CI happened to publish that day. When the resolver picks a +# cu128 torch, torch.utils.cpp_extension._check_cuda_version aborts the +# LMCache editable install with: +# RuntimeError: The detected CUDA version (13.0) mismatches the version +# that was used to compile PyTorch (12.8). +# +# Pin to the cu130 sub-index so torch.version.cuda is always "13.0" and +# matches the base image. This also lets us drop the HTML-scraping + apt +# cuda-compiler alignment dance that lived here before. +# (See https://docs.vllm.ai/ install tips → Nightly → CUDA 13.0.) +# +# --reinstall-package for transformers / tokenizers / huggingface-hub / +# safetensors / vllm forces uv to uninstall-and-reinstall those packages +# even when it thinks the existing install is up to date. That is the +# minimum set to put the full `vllm serve` import chain on a freshly +# extracted wheel, which bypasses whatever filesystem-level mismatch in +# the base image was causing the GenerationConfig ImportError. +# +# When PINNED_VLLM_VERSION is non-empty (resolved by resolve-pinned-vllm.sh +# from the `buildkite_latest_tested_vllm` branch), pin to that exact wheel +# so every CI job matches the version most recently verified by the +# canary build. Empty falls back to "latest nightly". +# +# vLLM's nightly index (wheels.vllm.ai/nightly//) only keeps the +# *latest* wheel; older versions get rolled off within a day or two and +# pinning them there fails with "no version of vllm==X". Historical +# wheels are still served at wheels.vllm.ai///, +# which is a PEP 503 simple index. The canary records that archive URL +# directly in the pin file, so the common path needs zero extra API +# calls. As a fallback (e.g. an old-format pin file written before the +# canary started recording metadata, or a manual override via +# PINNED_VLLM_VERSION), we still expand the short SHA via the public +# GitHub commits API; GITHUB_TOKEN is honoured (5000 req/h) but optional. +PINNED_VLLM_INDEX_ARGS=() +if [[ -n "${PINNED_VLLM_VERSION:-}" ]]; then + VLLM_INSTALL_SPEC="vllm[runai,tensorizer,flashinfer]==${PINNED_VLLM_VERSION}" + echo "Installing vLLM pinned: ${VLLM_INSTALL_SPEC}" + archive_url="${PINNED_VLLM_ARCHIVE_INDEX_URL:-}" + if [[ -z "${archive_url}" ]]; then + # Old-format pin file or per-build override: resolve on demand. + short_sha="${PINNED_VLLM_VERSION##*+g}" + if [[ "${short_sha}" != "${PINNED_VLLM_VERSION}" \ + && "${short_sha}" =~ ^[0-9a-f]+$ ]]; then + gh_auth_args=() + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + gh_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}") + fi + full_sha="" + for attempt in 1 2 3; do + full_sha="$(curl -fsSL --connect-timeout 5 --max-time 10 \ + -H "Accept: application/vnd.github+json" \ + "${gh_auth_args[@]+"${gh_auth_args[@]}"}" \ + "https://api.github.com/repos/vllm-project/vllm/commits/${short_sha}" \ + 2>/dev/null \ + | awk -F'"' '/"sha":/ {print $4; exit}')" || true + if [[ "${full_sha}" =~ ^[0-9a-f]{40}$ ]]; then + break + fi + echo "[INFO] GitHub commit lookup attempt ${attempt} for" \ + "${short_sha} returned no SHA; retrying..." >&2 + sleep 2 + done + if [[ "${full_sha}" =~ ^[0-9a-f]{40}$ ]]; then + archive_url="https://wheels.vllm.ai/${full_sha}/cu130" + else + echo "[WARN] could not resolve full SHA for ${short_sha}; pip" \ + "may fail if the wheel has rolled off the nightly index" >&2 + fi + fi + fi + if [[ -n "${archive_url}" ]]; then + echo "Adding commit-archived index: ${archive_url}" + PINNED_VLLM_INDEX_ARGS+=(--extra-index-url "${archive_url}") + fi +else + VLLM_INSTALL_SPEC="vllm[runai,tensorizer,flashinfer]" + echo "Installing latest vLLM nightly (no pin)" +fi +uv pip install -U "${VLLM_INSTALL_SPEC}" --pre \ + --reinstall-package transformers \ + --reinstall-package tokenizers \ + --reinstall-package huggingface-hub \ + --reinstall-package safetensors \ + --reinstall-package vllm \ + "${PINNED_VLLM_INDEX_ARGS[@]+"${PINNED_VLLM_INDEX_ARGS[@]}"}" \ + --extra-index-url https://wheels.vllm.ai/nightly/cu130 \ + --extra-index-url https://download.pytorch.org/whl/cu130 \ + --index-strategy unsafe-best-match + +# Pre-import transformers on the main thread via sitecustomize.py so +# vllm's BG-thread preload can't race ahead of _LazyModule init. +cat > /opt/venv/lib/python3.12/site-packages/sitecustomize.py <<'PY' +try: + import transformers # noqa: F401 +except Exception: + pass +PY + +# Probe the vLLM CLI by invoking `vllm --help` as a subprocess. This is the +# only probe that exercises the full import chain that `vllm serve` runs: +# vllm.entrypoints.cli.main.main() triggers `import vllm.entrypoints.cli. +# benchmark.main` *inside* the function body, which in turn loads +# vllm.config.model -> vllm.transformers_utils.config -> `from transformers +# import GenerationConfig, PretrainedConfig`. A plain `from vllm.entrypoints. +# cli.main import main` only resolves the `main` symbol; it never executes +# the function, so it silently passes even when the CLI is broken. Shared +# between the pre-install auto-heal loop below and the post-install hard +# probe at the end of this script. +probe_vllm_cli() { + vllm --help 2>&1 >/dev/null +} + +# vLLM nightlies periodically add eager imports of packages that aren't in +# their declared deps (e.g. `pandas` from vllm/_aiter_ops.py). Auto-install +# any ModuleNotFoundError modules so the job keeps going. Capped to avoid +# infinite loops; every auto-install is logged so the drift is visible in +# the build output. ImportError with a missing top-level name (e.g. a +# transformers/vLLM API break) bails immediately since reinstalling the +# package wouldn't recover. +dump_transformers_state() { + # Called whenever a CLI probe fails with anything other than a clean + # ModuleNotFoundError. This failure mode has been reproducible only on + # the K3s pods, never on a fresh local venv with identical versions, + # so we need a direct view of the running pod's filesystem + Python + # state to make progress. + echo "=================== DIAGNOSTIC DUMP ===================" >&2 + echo "--- uv pip list (relevant packages) ---" >&2 + uv pip list 2>/dev/null | grep -iE "^(transformers|tokenizers|huggingface|safetensors|vllm|torch) " >&2 || true + local tf_dir=/opt/venv/lib/python3.12/site-packages/transformers + echo "--- transformers directory listing ---" >&2 + ls -la "${tf_dir}/__init__.py" "${tf_dir}/__pycache__/__init__.cpython-312.pyc" 2>&1 >&2 || true + echo "--- dist-info METADATA version ---" >&2 + grep -m1 "^Version:" /opt/venv/lib/python3.12/site-packages/transformers-*.dist-info/METADATA 2>/dev/null >&2 || true + echo "--- transformers/__init__.py: __version__ line ---" >&2 + grep -n "^__version__" "${tf_dir}/__init__.py" 2>/dev/null >&2 || true + echo "--- transformers/__init__.py: 'generation' key in _import_structure ---" >&2 + awk '/"generation":/,/\]/' "${tf_dir}/__init__.py" 2>/dev/null | head -20 >&2 || true + echo "--- Python sees: ---" >&2 + python - <<'PY' >&2 2>&1 || true +import sys, importlib +print(f"sys.executable = {sys.executable}") +print(f"sys.path = {sys.path}") +try: + import transformers + print(f"transformers.__file__ = {transformers.__file__}") + print(f"transformers.__version__ = {transformers.__version__}") + print(f"type(transformers) = {type(transformers).__name__}") + print(f"'GenerationConfig' in dir(transformers) = {'GenerationConfig' in dir(transformers)}") + cs2m = getattr(transformers, "_class_to_module", None) + print(f"has _class_to_module = {cs2m is not None}") + if cs2m is not None: + print(f"GenerationConfig in _class_to_module = {'GenerationConfig' in cs2m}") + print(f"first 5 keys = {list(cs2m.keys())[:5]}") + import_struct = getattr(transformers, "_import_structure", None) + print(f"has _import_structure = {import_struct is not None}") + if import_struct is not None: + print(f"'generation' in _import_structure = {'generation' in import_struct}") + print(f"_import_structure['generation'] = {import_struct.get('generation')}") + try: + from transformers import GenerationConfig + print("DIRECT IMPORT of GenerationConfig WORKED") + except Exception as e: + print(f"DIRECT IMPORT failed: {type(e).__name__}: {e}") +except Exception as e: + import traceback + traceback.print_exc() +PY + echo "=================== END DIAGNOSTIC DUMP ===================" >&2 +} + +MAX_AUTO_INSTALL=5 +for i in $(seq 1 "$MAX_AUTO_INSTALL"); do + if err=$(probe_vllm_cli); then + break + fi + mod=$(printf '%s\n' "$err" | sed -n "s/.*No module named '\([^']*\)'.*/\1/p" | head -1) + if [[ -z "$mod" ]]; then + echo "vLLM import failed with a non-ModuleNotFoundError:" >&2 + echo "$err" >&2 + dump_transformers_state + exit 1 + fi + if [[ "$i" == "$MAX_AUTO_INSTALL" ]]; then + echo "Hit $MAX_AUTO_INSTALL auto-install retries; last missing module: $mod" >&2 + echo "$err" >&2 + exit 1 + fi + echo "Auto-installing missing vLLM runtime dep: $mod" + uv pip install "$mod" +done + +echo "--- :mag: Verifying torch CUDA matches system nvcc" +# Sanity check: fail fast with a clear message if the cu130 pin above +# somehow didn't produce a cu13x torch. Previously this mismatch surfaced +# deep inside ninja as a cryptic `cusparse.h: No such file or directory`; +# catching it here makes the failure mode obvious. +python - <<'PY' +import subprocess, sys, torch +tc = torch.version.cuda or "" +try: + nv = subprocess.check_output(["nvcc", "--version"], text=True) + sys_major = next( + (line.split("release ")[1].split(",")[0].split(".")[0] + for line in nv.splitlines() if "release " in line), + "", + ) +except Exception: + sys_major = "" +torch_major = tc.split(".")[0] if tc else "" +print(f"torch.version.cuda={tc!r}; system nvcc major={sys_major!r}") +if torch_major and sys_major and torch_major != sys_major: + sys.exit( + f"CUDA major mismatch: torch={torch_major} vs nvcc={sys_major}. " + "Check the vLLM nightly cu130 index pin in setup-env.sh." + ) +PY + +echo "--- :python: Installing LMCache from source" +# Skip setuptools_scm git describe; the repo carries non-PEP-440 tags +# (nightly, nightly-cu13) that crash the newer vcs_versioning backend. +export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE="${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE:-0.0.0+ci}" +uv pip freeze | sort > /tmp/env-before-lmcache.txt +uv pip install -e . --no-build-isolation +uv pip freeze | sort > /tmp/env-after-lmcache.txt +if ! diff -q /tmp/env-before-lmcache.txt /tmp/env-after-lmcache.txt >/dev/null; then + echo "--- :warning: Packages changed during LMCache install" + diff /tmp/env-before-lmcache.txt /tmp/env-after-lmcache.txt || true +fi + +echo "--- :broom: Post-install bytecode eviction" +# Belt-and-suspenders companion to the pre-install eviction above: clear +# __pycache__ again after the LMCache editable install, which may have +# triggered fresh imports (setuptools_scm, pyproject build backend) and +# deposited new .pyc files that reference now-downgraded packages. +find /opt/venv/lib/python3.12/site-packages -type d -name __pycache__ \ + -exec rm -rf {} + 2>/dev/null || true + +echo "--- :mag: Post-install CLI chain probe" +# The LMCache editable install can downgrade transitive deps to honor the +# caps in requirements/common.txt. If that leaves the env in a state where +# `vllm --help` cannot complete its import chain (vllm.entrypoints.cli. +# main.main() -> vllm.entrypoints.cli.benchmark.main -> vllm.config -> +# vllm.transformers_utils.config -> `from transformers import ...`), the +# only other signal is a 180s `wait_for_server` timeout inside each test +# harness. Re-probe the full chain here so broken envs fail fast with the +# actual traceback instead of a generic timeout. +if err=$(probe_vllm_cli); then + echo "vLLM CLI import chain OK post-install." +else + echo "FATAL: vLLM CLI import chain broken after LMCache install." >&2 + echo "--- Traceback ---" >&2 + echo "$err" >&2 + echo "--- Installed packages ---" >&2 + uv pip freeze >&2 || true + exit 1 +fi + +echo "--- :white_check_mark: Environment ready" +python -c "import vllm; import lmcache; print(f'vLLM={vllm.__version__}, LMCache installed from source with no build isolation')" diff --git a/.buildkite/k3_harness/setup-gpu-monitor.sh b/.buildkite/k3_harness/setup-gpu-monitor.sh new file mode 100755 index 0000000..7cdae33 --- /dev/null +++ b/.buildkite/k3_harness/setup-gpu-monitor.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Install the GPU monitor as a cron job on the CI host machine. +# Run this once during initial cluster setup. +# +# Usage: sudo bash setup-gpu-monitor.sh +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MONITOR_SCRIPT="${SCRIPT_DIR}/gpu-monitor.sh" +LOG_FILE="/var/log/gpu-monitor.log" +CRON_INTERVAL="*/10" # Every 10 minutes + +if [[ ! -f "$MONITOR_SCRIPT" ]]; then + echo "ERROR: gpu-monitor.sh not found at $MONITOR_SCRIPT" + exit 1 +fi + +chmod +x "$MONITOR_SCRIPT" + +# Create log file with rotation +touch "$LOG_FILE" + +# Set up logrotate +cat > /etc/logrotate.d/gpu-monitor </dev/null | grep -v "gpu-monitor.sh" || true; echo "$CRON_CMD") | crontab - + +echo "GPU monitor installed:" +echo " Script: $MONITOR_SCRIPT" +echo " Log: $LOG_FILE" +echo " Cron: $CRON_CMD" +echo "" +echo "View logs: tail -f $LOG_FILE" +echo "Remove cron: crontab -l | grep -v gpu-monitor.sh | crontab -" diff --git a/.buildkite/k3_harness/setup-lmcache-only-env.sh b/.buildkite/k3_harness/setup-lmcache-only-env.sh new file mode 100755 index 0000000..1758ebf --- /dev/null +++ b/.buildkite/k3_harness/setup-lmcache-only-env.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Per-job environment setup for jobs that DON'T need vLLM (e.g. unit tests). +# Installs LMCache from source on top of the ci-base image, which already +# has torch + requirements/cuda.txt + build.txt baked in. Much faster than +# setup-env.sh since it skips the vLLM nightly install entirely. +set -euo pipefail + +trap 'echo "ERROR: setup-lmcache-only-env.sh failed at line $LINENO (exit code $?)" >&2' ERR + +# ── GPU health pre-check ──────────────────────────────────── +# Fail fast if GPUs are occupied by stale host processes. +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh" +check_gpu_health 80 + +echo "--- :python: Installing LMCache from source (no vLLM)" +# Skip setuptools_scm git describe; the repo carries non-PEP-440 tags +# (nightly, nightly-cu13) that crash the newer vcs_versioning backend. +export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE="${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE:-0.0.0+ci}" +uv pip install -e . --no-build-isolation + +echo "--- :white_check_mark: Environment ready (LMCache only, no vLLM)" +python -c "import lmcache; print('LMCache installed from source')" diff --git a/.buildkite/k3_harness/setup-sglang-env.sh b/.buildkite/k3_harness/setup-sglang-env.sh new file mode 100755 index 0000000..a3cffcd --- /dev/null +++ b/.buildkite/k3_harness/setup-sglang-env.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Per-job env setup for the SGLang + LMCache MP integration tests. +# Drop the fork install once https://github.com/sgl-project/sglang/pull/24089 +# lands. +set -euo pipefail +trap 'echo "ERROR: setup-sglang-env.sh failed at line $LINENO (exit code $?)" >&2' ERR + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh" +check_gpu_health 80 + +echo "--- :wrench: System tools (rustup, protoc, libnuma1)" +# rustup: sglang-grpc needs Rust 1.85+ (apt's rustc is too old). +# protoc: sglang-grpc's prost-build shells out to it. +# libnuma1: sgl_kernel's sm100 .so dynamically links to libnuma.so.1. +if ! command -v rustc >/dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \ + -y --default-toolchain stable --profile minimal --no-modify-path +fi +# shellcheck disable=SC1091 +. "${HOME}/.cargo/env" +rustc --version + +APT_NEEDED=() +command -v protoc >/dev/null 2>&1 || APT_NEEDED+=("protobuf-compiler") +[[ -e /usr/lib/x86_64-linux-gnu/libnuma.so.1 || -e /lib/x86_64-linux-gnu/libnuma.so.1 ]] || APT_NEEDED+=("libnuma1") +if [[ ${#APT_NEEDED[@]} -gt 0 ]]; then + apt-get update + apt-get install -y --no-install-recommends "${APT_NEEDED[@]}" +fi +protoc --version + +echo "--- :package: SGLang + LMCache install" +SGLANG_URL="git+https://github.com/sgl-project/sglang.git@main#subdirectory=python" +uv pip install "${SGLANG_URL}" +export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE="${SETUPTOOLS_SCM_PRETEND_VERSION_FOR_LMCACHE:-0.0.0+ci}" + +uv pip uninstall cupy-cuda12x 2>/dev/null || true +uv pip install -e . --no-build-isolation + +python -c "import lmcache, sglang; print(f'sglang={sglang.__version__}; lmcache OK')" +python -c "import cupy; print(f'cupy={cupy.__version__}')" diff --git a/.buildkite/k3_harness/smoke-test.sh b/.buildkite/k3_harness/smoke-test.sh new file mode 100755 index 0000000..7e7971d --- /dev/null +++ b/.buildkite/k3_harness/smoke-test.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Verify K3s + GPU Operator works: schedule a pod that runs nvidia-smi. +set -euo pipefail + +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + +echo "=== Smoke Test ===" + +# 1. Check cluster +echo "→ Cluster" +kubectl get nodes -o wide +echo "" + +# 2. Check GPU resources +GPU_COUNT=$(kubectl get node -o jsonpath='{.items[0].status.allocatable.nvidia\.com/gpu}') +echo "→ GPUs allocatable: ${GPU_COUNT}" +if [[ "${GPU_COUNT}" -lt 1 ]]; then + echo "✗ No GPUs found on node" + exit 1 +fi + +# 3. Run GPU pod +echo "→ Launching GPU test pod..." +kubectl delete pod gpu-smoke-test --ignore-not-found --wait=false &>/dev/null + +cat <<'EOF' | kubectl apply -f - +apiVersion: v1 +kind: Pod +metadata: + name: gpu-smoke-test +spec: + restartPolicy: Never + containers: + - name: test + image: nvidia/cuda:12.8.0-base-ubuntu24.04 + command: ["nvidia-smi"] + resources: + limits: + nvidia.com/gpu: "1" +EOF + +kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/gpu-smoke-test --timeout=120s +echo "" +echo "→ Pod logs:" +kubectl logs gpu-smoke-test +kubectl delete pod gpu-smoke-test + +echo "" +echo "=== Smoke test passed ===" diff --git a/.buildkite/k3_harness/teardown.sh b/.buildkite/k3_harness/teardown.sh new file mode 100755 index 0000000..165b773 --- /dev/null +++ b/.buildkite/k3_harness/teardown.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Tear down K3s stack. Removes: agent-stack-k8s, GPU Operator, K3s. +# Host data volumes (/data/*) are NOT deleted. +set -euo pipefail + +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + +echo "=== Teardown ===" + +# agent-stack-k8s +if helm status agent-stack-k8s -n buildkite &>/dev/null; then + echo "→ Removing agent-stack-k8s..." + helm uninstall agent-stack-k8s -n buildkite --wait +fi + +# GPU Operator +if helm status gpu-operator -n gpu-operator &>/dev/null; then + echo "→ Removing GPU Operator..." + helm uninstall gpu-operator -n gpu-operator --wait + kubectl delete namespace gpu-operator --ignore-not-found +fi + +# K3s +if command -v k3s-uninstall.sh &>/dev/null; then + echo "→ Removing K3s..." + /usr/local/bin/k3s-uninstall.sh +fi + +echo "✓ Teardown complete" +echo " Note: /data/{huggingface,datasets} preserved" diff --git a/.buildkite/k3_harness/values.yaml b/.buildkite/k3_harness/values.yaml new file mode 100644 index 0000000..04fa55e --- /dev/null +++ b/.buildkite/k3_harness/values.yaml @@ -0,0 +1,25 @@ +# Reference Helm values for agent-stack-k8s. +# install-agent-stack.sh sets these automatically. +# This file is for documentation only. +# Docs: https://github.com/buildkite/agent-stack-k8s + +agentToken: "" + +config: + queue: "k8s" # override with BUILDKITE_QUEUE env var + + # HTTPS checkout using GitHub token (replaces SSH key) + git-credentials-secret: + name: buildkite-git-creds # created by install-agent-stack.sh + key: git-credentials # contains https://x-access-token:@github.com + + # Inject GITHUB_TOKEN into job containers for git push operations + pod-spec-patch: + containers: + - name: container-0 + env: + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-git-creds + key: GITHUB_TOKEN diff --git a/.buildkite/k3_tests/README.md b/.buildkite/k3_tests/README.md new file mode 100644 index 0000000..dd7016b --- /dev/null +++ b/.buildkite/k3_tests/README.md @@ -0,0 +1,154 @@ +# K8s Test Pipelines + +Each subdirectory under `k3_tests/` is a self-contained test with these files: + +| File | Purpose | +|------|---------| +| `run.sh` | Test script (sources `k3_harness/setup-env.sh`, then runs tests) | +| `pipeline.yml` | K8s pod spec — GPU count, volumes, timeouts | +| `buildkite-pipeline.yml` | What to paste into the Buildkite UI "Steps" editor (runs path filter, then uploads `pipeline.yml`) | +| `BK_WEB_SETUP.md` | Full Buildkite UI setup instructions: env vars, trigger filters, recommendations | + +## Buildkite Web UI Setup + +### Prerequisites + +Before creating pipelines, make sure a queue named `k8s` exists in your Buildkite cluster. Go to **Organization Settings → Default cluster → Queues → New Queue** and create it. The queue needs no configuration and no agents — agent-stack-k8s creates ephemeral pod-based agents automatically when jobs arrive. + +### Per-pipeline setup + +For each test directory, create a pipeline in the Buildkite UI. +Each directory has a `BK_WEB_SETUP.md` with the exact settings — env vars, GitHub trigger filters, and recommendations for that test. The short version: + +1. Go to your org → **New Pipeline** +2. In the **Steps** editor, paste the contents of that test's `buildkite-pipeline.yml`: + ```yaml + agents: + queue: "k8s" + + env: + HF_TOKEN: "" + + steps: + - label: ":pipeline: Upload pipeline" + command: buildkite-agent pipeline upload .buildkite/k3_tests//pipeline.yml + ``` + The `agents.queue` must match the queue you created above. This routes the upload step to agent-stack-k8s, which checks out the repo, runs the path filter, and (if the build isn't skipped) uploads the real `pipeline.yml`. Each subsequent step also targets the same queue. +3. `HF_TOKEN` is needed for gated model access (e.g., Llama, Qwen). Set it in the `env` block as shown above, or under **Pipeline Settings → Environment Variables** in the UI — both work +4. Under **GitHub Settings**, configure trigger filters per the test's `BK_WEB_SETUP.md` +5. Save — jobs will run on the K8s queue automatically + +### Path-based skip (auto-pass on docs-only changes) + +The upload step in `buildkite-pipeline.yml` runs `common_scripts/upload-pipeline.sh` +instead of `buildkite-agent pipeline upload` directly. The wrapper +(`common_scripts/path-filter.sh`) inspects the changed files in the build and: + +- **Skips** the build (exits 0 without uploading `pipeline.yml` → build is + green with just the upload step) when *all* changed files match a "trivial" + pattern: `*.md`, `LICENSE*`, `NOTICE*`, `.gitignore`, `.gitattributes`, + `.editorconfig`, `.mailmap`, `CODEOWNERS`, or anything under `docs/` or + `.github/`. (`.github/` is trivial here because k3 tests run on Buildkite, + not GitHub Actions, so workflow / CODEOWNERS / template changes do not + affect them.) +- **Force-runs** the build when any changed file lives under `.buildkite/` — + those PRs are usually fixing the k3 CI itself, so we want them tested on + the PR rather than after merge. +- **Runs** the build whenever there is at least one non-trivial file by + uploading `pipeline.yml`, which contains the real test steps. + +Detection: +- PR builds diff against `origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}` + (default `main`) using the merge-base. +- Push builds diff `HEAD~1..HEAD`. +- Scheduled builds (`BUILDKITE_SOURCE=schedule`) are never skipped. +- If the script can't determine the changed files (shallow clone with no + parent, missing base branch, etc.) it falls back to "do not skip". + +To bypass the skip and force a full run, add the **`force-ci`** label to the +PR on GitHub. Buildkite picks up PR labels automatically; when the filter +sees `force-ci` it runs the full pipeline regardless of which files changed. + +### Trigger strategy + +Not all tests should run on every push. The general pattern: + +| Test weight | When to trigger | Example filter condition | +|-------------|----------------|------------------------| +| Lightweight (1 GPU, <20 min) | Every push / every PR | *(no filter)* | +| Heavy (multi-GPU, >30 min) | PR label or main branch only | `build.pull_request.labels includes "full" \|\| build.branch == 'dev'` | + +Set **"Rebuild on PR label change"** to `Yes` for label-triggered pipelines so adding a label to an existing PR kicks off the build. + +## Adding a New Test + +1. Create a directory: `.buildkite/k3_tests//` + +2. Write a `run.sh` that sources the shared environment setup: + ```bash + #!/usr/bin/env bash + set -euo pipefail + cd "$(cd "$(dirname "$0")/../../.." && pwd)" + source .buildkite/k3_harness/setup-env.sh + # ... your test commands ... + ``` + +3. Write a `pipeline.yml`. Set the GPU limit to what your test needs: + ```yaml + steps: + - label: ":test_tube: My Test" + command: .buildkite/k3_tests//run.sh + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never # local image, imported into K3s containerd + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - { name: hf-cache, mountPath: /root/.cache/huggingface } + volumes: + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + ``` + +4. Write a `buildkite-pipeline.yml` (the snippet pasted into the Buildkite UI's Steps + editor). Use `common_scripts/upload-pipeline.sh` so the test gets path-based skip: + ```yaml + agents: + queue: "k8s" + + steps: + - label: ":pipeline: Upload pipeline" + command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests//pipeline.yml + ``` + +5. Write a `BK_WEB_SETUP.md` documenting the Buildkite UI settings for this test (env vars, trigger filters, recommendations). Use an existing test's `BK_WEB_SETUP.md` as a template. + +6. `chmod +x` your `run.sh` and create the pipeline in the Buildkite UI. + +### Optional: datasets volume + +If your test needs pre-downloaded data (e.g., ShareGPT), add the datasets volume: +```yaml +volumeMounts: + - { name: datasets, mountPath: /root/correctness } +volumes: + - { name: datasets, hostPath: { path: /data/datasets, type: DirectoryOrCreate } } +``` + +### Optional: Docker-in-Docker + +If your test runs Docker containers inside the pod: +```yaml +securityContext: + privileged: true +volumeMounts: + - { name: docker-sock, mountPath: /var/run/docker.sock } +volumes: + - { name: docker-sock, hostPath: { path: /var/run/docker.sock } } +``` diff --git a/.buildkite/k3_tests/blend/buildkite-pipeline.yml b/.buildkite/k3_tests/blend/buildkite-pipeline.yml new file mode 100644 index 0000000..327d74e --- /dev/null +++ b/.buildkite/k3_tests/blend/buildkite-pipeline.yml @@ -0,0 +1,20 @@ +# Paste this into the Buildkite pipeline's "Steps" editor. +# It uploads the real pipeline definition from the repo. +# +# Fill in HF_TOKEN (gated model access), GH_TOKEN (read access to the private +# plugin repo), and CB_PLUGIN_REPO (owner/name of the plugin repo). CB_PLUGIN_REF +# selects which plugin commit to test the PR against; CB_PLUGIN_DIR is the clone +# destination in the pod. +agents: + queue: "k8s" + +env: + HF_TOKEN: "" # HuggingFace token — gated model access (e.g. Llama) + GH_TOKEN: "" # GitHub token — read access to the private plugin repo + CB_PLUGIN_REPO: "" # owner/name of the plugin repo (required; set here or in the UI) + CB_PLUGIN_REF: "main" # plugin git ref to test the PR against + CB_PLUGIN_DIR: "/tmp/cb-plugin" # clone destination in the pod + +steps: + - label: ":pipeline: Upload pipeline" + command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/blend/pipeline.yml diff --git a/.buildkite/k3_tests/blend/pipeline.yml b/.buildkite/k3_tests/blend/pipeline.yml new file mode 100644 index 0000000..9665dd0 --- /dev/null +++ b/.buildkite/k3_tests/blend/pipeline.yml @@ -0,0 +1,34 @@ +# CacheBlend <-> LMCache compatibility test. +# +# Builds the PR's LMCache from source (compiling c_ops for the pod's GPU arch), +# pulls the external cacheblend-plugin from GitHub, installs it editable, and +# runs the plugin's vllm_compat_check (--skip-correctness): one blend server + +# vLLM stack, five V3 contract checks ending in a non-prefix blend that must +# engage. Catches LMCache changes that break the external plugin *before* merge +# — the recurring rebase/version-skew class (RPC key renames, handler-signature +# drift, KV-layout changes, etc.). +# +# Needs one GPU. Set HF_TOKEN (gated model) and GH_TOKEN (private plugin repo) +# in the Buildkite UI — see BK_WEB_SETUP.md. + +steps: + - label: ":jigsaw: CacheBlend plugin compat (Llama-8B)" + command: .buildkite/k3_tests/blend/run.sh + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never # local image, imported into K3s containerd + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - { name: hf-cache, mountPath: /root/.cache/huggingface } + volumes: + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + artifact_paths: + - "cb_compat_logs/**/*" diff --git a/.buildkite/k3_tests/blend/run.sh b/.buildkite/k3_tests/blend/run.sh new file mode 100755 index 0000000..44626eb --- /dev/null +++ b/.buildkite/k3_tests/blend/run.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +cd "${REPO_ROOT}" + +# ── Environment setup ──────────────────────────────────────── +source .buildkite/k3_harness/setup-blend-env.sh + +# ── Ensure all scripts are executable ──────────────────────── +chmod +x "${SCRIPT_DIR}"/scripts/*.sh + +# ── Run the actual test logic ──────────────────────────────── +exec bash "${SCRIPT_DIR}/scripts/run-compat.sh" "$@" diff --git a/.buildkite/k3_tests/blend/scripts/run-compat.sh b/.buildkite/k3_tests/blend/scripts/run-compat.sh new file mode 100755 index 0000000..2ec8799 --- /dev/null +++ b/.buildkite/k3_tests/blend/scripts/run-compat.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# CacheBlend <-> LMCache compatibility check. +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" + +cd "${REPO_ROOT}" +source .buildkite/k3_tests/common_scripts/helpers.sh + +# Fail fast if the GPUs are occupied by stale host processes. +check_gpu_health 80 + +: "${CB_PLUGIN_DIR:?CB_PLUGIN_DIR not set — source k3_harness/setup-blend-env.sh first}" + +MODEL="${CB_MODEL:-meta-llama/Llama-3.1-8B-Instruct}" +MAX_LEN="${CB_MAX_MODEL_LEN:-16384}" +LOG_DIR="${REPO_ROOT}/cb_compat_logs" # workspace-relative so artifact_paths can collect it +mkdir -p "${LOG_DIR}" + +# Resolve the venv that owns vllm/lmcache. In the CI pod this is the image's +# /opt/venv (on PATH via setup-env.sh); CB_VENV overrides for local runs. +if [ -n "${CB_VENV:-}" ]; then + VENV="${CB_VENV}" +elif [ -n "${VIRTUAL_ENV:-}" ]; then + VENV="${VIRTUAL_ENV}" +elif command -v vllm >/dev/null 2>&1; then + VENV="$(cd "$(dirname "$(command -v vllm)")/.." && pwd)" +else + VENV="/opt/venv" +fi + +echo "+++ :jigsaw: vllm_compat_check model=${MODEL} venv=${VENV} plugin=${CB_PLUGIN_DIR}" +python "${CB_PLUGIN_DIR}/.buildkite/harness/vllm_compat_check.py" \ + --model "${MODEL}" \ + --gpu-sync 0 \ + --skip-correctness \ + --venv "${VENV}" \ + --max-model-len "${MAX_LEN}" \ + --log-dir "${LOG_DIR}" diff --git a/.buildkite/k3_tests/common_scripts/helpers.sh b/.buildkite/k3_tests/common_scripts/helpers.sh new file mode 100755 index 0000000..770c424 --- /dev/null +++ b/.buildkite/k3_tests/common_scripts/helpers.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Shared helper functions for K3s test scripts. +# Source this file from every scripts/*.sh. + +# Track background PIDs for cleanup +TRACKED_PIDS=() + +# Check that all visible GPUs have sufficient free memory. +# Fails fast with a clear message if a host-level process is hogging GPU memory. +# Usage: check_gpu_health [min_free_percent] +# min_free_percent: minimum percentage of GPU memory that must be free (default: 80) +check_gpu_health() { + local min_free_pct="${1:-80}" + echo "--- :mag: GPU health check (require ${min_free_pct}% free memory)" + + if ! command -v nvidia-smi &>/dev/null; then + echo " nvidia-smi not found, skipping GPU health check" + return 0 + fi + + # Parse GPU info into a temp file to avoid subshell variable scoping issues + local gpu_info + gpu_info=$(nvidia-smi --query-gpu=index,memory.total,memory.used,memory.free --format=csv,noheader,nounits 2>/dev/null | sed 's/ //g') + + if [[ -z "$gpu_info" ]]; then + echo " No GPUs detected, skipping" + return 0 + fi + + local has_problem=false + while IFS=, read -r idx total used free; do + + if [[ "$total" -eq 0 ]]; then + continue + fi + + local free_pct=$((free * 100 / total)) + + if [[ "$free_pct" -lt "$min_free_pct" ]]; then + echo " WARNING: GPU $idx has only ${free_pct}% free (${free} MiB free / ${total} MiB total, ${used} MiB used)" + # Show which processes are using this GPU + nvidia-smi --query-compute-apps=pid,gpu_uuid,used_memory --format=csv,noheader -i "$idx" 2>/dev/null | while IFS= read -r proc; do + echo " Process: $proc" + done + has_problem=true + else + echo " GPU $idx: OK (${free_pct}% free, ${free} MiB / ${total} MiB)" + fi + done <<< "$gpu_info" + + if [[ "$has_problem" == "true" ]]; then + echo "" + echo "FATAL: One or more GPUs have insufficient free memory." + echo "This usually means a stale process on the host is consuming GPU memory." + echo "Check the host with: nvidia-smi" + echo "To fix: kill the offending host processes, then re-run the CI job." + return 1 + fi + + echo " All GPUs healthy." + return 0 +} + +# Find an available TCP port starting from a given port number. +# Usage: find_free_port [start_port] +find_free_port() { + local port="${1:-8000}" + while [ "$port" -lt 65536 ]; do + if ! lsof -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1 && + ! timeout 1 bash -c "/dev/null; then + echo "$port" + return 0 + fi + ((port++)) + done + echo "ERROR: No available port found starting from ${1:-8000}" >&2 + return 1 +} + +# Wait for a vLLM server to become ready by polling /v1/models. +# Usage: wait_for_server [timeout_secs] [log_file] +# If log_file is provided, its tail is dumped to stderr on timeout so the +# real failure (e.g. an ImportError during startup) is visible inline in the +# job output instead of requiring a trip through build artifacts. +wait_for_server() { + local port="$1" + local timeout="${2:-180}" + local log_file="${3:-}" + echo "Waiting for vLLM on port $port (timeout=${timeout}s)..." + for ((i = 0; i < timeout; i++)); do + if curl -sf "http://localhost:${port}/v1/models" >/dev/null 2>&1; then + echo "vLLM ready on port $port (${i}s)" + return 0 + fi + sleep 1 + done + echo "vLLM failed to start on port $port within ${timeout}s" >&2 + if [[ -n "$log_file" && -f "$log_file" ]]; then + echo "--- :page_facing_up: Last 200 lines of ${log_file}" >&2 + tail -n 200 "$log_file" >&2 + fi + return 1 +} + +# Kill all tracked background PIDs and wait for them. +# Call this in a trap handler. +cleanup_pids() { + echo "--- Cleaning up background processes..." + for pid in "${TRACKED_PIDS[@]}"; do + if kill -0 "$pid" 2>/dev/null; then + echo " Killing PID $pid" + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fi + done + TRACKED_PIDS=() + # Give GPU memory a moment to release + sleep 2 +} diff --git a/.buildkite/k3_tests/common_scripts/path-filter.sh b/.buildkite/k3_tests/common_scripts/path-filter.sh new file mode 100755 index 0000000..7b99f2b --- /dev/null +++ b/.buildkite/k3_tests/common_scripts/path-filter.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Path filter: decide whether a CI build can be skipped based on which files +# changed since the base commit. +# +# Usage: +# source path-filter.sh +# if should_skip_ci; then +# # all changed files are trivial (docs, etc.) +# fi +# +# Rules: +# - If ANY changed file lives under .buildkite/, the build runs. +# (Those PRs are usually fixing the k3 CI itself, so we want to test on +# the PR instead of waiting for it to land on main.) +# - Otherwise, if EVERY changed file matches a "trivial" pattern (markdown, +# LICENSE, anything under docs/ or .github/, etc.), the build can be +# skipped. .github/ is trivial for k3 because k3 tests run on Buildkite, +# not GitHub Actions, so workflow/CODEOWNERS/template changes do not +# affect what the k3 tests do. +# - Anything else → build runs. +# +# Opt-out: add a "force-ci" label to the PR on GitHub. Buildkite exposes +# PR labels via BUILDKITE_PULL_REQUEST_LABELS; if "force-ci" is present +# the filter is bypassed and the full pipeline runs. +# +# Detection of "changed files": +# - PR builds → diff against the merge-base with BUILDKITE_PULL_REQUEST_BASE_BRANCH. +# - Push builds → diff HEAD~1..HEAD. +# - Anything we can't figure out → fall back to "do not skip". + +set -uo pipefail + +# ── Pattern lists ───────────────────────────────────────────── +# bash `case` patterns: `*` matches any string including `/`, so `docs/*` +# matches `docs/foo/bar.png` as well as `docs/foo`. + +_path_filter_is_always_trigger() { + case "$1" in + .buildkite/*) return 0 ;; + esac + return 1 +} + +_path_filter_is_trivial() { + case "$1" in + *.md) return 0 ;; + LICENSE|LICENSE.*) return 0 ;; + NOTICE|NOTICE.*) return 0 ;; + .gitignore|.gitattributes|.editorconfig|.mailmap) return 0 ;; + CODEOWNERS) return 0 ;; + docs/*) return 0 ;; + .github/*) return 0 ;; + esac + return 1 +} + +# ── Changed-files detection ─────────────────────────────────── + +_path_filter_get_changed_files() { + local base_branch base merge_base + + # Ephemeral pods may not have GitHub's SSH host key yet. + # Accept new keys automatically so git-fetch doesn't hang on a prompt. + export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new -o LogLevel=ERROR" + + if [[ -n "${BUILDKITE_PULL_REQUEST:-}" && "${BUILDKITE_PULL_REQUEST:-}" != "false" ]]; then + base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" + # Buildkite checks out shallow; fetch enough history to find the merge-base. + git fetch --no-tags --depth=200 origin "$base_branch" 2>/dev/null || \ + git fetch --no-tags origin "$base_branch" 2>/dev/null || true + + if base=$(git rev-parse --verify "origin/${base_branch}" 2>/dev/null); then + if merge_base=$(git merge-base HEAD "$base" 2>/dev/null); then + git diff --name-only "$merge_base" HEAD + return 0 + fi + # No merge-base (history not deep enough): diff directly. + git diff --name-only "$base" HEAD + return 0 + fi + echo "path-filter: could not resolve origin/${base_branch}" >&2 + return 1 + fi + + # Push build (or unknown context): diff against the previous commit. + if git rev-parse --verify HEAD~1 >/dev/null 2>&1; then + git diff --name-only HEAD~1 HEAD + return 0 + fi + + echo "path-filter: no parent commit available" >&2 + return 1 +} + +# ── Public entry point ──────────────────────────────────────── + +# Returns 0 if the build can be safely skipped, non-zero otherwise. +# Prints a classification of every changed file to stderr for the build log. +should_skip_ci() { + # PR label opt-out: adding "force-ci" on GitHub forces a full run. + if [[ ",${BUILDKITE_PULL_REQUEST_LABELS:-}," == *",force-ci,"* ]]; then + echo "path-filter: PR has 'force-ci' label → not skipping" >&2 + return 1 + fi + + # Never skip scheduled builds (e.g. nightly baselines with NEED_UPLOAD=true). + if [[ "${BUILDKITE_SOURCE:-}" == "schedule" ]]; then + echo "path-filter: scheduled build (BUILDKITE_SOURCE=schedule) → not skipping" >&2 + return 1 + fi + + local changed_files + if ! changed_files=$(_path_filter_get_changed_files); then + echo "path-filter: could not determine changed files → not skipping" >&2 + return 1 + fi + + if [[ -z "$changed_files" ]]; then + echo "path-filter: no changed files reported → not skipping (safer default)" >&2 + return 1 + fi + + local has_always_trigger=0 + local has_non_trivial=0 + local trivial_count=0 + local total=0 + + echo "path-filter: classifying changed files:" >&2 + while IFS= read -r f; do + [[ -z "$f" ]] && continue + total=$((total + 1)) + if _path_filter_is_always_trigger "$f"; then + has_always_trigger=1 + echo " [force-trigger] $f" >&2 + elif _path_filter_is_trivial "$f"; then + trivial_count=$((trivial_count + 1)) + echo " [trivial] $f" >&2 + else + has_non_trivial=1 + echo " [non-trivial] $f" >&2 + fi + done <<< "$changed_files" + + echo "path-filter: ${total} files changed (${trivial_count} trivial)" >&2 + + if [[ "$has_always_trigger" -eq 1 ]]; then + echo "path-filter: at least one file under .buildkite/ → not skipping" >&2 + return 1 + fi + + if [[ "$has_non_trivial" -eq 1 ]]; then + echo "path-filter: non-trivial files changed → not skipping" >&2 + return 1 + fi + + echo "path-filter: all changed files are trivial → SKIP" >&2 + return 0 +} diff --git a/.buildkite/k3_tests/common_scripts/upload-pipeline.sh b/.buildkite/k3_tests/common_scripts/upload-pipeline.sh new file mode 100755 index 0000000..361808e --- /dev/null +++ b/.buildkite/k3_tests/common_scripts/upload-pipeline.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Wraps `buildkite-agent pipeline upload` with a path-based skip check. +# +# Usage (called from each test's buildkite-pipeline.yml upload step): +# command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh \ +# .buildkite/k3_tests//pipeline.yml +# +# If every changed file in this build is trivial (markdown, LICENSE, .github, +# etc.) and none touch .buildkite/, this script: +# - Annotates the build with a "skipped" note +# - Exits 0 without uploading any further steps → the build is green +# Otherwise it execs `buildkite-agent pipeline upload `, adding +# the real test steps to the build. +# +# Add a "force-ci" label to the PR on GitHub to bypass the check. + +set -euo pipefail + +PIPELINE_FILE="${1:?Usage: upload-pipeline.sh }" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +# shellcheck source=path-filter.sh +source "${SCRIPT_DIR}/path-filter.sh" + +if should_skip_ci; then + echo "+++ :fast_forward: Skipping CI — only trivial files changed" + if command -v buildkite-agent >/dev/null 2>&1; then + buildkite-agent annotate \ + --style success \ + --context "path-filter-skip" \ + "Skipped: only trivial files (docs, license, etc.) changed. Add a \`force-ci\` label to the PR to force a full run." \ + || true + fi + exit 0 +fi + +echo "--- :pipeline: Uploading ${PIPELINE_FILE}" +exec buildkite-agent pipeline upload "${PIPELINE_FILE}" diff --git a/.buildkite/k3_tests/comprehensive/BK_WEB_SETUP.md b/.buildkite/k3_tests/comprehensive/BK_WEB_SETUP.md new file mode 100644 index 0000000..a37b8dd --- /dev/null +++ b/.buildkite/k3_tests/comprehensive/BK_WEB_SETUP.md @@ -0,0 +1,27 @@ +# Buildkite Web UI Setup: Comprehensive Tests + +**Steps editor**: paste contents of `buildkite-pipeline.yml` (fill in `HF_TOKEN`). + +**GitHub trigger settings**: +- Filter: `build.pull_request.labels includes "full" || build.branch == 'dev'` +- Rebuild on PR label change: Yes +- Skip queued / cancel running branch builds: Yes + +Heavy test (10 parallel GPU steps) — run on `"full"` label or dev push, not every PR. + +> Builds whose only changes are docs/`*.md`/`LICENSE`/`.github/**` auto-pass +> via the [path filter](../README.md#path-based-skip-auto-pass-on-docs-only-changes). +> Changes under `.buildkite/` always run. Add `force-ci` label to the PR to +> bypass. + +## Nightly Scheduled Build (rolling baselines) + +Create a **Scheduled Build** on this same pipeline to upload performance baselines: + +- **Schedule**: daily (e.g. `0 2 * * *` — 2am UTC) +- **Branch**: `dev` +- **Extra Environment Variables**: `NEED_UPLOAD=true` + +Each config step writes a date-stamped baseline (`-YYYYMMDD.json`) as a Buildkite artifact. The finalize step (`scripts/upload-baselines.sh`) collects them all, prunes files older than 5 days, and pushes a single commit to the `benchmarks-main` branch. + +PR builds automatically compare against the rolling 5-day worst-case (max latency) baseline. diff --git a/.buildkite/k3_tests/comprehensive/buildkite-pipeline.yml b/.buildkite/k3_tests/comprehensive/buildkite-pipeline.yml new file mode 100644 index 0000000..559fa29 --- /dev/null +++ b/.buildkite/k3_tests/comprehensive/buildkite-pipeline.yml @@ -0,0 +1,11 @@ +# Paste this into the Buildkite pipeline's "Steps" editor. +# It uploads the real pipeline definition from the repo. +agents: + queue: "k8s" + +env: + HF_TOKEN: "" # set your HuggingFace token here (needed for gated model access) + +steps: + - label: ":pipeline: Upload pipeline" + command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/comprehensive/pipeline.yml diff --git a/.buildkite/k3_tests/comprehensive/pipeline.yml b/.buildkite/k3_tests/comprehensive/pipeline.yml new file mode 100644 index 0000000..8d80ca5 --- /dev/null +++ b/.buildkite/k3_tests/comprehensive/pipeline.yml @@ -0,0 +1,143 @@ +# Comprehensive integration tests — one step per config, run in parallel. +# Each step gets its own K8s pod with the exact GPU count it needs. +# 2-GPU group is listed first so these heavier jobs get scheduled before 1-GPU jobs. + +x-perf-retry: &perf-retry + automatic: + - exit_status: -1 + limit: 2 + - exit_status: 1 + limit: 2 + +steps: + - group: ":test_tube: Comprehensive (2-GPU)" + steps: + - label: ":test_tube: pd" + command: .buildkite/k3_tests/comprehensive/run.sh pd.yaml + timeout_in_minutes: 30 + retry: *perf-retry + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: &pod-2gpu + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: { limits: { "nvidia.com/gpu": "2" } } + volumeMounts: &vol-mounts + - { name: hf-cache, mountPath: /root/.cache/huggingface } + - { name: datasets, mountPath: /root/correctness } + volumes: &vols + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + - { name: datasets, hostPath: { path: /data/datasets, type: DirectoryOrCreate } } + artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"] + + - label: ":test_tube: p2p" + command: .buildkite/k3_tests/comprehensive/run.sh p2p.yaml + timeout_in_minutes: 30 + retry: *perf-retry + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-2gpu } }] + artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"] + + - label: ":test_tube: p2p_with_v3" + command: .buildkite/k3_tests/comprehensive/run.sh p2p_with_v3.yaml + timeout_in_minutes: 30 + retry: *perf-retry + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-2gpu } }] + artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"] + + - group: ":test_tube: Comprehensive (1-GPU)" + steps: + - label: ":test_tube: local_cpu" + command: .buildkite/k3_tests/comprehensive/run.sh local_cpu.yaml + timeout_in_minutes: 30 + retry: *perf-retry + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: &pod-1gpu + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: { limits: { "nvidia.com/gpu": "1" } } + volumeMounts: *vol-mounts + volumes: *vols + artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"] + + - label: ":test_tube: local_disk" + command: .buildkite/k3_tests/comprehensive/run.sh local_disk.yaml + timeout_in_minutes: 30 + retry: *perf-retry + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"] + + - label: ":test_tube: async" + command: .buildkite/k3_tests/comprehensive/run.sh async.yaml + timeout_in_minutes: 30 + retry: *perf-retry + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"] + + - label: ":test_tube: local_cpu_with_v3" + command: .buildkite/k3_tests/comprehensive/run.sh local_cpu_with_v3.yaml + timeout_in_minutes: 30 + retry: *perf-retry + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"] + + - label: ":test_tube: local_disk_with_v3" + command: .buildkite/k3_tests/comprehensive/run.sh local_disk_with_v3.yaml + timeout_in_minutes: 30 + retry: *perf-retry + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"] + + - label: ":test_tube: multi_device" + command: .buildkite/k3_tests/comprehensive/run.sh multi_device.yaml + timeout_in_minutes: 30 + retry: *perf-retry + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"] + + - label: ":test_tube: layerwise" + command: .buildkite/k3_tests/comprehensive/run.sh layerwise.yaml + timeout_in_minutes: 30 + retry: *perf-retry + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log", "benchmarks/long_doc_qa/*.json"] + + # Nightly baseline upload: only runs when NEED_UPLOAD=true (scheduled build). + # Collects date-stamped baseline artifacts from all config steps, + # prunes files older than 5 days, and pushes a single commit to benchmarks-main. + - wait: ~ + if: build.env("NEED_UPLOAD") == "true" + + - label: ":arrow_up: Upload rolling baselines" + if: build.env("NEED_UPLOAD") == "true" + command: .buildkite/k3_tests/comprehensive/scripts/upload-baselines.sh + timeout_in_minutes: 5 + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: { limits: { cpu: "1", memory: "2Gi" } } + env: + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-git-creds + key: GITHUB_TOKEN diff --git a/.buildkite/k3_tests/comprehensive/run.sh b/.buildkite/k3_tests/comprehensive/run.sh new file mode 100755 index 0000000..1b1186d --- /dev/null +++ b/.buildkite/k3_tests/comprehensive/run.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Comprehensive integration test entrypoint for K8s pods. +# Usage: run.sh +# Thin wrapper: sets up environment, then delegates to scripts/. +set -euo pipefail + +CFG_NAME="${1:?Usage: $0 }" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +cd "${REPO_ROOT}" + +# ── Environment setup ──────────────────────────────────────── +source .buildkite/k3_harness/setup-env.sh + +# Install test utilities (yq for YAML parsing, jq for JSON, openai/pandas/matplotlib for benchmarks) +uv pip install yq jq openai pandas matplotlib 2>/dev/null || true + +# ── Ensure all scripts are executable ──────────────────────── +chmod +x "${SCRIPT_DIR}"/scripts/*.sh + +# ── Run the actual test logic ──────────────────────────────── +exec bash "${SCRIPT_DIR}/scripts/run-single-config.sh" "$CFG_NAME" diff --git a/.buildkite/k3_tests/comprehensive/scripts/run-single-config.sh b/.buildkite/k3_tests/comprehensive/scripts/run-single-config.sh new file mode 100755 index 0000000..43d6790 --- /dev/null +++ b/.buildkite/k3_tests/comprehensive/scripts/run-single-config.sh @@ -0,0 +1,531 @@ +#!/usr/bin/env bash +# Comprehensive integration test: run a single config natively in a K8s pod. +# Replaces the old Docker-based vllm-integration-tests.sh. +# +# Usage: run-single-config.sh +set -euo pipefail + +CFG_NAME="${1:?Usage: $0 }" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" + +cd "${REPO_ROOT}" +source .buildkite/k3_tests/common_scripts/helpers.sh + +CONFIG_DIR="${REPO_ROOT}/.buildkite/configs" +LOGFILE="${REPO_ROOT}/${CFG_NAME%.yaml}.log" +BUILD_ID="${BUILDKITE_BUILD_ID:-local_$$}" +NEED_UPLOAD="${NEED_UPLOAD:-false}" +SERVER_WAIT_TIMEOUT="${SERVER_WAIT_TIMEOUT:-240}" + +# Tee all output so Buildkite can collect it as an artifact +exec > >(tee -a "$LOGFILE") 2>&1 + +# ── Validate config ────────────────────────────────────────── +cfg_file="${CONFIG_DIR}/${CFG_NAME}" +if [[ ! -f "$cfg_file" ]]; then + echo "Config not found: ${cfg_file}" + exit 1 +fi + +feature_type=$(yq -r '.feature.type // ""' "$cfg_file") +echo -e "\033[1;33m===== Testing LMCache with ${CFG_NAME} (type=${feature_type:-standard}) =====\033[0m" + +# Prevent git from hanging on prompts +export GIT_TERMINAL_PROMPT=0 + +# PID tracking for cleanup +PIDS=() +on_exit() { + local rc=$? + echo "--- Cleaning up (exit code: $rc)..." + for p in "${PIDS[@]}"; do + kill "$p" 2>/dev/null || true + wait "$p" 2>/dev/null || true + done + # Copy vLLM server logs to repo root so Buildkite can collect them as artifacts + cp /tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}*.log "${REPO_ROOT}/" 2>/dev/null || true + sleep 5 +} +trap on_exit EXIT INT TERM + +############### +# UTILITIES # +############### + +# Start a vLLM process natively (no Docker). +# Reads env vars from a yq-extracted docker section and vllm args from a vllm section. +start_single_server() { + local docker_section="$1" + local vllm_section="$2" + local port="$3" + local gpu="${4:-}" + local logfile="${5:-/tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}.log}" + + # Collect env vars from docker.env[] + local -a env_cmd=() + while IFS= read -r e; do + [[ -n "$e" ]] && env_cmd+=("$e") + done < <(yq -r '.env[]?' <<<"$docker_section") + + # Standard env vars + env_cmd+=("VLLM_USE_FLASHINFER_SAMPLER=0") + [[ -n "${HF_TOKEN:-}" ]] && env_cmd+=("HF_TOKEN=${HF_TOKEN}") + [[ -n "$gpu" ]] && env_cmd+=("CUDA_VISIBLE_DEVICES=${gpu}") + + # Override socket prefix so each server gets its own socket directory. + # In the old Docker setup, a volume mount mapped /tmp/lmcache_internal_api_server + # inside the container to /tmp/lmcache_internal_api_server/${port} on the host. + # Without Docker, we replicate this by setting the prefix to include the port. + mkdir -p "/tmp/lmcache_internal_api_server/${port}" + env_cmd+=("LMCACHE_INTERNAL_API_SERVER_SOCKET_PATH_PREFIX=/tmp/lmcache_internal_api_server/${port}/socket") + + # Parse vllm model and args + local vllm_model + vllm_model="$(yq -r '.model' <<<"$vllm_section")" + local -a vllm_cli_args=() + mapfile -t vllm_cli_args < <(yq -r '.args // [] | .[]' <<<"$vllm_section") + + echo "Starting vLLM: model=$vllm_model port=$port gpu=${gpu:-all}" + env "${env_cmd[@]}" \ + vllm serve "$vllm_model" "${vllm_cli_args[@]}" --port "$port" \ + >"$logfile" 2>&1 & + local pid=$! + PIDS+=("$pid") + echo " PID=$pid log=$logfile" + + # Wait for readiness + if ! wait_for_server "$port" "$SERVER_WAIT_TIMEOUT"; then + echo "Server failed to start. Last 50 lines of log:" + tail -50 "$logfile" || true + return 1 + fi +} + +############### +# LAUNCH # +############### + +PORT=$(find_free_port 8000) +PORT1="" +PORT2="" +model="" + +if [[ "$feature_type" == "pd" ]]; then + # ── Prefiller-Decoder mode (2 GPUs) ───────────────────── + PORT1=$(find_free_port 8100) + PORT2=$(find_free_port $((PORT1 + 100))) + + prefiller_docker="$(yq '.["docker-prefiller"]' "$cfg_file")" + prefiller_vllm="$(yq '.["vllm-prefiller"]' "$cfg_file")" + decoder_docker="$(yq '.["docker-decoder"]' "$cfg_file")" + decoder_vllm="$(yq '.["vllm-decoder"]' "$cfg_file")" + model="$(yq -r '.["vllm-prefiller"].model' "$cfg_file")" + + # Extra PD env vars + proxy=$(yq -er '.["docker-prefiller"]["proxy-port"]' "$cfg_file" 2>/dev/null || echo "7500") + init=$(yq -er '.["docker-decoder"]["init-port"]' "$cfg_file" 2>/dev/null || echo "7300") + alloc=$(yq -er '.["docker-decoder"]["alloc-port"]' "$cfg_file" 2>/dev/null || echo "7400") + + # Inject PD-specific env vars into docker sections + prefiller_docker=$(echo "$prefiller_docker" | yq -y --arg proxy "$proxy" '. + {"env": (.env + ["LMCACHE_PD_PROXY_PORT=" + $proxy])}') + decoder_docker=$(echo "$decoder_docker" | yq -y --arg init "$init" --arg alloc "$alloc" '. + {"env": (.env + ["LMCACHE_PD_PEER_INIT_PORT=" + $init, "LMCACHE_PD_PEER_ALLOC_PORT=" + $alloc])}') + + echo "--- Starting prefiller on port $PORT1 (GPU 0)" + # Prefiller needs UCX_TLS for NIXL transport + prefiller_docker=$(echo "$prefiller_docker" | yq -y ". + {\"env\": (.env + [\"UCX_TLS=cuda_ipc,cuda_copy,tcp\"])}") + start_single_server "$prefiller_docker" "$prefiller_vllm" "$PORT1" "0" \ + "${REPO_ROOT}/${CFG_NAME%.yaml}-prefiller.log" + + echo "--- Starting decoder on port $PORT2 (GPU 1)" + decoder_docker=$(echo "$decoder_docker" | yq -y ". + {\"env\": (.env + [\"UCX_TLS=cuda_ipc,cuda_copy,tcp\"])}") + start_single_server "$decoder_docker" "$decoder_vllm" "$PORT2" "1" \ + "${REPO_ROOT}/${CFG_NAME%.yaml}-decoder.log" + + # Start disagg proxy + echo "--- Starting PD proxy on port $PORT" + python3 "${REPO_ROOT}/examples/disagg_prefill/disagg_proxy_server.py" \ + --port "$PORT" \ + --prefiller-port "$PORT1" \ + --decoder-port "$PORT2" \ + --decoder-init-port "$init" \ + --decoder-alloc-port "$alloc" \ + --proxy-port "$proxy" \ + > "${REPO_ROOT}/${CFG_NAME%.yaml}-proxy.log" 2>&1 & + PIDS+=($!) + sleep 10 + +elif [[ "$feature_type" == "p2p" ]]; then + # ── Peer-to-Peer mode (2 GPUs) ────────────────────────── + PORT1=$(find_free_port 8177) + PORT2=$(find_free_port $((PORT1 + 100))) + + docker1="$(yq '.docker1' "$cfg_file")" + vllm1="$(yq '.vllm1' "$cfg_file")" + docker2="$(yq '.docker2' "$cfg_file")" + vllm2="$(yq '.vllm2' "$cfg_file")" + model="$(yq -r '.vllm1.model' "$cfg_file")" + + # Controller ports + pull=$(yq -er '.docker1["pull-port"]' "$cfg_file" 2>/dev/null || echo "8300") + reply=$(yq -er '.docker1["reply-port"]' "$cfg_file" 2>/dev/null || echo "8400") + + # Inject controller URLs + docker1=$(echo "$docker1" | yq -y --arg pull "$pull" --arg reply "$reply" '. + {"env": (.env + ["LMCACHE_CONTROLLER_PULL_URL=localhost:" + $pull, "LMCACHE_CONTROLLER_REPLY_URL=localhost:" + $reply, "UCX_TLS=tcp"])}') + + pull2=$(yq -er '.docker2["pull-port"]' "$cfg_file" 2>/dev/null || echo "$pull") + reply2=$(yq -er '.docker2["reply-port"]' "$cfg_file" 2>/dev/null || echo "$reply") + docker2=$(echo "$docker2" | yq -y --arg pull2 "$pull2" --arg reply2 "$reply2" '. + {"env": (.env + ["LMCACHE_CONTROLLER_PULL_URL=localhost:" + $pull2, "LMCACHE_CONTROLLER_REPLY_URL=localhost:" + $reply2, "UCX_TLS=tcp"])}') + + # Start controller + echo "--- Starting P2P controller on port $PORT" + PYTHONHASHSEED=123 lmcache_controller \ + --host localhost \ + --port "$PORT" \ + --monitor-ports "{\"pull\": ${pull}, \"reply\": ${reply}}" \ + > "/tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}_controller.log" 2>&1 & + PIDS+=($!) + sleep 10 + + echo "--- Starting instance 1 on port $PORT1 (GPU 0)" + start_single_server "$docker1" "$vllm1" "$PORT1" "0" \ + "/tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}1.log" + + echo "--- Starting instance 2 on port $PORT2 (GPU 1)" + start_single_server "$docker2" "$vllm2" "$PORT2" "1" \ + "/tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}2.log" + +else + # ── Single server mode (1 GPU) ────────────────────────── + docker_section="$(yq '.docker' "$cfg_file")" + vllm_section="$(yq '.vllm' "$cfg_file")" + model="$(yq -r '.vllm.model' "$cfg_file")" + + echo "--- Starting single server on port $PORT" + start_single_server "$docker_section" "$vllm_section" "$PORT" "" \ + "/tmp/build_${BUILD_ID}_${CFG_NAME%.yaml}.log" +fi + +############### +# WORKLOAD # +############### + +test_mode="$(yq -r '.workload.type' "$cfg_file")" + +if [[ "$test_mode" == "dummy" ]]; then + echo "--- Running dummy test (simple HTTP request)" + http_status_code=$( + curl --max-time 60 "http://localhost:${PORT}/v1/completions" \ + -w "%{http_code}" -o response-file.txt \ + -H "Content-Type: application/json" \ + -d '{ + "model": "'"$model"'", + "prompt": "<|begin_of_text|><|system|>\nYou are a helpful AI assistant.\n<|user|>\nWhat is the capital of France?\n<|assistant|>", + "max_tokens": 100, + "temperature": 0.7 + }' + ) + if [[ "$http_status_code" -ne 200 ]]; then + echo "FAIL: Dummy test HTTP status ${http_status_code}" + cat response-file.txt + exit 1 + fi + echo "PASS: Dummy test succeeded" + cat response-file.txt + +elif [[ "$test_mode" == "long_doc_qa" ]]; then + echo "--- Running long_doc_qa workload" + + # Build workload JSON (merge workload section with model, strip non-CLI fields) + # Fields like expected-latency-gain are used by the checking logic, not long_doc_qa.py. + # "completion" -> "completions" rename to match the argparse flag. + workload_yaml="$(yq --arg model "$model" '(.workload * {"model": $model}) | del(.type) | del(.["expected-latency-gain"]) | if .completion then .completions = .completion | del(.completion) else . end' "$cfg_file")" + cfg_json="$(yq '.' "$cfg_file")" + + # Determine which checking fields are requested + check_warmup_round_time_per_prompt=$( + jq -e '(.["checking-fields"] // []) | index("warmup_round_time_per_prompt") != null' \ + <<< "$cfg_json" >/dev/null && echo true || echo false + ) + check_query_round_time_per_prompt=$( + jq -e '(.["checking-fields"] // []) | index("query_round_time_per_prompt") != null' \ + <<< "$cfg_json" >/dev/null && echo true || echo false + ) + check_query_ttft_per_prompt=$( + jq -e '(.["checking-fields"] // []) | index("warmup_ttft_per_prompt") != null' \ + <<< "$cfg_json" >/dev/null && echo true || echo false + ) + + run_long_doc_qa_workload() { + local workload_config="$1" + local port="$2" + local check_warmup="${3:-false}" + local check_ttft="${4:-false}" + local check_round="${5:-false}" + local feat_type="${6:-dummy}" + local need_upload="${7:-false}" + + echo "Running long_doc_qa on port $port" + + # Build args array from JSON config + local -a workload_args=() + mapfile -d '' -t workload_args < <( + jq -j ' + to_entries[] + | select(.value != null and (.value|tostring) != "") + | ( + if (.value|type) == "boolean" then + if .value then ["--\(.key)", "\u0000"] else [] end + elif (.value|type) == "array" then + .value[] + | ["--\(.key)", "\u0000", + (if (type=="string") then . else tostring end), + "\u0000"] + else + ["--\(.key)", "\u0000", + (if ((.value|type)=="string") then .value else (.value|tostring) end), + "\u0000"] + end + ) + | .[] + ' <<<"$workload_config" + ) + + local json + json=$( + python3 "${REPO_ROOT}/benchmarks/long_doc_qa/long_doc_qa.py" \ + "${workload_args[@]}" \ + --port="$port" \ + --output="response.txt" \ + --json-output \ + 2>>response.txt | tail -n 1 + ) + local query_ttft_per_prompt query_round_time_per_prompt warmup_round_time_per_prompt + query_ttft_per_prompt=$(echo "$json" | jq -r '.query_ttft_per_prompt') + query_round_time_per_prompt=$(echo "$json" | jq -r '.query_round_time_per_prompt') + warmup_round_time_per_prompt=$(echo "$json" | jq -r '.warmup_round_time_per_prompt') + + if [[ "$need_upload" == "true" ]]; then + # Nightly: write date-stamped file and upload as Buildkite artifact. + # A finalize step (upload-baselines.sh) will collect all artifacts, + # prune old files, and make a single commit to benchmarks-main. + local datestamp + datestamp="$(date +%Y%m%d)" + local artifact_name="${feat_type}-${datestamp}.json" + local artifact_path="${REPO_ROOT}/benchmarks/long_doc_qa/${artifact_name}" + + echo "$json" + mkdir -p "$(dirname "$artifact_path")" + printf '%s\n' "$json" > "$artifact_path" + + echo "[INFO] Uploading baseline artifact: ${artifact_name}" + buildkite-agent artifact upload "$artifact_path" 2>/dev/null || { + echo "[WARN] buildkite-agent not available; wrote $artifact_path locally" + } + return 0 + fi + + # ── PR comparison: rolling 5-day worst-case baseline ────── + # Fetch all date-stamped baselines for this feature from benchmarks-main. + # Use max() of each metric (= worst/slowest) as the threshold baseline. + # This makes the gate more tolerant of nightly hardware noise. + timeout 30 git fetch origin benchmarks-main >/dev/null 2>&1 || true + + # List all baseline files for this feature (date-stamped and legacy). + # Note: git ls-tree does NOT expand shell globs in pathspecs, so we + # list the directory and filter with grep instead. + local -a baseline_files=() + mapfile -t baseline_files < <( + git ls-tree --name-only origin/benchmarks-main -- \ + benchmarks/long_doc_qa/ 2>/dev/null \ + | grep -E "^benchmarks/long_doc_qa/${feat_type}(-[0-9]{8})?\.json$" \ + || true + ) + + if [[ ${#baseline_files[@]} -eq 0 ]]; then + if [[ "$feat_type" != "dummy" ]]; then + echo "No baselines found for ${feat_type} -- skipping performance comparisons" + echo "Current metrics: TTFT=$query_ttft_per_prompt, Latency=$query_round_time_per_prompt, Warmup=$warmup_round_time_per_prompt" + fi + return 0 + fi + + echo "[INFO] Found ${#baseline_files[@]} baseline file(s) for ${feat_type}:" + printf " %s\n" "${baseline_files[@]}" + + # Compute rolling max (worst-case) across all baselines + local expected_query_ttft=0 expected_query_round=0 expected_warmup=0 + for bf in "${baseline_files[@]}"; do + local bj + bj=$(git show "origin/benchmarks-main:${bf}" 2>/dev/null || echo "") + if [[ -z "$bj" ]] || ! echo "$bj" | jq -e . >/dev/null 2>&1; then + echo " Skipping invalid baseline: $bf" + continue + fi + # Take max of each metric (higher time = worse perf = more permissive gate) + expected_query_ttft=$(awk "BEGIN { a=$expected_query_ttft; b=$(echo "$bj" | jq -r '.query_ttft_per_prompt // 0'); print (a>b?a:b) }") + expected_query_round=$(awk "BEGIN { a=$expected_query_round; b=$(echo "$bj" | jq -r '.query_round_time_per_prompt // 0'); print (a>b?a:b) }") + expected_warmup=$(awk "BEGIN { a=$expected_warmup; b=$(echo "$bj" | jq -r '.warmup_round_time_per_prompt // 0'); print (a>b?a:b) }") + done + + echo "[INFO] Rolling worst-case baseline: TTFT=$expected_query_ttft, Latency=$expected_query_round, Warmup=$expected_warmup" + + # Sanity check: if all baselines were invalid, skip comparison + if awk "BEGIN { exit ($expected_query_ttft == 0 && $expected_query_round == 0 && $expected_warmup == 0) ? 0 : 1 }"; then + echo "All baselines were invalid or empty -- skipping performance comparisons" + return 0 + fi + + # Adaptive threshold: with fewer baselines the worst-case max is less + # representative, so we allow more headroom. + # 5+ baselines → 1.2x (20% tolerance, full confidence) + # 4 baselines → 1.3x + # 3 baselines → 1.4x + # 2 baselines → 1.5x + # 1 baseline → 1.6x (60% tolerance, low confidence) + local num_baselines=${#baseline_files[@]} + local threshold + threshold=$(awk -v n="$num_baselines" 'BEGIN { + gap = (5 - n); if (gap < 0) gap = 0 + printf "%.1f", 1.2 + gap * 0.1 + }') + local pct + pct=$(awk -v t="$threshold" 'BEGIN { printf "%d", (t - 1) * 100 }') + echo "[INFO] Baselines: ${num_baselines}/5 — using ${pct}% tolerance (threshold=${threshold}x)" + + if [[ "$check_ttft" == "true" ]]; then + echo "Worst-case baseline query ttft per prompt: $expected_query_ttft" + echo "Actual query ttft per prompt: $query_ttft_per_prompt" + awk -v expected="$expected_query_ttft" -v actual="$query_ttft_per_prompt" -v t="$threshold" -v pct="$pct" 'BEGIN { + if (actual > expected * t) { printf "FAIL: Query ttft per prompt >%d%% worse than rolling baseline\n", pct; exit 1 } + else { print "PASS: Query ttft per prompt within threshold" } + }' + fi + + if [[ "$check_round" == "true" ]]; then + echo "Worst-case baseline query round time per prompt: $expected_query_round" + echo "Actual query round time per prompt: $query_round_time_per_prompt" + awk -v expected="$expected_query_round" -v actual="$query_round_time_per_prompt" -v t="$threshold" -v pct="$pct" 'BEGIN { + if (actual > expected * t) { printf "FAIL: Query round time per prompt >%d%% worse than rolling baseline\n", pct; exit 1 } + else { print "PASS: Query round time per prompt within threshold" } + }' + fi + + if [[ "$check_warmup" == "true" ]]; then + echo "Worst-case baseline warmup round time per prompt: $expected_warmup" + echo "Actual warmup round time per prompt: $warmup_round_time_per_prompt" + awk -v expected="$expected_warmup" -v actual="$warmup_round_time_per_prompt" -v t="$threshold" -v pct="$pct" 'BEGIN { + if (actual > expected * t) { printf "FAIL: Warmup round time per prompt >%d%% worse than rolling baseline\n", pct; exit 1 } + else { print "PASS: Warmup round time per prompt within threshold" } + }' + fi + } + + if [[ "$feature_type" == "p2p" ]]; then + run_long_doc_qa_workload "$workload_yaml" "$PORT1" + run_long_doc_qa_workload "$workload_yaml" "$PORT2" \ + "$check_warmup_round_time_per_prompt" "$check_query_ttft_per_prompt" \ + "$check_query_round_time_per_prompt" "${CFG_NAME%.yaml}" "$NEED_UPLOAD" + else + run_long_doc_qa_workload "$workload_yaml" "$PORT" \ + "$check_warmup_round_time_per_prompt" "$check_query_ttft_per_prompt" \ + "$check_query_round_time_per_prompt" "${CFG_NAME%.yaml}" "$NEED_UPLOAD" + fi +fi + +############### +# MEMORY LEAK # +############### + +check_memory_leak() { + local port="$1" + local socket_path="/tmp/lmcache_internal_api_server/${port}/socket_7000" + + local use_hot + use_hot=$(curl -s --unix-socket "$socket_path" "http://localhost/conf" 2>/dev/null | jq -r '.local_cpu // false') + if [[ -z "$use_hot" || "$use_hot" == "null" ]]; then + use_hot="false" + fi + + echo "Checking memory leak on socket_path $socket_path (use_hot=$use_hot)..." + + local metrics + metrics=$(curl -s --unix-socket "$socket_path" "http://localhost/metrics" 2>/dev/null) + if [[ -z "$metrics" ]]; then + echo "ERROR: Failed to fetch metrics from $socket_path" + return 1 + fi + + local local_cpu_hot_cache_count active_memory_objs_count pinned_memory_objs_count pin_monitor_pinned_objects_count + local_cpu_hot_cache_count=$(echo "$metrics" | grep -E '^lmcache:local_cpu_hot_cache_count\b' | awk '{print $2}' | head -n 1) + active_memory_objs_count=$(echo "$metrics" | grep -E '^lmcache:active_memory_objs_count\b' | awk '{print $2}' | head -n 1) + pinned_memory_objs_count=$(echo "$metrics" | grep -E '^lmcache:pinned_memory_objs_count\b' | awk '{print $2}' | head -n 1) + pin_monitor_pinned_objects_count=$(echo "$metrics" | grep -E '^lmcache:pin_monitor_pinned_objects_count\b' | awk '{print $2}' | head -n 1) + + local_cpu_hot_cache_count=$(printf "%.0f" "${local_cpu_hot_cache_count:-0}") + active_memory_objs_count=$(printf "%.0f" "${active_memory_objs_count:-0}") + pinned_memory_objs_count=$(printf "%.0f" "${pinned_memory_objs_count:-0}") + pin_monitor_pinned_objects_count=$(printf "%.0f" "${pin_monitor_pinned_objects_count:-0}") + + echo " local_cpu_hot_cache_count: $local_cpu_hot_cache_count" + echo " active_memory_objs_count: $active_memory_objs_count" + echo " pinned_memory_objs_count: $pinned_memory_objs_count" + echo " pin_monitor_pinned_objects_count: $pin_monitor_pinned_objects_count" + + local has_leak=false + + if [[ "$pinned_memory_objs_count" -ne 0 ]]; then + echo "ERROR: Memory leak detected - pinned_memory_objs_count ($pinned_memory_objs_count) should be 0" + has_leak=true + fi + + if [[ "$use_hot" == "false" || "$use_hot" == "False" ]]; then + if [[ "$local_cpu_hot_cache_count" -ne 0 ]]; then + echo "ERROR: Memory leak - local_cpu_hot_cache_count ($local_cpu_hot_cache_count) should be 0 when use_hot=false" + has_leak=true + fi + if [[ "$active_memory_objs_count" -ne 0 ]]; then + echo "ERROR: Memory leak - active_memory_objs_count ($active_memory_objs_count) should be 0 when use_hot=false" + has_leak=true + fi + else + if [[ "$active_memory_objs_count" -ne "$local_cpu_hot_cache_count" ]]; then + echo "ERROR: Memory leak - active_memory_objs_count ($active_memory_objs_count) should equal local_cpu_hot_cache_count ($local_cpu_hot_cache_count) when use_hot=true" + has_leak=true + fi + fi + + if [[ "$has_leak" == "true" ]]; then + echo "$metrics" + return 1 + fi + + echo " Memory leak check passed!" + return 0 +} + +# Wait for metrics to settle +sleep 30 +echo "--- Checking for memory leaks..." + +if [[ "$feature_type" == "p2p" ]]; then + echo "Skipping memory leak check for p2p (known issue with pinned_memory_objs not clearing)." +elif [[ "$feature_type" == "pd" ]]; then + if ! check_memory_leak "$PORT1"; then + echo "Memory leak check failed for instance 1 (port $PORT1)" + exit 1 + fi + if ! check_memory_leak "$PORT2"; then + echo "Memory leak check failed for instance 2 (port $PORT2)" + exit 1 + fi +elif [[ "$CFG_NAME" == "multi_device.yaml" || "$CFG_NAME" == "layerwise.yaml" ]]; then + echo "Skipping memory leak check for $CFG_NAME case as it's a flaky test while run check_memory_leak check." +else + if ! check_memory_leak "$PORT"; then + echo "Memory leak check failed for $CFG_NAME" + exit 1 + fi +fi + +echo "===== PASS: ${CFG_NAME} =====" diff --git a/.buildkite/k3_tests/comprehensive/scripts/upload-baselines.sh b/.buildkite/k3_tests/comprehensive/scripts/upload-baselines.sh new file mode 100755 index 0000000..1080145 --- /dev/null +++ b/.buildkite/k3_tests/comprehensive/scripts/upload-baselines.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Finalize nightly baseline upload. +# Runs AFTER all comprehensive config steps complete (via `wait`). +# +# 1. Downloads all *-YYYYMMDD.json artifacts from this build +# 2. Checks out benchmarks-main +# 3. Copies new date-stamped files, prunes entries older than 5 days +# 4. Makes a single commit and pushes to benchmarks-main +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" +BASELINE_DIR="${REPO_ROOT}/benchmarks/long_doc_qa" +KEEP_DAYS=5 + +cd "${REPO_ROOT}" + +# Prevent git from hanging on prompts +export GIT_TERMINAL_PROMPT=0 + +############### +# DOWNLOAD # +############### + +ARTIFACT_DIR="/tmp/baseline_artifacts" +mkdir -p "$ARTIFACT_DIR" + +echo "--- Downloading baseline artifacts from this build" +buildkite-agent artifact download "benchmarks/long_doc_qa/*-*.json" "$ARTIFACT_DIR" 2>/dev/null || { + echo "No baseline artifacts found. This is expected if no long_doc_qa configs ran." + exit 0 +} + +NEW_FILES=("$ARTIFACT_DIR"/benchmarks/long_doc_qa/*-*.json) +if [[ ! -e "${NEW_FILES[0]}" ]]; then + echo "No date-stamped baseline files found in artifacts." + exit 0 +fi + +echo "Downloaded ${#NEW_FILES[@]} baseline file(s):" +printf " %s\n" "${NEW_FILES[@]}" + +############### +# CHECKOUT # +############### + +# Work in a detached worktree to avoid touching the main checkout +WORK_DIR="/tmp/baselines_push_$$" +trap 'rm -rf "$WORK_DIR"' EXIT + +# Push baselines to a dedicated CI repo instead of the main LMCache repo. +CI_REPO="LMCache/LMCache" +CI_BRANCH="benchmarks-main" + +if [[ -n "${GITHUB_TOKEN:-}" ]]; then + CI_REPO_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${CI_REPO}.git" +else + echo "[WARN] GITHUB_TOKEN not set — push may fail without credentials" + CI_REPO_URL="https://github.com/${CI_REPO}.git" +fi + +echo "--- Preparing ${CI_BRANCH} branch from ${CI_REPO}" +git clone --depth=1 --branch "${CI_BRANCH}" "${CI_REPO_URL}" "$WORK_DIR" 2>/dev/null || { + # Branch doesn't exist yet — create an orphan + mkdir -p "$WORK_DIR" + git -C "$WORK_DIR" init + git -C "$WORK_DIR" remote add origin "${CI_REPO_URL}" + git -C "$WORK_DIR" checkout --orphan "${CI_BRANCH}" +} + +############### +# COPY + PRUNE# +############### + +mkdir -p "$WORK_DIR/benchmarks/long_doc_qa" + +# Copy new files +for f in "${NEW_FILES[@]}"; do + cp "$f" "$WORK_DIR/benchmarks/long_doc_qa/" +done + +# Prune date-stamped files older than KEEP_DAYS +echo "--- Pruning baselines older than ${KEEP_DAYS} days" +CUTOFF_DATE="$(date -d "${KEEP_DAYS} days ago" +%Y%m%d 2>/dev/null || date -v-${KEEP_DAYS}d +%Y%m%d)" + +for f in "$WORK_DIR"/benchmarks/long_doc_qa/*-*.json; do + [[ ! -e "$f" ]] && continue + fname="$(basename "$f")" + # Extract YYYYMMDD from filename like "local_cpu-20260301.json" + file_date="${fname##*-}" + file_date="${file_date%.json}" + if [[ "$file_date" =~ ^[0-9]{8}$ ]] && [[ "$file_date" < "$CUTOFF_DATE" ]]; then + echo " Removing old baseline: $fname" + rm "$f" + fi +done + +# Also remove legacy single-point files (feature.json without date) since we now use rolling +# Keep them if they exist for backward compat during transition, but don't create new ones. + +############### +# COMMIT+PUSH # +############### + +cd "$WORK_DIR" +git add benchmarks/long_doc_qa/ + +if git diff --cached --quiet 2>/dev/null; then + echo "No changes to commit." + exit 0 +fi + +TODAY="$(date +%Y-%m-%d)" +git -c user.email="ci@lmcache.ai" -c user.name="LMCache CI" \ + commit -m "Update rolling baselines: ${TODAY}" || true + +echo "--- Pushing to ${CI_REPO} ${CI_BRANCH}" +# Try normal push first; fall back to force-push if history diverged +if ! git push origin "HEAD:${CI_BRANCH}" 2>/dev/null; then + echo "[WARN] Normal push failed, force-pushing..." + git push origin "+HEAD:${CI_BRANCH}" 2>/dev/null || { + echo "[ERROR] Failed to push baselines to ${CI_REPO} ${CI_BRANCH}" + exit 1 + } +fi + +echo "--- Baselines uploaded successfully" +git log --oneline -1 +ls -la benchmarks/long_doc_qa/ diff --git a/.buildkite/k3_tests/correctness/BK_WEB_SETUP.md b/.buildkite/k3_tests/correctness/BK_WEB_SETUP.md new file mode 100644 index 0000000..77bbc2d --- /dev/null +++ b/.buildkite/k3_tests/correctness/BK_WEB_SETUP.md @@ -0,0 +1,14 @@ +# Buildkite Web UI Setup: Correctness Tests + +**Steps editor**: paste contents of `buildkite-pipeline.yml` (fill in `HF_TOKEN`). + +**GitHub trigger settings**: +- Filter: *(none — runs on every push/PR)* +- Skip queued / cancel running branch builds: Yes + +Lightweight (1 GPU) — good candidate for a required GitHub status check. + +> Builds whose only changes are docs/`*.md`/`LICENSE`/`.github/**` auto-pass +> via the [path filter](../README.md#path-based-skip-auto-pass-on-docs-only-changes). +> Changes under `.buildkite/` always run. Add `force-ci` label to the PR to +> bypass. diff --git a/.buildkite/k3_tests/correctness/buildkite-pipeline.yml b/.buildkite/k3_tests/correctness/buildkite-pipeline.yml new file mode 100644 index 0000000..ed9fb33 --- /dev/null +++ b/.buildkite/k3_tests/correctness/buildkite-pipeline.yml @@ -0,0 +1,11 @@ +# Paste this into the Buildkite pipeline's "Steps" editor. +# It uploads the real pipeline definition from the repo. +agents: + queue: "k8s" + +env: + HF_TOKEN: "" # set your HuggingFace token here (needed for gated model access) + +steps: + - label: ":pipeline: Upload pipeline" + command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/correctness/pipeline.yml diff --git a/.buildkite/k3_tests/correctness/pipeline.yml b/.buildkite/k3_tests/correctness/pipeline.yml new file mode 100644 index 0000000..935c377 --- /dev/null +++ b/.buildkite/k3_tests/correctness/pipeline.yml @@ -0,0 +1,27 @@ +# Correctness test — verifies LMCache produces identical output to base vLLM. +# Runs: base vLLM → LMCache vLLM → man bash prefix test → ShareGPT diff. +# Requires pre-downloaded model weights and ShareGPT dataset. + +steps: + - label: ":white_check_mark: Correctness" + command: .buildkite/k3_tests/correctness/run.sh + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - { name: hf-cache, mountPath: /root/.cache/huggingface } + - { name: datasets, mountPath: /root/correctness } + volumes: + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + - { name: datasets, hostPath: { path: /data/datasets, type: DirectoryOrCreate } } + artifact_paths: + - "build_*.log" diff --git a/.buildkite/k3_tests/correctness/run.sh b/.buildkite/k3_tests/correctness/run.sh new file mode 100755 index 0000000..eed3b5a --- /dev/null +++ b/.buildkite/k3_tests/correctness/run.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Correctness test entrypoint for K8s pods. +# Thin wrapper: sets up environment, then delegates to scripts/. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +cd "${REPO_ROOT}" + +# ── Environment setup ──────────────────────────────────────── +source .buildkite/k3_harness/setup-env.sh +uv pip install aiohttp tqdm pandas huggingface_hub + +# ── Ensure all scripts are executable ──────────────────────── +chmod +x "${SCRIPT_DIR}"/scripts/*.sh + +# ── Run the actual test logic ──────────────────────────────── +# Retry up to 3 times. vLLM's `VLLM_BATCH_INVARIANT=1` guarantee +# currently does not hold across process restarts in the vllm nightlies +# we pin to (reproduced locally: two back-to-back vanilla vllm-only runs +# produce ~55 different outputs out of 100). Phase 4's bitwise +# comparison between the base-vllm run and the vllm+LMCache run is +# bimodal (0 diff or ~50 diff) depending on which cuBLAS/kernel plans +# the two separate processes happen to pick. Retrying lets a lucky +# scheduling pass; a real LMCache regression will persistently fail. +MAX_ATTEMPTS="${CORRECTNESS_MAX_ATTEMPTS:-3}" +for attempt in $(seq 1 "${MAX_ATTEMPTS}"); do + echo "=== Correctness attempt ${attempt}/${MAX_ATTEMPTS} ===" + if bash "${SCRIPT_DIR}/scripts/run-correctness.sh"; then + exit 0 + fi + echo "[INFO] Attempt ${attempt} failed." +done +echo "[FAIL] Correctness persistently failed after ${MAX_ATTEMPTS} attempts." >&2 +exit 1 diff --git a/.buildkite/k3_tests/correctness/scripts/run-correctness.sh b/.buildkite/k3_tests/correctness/scripts/run-correctness.sh new file mode 100755 index 0000000..3f3d8c1 --- /dev/null +++ b/.buildkite/k3_tests/correctness/scripts/run-correctness.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# Self-contained correctness test for K8s pods. +# Replaces the old vllm-correctness.sh which assumed a pre-existing venv +# and pick-free-gpu.sh. Here everything runs natively in the pod. +# +# Tests that LMCache produces identical output to base vLLM. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" + +cd "${REPO_ROOT}" +source .buildkite/k3_tests/common_scripts/helpers.sh + +############### +# CONFIG # +############### + +BUILD_ID="${BUILDKITE_BUILD_ID:-local_$$}" +MODEL="Qwen/Qwen2.5-14B-Instruct" +WORK_LOG="/tmp/build_${BUILD_ID}_correctness.log" +VLLM_LOG="/tmp/build_${BUILD_ID}_vllm.log" +ARTIFACT="build_${BUILD_ID}.log" +SERVER_WAIT_TIMEOUT=180 +CORRECTNESS_DIR=".buildkite/correctness" +REQUEST_NUMBER=100 +MAX_CONCURRENCY=40 + +# K8s assigns GPUs via device plugin +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" + +############### +# DATA SETUP # +############### + +SHAREGPT_PATH="/root/correctness/.ShareGPT_V3_unfiltered_cleaned_split.json" +if [[ ! -f "$SHAREGPT_PATH" ]]; then + echo "[INFO] ShareGPT dataset not found, downloading..." + mkdir -p /root/correctness + wget -q \ + "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" \ + -O "$SHAREGPT_PATH" +fi + +if [[ ! -s "$SHAREGPT_PATH" ]]; then + echo "[ERROR] ShareGPT dataset file is empty: $SHAREGPT_PATH" + exit 1 +fi + +############### +# HELPERS # +############### + +VLLM_PID="" + +CI_CACHE_DIR="$PWD/.vllm_cache_${BUILD_ID}" +mkdir -p "$CI_CACHE_DIR" +export FLASHINFER_WORKSPACE_DIR="$CI_CACHE_DIR/flashinfer" + +collect_artifact() { + echo "[INFO] Collecting logs into ${ARTIFACT}" + cat "${WORK_LOG}" "${VLLM_LOG}" > "${ARTIFACT}" 2>/dev/null || true +} + +stop_vllm() { + if [[ -n "${VLLM_PID:-}" ]]; then + echo "[INFO] Stopping vLLM process (PID: ${VLLM_PID})" + kill "${VLLM_PID}" >/dev/null 2>&1 || true + wait "${VLLM_PID}" 2>/dev/null || true + VLLM_PID="" + sleep 5 + fi +} + +trap 'rc=$?; stop_vllm; collect_artifact; exit $rc' EXIT INT TERM + +exec > >(tee -a "${WORK_LOG}") 2>&1 + +echo "=== DIAGNOSTICS: GPU STATE before CI ===" +nvidia-smi +echo "[INFO] Using GPU(s): ${CUDA_VISIBLE_DEVICES}" + +echo "[INFO] Converting ShareGPT dataset to OpenAI format..." +python "${CORRECTNESS_DIR}/sharegpt2openai.py" -i "$SHAREGPT_PATH" -o "./shareGPT_dataset.json" + +############### +# PHASE 1 # +############### +# Base Server (Baseline) -- vLLM WITHOUT LMCache + +PORT=$(find_free_port 8000) +echo "[INFO] Starting BASE vLLM server on port ${PORT}..." + +VLLM_SERVER_DEV_MODE=1 \ +VLLM_BATCH_INVARIANT=1 \ +vllm serve "${MODEL}" \ + --port "${PORT}" \ + --trust-remote-code \ + --enforce-eager \ + --attention-backend FLASH_ATTN \ + --gpu-memory-utilization 0.8 \ + >"${VLLM_LOG}" 2>&1 & +VLLM_PID=$! + +echo "[INFO] Waiting for Base server readiness..." +if ! wait_for_server "$PORT" "$SERVER_WAIT_TIMEOUT"; then + echo "[ERROR] Base vLLM failed to start" + exit 1 +fi + +echo "[TEST] Running ShareGPT Batch (Base)..." +python "${CORRECTNESS_DIR}/async_request.py" \ + --model "${MODEL}" \ + --endpoint "http://localhost:${PORT}/v1/chat/completions" \ + --output-file "sharegpt_base.txt" \ + --dataset_file "./shareGPT_dataset.json" \ + --request-number "${REQUEST_NUMBER}" \ + --max-concurrency "${MAX_CONCURRENCY}" + +stop_vllm + +############### +# PHASE 2 # +############### +# LMCache Server -- vLLM WITH LMCache + +echo "[INFO] Preparing LMCache config (cpu.yaml)..." +cat < cpu.yaml +chunk_size: 256 +local_cpu: true +max_local_cpu_size: 50 +EOF + +PORT=$(find_free_port 8000) +echo "[INFO] Starting LMCACHE vLLM server on port ${PORT}..." + +LMCACHE_CONFIG_FILE=cpu.yaml \ +VLLM_SERVER_DEV_MODE=1 \ +VLLM_BATCH_INVARIANT=1 \ +vllm serve "${MODEL}" \ + --port "${PORT}" \ + --trust-remote-code \ + --enforce-eager \ + --attention-backend FLASH_ATTN \ + --gpu-memory-utilization 0.8 \ + --kv-transfer-config '{"kv_connector":"LMCacheConnectorV1","kv_role":"kv_both"}' \ + >>"${VLLM_LOG}" 2>&1 & +VLLM_PID=$! + +echo "[INFO] Waiting for LMCache server readiness..." +if ! wait_for_server "$PORT" "$SERVER_WAIT_TIMEOUT"; then + echo "[ERROR] LMCache vLLM failed to start" + exit 1 +fi + +echo "[TEST] Running ShareGPT Batch (LMCache)..." +python "${CORRECTNESS_DIR}/async_request.py" \ + --model "${MODEL}" \ + --endpoint "http://localhost:${PORT}/v1/chat/completions" \ + --output-file "sharegpt_lmcache.txt" \ + --dataset_file "./shareGPT_dataset.json" \ + --request-number "${REQUEST_NUMBER}" \ + --max-concurrency "${MAX_CONCURRENCY}" + +############### +# PHASE 3 # +############### +# Man bash correctness test (on the LMCache server) + +echo "[TEST] Running technical man bash correctness test..." +CONTEXT="$(man bash | sed 's/.\x08//g' | tr -s '[:space:]' ' ' | awk '{for(i=1;i<=NF;i++){printf "%s ",$i; if(++c==5000) exit}}')" +HALF_CONTEXT="$(man bash | sed 's/.\x08//g' | tr -s '[:space:]' ' ' | awk '{for(i=1;i<=NF;i++){printf "%s ",$i; if(++c==2500) exit}}')" + +send_completion() { + curl -s "http://localhost:${PORT}/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "$(jq -n --arg model "${MODEL}" --arg content "$1" '{model: $model, temperature: 0, max_tokens: 100, messages: [{role:"user",content:$content}]}')" | + jq -r '.choices[0].message.content' +} + +echo "[STEP 1] Full context (LMCache)" +OUT1="$(send_completion "${CONTEXT}")" + +echo "[STEP 2] Reset prefix cache" +curl -s -X POST "http://localhost:${PORT}/reset_prefix_cache" >/dev/null + +echo "[STEP 3] Half context" +send_completion "${HALF_CONTEXT}" >/dev/null + +echo "[STEP 4] Full context again" +OUT2="$(send_completion "${CONTEXT}")" + +if [[ "${OUT1}" != "${OUT2}" ]]; then + echo "[FAIL] man bash output mismatch!" + exit 1 +fi + +############### +# PHASE 4 # +############### +# ShareGPT File Comparison + +echo "[INFO] Comparing ShareGPT results..." +COMPARE_OUT=$(python "${CORRECTNESS_DIR}/compare_files.py" --file1 "sharegpt_base.txt" --file2 "sharegpt_lmcache.txt") + +echo "--- COMPARISON RESULTS ---" +echo "${COMPARE_OUT}" +echo "--------------------------" + +DIFFERENT_COUNT=$(echo "${COMPARE_OUT}" | grep "^Different IDs:" | grep -oE '[0-9]+' | tail -1 || echo "0") +ONLY_FILE1_COUNT=$(echo "${COMPARE_OUT}" | awk '/^—— Only in File 1 ——$/,/^—— Only in File 2 ——$/ {print}' | grep -cE "^chatcmpl-" || echo "0") +ONLY_FILE2_COUNT=$(echo "${COMPARE_OUT}" | awk '/^—— Only in File 2 ——$/,/^$/ {print}' | grep -cE "^chatcmpl-" || echo "0") + +echo "[INFO] Analysis: Different=${DIFFERENT_COUNT}, Only in Base=${ONLY_FILE1_COUNT}, Only in LMCache=${ONLY_FILE2_COUNT}" + +if [[ "${DIFFERENT_COUNT}" -gt 0 ]] || [[ "${ONLY_FILE1_COUNT}" -gt 0 ]] || [[ "${ONLY_FILE2_COUNT}" -gt 0 ]]; then + echo "[FAIL] Inconsistency detected between Base and LMCache outputs." + exit 1 +fi + +echo "[PASS] All correctness tests passed -- identical output verified." diff --git a/.buildkite/k3_tests/integration/BK_WEB_SETUP.md b/.buildkite/k3_tests/integration/BK_WEB_SETUP.md new file mode 100644 index 0000000..8d2fcea --- /dev/null +++ b/.buildkite/k3_tests/integration/BK_WEB_SETUP.md @@ -0,0 +1,14 @@ +# Buildkite Web UI Setup: Integration Tests + +**Steps editor**: paste contents of `buildkite-pipeline.yml` (fill in `HF_TOKEN`). + +**GitHub trigger settings**: +- Filter: *(none — runs on every push/PR)* +- Skip queued / cancel running branch builds: Yes + +Lightweight (1 GPU) — good candidate for a required GitHub status check. + +> Builds whose only changes are docs/`*.md`/`LICENSE`/`.github/**` auto-pass +> via the [path filter](../README.md#path-based-skip-auto-pass-on-docs-only-changes). +> Changes under `.buildkite/` always run. Add `force-ci` label to the PR to +> bypass. \ No newline at end of file diff --git a/.buildkite/k3_tests/integration/buildkite-pipeline.yml b/.buildkite/k3_tests/integration/buildkite-pipeline.yml new file mode 100644 index 0000000..ffed4f5 --- /dev/null +++ b/.buildkite/k3_tests/integration/buildkite-pipeline.yml @@ -0,0 +1,11 @@ +# Paste this into the Buildkite pipeline's "Steps" editor. +# It uploads the real pipeline definition from the repo. +agents: + queue: "k8s" + +env: + HF_TOKEN: "" # set your HuggingFace token here (needed for gated model access) + +steps: + - label: ":pipeline: Upload pipeline" + command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/integration/pipeline.yml diff --git a/.buildkite/k3_tests/integration/pipeline.yml b/.buildkite/k3_tests/integration/pipeline.yml new file mode 100644 index 0000000..9d6c114 --- /dev/null +++ b/.buildkite/k3_tests/integration/pipeline.yml @@ -0,0 +1,49 @@ +# Integration tests — starts vLLM with LMCache directly (no Docker), +# tests CPU and disk backends via the OpenAI API. + +steps: + - label: ":test_tube: Integration — CPU backend" + command: .buildkite/k3_tests/integration/run.sh cpu + timeout_in_minutes: 20 + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - { name: hf-cache, mountPath: /root/.cache/huggingface } + volumes: + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + artifact_paths: + - "*.csv" + - "*.pdf" + - "*.log" + + - label: ":test_tube: Integration — Disk backend" + command: .buildkite/k3_tests/integration/run.sh disk + timeout_in_minutes: 20 + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - { name: hf-cache, mountPath: /root/.cache/huggingface } + volumes: + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + artifact_paths: + - "*.csv" + - "*.pdf" + - "*.log" diff --git a/.buildkite/k3_tests/integration/run.sh b/.buildkite/k3_tests/integration/run.sh new file mode 100755 index 0000000..c2d23d6 --- /dev/null +++ b/.buildkite/k3_tests/integration/run.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Integration test entrypoint for K8s pods. +# Thin wrapper: sets up environment, then delegates to scripts/. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +cd "${REPO_ROOT}" + +# ── Environment setup ──────────────────────────────────────── +source .buildkite/k3_harness/setup-env.sh +uv pip install aiohttp + +# ── Ensure all scripts are executable ──────────────────────── +chmod +x "${SCRIPT_DIR}"/scripts/*.sh + +# ── Run the actual test logic ──────────────────────────────── +exec bash "${SCRIPT_DIR}/scripts/run-integration.sh" "$@" diff --git a/.buildkite/k3_tests/integration/scripts/run-integration.sh b/.buildkite/k3_tests/integration/scripts/run-integration.sh new file mode 100755 index 0000000..3d805a2 --- /dev/null +++ b/.buildkite/k3_tests/integration/scripts/run-integration.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Integration test logic: starts vLLM with LMCache (CPU and disk backends), +# sends requests via the OpenAI API, and verifies responses. +# Runs directly in the K8s pod -- no Docker. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" + +cd "${REPO_ROOT}" +source .buildkite/k3_tests/common_scripts/helpers.sh + +TEST_NAME="${1:-all}" # "cpu", "disk", or "all" + +MODEL="meta-llama/Llama-3.2-1B-Instruct" +PORT=8000 +VLLM_PID="" + +cleanup() { + if [[ -n "$VLLM_PID" ]]; then + echo "--- Shutting down vLLM (PID=$VLLM_PID)" + kill "$VLLM_PID" 2>/dev/null || true + wait "$VLLM_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +send_request() { + local port=$1 prompt=$2 + curl -sf "http://localhost:${port}/v1/completions" \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"${MODEL}\", + \"prompt\": \"${prompt}\", + \"max_tokens\": 64, + \"temperature\": 0 + }" +} + +run_test() { + local test_name=$1 + shift + local env_vars=("$@") + + echo "--- :test_tube: ${test_name}" + + # Start vLLM with LMCache + env "${env_vars[@]}" \ + vllm serve "$MODEL" \ + --port "$PORT" \ + --load-format dummy \ + --enforce-eager \ + --no-enable-prefix-caching \ + --gpu-memory-utilization 0.8 \ + --kv-transfer-config '{"kv_connector":"LMCacheConnectorV1","kv_role":"kv_both"}' \ + > "${test_name}-vllm.log" 2>&1 & + VLLM_PID=$! + + wait_for_server "$PORT" 300 "${test_name}-vllm.log" + + # Send requests -- same prompt twice to exercise the cache (second hit should be cached) + local prompt="The quick brown fox jumps over the lazy dog. Explain this sentence in detail." + + echo "Sending request 1 (cold)..." + local resp1 + resp1=$(send_request "$PORT" "$prompt") + echo "Response 1: $(echo "$resp1" | python3 -c 'import sys,json; r=json.load(sys.stdin); print(r["choices"][0]["text"][:100])' 2>/dev/null || echo "PARSE_ERROR")" + + echo "Sending request 2 (should hit cache)..." + local resp2 + resp2=$(send_request "$PORT" "$prompt") + echo "Response 2: $(echo "$resp2" | python3 -c 'import sys,json; r=json.load(sys.stdin); print(r["choices"][0]["text"][:100])' 2>/dev/null || echo "PARSE_ERROR")" + + # Verify we got valid responses + for resp_name in resp1 resp2; do + local resp_val="${!resp_name}" + if ! echo "$resp_val" | python3 -c 'import sys,json; r=json.load(sys.stdin); assert r["choices"][0]["text"]' 2>/dev/null; then + echo "FAIL: ${test_name} -- invalid response in ${resp_name}" + echo "$resp_val" + kill "$VLLM_PID" 2>/dev/null; wait "$VLLM_PID" 2>/dev/null || true; VLLM_PID="" + return 1 + fi + done + + echo "PASS: ${test_name}" + + # Shut down for next test + kill "$VLLM_PID" 2>/dev/null; wait "$VLLM_PID" 2>/dev/null || true; VLLM_PID="" + sleep 2 +} + +# ── Test 1: CPU backend ─────────────────────────────────────── +if [[ "$TEST_NAME" == "cpu" || "$TEST_NAME" == "all" ]]; then + run_test "local_cpu" \ + "LMCACHE_CHUNK_SIZE=256" \ + "LMCACHE_LOCAL_CPU=True" \ + "LMCACHE_MAX_LOCAL_CPU_SIZE=5" +fi + +# ── Test 2: Disk backend ───────────────────────────────────── +if [[ "$TEST_NAME" == "disk" || "$TEST_NAME" == "all" ]]; then + DISK_DIR=$(mktemp -d) + trap "cleanup; rm -rf $DISK_DIR" EXIT + + run_test "local_disk" \ + "LMCACHE_CHUNK_SIZE=256" \ + "LMCACHE_LOCAL_CPU=True" \ + "LMCACHE_MAX_LOCAL_CPU_SIZE=5" \ + "LMCACHE_LOCAL_DISK=file://${DISK_DIR}" +fi + +echo "--- :white_check_mark: Integration tests passed (${TEST_NAME})" diff --git a/.buildkite/k3_tests/multiprocess/BK_WEB_SETUP.md b/.buildkite/k3_tests/multiprocess/BK_WEB_SETUP.md new file mode 100644 index 0000000..eb0244d --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/BK_WEB_SETUP.md @@ -0,0 +1,15 @@ +# Buildkite Web UI Setup: Multiprocess Tests + +**Steps editor**: paste contents of `buildkite-pipeline.yml` (fill in `HF_TOKEN`). + +**GitHub trigger settings**: +- Filter: `build.pull_request.labels includes "mp" || build.pull_request.labels includes "full" || build.branch == 'dev'` +- Rebuild on PR label change: Yes +- Skip queued / cancel running branch builds: Yes + +Heavy test (2 GPUs, Docker-in-Docker, ~45 min) — run on `"mp"`/`"full"` label or dev push, not every PR. + +> Builds whose only changes are docs/`*.md`/`LICENSE`/`.github/**` auto-pass +> via the [path filter](../README.md#path-based-skip-auto-pass-on-docs-only-changes). +> Changes under `.buildkite/` always run. Add `force-ci` label to the PR to +> bypass. diff --git a/.buildkite/k3_tests/multiprocess/buildkite-pipeline.yml b/.buildkite/k3_tests/multiprocess/buildkite-pipeline.yml new file mode 100644 index 0000000..0178e84 --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/buildkite-pipeline.yml @@ -0,0 +1,11 @@ +# Paste this into the Buildkite pipeline's "Steps" editor. +# It uploads the real pipeline definition from the repo. +agents: + queue: "k8s" + +env: + HF_TOKEN: "" # set your HuggingFace token here (needed for gated model access) + +steps: + - label: ":pipeline: Upload pipeline" + command: bash .buildkite/k3_tests/common_scripts/upload-pipeline.sh .buildkite/k3_tests/multiprocess/pipeline.yml diff --git a/.buildkite/k3_tests/multiprocess/pipeline.yml b/.buildkite/k3_tests/multiprocess/pipeline.yml new file mode 100644 index 0000000..bf9936b --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/pipeline.yml @@ -0,0 +1,279 @@ +# Multiprocess tests — one step per test, run in parallel. +# Each step gets its own K8s pod with the exact GPU count it needs. +# Tests are grouped by GPU requirement; the 2-GPU group is listed first so +# these heavier jobs get scheduled before 1-GPU jobs. +# 2-GPU: LMCache+vLLM on GPU 0, baseline on GPU 1 (or TP=2 across both). +# 1-GPU: LMCache+vLLM on GPU 0 only (no baseline server). + +# Retry policy for known-flaky long-running multiprocess steps. +# Note: exit_status "*" matches 1-255 only; -1 (agent lost) needs a separate rule. +x-flaky-retry: &flaky-retry + automatic: + - exit_status: -1 + limit: 2 + - exit_status: "*" + limit: 2 + +steps: + - group: ":compression: Multiprocess (2-GPU)" + if: build.env("VERIFY_AND_PIN_VLLM") !~ /true/ + steps: + - label: ":compression: vllm_bench" + command: .buildkite/k3_tests/multiprocess/run.sh vllm_bench + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: &pod-2gpu + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: { limits: { "nvidia.com/gpu": "2" } } + volumeMounts: &vol-mounts + - { name: hf-cache, mountPath: /root/.cache/huggingface } + volumes: &vols + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + artifact_paths: ["*.log"] + + - label: ":compression: long_doc_qa" + command: .buildkite/k3_tests/multiprocess/run.sh long_doc_qa + timeout_in_minutes: 30 + retry: *flaky-retry + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-2gpu } }] + artifact_paths: ["*.log"] + + - label: ":compression: long_doc_qa_l2" + command: .buildkite/k3_tests/multiprocess/run.sh long_doc_qa_l2 + timeout_in_minutes: 30 + retry: *flaky-retry + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-2gpu } }] + artifact_paths: ["*.log"] + + - label: ":compression: deadlock" + command: .buildkite/k3_tests/multiprocess/run.sh deadlock + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-2gpu } }] + artifact_paths: ["*.log"] + + # Two LMCache MP instances + two vLLM instances + one coordinator on a + # single 2-GPU pod (localhost). Verifies engine B serves a long prompt's + # KV from engine A over P2P. + - label: ":compression: p2p" + command: .buildkite/k3_tests/multiprocess/run.sh p2p + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-2gpu } }] + artifact_paths: ["*.log"] + + - group: ":compression: Multiprocess (1-GPU)" + if: build.env("VERIFY_AND_PIN_VLLM") !~ /true/ + steps: + - label: ":compression: lm_eval" + command: .buildkite/k3_tests/multiprocess/run.sh lm_eval + timeout_in_minutes: 30 + retry: *flaky-retry + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: &pod-1gpu + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: { limits: { "nvidia.com/gpu": "1" } } + volumeMounts: *vol-mounts + volumes: *vols + artifact_paths: ["*.log"] + + # Engine-driven transfer mode correctness on GPU (SHM transport). + # Default CUDA dispatch is lmcache_driven (server-side IPC handle); + # this step forces engine_driven (worker-side gather/scatter via SHM) + # and verifies gsm8k outputs are still bit-exact across cache-population + # and cache-hit runs. + # + # To enable SHM transport: + # - L1_USE_LAZY=false disables lazy allocation so L1 is backed by a + # POSIX SHM segment that the worker can mmap. + # - CPU_BUFFER_SIZE=20 caps L1 at 20GB (needs /dev/shm >= 20GB). + # - dshm volume provides a 24Gi tmpfs mount at /dev/shm. + - label: ":compression: lm_eval_engine_driven" + command: .buildkite/k3_tests/multiprocess/run.sh lm_eval + timeout_in_minutes: 30 + env: + LMCACHE_MP_TRANSFER_MODE: "engine_driven" + L1_USE_LAZY: "false" + CPU_BUFFER_SIZE: "80" + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: { limits: { "nvidia.com/gpu": "1" } } + volumeMounts: + - { name: hf-cache, mountPath: /root/.cache/huggingface } + - { name: dshm, mountPath: /dev/shm } + volumes: + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + - { name: dshm, emptyDir: { medium: Memory, sizeLimit: 128Gi } } + artifact_paths: ["*.log"] + + # HMA correctness check on google/gemma-4-31B-it (a hybrid model whose KV + # cache groups get different block sizes). Runs gsm8k, resets vLLM's prefix + # cache (LMCache preserved), reruns served by LMCache, and asserts the two + # runs' scores match. Single GPU, no baseline. + - label: ":compression: hma_lm_eval_gemma4" + command: .buildkite/k3_tests/multiprocess/run.sh hma_lm_eval_gemma4 + timeout_in_minutes: 60 + retry: *flaky-retry + env: + MODEL: "google/gemma-4-31B-it" + # Allow a small score difference between the two runs. + SCORE_TOLERANCE: "0.03" + ATTENTION_BACKEND: "auto" + GPU_MEMORY_UTILIZATION: "0.85" + # 31B load + CUDA-graph capture is slow; raise the readiness timeout + # above the 300s default. + MAX_WAIT_SECONDS: "600" + # Cap samples and enlarge the CPU pool so the retrieve run stays + # cache-served (31B's per-token KV is large). + LIMIT: "100" + CPU_BUFFER_SIZE: "200" + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log"] + + - label: ":compression: fault_tolerance" + command: .buildkite/k3_tests/multiprocess/run.sh fault_tolerance + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log"] + + - label: ":compression: restart_recovery" + command: .buildkite/k3_tests/multiprocess/run.sh restart_recovery + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log"] + + - label: ":compression: cache_stats" + command: .buildkite/k3_tests/multiprocess/run.sh cache_stats + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log"] + + - label: ":compression: http_api" + command: .buildkite/k3_tests/multiprocess/run.sh http_api + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: [{ kubernetes: { podSpec: *pod-1gpu } }] + artifact_paths: ["*.log"] + + - label: ":compression: gds_smoke_test" + command: .buildkite/k3_tests/multiprocess/run.sh gds_smoke_test + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: { limits: { "nvidia.com/gpu": "1" } } + volumeMounts: + - { name: hf-cache, mountPath: /root/.cache/huggingface } + - { name: scratch, mountPath: /scratch } + - { name: udev, mountPath: /run/udev, readOnly: true } + volumes: + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + - { name: scratch, hostPath: { path: /data/gds-scratch, type: DirectoryOrCreate } } + - { name: udev, hostPath: { path: /run/udev, type: Directory } } + artifact_paths: ["*.log"] + + - group: ":compression: Multiprocess (CPU-only)" + if: build.env("VERIFY_AND_PIN_VLLM") !~ /true/ + steps: + # Note: LMCACHE_SHM_NAME uses `${VAR-default}` (not `:-`) in the + # runner script, so an empty string ("") is meaningful and differs + # from unset. `__default__` is the sentinel meaning "shm mode". + - label: ":compression: cpu_e2e_validation ({{matrix.mode}})" + # matrix.adjustments does not support `env:` in Buildkite. Instead + # we branch on {{matrix.mode}} inside the command to set the + # per-mode env before invoking the shared runner script. + command: | + case "{{matrix.mode}}" in + shm) ;; # rely on script defaults (engine-driven + shm transport) + pickle) export LMCACHE_SHM_NAME="" ;; + server_copy) export LMCACHE_MP_TRANSFER_MODE="lmcache_driven" ;; + esac + bash .github/scripts/run-cpu-e2e-validation.sh + matrix: + setup: + mode: + - "shm" + - "pickle" + - "server_copy" + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: + requests: + cpu: "8" + memory: "256Gi" + limits: + cpu: "8" + memory: "256Gi" + volumeMounts: + - { name: hf-cache, mountPath: /root/.cache/huggingface } + - { name: dshm, mountPath: /dev/shm } + volumes: + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + - { name: dshm, emptyDir: { medium: Memory, sizeLimit: 4Gi } } + + # Canary path for VERIFY_AND_PIN_VLLM=true: skips every real test group + # above (each group's `if` evaluates to false) and instead runs a single + # vllm_bench step that proves the freshly-installed vLLM nightly works + # with LMCache. On success, the verified vllm wheel version is pushed to + # the `buildkite_latest_tested_vllm` branch so downstream builds can pin + # to a known-good nightly instead of resolving "latest" again. + - label: ":pushpin: Verify & pin vLLM nightly" + if: build.env("VERIFY_AND_PIN_VLLM") =~ /true/ + command: | + .buildkite/k3_tests/multiprocess/run.sh vllm_bench + bash .buildkite/k3_tests/multiprocess/scripts/pin-tested-vllm.sh + timeout_in_minutes: 30 + agents: { queue: "k8s" } + plugins: + - kubernetes: + podSpec: + containers: + - name: container-0 + image: lmcache/ci-base:latest + imagePullPolicy: Never + resources: { limits: { "nvidia.com/gpu": "2" } } + env: + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: buildkite-git-creds + key: GITHUB_TOKEN + volumeMounts: + - { name: hf-cache, mountPath: /root/.cache/huggingface } + volumes: + - { name: hf-cache, hostPath: { path: /data/huggingface, type: DirectoryOrCreate } } + artifact_paths: ["*.log"] diff --git a/.buildkite/k3_tests/multiprocess/run.sh b/.buildkite/k3_tests/multiprocess/run.sh new file mode 100755 index 0000000..88df1cd --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/run.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Multiprocess test entrypoint for K8s pods. +# Usage: run.sh +# test_name: lm_eval | hma_lm_eval_gemma4 | vllm_bench | long_doc_qa +# | long_doc_qa_l2 | fault_tolerance | deadlock | restart_recovery +# | gds_smoke_test | p2p +# Thin wrapper: sets up environment, then delegates to scripts/. +# No Docker -- all processes run natively in the pod. +set -euo pipefail + +TEST_NAME="${1:?Usage: $0 (lm_eval|hma_lm_eval_gemma4|vllm_bench|long_doc_qa|long_doc_qa_l2|fault_tolerance|deadlock|restart_recovery|cache_stats|http_api|p2p)}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +cd "${REPO_ROOT}" + +# ── Environment setup ──────────────────────────────────────── +source .buildkite/k3_harness/setup-env.sh + +# Install test extras (lm-eval for eval workload, openai/pandas/matplotlib for benchmarks) +uv pip install 'lm-eval[api]' openai pandas matplotlib + +# ── Ensure all scripts are executable ──────────────────────── +chmod +x "${SCRIPT_DIR}"/scripts/*.sh + +# ── Run the actual test logic ──────────────────────────────── +exec bash "${SCRIPT_DIR}/scripts/run-single-test.sh" "$TEST_NAME" diff --git a/.buildkite/k3_tests/multiprocess/scripts/cleanup.sh b/.buildkite/k3_tests/multiprocess/scripts/cleanup.sh new file mode 100755 index 0000000..2a61601 --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/scripts/cleanup.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Cleanup background processes launched for multiprocessing tests. +# This script should always be called, even on failure. + +BUILD_ID="${BUILD_ID:-local_$$}" +PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}" + +echo "=== Cleaning up background processes ===" + +if [[ -f "$PID_FILE" ]]; then + while IFS= read -r pid; do + if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then + echo "Killing PID $pid" + # Capture a snippet of the process log before killing + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + echo "PID $pid stopped" + fi + done < "$PID_FILE" + rm -f "$PID_FILE" +else + echo "No PID file found at $PID_FILE" +fi + +# Also kill any stray vllm/lmcache processes from this build +# (safety net in case PIDs weren't recorded) +for port in "${VLLM_PORT:-8000}" "${VLLM_BASELINE_PORT:-9000}" "${LMCACHE_PORT:-6555}"; do + fuser -k "${port}/tcp" 2>/dev/null || true +done + +# Remove the GDS slab scratch dir (only set for gds_* tests). It lives on the +# /scratch hostPath (host-local NVMe), so it persists past the pod and the +# preallocated slab is large -- drop it now that the server is stopped. +if [[ -n "${GDS_L1_PATH:-}" ]]; then + echo "Removing GDS slab dir: $GDS_L1_PATH" + rm -rf "${GDS_L1_PATH}" 2>/dev/null || true +fi + +echo "=== Cleanup complete ===" + +# Copy server logs to the workspace so Buildkite can collect them as artifacts +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" +cp /tmp/build_${BUILD_ID}_*.log "${REPO_ROOT}/" 2>/dev/null || true + +# Wait for GPU memory to be fully released +echo "Waiting 5 seconds for GPU memory to be released..." +sleep 5 diff --git a/.buildkite/k3_tests/multiprocess/scripts/launch-processes.sh b/.buildkite/k3_tests/multiprocess/scripts/launch-processes.sh new file mode 100755 index 0000000..4e40052 --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/scripts/launch-processes.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# Launch LMCache MP server, vLLM with LMCache, and vLLM baseline +# as native background processes (no Docker). +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" + +source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh" + +# Configuration (inherited from run-mp-test.sh) +LMCACHE_PORT="${LMCACHE_PORT:-6555}" +vllm_port="${VLLM_PORT:-8000}" +vllm_baseline_port="${VLLM_BASELINE_PORT:-9000}" +CPU_BUFFER_SIZE="${CPU_BUFFER_SIZE:-80}" +MAX_WORKERS="${MAX_WORKERS:-4}" +MODEL="${MODEL:-Qwen/Qwen3-14B}" +BUILD_ID="${BUILD_ID:-local_$$}" + +# K8s assigns exactly 2 GPUs as devices 0 and 1 (overridable for local runs). +GPU_FOR_VLLM="${GPU_FOR_VLLM:-0}" +GPU_FOR_BASELINE="${GPU_FOR_BASELINE:-1}" +echo "Using GPU $GPU_FOR_VLLM for vLLM with LMCache" +echo "Using GPU $GPU_FOR_BASELINE for vLLM baseline" + +# Check GPU memory and set gpu-memory-utilization for very large GPUs. +# Without this, vLLM allocates so much KV cache that APC covers all prefixes +# and LMCache's cache path is never exercised, making the test pass vacuously. +GPU_MEMORY_UTIL_ARG="" +GPU_MEMORY_MB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits -i "${GPU_FOR_VLLM}" | tr -d ' ') +GPU_MEMORY_GB=$((GPU_MEMORY_MB / 1024)) +echo "Detected GPU memory: ${GPU_MEMORY_GB}GB (${GPU_MEMORY_MB}MB)" + +if [ -n "${GPU_MEMORY_UTILIZATION:-}" ]; then + # Explicit override (e.g. large models like gemma-4-31B whose ~63GB of + # weights alone exceed the default 0.5 fraction and would fail to load). + echo "Using configured --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION}" + GPU_MEMORY_UTIL_ARG="--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION}" +elif [ "$GPU_MEMORY_GB" -gt 90 ]; then + echo "GPU memory > 90GB, adding --gpu-memory-utilization 0.5" + GPU_MEMORY_UTIL_ARG="--gpu-memory-utilization 0.5" +fi + +# Attention backend for both vLLM servers. Defaults to FLASH_ATTN (what the +# batch-invariant lm_eval needs). Models with heterogeneous head dimensions +# (e.g. gemma-4) must NOT pin FLASH_ATTN -- set ATTENTION_BACKEND=auto so vLLM +# selects the backend itself (gemma-4 auto-forces TRITON_ATTN). +ATTENTION_BACKEND="${ATTENTION_BACKEND:-FLASH_ATTN}" +ATTENTION_BACKEND_ARG="" +if [ -n "$ATTENTION_BACKEND" ] && [ "$ATTENTION_BACKEND" != "auto" ]; then + ATTENTION_BACKEND_ARG="--attention-backend $ATTENTION_BACKEND" +fi + +# Optionally run vLLM in eager mode (skip CUDA graph capture) for both servers. +# Off by default: verified to break the bit-exact run1 == run2 check in the +# determinism tests (lm_eval) -- eager changes the kernel path enough to diverge +# across the cold/warm batch difference even under VLLM_BATCH_INVARIANT. Enable +# (ENFORCE_EAGER=1) only for large models whose CUDA-graph capture would +# otherwise time out at launch (those tests use a tolerance, not bit-exactness). +ENFORCE_EAGER_ARG="" +if [ "${ENFORCE_EAGER:-0}" = "1" ] || [ "${ENFORCE_EAGER:-0}" = "true" ]; then + ENFORCE_EAGER_ARG="--enforce-eager" +fi + +# Pin max model length for both servers; defaults to "auto" (vLLM derives the +# largest length that fits KV memory). Verified not to affect the bit-exact +# determinism tests. Override via MAX_MODEL_LEN if a model needs a fixed length. +MAX_MODEL_LEN="${MAX_MODEL_LEN:-auto}" +MAX_MODEL_LEN_ARG="--max-model-len ${MAX_MODEL_LEN}" + +# LMCache server chunk size in tokens. Empty -> server default. +CHUNK_SIZE_ARG="" +if [ -n "${CHUNK_SIZE:-}" ]; then + CHUNK_SIZE_ARG="--chunk-size ${CHUNK_SIZE}" +fi + +# vLLM batch-invariant mode. On by default; GDN/Mamba backends do not support it. +BATCH_INVARIANT="${BATCH_INVARIANT:-1}" + +# Mamba KV cache mode + prefix caching, set only for hybrid Mamba models. +MAMBA_ARGS="" +if [ -n "${MAMBA_CACHE_MODE:-}" ]; then + MAMBA_ARGS="--mamba-cache-mode ${MAMBA_CACHE_MODE} --enable-prefix-caching" +fi + +# Max tokens per scheduler step. Empty -> vLLM default. +MAX_NUM_BATCHED_TOKENS_ARG="" +if [ -n "${MAX_NUM_BATCHED_TOKENS:-}" ]; then + MAX_NUM_BATCHED_TOKENS_ARG="--max-num-batched-tokens ${MAX_NUM_BATCHED_TOKENS}" +fi + +# L1 lazy allocation mode. Default is lazy (--l1-use-lazy). Set L1_USE_LAZY=false +# to disable lazy allocation, which enables POSIX SHM-backed L1 pool for the +# engine_driven SHM transfer path. When lazy is enabled (default), the SHM pool +# is disabled and engine_driven falls back to pickle transport. +L1_LAZY_ARG="" +if [ "${L1_USE_LAZY:-true}" = "false" ]; then + L1_LAZY_ARG="--no-l1-use-lazy" + echo "L1 lazy allocation disabled (SHM transport enabled)" +fi + +# Store PIDs in a file so cleanup.sh can find them +PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}" +> "$PID_FILE" + +# ── 1. LMCache Multiprocess Server ────────────────────────── +echo "=== Launching LMCache MP server ===" +echo "Port: $LMCACHE_PORT" + +# Optional GDS L1 slab tier (gds_* tests). When GDS_L1_PATH is set, the L1 +# medium becomes an NVMe slab accessed via cuFile DMA instead of pinned DRAM; +# --l1-size-gb then sizes the slab. The path must be on a GDS-capable +# filesystem (local NVMe), provided by the /scratch hostPath mount. +GDS_L1_ARG="" +if [ -n "${GDS_L1_PATH:-}" ]; then + echo "GDS L1 tier enabled; slab directory: $GDS_L1_PATH" + GDS_L1_ARG="--gds-l1-path ${GDS_L1_PATH}" +fi + +CUDA_VISIBLE_DEVICES="${GPU_FOR_VLLM}" \ +lmcache server \ + --l1-size-gb "$CPU_BUFFER_SIZE" \ + --eviction-policy LRU \ + --max-workers "$MAX_WORKERS" \ + $CHUNK_SIZE_ARG \ + --port "$LMCACHE_PORT" \ + ${GDS_L1_ARG} \ + ${L1_LAZY_ARG} \ + > "/tmp/build_${BUILD_ID}_lmcache.log" 2>&1 & + +LMCACHE_PID=$! +echo "$LMCACHE_PID" >> "$PID_FILE" +echo "LMCache MP server started (PID=$LMCACHE_PID)" + +# Wait for LMCache to initialize +echo "Waiting for LMCache to initialize..." +sleep 10 + +# Unset VLLM_PORT so vLLM's internal get_open_port() picks a random +# ephemeral port for torch.distributed instead of trying serving_port+1. +# Without this, both instances fight over the same internal port. +unset VLLM_PORT + +# ── 2. vLLM with LMCache ──────────────────────────────────── +echo "=== Launching vLLM with LMCache ===" +echo "Model: $MODEL" +echo "Port: $vllm_port" + +CUDA_VISIBLE_DEVICES="${GPU_FOR_VLLM}" \ +VLLM_ENABLE_V1_MULTIPROCESSING=0 \ +VLLM_SERVER_DEV_MODE=1 \ +VLLM_BATCH_INVARIANT=${BATCH_INVARIANT} \ +PYTHONHASHSEED=0 \ +vllm serve "$MODEL" \ + --kv-transfer-config "{\"kv_connector\":\"LMCacheMPConnector\", \"kv_role\":\"kv_both\", \"kv_load_failure_policy\": \"recompute\", \"kv_connector_extra_config\": {\"lmcache.mp.port\": $LMCACHE_PORT, \"lmcache.mp.mq_timeout\": 10}}" \ + $ATTENTION_BACKEND_ARG \ + --port "$vllm_port" \ + --no-async-scheduling \ + $MAX_MODEL_LEN_ARG \ + $ENFORCE_EAGER_ARG \ + $GPU_MEMORY_UTIL_ARG \ + $MAMBA_ARGS \ + $MAX_NUM_BATCHED_TOKENS_ARG \ + > "/tmp/build_${BUILD_ID}_vllm.log" 2>&1 & + +VLLM_PID=$! +echo "$VLLM_PID" >> "$PID_FILE" +echo "vLLM with LMCache started (PID=$VLLM_PID)" + +# ── 3. vLLM Baseline (without LMCache) ────────────────────── +# Only launched for tests that compare against a baseline (2-GPU pods). +# Single-GPU tests set LAUNCH_BASELINE=false and skip this entirely. +if [[ "${LAUNCH_BASELINE:-true}" == "true" ]]; then + echo "=== Launching vLLM baseline ===" + echo "Port: $vllm_baseline_port" + + CUDA_VISIBLE_DEVICES="${GPU_FOR_BASELINE}" \ + VLLM_ENABLE_V1_MULTIPROCESSING=0 \ + VLLM_SERVER_DEV_MODE=1 \ + VLLM_BATCH_INVARIANT=1 \ + PYTHONHASHSEED=0 \ + vllm serve "$MODEL" \ + $ATTENTION_BACKEND_ARG \ + --port "$vllm_baseline_port" \ + --no-async-scheduling \ + $MAX_MODEL_LEN_ARG \ + $ENFORCE_EAGER_ARG \ + $GPU_MEMORY_UTIL_ARG \ + > "/tmp/build_${BUILD_ID}_vllm_baseline.log" 2>&1 & + + VLLM_BASELINE_PID=$! + echo "$VLLM_BASELINE_PID" >> "$PID_FILE" + echo "vLLM baseline started (PID=$VLLM_BASELINE_PID)" +else + echo "=== Skipping vLLM baseline (LAUNCH_BASELINE=false, 1-GPU test) ===" +fi + +echo "=== All processes launched ===" +echo "PIDs: LMCache=$LMCACHE_PID, vLLM=$VLLM_PID, Baseline=${VLLM_BASELINE_PID:-skipped}" diff --git a/.buildkite/k3_tests/multiprocess/scripts/pin-tested-vllm.sh b/.buildkite/k3_tests/multiprocess/scripts/pin-tested-vllm.sh new file mode 100644 index 0000000..9a5c3c8 --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/scripts/pin-tested-vllm.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +# Pin the currently-installed vLLM nightly to the +# `buildkite_latest_tested_vllm` branch. +# +# Runs ONLY when the canary vllm_bench step has succeeded and +# VERIFY_AND_PIN_VLLM=true. Records the just-verified vllm wheel version so +# downstream builds can install the same version deterministically instead +# of resolving "latest nightly" again. +# +# Two files are maintained on the dedicated branch: +# tested_vllm_versions.csv -- JSON Lines, append-only history. Each line +# is one self-contained record. +# latest_tested_vllm.txt -- Plain text, single line: the most recent +# verified version. Overwritten every run. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +cd "${REPO_ROOT}" + +export GIT_TERMINAL_PROMPT=0 + +# ── Resolve the version that's actually installed in this pod ─────────── +VLLM_VERSION="$(python -c 'import vllm; print(vllm.__version__)')" +if [[ -z "${VLLM_VERSION}" ]]; then + echo "[ERROR] could not read vllm.__version__ from the live env" >&2 + exit 1 +fi +echo "Verified vLLM version: ${VLLM_VERSION}" + +# ── Resolve commit SHAs so consumers don't need to call any external API ─ +# The PEP 440 local version after `+g` is the short commit SHA, e.g. +# 0.23.1rc1.dev508+gc6dd32a81 -> c6dd32a81. Expand it to the full 40-char +# SHA via the public GitHub commits API; we already have GITHUB_TOKEN in +# the env (5000 req/h). The full SHA gives us the permanent +# wheels.vllm.ai/// archive URL, which keeps working even +# after the rolling nightly index has dropped the wheel. +VLLM_SHORT_SHA="${VLLM_VERSION##*+g}" +if [[ "${VLLM_SHORT_SHA}" == "${VLLM_VERSION}" \ + || ! "${VLLM_SHORT_SHA}" =~ ^[0-9a-f]+$ ]]; then + VLLM_SHORT_SHA="" +fi + +VLLM_FULL_SHA="" +if [[ -n "${VLLM_SHORT_SHA}" ]]; then + gh_auth_args=() + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + gh_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}") + fi + for attempt in 1 2 3; do + VLLM_FULL_SHA="$(curl -fsSL --connect-timeout 5 --max-time 10 \ + -H "Accept: application/vnd.github+json" \ + "${gh_auth_args[@]+"${gh_auth_args[@]}"}" \ + "https://api.github.com/repos/vllm-project/vllm/commits/${VLLM_SHORT_SHA}" \ + 2>/dev/null \ + | jq -r '.sha // empty')" || true + if [[ "${VLLM_FULL_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + break + fi + VLLM_FULL_SHA="" + echo "[INFO] GitHub commit lookup attempt ${attempt} for" \ + "${VLLM_SHORT_SHA} returned no SHA; retrying..." >&2 + sleep 2 + done +fi + +if [[ -n "${VLLM_FULL_SHA}" ]]; then + VLLM_ARCHIVE_INDEX="https://wheels.vllm.ai/${VLLM_FULL_SHA}/cu130" + echo "Resolved full SHA: ${VLLM_FULL_SHA}" + echo "Archive index: ${VLLM_ARCHIVE_INDEX}" +else + VLLM_ARCHIVE_INDEX="" + echo "[WARN] could not resolve full SHA for short SHA" \ + "'${VLLM_SHORT_SHA:-}'; archive_index_url will be empty" \ + "and consumers will fall back to live API lookup" >&2 +fi + +CI_REPO="LMCache/LMCache" +CI_BRANCH="buildkite_latest_tested_vllm" + +if [[ -n "${GITHUB_TOKEN:-}" ]]; then + CI_REPO_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${CI_REPO}.git" +else + echo "[WARN] GITHUB_TOKEN not set — push will likely fail" >&2 + CI_REPO_URL="https://github.com/${CI_REPO}.git" +fi + +WORK_DIR="/tmp/pin_vllm_$$" +trap 'rm -rf "${WORK_DIR}"' EXIT + +echo "--- Preparing ${CI_BRANCH} branch from ${CI_REPO}" +if ! git clone --depth=1 --branch "${CI_BRANCH}" "${CI_REPO_URL}" \ + "${WORK_DIR}" 2>/dev/null; then + # Branch does not exist yet -- create an orphan with no parent history. + rm -rf "${WORK_DIR}" + mkdir -p "${WORK_DIR}" + git -C "${WORK_DIR}" init -q + git -C "${WORK_DIR}" remote add origin "${CI_REPO_URL}" + git -C "${WORK_DIR}" checkout --orphan "${CI_BRANCH}" + # Drop anything that the orphan checkout might have staged from HEAD. + git -C "${WORK_DIR}" rm -rf --cached . >/dev/null 2>&1 || true + find "${WORK_DIR}" -mindepth 1 -maxdepth 1 ! -name ".git" \ + -exec rm -rf {} + +fi + +# ── Update files ──────────────────────────────────────────────────────── +HISTORY_FILE="${WORK_DIR}/tested_vllm_versions.csv" +LATEST_FILE="${WORK_DIR}/latest_tested_vllm.txt" + +TIMESTAMP="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +BUILD_URL="${BUILDKITE_BUILD_URL:-}" +BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-}" +COMMIT_SHA="${BUILDKITE_COMMIT:-}" + +# Append-only history (JSON Lines). Built via python so quoting is safe. +# Use a quoted heredoc (<<'PY') to disable Bash expansion inside the script, +# and pass variables via the environment to avoid syntax errors from special +# characters in values like BUILD_URL. +TIMESTAMP="${TIMESTAMP}" \ +VLLM_VERSION="${VLLM_VERSION}" \ +VLLM_SHORT_SHA="${VLLM_SHORT_SHA}" \ +VLLM_FULL_SHA="${VLLM_FULL_SHA}" \ +VLLM_ARCHIVE_INDEX="${VLLM_ARCHIVE_INDEX}" \ +BUILD_NUMBER="${BUILD_NUMBER}" \ +BUILD_URL="${BUILD_URL}" \ +COMMIT_SHA="${COMMIT_SHA}" \ +python - "$HISTORY_FILE" <<'PY' +import json, os, sys +path = sys.argv[1] +record = { + "timestamp": os.environ.get("TIMESTAMP", ""), + "vllm_version": os.environ.get("VLLM_VERSION", ""), + "vllm_short_sha": os.environ.get("VLLM_SHORT_SHA", ""), + "vllm_full_sha": os.environ.get("VLLM_FULL_SHA", ""), + "archive_index_url": os.environ.get("VLLM_ARCHIVE_INDEX", ""), + "build_number": os.environ.get("BUILD_NUMBER", ""), + "build_url": os.environ.get("BUILD_URL", ""), + "commit": os.environ.get("COMMIT_SHA", ""), +} +with open(path, "a", encoding="utf-8") as f: + f.write(json.dumps(record) + "\n") +PY + +# Latest pointer. The first line is the bare version so older consumers +# that just `head -n1` keep working; the trailing key=value lines let new +# consumers skip the live GitHub API lookup entirely. +{ + printf '%s\n' "${VLLM_VERSION}" + printf 'short_sha=%s\n' "${VLLM_SHORT_SHA}" + printf 'full_sha=%s\n' "${VLLM_FULL_SHA}" + printf 'archive_index_url=%s\n' "${VLLM_ARCHIVE_INDEX}" +} > "${LATEST_FILE}" + +# ── Commit + push ─────────────────────────────────────────────────────── +cd "${WORK_DIR}" +git add tested_vllm_versions.csv latest_tested_vllm.txt + +if git diff --cached --quiet 2>/dev/null; then + echo "No changes to commit (version unchanged?)." + exit 0 +fi + +git -c user.email="ci@lmcache.ai" -c user.name="LMCache CI" \ + commit -m "Pin verified vLLM nightly: ${VLLM_VERSION}" + +echo "--- Pushing to ${CI_REPO} ${CI_BRANCH}" +if ! git push origin "HEAD:${CI_BRANCH}" 2>/dev/null; then + echo "[WARN] Normal push failed, force-pushing..." >&2 + git push origin "+HEAD:${CI_BRANCH}" 2>/dev/null || { + echo "[ERROR] Failed to push to ${CI_REPO} ${CI_BRANCH}" >&2 + exit 1 + } +fi + +echo "--- Pinned vLLM ${VLLM_VERSION} successfully" +git log --oneline -1 diff --git a/.buildkite/k3_tests/multiprocess/scripts/run-cache-stats.sh b/.buildkite/k3_tests/multiprocess/scripts/run-cache-stats.sh new file mode 100755 index 0000000..5129767 --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/scripts/run-cache-stats.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# Test that kv_transfer_params / cached_token_stats flows end-to-end +# through the OpenAI-compatible API when LMCache MP mode is active. +# +# Flow: +# 1. Send a long prompt (cold — populates LMCache, no cache hit) +# 2. Send the same prompt again (warm — should hit LMCache) +# 3. Verify the response contains cached_token_stats with expected values +set -e +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" + +source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh" + +# Configuration (inherited from run-single-test.sh) +VLLM_PORT="${VLLM_PORT:-8000}" +MODEL="${MODEL:-Qwen/Qwen3-14B}" +BUILD_ID="${BUILD_ID:-local_$$}" +RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}" + +STATS_DIR="$RESULTS_DIR/cache_stats" +mkdir -p "$STATS_DIR" + +echo "=== Cache Stats Reporting Test ===" +echo "Model: $MODEL" +echo "vLLM Port: $VLLM_PORT" +echo "Results dir: $STATS_DIR" +echo "" + +# Build a prompt long enough to span multiple LMCache chunks (default +# chunk_size=256 tokens). Repeating a sentence gives us ~600+ tokens. +LONG_CONTENT="Explain the history of computer science in great detail. $(printf 'The Turing machine is a fundamental concept in theoretical computer science that defines an abstract machine capable of manipulating symbols on a strip of tape according to a table of rules. %.0s' {1..20})" + +send_request() { + local label="$1" + local output_file="$2" + + echo "--- Sending request: $label ---" + local http_code + http_code=$(curl -s -o "$output_file" -w "%{http_code}" \ + -X POST "http://localhost:${VLLM_PORT}/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"${MODEL}\", + \"messages\": [{\"role\": \"user\", \"content\": $(python3 -c "import json; print(json.dumps('$LONG_CONTENT'))")}], + \"max_tokens\": 1, + \"kv_transfer_params\": {\"cached_token_stats\": true} + }") + + if [ "$http_code" -ne 200 ]; then + echo "FAIL: $label returned HTTP $http_code" + cat "$output_file" + return 1 + fi + echo "$label: HTTP 200 OK" +} + +validate_stats_present() { + local label="$1" + local response_file="$2" + + python3 -c " +import json, sys + +with open('$response_file') as f: + data = json.load(f) + +kv_params = data.get('kv_transfer_params') +if kv_params is None: + print('FAIL: $label — kv_transfer_params is missing from response') + sys.exit(1) + +stats = kv_params.get('cached_token_stats') +if stats is None: + print('FAIL: $label — cached_token_stats is missing from kv_transfer_params') + print(f' kv_transfer_params = {kv_params}') + sys.exit(1) + +required_keys = [ + 'num_vllm_cached_tokens', + 'num_lmcache_cached_tokens', + 'num_lmcache_extra_cached_tokens', +] +missing = [k for k in required_keys if k not in stats] +if missing: + print(f'FAIL: $label — missing keys in cached_token_stats: {missing}') + print(f' cached_token_stats = {stats}') + sys.exit(1) + +for k in required_keys: + v = stats[k] + if not isinstance(v, int) or v < 0: + print(f'FAIL: $label — {k} should be a non-negative integer, got {v!r}') + sys.exit(1) + +print(f'PASS: $label — cached_token_stats present with all required keys') +print(f' num_vllm_cached_tokens: {stats[\"num_vllm_cached_tokens\"]}') +print(f' num_lmcache_cached_tokens: {stats[\"num_lmcache_cached_tokens\"]}') +print(f' num_lmcache_extra_cached_tokens: {stats[\"num_lmcache_extra_cached_tokens\"]}') +" +} + +validate_warm_hit() { + local cold_file="$1" + local warm_file="$2" + + python3 -c " +import json, sys + +with open('$cold_file') as f: + cold = json.load(f) +with open('$warm_file') as f: + warm = json.load(f) + +cold_stats = cold['kv_transfer_params']['cached_token_stats'] +warm_stats = warm['kv_transfer_params']['cached_token_stats'] + +cold_lmcache = cold_stats['num_lmcache_cached_tokens'] +warm_lmcache = warm_stats['num_lmcache_cached_tokens'] + +print(f'Cold request — num_lmcache_cached_tokens: {cold_lmcache}') +print(f'Warm request — num_lmcache_cached_tokens: {warm_lmcache}') + +if warm_lmcache <= cold_lmcache: + print(f'FAIL: warm request should have more LMCache hits than cold request') + print(f' cold={cold_lmcache}, warm={warm_lmcache}') + sys.exit(1) + +if warm_lmcache == 0: + print('FAIL: warm request has 0 LMCache cached tokens (cache not populated?)') + sys.exit(1) + +print(f'PASS: warm request has more LMCache hits ({warm_lmcache} > {cold_lmcache})') +" +} + +# ── Step 1: Cold request (populates LMCache) ────────────────── +echo "============================================" +echo "=== Step 1: Cold request ===" +echo "============================================" +if ! send_request "Cold" "$STATS_DIR/cold_response.json"; then + exit 1 +fi +if ! validate_stats_present "Cold" "$STATS_DIR/cold_response.json"; then + exit 1 +fi +echo "" + +# Small delay to let the store operation complete in LMCache +sleep 2 + +# ── Step 2: Warm request (same prompt, should hit cache) ────── +echo "============================================" +echo "=== Step 2: Warm request ===" +echo "============================================" +if ! send_request "Warm" "$STATS_DIR/warm_response.json"; then + exit 1 +fi +if ! validate_stats_present "Warm" "$STATS_DIR/warm_response.json"; then + exit 1 +fi +echo "" + +# ── Step 3: Validate cache hit improvement ──────────────────── +echo "============================================" +echo "=== Step 3: Validate cache hit ===" +echo "============================================" +if ! validate_warm_hit "$STATS_DIR/cold_response.json" "$STATS_DIR/warm_response.json"; then + exit 1 +fi +echo "" + +# ── Step 4: Verify opt-in behavior ──────────────────────────── +# Request WITHOUT kv_transfer_params should NOT have stats in response. +echo "============================================" +echo "=== Step 4: Verify opt-in (no stats without opt-in) ===" +echo "============================================" + +echo "--- Sending request without kv_transfer_params ---" +http_code=$(curl -s -o "$STATS_DIR/no_opt_in_response.json" -w "%{http_code}" \ + -X POST "http://localhost:${VLLM_PORT}/v1/chat/completions" \ + -H "Content-Type: application/json" \ + -d "{ + \"model\": \"${MODEL}\", + \"messages\": [{\"role\": \"user\", \"content\": \"Hello, how are you?\"}], + \"max_tokens\": 1 + }") + +if [ "$http_code" -ne 200 ]; then + echo "FAIL: no-opt-in request returned HTTP $http_code" + exit 1 +fi + +python3 -c " +import json, sys + +with open('$STATS_DIR/no_opt_in_response.json') as f: + data = json.load(f) + +kv_params = data.get('kv_transfer_params') +if kv_params is not None: + print(f'FAIL: kv_transfer_params should be absent without opt-in, got {kv_params}') + sys.exit(1) + +print('PASS: kv_transfer_params correctly absent when not opted in') +" +echo "" + +# ── Summary ─────────────────────────────────────────────────── +echo "============================================" +echo "=== Cache Stats Reporting Test PASSED ===" +echo "============================================" diff --git a/.buildkite/k3_tests/multiprocess/scripts/run-deadlock.sh b/.buildkite/k3_tests/multiprocess/scripts/run-deadlock.sh new file mode 100644 index 0000000..77a84e2 --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/scripts/run-deadlock.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Self-contained deadlock regression test. +# +# Launches DeepSeek-V2-Lite-Chat with TP=2 (both GPUs) + LMCache server, +# sends 50 requests with ~30K token prefixes, and verifies they all +# complete within 3 minutes. A CUDA-driver/GIL deadlock would cause +# requests to hang indefinitely, failing the timeout. +# +# This test is self-contained: it handles its own server lifecycle +# instead of using the standard launch-processes.sh / wait-for-servers.sh. +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" + +source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh" + +# ── Configuration ─────────────────────────────────────────── +MODEL="deepseek-ai/DeepSeek-V2-Lite-Chat" +LMCACHE_PORT="${LMCACHE_PORT:-15554}" +VLLM_PORT="${VLLM_PORT:-8000}" +BUILD_ID="${BUILD_ID:-local_$$}" +PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}" +TIMEOUT_SECONDS=180 # 3 minutes + +# ── Install py-spy for deadlock diagnosis ────────────────── +echo "=== Installing py-spy ===" +uv pip install py-spy +PY_SPY="$(which py-spy)" +echo "py-spy installed at: $PY_SPY" + +PYSPY_LOG="/tmp/build_${BUILD_ID}_pyspy.log" + +# ── Helper: dump stacks of server processes via py-spy ───── +dump_stacks() { + echo "" | tee -a "$PYSPY_LOG" + echo "=== py-spy stack dump (native + Python) ===" | tee -a "$PYSPY_LOG" + + if kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "" | tee -a "$PYSPY_LOG" + echo "--- LMCache server (PID=$LMCACHE_PID) ---" | tee -a "$PYSPY_LOG" + sudo "$PY_SPY" dump --pid "$LMCACHE_PID" --native 2>&1 | tee -a "$PYSPY_LOG" || true + fi + + # Copy to repo root so cleanup.sh collects it as a Buildkite artifact + cp "$PYSPY_LOG" "${REPO_ROOT}/build_${BUILD_ID}_pyspy.log" 2>/dev/null || true +} + +# ── 1. Launch LMCache server ─────────────────────────────── +echo "=== Launching LMCache server ===" +echo "Port: $LMCACHE_PORT" + +lmcache server \ + --host localhost \ + --port "$LMCACHE_PORT" \ + --chunk-size 256 \ + --l1-size-gb 50 \ + --eviction-policy LRU \ + --max-workers 2 \ + > "/tmp/build_${BUILD_ID}_lmcache.log" 2>&1 & + +LMCACHE_PID=$! +echo "$LMCACHE_PID" >> "$PID_FILE" +echo "LMCache server started (PID=$LMCACHE_PID)" +sleep 10 + +# ── 2. Launch vLLM with DeepSeek TP=2 ───────────────────── +echo "=== Launching vLLM (DeepSeek TP=2) ===" +echo "Model: $MODEL" +echo "Port: $VLLM_PORT" + +# Save VLLM_PORT before unsetting — vLLM's internal get_open_port() +# would otherwise collide with the serving port for torch.distributed. +SAVED_VLLM_PORT="$VLLM_PORT" +unset VLLM_PORT + +FLASHINFER_DISABLE_VERSION_CHECK=1 \ +VLLM_SERVER_DEV_MODE=1 \ +vllm serve "$MODEL" \ + --tensor-parallel-size 2 \ + --distributed-executor-backend mp \ + --block-size 64 \ + --trust-remote-code \ + --load-format dummy \ + --enable-prefix-caching \ + --enable-chunked-prefill \ + --gpu-memory-utilization 0.8 \ + --max-model-len 65536 \ + --hf-overrides '{"max_position_embeddings":65536}' \ + --max-num-seqs 32 \ + --max-num-batched-tokens 16000 \ + --scheduling-policy fcfs \ + --port "$SAVED_VLLM_PORT" \ + --enforce-eager \ + --kv-transfer-config "{\"kv_connector\":\"LMCacheMPConnector\", \"kv_role\":\"kv_both\", \"kv_load_failure_policy\": \"recompute\", \"kv_connector_extra_config\": {\"lmcache.mp.port\": $LMCACHE_PORT, \"lmcache.mp.mq_timeout\": 60}}" \ + > "/tmp/build_${BUILD_ID}_vllm.log" 2>&1 & + +VLLM_PID=$! +echo "$VLLM_PID" >> "$PID_FILE" +echo "vLLM started (PID=$VLLM_PID)" + +VLLM_PORT="$SAVED_VLLM_PORT" + +# ── 3. Wait for vLLM to be ready ────────────────────────── +echo "=== Waiting for vLLM to be ready ===" +if ! wait_for_server "$VLLM_PORT" 600; then + echo "vLLM failed to start. Last 100 lines of log:" + tail -100 "/tmp/build_${BUILD_ID}_vllm.log" 2>/dev/null || true + exit 1 +fi + +# ── 4. Run benchmark with timeout ───────────────────────── +echo "=== Running lmcache bench engine (random-prefill, 50 reqs, ~30K tokens) ===" +echo "Timeout: ${TIMEOUT_SECONDS}s" + +if ! timeout "$TIMEOUT_SECONDS" lmcache bench engine \ + --engine-url "http://localhost:${VLLM_PORT}" \ + --workload random-prefill \ + --tokens-per-gb-kvcache 6000 \ + --rp-request-length 30000 \ + --rp-num-requests 50 \ + --no-interactive \ + --no-csv \ + -q; then + echo "FAIL: Benchmark failed or timed out (possible deadlock)" + echo "" + echo "=== LMCache log (last 50 lines) ===" + tail -50 "/tmp/build_${BUILD_ID}_lmcache.log" 2>/dev/null || true + echo "" + echo "=== vLLM log (last 50 lines) ===" + tail -50 "/tmp/build_${BUILD_ID}_vllm.log" 2>/dev/null || true + exit 1 +fi + +echo "" +echo "=== Benchmark completed within ${TIMEOUT_SECONDS}s ===" +echo "PASS: No deadlock detected" diff --git a/.buildkite/k3_tests/multiprocess/scripts/run-fault-tolerance.sh b/.buildkite/k3_tests/multiprocess/scripts/run-fault-tolerance.sh new file mode 100755 index 0000000..2bbfbae --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/scripts/run-fault-tolerance.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +# Test LMCache fault tolerance: verify vLLM requests complete after +# the LMCache MP server is killed mid-flight. +# +# Flow: +# 1. Run a warmup bench (measures baseline timing) +# 2. Run bench again, killing LMCache server mid-flight +# 3. Run bench fully without LMCache server +# 4. Verify all prompts completed in every phase +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" + +source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh" + +# Configuration (inherited from run-mp-test.sh) +VLLM_PORT="${VLLM_PORT:-8000}" +MODEL="${MODEL:-Qwen/Qwen3-14B}" +BUILD_ID="${BUILD_ID:-local_$$}" +RESULTS_DIR="${RESULTS_DIR:-/tmp/lmcache_ci_results_${BUILD_ID}}" +LMCACHE_PORT="${LMCACHE_PORT:-6555}" + +# Bench parameters +NUM_PROMPTS="${NUM_PROMPTS:-50}" +RANDOM_INPUT_LEN="${RANDOM_INPUT_LEN:-10000}" +RANDOM_OUTPUT_LEN="${RANDOM_OUTPUT_LEN:-1}" +RANDOM_SEED="${RANDOM_SEED:-42}" + +CPU_BUFFER_SIZE="${CPU_BUFFER_SIZE:-80}" +MAX_WORKERS="${MAX_WORKERS:-4}" + +# Output directory +FT_DIR="$RESULTS_DIR/fault_tolerance" +mkdir -p "$FT_DIR" + +echo "=== Fault Tolerance Test ===" +echo "Model: $MODEL" +echo "vLLM Port: $VLLM_PORT" +echo "LMCache Port: $LMCACHE_PORT" +echo "Bench: $NUM_PROMPTS prompts, input_len=$RANDOM_INPUT_LEN, output_len=$RANDOM_OUTPUT_LEN" +echo "Results dir: $FT_DIR" +echo "" + +# ── Step 0: Restart LMCache + vLLM with fresh state ───────── +# Previous steps may have left processes in an unknown state. +# Restart both so vLLM registers its GPU context with the new server. +echo "============================================" +echo "=== Restarting LMCache + vLLM ===" +echo "============================================" + +PID_FILE="/tmp/lmcache_mp_pids_${BUILD_ID}" +GPU_DEVICE="${GPU_FOR_VLLM:-0}" + +# Kill existing LMCache + vLLM (keep baseline on line 3) +if [ -f "$PID_FILE" ]; then + OLD_LMCACHE_PID=$(sed -n '1p' "$PID_FILE") + OLD_VLLM_PID=$(sed -n '2p' "$PID_FILE") + for pid in $OLD_LMCACHE_PID $OLD_VLLM_PID; do + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + echo "Killing PID $pid" + kill "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fi + done + sleep 2 +fi + +# Launch LMCache with L1 config +CUDA_VISIBLE_DEVICES="${GPU_DEVICE}" \ +lmcache server \ + --l1-size-gb "$CPU_BUFFER_SIZE" \ + --eviction-policy LRU \ + --max-workers "$MAX_WORKERS" \ + --port "$LMCACHE_PORT" \ + > "/tmp/build_${BUILD_ID}_lmcache_ft.log" 2>&1 & + +NEW_LMCACHE_PID=$! +echo "LMCache server started (PID=$NEW_LMCACHE_PID)" +sleep 10 + +# Launch vLLM with LMCache +GPU_MEMORY_UTIL_ARG="" +GPU_MEMORY_MB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits -i "${GPU_DEVICE}" | tr -d ' ') +GPU_MEMORY_GB=$((GPU_MEMORY_MB / 1024)) +if [ "$GPU_MEMORY_GB" -gt 90 ]; then + GPU_MEMORY_UTIL_ARG="--gpu-memory-utilization 0.5" +fi + +env -u VLLM_PORT \ + CUDA_VISIBLE_DEVICES="${GPU_DEVICE}" \ + VLLM_ENABLE_V1_MULTIPROCESSING=0 \ + VLLM_SERVER_DEV_MODE=1 \ + VLLM_BATCH_INVARIANT=1 \ + PYTHONHASHSEED=0 \ +vllm serve "$MODEL" \ + --kv-transfer-config "{\"kv_connector\":\"LMCacheMPConnector\", \"kv_role\":\"kv_both\", \"kv_load_failure_policy\": \"recompute\", \"kv_connector_extra_config\": {\"lmcache.mp.port\": $LMCACHE_PORT, \"lmcache.mp.mq_timeout\": 10}}" \ + --attention-backend FLASH_ATTN \ + --port "$VLLM_PORT" \ + --no-async-scheduling \ + $GPU_MEMORY_UTIL_ARG \ + > "/tmp/build_${BUILD_ID}_vllm_ft.log" 2>&1 & + +NEW_VLLM_PID=$! +echo "vLLM started (PID=$NEW_VLLM_PID)" + +# Update PID file +if [ -f "$PID_FILE" ]; then + sed -i "1s/.*/$NEW_LMCACHE_PID/" "$PID_FILE" + sed -i "2s/.*/$NEW_VLLM_PID/" "$PID_FILE" +else + echo "$NEW_LMCACHE_PID" > "$PID_FILE" + echo "$NEW_VLLM_PID" >> "$PID_FILE" +fi + +if ! wait_for_server "$VLLM_PORT" 300; then + echo "vLLM failed to start" + tail -50 "/tmp/build_${BUILD_ID}_lmcache_ft.log" || true + tail -50 "/tmp/build_${BUILD_ID}_vllm_ft.log" || true + exit 1 +fi +echo "" + +# ── Helpers ────────────────────────────────────────────────── + +run_bench() { + local description="$1" + local result_file="$2" + + echo "" + echo "--- $description ---" + + vllm bench serve \ + --seed "$RANDOM_SEED" \ + --port "$VLLM_PORT" \ + --model "$MODEL" \ + --dataset-name random \ + --random-input-len "$RANDOM_INPUT_LEN" \ + --random-output-len "$RANDOM_OUTPUT_LEN" \ + --num-prompts "$NUM_PROMPTS" \ + --ignore-eos \ + --backend openai-chat \ + --endpoint /v1/chat/completions \ + --result-dir "$FT_DIR" \ + --result-filename "$result_file" \ + --save-result + + local completed + completed=$(python3 -c " +import json +with open('$FT_DIR/$result_file') as f: + data = json.load(f) +print(data.get('completed', 0)) +") + + echo "$description: $completed / $NUM_PROMPTS completed" + + if [ "$completed" -ne "$NUM_PROMPTS" ]; then + echo "FAIL: Expected $NUM_PROMPTS completed, got $completed" + return 1 + fi + + echo "PASS: All $NUM_PROMPTS prompts completed" + return 0 +} + +get_lmcache_pid() { + local pid_file="/tmp/lmcache_mp_pids_${BUILD_ID}" + if [[ -f "$pid_file" ]]; then + head -1 "$pid_file" + fi +} + +# ── Step 1: Warmup bench ───────────────────────────────────── +echo "============================================" +echo "=== Fault Tolerance Step 1: Warmup bench ===" +echo "============================================" + +if ! run_bench "Warmup (with LMCache)" "ft_warmup.json"; then + echo "FAIL: Warmup bench failed" + exit 1 +fi + +# Extract duration to calibrate kill timing +WARMUP_DURATION=$(python3 -c "import json; print(json.load(open('$FT_DIR/ft_warmup.json'))['duration'])") +KILL_DELAY=$(python3 -c "print(max(3, int($WARMUP_DURATION * 0.4)))") +echo "Warmup took ${WARMUP_DURATION}s. Will kill LMCache after ${KILL_DELAY}s in next run." + +# ── Step 2: Bench with mid-flight LMCache kill ─────────────── +echo "" +echo "============================================" +echo "=== Fault Tolerance Step 2: Mid-flight kill ===" +echo "============================================" + +LMCACHE_PID=$(get_lmcache_pid) +if [ -z "$LMCACHE_PID" ] || ! kill -0 "$LMCACHE_PID" 2>/dev/null; then + echo "FAIL: LMCache server not running (PID=$LMCACHE_PID)" + exit 1 +fi + +echo "LMCache server PID: $LMCACHE_PID" +echo "Will kill after ${KILL_DELAY}s into bench." + +# Start bench in background +run_bench "Mid-flight kill" "ft_midflight.json" & +BENCH_PID=$! + +# Wait, then kill LMCache +sleep "$KILL_DELAY" +echo "Killing LMCache server (PID: $LMCACHE_PID)..." +kill "$LMCACHE_PID" 2>/dev/null +wait "$LMCACHE_PID" 2>/dev/null || true +echo "LMCache server killed at +${KILL_DELAY}s." + +# Wait for bench to finish +echo "Waiting for bench to complete..." +if ! wait "$BENCH_PID"; then + echo "FAIL: Bench did not complete after mid-flight LMCache kill." + echo "--- vLLM log (last 50 lines) ---" + tail -50 "/tmp/build_${BUILD_ID}_vllm.log" 2>/dev/null || true + exit 1 +fi + +# ── Step 3: Bench fully without LMCache server ─────────────── +echo "" +echo "============================================" +echo "=== Fault Tolerance Step 3: Without LMCache ===" +echo "============================================" + +if ! run_bench "Without LMCache" "ft_without_lmcache.json"; then + echo "FAIL: Bench failed without LMCache server." + echo "--- vLLM log (last 50 lines) ---" + tail -50 "/tmp/build_${BUILD_ID}_vllm.log" 2>/dev/null || true + exit 1 +fi + +# ── Summary ────────────────────────────────────────────────── +echo "" +echo "============================================" +echo "=== Fault Tolerance Test PASSED ===" +echo "============================================" + +warmup_completed=$(python3 -c "import json; print(json.load(open('$FT_DIR/ft_warmup.json'))['completed'])") +warmup_duration=$(python3 -c "import json; print(f\"{json.load(open('$FT_DIR/ft_warmup.json'))['duration']:.1f}\")") +midflight_completed=$(python3 -c "import json; print(json.load(open('$FT_DIR/ft_midflight.json'))['completed'])") +midflight_duration=$(python3 -c "import json; print(f\"{json.load(open('$FT_DIR/ft_midflight.json'))['duration']:.1f}\")") +without_completed=$(python3 -c "import json; print(json.load(open('$FT_DIR/ft_without_lmcache.json'))['completed'])") +without_duration=$(python3 -c "import json; print(f\"{json.load(open('$FT_DIR/ft_without_lmcache.json'))['duration']:.1f}\")") + +echo " Warmup (with LMCache): $warmup_completed/$NUM_PROMPTS in ${warmup_duration}s" +echo " Mid-flight kill: $midflight_completed/$NUM_PROMPTS in ${midflight_duration}s (killed at +${KILL_DELAY}s)" +echo " Without LMCache: $without_completed/$NUM_PROMPTS in ${without_duration}s" +echo "" diff --git a/.buildkite/k3_tests/multiprocess/scripts/run-gds-smoke.sh b/.buildkite/k3_tests/multiprocess/scripts/run-gds-smoke.sh new file mode 100755 index 0000000..c33e5c6 --- /dev/null +++ b/.buildkite/k3_tests/multiprocess/scripts/run-gds-smoke.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# GDS L1 smoke test. Sends a few completions (cold) to store KV to the slab, +# resets vLLM's prefix cache, then re-sends them (warm) to read the KV back from +# LMCache/GDS. Passes if every request returns HTTP 200, a real LMCache retrieve +# happened, and the warm (GDS-retrieved) outputs match the cold (recomputed) +# ones -- i.e. the GDS store/retrieve path works and is correct. +# +# Expects the GDS-enabled LMCache server + vLLM to already be running, with +# VLLM_SERVER_DEV_MODE=1 (for /reset_prefix_cache). +set -e +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" +source "${REPO_ROOT}/.buildkite/k3_tests/common_scripts/helpers.sh" + +VLLM_PORT="${VLLM_PORT:-8000}" +MODEL="${MODEL:-Qwen/Qwen3-14B}" +BUILD_ID="${BUILD_ID:-local_$$}" +LMCACHE_LOG="/tmp/build_${BUILD_ID}_lmcache.log" +N_PROMPTS="${GDS_SMOKE_PROMPTS:-4}" +OUT_DIR="$(mktemp -d)" +trap 'rm -rf "$OUT_DIR"' EXIT + +# A long-ish prompt so each request stores at least one LMCache chunk. +build_prompt() { # $1 = unique id + local filler="The key-value cache stores attention keys and values across transformer layers. " + local body="" i + for i in $(seq 1 80); do body="${body}${filler}"; done + printf 'Document %s. %s' "$1" "$body" +} + +# Send N_PROMPTS completions; capture each generated text to +# $OUT_DIR/