wehub-resource-sync
324 lines
12 KiB
Docker
324 lines
12 KiB
Docker
# This vLLM Dockerfile is used to build images that can run vLLM on both x86_64 and arm64 CPU platforms.
|
|
#
|
|
# Supported platforms:
|
|
# - linux/amd64 (x86_64)
|
|
# - linux/arm64 (aarch64)
|
|
#
|
|
# Use the `--platform` option with `docker buildx build` to specify the target architecture, e.g.:
|
|
# docker buildx build --platform=linux/arm64 -f docker/Dockerfile.cpu .
|
|
#
|
|
# Build targets:
|
|
# vllm-openai (default): used for serving deployment
|
|
# vllm-openai-zen: vLLM from source + zentorch from PyPI via vllm[zen]
|
|
# vllm-test: used for CI tests
|
|
# vllm-dev: used for development
|
|
#
|
|
# Build arguments:
|
|
# PYTHON_VERSION=3.13|3.12 (default)|3.11|3.10
|
|
# VLLM_CPU_X86=false (default)|true (for cross-compilation)
|
|
# VLLM_CPU_ARM_BF16=false (default)|true (for cross-compilation)
|
|
#
|
|
|
|
######################### COMMON BASE IMAGE #########################
|
|
FROM ubuntu:22.04 AS base-common
|
|
|
|
WORKDIR /workspace
|
|
|
|
ARG PYTHON_VERSION=3.12
|
|
|
|
ARG max_jobs=32
|
|
ENV MAX_JOBS=${max_jobs}
|
|
|
|
# Install minimal dependencies and uv
|
|
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
|
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
|
apt-get update -y \
|
|
&& apt-get install -y --no-install-recommends sudo ccache git curl wget ca-certificates zlib1g-dev \
|
|
gcc-12 g++-12 libtcmalloc-minimal4 libnuma-dev ffmpeg libsm6 libxext6 libgl1 jq lsof make xz-utils \
|
|
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \
|
|
&& curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
|
|
# Compiler and linker environment
|
|
ENV CC=/usr/bin/gcc-12 CXX=/usr/bin/g++-12
|
|
ENV CCACHE_DIR=/root/.cache/ccache
|
|
ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
|
|
|
|
ENV PATH="/root/.local/bin:$PATH"
|
|
ENV VIRTUAL_ENV="/opt/venv"
|
|
ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
|
|
RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
|
|
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
|
|
|
|
ENV UV_HTTP_TIMEOUT=500
|
|
|
|
# Install Python dependencies
|
|
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
|
ENV UV_LINK_MODE="copy"
|
|
|
|
# Copy requirements files for installation
|
|
COPY requirements/common.txt requirements/common.txt
|
|
COPY requirements/cpu.txt requirements/cpu.txt
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv pip install --upgrade pip && \
|
|
uv pip install -r requirements/cpu.txt --torch-backend cpu
|
|
|
|
ARG TARGETARCH
|
|
ENV TARGETARCH=${TARGETARCH}
|
|
|
|
######################### x86_64 BASE IMAGE #########################
|
|
FROM base-common AS base-amd64
|
|
|
|
ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/opt/venv/lib/libiomp5.so"
|
|
|
|
######################### arm64 BASE IMAGE #########################
|
|
FROM base-common AS base-arm64
|
|
|
|
ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4"
|
|
|
|
######################### BASE IMAGE #########################
|
|
FROM base-${TARGETARCH} AS base
|
|
|
|
RUN echo 'ulimit -c 0' >> ~/.bashrc
|
|
|
|
######################### RUST BUILD IMAGE #########################
|
|
# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the wheel build
|
|
# stage doesn't need the rust toolchain or protoc. This stage runs in parallel
|
|
# with the main vllm-build stage.
|
|
FROM ubuntu:22.04 AS rust-build
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
RUN apt-get update -y \
|
|
&& apt-get install -y --no-install-recommends \
|
|
ca-certificates curl git build-essential unzip python3 python3-pip \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
COPY tools/install_protoc.sh /tmp/install_protoc.sh
|
|
RUN /tmp/install_protoc.sh && rm /tmp/install_protoc.sh
|
|
|
|
WORKDIR /workspace
|
|
|
|
COPY requirements/build/rust.txt requirements/build/rust.txt
|
|
RUN python3 -m pip install --no-cache-dir -r requirements/build/rust.txt
|
|
|
|
# Copy only the Rust build inputs; build_rust.sh publishes artifacts needed
|
|
# by the wheel build stage.
|
|
COPY rust rust
|
|
COPY rust-toolchain.toml rust-toolchain.toml
|
|
COPY tools/build_rust.py tools/build_rust.py
|
|
COPY build_rust.sh build_rust.sh
|
|
|
|
# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
|
|
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
|
|
ENV CARGO_BUILD_JOBS=4
|
|
|
|
# Build the release artifacts. Cache cargo registry/git, but not target/,
|
|
# because stale target metadata can outlive source updates across BuildKit
|
|
# cache reuse.
|
|
RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
|
|
--mount=type=cache,target=/root/.cargo/git,sharing=locked \
|
|
bash build_rust.sh
|
|
|
|
######################### BUILD IMAGE #########################
|
|
FROM base AS vllm-build
|
|
|
|
ARG GIT_REPO_CHECK=0
|
|
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
|
|
ARG VLLM_CPU_X86=0
|
|
ENV VLLM_CPU_X86=${VLLM_CPU_X86}
|
|
# Support for cross-compilation with ARM BF16 ISA: docker build --build-arg VLLM_CPU_ARM_BF16="true" ...
|
|
ARG VLLM_CPU_ARM_BF16=0
|
|
ENV VLLM_CPU_ARM_BF16=${VLLM_CPU_ARM_BF16}
|
|
|
|
WORKDIR /vllm-workspace
|
|
|
|
# Validate build arguments - prevent mixing incompatible ISA flags
|
|
RUN if [ "$TARGETARCH" = "arm64" ] && [ "$VLLM_CPU_X86" != "0" ]; then \
|
|
echo "ERROR: Cannot use x86-specific ISA flags (AVX2, AVX512, etc.) when building for ARM64 (--platform=linux/arm64)"; \
|
|
exit 1; \
|
|
fi && \
|
|
if [ "$TARGETARCH" = "amd64" ] && [ "$VLLM_CPU_ARM_BF16" != "0" ]; then \
|
|
echo "ERROR: Cannot use ARM-specific ISA flags (ARM_BF16) when building for x86_64 (--platform=linux/amd64)"; \
|
|
exit 1; \
|
|
fi
|
|
|
|
# Copy build requirements
|
|
COPY requirements/build/cpu.txt requirements/build/cpu.txt
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv pip install -r requirements/build/cpu.txt --torch-backend cpu
|
|
|
|
COPY . .
|
|
|
|
# Drop the pre-built Rust artifacts into the source tree. setup.py detects
|
|
# them and ships them as-is, skipping the local Rust build.
|
|
COPY --from=rust-build /workspace/vllm/vllm-rs vllm/vllm-rs
|
|
COPY --from=rust-build /workspace/vllm/_rust_*.so vllm/
|
|
|
|
RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
--mount=type=cache,target=/root/.cache/ccache \
|
|
--mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
|
|
VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38
|
|
|
|
######################### TRITON-CPU BUILD IMAGE #########################
|
|
FROM base AS vllm-triton-cpu-build
|
|
|
|
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
|
|
# Re-declared here because this stage is `FROM base` (not `vllm-build`), so it
|
|
# does not inherit the ARG/ENV defined there. Without it, the guard below would
|
|
# see an empty value and build triton-cpu on non-x86 targets (e.g. arm64).
|
|
ARG VLLM_CPU_X86=0
|
|
|
|
WORKDIR /vllm-workspace
|
|
|
|
RUN mkdir dist
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
--mount=type=cache,target=/root/.cache/ccache \
|
|
--mount=type=cache,target=/vllm-workspace/.deps,sharing=locked \
|
|
if [ "$TARGETARCH" = "amd64" ] || [ "$VLLM_CPU_X86" != "0" ]; then \
|
|
git clone --recurse-submodules "https://github.com/triton-lang/triton-cpu.git"; \
|
|
cd triton-cpu; \
|
|
git checkout "270e696d"; \
|
|
uv build --wheel --out-dir=../dist; \
|
|
fi
|
|
|
|
######################### TEST DEPS #########################
|
|
FROM base AS vllm-test-deps
|
|
|
|
WORKDIR /vllm-workspace
|
|
|
|
# Test requirements are compiled from requirements/test/cuda.in into
|
|
# requirements/test/cpu.txt by the pip-compile-cpu pre-commit hook, which
|
|
# resolves CPU wheels via uv's --torch-backend cpu.
|
|
COPY requirements/test/cpu.txt requirements/test/cpu.txt
|
|
|
|
# cpu.txt is compiled for x86_64, so platform markers are resolved away. Drop
|
|
# packages unavailable on aarch64 (decord, terratorch) for arm builds.
|
|
RUN case "$(uname -m)" in \
|
|
aarch64|arm64) sed -i '/^decord==/d; /^terratorch==/d' requirements/test/cpu.txt ;; \
|
|
esac
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv pip install -r requirements/test/cpu.txt --torch-backend cpu
|
|
|
|
######################### DEV IMAGE #########################
|
|
FROM vllm-build AS vllm-dev
|
|
|
|
WORKDIR /vllm-workspace
|
|
|
|
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
|
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
|
apt-get install -y --no-install-recommends vim numactl clangd-14
|
|
|
|
RUN ln -s /usr/bin/clangd-14 /usr/bin/clangd
|
|
|
|
# install development dependencies (for testing)
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv pip install --no-build-isolation -e tests/vllm_test_utils
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
--mount=type=cache,target=/root/.cache/ccache \
|
|
--mount=type=bind,source=.git,target=.git \
|
|
VLLM_TARGET_DEVICE=cpu python3 setup.py develop
|
|
|
|
COPY --from=vllm-test-deps /vllm-workspace/requirements/test/cpu.txt requirements/test/cpu.txt
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv pip install -r requirements/lint.txt && \
|
|
uv pip install -r requirements/test/cpu.txt --torch-backend cpu && \
|
|
pre-commit install --hook-type pre-commit --hook-type commit-msg
|
|
|
|
ENTRYPOINT ["bash"]
|
|
|
|
######################### TEST IMAGE #########################
|
|
FROM vllm-test-deps AS vllm-test
|
|
|
|
WORKDIR /vllm-workspace
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
--mount=type=bind,from=vllm-build,src=/vllm-workspace/dist,target=dist \
|
|
uv pip install dist/*.whl
|
|
|
|
ADD ./tests/ ./tests/
|
|
ADD ./examples/ ./examples/
|
|
ADD ./benchmarks/ ./benchmarks/
|
|
ADD ./vllm/collect_env.py .
|
|
ADD ./docker/ ./docker/
|
|
ADD ./.buildkite/ ./.buildkite/
|
|
|
|
# install development dependencies (for testing)
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv pip install -e tests/vllm_test_utils
|
|
|
|
# enable fast downloads from hf (for testing)
|
|
ENV HF_XET_HIGH_PERFORMANCE 1
|
|
|
|
# increase timeout for hf downloads (for testing)
|
|
ENV HF_HUB_DOWNLOAD_TIMEOUT 60
|
|
|
|
######################### RELEASE IMAGE #########################
|
|
FROM base AS vllm-openai
|
|
|
|
# Re-declared here because this stage is `FROM base` (not `vllm-build`), so the
|
|
# RUN below that gates the triton-cpu wheel install on $VLLM_CPU_X86 would
|
|
# otherwise see an empty value and try to install it on non-x86 targets.
|
|
ARG VLLM_CPU_X86=0
|
|
|
|
WORKDIR /vllm-workspace
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
--mount=type=cache,target=/root/.cache/ccache \
|
|
--mount=type=bind,from=vllm-build,src=/vllm-workspace/dist,target=dist \
|
|
uv pip install "$(realpath dist/*.whl)[audio]"
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
--mount=type=cache,target=/root/.cache/ccache \
|
|
--mount=type=bind,from=vllm-triton-cpu-build,src=/vllm-workspace/dist,target=dist \
|
|
if [ "$TARGETARCH" = "amd64" ] || [ "$VLLM_CPU_X86" != "0" ]; then \
|
|
uv pip install "$(realpath dist/*.whl)"; \
|
|
fi
|
|
|
|
# Add labels to document build configuration
|
|
LABEL org.opencontainers.image.title="vLLM CPU"
|
|
LABEL org.opencontainers.image.description="vLLM inference engine for CPU platforms"
|
|
LABEL org.opencontainers.image.vendor="vLLM Project"
|
|
LABEL org.opencontainers.image.source="https://github.com/vllm-project/vllm"
|
|
|
|
# Build configuration labels
|
|
ARG TARGETARCH
|
|
ARG VLLM_CPU_X86
|
|
ARG VLLM_CPU_ARM_BF16
|
|
ARG PYTHON_VERSION
|
|
|
|
LABEL ai.vllm.build.target-arch="${TARGETARCH}"
|
|
LABEL ai.vllm.build.cpu-x86="${VLLM_CPU_X86:-false}"
|
|
LABEL ai.vllm.build.cpu-arm-bf16="${VLLM_CPU_ARM_BF16:-false}"
|
|
LABEL ai.vllm.build.python-version="${PYTHON_VERSION:-3.12}"
|
|
|
|
# Copy the examples directory (including the chat/tool templates) so it is
|
|
# present in the released image, as the CUDA image ships it too. The vllm-test
|
|
# stage above adds examples/ for testing only, so without this the published
|
|
# vllm-openai-cpu image would not ship examples/*.jinja.
|
|
COPY examples examples
|
|
|
|
ENTRYPOINT ["vllm", "serve"]
|
|
|
|
|
|
######################### ZEN CPU PYPI IMAGE #########################
|
|
FROM vllm-openai AS vllm-openai-zen
|
|
|
|
ARG TARGETARCH
|
|
|
|
RUN if [ "$TARGETARCH" != "amd64" ]; then \
|
|
echo "ERROR: vllm-openai-amd only supports --platform=linux/amd64"; \
|
|
exit 1; \
|
|
fi
|
|
|
|
RUN --mount=type=cache,target=/root/.cache/uv \
|
|
uv pip install "vllm[zen]"
|
|
|
|
ENTRYPOINT ["vllm", "serve"]
|