mirrors/kvcache-ai--ktransformers
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
Book-CI / test (macos-latest) (push) Has been cancelled
Details
Book-CI / test (ubuntu-latest) (push) Has been cancelled
Details
Book-CI / test (windows-latest) (push) Has been cancelled
Details
Release Fake Tag / publish (push) Has been cancelled
Details
Deploy / deploy (macos-latest) (push) Has been cancelled
Details
Deploy / deploy (ubuntu-latest) (push) Has been cancelled
Details
Deploy / deploy (windows-latest) (push) Has been cancelled
Details
Release to PyPI / Build & publish sglang-kt (push) Has been cancelled
Details
Release to PyPI / Build kt-kernel (Python 3.11) (push) Has been cancelled
Details
Release to PyPI / Build kt-kernel (Python 3.12) (push) Has been cancelled
Details
Release to PyPI / Publish kt-kernel to PyPI (push) Has been cancelled
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:30:03 +08:00
commit ec436095dd
1232 changed files with 404407 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docker/Dockerfile
+434
View File
@@ -0,0 +1,434 @@
ARG CUDA_VERSION=12.8.1
FROM docker.1ms.run/nvidia/cuda:${CUDA_VERSION}-cudnn-devel-ubuntu24.04 AS base
ARG TARGETARCH
ARG GRACE_BLACKWELL=0
ARG HOPPER_SBO=0
ARG CPU_VARIANT=x86-intel-multi
ARG BUILD_ALL_CPU_VARIANTS=1
# Proxy settings for build-time network access
ARG HTTP_PROXY
ARG HTTPS_PROXY
ARG http_proxy
ARG https_proxy
ENV HTTP_PROXY=${HTTP_PROXY} \
HTTPS_PROXY=${HTTPS_PROXY} \
http_proxy=${http_proxy} \
https_proxy=${https_proxy}
ARG GRACE_BLACKWELL_DEEPEP_BRANCH=gb200_blog_part_2
ARG HOPPER_SBO_DEEPEP_COMMIT=9f2fc4b3182a51044ae7ecb6610f7c9c3258c4d6
ARG DEEPEP_COMMIT=9af0e0d0e74f3577af1979c9b9e1ac2cad0104ee
ARG BUILD_AND_DOWNLOAD_PARALLEL=8
ARG SGL_KERNEL_VERSION=0.3.19
ARG SGL_VERSION=0.5.6.post1
ARG USE_LATEST_SGLANG=0
ARG GDRCOPY_VERSION=2.5.1
ARG UBUNTU_MIRROR
ARG GITHUB_ARTIFACTORY=github.com
ARG FLASHINFER_VERSION=0.5.3
# ktransformers wheel version (cu128torch28 for CUDA 12.8 + PyTorch 2.8)
ARG KTRANSFORMERS_VERSION=0.5.3
ARG KTRANSFORMERS_WHEEL=ktransformers-0.5.3+cu128torch28fancy-cp312-cp312-linux_x86_64.whl
# flash_attn wheel for fine-tune env
ARG FLASH_ATTN_WHEEL=flash_attn-2.8.3+cu12torch2.8cxx11abiTRUE-cp312-cp312-linux_x86_64.whl
ENV DEBIAN_FRONTEND=noninteractive \
CUDA_HOME=/usr/local/cuda \
GDRCOPY_HOME=/usr/src/gdrdrv-${GDRCOPY_VERSION}/ \
FLASHINFER_VERSION=${FLASHINFER_VERSION}
# Add GKE default lib and bin locations
ENV PATH="${PATH}:/usr/local/nvidia/bin" \
LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/nvidia/lib:/usr/local/nvidia/lib64"
# Replace Ubuntu sources with Tsinghua mirror for Ubuntu 24.04 (noble)
RUN if [ -n "$UBUNTU_MIRROR" ]; then \
echo "deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ noble main restricted universe multiverse" > /etc/apt/sources.list && \
echo "deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ noble-updates main restricted universe multiverse" >> /etc/apt/sources.list && \
echo "deb https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ noble-backports main restricted universe multiverse" >> /etc/apt/sources.list && \
echo "deb http://security.ubuntu.com/ubuntu/ noble-security main restricted universe multiverse" >> /etc/apt/sources.list && \
rm -f /etc/apt/sources.list.d/ubuntu.sources; \
fi
# Install system dependencies (organized by category for better caching)
RUN --mount=type=cache,target=/var/cache/apt,id=base-apt \
echo 'tzdata tzdata/Areas select America' | debconf-set-selections \
&& echo 'tzdata tzdata/Zones/America select Los_Angeles' | debconf-set-selections \
&& apt-get update && apt-get install -y --no-install-recommends --allow-change-held-packages \
# Core system utilities
tzdata \
ca-certificates \
software-properties-common \
netcat-openbsd \
kmod \
unzip \
openssh-server \
curl \
wget \
lsof \
locales \
# Build essentials
build-essential \
cmake \
perl \
patchelf \
ccache \
git \
git-lfs \
# MPI and NUMA
libopenmpi-dev \
libnuma1 \
libnuma-dev \
numactl \
# transformers multimodal VLM
ffmpeg \
# InfiniBand/RDMA
libibverbs-dev \
libibverbs1 \
libibumad3 \
librdmacm1 \
libnl-3-200 \
libnl-route-3-200 \
libnl-route-3-dev \
libnl-3-dev \
ibverbs-providers \
infiniband-diags \
perftest \
# Development libraries
libgoogle-glog-dev \
libgtest-dev \
libjsoncpp-dev \
libunwind-dev \
libboost-all-dev \
libssl-dev \
libgrpc-dev \
libgrpc++-dev \
libprotobuf-dev \
protobuf-compiler \
protobuf-compiler-grpc \
pybind11-dev \
libhiredis-dev \
libcurl4-openssl-dev \
libczmq4 \
libczmq-dev \
libfabric-dev \
# Package building tools
devscripts \
debhelper \
fakeroot \
dkms \
check \
libsubunit0 \
libsubunit-dev \
# Development tools
gdb \
ninja-build \
vim \
tmux \
htop \
zsh \
tree \
less \
rdma-core \
# NCCL
libnccl2 \
libnccl-dev \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get clean
# GDRCopy installation
RUN mkdir -p /tmp/gdrcopy && cd /tmp \
&& curl --retry 3 --retry-delay 2 -fsSL -o v${GDRCOPY_VERSION}.tar.gz \
https://${GITHUB_ARTIFACTORY}/NVIDIA/gdrcopy/archive/refs/tags/v${GDRCOPY_VERSION}.tar.gz \
&& tar -xzf v${GDRCOPY_VERSION}.tar.gz && rm v${GDRCOPY_VERSION}.tar.gz \
&& cd gdrcopy-${GDRCOPY_VERSION}/packages \
&& CUDA=/usr/local/cuda ./build-deb-packages.sh \
&& dpkg -i gdrdrv-dkms_*.deb libgdrapi_*.deb gdrcopy-tests_*.deb gdrcopy_*.deb \
&& cd / && rm -rf /tmp/gdrcopy
# Fix DeepEP IBGDA symlink
RUN ln -sf /usr/lib/$(uname -m)-linux-gnu/libmlx5.so.1 /usr/lib/$(uname -m)-linux-gnu/libmlx5.so
# Set up locale
RUN locale-gen en_US.UTF-8
ENV LANG=en_US.UTF-8 \
LANGUAGE=en_US:en \
LC_ALL=en_US.UTF-8
########################################################
########## Install Miniconda ###########################
########################################################
RUN mkdir -p /opt/miniconda3 \
&& wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /opt/miniconda3/miniconda.sh \
&& bash /opt/miniconda3/miniconda.sh -b -u -p /opt/miniconda3 \
&& rm /opt/miniconda3/miniconda.sh
# Add conda to PATH
ENV PATH="/opt/miniconda3/bin:${PATH}"
# Accept conda TOS
RUN conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/main \
&& conda tos accept --override-channels --channel https://repo.anaconda.com/pkgs/r
# Configure conda to use Tsinghua mirror
RUN conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/main \
&& conda config --add channels https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free \
&& conda config --set show_channel_urls yes
########################################################
########## Dual Conda Environment Setup ################
########################################################
FROM base AS framework
ARG CUDA_VERSION
ARG BUILD_AND_DOWNLOAD_PARALLEL
ARG SGL_KERNEL_VERSION
ARG SGL_VERSION
ARG USE_LATEST_SGLANG
ARG FLASHINFER_VERSION
ARG GRACE_BLACKWELL
ARG GRACE_BLACKWELL_DEEPEP_BRANCH
ARG HOPPER_SBO
ARG HOPPER_SBO_DEEPEP_COMMIT
ARG DEEPEP_COMMIT
ARG GITHUB_ARTIFACTORY
ARG KTRANSFORMERS_VERSION
ARG KTRANSFORMERS_WHEEL
ARG FLASH_ATTN_WHEEL
ARG FUNCTIONALITY=sft
WORKDIR /workspace
# Create conda environments (fine-tune only needed for sft mode)
RUN conda create -n serve python=3.12 -y \
&& if [ "$FUNCTIONALITY" = "sft" ]; then conda create -n fine-tune python=3.12 -y; fi
# Set pip mirror for conda envs
RUN /opt/miniconda3/envs/serve/bin/pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple \
&& if [ "$FUNCTIONALITY" = "sft" ]; then \
/opt/miniconda3/envs/fine-tune/bin/pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple; \
fi
# Clone repositories (sglang is included as a submodule in ktransformers)
RUN git clone --depth 1 https://${GITHUB_ARTIFACTORY}/kvcache-ai/ktransformers.git /workspace/ktransformers \
&& cd /workspace/ktransformers && git submodule update --init --recursive \
&& ln -s /workspace/ktransformers/third_party/sglang /workspace/sglang \
&& if [ "$FUNCTIONALITY" = "sft" ]; then \
git clone --depth 1 https://${GITHUB_ARTIFACTORY}/hiyouga/LLaMA-Factory.git /workspace/LLaMA-Factory; \
fi
# Download ktransformers wheel and flash_attn wheel for fine-tune env (sft mode only)
RUN if [ "$FUNCTIONALITY" = "sft" ]; then \
curl --retry 3 --retry-delay 2 -fsSL -o /workspace/${KTRANSFORMERS_WHEEL} \
https://${GITHUB_ARTIFACTORY}/kvcache-ai/ktransformers/releases/download/v${KTRANSFORMERS_VERSION}/${KTRANSFORMERS_WHEEL} \
&& curl --retry 3 --retry-delay 2 -fsSL -o /workspace/${FLASH_ATTN_WHEEL} \
https://${GITHUB_ARTIFACTORY}/Dao-AILab/flash-attention/releases/download/v2.8.3/${FLASH_ATTN_WHEEL}; \
fi
########################################################
# Environment 1: serve (sglang + kt-kernel)
########################################################
# Upgrade pip and install basic tools in serve env
RUN --mount=type=cache,target=/root/.cache/pip \
/opt/miniconda3/envs/serve/bin/pip install --upgrade pip setuptools wheel html5lib six
# Install sgl-kernel
RUN --mount=type=cache,target=/root/.cache/pip \
case "$CUDA_VERSION" in \
12.6.1) CUINDEX=126 ;; \
12.8.1) CUINDEX=128 ;; \
12.9.1) CUINDEX=129 ;; \
13.0.1) CUINDEX=130 ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
esac \
&& if [ "$CUDA_VERSION" = "12.6.1" ]; then \
/opt/miniconda3/envs/serve/bin/pip install https://${GITHUB_ARTIFACTORY}/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sgl_kernel-${SGL_KERNEL_VERSION}+cu124-cp310-abi3-manylinux2014_$(uname -m).whl --force-reinstall --no-deps \
; \
elif [ "$CUDA_VERSION" = "12.8.1" ] || [ "$CUDA_VERSION" = "12.9.1" ]; then \
/opt/miniconda3/envs/serve/bin/pip install sgl-kernel==${SGL_KERNEL_VERSION} \
; \
elif [ "$CUDA_VERSION" = "13.0.1" ]; then \
/opt/miniconda3/envs/serve/bin/pip install https://github.com/sgl-project/whl/releases/download/v${SGL_KERNEL_VERSION}/sgl_kernel-${SGL_KERNEL_VERSION}+cu130-cp310-abi3-manylinux2014_$(uname -m).whl --force-reinstall --no-deps \
; \
fi
# Install SGLang in serve env (version aligned with ktransformers)
RUN --mount=type=cache,target=/root/.cache/pip \
case "$CUDA_VERSION" in \
12.6.1) CUINDEX=126 ;; \
12.8.1) CUINDEX=128 ;; \
12.9.1) CUINDEX=129 ;; \
13.0.1) CUINDEX=130 ;; \
esac \
&& export SGLANG_KT_VERSION=$(python3 -c "exec(open('/workspace/ktransformers/version.py').read()); print(__version__)") \
&& echo "Installing sglang-kt v${SGLANG_KT_VERSION}" \
&& cd /workspace/sglang \
&& /opt/miniconda3/envs/serve/bin/pip install -e "python[all]" --extra-index-url https://download.pytorch.org/whl/cu${CUINDEX}
# Download FlashInfer cubin for serve env
RUN --mount=type=cache,target=/root/.cache/pip \
FLASHINFER_CUBIN_DOWNLOAD_THREADS=${BUILD_AND_DOWNLOAD_PARALLEL} FLASHINFER_LOGGING_LEVEL=warning \
/opt/miniconda3/envs/serve/bin/python -m flashinfer --download-cubin
# Install DeepEP in serve env
RUN set -eux; \
if [ "$GRACE_BLACKWELL" = "1" ]; then \
git clone https://github.com/fzyzcjy/DeepEP.git /workspace/DeepEP && \
cd /workspace/DeepEP && \
git checkout ${GRACE_BLACKWELL_DEEPEP_BRANCH} && \
sed -i 's/#define NUM_CPU_TIMEOUT_SECS 100/#define NUM_CPU_TIMEOUT_SECS 1000/' csrc/kernels/configs.cuh; \
elif [ "$HOPPER_SBO" = "1" ]; then \
git clone https://github.com/deepseek-ai/DeepEP.git -b antgroup-opt /workspace/DeepEP && \
cd /workspace/DeepEP && \
git checkout ${HOPPER_SBO_DEEPEP_COMMIT} && \
sed -i 's/#define NUM_CPU_TIMEOUT_SECS 100/#define NUM_CPU_TIMEOUT_SECS 1000/' csrc/kernels/configs.cuh; \
else \
curl --retry 3 --retry-delay 2 -fsSL -o /tmp/${DEEPEP_COMMIT}.zip \
https://${GITHUB_ARTIFACTORY}/deepseek-ai/DeepEP/archive/${DEEPEP_COMMIT}.zip && \
unzip -q /tmp/${DEEPEP_COMMIT}.zip -d /tmp && rm /tmp/${DEEPEP_COMMIT}.zip && \
mv /tmp/DeepEP-${DEEPEP_COMMIT} /workspace/DeepEP && \
cd /workspace/DeepEP && \
sed -i 's/#define NUM_CPU_TIMEOUT_SECS 100/#define NUM_CPU_TIMEOUT_SECS 1000/' csrc/kernels/configs.cuh; \
fi
RUN --mount=type=cache,target=/root/.cache/pip \
cd /workspace/DeepEP && \
case "$CUDA_VERSION" in \
12.6.1) CHOSEN_TORCH_CUDA_ARCH_LIST='9.0' ;; \
12.8.1) CHOSEN_TORCH_CUDA_ARCH_LIST='9.0;10.0' ;; \
12.9.1|13.0.1) CHOSEN_TORCH_CUDA_ARCH_LIST='9.0;10.0;10.3' ;; \
*) echo "Unsupported CUDA version: $CUDA_VERSION" && exit 1 ;; \
esac && \
. /opt/miniconda3/etc/profile.d/conda.sh && conda activate serve && \
TORCH_CUDA_ARCH_LIST="${CHOSEN_TORCH_CUDA_ARCH_LIST}" MAX_JOBS=${BUILD_AND_DOWNLOAD_PARALLEL} \
pip install --no-build-isolation .
# Install NCCL for serve env
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
/opt/miniconda3/envs/serve/bin/pip install nvidia-nccl-cu12==2.28.3 --force-reinstall --no-deps ; \
elif [ "${CUDA_VERSION%%.*}" = "13" ]; then \
/opt/miniconda3/envs/serve/bin/pip install nvidia-nccl-cu13==2.28.3 --force-reinstall --no-deps ; \
fi
# Install kt-kernel in serve env with all CPU variants
RUN . /opt/miniconda3/etc/profile.d/conda.sh && conda activate serve \
&& cd /workspace/ktransformers/kt-kernel \
&& CPUINFER_BUILD_ALL_VARIANTS=1 ./install.sh build
########################################################
# Environment 2: fine-tune (LLaMA-Factory + ktransformers) - sft mode only
########################################################
# Install dependency libraries for ktransformers (CUDA 11.8 runtime required)
RUN if [ "$FUNCTIONALITY" = "sft" ]; then \
conda install -n fine-tune -y -c conda-forge libstdcxx-ng gcc_impl_linux-64 \
&& conda install -n fine-tune -y -c nvidia/label/cuda-11.8.0 cuda-runtime; \
fi
# Install PyTorch 2.8 in fine-tune env
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "$FUNCTIONALITY" = "sft" ]; then \
case "$CUDA_VERSION" in \
12.6.1) CUINDEX=126 ;; \
12.8.1) CUINDEX=128 ;; \
12.9.1) CUINDEX=129 ;; \
13.0.1) CUINDEX=130 ;; \
esac \
&& /opt/miniconda3/envs/fine-tune/bin/pip install --upgrade pip setuptools wheel hatchling \
&& /opt/miniconda3/envs/fine-tune/bin/pip install \
torch==2.8.0 \
torchvision \
torchaudio \
--extra-index-url https://download.pytorch.org/whl/cu${CUINDEX}; \
fi
# Install LLaMA-Factory in fine-tune env
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "$FUNCTIONALITY" = "sft" ]; then \
cd /workspace/LLaMA-Factory \
&& /opt/miniconda3/envs/fine-tune/bin/pip install -e ".[torch,metrics]" --no-build-isolation; \
fi
# Install ktransformers wheel in fine-tune env
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "$FUNCTIONALITY" = "sft" ]; then \
/opt/miniconda3/envs/fine-tune/bin/pip install /workspace/${KTRANSFORMERS_WHEEL}; \
fi
# Install flash_attn wheel in fine-tune env
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "$FUNCTIONALITY" = "sft" ]; then \
/opt/miniconda3/envs/fine-tune/bin/pip install /workspace/${FLASH_ATTN_WHEEL}; \
fi
# Install NCCL for fine-tune env
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "$FUNCTIONALITY" = "sft" ]; then \
if [ "${CUDA_VERSION%%.*}" = "12" ]; then \
/opt/miniconda3/envs/fine-tune/bin/pip install nvidia-nccl-cu12==2.28.3 --force-reinstall --no-deps ; \
elif [ "${CUDA_VERSION%%.*}" = "13" ]; then \
/opt/miniconda3/envs/fine-tune/bin/pip install nvidia-nccl-cu13==2.28.3 --force-reinstall --no-deps ; \
fi; \
fi
########################################################
# Cleanup and final setup
########################################################
# Clean up downloaded wheels
RUN if [ "$FUNCTIONALITY" = "sft" ]; then \
rm -f /workspace/${KTRANSFORMERS_WHEEL} /workspace/${FLASH_ATTN_WHEEL}; \
fi
# Initialize conda for bash
RUN /opt/miniconda3/bin/conda init bash
# Create shell aliases for convenience
RUN echo '\n# Conda environment aliases\nalias serve="conda activate serve"' >> /root/.bashrc \
&& if [ "$FUNCTIONALITY" = "sft" ]; then \
echo 'alias finetune="conda activate fine-tune"' >> /root/.bashrc; \
fi
########################################################
# Extract version information for image naming
########################################################
# Extract versions from each component and save to versions.env
RUN set -x && \
# KTransformers version (single source of truth for both kt-kernel and sglang-kt)
cd /workspace/ktransformers && \
KTRANSFORMERS_VERSION=$(python3 -c "exec(open('version.py').read()); print(__version__)" 2>/dev/null || echo "unknown") && \
echo "KTRANSFORMERS_VERSION=$KTRANSFORMERS_VERSION" > /workspace/versions.env && \
echo "Extracted KTransformers version: $KTRANSFORMERS_VERSION" && \
\
# sglang-kt version = ktransformers version (aligned)
echo "SGLANG_KT_VERSION=$KTRANSFORMERS_VERSION" >> /workspace/versions.env && \
echo "sglang-kt version (aligned): $KTRANSFORMERS_VERSION" && \
\
# LLaMA-Factory version (from fine-tune environment, sft mode only)
if [ "$FUNCTIONALITY" = "sft" ]; then \
. /opt/miniconda3/etc/profile.d/conda.sh && conda activate fine-tune && \
cd /workspace/LLaMA-Factory && \
LLAMAFACTORY_VERSION=$(python -c "import sys; sys.path.insert(0, 'src'); from llamafactory import __version__; print(__version__)" 2>/dev/null || echo "unknown") && \
echo "LLAMAFACTORY_VERSION=$LLAMAFACTORY_VERSION" >> /workspace/versions.env && \
echo "Extracted LLaMA-Factory version: $LLAMAFACTORY_VERSION"; \
else \
echo "LLAMAFACTORY_VERSION=none" >> /workspace/versions.env && \
echo "LLaMA-Factory not installed (infer mode)"; \
fi && \
\
# Display all versions
echo "=== Version Summary ===" && \
cat /workspace/versions.env
WORKDIR /workspace
CMD ["/bin/bash"]
docker/README-packaging.md
+387
View File
@@ -0,0 +1,387 @@
# KTransformers Docker Packaging Guide
This directory contains scripts for building and distributing KTransformers Docker images with standardized naming conventions.
## Overview
The packaging system provides:
- **Automated version detection** from sglang, ktransformers, and LLaMA-Factory
- **Multi-CPU variant support** (AMX, AVX512, AVX2) with runtime auto-detection
- **Standardized naming convention** for easy identification and management
- **Two distribution methods**:
- Local tar file export for offline distribution
- DockerHub publishing for online distribution
## Naming Convention
Docker images follow this naming pattern:
```
sglang-v{sglang版本}_ktransformers-v{ktransformers版本}_{cpu信息}_{gpu信息}_{功能模式}_{时间戳}
```
### Example Names
**Tar file:**
```
sglang-v0.5.6_ktransformers-v0.5.3_x86-intel-multi_cu128_sft_llamafactory-v0.9.3_20241212143022.tar
```
**DockerHub tags:**
```
Full tag:
kvcache/ktransformers:sglang-v0.5.6_ktransformers-v0.5.3_x86-intel-multi_cu128_sft_llamafactory-v0.9.3_20241212143022
Simplified tag:
kvcache/ktransformers:v0.5.3-cu128
```
### Name Components
| Component | Description | Example |
|-----------|-------------|---------|
| sglang version | SGLang package version | `v0.5.6` |
| ktransformers version | KTransformers version | `v0.5.3` |
| cpu info | CPU instruction set support | `x86-intel-multi` (includes AMX/AVX512/AVX2) |
| gpu info | CUDA version | `cu128` (CUDA 12.8) |
| functionality | Feature mode | `sft_llamafactory-v0.9.3` or `infer` |
| timestamp | Build time (Beijing/UTC+8) | `20241212143022` |
## Files
| File | Purpose |
|------|---------|
| `Dockerfile` | Main Dockerfile with multi-CPU build and version extraction |
| `docker-utils.sh` | Shared utility functions for both scripts |
| `build-docker-tar.sh` | Build and export Docker image to tar file |
| `push-to-dockerhub.sh` | Build and push Docker image to DockerHub |
## Prerequisites
- Docker installed and running
- For DockerHub push: Docker Hub account and login (`docker login`)
- Sufficient disk space (at least 20GB recommended)
- Internet access (or local mirrors configured)
## Quick Start
### Build Local Tar File
```bash
cd docker
# Basic build
./build-docker-tar.sh
# With specific CUDA version and mirror
./build-docker-tar.sh \
--cuda-version 12.8.1 \
--ubuntu-mirror 1
# With proxy
./build-docker-tar.sh \
--cuda-version 12.8.1 \
--ubuntu-mirror 1 \
--http-proxy "http://127.0.0.1:16981" \
--https-proxy "http://127.0.0.1:16981" \
--output-dir /path/to/output
```
### Push to DockerHub
```bash
cd docker
# Basic push (requires --repository)
./push-to-dockerhub.sh \
--repository kvcache/ktransformers
# With simplified tag
./push-to-dockerhub.sh \
--cuda-version 12.8.1 \
--repository kvcache/ktransformers \
--also-push-simplified
# Skip build if image exists
./push-to-dockerhub.sh \
--repository kvcache/ktransformers \
--skip-build
```
## Script Options
### build-docker-tar.sh
```
Build Configuration:
--cuda-version VERSION CUDA version (default: 12.8.1)
--ubuntu-mirror 0|1 Use Tsinghua mirror (default: 0)
--http-proxy URL HTTP proxy URL
--https-proxy URL HTTPS proxy URL
--cpu-variant VARIANT CPU variant (default: x86-intel-multi)
--functionality TYPE Mode: sft or infer (default: sft)
Paths:
--dockerfile PATH Path to Dockerfile (default: ./Dockerfile)
--context-dir PATH Build context directory (default: .)
--output-dir PATH Output directory for tar (default: .)
Options:
--dry-run Preview without building
--keep-image Keep Docker image after export
--build-arg KEY=VALUE Additional build arguments
-h, --help Show help message
```
### push-to-dockerhub.sh
```
All options from build-docker-tar.sh, plus:
Registry Settings:
--registry REGISTRY Docker registry (default: docker.io)
--repository REPO Repository name (REQUIRED)
Options:
--skip-build Skip build if image exists
--also-push-simplified Also push simplified tag
--max-retries N Max push retries (default: 3)
--retry-delay SECONDS Delay between retries (default: 5)
```
## Usage Examples
### Example 1: Local Development Build
For testing on your local machine:
```bash
./build-docker-tar.sh \
--cuda-version 12.8.1 \
--output-dir ./builds \
--keep-image
```
This will:
1. Build the Docker image
2. Export to tar in `./builds/` directory
3. Keep the Docker image for local testing
### Example 2: Production Build for Distribution
For creating a production build with mirrors and proxy:
```bash
./build-docker-tar.sh \
--cuda-version 12.8.1 \
--ubuntu-mirror 1 \
--http-proxy "http://127.0.0.1:16981" \
--https-proxy "http://127.0.0.1:16981" \
--output-dir /mnt/data/releases
```
### Example 3: Publish to DockerHub
For publishing to DockerHub:
```bash
# First, login to Docker Hub
docker login
# Then push
./push-to-dockerhub.sh \
--cuda-version 12.8.1 \
--repository kvcache/ktransformers \
--also-push-simplified
```
This creates two tags:
- Full: `kvcache/ktransformers:sglang-v0.5.6_ktransformers-v0.5.3_x86-intel-multi_cu128_sft_llamafactory-v0.9.3_20241212143022`
- Simplified: `kvcache/ktransformers:v0.5.3-cu128`
### Example 4: Dry Run
Preview the build without actually building:
```bash
./build-docker-tar.sh --cuda-version 12.8.1 --dry-run
```
### Example 5: Custom Build Arguments
Pass additional Docker build arguments:
```bash
./build-docker-tar.sh \
--cuda-version 12.8.1 \
--build-arg SGL_VERSION=0.5.7 \
--build-arg FLASHINFER_VERSION=0.5.4
```
## Using the Built Images
### Load from Tar File
```bash
# Load the image
docker load -i sglang-v0.5.6_ktransformers-v0.5.3_x86-intel-multi_cu128_sft_llamafactory-v0.9.3_20241212143022.tar
# Run the container
docker run -it --rm \
--gpus all \
sglang-v0.5.6_ktransformers-v0.5.3_x86-intel-multi_cu128_sft_llamafactory-v0.9.3_20241212143022 \
/bin/bash
```
### Pull from DockerHub
```bash
# Pull with full tag
docker pull kvcache/ktransformers:sglang-v0.5.6_ktransformers-v0.5.3_x86-intel-multi_cu128_sft_llamafactory-v0.9.3_20241212143022
# Or pull with simplified tag
docker pull kvcache/ktransformers:v0.5.3-cu128
# Run the container
docker run -it --rm \
--gpus all \
kvcache/ktransformers:v0.5.3-cu128 \
/bin/bash
```
### Inside the Container
The image contains two conda environments:
```bash
# Activate serve environment (for inference with sglang)
conda activate serve
# or use the alias:
serve
# Activate fine-tune environment (for training with LLaMA-Factory)
conda activate fine-tune
# or use the alias:
finetune
```
## Multi-CPU Variant Support
The Docker image includes all three CPU variants:
- **AMX** - For Intel Sapphire Rapids and newer (4th Gen Xeon+)
- **AVX512** - For Intel Skylake-X, Ice Lake, Cascade Lake
- **AVX2** - Maximum compatibility for older CPUs
The runtime automatically detects your CPU and loads the appropriate variant. To override:
```bash
# Force use of AVX2 variant
export KT_KERNEL_CPU_VARIANT=avx2
python your_script.py
# Enable debug output to see which variant is loaded
export KT_KERNEL_DEBUG=1
python your_script.py
```
## Version Extraction
Versions are automatically extracted during Docker build from:
- **SGLang**: From `sglang.__version__` in serve environment
- **KTransformers**: From `version.py` in ktransformers repository
- **LLaMA-Factory**: From `llamafactory.__version__` in fine-tune environment
The versions are saved to `/workspace/versions.env` in the image:
```bash
# View versions in running container
cat /workspace/versions.env
# Output:
SGLANG_VERSION=0.5.6
KTRANSFORMERS_VERSION=0.5.3
LLAMAFACTORY_VERSION=0.9.3
```
## Troubleshooting
### Build Fails with Out of Disk Space
Check available disk space:
```bash
df -h
```
The build requires approximately 15-20GB of disk space. Clean up Docker:
```bash
docker system prune -a
```
### Version Extraction Fails
If version extraction fails (shows "unknown"), check:
1. The cloned repositories have the correct branches
2. Python packages are properly installed in conda environments
3. Version files exist in expected locations
You can manually verify by running:
```bash
docker run --rm <image> /bin/bash -c "
source /opt/miniconda3/etc/profile.d/conda.sh &&
conda activate serve &&
python -c 'import sglang; print(sglang.__version__)'
"
```
### Push to DockerHub Fails
1. **Check login**: `docker login`
2. **Check repository name**: Must include namespace (e.g., `kvcache/ktransformers`, not just `ktransformers`)
3. **Network issues**: Use `--max-retries` and `--retry-delay` options
4. **Rate limiting**: DockerHub has pull/push rate limits for free accounts
## Advanced Topics
### Custom Dockerfile Location
```bash
./build-docker-tar.sh \
--dockerfile /path/to/custom/Dockerfile \
--context-dir /path/to/build/context
```
### Building Only Inference Image (Future)
Currently, the image always includes both serve and fine-tune environments. To create an inference-only image, modify the Dockerfile to skip the fine-tune environment section.
### Customizing CPU Variants
To build only specific CPU variants, modify `kt-kernel/install.sh` or set environment variables in the Dockerfile.
### CI/CD Integration
The scripts are designed for manual execution but can be integrated into CI/CD pipelines:
```yaml
# Example GitHub Actions workflow
- name: Build and push Docker image
run: |
cd docker
./push-to-dockerhub.sh \
--cuda-version ${{ matrix.cuda_version }} \
--repository ${{ secrets.DOCKER_REPOSITORY }} \
--also-push-simplified
```
## Support
For issues and questions:
- File an issue at: https://github.com/kvcache-ai/ktransformers/issues
- Check documentation: https://github.com/kvcache-ai/ktransformers
## License
This packaging system is part of KTransformers and follows the same license.
docker/docker-utils.sh
Executable
+372
View File
@@ -0,0 +1,372 @@
#!/usr/bin/env bash
#
# docker-utils.sh - Shared utility functions for Docker image build and publish scripts
#
# This script provides common functions for:
# - Timestamp generation (Beijing timezone)
# - Version extraction from Docker images
# - Image name generation following naming conventions
# - Colored logging
# - Validation and error handling
#
# Usage: source docker-utils.sh
set -euo pipefail
# Color codes for logging
COLOR_RED='\033[0;31m'
COLOR_GREEN='\033[0;32m'
COLOR_YELLOW='\033[1;33m'
COLOR_BLUE='\033[0;34m'
COLOR_CYAN='\033[0;36m'
COLOR_RESET='\033[0m'
################################################################################
# Logging Functions
################################################################################
log_info() {
echo -e "${COLOR_BLUE}[INFO]${COLOR_RESET} $*"
}
log_success() {
echo -e "${COLOR_GREEN}[SUCCESS]${COLOR_RESET} $*"
}
log_warning() {
echo -e "${COLOR_YELLOW}[WARNING]${COLOR_RESET} $*"
}
log_error() {
echo -e "${COLOR_RED}[ERROR]${COLOR_RESET} $*" >&2
}
log_step() {
echo -e "\n${COLOR_CYAN}==>${COLOR_RESET} $*"
}
################################################################################
# Timestamp Functions
################################################################################
# Generate timestamp in Beijing timezone (UTC+8)
# Format: YYYYMMDDHHMMSS
# Example: 20241212143022
get_beijing_timestamp() {
# Try to use TZ environment variable approach
if date --version &>/dev/null 2>&1; then
# GNU date (Linux)
TZ='Asia/Shanghai' date '+%Y%m%d%H%M%S'
else
# BSD date (macOS)
TZ='Asia/Shanghai' date '+%Y%m%d%H%M%S'
fi
}
################################################################################
# CUDA Version Parsing
################################################################################
# Parse CUDA version to short format
# Input: 12.8.1 or 12.8 or 13.0.1
# Output: cu128 or cu130
parse_cuda_short_version() {
local cuda_version="$1"
# Extract major and minor version
local major minor
major=$(echo "$cuda_version" | cut -d. -f1)
minor=$(echo "$cuda_version" | cut -d. -f2)
# Validate
if [[ ! "$major" =~ ^[0-9]+$ ]] || [[ ! "$minor" =~ ^[0-9]+$ ]]; then
log_error "Invalid CUDA version format: $cuda_version"
log_error "Expected format: X.Y.Z (e.g., 12.8.1)"
return 1
fi
echo "cu${major}${minor}"
}
################################################################################
# Version Extraction
################################################################################
# Extract versions from built Docker image
# Input: image tag (e.g., ktransformers:temp-build-20241212)
# Output: Sets environment variables or prints to stdout
# SGLANG_VERSION=x.y.z
# KTRANSFORMERS_VERSION=x.y.z
# LLAMAFACTORY_VERSION=x.y.z
extract_versions_from_image() {
local image_tag="$1"
log_step "Extracting versions from image: $image_tag"
# Check if image exists
if ! docker image inspect "$image_tag" &>/dev/null; then
log_error "Image not found: $image_tag"
return 1
fi
# Extract versions.env file from the image
local versions_content
versions_content=$(docker run --rm "$image_tag" cat /workspace/versions.env 2>/dev/null)
if [ -z "$versions_content" ]; then
log_error "Failed to extract versions from image"
log_error "The /workspace/versions.env file may not exist in the image"
return 1
fi
# Parse and display versions
log_info "Extracted versions:"
echo "$versions_content" | while IFS= read -r line; do
log_info " $line"
done
# Output the content (caller can parse this or eval it)
echo "$versions_content"
}
# Validate that all required versions were extracted
# Input: versions string (output from extract_versions_from_image)
validate_versions() {
local versions="$1"
local all_valid=true
# Check each required version
for var in SGLANG_VERSION KTRANSFORMERS_VERSION LLAMAFACTORY_VERSION; do
local value
value=$(echo "$versions" | grep "^${var}=" | cut -d= -f2)
if [ -z "$value" ]; then
log_error "Missing version: $var"
all_valid=false
elif [ "$value" = "unknown" ]; then
log_warning "Version is 'unknown': $var"
# Don't fail, but warn user
fi
done
if [ "$all_valid" = false ]; then
return 1
fi
return 0
}
################################################################################
# Image Naming
################################################################################
# Generate standardized image name
# Input:
# $1: versions string (from extract_versions_from_image)
# $2: cuda_version (e.g., 12.8.1)
# $3: cpu_variant (e.g., x86-intel-multi)
# $4: functionality (e.g., sft_llamafactory or infer)
# $5: timestamp (optional, will generate if not provided)
# Output: Standardized image name
# Format: sglang-v{ver}_ktransformers-v{ver}_{cpu}_{gpu}_{func}_{timestamp}
generate_image_name() {
local versions="$1"
local cuda_version="$2"
local cpu_variant="$3"
local functionality="$4"
local timestamp="${5:-$(get_beijing_timestamp)}"
# Parse versions from the versions string
local sglang_ver ktrans_ver llama_ver
sglang_ver=$(echo "$versions" | grep "^SGLANG_VERSION=" | cut -d= -f2)
ktrans_ver=$(echo "$versions" | grep "^KTRANSFORMERS_VERSION=" | cut -d= -f2)
llama_ver=$(echo "$versions" | grep "^LLAMAFACTORY_VERSION=" | cut -d= -f2)
# Validate versions were extracted
if [ -z "$sglang_ver" ] || [ -z "$ktrans_ver" ]; then
log_error "Failed to parse versions from input"
return 1
fi
# Parse CUDA short version
local cuda_short
cuda_short=$(parse_cuda_short_version "$cuda_version")
# Build functionality string
local func_str
if [ "$functionality" = "sft" ]; then
func_str="sft_llamafactory-v${llama_ver}"
else
func_str="infer"
fi
# Generate full image name
# Format: sglang-v{ver}_ktransformers-v{ver}_{cpu}_{gpu}_{func}_{timestamp}
local image_name
image_name="sglang-v${sglang_ver}_ktransformers-v${ktrans_ver}_${cpu_variant}_${cuda_short}_${func_str}_${timestamp}"
echo "$image_name"
}
# Generate simplified tag for DockerHub
# Input:
# $1: ktransformers_version (e.g., 0.5.3)
# $2: cuda_version (e.g., 12.8.1)
# Output: Simplified tag (e.g., v0.5.3-cu128)
generate_simplified_tag() {
local ktrans_ver="$1"
local cuda_version="$2"
local cuda_short
cuda_short=$(parse_cuda_short_version "$cuda_version")
echo "v${ktrans_ver}-${cuda_short}"
}
################################################################################
# Validation Functions
################################################################################
# Check if Docker daemon is running
check_docker_running() {
if ! docker info &>/dev/null; then
log_error "Docker daemon is not running"
log_error "Please start Docker and try again"
return 1
fi
return 0
}
# Check if user is logged into Docker registry
# Input: registry (optional, default: docker.io)
check_docker_login() {
local registry="${1:-docker.io}"
# Try to check auth by attempting a trivial operation
if ! docker login --help &>/dev/null; then
log_error "Docker CLI is not available"
return 1
fi
# Note: This is a best-effort check
# docker login status is not always easy to check programmatically
log_info "Assuming Docker login is configured"
log_info "If push fails, please run: docker login $registry"
return 0
}
# Validate CUDA version format
validate_cuda_version() {
local cuda_version="$1"
if [[ ! "$cuda_version" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then
log_error "Invalid CUDA version format: $cuda_version"
log_error "Expected format: X.Y or X.Y.Z (e.g., 12.8 or 12.8.1)"
return 1
fi
return 0
}
# Check available disk space
# Input: required space in GB
check_disk_space() {
local required_gb="$1"
local output_dir="${2:-.}"
# Get available space in GB (works on Linux and macOS)
local available_kb
if df -k "$output_dir" &>/dev/null; then
available_kb=$(df -k "$output_dir" | tail -1 | awk '{print $4}')
local available_gb=$((available_kb / 1024 / 1024))
log_info "Available disk space: ${available_gb}GB"
if [ "$available_gb" -lt "$required_gb" ]; then
log_warning "Low disk space: ${available_gb}GB available, ${required_gb}GB recommended"
return 1
fi
else
log_warning "Unable to check disk space"
fi
return 0
}
# Check if file/directory exists and is writable
check_writable() {
local path="$1"
if [ -e "$path" ]; then
if [ ! -w "$path" ]; then
log_error "Path exists but is not writable: $path"
return 1
fi
else
# Try to create parent directory to test writability
local parent_dir
parent_dir=$(dirname "$path")
if [ ! -w "$parent_dir" ]; then
log_error "Parent directory is not writable: $parent_dir"
return 1
fi
fi
return 0
}
################################################################################
# Cleanup Functions
################################################################################
# Remove intermediate Docker images
cleanup_temp_images() {
local image_tag="$1"
log_step "Cleaning up temporary image: $image_tag"
if docker image inspect "$image_tag" &>/dev/null; then
docker rmi "$image_tag" &>/dev/null || true
log_success "Cleaned up temporary image"
fi
}
################################################################################
# Display Functions
################################################################################
# Display a summary box
display_summary() {
local title="$1"
shift
local lines=("$@")
local width=80
local border=$(printf '=%.0s' $(seq 1 $width))
echo ""
echo "$border"
echo " $title"
echo "$border"
for line in "${lines[@]}"; do
echo " $line"
done
echo "$border"
echo ""
}
################################################################################
# Export functions
################################################################################
# Export all functions so they can be used by scripts that source this file
export -f log_info log_success log_warning log_error log_step
export -f get_beijing_timestamp
export -f parse_cuda_short_version
export -f extract_versions_from_image validate_versions
export -f generate_image_name generate_simplified_tag
export -f check_docker_running check_docker_login validate_cuda_version
export -f check_disk_space check_writable
export -f cleanup_temp_images
export -f display_summary
docker/push-to-dockerhub.sh
Executable
+1144
View File
File diff suppressed because it is too large Load Diff