chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
ARG DOCKER_IMAGE_BASE_TEST=cr.ray.io/rayproject/oss-ci-base_test
|
||||
FROM $DOCKER_IMAGE_BASE_TEST
|
||||
|
||||
ARG RAYCI_DISABLE_JAVA=false
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash -i
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "$RAYCI_DISABLE_JAVA" != "true" ]]; then
|
||||
apt-get update -y
|
||||
apt-get install -y -qq maven openjdk-8-jre openjdk-8-jdk
|
||||
# Ensure Java 8 is the default; Ubuntu 22.04 defaults to Java 11.
|
||||
if [[ "$(dpkg --print-architecture)" == "arm64" ]]; then
|
||||
update-alternatives --set java /usr/lib/jvm/java-8-openjdk-arm64/jre/bin/java
|
||||
else
|
||||
update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
|
||||
fi
|
||||
fi
|
||||
|
||||
BUILD=1 ./ci/ci.sh init
|
||||
|
||||
EOF
|
||||
|
||||
ENV CC=clang
|
||||
ENV CXX=clang++-14
|
||||
@@ -0,0 +1,17 @@
|
||||
name: "oss-ci-base_build-aarch64"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_test-aarch64"]
|
||||
dockerfile: ci/docker/base.build.Dockerfile
|
||||
srcs:
|
||||
- .bazelrc
|
||||
- ci/ci.sh
|
||||
- ci/env/install-dependencies.sh
|
||||
- ci/env/install-llvm-binaries.sh
|
||||
- ci/suppress_output
|
||||
- python/requirements.txt
|
||||
- python/requirements_compiled.txt
|
||||
- python/requirements/test-requirements.txt
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_TEST=cr.ray.io/rayproject/oss-ci-base_test-aarch64
|
||||
- RAYCI_DISABLE_JAVA
|
||||
tags:
|
||||
- cr.ray.io/rayproject/oss-ci-base_build-aarch64
|
||||
@@ -0,0 +1,17 @@
|
||||
name: "oss-ci-base_build-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON"]
|
||||
dockerfile: ci/docker/base.build.Dockerfile
|
||||
srcs:
|
||||
- .bazelrc
|
||||
- ci/ci.sh
|
||||
- ci/env/install-dependencies.sh
|
||||
- ci/env/install-llvm-binaries.sh
|
||||
- ci/suppress_output
|
||||
- python/requirements.txt
|
||||
- python/requirements_compiled.txt
|
||||
- python/requirements/test-requirements.txt
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_TEST=cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON
|
||||
- RAYCI_DISABLE_JAVA
|
||||
tags:
|
||||
- cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
|
||||
@@ -0,0 +1,18 @@
|
||||
name: "oss-ci-base_cu128-py$PYTHON"
|
||||
froms: ["nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04"]
|
||||
dockerfile: ci/docker/base.gpu.Dockerfile
|
||||
srcs:
|
||||
- .bazelrc
|
||||
- .bazelversion
|
||||
- ci/env/install-dependencies.sh
|
||||
- ci/env/install-llvm-binaries.sh
|
||||
- ci/env/install-bazel.sh
|
||||
- ci/env/install-miniforge.sh
|
||||
- ci/suppress_output
|
||||
build_args:
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
- PYTHON
|
||||
- BASE_IMAGE=nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04
|
||||
- CUDA_VERSION=12.8.1
|
||||
tags:
|
||||
- cr.ray.io/rayproject/oss-ci-base_cu128-py$PYTHON
|
||||
@@ -0,0 +1,18 @@
|
||||
name: "oss-ci-base_cu130-py$PYTHON"
|
||||
froms: ["nvidia/cuda:13.0.0-cudnn-devel-ubuntu22.04"]
|
||||
dockerfile: ci/docker/base.gpu.Dockerfile
|
||||
srcs:
|
||||
- .bazelrc
|
||||
- .bazelversion
|
||||
- ci/env/install-dependencies.sh
|
||||
- ci/env/install-llvm-binaries.sh
|
||||
- ci/env/install-bazel.sh
|
||||
- ci/env/install-miniforge.sh
|
||||
- ci/suppress_output
|
||||
build_args:
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
- PYTHON
|
||||
- BASE_IMAGE=nvidia/cuda:13.0.0-cudnn-devel-ubuntu22.04
|
||||
- CUDA_VERSION=13.0.0
|
||||
tags:
|
||||
- cr.ray.io/rayproject/oss-ci-base_cu130-py$PYTHON
|
||||
@@ -0,0 +1,72 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
ARG BASE_IMAGE=nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04
|
||||
FROM $BASE_IMAGE
|
||||
|
||||
ARG BUILDKITE_BAZEL_CACHE_URL
|
||||
ARG PYTHON=3.10
|
||||
ARG CUDA_VERSION=12.8.1
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV TZ=America/Los_Angeles
|
||||
|
||||
ENV RAY_BUILD_ENV=ubuntu22.04_clang14_cuda${CUDA_VERSION}_py$PYTHON
|
||||
ENV BUILDKITE=true
|
||||
ENV CI=true
|
||||
ENV PYTHON=$PYTHON
|
||||
ENV RAY_USE_RANDOM_PORTS=1
|
||||
ENV RAY_DEFAULT_BUILD=1
|
||||
ENV RAY_INSTALL_JAVA=0
|
||||
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
apt-get update -qq && apt-get upgrade -qq
|
||||
apt-get install -y -qq \
|
||||
curl python-is-python3 git build-essential \
|
||||
sudo zip unzip unrar apt-utils dialog tzdata wget rsync \
|
||||
language-pack-en tmux cmake gdb vim htop \
|
||||
libgtk2.0-dev zlib1g-dev libgl1-mesa-dev \
|
||||
clang-format-14 jq \
|
||||
clang-tidy-14 clang-14
|
||||
ln -s /usr/bin/clang-format-14 /usr/bin/clang-format
|
||||
ln -s /usr/bin/clang-tidy-14 /usr/bin/clang-tidy
|
||||
ln -s /usr/bin/clang-14 /usr/bin/clang
|
||||
|
||||
# Install docker CLI
|
||||
mkdir -p /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
|
||||
chmod a+r /etc/apt/keyrings/docker.asc
|
||||
echo \
|
||||
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
|
||||
$(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
|
||||
tee /etc/apt/sources.list.d/docker.list > /dev/null
|
||||
apt-get update
|
||||
apt-get install -y docker-ce-cli
|
||||
|
||||
echo "build --remote_cache=${BUILDKITE_BAZEL_CACHE_URL}" >> /root/.bazelrc
|
||||
|
||||
curl -fsSL https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="/usr/local/bin" sh
|
||||
|
||||
EOF
|
||||
|
||||
ENV CC=clang
|
||||
ENV CXX=clang++-14
|
||||
|
||||
# System conf for tests
|
||||
RUN locale -a
|
||||
ENV LC_ALL=en_US.utf8
|
||||
ENV LANG=en_US.utf8
|
||||
RUN echo "ulimit -c 0" >> /root/.bashrc
|
||||
|
||||
# Install some dependencies (miniforge, pip dependencies, etc)
|
||||
RUN mkdir /ray
|
||||
WORKDIR /ray
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN bash --login -ie -c '\
|
||||
BUILD=1 SKIP_PYTHON_PACKAGES=1 ./ci/env/install-dependencies.sh \
|
||||
'
|
||||
@@ -0,0 +1,17 @@
|
||||
name: "oss-ci-base_gpu-py$PYTHON"
|
||||
froms: ["nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04"]
|
||||
dockerfile: ci/docker/base.gpu.Dockerfile
|
||||
srcs:
|
||||
- .bazelrc
|
||||
- .bazelversion
|
||||
- ci/env/install-dependencies.sh
|
||||
- ci/env/install-llvm-binaries.sh
|
||||
- ci/env/install-bazel.sh
|
||||
- ci/env/install-miniforge.sh
|
||||
- ci/suppress_output
|
||||
build_args:
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
- PYTHON
|
||||
- CUDA_VERSION
|
||||
tags:
|
||||
- cr.ray.io/rayproject/oss-ci-base_gpu-py$PYTHON
|
||||
@@ -0,0 +1,18 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
ARG DOCKER_IMAGE_BASE_TEST=cr.ray.io/rayproject/oss-ci-base_test
|
||||
FROM $DOCKER_IMAGE_BASE_TEST
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash -i
|
||||
|
||||
set -e
|
||||
|
||||
BUILD=1 ./ci/ci.sh init
|
||||
RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh
|
||||
|
||||
pip uninstall -y ray
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,24 @@
|
||||
name: "oss-ci-base_ml-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON"]
|
||||
dockerfile: ci/docker/base.ml.Dockerfile
|
||||
srcs:
|
||||
- .bazelrc
|
||||
- ci/ci.sh
|
||||
- ci/env/install-dependencies.sh
|
||||
- ci/env/install-llvm-binaries.sh
|
||||
- ci/suppress_output
|
||||
- python/requirements.txt
|
||||
- python/requirements_compiled.txt
|
||||
- python/requirements/test-requirements.txt
|
||||
- python/requirements/ml/rllib-requirements.txt
|
||||
- python/requirements/ml/rllib-test-requirements.txt
|
||||
- python/requirements/ml/train-requirements.txt
|
||||
- python/requirements/ml/train-test-requirements.txt
|
||||
- python/requirements/ml/tune-requirements.txt
|
||||
- python/requirements/ml/tune-test-requirements.txt
|
||||
- python/requirements/ml/dl-cpu-requirements.txt
|
||||
- python/requirements/ml/core-requirements.txt
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_TEST=cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON
|
||||
tags:
|
||||
- cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
|
||||
@@ -0,0 +1,68 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
FROM ubuntu:jammy
|
||||
|
||||
ARG BUILDKITE_BAZEL_CACHE_URL
|
||||
ARG PYTHON=3.10
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV TZ=America/Los_Angeles
|
||||
|
||||
ENV RAY_BUILD_ENV=ubuntu22.04_clang14_py$PYTHON
|
||||
ENV BUILDKITE=true
|
||||
ENV CI=true
|
||||
ENV PYTHON=$PYTHON
|
||||
ENV RAY_USE_RANDOM_PORTS=1
|
||||
ENV RAY_DEFAULT_BUILD=1
|
||||
ENV RAY_INSTALL_JAVA=0
|
||||
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
apt-get update -qq && apt-get upgrade -qq
|
||||
apt-get install -y -qq \
|
||||
curl python-is-python3 git build-essential \
|
||||
sudo zip unzip unrar apt-utils dialog tzdata wget rsync \
|
||||
language-pack-en tmux cmake gdb vim htop graphviz \
|
||||
libgtk2.0-dev zlib1g-dev libgl1-mesa-dev \
|
||||
liblz4-dev libunwind-dev libncurses6 \
|
||||
clang-format-14 jq \
|
||||
clang-tidy-14 clang-14
|
||||
|
||||
ln -s /usr/bin/clang-format-14 /usr/bin/clang-format
|
||||
ln -s /usr/bin/clang-tidy-14 /usr/bin/clang-tidy
|
||||
ln -s /usr/bin/clang-14 /usr/bin/clang
|
||||
|
||||
# Install docker CLI
|
||||
mkdir -p /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
|
||||
chmod a+r /etc/apt/keyrings/docker.asc
|
||||
echo \
|
||||
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
|
||||
$(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
|
||||
tee /etc/apt/sources.list.d/docker.list > /dev/null
|
||||
apt-get update
|
||||
apt-get install -y docker-ce-cli
|
||||
|
||||
EOF
|
||||
|
||||
ENV CC=clang
|
||||
ENV CXX=clang++-14
|
||||
|
||||
# System conf for tests
|
||||
RUN locale -a
|
||||
ENV LC_ALL=en_US.utf8
|
||||
ENV LANG=en_US.utf8
|
||||
RUN echo "ulimit -c 0" >> /root/.bashrc
|
||||
|
||||
# Install some dependencies (miniforge, pip dependencies, etc)
|
||||
RUN mkdir /ray
|
||||
WORKDIR /ray
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN ./ci/env/install-miniforge.sh
|
||||
RUN ./ci/env/install-bazel.sh
|
||||
@@ -0,0 +1,13 @@
|
||||
name: "oss-ci-base_test-aarch64"
|
||||
froms: ["ubuntu:jammy"]
|
||||
dockerfile: ci/docker/base.test.Dockerfile
|
||||
srcs:
|
||||
- ci/env/install-bazel.sh
|
||||
- ci/env/install-miniforge.sh
|
||||
- ci/suppress_output
|
||||
- .bazelversion
|
||||
build_args:
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
- PYTHON=3.10
|
||||
tags:
|
||||
- cr.ray.io/rayproject/oss-ci-base_test-aarch64
|
||||
@@ -0,0 +1,13 @@
|
||||
name: "oss-ci-base_test-py$PYTHON"
|
||||
froms: ["ubuntu:jammy"]
|
||||
dockerfile: ci/docker/base.test.Dockerfile
|
||||
srcs:
|
||||
- ci/env/install-bazel.sh
|
||||
- ci/env/install-miniforge.sh
|
||||
- ci/suppress_output
|
||||
- .bazelversion
|
||||
build_args:
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
- PYTHON
|
||||
tags:
|
||||
- cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON
|
||||
@@ -0,0 +1,23 @@
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py3.10
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ARG PYTHON=3.10
|
||||
ARG BUILD_VARIANT=build
|
||||
ARG PYTHON_DEPSET=python/deplocks/ci/core-${BUILD_VARIANT}-ci_depset_py${PYTHON}.lock
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
|
||||
|
||||
uv pip uninstall --system ray
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,11 @@
|
||||
name: "core$BUILD_VARIANT-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON"]
|
||||
dockerfile: ci/docker/core.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/ci/core-$BUILD_VARIANT-ci_depset_py$PYTHON.lock
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON
|
||||
- PYTHON
|
||||
- BUILD_VARIANT
|
||||
tags:
|
||||
- cr.ray.io/rayproject/core$BUILD_VARIANT-py$PYTHON
|
||||
@@ -0,0 +1,10 @@
|
||||
# Dedicated image for the Ray Serve LLM custom-vLLM-model docs test. It layers
|
||||
# the custom reward-model plugin on top of the standard LLM GPU test image so
|
||||
# the plugin is available on every node without touching the shared image.
|
||||
ARG DOCKER_IMAGE_BASE=cr.ray.io/rayproject/llmgpubuild
|
||||
FROM $DOCKER_IMAGE_BASE
|
||||
|
||||
COPY . .
|
||||
|
||||
# The plugin's entry point registers Qwen3CustomRewardModel in every vLLM process.
|
||||
RUN pip install --no-deps doc/source/llm/doc_code/serve/custom_vllm
|
||||
@@ -0,0 +1,13 @@
|
||||
name: "custom-vllm-plugin-gpubuild"
|
||||
froms: ["cr.ray.io/rayproject/llmgpubuild"]
|
||||
dockerfile: ci/docker/custom-vllm-plugin.build.Dockerfile
|
||||
srcs:
|
||||
# The plugin source, installed on top of the LLM GPU image (see the Dockerfile).
|
||||
- doc/source/llm/doc_code/serve/custom_vllm/setup.py
|
||||
- doc/source/llm/doc_code/serve/custom_vllm/qwen3_reward_plugin/__init__.py
|
||||
- doc/source/llm/doc_code/serve/custom_vllm/qwen3_reward_plugin/qwen3_rm.py
|
||||
- doc/source/llm/doc_code/serve/custom_vllm/qwen3_reward_plugin/serve_hook.py
|
||||
tags:
|
||||
- cr.ray.io/rayproject/custom-vllm-plugin-gpubuild
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE=cr.ray.io/rayproject/llmgpubuild
|
||||
@@ -0,0 +1,52 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py3.10
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ARG RAY_CI_JAVA_BUILD=
|
||||
ARG IMAGE_TYPE=base
|
||||
ARG PYTHON=3.10
|
||||
ARG PYTHON_DEPSET=python/deplocks/ci/data-$IMAGE_TYPE-ci_depset_py$PYTHON.lock
|
||||
|
||||
COPY $PYTHON_DEPSET /home/ray/python_depset.lock
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
|
||||
|
||||
if [[ "$IMAGE_TYPE" == "pyarrow-nightly" ]]; then
|
||||
uv pip install \
|
||||
--system \
|
||||
--prerelease allow \
|
||||
--extra-index-url https://pypi.fury.io/arrow-nightlies/ \
|
||||
--upgrade-package pyarrow \
|
||||
pyarrow
|
||||
fi
|
||||
|
||||
curl -fsSL https://pgp.mongodb.com/server-8.0.asc | \
|
||||
sudo gpg -o /usr/share/keyrings/mongodb-server-8.0.gpg --dearmor
|
||||
echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-8.0.gpg ] \
|
||||
https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/8.0 multiverse" | \
|
||||
sudo tee /etc/apt/sources.list.d/mongodb-org-8.0.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y mongodb-org
|
||||
|
||||
if [[ $RAY_CI_JAVA_BUILD == 1 ]]; then
|
||||
# These packages increase the image size quite a bit, so we only install them
|
||||
# as needed.
|
||||
sudo apt-get install -y -qq maven openjdk-8-jre openjdk-8-jdk
|
||||
# Ensure Java 8 is the default; Ubuntu 22.04 defaults to Java 11 which
|
||||
# breaks Spark's reflective access to DirectByteBuffer.
|
||||
if [[ "$(dpkg --print-architecture)" == "arm64" ]]; then
|
||||
sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-arm64/jre/bin/java
|
||||
else
|
||||
sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
|
||||
fi
|
||||
fi
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,11 @@
|
||||
name: "data17build-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
|
||||
dockerfile: ci/docker/data.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/ci/data-pyarrow-v17-ci_depset_py$PYTHON.lock
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
|
||||
- IMAGE_TYPE=pyarrow-v17
|
||||
- PYTHON=$PYTHON
|
||||
tags:
|
||||
- cr.ray.io/rayproject/data17build-py$PYTHON
|
||||
@@ -0,0 +1,11 @@
|
||||
name: "datalbuild-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
|
||||
dockerfile: ci/docker/data.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/ci/data-pyarrow-latest-ci_depset_py$PYTHON.lock
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
|
||||
- IMAGE_TYPE=pyarrow-latest
|
||||
- PYTHON=$PYTHON
|
||||
tags:
|
||||
- cr.ray.io/rayproject/datalbuild-py$PYTHON
|
||||
@@ -0,0 +1,12 @@
|
||||
name: "datamongobuild-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
|
||||
dockerfile: ci/docker/data.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/ci/data-mongo-ci_depset_py$PYTHON.lock
|
||||
build_args:
|
||||
- RAY_CI_JAVA_BUILD=1
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
|
||||
- IMAGE_TYPE=mongo
|
||||
- PYTHON=$PYTHON
|
||||
tags:
|
||||
- cr.ray.io/rayproject/datamongobuild-py$PYTHON
|
||||
@@ -0,0 +1,11 @@
|
||||
name: "datanbuild-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
|
||||
dockerfile: ci/docker/data.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/ci/data-pyarrow-nightly-ci_depset_py$PYTHON.lock
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
|
||||
- IMAGE_TYPE=pyarrow-nightly
|
||||
- PYTHON=$PYTHON
|
||||
tags:
|
||||
- cr.ray.io/rayproject/datanbuild-py$PYTHON
|
||||
@@ -0,0 +1,24 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py3.10
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ARG ARROW_VERSION=14.*
|
||||
ARG ARROW_MONGO_VERSION=
|
||||
ARG RAY_CI_JAVA_BUILD=
|
||||
ARG PYTHON=3.12
|
||||
ARG PYTHON_DEPSET=python/deplocks/tests/data/datatfds_py$PYTHON.lock
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,10 @@
|
||||
name: "datatfdsbuild-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
|
||||
dockerfile: ci/docker/datatfds.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/tests/data/datatfds_py$PYTHON.lock
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
|
||||
- PYTHON
|
||||
tags:
|
||||
- cr.ray.io/rayproject/datatfdsbuild-py$PYTHON
|
||||
@@ -0,0 +1,29 @@
|
||||
ARG DOCKER_IMAGE_RAY_CORE=cr.ray.io/rayproject/ray-core-py3.10
|
||||
ARG DOCKER_IMAGE_RAY_DASHBOARD=cr.ray.io/rayproject/ray-dashboard
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py3.10
|
||||
|
||||
FROM $DOCKER_IMAGE_RAY_CORE AS ray_core
|
||||
FROM $DOCKER_IMAGE_RAY_DASHBOARD AS ray_dashboard
|
||||
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
COPY . .
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
RUN --mount=type=bind,from=ray_core,target=/mnt/ray-core \
|
||||
--mount=type=bind,from=ray_dashboard,target=/mnt/ray-dashboard \
|
||||
<<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p /opt/ray-build
|
||||
|
||||
cp /mnt/ray-core/ray_pkg.zip /opt/ray-build/ray_pkg.zip
|
||||
cp /mnt/ray-core/ray_py_proto.zip /opt/ray-build/ray_py_proto.zip
|
||||
cp /mnt/ray-dashboard/dashboard.tar.gz /opt/ray-build/dashboard.tar.gz
|
||||
|
||||
pip install -r python/deplocks/docs/docbuild_depset_py${PYTHON}.lock
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,14 @@
|
||||
name: "docbuild-py$PYTHON"
|
||||
froms:
|
||||
- cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
|
||||
- cr.ray.io/rayproject/ray-core-py$PYTHON
|
||||
- cr.ray.io/rayproject/ray-dashboard
|
||||
dockerfile: ci/docker/doc.build.Dockerfile
|
||||
srcs:
|
||||
- $REQUIREMENTS_FILE
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
|
||||
- DOCKER_IMAGE_RAY_CORE=cr.ray.io/rayproject/ray-core-py$PYTHON
|
||||
- DOCKER_IMAGE_RAY_DASHBOARD=cr.ray.io/rayproject/ray-dashboard
|
||||
tags:
|
||||
- cr.ray.io/rayproject/docbuild-py$PYTHON
|
||||
@@ -0,0 +1,25 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_gpu-py3.10
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ARG PYTHON
|
||||
ARG PYTHON_DEPSET=python/deplocks/ci/docgpu_depset_py$PYTHON.lock
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
|
||||
|
||||
# Remove installed ray so the source overlay at /rayci/ is used at test time
|
||||
pip uninstall -y ray
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,10 @@
|
||||
name: "docgpubuild-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_gpu-py$PYTHON"]
|
||||
dockerfile: ci/docker/docgpu.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/ci/docgpu_depset_py$PYTHON.lock
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_gpu-py$PYTHON
|
||||
- PYTHON
|
||||
tags:
|
||||
- cr.ray.io/rayproject/docgpubuild-py$PYTHON
|
||||
@@ -0,0 +1,22 @@
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ARG PYTHON=3.10
|
||||
ARG PYTHON_DEPSET=python/deplocks/docs/doctest_depset_py$PYTHON.lock
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
COPY . .
|
||||
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y tesseract-ocr libosmesa6-dev libglfw3 patchelf
|
||||
|
||||
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,10 @@
|
||||
name: "doctestbuild"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON"]
|
||||
dockerfile: ci/docker/doctest.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/docs/doctest_depset_py$PYTHON.lock
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
|
||||
- PYTHON
|
||||
tags:
|
||||
- cr.ray.io/rayproject/doctestbuild
|
||||
@@ -0,0 +1,151 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
FROM ubuntu:22.04
|
||||
|
||||
ARG BUILDKITE_BAZEL_CACHE_URL
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV PATH="/home/forge/.local/bin:${PATH}"
|
||||
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
|
||||
ENV RAY_BUILD_ENV=ubuntu22.04_forge
|
||||
|
||||
RUN \
|
||||
--mount=type=bind,source=ci/k8s/install-k8s-tools.sh,target=install-k8s-tools.sh \
|
||||
<<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
apt-get update
|
||||
apt-get upgrade -y
|
||||
apt-get install -y ca-certificates curl zip unzip sudo gnupg tzdata git apt-transport-https lsb-release
|
||||
|
||||
# Add docker client APT repository
|
||||
mkdir -p /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
|
||||
# Download and install Microsoft signing key
|
||||
curl -fsSL https://packages.microsoft.com/keys/microsoft.asc |
|
||||
gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null
|
||||
chmod go+r /etc/apt/keyrings/microsoft.gpg
|
||||
|
||||
echo \
|
||||
"deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
|
||||
"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
|
||||
tee /etc/apt/sources.list.d/docker.list > /dev/null
|
||||
|
||||
# Add NodeJS APT repository
|
||||
curl -fsSL https://deb.nodesource.com/setup_18.x | bash -
|
||||
|
||||
AZ_VER=2.72.0
|
||||
AZ_DIST="$(lsb_release -cs)"
|
||||
|
||||
# Add Azure CLI repository
|
||||
echo "Types: deb
|
||||
URIs: https://packages.microsoft.com/repos/azure-cli/
|
||||
Suites: ${AZ_DIST}
|
||||
Components: main
|
||||
Architectures: $(dpkg --print-architecture)
|
||||
Signed-by: /etc/apt/keyrings/microsoft.gpg" | tee /etc/apt/sources.list.d/azure-cli.sources
|
||||
|
||||
# Add Google Cloud CLI repository
|
||||
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg |
|
||||
gpg --dearmor -o /etc/apt/keyrings/cloud.google.gpg
|
||||
echo "deb [signed-by=/etc/apt/keyrings/cloud.google.gpg] \
|
||||
https://packages.cloud.google.com/apt cloud-sdk main" |
|
||||
tee /etc/apt/sources.list.d/google-cloud-sdk.list
|
||||
|
||||
# Install packages
|
||||
|
||||
apt-get update
|
||||
apt-get install -y \
|
||||
awscli nodejs build-essential python-is-python3 \
|
||||
python3-pip openjdk-8-jre wget jq \
|
||||
docker-ce-cli azure-cli="${AZ_VER}"-1~"${AZ_DIST}" \
|
||||
google-cloud-cli
|
||||
|
||||
# Install uv
|
||||
curl -fsSL https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="/usr/local/bin" sh
|
||||
|
||||
mkdir -p /usr/local/python
|
||||
# Install Python using uv
|
||||
UV_PYTHON_VERSION=3.10
|
||||
uv python install --install-dir /usr/local/python "$UV_PYTHON_VERSION"
|
||||
|
||||
export UV_PYTHON_INSTALL_DIR=/usr/local/python
|
||||
# Make Python from uv the default by creating symlinks
|
||||
UV_PYTHON_BIN="$(uv python find --no-project "$UV_PYTHON_VERSION")"
|
||||
echo "uv python binary location: $UV_PYTHON_BIN"
|
||||
ln -s "$UV_PYTHON_BIN" "/usr/local/bin/python${UV_PYTHON_VERSION}"
|
||||
ln -s "$UV_PYTHON_BIN" /usr/local/bin/python3
|
||||
ln -s "$UV_PYTHON_BIN" /usr/local/bin/python
|
||||
|
||||
# As a convention, we pin all python packages to a specific version. This
|
||||
# is to to make sure we can control version upgrades through code changes.
|
||||
uv pip install --system pip==25.0 cffi==1.16.0
|
||||
|
||||
# Needs to be synchronized to the host group id as we map /var/run/docker.sock
|
||||
# into the container.
|
||||
addgroup --gid 1001 docker0 # Used on old buildkite AMIs before 2023
|
||||
addgroup --gid 993 docker1
|
||||
addgroup --gid 992 docker # buildkite AMI as of 2025-06-07
|
||||
|
||||
# Install bazelisk
|
||||
npm install -g @bazel/bazelisk
|
||||
ln -s /usr/local/bin/bazel /usr/local/bin/bazelisk
|
||||
|
||||
# A non-root user. Use 2000, which is the same as our buildkite agent VM uses.
|
||||
adduser --home /home/forge --uid 2000 forge --gid 100
|
||||
usermod -a -G docker0 forge
|
||||
usermod -a -G docker1 forge
|
||||
usermod -a -G docker forge
|
||||
|
||||
# Create a shared directory for the ray repository checkout.
|
||||
mkdir /rayci
|
||||
chown forge:users /rayci
|
||||
|
||||
if [[ "$(uname -i)" == "x86_64" ]]; then
|
||||
bash install-k8s-tools.sh
|
||||
fi
|
||||
|
||||
# Install crane (container registry tool)
|
||||
CRANE_VERSION=0.19.0
|
||||
case "$(uname -m)" in
|
||||
x86_64|amd64)
|
||||
CRANE_ARCH="x86_64"
|
||||
;;
|
||||
aarch64|arm64)
|
||||
CRANE_ARCH="arm64"
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported architecture: $(uname -m)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
curl -fsSL "https://github.com/google/go-containerregistry/releases/download/v${CRANE_VERSION}/go-containerregistry_Linux_${CRANE_ARCH}.tar.gz" \
|
||||
| tar -xzf - -C /usr/local/bin crane
|
||||
|
||||
chmod +x /usr/local/bin/crane
|
||||
EOF
|
||||
|
||||
USER forge
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
{
|
||||
echo "build --config=ci"
|
||||
echo "build --announce_rc"
|
||||
echo "build --remote_cache=${BUILDKITE_BAZEL_CACHE_URL}"
|
||||
} > ~/.bazelrc
|
||||
|
||||
EOF
|
||||
|
||||
ENV DOCKER_API_VERSION=1.43
|
||||
|
||||
CMD ["echo", "ray forge"]
|
||||
|
||||
|
||||
# last update: 2026-02-10
|
||||
@@ -0,0 +1,8 @@
|
||||
name: "forge-aarch64"
|
||||
froms:
|
||||
- ubuntu:22.04
|
||||
build_args:
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
dockerfile: ci/docker/forge.Dockerfile
|
||||
srcs:
|
||||
- ci/k8s/install-k8s-tools.sh
|
||||
@@ -0,0 +1,8 @@
|
||||
name: "forge"
|
||||
froms:
|
||||
- ubuntu:22.04
|
||||
build_args:
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
dockerfile: ci/docker/forge.Dockerfile
|
||||
srcs:
|
||||
- ci/k8s/install-k8s-tools.sh
|
||||
@@ -0,0 +1,36 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
FROM cr.ray.io/rayproject/forge
|
||||
|
||||
# Add forge user to sudo group
|
||||
USER root
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
usermod -aG sudo forge && \
|
||||
echo "forge ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/forge && \
|
||||
chmod 0440 /etc/sudoers.d/forge
|
||||
|
||||
EOF
|
||||
|
||||
USER forge
|
||||
|
||||
# Install fossa and askalono
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
curl -sSfL https://github.com/fossas/fossa-cli/releases/download/v3.10.6/fossa_3.10.6_linux_amd64.tar.gz \
|
||||
| sudo tar -xvzf - -C /usr/local/bin/
|
||||
|
||||
wget -O /tmp/askalono.zip https://github.com/jpeddicord/askalono/releases/download/0.5.0/askalono-Linux.zip
|
||||
unzip -d /tmp/ /tmp/askalono.zip
|
||||
sudo mv /tmp/askalono /usr/local/bin/
|
||||
rm /tmp/askalono.zip
|
||||
|
||||
EOF
|
||||
|
||||
CMD ["echo", "ray fossa"]
|
||||
@@ -0,0 +1,4 @@
|
||||
name: "fossa-base"
|
||||
froms:
|
||||
- cr.ray.io/rayproject/forge
|
||||
dockerfile: ci/docker/fossa.Dockerfile
|
||||
@@ -0,0 +1,22 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
FROM cr.ray.io/rayproject/oss-ci-base_build-py3.10
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
bash ci/k8s/install-k8s-tools.sh
|
||||
|
||||
pip install -U --ignore-installed \
|
||||
-r python/requirements.txt -c python/requirements_compiled.txt \
|
||||
|
||||
pip install -U --ignore-installed \
|
||||
docker -c python/requirements_compiled.txt \
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,9 @@
|
||||
name: k8sbuild
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_build-py3.10"]
|
||||
dockerfile: ci/docker/k8s.build.Dockerfile
|
||||
srcs:
|
||||
- python/requirements.txt
|
||||
- python/requirements_compiled.txt
|
||||
- ci/k8s/install-k8s-tools.sh
|
||||
tags:
|
||||
- cr.ray.io/rayproject/k8sbuild
|
||||
@@ -0,0 +1,59 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py3.11
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ARG RAY_CI_JAVA_BUILD=
|
||||
ARG RAY_CUDA_CODE=cpu
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SKIP_PYTHON_PACKAGES=1 ./ci/env/install-dependencies.sh
|
||||
|
||||
PYTHON_CODE="$(python -c "import sys; v=sys.version_info; print(f'py{v.major}{v.minor}')")"
|
||||
pip install --no-deps -r python/deplocks/llm/rayllm_test_${PYTHON_CODE}_${RAY_CUDA_CODE}.lock
|
||||
|
||||
# Include the CUDA device index in vLLM's compile cache paths so a worker never
|
||||
# reloads a torch.compile artifact built for a different physical GPU.
|
||||
# TODO (jeffreywang): Remove this patch once https://github.com/vllm-project/vllm/pull/38962 lands.
|
||||
VLLM_DEVICE_AWARE_COMPILE_CACHE_PATCH="$(pwd)/python/requirements/llm/patches/vllm-device-aware-compile-cache.patch"
|
||||
VLLM_SITE_PACKAGES="$(python - <<'PY'
|
||||
import site
|
||||
import sysconfig
|
||||
from pathlib import Path
|
||||
|
||||
candidate_dirs = [
|
||||
Path(sysconfig.get_paths()["purelib"]),
|
||||
Path(sysconfig.get_paths()["platlib"]),
|
||||
*(Path(path) for path in site.getsitepackages()),
|
||||
]
|
||||
|
||||
for base_dir in dict.fromkeys(candidate_dirs):
|
||||
import_utils = base_dir / "vllm" / "utils" / "import_utils.py"
|
||||
if import_utils.exists():
|
||||
print(base_dir)
|
||||
break
|
||||
else:
|
||||
raise SystemExit("vLLM import_utils.py not found")
|
||||
PY
|
||||
)"
|
||||
(
|
||||
cd "${VLLM_SITE_PACKAGES}"
|
||||
git apply "${VLLM_DEVICE_AWARE_COMPILE_CACHE_PATCH}"
|
||||
)
|
||||
|
||||
EOF
|
||||
|
||||
|
||||
# vLLM 0.21.0 selects the FlashInfer top-k/top-p sampler during engine initialization
|
||||
# instead of the previous PyTorch-native/Triton sampling path. The FlashInfer sampler
|
||||
# introduces longer adds a large one-time engine initialization cost. To avoid performance
|
||||
# surprises, we disable the FlashInfer sampler by default.
|
||||
ENV VLLM_USE_FLASHINFER_SAMPLER=0
|
||||
@@ -0,0 +1,15 @@
|
||||
name: "llm$BUILD_VARIANT"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON"]
|
||||
dockerfile: ci/docker/llm.build.Dockerfile
|
||||
srcs:
|
||||
- ci/env/install-dependencies.sh
|
||||
- ci/env/install-llvm-binaries.sh
|
||||
- ci/suppress_output
|
||||
- python/deplocks/llm/rayllm_test_py312_cpu.lock
|
||||
- python/deplocks/llm/rayllm_test_py312_cu130.lock
|
||||
- python/requirements/llm/patches/vllm-device-aware-compile-cache.patch
|
||||
tags:
|
||||
- cr.ray.io/rayproject/llm$BUILD_VARIANT
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON
|
||||
- RAY_CUDA_CODE=$RAY_CUDA_CODE
|
||||
@@ -0,0 +1,10 @@
|
||||
name: "manylinux-cibase$JDK_SUFFIX-$HOSTTYPE"
|
||||
froms:
|
||||
- quay.io/pypa/manylinux2014_$HOSTTYPE:2026.01.02-1
|
||||
srcs:
|
||||
- ci/build/build-manylinux-forge.sh
|
||||
build_args:
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
- RAYCI_DISABLE_JAVA
|
||||
- HOSTTYPE
|
||||
dockerfile: ci/docker/manylinux.Dockerfile
|
||||
@@ -0,0 +1,30 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
# NOTE(andrew-anyscale): This Dockerfile is used to retag the manylinux image to
|
||||
# a new tag with Wanda. This is kept as-is until we can rework LinuxContainer to
|
||||
# use a hardcoded image tag not hosted in ECR. See:
|
||||
# https://github.com/ray-project/ray/blob/6ab10189b0f6506158bac76437a97b28c2643155/ci/ray_ci/builder_container.py#L16
|
||||
# https://github.com/ray-project/ray/blob/master/ci/ray_ci/container.py#L57C49-L57C59
|
||||
|
||||
ARG MANYLINUX_VERSION
|
||||
ARG HOSTTYPE
|
||||
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE}
|
||||
|
||||
ARG BUILDKITE_BAZEL_CACHE_URL
|
||||
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
|
||||
|
||||
# Still keep bazelrc updates to allow BUILDKITE_BAZEL_CACHE_URL to be used.
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
{
|
||||
echo "build --config=ci"
|
||||
echo "build --announce_rc"
|
||||
if [[ "${BUILDKITE_BAZEL_CACHE_URL:-}" != "" ]]; then
|
||||
echo "build:ci --remote_cache=${BUILDKITE_BAZEL_CACHE_URL:-}"
|
||||
fi
|
||||
} > "$HOME"/.bazelrc
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,54 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
ARG HOSTTYPE
|
||||
FROM quay.io/pypa/manylinux2014_${HOSTTYPE}:2026.01.02-1
|
||||
|
||||
ARG BUILDKITE_BAZEL_CACHE_URL
|
||||
ARG RAYCI_DISABLE_JAVA=false
|
||||
|
||||
# uid needs to be synced with forge.Dockerfile
|
||||
ARG FORGE_UID=2000
|
||||
|
||||
ENV BUILD_JAR=1
|
||||
ENV RAYCI_DISABLE_JAVA=$RAYCI_DISABLE_JAVA
|
||||
ENV RAY_INSTALL_JAVA=1
|
||||
ENV BUILDKITE_BAZEL_CACHE_URL=$BUILDKITE_BAZEL_CACHE_URL
|
||||
|
||||
RUN yum -y install sudo
|
||||
|
||||
RUN curl -LsSf https://astral.sh/uv/0.8.17/install.sh | \
|
||||
env UV_INSTALL_DIR=/usr/local/bin sh
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
useradd -m -u "$FORGE_UID" -g users -d /home/forge forge
|
||||
echo 'forge ALL=NOPASSWD: ALL' >> /etc/sudoers
|
||||
|
||||
EOF
|
||||
|
||||
COPY ci/build/build-manylinux-forge.sh /tmp/build-manylinux-forge.sh
|
||||
|
||||
RUN ./tmp/build-manylinux-forge.sh
|
||||
|
||||
USER forge
|
||||
ENV HOME=/home/forge
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
{
|
||||
echo "build --config=ci"
|
||||
echo "build --announce_rc"
|
||||
if [[ "${BUILDKITE_BAZEL_CACHE_URL:-}" != "" ]]; then
|
||||
echo "build:ci --remote_cache=${BUILDKITE_BAZEL_CACHE_URL:-}"
|
||||
fi
|
||||
} > "$HOME"/.bazelrc
|
||||
|
||||
EOF
|
||||
|
||||
# last kick: 2025-10-08
|
||||
@@ -0,0 +1,8 @@
|
||||
name: "manylinux-$HOSTTYPE"
|
||||
froms:
|
||||
- rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE
|
||||
build_args:
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
- MANYLINUX_VERSION
|
||||
- HOSTTYPE
|
||||
dockerfile: ci/docker/manylinux-retag.Dockerfile
|
||||
@@ -0,0 +1,77 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
ARG PYTHON_VERSION=3.10
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON_VERSION
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ARG PYTHON_VERSION
|
||||
ARG EXTRA_DEPENDENCY
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# minimal dependencies
|
||||
BUILD=1 MINIMAL_INSTALL=1 PYTHON=${PYTHON_VERSION} ./ci/ci.sh init
|
||||
rm -rf python/ray/thirdparty_files
|
||||
|
||||
# Re-activate conda environment (ci.sh runs in a subprocess so PATH changes are lost)
|
||||
# For Python 3.14+, a separate conda env is created and symlinked to /opt/miniforge/bin
|
||||
# We can't use `conda activate` because the conda command itself breaks after symlinking
|
||||
# (it tries to use the new Python which doesn't have conda installed)
|
||||
if [[ -d /opt/miniforge/envs/py${PYTHON_VERSION} ]]; then
|
||||
export PATH="/opt/miniforge/envs/py${PYTHON_VERSION}/bin:/opt/miniforge/bin:$PATH"
|
||||
else
|
||||
export PATH="/opt/miniforge/bin:$PATH"
|
||||
fi
|
||||
|
||||
# install test requirements
|
||||
python -m pip install -U pytest==8.3.3 pip-tools==7.4.1
|
||||
|
||||
# Verify pytest was installed and is accessible
|
||||
python -m pytest --version || echo "WARNING: pytest not found in PATH"
|
||||
|
||||
# install extra dependencies
|
||||
if [[ "${EXTRA_DEPENDENCY}" == "core" ]]; then
|
||||
pip-compile -o min_requirements.txt python/setup.py
|
||||
elif [[ "${EXTRA_DEPENDENCY}" == "ml" ]]; then
|
||||
pip-compile -o min_requirements.txt python/setup.py --extra tune
|
||||
elif [[ "${EXTRA_DEPENDENCY}" == "default" ]]; then
|
||||
pip-compile -o min_requirements.txt python/setup.py --extra default
|
||||
elif [[ "${EXTRA_DEPENDENCY}" == "serve" ]]; then
|
||||
echo "httpx==0.27.2" >> /tmp/min_build_requirements.txt
|
||||
echo "pytest-asyncio==1.1.0" >> /tmp/min_build_requirements.txt
|
||||
pip-compile -o min_requirements.txt /tmp/min_build_requirements.txt python/setup.py --extra "serve-grpc"
|
||||
rm /tmp/min_build_requirements.txt
|
||||
fi
|
||||
|
||||
pip install -r min_requirements.txt
|
||||
|
||||
# Core wants to eagerly be tested with some of its prerelease dependencies.
|
||||
if [[ "${EXTRA_DEPENDENCY}" == "core" ]]; then
|
||||
./ci/env/install-core-prerelease-dependencies.sh
|
||||
fi
|
||||
|
||||
# Verify pytest was installed
|
||||
python -m pytest --version
|
||||
|
||||
EOF
|
||||
|
||||
# For Python 3.14+, the PATH needs to include the conda environment directory
|
||||
# Create a symlink from the env-specific python to a standard location
|
||||
RUN bash -c 'if [[ -d /opt/miniforge/envs/py${PYTHON_VERSION}/bin ]]; then \
|
||||
for f in /opt/miniforge/envs/py${PYTHON_VERSION}/bin/*; do \
|
||||
ln -sf "$f" "/opt/miniforge/bin/$(basename "$f")" 2>/dev/null || true; \
|
||||
done \
|
||||
fi'
|
||||
|
||||
# Ensure /opt/miniforge/bin is in PATH for all shells (including non-interactive)
|
||||
# This is needed for tests.env.Dockerfile which uses bash -i
|
||||
# For Python 3.14+, also add the env-specific bin directory to PATH
|
||||
# so that console scripts installed later (like 'ray') are accessible
|
||||
ENV PATH="/opt/miniforge/bin:/opt/miniforge/envs/py${PYTHON_VERSION}/bin:${PATH}"
|
||||
@@ -0,0 +1,21 @@
|
||||
name: "minbuild-$EXTRA_DEPENDENCY-py$PYTHON_VERSION"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON_VERSION"]
|
||||
dockerfile: ci/docker/min.build.Dockerfile
|
||||
srcs:
|
||||
- .bazelrc
|
||||
- ci/ci.sh
|
||||
- ci/env/install-dependencies.sh
|
||||
- ci/env/install-llvm-binaries.sh
|
||||
- ci/env/install-core-prerelease-dependencies.sh
|
||||
- ci/suppress_output
|
||||
- python/requirements.txt
|
||||
- python/requirements_compiled.txt
|
||||
- python/setup.py
|
||||
- python/LICENSE.txt
|
||||
- python/ray/_version.py
|
||||
- README.rst
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- EXTRA_DEPENDENCY
|
||||
tags:
|
||||
- cr.ray.io/rayproject/minbuild-$EXTRA_DEPENDENCY-py$PYTHON_VERSION
|
||||
@@ -0,0 +1,61 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py3.10
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ARG RAYCI_IS_GPU_BUILD=false
|
||||
ARG PYTHON
|
||||
ARG IMAGE_TYPE=build
|
||||
ARG PYTHON_DEPSET=python/deplocks/ci/ml-$IMAGE_TYPE-ci_depset_py$PYTHON.lock
|
||||
ARG THIRDPARTY_DEPSET=python/deplocks/ci/ci_ml_thirdparty_depset_py$PYTHON.lock
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
|
||||
COPY "$THIRDPARTY_DEPSET" /home/ray/thirdparty_depset.lock
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
set -x
|
||||
|
||||
./ci/env/install-hdfs.sh
|
||||
|
||||
# Install system packages for doc testing (not in base image which lacks DOC_TESTING=1)
|
||||
sudo apt-get update -qq && sudo apt-get install -y graphviz tesseract-ocr
|
||||
|
||||
# Install HEBO for testing (not supported on Python 3.12+)
|
||||
if [[ "${PYTHON-}" != "3.12" ]]; then
|
||||
pip install HEBO==0.3.5
|
||||
fi
|
||||
|
||||
# Install doc-testing Python packages not covered by depsets
|
||||
pip install pydot pytesseract==0.3.13
|
||||
|
||||
# Install thirdparty dependencies
|
||||
mkdir -p python/ray/thirdparty_files
|
||||
uv pip install -r /home/ray/thirdparty_depset.lock --no-deps --target=python/ray/thirdparty_files
|
||||
|
||||
# Install Python dependencies from depset lock file
|
||||
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
|
||||
|
||||
# Inject our own mirror for the CIFAR10 dataset
|
||||
SITE_PACKAGES=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')
|
||||
|
||||
TF_CIFAR="${SITE_PACKAGES}/tensorflow/python/keras/datasets/cifar10.py"
|
||||
TF_KERAS_CIFAR="${SITE_PACKAGES}/tf_keras/src/datasets/cifar10.py"
|
||||
TORCH_CIFAR="${SITE_PACKAGES}/torchvision/datasets/cifar.py"
|
||||
KERAS_CIFAR="${SITE_PACKAGES}/keras/src/datasets/cifar10.py"
|
||||
|
||||
for f in "$TF_CIFAR" "$TF_KERAS_CIFAR" "$TORCH_CIFAR" "$KERAS_CIFAR"; do
|
||||
[ -f "$f" ] && sed -i 's https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz https://air-example-data.s3.us-west-2.amazonaws.com/cifar-10-python.tar.gz g' "$f"
|
||||
done
|
||||
|
||||
# Remove installed ray.
|
||||
pip uninstall -y ray
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,14 @@
|
||||
name: "ml$IMAGE_TYPE-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON"]
|
||||
dockerfile: ci/docker/ml.build.Dockerfile
|
||||
srcs:
|
||||
- ci/env/install-hdfs.sh
|
||||
- python/deplocks/ci/ml-$IMAGE_TYPE-ci_depset_py$PYTHON.lock
|
||||
- python/deplocks/ci/ci_ml_thirdparty_depset_py$PYTHON.lock
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON
|
||||
- RAYCI_IS_GPU_BUILD
|
||||
- PYTHON
|
||||
- IMAGE_TYPE
|
||||
tags:
|
||||
- cr.ray.io/rayproject/ml$IMAGE_TYPE-py$PYTHON
|
||||
@@ -0,0 +1,20 @@
|
||||
# Ray Anyscale CPU Image (for release tests)
|
||||
# Installs ray wheel into the CPU base-extra-testdeps image, which includes
|
||||
# additional test dependencies.
|
||||
#
|
||||
# This produces the anyscale test image for CPU-only release tests.
|
||||
#
|
||||
name: "ray-anyscale-py$PYTHON_VERSION-cpu$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base-extra-testdeps" # CPU base with test deps
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- IMAGE_TYPE=ray
|
||||
- PLATFORM=cpu
|
||||
- BASE_VARIANT=base-extra-testdeps
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,20 @@
|
||||
# Ray Anyscale CUDA Image (for release tests)
|
||||
# Installs ray wheel into the CUDA base-extra-testdeps image, which includes
|
||||
# additional test dependencies.
|
||||
#
|
||||
# This produces the anyscale test image for GPU release tests.
|
||||
#
|
||||
name: "ray-anyscale-py$PYTHON_VERSION-$GPU$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-$GPU-base-extra-testdeps" # CUDA base with test deps
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- IMAGE_TYPE=ray
|
||||
- BASE_VARIANT=base-extra-testdeps
|
||||
- PLATFORM=$GPU
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,72 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
ARG ARCH_SUFFIX=
|
||||
ARG HOSTTYPE=x86_64
|
||||
ARG MANYLINUX_VERSION
|
||||
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
|
||||
|
||||
ARG PYTHON_VERSION=3.10
|
||||
ARG BUILDKITE_BAZEL_CACHE_URL
|
||||
ARG BUILDKITE_CACHE_READONLY
|
||||
ARG HOSTTYPE
|
||||
ARG IS_LOCAL_BUILD=false
|
||||
ARG CACHE_DIR=/opt/cache
|
||||
|
||||
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
|
||||
ENV BUILDKITE_CACHE_READONLY=${BUILDKITE_CACHE_READONLY}
|
||||
ENV IS_LOCAL_BUILD=${IS_LOCAL_BUILD}
|
||||
ENV CACHE_DIR=${CACHE_DIR}
|
||||
ENV DOWNLOAD_CACHE=${CACHE_DIR}/downloads
|
||||
ENV BAZEL_CACHE=${CACHE_DIR}/bazel
|
||||
|
||||
WORKDIR /home/forge/ray
|
||||
|
||||
COPY --chown=forge . .
|
||||
|
||||
RUN --mount=type=cache,target=${DOWNLOAD_CACHE},uid=2000,gid=100,id=ray-downloads-${HOSTTYPE} \
|
||||
--mount=type=cache,target=${BAZEL_CACHE},uid=2000,gid=100,id=ray-bazel-${HOSTTYPE}-py${PYTHON_VERSION} \
|
||||
<<'EOF'
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
export BAZELISK_HOME=$DOWNLOAD_CACHE/bazelisk
|
||||
REPOSITORY_CACHE=$DOWNLOAD_CACHE/repo
|
||||
|
||||
PY_CODE="${PYTHON_VERSION//./}"
|
||||
PY_BIN="cp${PY_CODE}-cp${PY_CODE}"
|
||||
|
||||
export RAY_BUILD_ENV="manylinux_py${PY_BIN}"
|
||||
|
||||
sudo ln -sf "/opt/python/${PY_BIN}/bin/python3" /usr/local/bin/python3
|
||||
sudo ln -sf /usr/local/bin/python3 /usr/local/bin/python
|
||||
|
||||
BAZEL_CACHE_ARGS=""
|
||||
if [[ -z "${BUILDKITE_BAZEL_CACHE_URL:-}" ]]; then
|
||||
# Disable remote cache for local builds (no credentials)
|
||||
BAZEL_CACHE_ARGS="--remote_cache="
|
||||
else
|
||||
# Override whatever the base image baked into ~/.bazelrc
|
||||
BAZEL_CACHE_ARGS="--remote_cache=${BUILDKITE_BAZEL_CACHE_URL}"
|
||||
if [[ "${BUILDKITE_CACHE_READONLY:-}" == "true" ]]; then
|
||||
BAZEL_CACHE_ARGS="$BAZEL_CACHE_ARGS --remote_upload_local_results=false"
|
||||
fi
|
||||
fi
|
||||
|
||||
BAZEL_RESOURCE_FLAGS=""
|
||||
if [[ "$IS_LOCAL_BUILD" == "true" ]]; then
|
||||
BAZEL_RESOURCE_FLAGS=$(python3 "$HOME/ray/ci/build/container_resource_utils.py")
|
||||
fi
|
||||
|
||||
bazelisk --output_base=$BAZEL_CACHE build --config=ci \
|
||||
--repository_cache=$REPOSITORY_CACHE \
|
||||
$BAZEL_CACHE_ARGS \
|
||||
$BAZEL_RESOURCE_FLAGS \
|
||||
//:ray_pkg_zip //:ray_py_proto_zip
|
||||
|
||||
cp bazel-bin/ray_pkg.zip /home/forge/ray_pkg.zip
|
||||
cp bazel-bin/ray_py_proto.zip /home/forge/ray_py_proto.zip
|
||||
|
||||
EOF
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=builder /home/forge/ray_pkg.zip /home/forge/ray_py_proto.zip /
|
||||
@@ -0,0 +1,31 @@
|
||||
name: "ray-core-py$PYTHON_VERSION$ARCH_SUFFIX"
|
||||
froms: ["rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"]
|
||||
dockerfile: ci/docker/ray-core.Dockerfile
|
||||
srcs:
|
||||
- .bazelversion
|
||||
- .bazelrc
|
||||
- WORKSPACE
|
||||
- BUILD.bazel
|
||||
- bazel/
|
||||
- thirdparty/
|
||||
- src/
|
||||
- gen_ray_pkg.py
|
||||
- python/ray/__init__.py
|
||||
- python/ray/_raylet.pxd
|
||||
- python/ray/_raylet.pyi
|
||||
- python/ray/_raylet.pyx
|
||||
- python/ray/includes/
|
||||
- java/BUILD.bazel
|
||||
- java/dependencies.bzl
|
||||
- release/BUILD.bazel
|
||||
- release/requirements_py310.txt
|
||||
- ci/build/container_resource_utils.py
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
- HOSTTYPE
|
||||
- MANYLINUX_VERSION
|
||||
build_hint_args:
|
||||
- BUILDKITE_CACHE_READONLY
|
||||
- IS_LOCAL_BUILD
|
||||
@@ -0,0 +1,67 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
#
|
||||
# Ray C++ Core Artifacts Builder
|
||||
# ==============================
|
||||
# Builds ray_cpp_pkg.zip containing C++ headers, libraries, and examples.
|
||||
#
|
||||
ARG ARCH_SUFFIX=
|
||||
ARG HOSTTYPE=x86_64
|
||||
ARG MANYLINUX_VERSION
|
||||
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
|
||||
|
||||
ARG PYTHON_VERSION=3.10
|
||||
ARG BUILDKITE_BAZEL_CACHE_URL
|
||||
ARG BUILDKITE_CACHE_READONLY
|
||||
ARG HOSTTYPE
|
||||
ARG CACHE_DIR=/opt/cache
|
||||
|
||||
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
|
||||
ENV BUILDKITE_CACHE_READONLY=${BUILDKITE_CACHE_READONLY}
|
||||
ENV CACHE_DIR=${CACHE_DIR}
|
||||
ENV DOWNLOAD_CACHE=${CACHE_DIR}/downloads
|
||||
ENV BAZEL_CACHE=${CACHE_DIR}/bazel
|
||||
|
||||
WORKDIR /home/forge/ray
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN --mount=type=cache,target=${DOWNLOAD_CACHE},uid=2000,gid=100,id=ray-downloads-${HOSTTYPE} \
|
||||
--mount=type=cache,target=${BAZEL_CACHE},uid=2000,gid=100,id=ray-bazel-${HOSTTYPE} \
|
||||
<<'EOF'
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
export BAZELISK_HOME=$DOWNLOAD_CACHE/bazelisk
|
||||
REPOSITORY_CACHE=$DOWNLOAD_CACHE/repo
|
||||
|
||||
PY_CODE="${PYTHON_VERSION//./}"
|
||||
PY_BIN="cp${PY_CODE}-cp${PY_CODE}"
|
||||
export RAY_BUILD_ENV="manylinux_py${PY_BIN}"
|
||||
|
||||
sudo ln -sf "/opt/python/${PY_BIN}/bin/python3" /usr/local/bin/python3
|
||||
sudo ln -sf /usr/local/bin/python3 /usr/local/bin/python
|
||||
|
||||
BAZEL_CACHE_ARGS=""
|
||||
if [[ -z "${BUILDKITE_BAZEL_CACHE_URL:-}" ]]; then
|
||||
# Disable remote cache for local builds (no credentials)
|
||||
BAZEL_CACHE_ARGS="--remote_cache="
|
||||
else
|
||||
# Override whatever the base image baked into ~/.bazelrc
|
||||
BAZEL_CACHE_ARGS="--remote_cache=${BUILDKITE_BAZEL_CACHE_URL}"
|
||||
if [[ "${BUILDKITE_CACHE_READONLY:-}" == "true" ]]; then
|
||||
BAZEL_CACHE_ARGS="$BAZEL_CACHE_ARGS --remote_upload_local_results=false"
|
||||
fi
|
||||
fi
|
||||
|
||||
bazelisk --output_base=$BAZEL_CACHE build --config=ci \
|
||||
--repository_cache=$REPOSITORY_CACHE \
|
||||
$BAZEL_CACHE_ARGS \
|
||||
//cpp:ray_cpp_pkg_zip
|
||||
|
||||
cp bazel-bin/cpp/ray_cpp_pkg.zip /home/forge/ray_cpp_pkg.zip
|
||||
|
||||
EOF
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=builder /home/forge/ray_cpp_pkg.zip /
|
||||
@@ -0,0 +1,29 @@
|
||||
name: "ray-cpp-core$ARCH_SUFFIX"
|
||||
froms: ["rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"]
|
||||
dockerfile: ci/docker/ray-cpp-core.Dockerfile
|
||||
srcs:
|
||||
- .bazelversion
|
||||
- .bazelrc
|
||||
- WORKSPACE
|
||||
- BUILD.bazel
|
||||
- bazel/
|
||||
- thirdparty/
|
||||
- src/
|
||||
- cpp/
|
||||
- python/ray/__init__.py
|
||||
- python/ray/_raylet.pxd
|
||||
- python/ray/_raylet.pyi
|
||||
- python/ray/_raylet.pyx
|
||||
- python/ray/includes/
|
||||
- java/BUILD.bazel
|
||||
- java/dependencies.bzl
|
||||
- release/BUILD.bazel
|
||||
- release/requirements.txt
|
||||
- release/requirements_py310.txt
|
||||
build_args:
|
||||
- ARCH_SUFFIX
|
||||
- HOSTTYPE
|
||||
- MANYLINUX_VERSION
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
build_hint_args:
|
||||
- BUILDKITE_CACHE_READONLY
|
||||
@@ -0,0 +1,89 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
#
|
||||
# Ray C++ Wheel Builder
|
||||
# =====================
|
||||
# Builds manylinux2014-compatible ray-cpp wheel using pre-built C++ artifacts from wanda cache.
|
||||
#
|
||||
# This is a minimal Dockerfile for ray-cpp wheel builds only.
|
||||
# It copies only the files needed for the cpp wheel, reducing build context size.
|
||||
|
||||
ARG RAY_CORE_IMAGE
|
||||
ARG RAY_CPP_CORE_IMAGE
|
||||
ARG RAY_JAVA_IMAGE
|
||||
ARG RAY_DASHBOARD_IMAGE
|
||||
ARG MANYLINUX_VERSION
|
||||
ARG HOSTTYPE
|
||||
|
||||
FROM ${RAY_CORE_IMAGE} AS ray-core
|
||||
FROM ${RAY_CPP_CORE_IMAGE} AS ray-cpp-core
|
||||
FROM ${RAY_JAVA_IMAGE} AS ray-java
|
||||
FROM ${RAY_DASHBOARD_IMAGE} AS ray-dashboard
|
||||
|
||||
# Main build stage - manylinux2014 provides GLIBC 2.17
|
||||
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
|
||||
|
||||
ARG BUILDKITE_COMMIT
|
||||
|
||||
WORKDIR /home/forge/ray
|
||||
|
||||
# Copy artifacts from all stages
|
||||
COPY --from=ray-core /ray_pkg.zip /tmp/
|
||||
COPY --from=ray-core /ray_py_proto.zip /tmp/
|
||||
COPY --from=ray-java /ray_java_pkg.zip /tmp/
|
||||
COPY --from=ray-dashboard /dashboard.tar.gz /tmp/
|
||||
|
||||
# Minimal source files needed for cpp wheel build
|
||||
COPY --chown=forge ci/build/build-ray-cpp-wheel.sh ci/build/
|
||||
COPY --chown=forge README.rst pyproject.toml ./
|
||||
COPY --chown=forge python/setup.py python/
|
||||
COPY --chown=forge python/LICENSE.txt python/
|
||||
COPY --chown=forge python/MANIFEST.in python/
|
||||
COPY --chown=forge python/ray/_version.py python/ray/
|
||||
|
||||
USER forge
|
||||
ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT:-unknown}
|
||||
RUN --mount=from=ray-cpp-core,source=/,target=/ray-cpp-core,ro \
|
||||
<<'EOF'
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Clean extraction dirs to avoid stale leftovers
|
||||
rm -rf /tmp/ray_pkg /tmp/ray_java_pkg /tmp/ray_cpp_pkg
|
||||
mkdir -p /tmp/ray_pkg /tmp/ray_java_pkg /tmp/ray_cpp_pkg
|
||||
|
||||
# Unpack pre-built artifacts
|
||||
unzip -o /tmp/ray_pkg.zip -d /tmp/ray_pkg
|
||||
unzip -o /tmp/ray_py_proto.zip -d python/
|
||||
unzip -o /tmp/ray_java_pkg.zip -d /tmp/ray_java_pkg
|
||||
mkdir -p python/ray/dashboard/client/build
|
||||
tar -xzf /tmp/dashboard.tar.gz -C python/ray/dashboard/client/build/
|
||||
|
||||
# C++ core artifacts
|
||||
cp -r /tmp/ray_pkg/ray/* python/ray/
|
||||
|
||||
# Java JARs
|
||||
cp -r /tmp/ray_java_pkg/ray/* python/ray/
|
||||
|
||||
# C++ API artifacts (headers, libs, examples)
|
||||
unzip -o /ray-cpp-core/ray_cpp_pkg.zip -d /tmp/ray_cpp_pkg
|
||||
cp -r /tmp/ray_cpp_pkg/ray/cpp python/ray/
|
||||
|
||||
# Build Python-agnostic cpp wheel
|
||||
./ci/build/build-ray-cpp-wheel.sh
|
||||
|
||||
# Sanity check: ensure wheels exist
|
||||
if [[ ! -d .whl ]]; then
|
||||
echo "ERROR: .whl directory not created"
|
||||
exit 1
|
||||
fi
|
||||
wheels=($(find .whl -maxdepth 1 -name '*.whl'))
|
||||
if (( ${#wheels[@]} == 0 )); then
|
||||
echo "ERROR: No wheels produced in .whl/"
|
||||
ls -la .whl
|
||||
exit 1
|
||||
fi
|
||||
|
||||
EOF
|
||||
|
||||
FROM scratch
|
||||
COPY --from=builder /home/forge/ray/.whl/*.whl /opt/artifacts/
|
||||
@@ -0,0 +1,31 @@
|
||||
# Python-agnostic ray-cpp wheel.
|
||||
# Uses upstream images from one Python version but produces a py3-none wheel
|
||||
# that works with any Python 3. PYTHON_VERSION is set at buildkite level.
|
||||
name: "ray-cpp-wheel$ARCH_SUFFIX"
|
||||
froms:
|
||||
- "rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"
|
||||
- "cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX" # C++ binaries (ray_pkg.zip)
|
||||
- "cr.ray.io/rayproject/ray-cpp-core$ARCH_SUFFIX" # C++ headers/libs (ray_cpp_pkg.zip)
|
||||
- "cr.ray.io/rayproject/ray-java-build$ARCH_SUFFIX" # Java JARs
|
||||
- "cr.ray.io/rayproject/ray-dashboard$ARCH_SUFFIX" # Dashboard
|
||||
dockerfile: ci/docker/ray-cpp-wheel.Dockerfile
|
||||
srcs:
|
||||
- pyproject.toml
|
||||
- README.rst
|
||||
- ci/build/build-ray-cpp-wheel.sh
|
||||
- python/setup.py
|
||||
- python/LICENSE.txt
|
||||
- python/MANIFEST.in
|
||||
- python/ray/_version.py
|
||||
build_args:
|
||||
- MANYLINUX_VERSION
|
||||
- HOSTTYPE
|
||||
- BUILDKITE_COMMIT
|
||||
- ARCH_SUFFIX
|
||||
- RAY_CORE_IMAGE=cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX
|
||||
- RAY_CPP_CORE_IMAGE=cr.ray.io/rayproject/ray-cpp-core$ARCH_SUFFIX
|
||||
- RAY_JAVA_IMAGE=cr.ray.io/rayproject/ray-java-build$ARCH_SUFFIX
|
||||
- RAY_DASHBOARD_IMAGE=cr.ray.io/rayproject/ray-dashboard$ARCH_SUFFIX
|
||||
artifacts:
|
||||
- src: "/opt/artifacts/"
|
||||
dst: "./"
|
||||
@@ -0,0 +1,32 @@
|
||||
ARG HOSTTYPE=x86_64
|
||||
ARG MANYLINUX_VERSION
|
||||
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
|
||||
|
||||
WORKDIR /home/forge/ray
|
||||
|
||||
COPY --chown=forge:users . .
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
export PATH="/usr/local/node/bin:$PATH"
|
||||
|
||||
(
|
||||
cd python/ray/dashboard/client
|
||||
|
||||
echo "--- npm ci"
|
||||
npm ci
|
||||
|
||||
echo "--- npm run build"
|
||||
npm run build
|
||||
|
||||
tar -czf /home/forge/dashboard.tar.gz -C build .
|
||||
)
|
||||
|
||||
EOF
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=builder /home/forge/dashboard.tar.gz /dashboard.tar.gz
|
||||
@@ -0,0 +1,8 @@
|
||||
name: ray-dashboard$ARCH_SUFFIX
|
||||
froms: ["rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"]
|
||||
dockerfile: ci/docker/ray-dashboard.Dockerfile
|
||||
srcs:
|
||||
- python/ray/dashboard/client/
|
||||
build_args:
|
||||
- HOSTTYPE
|
||||
- MANYLINUX_VERSION
|
||||
@@ -0,0 +1,14 @@
|
||||
name: "ray-extra-py$PYTHON_VERSION-cpu$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base-extra$ARCH_SUFFIX" # CPU base-extra image
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- IMAGE_TYPE=ray
|
||||
- PLATFORM=cpu
|
||||
- BASE_VARIANT=base-extra
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,14 @@
|
||||
name: "ray-extra-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra$ARCH_SUFFIX" # CUDA base-extra image
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- IMAGE_TYPE=ray
|
||||
- PLATFORM=cu$CUDA_VERSION
|
||||
- BASE_VARIANT=base-extra
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,12 @@
|
||||
name: "ray-py$PYTHON_VERSION-cpu$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX" # CPU base image with Python + deps
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- PLATFORM=cpu
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,12 @@
|
||||
name: "ray-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX" # CUDA base image
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- PLATFORM=cu$CUDA_VERSION
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,12 @@
|
||||
name: "ray-py$PYTHON_VERSION-tpu$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-tpu-base$ARCH_SUFFIX" # TPU base image with Python + deps
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- PLATFORM=tpu
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,77 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
#
|
||||
# Ray Image Builder
|
||||
# ==============================
|
||||
# Installs the Ray wheel into a base image (CPU or CUDA), includes
|
||||
# pip freeze output for reproducibility.
|
||||
#
|
||||
ARG PYTHON_VERSION=3.10
|
||||
ARG PLATFORM=cpu
|
||||
ARG ARCH_SUFFIX=
|
||||
ARG IMAGE_TYPE=ray
|
||||
ARG BASE_VARIANT=base
|
||||
ARG BASE_IMAGE=cr.ray.io/rayproject/${IMAGE_TYPE}-py${PYTHON_VERSION}-${PLATFORM}-${BASE_VARIANT}${ARCH_SUFFIX}
|
||||
ARG RAY_WHEEL_IMAGE=cr.ray.io/rayproject/ray-wheel-py${PYTHON_VERSION}${ARCH_SUFFIX}
|
||||
|
||||
FROM ${RAY_WHEEL_IMAGE} AS wheel-source
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ARG IMAGE_TYPE=ray
|
||||
ARG PLATFORM=cpu
|
||||
ARG RAY_COMMIT=unknown-commit
|
||||
ARG RAY_VERSION=3.0.0.dev0
|
||||
|
||||
LABEL io.ray.ray-commit="${RAY_COMMIT}"
|
||||
LABEL io.ray.ray-version="${RAY_VERSION}"
|
||||
|
||||
COPY --from=wheel-source /opt/artifacts/*.whl /home/ray/
|
||||
|
||||
# Install Ray wheel with all extras
|
||||
# Uses requirements_compiled.txt from base image (already at /home/ray/)
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
WHEEL_FILES=(/home/ray/ray-*.whl)
|
||||
if [[ ${#WHEEL_FILES[@]} -ne 1 ]]; then
|
||||
echo "Error: Expected 1 ray wheel file, but found ${#WHEEL_FILES[@]} in /home/ray/." >&2
|
||||
ls -l /home/ray/*.whl >&2
|
||||
exit 1
|
||||
fi
|
||||
WHEEL_FILE="${WHEEL_FILES[0]}"
|
||||
|
||||
echo "Installing wheel: $WHEEL_FILE"
|
||||
|
||||
if [[ "${IMAGE_TYPE}" == "ray-llm" ]]; then
|
||||
RAY_EXTRAS="default,data,serve"
|
||||
else
|
||||
RAY_EXTRAS="all"
|
||||
fi
|
||||
|
||||
# TODO(cu130): ray[all]'s cgraph extra hard-pins cupy-cuda12x, so this install
|
||||
# always pulls the CUDA-12 build even on cu130 images (no PEP 508 marker exists
|
||||
# to select cupy by CUDA version). Until the cgraph extra can resolve cupy per
|
||||
# CUDA runtime (or cupy ships a unified package), we patch it up with the
|
||||
# uninstall/reinstall swap below. Drop that swap once this install can pick the
|
||||
# right cupy directly.
|
||||
$HOME/anaconda3/bin/pip --no-cache-dir install \
|
||||
-c /home/ray/requirements_compiled.txt \
|
||||
"${WHEEL_FILE}[${RAY_EXTRAS}]"
|
||||
|
||||
# ray[all]'s cgraph extra hard-pins cupy-cuda12x (a CUDA-12 build), but cu130
|
||||
# images ship a CUDA-13 runtime where that build is broken. Swap it for the
|
||||
# matching CUDA-13 build. cupy-cuda12x and cupy-cuda13x both own the top-level
|
||||
# `cupy` package and cannot coexist, so this is an uninstall-then-install.
|
||||
# Scoped to IMAGE_TYPE=ray (covers ray + ray-extra); ray-llm flows through this
|
||||
# same Dockerfile but manages cupy via its own llm locks, so leave it untouched.
|
||||
if [[ "${IMAGE_TYPE}" == "ray" && "${PLATFORM}" == cu13* ]]; then
|
||||
$HOME/anaconda3/bin/pip --no-cache-dir uninstall -y cupy-cuda12x
|
||||
$HOME/anaconda3/bin/pip --no-cache-dir install "cupy-cuda13x==13.6.0"
|
||||
fi
|
||||
|
||||
$HOME/anaconda3/bin/pip freeze > /home/ray/pip-freeze.txt
|
||||
|
||||
echo "Ray version: $($HOME/anaconda3/bin/python -c 'import ray; print(ray.__version__)')"
|
||||
EOF
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
@@ -0,0 +1,65 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
ARG ARCH_SUFFIX
|
||||
ARG HOSTTYPE=x86_64
|
||||
ARG MANYLINUX_VERSION
|
||||
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
|
||||
|
||||
ARG BUILDKITE_BAZEL_CACHE_URL
|
||||
ARG BUILDKITE_CACHE_READONLY
|
||||
ARG HOSTTYPE
|
||||
ARG IS_LOCAL_BUILD=false
|
||||
ARG CACHE_DIR=/opt/cache
|
||||
|
||||
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
|
||||
ENV BUILDKITE_CACHE_READONLY=${BUILDKITE_CACHE_READONLY}
|
||||
ENV IS_LOCAL_BUILD=${IS_LOCAL_BUILD}
|
||||
ENV CACHE_DIR=${CACHE_DIR}
|
||||
ENV DOWNLOAD_CACHE=${CACHE_DIR}/downloads
|
||||
ENV BAZEL_CACHE=${CACHE_DIR}/bazel
|
||||
|
||||
WORKDIR /home/forge/ray
|
||||
|
||||
COPY --chown=forge . .
|
||||
|
||||
RUN --mount=type=cache,target=${DOWNLOAD_CACHE},uid=2000,gid=100,id=ray-downloads-${HOSTTYPE} \
|
||||
--mount=type=cache,target=${BAZEL_CACHE},uid=2000,gid=100,id=ray-bazel-${HOSTTYPE} \
|
||||
<<'EOF'
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
export BAZELISK_HOME=$DOWNLOAD_CACHE/bazelisk
|
||||
export COURSIER_CACHE=$DOWNLOAD_CACHE/coursier
|
||||
REPOSITORY_CACHE=$DOWNLOAD_CACHE/repo
|
||||
|
||||
export RAY_BUILD_ENV="manylinux"
|
||||
|
||||
BAZEL_CACHE_ARGS=""
|
||||
if [[ -z "${BUILDKITE_BAZEL_CACHE_URL:-}" ]]; then
|
||||
# Disable remote cache for local builds (no credentials)
|
||||
BAZEL_CACHE_ARGS="--remote_cache="
|
||||
else
|
||||
# Override whatever the base image baked into ~/.bazelrc
|
||||
BAZEL_CACHE_ARGS="--remote_cache=${BUILDKITE_BAZEL_CACHE_URL}"
|
||||
if [[ "${BUILDKITE_CACHE_READONLY:-}" == "true" ]]; then
|
||||
BAZEL_CACHE_ARGS="$BAZEL_CACHE_ARGS --remote_upload_local_results=false"
|
||||
fi
|
||||
fi
|
||||
|
||||
BAZEL_RESOURCE_FLAGS=""
|
||||
if [[ "$IS_LOCAL_BUILD" == "true" ]]; then
|
||||
BAZEL_RESOURCE_FLAGS=$(python3 "$HOME/ray/ci/build/container_resource_utils.py")
|
||||
fi
|
||||
|
||||
bazelisk --output_base=$BAZEL_CACHE run --config=ci \
|
||||
--repository_cache=$REPOSITORY_CACHE \
|
||||
$BAZEL_CACHE_ARGS \
|
||||
$BAZEL_RESOURCE_FLAGS \
|
||||
//java:gen_ray_java_pkg
|
||||
|
||||
cp bazel-bin/java/ray_java_pkg.zip /home/forge/ray_java_pkg.zip
|
||||
|
||||
EOF
|
||||
|
||||
FROM scratch
|
||||
|
||||
COPY --from=builder /home/forge/ray_java_pkg.zip /
|
||||
@@ -0,0 +1,24 @@
|
||||
name: ray-java-build$ARCH_SUFFIX
|
||||
froms: ["rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"]
|
||||
dockerfile: ci/docker/ray-java.Dockerfile
|
||||
srcs:
|
||||
- .bazelversion
|
||||
- .bazelrc
|
||||
- WORKSPACE
|
||||
- BUILD.bazel
|
||||
- bazel/
|
||||
- thirdparty/
|
||||
- src/
|
||||
- java/
|
||||
- gen_ray_pkg.py
|
||||
- release/BUILD.bazel
|
||||
- release/requirements_py310.txt
|
||||
- ci/build/container_resource_utils.py
|
||||
build_args:
|
||||
- ARCH_SUFFIX
|
||||
- BUILDKITE_BAZEL_CACHE_URL
|
||||
- HOSTTYPE
|
||||
- MANYLINUX_VERSION
|
||||
build_hint_args:
|
||||
- BUILDKITE_CACHE_READONLY
|
||||
- IS_LOCAL_BUILD
|
||||
@@ -0,0 +1,20 @@
|
||||
# Ray-LLM Anyscale CUDA Image (for release tests)
|
||||
# Installs ray wheel into the ray-llm CUDA base-extra-testdeps image, which
|
||||
# includes extra test dependencies.
|
||||
#
|
||||
# This produces the anyscale test image for ray-llm GPU release tests.
|
||||
#
|
||||
name: "ray-llm-anyscale-py$PYTHON_VERSION-$GPU$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-llm-py$PYTHON_VERSION-$GPU-base-extra-testdeps" # ray-llm base with test deps
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- BASE_VARIANT=base-extra-testdeps
|
||||
- IMAGE_TYPE=ray-llm
|
||||
- PLATFORM=$GPU
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,14 @@
|
||||
name: "ray-llm-extra-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-llm-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra$ARCH_SUFFIX" # LLM base-extra image
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- IMAGE_TYPE=ray-llm
|
||||
- PLATFORM=cu$CUDA_VERSION
|
||||
- BASE_VARIANT=base-extra
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,13 @@
|
||||
name: "ray-llm-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-llm-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX" # LLM base image
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- IMAGE_TYPE=ray-llm
|
||||
- PLATFORM=cu$CUDA_VERSION
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,20 @@
|
||||
# Ray-ML Anyscale CUDA Image (for release tests)
|
||||
# Installs ray wheel into the ray-ml CUDA base-extra-testdeps image, which includes
|
||||
# extra test dependencies.
|
||||
#
|
||||
# This produces the anyscale test image for ray-ml GPU release tests.
|
||||
#
|
||||
name: "ray-ml-anyscale-py$PYTHON_VERSION-$GPU$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "cr.ray.io/rayproject/ray-ml-py$PYTHON_VERSION-$GPU-base-extra-testdeps" # ray-ml base with test deps
|
||||
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
|
||||
dockerfile: ci/docker/ray-image.Dockerfile
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- ARCH_SUFFIX
|
||||
- BASE_VARIANT=base-extra-testdeps
|
||||
- IMAGE_TYPE=ray-ml
|
||||
- PLATFORM=$GPU
|
||||
- RAY_COMMIT=$BUILDKITE_COMMIT
|
||||
- RAY_VERSION
|
||||
@@ -0,0 +1,95 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
#
|
||||
# Ray Wheel Builder
|
||||
# =================
|
||||
# Builds manylinux2014-compatible ray wheel using pre-built C++ artifacts from wanda cache.
|
||||
#
|
||||
# GLIBC Compatibility:
|
||||
# --------------------
|
||||
# manylinux2014 requires GLIBC <= 2.17 for broad Linux compatibility.
|
||||
# The pre-built _raylet.so is compiled inside manylinux2014 with GLIBC 2.17.
|
||||
#
|
||||
|
||||
ARG RAY_CORE_IMAGE
|
||||
ARG RAY_JAVA_IMAGE=scratch
|
||||
ARG RAY_DASHBOARD_IMAGE=scratch
|
||||
ARG MANYLINUX_IMAGE
|
||||
|
||||
FROM ${RAY_CORE_IMAGE} AS ray-core
|
||||
FROM ${RAY_JAVA_IMAGE} AS ray-java
|
||||
FROM ${RAY_DASHBOARD_IMAGE} AS ray-dashboard
|
||||
|
||||
# Main build stage - manylinux2014 provides GLIBC 2.17
|
||||
FROM ${MANYLINUX_IMAGE} AS builder
|
||||
|
||||
ARG PYTHON_VERSION=3.10
|
||||
ARG BUILDKITE_COMMIT
|
||||
|
||||
WORKDIR /home/forge/ray
|
||||
|
||||
# Copy artifacts from all stages
|
||||
COPY --from=ray-core /ray_pkg.zip /tmp/
|
||||
COPY --from=ray-core /ray_py_proto.zip /tmp/
|
||||
|
||||
# Source files needed for wheel build
|
||||
COPY --chown=forge ci/build/build-manylinux-wheel.sh ci/build/
|
||||
COPY --chown=forge README.rst pyproject.toml ./
|
||||
COPY --chown=forge rllib/ rllib/
|
||||
COPY --chown=forge python/ python/
|
||||
|
||||
USER forge
|
||||
# - BUILDKITE_COMMIT: Used for ray.__commit__. Defaults to "unknown" for local builds.
|
||||
ENV PYTHON_VERSION=${PYTHON_VERSION} \
|
||||
BUILDKITE_COMMIT=${BUILDKITE_COMMIT:-unknown}
|
||||
RUN --mount=from=ray-java,target=/mnt/java \
|
||||
--mount=from=ray-dashboard,target=/mnt/dashboard \
|
||||
<<'EOF'
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Clean extraction dirs to avoid stale leftovers
|
||||
rm -rf /tmp/ray_pkg
|
||||
mkdir -p /tmp/ray_pkg
|
||||
|
||||
# Unpack pre-built artifacts
|
||||
unzip -o /tmp/ray_pkg.zip -d /tmp/ray_pkg
|
||||
unzip -o /tmp/ray_py_proto.zip -d python/
|
||||
|
||||
# Dashboard (optional)
|
||||
if [[ -f /mnt/dashboard/dashboard.tar.gz ]]; then
|
||||
mkdir -p python/ray/dashboard/client/build
|
||||
tar -xzf /mnt/dashboard/dashboard.tar.gz -C python/ray/dashboard/client/build/
|
||||
fi
|
||||
|
||||
# C++ core artifacts
|
||||
cp -r /tmp/ray_pkg/ray/* python/ray/
|
||||
|
||||
# Java JARs (optional)
|
||||
if [[ -f /mnt/java/ray_java_pkg.zip ]]; then
|
||||
mkdir -p /tmp/ray_java_pkg
|
||||
unzip -o /mnt/java/ray_java_pkg.zip -d /tmp/ray_java_pkg
|
||||
cp -r /tmp/ray_java_pkg/ray/* python/ray/
|
||||
fi
|
||||
|
||||
# Build ray wheel
|
||||
PY_VERSION="${PYTHON_VERSION//./}"
|
||||
PY_BIN="cp${PY_VERSION}-cp${PY_VERSION}"
|
||||
SKIP_BAZEL_BUILD=1 RAY_DISABLE_EXTRA_CPP=1 \
|
||||
./ci/build/build-manylinux-wheel.sh "$PY_BIN"
|
||||
|
||||
# Sanity check: ensure wheels exist
|
||||
if [[ ! -d .whl ]]; then
|
||||
echo "ERROR: .whl directory not created"
|
||||
exit 1
|
||||
fi
|
||||
wheels=($(find .whl -maxdepth 1 -name '*.whl'))
|
||||
if (( ${#wheels[@]} == 0 )); then
|
||||
echo "ERROR: No wheels produced in .whl/"
|
||||
ls -la .whl
|
||||
exit 1
|
||||
fi
|
||||
|
||||
EOF
|
||||
|
||||
FROM scratch
|
||||
COPY --from=builder /home/forge/ray/.whl/*.whl /opt/artifacts/
|
||||
@@ -0,0 +1,26 @@
|
||||
name: "ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX"
|
||||
disable_caching: true
|
||||
froms:
|
||||
- "rayproject/manylinux2014:$MANYLINUX_VERSION$JDK_SUFFIX-$HOSTTYPE"
|
||||
- "cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX" # C++ binaries (ray_pkg.zip)
|
||||
- "$RAY_JAVA_IMAGE" # Java JARs (scratch to skip)
|
||||
- "$RAY_DASHBOARD_IMAGE" # Dashboard (scratch to skip)
|
||||
dockerfile: ci/docker/ray-wheel.Dockerfile
|
||||
srcs:
|
||||
- pyproject.toml
|
||||
- README.rst
|
||||
- ci/build/build-manylinux-wheel.sh
|
||||
- python/
|
||||
- rllib/
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- BUILDKITE_COMMIT
|
||||
- ARCH_SUFFIX
|
||||
- WHEEL_TYPE=ray
|
||||
- MANYLINUX_IMAGE=rayproject/manylinux2014:$MANYLINUX_VERSION$JDK_SUFFIX-$HOSTTYPE
|
||||
- RAY_CORE_IMAGE=cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX
|
||||
- RAY_JAVA_IMAGE
|
||||
- RAY_DASHBOARD_IMAGE
|
||||
artifacts:
|
||||
- src: "/opt/artifacts/"
|
||||
dst: "./"
|
||||
@@ -0,0 +1,30 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py3.10
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ARG RAYCI_IS_GPU_BUILD
|
||||
ARG PYTHON
|
||||
ARG BUILD_VARIANT=build
|
||||
ARG PYTHON_DEPSET=python/deplocks/ci/rllib-$BUILD_VARIANT-ci_depset_py$PYTHON.lock
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
COPY . .
|
||||
|
||||
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Install system packages required for gymnasium rendering (MuJoCo, OpenGL, GLFW)
|
||||
sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends \
|
||||
libosmesa6-dev libgl1 libglfw3 patchelf
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Python dependencies from depset lock file
|
||||
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,11 @@
|
||||
name: "rllib$BUILD_VARIANT-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON"]
|
||||
dockerfile: ci/docker/rllib.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/ci/rllib-$BUILD_VARIANT-ci_depset_py$PYTHON.lock
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON
|
||||
- RAYCI_IS_GPU_BUILD
|
||||
- BUILD_VARIANT
|
||||
tags:
|
||||
- cr.ray.io/rayproject/rllib$BUILD_VARIANT-py$PYTHON
|
||||
@@ -0,0 +1,17 @@
|
||||
ARG BASE_IMAGE
|
||||
FROM $BASE_IMAGE
|
||||
|
||||
COPY python/ray/tests/runtime_env_container/ /home/ray/tests/
|
||||
|
||||
RUN pip install --no-cache-dir -c /home/ray/requirements_compiled.txt httpx
|
||||
|
||||
# Podman for runtime tests; docker-ce-cli so `docker save | podman load` can talk to
|
||||
# the mounted host socket (stock podman's docker-daemon: pull uses an old API).
|
||||
RUN sudo apt-get update -qq \
|
||||
&& sudo apt-get install -y -qq ca-certificates curl gnupg \
|
||||
&& sudo mkdir -p /etc/apt/keyrings \
|
||||
&& curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
|
||||
&& sudo chmod a+r /etc/apt/keyrings/docker.gpg \
|
||||
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu jammy stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
|
||||
&& sudo apt-get update \
|
||||
&& sudo apt-get install -y docker-ce-cli podman
|
||||
@@ -0,0 +1,31 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py3.10
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ARG ENABLE_TRACING
|
||||
ARG PYDANTIC_VERSION
|
||||
ARG IMAGE_TYPE="base"
|
||||
ARG PYTHON
|
||||
ARG PYTHON_DEPSET="python/deplocks/ci/serve_${IMAGE_TYPE}_depset_py${PYTHON}.lock"
|
||||
|
||||
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
|
||||
|
||||
SHELL ["/bin/bash", "-ice"]
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
uv pip install --system --no-cache-dir --no-deps --index-strategy unsafe-best-match \
|
||||
-r /home/ray/python_depset.lock
|
||||
|
||||
git clone --branch=4.2.0 --depth=1 https://github.com/wg/wrk.git /tmp/wrk
|
||||
make -C /tmp/wrk -j
|
||||
sudo cp /tmp/wrk/wrk /usr/local/bin/wrk
|
||||
rm -rf /tmp/wrk
|
||||
|
||||
EOF
|
||||
@@ -0,0 +1,11 @@
|
||||
name: "servebuild-py$PYTHON"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON"]
|
||||
dockerfile: ci/docker/serve.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/ci/serve_ci_depset_py$PYTHON.lock
|
||||
build_args:
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
|
||||
- PYTHON=$PYTHON
|
||||
- IMAGE_TYPE=ci
|
||||
tags:
|
||||
- cr.ray.io/rayproject/servebuild-py$PYTHON
|
||||
@@ -0,0 +1,12 @@
|
||||
name: "servetracingbuild"
|
||||
froms: ["cr.ray.io/rayproject/oss-ci-base_build-py3.10"]
|
||||
dockerfile: ci/docker/serve.build.Dockerfile
|
||||
srcs:
|
||||
- python/deplocks/ci/serve_tracing_depset_py3.10.lock
|
||||
build_args:
|
||||
- ENABLE_TRACING=1
|
||||
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py3.10
|
||||
- PYTHON=3.10
|
||||
- IMAGE_TYPE=tracing
|
||||
tags:
|
||||
- cr.ray.io/rayproject/servetracingbuild
|
||||
@@ -0,0 +1,9 @@
|
||||
ARG DOCKER_IMAGE_BASE_BUILD=rayproject/buildenv:windows
|
||||
FROM $DOCKER_IMAGE_BASE_BUILD
|
||||
|
||||
ENV PYTHON=3.10
|
||||
ENV PYTHON_FULL_VERSION=3.10.19
|
||||
ENV RAY_BUILD_ENV=win_py$PYTHON_FULL_VERSION
|
||||
|
||||
COPY . .
|
||||
RUN bash ci/ray_ci/windows/build_base.sh
|
||||
@@ -0,0 +1,13 @@
|
||||
name: "windowsbuild"
|
||||
froms: ["@rayproject/buildenv:windows"]
|
||||
dockerfile: ci/docker/windows.build.Dockerfile
|
||||
srcs:
|
||||
- python/requirements.txt
|
||||
- python/requirements_compiled.txt
|
||||
- python/requirements/test-requirements.txt
|
||||
- python/requirements/ml/dl-cpu-requirements.txt
|
||||
- ci/ray_ci/windows/build_base.sh
|
||||
- ci/ray_ci/windows/install_bazelisk.ps1
|
||||
- ci/ray_ci/windows/cleanup.ps1
|
||||
tags:
|
||||
- cr.ray.io/rayproject/windowsbuild
|
||||
Reference in New Issue
Block a user