chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 13:17:40 +08:00
commit f1825c8ceb
10096 changed files with 2364182 additions and 0 deletions
+29
View File
@@ -0,0 +1,29 @@
ARG DOCKER_IMAGE_BASE_TEST=cr.ray.io/rayproject/oss-ci-base_test
FROM $DOCKER_IMAGE_BASE_TEST
ARG RAYCI_DISABLE_JAVA=false
COPY . .
RUN <<EOF
#!/bin/bash -i
set -euo pipefail
if [[ "$RAYCI_DISABLE_JAVA" != "true" ]]; then
apt-get update -y
apt-get install -y -qq maven openjdk-8-jre openjdk-8-jdk
# Ensure Java 8 is the default; Ubuntu 22.04 defaults to Java 11.
if [[ "$(dpkg --print-architecture)" == "arm64" ]]; then
update-alternatives --set java /usr/lib/jvm/java-8-openjdk-arm64/jre/bin/java
else
update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
fi
fi
BUILD=1 ./ci/ci.sh init
EOF
ENV CC=clang
ENV CXX=clang++-14
+17
View File
@@ -0,0 +1,17 @@
name: "oss-ci-base_build-aarch64"
froms: ["cr.ray.io/rayproject/oss-ci-base_test-aarch64"]
dockerfile: ci/docker/base.build.Dockerfile
srcs:
- .bazelrc
- ci/ci.sh
- ci/env/install-dependencies.sh
- ci/env/install-llvm-binaries.sh
- ci/suppress_output
- python/requirements.txt
- python/requirements_compiled.txt
- python/requirements/test-requirements.txt
build_args:
- DOCKER_IMAGE_BASE_TEST=cr.ray.io/rayproject/oss-ci-base_test-aarch64
- RAYCI_DISABLE_JAVA
tags:
- cr.ray.io/rayproject/oss-ci-base_build-aarch64
+17
View File
@@ -0,0 +1,17 @@
name: "oss-ci-base_build-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON"]
dockerfile: ci/docker/base.build.Dockerfile
srcs:
- .bazelrc
- ci/ci.sh
- ci/env/install-dependencies.sh
- ci/env/install-llvm-binaries.sh
- ci/suppress_output
- python/requirements.txt
- python/requirements_compiled.txt
- python/requirements/test-requirements.txt
build_args:
- DOCKER_IMAGE_BASE_TEST=cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON
- RAYCI_DISABLE_JAVA
tags:
- cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
+18
View File
@@ -0,0 +1,18 @@
name: "oss-ci-base_cu128-py$PYTHON"
froms: ["nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04"]
dockerfile: ci/docker/base.gpu.Dockerfile
srcs:
- .bazelrc
- .bazelversion
- ci/env/install-dependencies.sh
- ci/env/install-llvm-binaries.sh
- ci/env/install-bazel.sh
- ci/env/install-miniforge.sh
- ci/suppress_output
build_args:
- BUILDKITE_BAZEL_CACHE_URL
- PYTHON
- BASE_IMAGE=nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04
- CUDA_VERSION=12.8.1
tags:
- cr.ray.io/rayproject/oss-ci-base_cu128-py$PYTHON
+18
View File
@@ -0,0 +1,18 @@
name: "oss-ci-base_cu130-py$PYTHON"
froms: ["nvidia/cuda:13.0.0-cudnn-devel-ubuntu22.04"]
dockerfile: ci/docker/base.gpu.Dockerfile
srcs:
- .bazelrc
- .bazelversion
- ci/env/install-dependencies.sh
- ci/env/install-llvm-binaries.sh
- ci/env/install-bazel.sh
- ci/env/install-miniforge.sh
- ci/suppress_output
build_args:
- BUILDKITE_BAZEL_CACHE_URL
- PYTHON
- BASE_IMAGE=nvidia/cuda:13.0.0-cudnn-devel-ubuntu22.04
- CUDA_VERSION=13.0.0
tags:
- cr.ray.io/rayproject/oss-ci-base_cu130-py$PYTHON
+72
View File
@@ -0,0 +1,72 @@
# syntax=docker/dockerfile:1.3-labs
ARG BASE_IMAGE=nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04
FROM $BASE_IMAGE
ARG BUILDKITE_BAZEL_CACHE_URL
ARG PYTHON=3.10
ARG CUDA_VERSION=12.8.1
ENV DEBIAN_FRONTEND=noninteractive
ENV TZ=America/Los_Angeles
ENV RAY_BUILD_ENV=ubuntu22.04_clang14_cuda${CUDA_VERSION}_py$PYTHON
ENV BUILDKITE=true
ENV CI=true
ENV PYTHON=$PYTHON
ENV RAY_USE_RANDOM_PORTS=1
ENV RAY_DEFAULT_BUILD=1
ENV RAY_INSTALL_JAVA=0
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
RUN <<EOF
#!/bin/bash
set -euo pipefail
apt-get update -qq && apt-get upgrade -qq
apt-get install -y -qq \
curl python-is-python3 git build-essential \
sudo zip unzip unrar apt-utils dialog tzdata wget rsync \
language-pack-en tmux cmake gdb vim htop \
libgtk2.0-dev zlib1g-dev libgl1-mesa-dev \
clang-format-14 jq \
clang-tidy-14 clang-14
ln -s /usr/bin/clang-format-14 /usr/bin/clang-format
ln -s /usr/bin/clang-tidy-14 /usr/bin/clang-tidy
ln -s /usr/bin/clang-14 /usr/bin/clang
# Install docker CLI
mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
chmod a+r /etc/apt/keyrings/docker.asc
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
$(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
tee /etc/apt/sources.list.d/docker.list > /dev/null
apt-get update
apt-get install -y docker-ce-cli
echo "build --remote_cache=${BUILDKITE_BAZEL_CACHE_URL}" >> /root/.bazelrc
curl -fsSL https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="/usr/local/bin" sh
EOF
ENV CC=clang
ENV CXX=clang++-14
# System conf for tests
RUN locale -a
ENV LC_ALL=en_US.utf8
ENV LANG=en_US.utf8
RUN echo "ulimit -c 0" >> /root/.bashrc
# Install some dependencies (miniforge, pip dependencies, etc)
RUN mkdir /ray
WORKDIR /ray
COPY . .
RUN bash --login -ie -c '\
BUILD=1 SKIP_PYTHON_PACKAGES=1 ./ci/env/install-dependencies.sh \
'
+17
View File
@@ -0,0 +1,17 @@
name: "oss-ci-base_gpu-py$PYTHON"
froms: ["nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04"]
dockerfile: ci/docker/base.gpu.Dockerfile
srcs:
- .bazelrc
- .bazelversion
- ci/env/install-dependencies.sh
- ci/env/install-llvm-binaries.sh
- ci/env/install-bazel.sh
- ci/env/install-miniforge.sh
- ci/suppress_output
build_args:
- BUILDKITE_BAZEL_CACHE_URL
- PYTHON
- CUDA_VERSION
tags:
- cr.ray.io/rayproject/oss-ci-base_gpu-py$PYTHON
+18
View File
@@ -0,0 +1,18 @@
# syntax=docker/dockerfile:1.3-labs
ARG DOCKER_IMAGE_BASE_TEST=cr.ray.io/rayproject/oss-ci-base_test
FROM $DOCKER_IMAGE_BASE_TEST
COPY . .
RUN <<EOF
#!/bin/bash -i
set -e
BUILD=1 ./ci/ci.sh init
RLLIB_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 bash --login -i ./ci/env/install-dependencies.sh
pip uninstall -y ray
EOF
+24
View File
@@ -0,0 +1,24 @@
name: "oss-ci-base_ml-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON"]
dockerfile: ci/docker/base.ml.Dockerfile
srcs:
- .bazelrc
- ci/ci.sh
- ci/env/install-dependencies.sh
- ci/env/install-llvm-binaries.sh
- ci/suppress_output
- python/requirements.txt
- python/requirements_compiled.txt
- python/requirements/test-requirements.txt
- python/requirements/ml/rllib-requirements.txt
- python/requirements/ml/rllib-test-requirements.txt
- python/requirements/ml/train-requirements.txt
- python/requirements/ml/train-test-requirements.txt
- python/requirements/ml/tune-requirements.txt
- python/requirements/ml/tune-test-requirements.txt
- python/requirements/ml/dl-cpu-requirements.txt
- python/requirements/ml/core-requirements.txt
build_args:
- DOCKER_IMAGE_BASE_TEST=cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON
tags:
- cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
+68
View File
@@ -0,0 +1,68 @@
# syntax=docker/dockerfile:1.3-labs
FROM ubuntu:jammy
ARG BUILDKITE_BAZEL_CACHE_URL
ARG PYTHON=3.10
ENV DEBIAN_FRONTEND=noninteractive
ENV TZ=America/Los_Angeles
ENV RAY_BUILD_ENV=ubuntu22.04_clang14_py$PYTHON
ENV BUILDKITE=true
ENV CI=true
ENV PYTHON=$PYTHON
ENV RAY_USE_RANDOM_PORTS=1
ENV RAY_DEFAULT_BUILD=1
ENV RAY_INSTALL_JAVA=0
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
RUN <<EOF
#!/bin/bash
set -euo pipefail
apt-get update -qq && apt-get upgrade -qq
apt-get install -y -qq \
curl python-is-python3 git build-essential \
sudo zip unzip unrar apt-utils dialog tzdata wget rsync \
language-pack-en tmux cmake gdb vim htop graphviz \
libgtk2.0-dev zlib1g-dev libgl1-mesa-dev \
liblz4-dev libunwind-dev libncurses6 \
clang-format-14 jq \
clang-tidy-14 clang-14
ln -s /usr/bin/clang-format-14 /usr/bin/clang-format
ln -s /usr/bin/clang-tidy-14 /usr/bin/clang-tidy
ln -s /usr/bin/clang-14 /usr/bin/clang
# Install docker CLI
mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
chmod a+r /etc/apt/keyrings/docker.asc
echo \
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
$(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
tee /etc/apt/sources.list.d/docker.list > /dev/null
apt-get update
apt-get install -y docker-ce-cli
EOF
ENV CC=clang
ENV CXX=clang++-14
# System conf for tests
RUN locale -a
ENV LC_ALL=en_US.utf8
ENV LANG=en_US.utf8
RUN echo "ulimit -c 0" >> /root/.bashrc
# Install some dependencies (miniforge, pip dependencies, etc)
RUN mkdir /ray
WORKDIR /ray
COPY . .
RUN ./ci/env/install-miniforge.sh
RUN ./ci/env/install-bazel.sh
+13
View File
@@ -0,0 +1,13 @@
name: "oss-ci-base_test-aarch64"
froms: ["ubuntu:jammy"]
dockerfile: ci/docker/base.test.Dockerfile
srcs:
- ci/env/install-bazel.sh
- ci/env/install-miniforge.sh
- ci/suppress_output
- .bazelversion
build_args:
- BUILDKITE_BAZEL_CACHE_URL
- PYTHON=3.10
tags:
- cr.ray.io/rayproject/oss-ci-base_test-aarch64
+13
View File
@@ -0,0 +1,13 @@
name: "oss-ci-base_test-py$PYTHON"
froms: ["ubuntu:jammy"]
dockerfile: ci/docker/base.test.Dockerfile
srcs:
- ci/env/install-bazel.sh
- ci/env/install-miniforge.sh
- ci/suppress_output
- .bazelversion
build_args:
- BUILDKITE_BAZEL_CACHE_URL
- PYTHON
tags:
- cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON
+23
View File
@@ -0,0 +1,23 @@
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py3.10
FROM $DOCKER_IMAGE_BASE_BUILD
ARG PYTHON=3.10
ARG BUILD_VARIANT=build
ARG PYTHON_DEPSET=python/deplocks/ci/core-${BUILD_VARIANT}-ci_depset_py${PYTHON}.lock
SHELL ["/bin/bash", "-ice"]
COPY . .
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
RUN <<EOF
#!/bin/bash
set -euo pipefail
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
uv pip uninstall --system ray
EOF
+11
View File
@@ -0,0 +1,11 @@
name: "core$BUILD_VARIANT-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON"]
dockerfile: ci/docker/core.build.Dockerfile
srcs:
- python/deplocks/ci/core-$BUILD_VARIANT-ci_depset_py$PYTHON.lock
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON
- PYTHON
- BUILD_VARIANT
tags:
- cr.ray.io/rayproject/core$BUILD_VARIANT-py$PYTHON
@@ -0,0 +1,10 @@
# Dedicated image for the Ray Serve LLM custom-vLLM-model docs test. It layers
# the custom reward-model plugin on top of the standard LLM GPU test image so
# the plugin is available on every node without touching the shared image.
ARG DOCKER_IMAGE_BASE=cr.ray.io/rayproject/llmgpubuild
FROM $DOCKER_IMAGE_BASE
COPY . .
# The plugin's entry point registers Qwen3CustomRewardModel in every vLLM process.
RUN pip install --no-deps doc/source/llm/doc_code/serve/custom_vllm
@@ -0,0 +1,13 @@
name: "custom-vllm-plugin-gpubuild"
froms: ["cr.ray.io/rayproject/llmgpubuild"]
dockerfile: ci/docker/custom-vllm-plugin.build.Dockerfile
srcs:
# The plugin source, installed on top of the LLM GPU image (see the Dockerfile).
- doc/source/llm/doc_code/serve/custom_vllm/setup.py
- doc/source/llm/doc_code/serve/custom_vllm/qwen3_reward_plugin/__init__.py
- doc/source/llm/doc_code/serve/custom_vllm/qwen3_reward_plugin/qwen3_rm.py
- doc/source/llm/doc_code/serve/custom_vllm/qwen3_reward_plugin/serve_hook.py
tags:
- cr.ray.io/rayproject/custom-vllm-plugin-gpubuild
build_args:
- DOCKER_IMAGE_BASE=cr.ray.io/rayproject/llmgpubuild
+52
View File
@@ -0,0 +1,52 @@
# syntax=docker/dockerfile:1.3-labs
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py3.10
FROM $DOCKER_IMAGE_BASE_BUILD
ARG RAY_CI_JAVA_BUILD=
ARG IMAGE_TYPE=base
ARG PYTHON=3.10
ARG PYTHON_DEPSET=python/deplocks/ci/data-$IMAGE_TYPE-ci_depset_py$PYTHON.lock
COPY $PYTHON_DEPSET /home/ray/python_depset.lock
SHELL ["/bin/bash", "-ice"]
RUN <<EOF
#!/bin/bash
set -ex
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
if [[ "$IMAGE_TYPE" == "pyarrow-nightly" ]]; then
uv pip install \
--system \
--prerelease allow \
--extra-index-url https://pypi.fury.io/arrow-nightlies/ \
--upgrade-package pyarrow \
pyarrow
fi
curl -fsSL https://pgp.mongodb.com/server-8.0.asc | \
sudo gpg -o /usr/share/keyrings/mongodb-server-8.0.gpg --dearmor
echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-8.0.gpg ] \
https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/8.0 multiverse" | \
sudo tee /etc/apt/sources.list.d/mongodb-org-8.0.list
sudo apt-get update
sudo apt-get install -y mongodb-org
if [[ $RAY_CI_JAVA_BUILD == 1 ]]; then
# These packages increase the image size quite a bit, so we only install them
# as needed.
sudo apt-get install -y -qq maven openjdk-8-jre openjdk-8-jdk
# Ensure Java 8 is the default; Ubuntu 22.04 defaults to Java 11 which
# breaks Spark's reflective access to DirectByteBuffer.
if [[ "$(dpkg --print-architecture)" == "arm64" ]]; then
sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-arm64/jre/bin/java
else
sudo update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
fi
fi
EOF
+11
View File
@@ -0,0 +1,11 @@
name: "data17build-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
dockerfile: ci/docker/data.build.Dockerfile
srcs:
- python/deplocks/ci/data-pyarrow-v17-ci_depset_py$PYTHON.lock
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
- IMAGE_TYPE=pyarrow-v17
- PYTHON=$PYTHON
tags:
- cr.ray.io/rayproject/data17build-py$PYTHON
+11
View File
@@ -0,0 +1,11 @@
name: "datalbuild-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
dockerfile: ci/docker/data.build.Dockerfile
srcs:
- python/deplocks/ci/data-pyarrow-latest-ci_depset_py$PYTHON.lock
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
- IMAGE_TYPE=pyarrow-latest
- PYTHON=$PYTHON
tags:
- cr.ray.io/rayproject/datalbuild-py$PYTHON
+12
View File
@@ -0,0 +1,12 @@
name: "datamongobuild-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
dockerfile: ci/docker/data.build.Dockerfile
srcs:
- python/deplocks/ci/data-mongo-ci_depset_py$PYTHON.lock
build_args:
- RAY_CI_JAVA_BUILD=1
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
- IMAGE_TYPE=mongo
- PYTHON=$PYTHON
tags:
- cr.ray.io/rayproject/datamongobuild-py$PYTHON
+11
View File
@@ -0,0 +1,11 @@
name: "datanbuild-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
dockerfile: ci/docker/data.build.Dockerfile
srcs:
- python/deplocks/ci/data-pyarrow-nightly-ci_depset_py$PYTHON.lock
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
- IMAGE_TYPE=pyarrow-nightly
- PYTHON=$PYTHON
tags:
- cr.ray.io/rayproject/datanbuild-py$PYTHON
+24
View File
@@ -0,0 +1,24 @@
# syntax=docker/dockerfile:1.3-labs
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py3.10
FROM $DOCKER_IMAGE_BASE_BUILD
ARG ARROW_VERSION=14.*
ARG ARROW_MONGO_VERSION=
ARG RAY_CI_JAVA_BUILD=
ARG PYTHON=3.12
ARG PYTHON_DEPSET=python/deplocks/tests/data/datatfds_py$PYTHON.lock
SHELL ["/bin/bash", "-ice"]
COPY . .
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
RUN <<EOF
#!/bin/bash
set -euo pipefail
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
EOF
+10
View File
@@ -0,0 +1,10 @@
name: "datatfdsbuild-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON"]
dockerfile: ci/docker/datatfds.build.Dockerfile
srcs:
- python/deplocks/tests/data/datatfds_py$PYTHON.lock
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py$PYTHON
- PYTHON
tags:
- cr.ray.io/rayproject/datatfdsbuild-py$PYTHON
+29
View File
@@ -0,0 +1,29 @@
ARG DOCKER_IMAGE_RAY_CORE=cr.ray.io/rayproject/ray-core-py3.10
ARG DOCKER_IMAGE_RAY_DASHBOARD=cr.ray.io/rayproject/ray-dashboard
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py3.10
FROM $DOCKER_IMAGE_RAY_CORE AS ray_core
FROM $DOCKER_IMAGE_RAY_DASHBOARD AS ray_dashboard
FROM $DOCKER_IMAGE_BASE_BUILD
COPY . .
SHELL ["/bin/bash", "-ice"]
RUN --mount=type=bind,from=ray_core,target=/mnt/ray-core \
--mount=type=bind,from=ray_dashboard,target=/mnt/ray-dashboard \
<<EOF
#!/bin/bash
set -euo pipefail
mkdir -p /opt/ray-build
cp /mnt/ray-core/ray_pkg.zip /opt/ray-build/ray_pkg.zip
cp /mnt/ray-core/ray_py_proto.zip /opt/ray-build/ray_py_proto.zip
cp /mnt/ray-dashboard/dashboard.tar.gz /opt/ray-build/dashboard.tar.gz
pip install -r python/deplocks/docs/docbuild_depset_py${PYTHON}.lock
EOF
+14
View File
@@ -0,0 +1,14 @@
name: "docbuild-py$PYTHON"
froms:
- cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
- cr.ray.io/rayproject/ray-core-py$PYTHON
- cr.ray.io/rayproject/ray-dashboard
dockerfile: ci/docker/doc.build.Dockerfile
srcs:
- $REQUIREMENTS_FILE
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
- DOCKER_IMAGE_RAY_CORE=cr.ray.io/rayproject/ray-core-py$PYTHON
- DOCKER_IMAGE_RAY_DASHBOARD=cr.ray.io/rayproject/ray-dashboard
tags:
- cr.ray.io/rayproject/docbuild-py$PYTHON
+25
View File
@@ -0,0 +1,25 @@
# syntax=docker/dockerfile:1.3-labs
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_gpu-py3.10
FROM $DOCKER_IMAGE_BASE_BUILD
ARG PYTHON
ARG PYTHON_DEPSET=python/deplocks/ci/docgpu_depset_py$PYTHON.lock
SHELL ["/bin/bash", "-ice"]
COPY . .
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
RUN <<EOF
#!/bin/bash
set -euo pipefail
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
# Remove installed ray so the source overlay at /rayci/ is used at test time
pip uninstall -y ray
EOF
+10
View File
@@ -0,0 +1,10 @@
name: "docgpubuild-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_gpu-py$PYTHON"]
dockerfile: ci/docker/docgpu.build.Dockerfile
srcs:
- python/deplocks/ci/docgpu_depset_py$PYTHON.lock
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_gpu-py$PYTHON
- PYTHON
tags:
- cr.ray.io/rayproject/docgpubuild-py$PYTHON
+22
View File
@@ -0,0 +1,22 @@
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
FROM $DOCKER_IMAGE_BASE_BUILD
ARG PYTHON=3.10
ARG PYTHON_DEPSET=python/deplocks/docs/doctest_depset_py$PYTHON.lock
SHELL ["/bin/bash", "-ice"]
COPY . .
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
RUN <<EOF
#!/bin/bash
set -euo pipefail
sudo apt-get update
sudo apt-get install -y tesseract-ocr libosmesa6-dev libglfw3 patchelf
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
EOF
+10
View File
@@ -0,0 +1,10 @@
name: "doctestbuild"
froms: ["cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON"]
dockerfile: ci/docker/doctest.build.Dockerfile
srcs:
- python/deplocks/docs/doctest_depset_py$PYTHON.lock
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
- PYTHON
tags:
- cr.ray.io/rayproject/doctestbuild
+151
View File
@@ -0,0 +1,151 @@
# syntax=docker/dockerfile:1.3-labs
FROM ubuntu:22.04
ARG BUILDKITE_BAZEL_CACHE_URL
ENV DEBIAN_FRONTEND=noninteractive
ENV PATH="/home/forge/.local/bin:${PATH}"
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
ENV RAY_BUILD_ENV=ubuntu22.04_forge
RUN \
--mount=type=bind,source=ci/k8s/install-k8s-tools.sh,target=install-k8s-tools.sh \
<<EOF
#!/bin/bash
set -euo pipefail
apt-get update
apt-get upgrade -y
apt-get install -y ca-certificates curl zip unzip sudo gnupg tzdata git apt-transport-https lsb-release
# Add docker client APT repository
mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
# Download and install Microsoft signing key
curl -fsSL https://packages.microsoft.com/keys/microsoft.asc |
gpg --dearmor | tee /etc/apt/keyrings/microsoft.gpg > /dev/null
chmod go+r /etc/apt/keyrings/microsoft.gpg
echo \
"deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
"$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
tee /etc/apt/sources.list.d/docker.list > /dev/null
# Add NodeJS APT repository
curl -fsSL https://deb.nodesource.com/setup_18.x | bash -
AZ_VER=2.72.0
AZ_DIST="$(lsb_release -cs)"
# Add Azure CLI repository
echo "Types: deb
URIs: https://packages.microsoft.com/repos/azure-cli/
Suites: ${AZ_DIST}
Components: main
Architectures: $(dpkg --print-architecture)
Signed-by: /etc/apt/keyrings/microsoft.gpg" | tee /etc/apt/sources.list.d/azure-cli.sources
# Add Google Cloud CLI repository
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg |
gpg --dearmor -o /etc/apt/keyrings/cloud.google.gpg
echo "deb [signed-by=/etc/apt/keyrings/cloud.google.gpg] \
https://packages.cloud.google.com/apt cloud-sdk main" |
tee /etc/apt/sources.list.d/google-cloud-sdk.list
# Install packages
apt-get update
apt-get install -y \
awscli nodejs build-essential python-is-python3 \
python3-pip openjdk-8-jre wget jq \
docker-ce-cli azure-cli="${AZ_VER}"-1~"${AZ_DIST}" \
google-cloud-cli
# Install uv
curl -fsSL https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="/usr/local/bin" sh
mkdir -p /usr/local/python
# Install Python using uv
UV_PYTHON_VERSION=3.10
uv python install --install-dir /usr/local/python "$UV_PYTHON_VERSION"
export UV_PYTHON_INSTALL_DIR=/usr/local/python
# Make Python from uv the default by creating symlinks
UV_PYTHON_BIN="$(uv python find --no-project "$UV_PYTHON_VERSION")"
echo "uv python binary location: $UV_PYTHON_BIN"
ln -s "$UV_PYTHON_BIN" "/usr/local/bin/python${UV_PYTHON_VERSION}"
ln -s "$UV_PYTHON_BIN" /usr/local/bin/python3
ln -s "$UV_PYTHON_BIN" /usr/local/bin/python
# As a convention, we pin all python packages to a specific version. This
# is to to make sure we can control version upgrades through code changes.
uv pip install --system pip==25.0 cffi==1.16.0
# Needs to be synchronized to the host group id as we map /var/run/docker.sock
# into the container.
addgroup --gid 1001 docker0 # Used on old buildkite AMIs before 2023
addgroup --gid 993 docker1
addgroup --gid 992 docker # buildkite AMI as of 2025-06-07
# Install bazelisk
npm install -g @bazel/bazelisk
ln -s /usr/local/bin/bazel /usr/local/bin/bazelisk
# A non-root user. Use 2000, which is the same as our buildkite agent VM uses.
adduser --home /home/forge --uid 2000 forge --gid 100
usermod -a -G docker0 forge
usermod -a -G docker1 forge
usermod -a -G docker forge
# Create a shared directory for the ray repository checkout.
mkdir /rayci
chown forge:users /rayci
if [[ "$(uname -i)" == "x86_64" ]]; then
bash install-k8s-tools.sh
fi
# Install crane (container registry tool)
CRANE_VERSION=0.19.0
case "$(uname -m)" in
x86_64|amd64)
CRANE_ARCH="x86_64"
;;
aarch64|arm64)
CRANE_ARCH="arm64"
;;
*)
echo "Unsupported architecture: $(uname -m)" >&2
exit 1
;;
esac
curl -fsSL "https://github.com/google/go-containerregistry/releases/download/v${CRANE_VERSION}/go-containerregistry_Linux_${CRANE_ARCH}.tar.gz" \
| tar -xzf - -C /usr/local/bin crane
chmod +x /usr/local/bin/crane
EOF
USER forge
RUN <<EOF
#!/bin/bash
set -euo pipefail
{
echo "build --config=ci"
echo "build --announce_rc"
echo "build --remote_cache=${BUILDKITE_BAZEL_CACHE_URL}"
} > ~/.bazelrc
EOF
ENV DOCKER_API_VERSION=1.43
CMD ["echo", "ray forge"]
# last update: 2026-02-10
+8
View File
@@ -0,0 +1,8 @@
name: "forge-aarch64"
froms:
- ubuntu:22.04
build_args:
- BUILDKITE_BAZEL_CACHE_URL
dockerfile: ci/docker/forge.Dockerfile
srcs:
- ci/k8s/install-k8s-tools.sh
+8
View File
@@ -0,0 +1,8 @@
name: "forge"
froms:
- ubuntu:22.04
build_args:
- BUILDKITE_BAZEL_CACHE_URL
dockerfile: ci/docker/forge.Dockerfile
srcs:
- ci/k8s/install-k8s-tools.sh
+36
View File
@@ -0,0 +1,36 @@
# syntax=docker/dockerfile:1.3-labs
FROM cr.ray.io/rayproject/forge
# Add forge user to sudo group
USER root
RUN <<EOF
#!/bin/bash
set -euo pipefail
usermod -aG sudo forge && \
echo "forge ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/forge && \
chmod 0440 /etc/sudoers.d/forge
EOF
USER forge
# Install fossa and askalono
RUN <<EOF
#!/bin/bash
set -euo pipefail
curl -sSfL https://github.com/fossas/fossa-cli/releases/download/v3.10.6/fossa_3.10.6_linux_amd64.tar.gz \
| sudo tar -xvzf - -C /usr/local/bin/
wget -O /tmp/askalono.zip https://github.com/jpeddicord/askalono/releases/download/0.5.0/askalono-Linux.zip
unzip -d /tmp/ /tmp/askalono.zip
sudo mv /tmp/askalono /usr/local/bin/
rm /tmp/askalono.zip
EOF
CMD ["echo", "ray fossa"]
+4
View File
@@ -0,0 +1,4 @@
name: "fossa-base"
froms:
- cr.ray.io/rayproject/forge
dockerfile: ci/docker/fossa.Dockerfile
+22
View File
@@ -0,0 +1,22 @@
# syntax=docker/dockerfile:1.3-labs
FROM cr.ray.io/rayproject/oss-ci-base_build-py3.10
SHELL ["/bin/bash", "-ice"]
COPY . .
RUN <<EOF
#!/bin/bash
set -euo pipefail
bash ci/k8s/install-k8s-tools.sh
pip install -U --ignore-installed \
-r python/requirements.txt -c python/requirements_compiled.txt \
pip install -U --ignore-installed \
docker -c python/requirements_compiled.txt \
EOF
+9
View File
@@ -0,0 +1,9 @@
name: k8sbuild
froms: ["cr.ray.io/rayproject/oss-ci-base_build-py3.10"]
dockerfile: ci/docker/k8s.build.Dockerfile
srcs:
- python/requirements.txt
- python/requirements_compiled.txt
- ci/k8s/install-k8s-tools.sh
tags:
- cr.ray.io/rayproject/k8sbuild
+59
View File
@@ -0,0 +1,59 @@
# syntax=docker/dockerfile:1.3-labs
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py3.11
FROM $DOCKER_IMAGE_BASE_BUILD
ARG RAY_CI_JAVA_BUILD=
ARG RAY_CUDA_CODE=cpu
SHELL ["/bin/bash", "-ice"]
COPY . .
RUN <<EOF
#!/bin/bash
set -euo pipefail
SKIP_PYTHON_PACKAGES=1 ./ci/env/install-dependencies.sh
PYTHON_CODE="$(python -c "import sys; v=sys.version_info; print(f'py{v.major}{v.minor}')")"
pip install --no-deps -r python/deplocks/llm/rayllm_test_${PYTHON_CODE}_${RAY_CUDA_CODE}.lock
# Include the CUDA device index in vLLM's compile cache paths so a worker never
# reloads a torch.compile artifact built for a different physical GPU.
# TODO (jeffreywang): Remove this patch once https://github.com/vllm-project/vllm/pull/38962 lands.
VLLM_DEVICE_AWARE_COMPILE_CACHE_PATCH="$(pwd)/python/requirements/llm/patches/vllm-device-aware-compile-cache.patch"
VLLM_SITE_PACKAGES="$(python - <<'PY'
import site
import sysconfig
from pathlib import Path
candidate_dirs = [
Path(sysconfig.get_paths()["purelib"]),
Path(sysconfig.get_paths()["platlib"]),
*(Path(path) for path in site.getsitepackages()),
]
for base_dir in dict.fromkeys(candidate_dirs):
import_utils = base_dir / "vllm" / "utils" / "import_utils.py"
if import_utils.exists():
print(base_dir)
break
else:
raise SystemExit("vLLM import_utils.py not found")
PY
)"
(
cd "${VLLM_SITE_PACKAGES}"
git apply "${VLLM_DEVICE_AWARE_COMPILE_CACHE_PATCH}"
)
EOF
# vLLM 0.21.0 selects the FlashInfer top-k/top-p sampler during engine initialization
# instead of the previous PyTorch-native/Triton sampling path. The FlashInfer sampler
# introduces longer adds a large one-time engine initialization cost. To avoid performance
# surprises, we disable the FlashInfer sampler by default.
ENV VLLM_USE_FLASHINFER_SAMPLER=0
+15
View File
@@ -0,0 +1,15 @@
name: "llm$BUILD_VARIANT"
froms: ["cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON"]
dockerfile: ci/docker/llm.build.Dockerfile
srcs:
- ci/env/install-dependencies.sh
- ci/env/install-llvm-binaries.sh
- ci/suppress_output
- python/deplocks/llm/rayllm_test_py312_cpu.lock
- python/deplocks/llm/rayllm_test_py312_cu130.lock
- python/requirements/llm/patches/vllm-device-aware-compile-cache.patch
tags:
- cr.ray.io/rayproject/llm$BUILD_VARIANT
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON
- RAY_CUDA_CODE=$RAY_CUDA_CODE
+10
View File
@@ -0,0 +1,10 @@
name: "manylinux-cibase$JDK_SUFFIX-$HOSTTYPE"
froms:
- quay.io/pypa/manylinux2014_$HOSTTYPE:2026.01.02-1
srcs:
- ci/build/build-manylinux-forge.sh
build_args:
- BUILDKITE_BAZEL_CACHE_URL
- RAYCI_DISABLE_JAVA
- HOSTTYPE
dockerfile: ci/docker/manylinux.Dockerfile
+30
View File
@@ -0,0 +1,30 @@
# syntax=docker/dockerfile:1.3-labs
# NOTE(andrew-anyscale): This Dockerfile is used to retag the manylinux image to
# a new tag with Wanda. This is kept as-is until we can rework LinuxContainer to
# use a hardcoded image tag not hosted in ECR. See:
# https://github.com/ray-project/ray/blob/6ab10189b0f6506158bac76437a97b28c2643155/ci/ray_ci/builder_container.py#L16
# https://github.com/ray-project/ray/blob/master/ci/ray_ci/container.py#L57C49-L57C59
ARG MANYLINUX_VERSION
ARG HOSTTYPE
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE}
ARG BUILDKITE_BAZEL_CACHE_URL
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
# Still keep bazelrc updates to allow BUILDKITE_BAZEL_CACHE_URL to be used.
RUN <<EOF
#!/bin/bash
set -euo pipefail
{
echo "build --config=ci"
echo "build --announce_rc"
if [[ "${BUILDKITE_BAZEL_CACHE_URL:-}" != "" ]]; then
echo "build:ci --remote_cache=${BUILDKITE_BAZEL_CACHE_URL:-}"
fi
} > "$HOME"/.bazelrc
EOF
+54
View File
@@ -0,0 +1,54 @@
# syntax=docker/dockerfile:1.3-labs
ARG HOSTTYPE
FROM quay.io/pypa/manylinux2014_${HOSTTYPE}:2026.01.02-1
ARG BUILDKITE_BAZEL_CACHE_URL
ARG RAYCI_DISABLE_JAVA=false
# uid needs to be synced with forge.Dockerfile
ARG FORGE_UID=2000
ENV BUILD_JAR=1
ENV RAYCI_DISABLE_JAVA=$RAYCI_DISABLE_JAVA
ENV RAY_INSTALL_JAVA=1
ENV BUILDKITE_BAZEL_CACHE_URL=$BUILDKITE_BAZEL_CACHE_URL
RUN yum -y install sudo
RUN curl -LsSf https://astral.sh/uv/0.8.17/install.sh | \
env UV_INSTALL_DIR=/usr/local/bin sh
RUN <<EOF
#!/bin/bash
set -euo pipefail
useradd -m -u "$FORGE_UID" -g users -d /home/forge forge
echo 'forge ALL=NOPASSWD: ALL' >> /etc/sudoers
EOF
COPY ci/build/build-manylinux-forge.sh /tmp/build-manylinux-forge.sh
RUN ./tmp/build-manylinux-forge.sh
USER forge
ENV HOME=/home/forge
RUN <<EOF
#!/bin/bash
set -euo pipefail
{
echo "build --config=ci"
echo "build --announce_rc"
if [[ "${BUILDKITE_BAZEL_CACHE_URL:-}" != "" ]]; then
echo "build:ci --remote_cache=${BUILDKITE_BAZEL_CACHE_URL:-}"
fi
} > "$HOME"/.bazelrc
EOF
# last kick: 2025-10-08
+8
View File
@@ -0,0 +1,8 @@
name: "manylinux-$HOSTTYPE"
froms:
- rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE
build_args:
- BUILDKITE_BAZEL_CACHE_URL
- MANYLINUX_VERSION
- HOSTTYPE
dockerfile: ci/docker/manylinux-retag.Dockerfile
+77
View File
@@ -0,0 +1,77 @@
# syntax=docker/dockerfile:1.3-labs
ARG PYTHON_VERSION=3.10
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON_VERSION
FROM $DOCKER_IMAGE_BASE_BUILD
ARG PYTHON_VERSION
ARG EXTRA_DEPENDENCY
SHELL ["/bin/bash", "-ice"]
COPY . .
RUN <<EOF
#!/bin/bash
set -euo pipefail
# minimal dependencies
BUILD=1 MINIMAL_INSTALL=1 PYTHON=${PYTHON_VERSION} ./ci/ci.sh init
rm -rf python/ray/thirdparty_files
# Re-activate conda environment (ci.sh runs in a subprocess so PATH changes are lost)
# For Python 3.14+, a separate conda env is created and symlinked to /opt/miniforge/bin
# We can't use `conda activate` because the conda command itself breaks after symlinking
# (it tries to use the new Python which doesn't have conda installed)
if [[ -d /opt/miniforge/envs/py${PYTHON_VERSION} ]]; then
export PATH="/opt/miniforge/envs/py${PYTHON_VERSION}/bin:/opt/miniforge/bin:$PATH"
else
export PATH="/opt/miniforge/bin:$PATH"
fi
# install test requirements
python -m pip install -U pytest==8.3.3 pip-tools==7.4.1
# Verify pytest was installed and is accessible
python -m pytest --version || echo "WARNING: pytest not found in PATH"
# install extra dependencies
if [[ "${EXTRA_DEPENDENCY}" == "core" ]]; then
pip-compile -o min_requirements.txt python/setup.py
elif [[ "${EXTRA_DEPENDENCY}" == "ml" ]]; then
pip-compile -o min_requirements.txt python/setup.py --extra tune
elif [[ "${EXTRA_DEPENDENCY}" == "default" ]]; then
pip-compile -o min_requirements.txt python/setup.py --extra default
elif [[ "${EXTRA_DEPENDENCY}" == "serve" ]]; then
echo "httpx==0.27.2" >> /tmp/min_build_requirements.txt
echo "pytest-asyncio==1.1.0" >> /tmp/min_build_requirements.txt
pip-compile -o min_requirements.txt /tmp/min_build_requirements.txt python/setup.py --extra "serve-grpc"
rm /tmp/min_build_requirements.txt
fi
pip install -r min_requirements.txt
# Core wants to eagerly be tested with some of its prerelease dependencies.
if [[ "${EXTRA_DEPENDENCY}" == "core" ]]; then
./ci/env/install-core-prerelease-dependencies.sh
fi
# Verify pytest was installed
python -m pytest --version
EOF
# For Python 3.14+, the PATH needs to include the conda environment directory
# Create a symlink from the env-specific python to a standard location
RUN bash -c 'if [[ -d /opt/miniforge/envs/py${PYTHON_VERSION}/bin ]]; then \
for f in /opt/miniforge/envs/py${PYTHON_VERSION}/bin/*; do \
ln -sf "$f" "/opt/miniforge/bin/$(basename "$f")" 2>/dev/null || true; \
done \
fi'
# Ensure /opt/miniforge/bin is in PATH for all shells (including non-interactive)
# This is needed for tests.env.Dockerfile which uses bash -i
# For Python 3.14+, also add the env-specific bin directory to PATH
# so that console scripts installed later (like 'ray') are accessible
ENV PATH="/opt/miniforge/bin:/opt/miniforge/envs/py${PYTHON_VERSION}/bin:${PATH}"
+21
View File
@@ -0,0 +1,21 @@
name: "minbuild-$EXTRA_DEPENDENCY-py$PYTHON_VERSION"
froms: ["cr.ray.io/rayproject/oss-ci-base_test-py$PYTHON_VERSION"]
dockerfile: ci/docker/min.build.Dockerfile
srcs:
- .bazelrc
- ci/ci.sh
- ci/env/install-dependencies.sh
- ci/env/install-llvm-binaries.sh
- ci/env/install-core-prerelease-dependencies.sh
- ci/suppress_output
- python/requirements.txt
- python/requirements_compiled.txt
- python/setup.py
- python/LICENSE.txt
- python/ray/_version.py
- README.rst
build_args:
- PYTHON_VERSION
- EXTRA_DEPENDENCY
tags:
- cr.ray.io/rayproject/minbuild-$EXTRA_DEPENDENCY-py$PYTHON_VERSION
+61
View File
@@ -0,0 +1,61 @@
# syntax=docker/dockerfile:1.3-labs
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py3.10
FROM $DOCKER_IMAGE_BASE_BUILD
ARG RAYCI_IS_GPU_BUILD=false
ARG PYTHON
ARG IMAGE_TYPE=build
ARG PYTHON_DEPSET=python/deplocks/ci/ml-$IMAGE_TYPE-ci_depset_py$PYTHON.lock
ARG THIRDPARTY_DEPSET=python/deplocks/ci/ci_ml_thirdparty_depset_py$PYTHON.lock
SHELL ["/bin/bash", "-ice"]
COPY . .
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
COPY "$THIRDPARTY_DEPSET" /home/ray/thirdparty_depset.lock
RUN <<EOF
#!/bin/bash
set -euo pipefail
set -x
./ci/env/install-hdfs.sh
# Install system packages for doc testing (not in base image which lacks DOC_TESTING=1)
sudo apt-get update -qq && sudo apt-get install -y graphviz tesseract-ocr
# Install HEBO for testing (not supported on Python 3.12+)
if [[ "${PYTHON-}" != "3.12" ]]; then
pip install HEBO==0.3.5
fi
# Install doc-testing Python packages not covered by depsets
pip install pydot pytesseract==0.3.13
# Install thirdparty dependencies
mkdir -p python/ray/thirdparty_files
uv pip install -r /home/ray/thirdparty_depset.lock --no-deps --target=python/ray/thirdparty_files
# Install Python dependencies from depset lock file
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
# Inject our own mirror for the CIFAR10 dataset
SITE_PACKAGES=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')
TF_CIFAR="${SITE_PACKAGES}/tensorflow/python/keras/datasets/cifar10.py"
TF_KERAS_CIFAR="${SITE_PACKAGES}/tf_keras/src/datasets/cifar10.py"
TORCH_CIFAR="${SITE_PACKAGES}/torchvision/datasets/cifar.py"
KERAS_CIFAR="${SITE_PACKAGES}/keras/src/datasets/cifar10.py"
for f in "$TF_CIFAR" "$TF_KERAS_CIFAR" "$TORCH_CIFAR" "$KERAS_CIFAR"; do
[ -f "$f" ] && sed -i 's https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz https://air-example-data.s3.us-west-2.amazonaws.com/cifar-10-python.tar.gz g' "$f"
done
# Remove installed ray.
pip uninstall -y ray
EOF
+14
View File
@@ -0,0 +1,14 @@
name: "ml$IMAGE_TYPE-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON"]
dockerfile: ci/docker/ml.build.Dockerfile
srcs:
- ci/env/install-hdfs.sh
- python/deplocks/ci/ml-$IMAGE_TYPE-ci_depset_py$PYTHON.lock
- python/deplocks/ci/ci_ml_thirdparty_depset_py$PYTHON.lock
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON
- RAYCI_IS_GPU_BUILD
- PYTHON
- IMAGE_TYPE
tags:
- cr.ray.io/rayproject/ml$IMAGE_TYPE-py$PYTHON
+20
View File
@@ -0,0 +1,20 @@
# Ray Anyscale CPU Image (for release tests)
# Installs ray wheel into the CPU base-extra-testdeps image, which includes
# additional test dependencies.
#
# This produces the anyscale test image for CPU-only release tests.
#
name: "ray-anyscale-py$PYTHON_VERSION-cpu$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base-extra-testdeps" # CPU base with test deps
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- IMAGE_TYPE=ray
- PLATFORM=cpu
- BASE_VARIANT=base-extra-testdeps
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
+20
View File
@@ -0,0 +1,20 @@
# Ray Anyscale CUDA Image (for release tests)
# Installs ray wheel into the CUDA base-extra-testdeps image, which includes
# additional test dependencies.
#
# This produces the anyscale test image for GPU release tests.
#
name: "ray-anyscale-py$PYTHON_VERSION-$GPU$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-$GPU-base-extra-testdeps" # CUDA base with test deps
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- IMAGE_TYPE=ray
- BASE_VARIANT=base-extra-testdeps
- PLATFORM=$GPU
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
+72
View File
@@ -0,0 +1,72 @@
# syntax=docker/dockerfile:1.3-labs
ARG ARCH_SUFFIX=
ARG HOSTTYPE=x86_64
ARG MANYLINUX_VERSION
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
ARG PYTHON_VERSION=3.10
ARG BUILDKITE_BAZEL_CACHE_URL
ARG BUILDKITE_CACHE_READONLY
ARG HOSTTYPE
ARG IS_LOCAL_BUILD=false
ARG CACHE_DIR=/opt/cache
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
ENV BUILDKITE_CACHE_READONLY=${BUILDKITE_CACHE_READONLY}
ENV IS_LOCAL_BUILD=${IS_LOCAL_BUILD}
ENV CACHE_DIR=${CACHE_DIR}
ENV DOWNLOAD_CACHE=${CACHE_DIR}/downloads
ENV BAZEL_CACHE=${CACHE_DIR}/bazel
WORKDIR /home/forge/ray
COPY --chown=forge . .
RUN --mount=type=cache,target=${DOWNLOAD_CACHE},uid=2000,gid=100,id=ray-downloads-${HOSTTYPE} \
--mount=type=cache,target=${BAZEL_CACHE},uid=2000,gid=100,id=ray-bazel-${HOSTTYPE}-py${PYTHON_VERSION} \
<<'EOF'
#!/bin/bash
set -euo pipefail
export BAZELISK_HOME=$DOWNLOAD_CACHE/bazelisk
REPOSITORY_CACHE=$DOWNLOAD_CACHE/repo
PY_CODE="${PYTHON_VERSION//./}"
PY_BIN="cp${PY_CODE}-cp${PY_CODE}"
export RAY_BUILD_ENV="manylinux_py${PY_BIN}"
sudo ln -sf "/opt/python/${PY_BIN}/bin/python3" /usr/local/bin/python3
sudo ln -sf /usr/local/bin/python3 /usr/local/bin/python
BAZEL_CACHE_ARGS=""
if [[ -z "${BUILDKITE_BAZEL_CACHE_URL:-}" ]]; then
# Disable remote cache for local builds (no credentials)
BAZEL_CACHE_ARGS="--remote_cache="
else
# Override whatever the base image baked into ~/.bazelrc
BAZEL_CACHE_ARGS="--remote_cache=${BUILDKITE_BAZEL_CACHE_URL}"
if [[ "${BUILDKITE_CACHE_READONLY:-}" == "true" ]]; then
BAZEL_CACHE_ARGS="$BAZEL_CACHE_ARGS --remote_upload_local_results=false"
fi
fi
BAZEL_RESOURCE_FLAGS=""
if [[ "$IS_LOCAL_BUILD" == "true" ]]; then
BAZEL_RESOURCE_FLAGS=$(python3 "$HOME/ray/ci/build/container_resource_utils.py")
fi
bazelisk --output_base=$BAZEL_CACHE build --config=ci \
--repository_cache=$REPOSITORY_CACHE \
$BAZEL_CACHE_ARGS \
$BAZEL_RESOURCE_FLAGS \
//:ray_pkg_zip //:ray_py_proto_zip
cp bazel-bin/ray_pkg.zip /home/forge/ray_pkg.zip
cp bazel-bin/ray_py_proto.zip /home/forge/ray_py_proto.zip
EOF
FROM scratch
COPY --from=builder /home/forge/ray_pkg.zip /home/forge/ray_py_proto.zip /
+31
View File
@@ -0,0 +1,31 @@
name: "ray-core-py$PYTHON_VERSION$ARCH_SUFFIX"
froms: ["rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"]
dockerfile: ci/docker/ray-core.Dockerfile
srcs:
- .bazelversion
- .bazelrc
- WORKSPACE
- BUILD.bazel
- bazel/
- thirdparty/
- src/
- gen_ray_pkg.py
- python/ray/__init__.py
- python/ray/_raylet.pxd
- python/ray/_raylet.pyi
- python/ray/_raylet.pyx
- python/ray/includes/
- java/BUILD.bazel
- java/dependencies.bzl
- release/BUILD.bazel
- release/requirements_py310.txt
- ci/build/container_resource_utils.py
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- BUILDKITE_BAZEL_CACHE_URL
- HOSTTYPE
- MANYLINUX_VERSION
build_hint_args:
- BUILDKITE_CACHE_READONLY
- IS_LOCAL_BUILD
+67
View File
@@ -0,0 +1,67 @@
# syntax=docker/dockerfile:1.3-labs
#
# Ray C++ Core Artifacts Builder
# ==============================
# Builds ray_cpp_pkg.zip containing C++ headers, libraries, and examples.
#
ARG ARCH_SUFFIX=
ARG HOSTTYPE=x86_64
ARG MANYLINUX_VERSION
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
ARG PYTHON_VERSION=3.10
ARG BUILDKITE_BAZEL_CACHE_URL
ARG BUILDKITE_CACHE_READONLY
ARG HOSTTYPE
ARG CACHE_DIR=/opt/cache
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
ENV BUILDKITE_CACHE_READONLY=${BUILDKITE_CACHE_READONLY}
ENV CACHE_DIR=${CACHE_DIR}
ENV DOWNLOAD_CACHE=${CACHE_DIR}/downloads
ENV BAZEL_CACHE=${CACHE_DIR}/bazel
WORKDIR /home/forge/ray
COPY . .
RUN --mount=type=cache,target=${DOWNLOAD_CACHE},uid=2000,gid=100,id=ray-downloads-${HOSTTYPE} \
--mount=type=cache,target=${BAZEL_CACHE},uid=2000,gid=100,id=ray-bazel-${HOSTTYPE} \
<<'EOF'
#!/bin/bash
set -euo pipefail
export BAZELISK_HOME=$DOWNLOAD_CACHE/bazelisk
REPOSITORY_CACHE=$DOWNLOAD_CACHE/repo
PY_CODE="${PYTHON_VERSION//./}"
PY_BIN="cp${PY_CODE}-cp${PY_CODE}"
export RAY_BUILD_ENV="manylinux_py${PY_BIN}"
sudo ln -sf "/opt/python/${PY_BIN}/bin/python3" /usr/local/bin/python3
sudo ln -sf /usr/local/bin/python3 /usr/local/bin/python
BAZEL_CACHE_ARGS=""
if [[ -z "${BUILDKITE_BAZEL_CACHE_URL:-}" ]]; then
# Disable remote cache for local builds (no credentials)
BAZEL_CACHE_ARGS="--remote_cache="
else
# Override whatever the base image baked into ~/.bazelrc
BAZEL_CACHE_ARGS="--remote_cache=${BUILDKITE_BAZEL_CACHE_URL}"
if [[ "${BUILDKITE_CACHE_READONLY:-}" == "true" ]]; then
BAZEL_CACHE_ARGS="$BAZEL_CACHE_ARGS --remote_upload_local_results=false"
fi
fi
bazelisk --output_base=$BAZEL_CACHE build --config=ci \
--repository_cache=$REPOSITORY_CACHE \
$BAZEL_CACHE_ARGS \
//cpp:ray_cpp_pkg_zip
cp bazel-bin/cpp/ray_cpp_pkg.zip /home/forge/ray_cpp_pkg.zip
EOF
FROM scratch
COPY --from=builder /home/forge/ray_cpp_pkg.zip /
+29
View File
@@ -0,0 +1,29 @@
name: "ray-cpp-core$ARCH_SUFFIX"
froms: ["rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"]
dockerfile: ci/docker/ray-cpp-core.Dockerfile
srcs:
- .bazelversion
- .bazelrc
- WORKSPACE
- BUILD.bazel
- bazel/
- thirdparty/
- src/
- cpp/
- python/ray/__init__.py
- python/ray/_raylet.pxd
- python/ray/_raylet.pyi
- python/ray/_raylet.pyx
- python/ray/includes/
- java/BUILD.bazel
- java/dependencies.bzl
- release/BUILD.bazel
- release/requirements.txt
- release/requirements_py310.txt
build_args:
- ARCH_SUFFIX
- HOSTTYPE
- MANYLINUX_VERSION
- BUILDKITE_BAZEL_CACHE_URL
build_hint_args:
- BUILDKITE_CACHE_READONLY
+89
View File
@@ -0,0 +1,89 @@
# syntax=docker/dockerfile:1.3-labs
#
# Ray C++ Wheel Builder
# =====================
# Builds manylinux2014-compatible ray-cpp wheel using pre-built C++ artifacts from wanda cache.
#
# This is a minimal Dockerfile for ray-cpp wheel builds only.
# It copies only the files needed for the cpp wheel, reducing build context size.
ARG RAY_CORE_IMAGE
ARG RAY_CPP_CORE_IMAGE
ARG RAY_JAVA_IMAGE
ARG RAY_DASHBOARD_IMAGE
ARG MANYLINUX_VERSION
ARG HOSTTYPE
FROM ${RAY_CORE_IMAGE} AS ray-core
FROM ${RAY_CPP_CORE_IMAGE} AS ray-cpp-core
FROM ${RAY_JAVA_IMAGE} AS ray-java
FROM ${RAY_DASHBOARD_IMAGE} AS ray-dashboard
# Main build stage - manylinux2014 provides GLIBC 2.17
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
ARG BUILDKITE_COMMIT
WORKDIR /home/forge/ray
# Copy artifacts from all stages
COPY --from=ray-core /ray_pkg.zip /tmp/
COPY --from=ray-core /ray_py_proto.zip /tmp/
COPY --from=ray-java /ray_java_pkg.zip /tmp/
COPY --from=ray-dashboard /dashboard.tar.gz /tmp/
# Minimal source files needed for cpp wheel build
COPY --chown=forge ci/build/build-ray-cpp-wheel.sh ci/build/
COPY --chown=forge README.rst pyproject.toml ./
COPY --chown=forge python/setup.py python/
COPY --chown=forge python/LICENSE.txt python/
COPY --chown=forge python/MANIFEST.in python/
COPY --chown=forge python/ray/_version.py python/ray/
USER forge
ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT:-unknown}
RUN --mount=from=ray-cpp-core,source=/,target=/ray-cpp-core,ro \
<<'EOF'
#!/bin/bash
set -euo pipefail
# Clean extraction dirs to avoid stale leftovers
rm -rf /tmp/ray_pkg /tmp/ray_java_pkg /tmp/ray_cpp_pkg
mkdir -p /tmp/ray_pkg /tmp/ray_java_pkg /tmp/ray_cpp_pkg
# Unpack pre-built artifacts
unzip -o /tmp/ray_pkg.zip -d /tmp/ray_pkg
unzip -o /tmp/ray_py_proto.zip -d python/
unzip -o /tmp/ray_java_pkg.zip -d /tmp/ray_java_pkg
mkdir -p python/ray/dashboard/client/build
tar -xzf /tmp/dashboard.tar.gz -C python/ray/dashboard/client/build/
# C++ core artifacts
cp -r /tmp/ray_pkg/ray/* python/ray/
# Java JARs
cp -r /tmp/ray_java_pkg/ray/* python/ray/
# C++ API artifacts (headers, libs, examples)
unzip -o /ray-cpp-core/ray_cpp_pkg.zip -d /tmp/ray_cpp_pkg
cp -r /tmp/ray_cpp_pkg/ray/cpp python/ray/
# Build Python-agnostic cpp wheel
./ci/build/build-ray-cpp-wheel.sh
# Sanity check: ensure wheels exist
if [[ ! -d .whl ]]; then
echo "ERROR: .whl directory not created"
exit 1
fi
wheels=($(find .whl -maxdepth 1 -name '*.whl'))
if (( ${#wheels[@]} == 0 )); then
echo "ERROR: No wheels produced in .whl/"
ls -la .whl
exit 1
fi
EOF
FROM scratch
COPY --from=builder /home/forge/ray/.whl/*.whl /opt/artifacts/
+31
View File
@@ -0,0 +1,31 @@
# Python-agnostic ray-cpp wheel.
# Uses upstream images from one Python version but produces a py3-none wheel
# that works with any Python 3. PYTHON_VERSION is set at buildkite level.
name: "ray-cpp-wheel$ARCH_SUFFIX"
froms:
- "rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"
- "cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX" # C++ binaries (ray_pkg.zip)
- "cr.ray.io/rayproject/ray-cpp-core$ARCH_SUFFIX" # C++ headers/libs (ray_cpp_pkg.zip)
- "cr.ray.io/rayproject/ray-java-build$ARCH_SUFFIX" # Java JARs
- "cr.ray.io/rayproject/ray-dashboard$ARCH_SUFFIX" # Dashboard
dockerfile: ci/docker/ray-cpp-wheel.Dockerfile
srcs:
- pyproject.toml
- README.rst
- ci/build/build-ray-cpp-wheel.sh
- python/setup.py
- python/LICENSE.txt
- python/MANIFEST.in
- python/ray/_version.py
build_args:
- MANYLINUX_VERSION
- HOSTTYPE
- BUILDKITE_COMMIT
- ARCH_SUFFIX
- RAY_CORE_IMAGE=cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX
- RAY_CPP_CORE_IMAGE=cr.ray.io/rayproject/ray-cpp-core$ARCH_SUFFIX
- RAY_JAVA_IMAGE=cr.ray.io/rayproject/ray-java-build$ARCH_SUFFIX
- RAY_DASHBOARD_IMAGE=cr.ray.io/rayproject/ray-dashboard$ARCH_SUFFIX
artifacts:
- src: "/opt/artifacts/"
dst: "./"
+32
View File
@@ -0,0 +1,32 @@
ARG HOSTTYPE=x86_64
ARG MANYLINUX_VERSION
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
WORKDIR /home/forge/ray
COPY --chown=forge:users . .
RUN <<EOF
#!/bin/bash
set -euo pipefail
export PATH="/usr/local/node/bin:$PATH"
(
cd python/ray/dashboard/client
echo "--- npm ci"
npm ci
echo "--- npm run build"
npm run build
tar -czf /home/forge/dashboard.tar.gz -C build .
)
EOF
FROM scratch
COPY --from=builder /home/forge/dashboard.tar.gz /dashboard.tar.gz
+8
View File
@@ -0,0 +1,8 @@
name: ray-dashboard$ARCH_SUFFIX
froms: ["rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"]
dockerfile: ci/docker/ray-dashboard.Dockerfile
srcs:
- python/ray/dashboard/client/
build_args:
- HOSTTYPE
- MANYLINUX_VERSION
+14
View File
@@ -0,0 +1,14 @@
name: "ray-extra-py$PYTHON_VERSION-cpu$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base-extra$ARCH_SUFFIX" # CPU base-extra image
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- IMAGE_TYPE=ray
- PLATFORM=cpu
- BASE_VARIANT=base-extra
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
+14
View File
@@ -0,0 +1,14 @@
name: "ray-extra-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra$ARCH_SUFFIX" # CUDA base-extra image
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- IMAGE_TYPE=ray
- PLATFORM=cu$CUDA_VERSION
- BASE_VARIANT=base-extra
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
+12
View File
@@ -0,0 +1,12 @@
name: "ray-py$PYTHON_VERSION-cpu$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX" # CPU base image with Python + deps
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- PLATFORM=cpu
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
+12
View File
@@ -0,0 +1,12 @@
name: "ray-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX" # CUDA base image
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- PLATFORM=cu$CUDA_VERSION
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
+12
View File
@@ -0,0 +1,12 @@
name: "ray-py$PYTHON_VERSION-tpu$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-tpu-base$ARCH_SUFFIX" # TPU base image with Python + deps
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- PLATFORM=tpu
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
+77
View File
@@ -0,0 +1,77 @@
# syntax=docker/dockerfile:1.3-labs
#
# Ray Image Builder
# ==============================
# Installs the Ray wheel into a base image (CPU or CUDA), includes
# pip freeze output for reproducibility.
#
ARG PYTHON_VERSION=3.10
ARG PLATFORM=cpu
ARG ARCH_SUFFIX=
ARG IMAGE_TYPE=ray
ARG BASE_VARIANT=base
ARG BASE_IMAGE=cr.ray.io/rayproject/${IMAGE_TYPE}-py${PYTHON_VERSION}-${PLATFORM}-${BASE_VARIANT}${ARCH_SUFFIX}
ARG RAY_WHEEL_IMAGE=cr.ray.io/rayproject/ray-wheel-py${PYTHON_VERSION}${ARCH_SUFFIX}
FROM ${RAY_WHEEL_IMAGE} AS wheel-source
FROM ${BASE_IMAGE}
ARG IMAGE_TYPE=ray
ARG PLATFORM=cpu
ARG RAY_COMMIT=unknown-commit
ARG RAY_VERSION=3.0.0.dev0
LABEL io.ray.ray-commit="${RAY_COMMIT}"
LABEL io.ray.ray-version="${RAY_VERSION}"
COPY --from=wheel-source /opt/artifacts/*.whl /home/ray/
# Install Ray wheel with all extras
# Uses requirements_compiled.txt from base image (already at /home/ray/)
RUN <<EOF
#!/bin/bash
set -euo pipefail
WHEEL_FILES=(/home/ray/ray-*.whl)
if [[ ${#WHEEL_FILES[@]} -ne 1 ]]; then
echo "Error: Expected 1 ray wheel file, but found ${#WHEEL_FILES[@]} in /home/ray/." >&2
ls -l /home/ray/*.whl >&2
exit 1
fi
WHEEL_FILE="${WHEEL_FILES[0]}"
echo "Installing wheel: $WHEEL_FILE"
if [[ "${IMAGE_TYPE}" == "ray-llm" ]]; then
RAY_EXTRAS="default,data,serve"
else
RAY_EXTRAS="all"
fi
# TODO(cu130): ray[all]'s cgraph extra hard-pins cupy-cuda12x, so this install
# always pulls the CUDA-12 build even on cu130 images (no PEP 508 marker exists
# to select cupy by CUDA version). Until the cgraph extra can resolve cupy per
# CUDA runtime (or cupy ships a unified package), we patch it up with the
# uninstall/reinstall swap below. Drop that swap once this install can pick the
# right cupy directly.
$HOME/anaconda3/bin/pip --no-cache-dir install \
-c /home/ray/requirements_compiled.txt \
"${WHEEL_FILE}[${RAY_EXTRAS}]"
# ray[all]'s cgraph extra hard-pins cupy-cuda12x (a CUDA-12 build), but cu130
# images ship a CUDA-13 runtime where that build is broken. Swap it for the
# matching CUDA-13 build. cupy-cuda12x and cupy-cuda13x both own the top-level
# `cupy` package and cannot coexist, so this is an uninstall-then-install.
# Scoped to IMAGE_TYPE=ray (covers ray + ray-extra); ray-llm flows through this
# same Dockerfile but manages cupy via its own llm locks, so leave it untouched.
if [[ "${IMAGE_TYPE}" == "ray" && "${PLATFORM}" == cu13* ]]; then
$HOME/anaconda3/bin/pip --no-cache-dir uninstall -y cupy-cuda12x
$HOME/anaconda3/bin/pip --no-cache-dir install "cupy-cuda13x==13.6.0"
fi
$HOME/anaconda3/bin/pip freeze > /home/ray/pip-freeze.txt
echo "Ray version: $($HOME/anaconda3/bin/python -c 'import ray; print(ray.__version__)')"
EOF
CMD ["/bin/bash"]
+65
View File
@@ -0,0 +1,65 @@
# syntax=docker/dockerfile:1.3-labs
ARG ARCH_SUFFIX
ARG HOSTTYPE=x86_64
ARG MANYLINUX_VERSION
FROM rayproject/manylinux2014:${MANYLINUX_VERSION}-jdk-${HOSTTYPE} AS builder
ARG BUILDKITE_BAZEL_CACHE_URL
ARG BUILDKITE_CACHE_READONLY
ARG HOSTTYPE
ARG IS_LOCAL_BUILD=false
ARG CACHE_DIR=/opt/cache
ENV BUILDKITE_BAZEL_CACHE_URL=${BUILDKITE_BAZEL_CACHE_URL}
ENV BUILDKITE_CACHE_READONLY=${BUILDKITE_CACHE_READONLY}
ENV IS_LOCAL_BUILD=${IS_LOCAL_BUILD}
ENV CACHE_DIR=${CACHE_DIR}
ENV DOWNLOAD_CACHE=${CACHE_DIR}/downloads
ENV BAZEL_CACHE=${CACHE_DIR}/bazel
WORKDIR /home/forge/ray
COPY --chown=forge . .
RUN --mount=type=cache,target=${DOWNLOAD_CACHE},uid=2000,gid=100,id=ray-downloads-${HOSTTYPE} \
--mount=type=cache,target=${BAZEL_CACHE},uid=2000,gid=100,id=ray-bazel-${HOSTTYPE} \
<<'EOF'
#!/bin/bash
set -euo pipefail
export BAZELISK_HOME=$DOWNLOAD_CACHE/bazelisk
export COURSIER_CACHE=$DOWNLOAD_CACHE/coursier
REPOSITORY_CACHE=$DOWNLOAD_CACHE/repo
export RAY_BUILD_ENV="manylinux"
BAZEL_CACHE_ARGS=""
if [[ -z "${BUILDKITE_BAZEL_CACHE_URL:-}" ]]; then
# Disable remote cache for local builds (no credentials)
BAZEL_CACHE_ARGS="--remote_cache="
else
# Override whatever the base image baked into ~/.bazelrc
BAZEL_CACHE_ARGS="--remote_cache=${BUILDKITE_BAZEL_CACHE_URL}"
if [[ "${BUILDKITE_CACHE_READONLY:-}" == "true" ]]; then
BAZEL_CACHE_ARGS="$BAZEL_CACHE_ARGS --remote_upload_local_results=false"
fi
fi
BAZEL_RESOURCE_FLAGS=""
if [[ "$IS_LOCAL_BUILD" == "true" ]]; then
BAZEL_RESOURCE_FLAGS=$(python3 "$HOME/ray/ci/build/container_resource_utils.py")
fi
bazelisk --output_base=$BAZEL_CACHE run --config=ci \
--repository_cache=$REPOSITORY_CACHE \
$BAZEL_CACHE_ARGS \
$BAZEL_RESOURCE_FLAGS \
//java:gen_ray_java_pkg
cp bazel-bin/java/ray_java_pkg.zip /home/forge/ray_java_pkg.zip
EOF
FROM scratch
COPY --from=builder /home/forge/ray_java_pkg.zip /
+24
View File
@@ -0,0 +1,24 @@
name: ray-java-build$ARCH_SUFFIX
froms: ["rayproject/manylinux2014:$MANYLINUX_VERSION-jdk-$HOSTTYPE"]
dockerfile: ci/docker/ray-java.Dockerfile
srcs:
- .bazelversion
- .bazelrc
- WORKSPACE
- BUILD.bazel
- bazel/
- thirdparty/
- src/
- java/
- gen_ray_pkg.py
- release/BUILD.bazel
- release/requirements_py310.txt
- ci/build/container_resource_utils.py
build_args:
- ARCH_SUFFIX
- BUILDKITE_BAZEL_CACHE_URL
- HOSTTYPE
- MANYLINUX_VERSION
build_hint_args:
- BUILDKITE_CACHE_READONLY
- IS_LOCAL_BUILD
@@ -0,0 +1,20 @@
# Ray-LLM Anyscale CUDA Image (for release tests)
# Installs ray wheel into the ray-llm CUDA base-extra-testdeps image, which
# includes extra test dependencies.
#
# This produces the anyscale test image for ray-llm GPU release tests.
#
name: "ray-llm-anyscale-py$PYTHON_VERSION-$GPU$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-llm-py$PYTHON_VERSION-$GPU-base-extra-testdeps" # ray-llm base with test deps
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- BASE_VARIANT=base-extra-testdeps
- IMAGE_TYPE=ray-llm
- PLATFORM=$GPU
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
@@ -0,0 +1,14 @@
name: "ray-llm-extra-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-llm-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra$ARCH_SUFFIX" # LLM base-extra image
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- IMAGE_TYPE=ray-llm
- PLATFORM=cu$CUDA_VERSION
- BASE_VARIANT=base-extra
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
+13
View File
@@ -0,0 +1,13 @@
name: "ray-llm-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-llm-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX" # LLM base image
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- IMAGE_TYPE=ray-llm
- PLATFORM=cu$CUDA_VERSION
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
+20
View File
@@ -0,0 +1,20 @@
# Ray-ML Anyscale CUDA Image (for release tests)
# Installs ray wheel into the ray-ml CUDA base-extra-testdeps image, which includes
# extra test dependencies.
#
# This produces the anyscale test image for ray-ml GPU release tests.
#
name: "ray-ml-anyscale-py$PYTHON_VERSION-$GPU$ARCH_SUFFIX"
disable_caching: true
froms:
- "cr.ray.io/rayproject/ray-ml-py$PYTHON_VERSION-$GPU-base-extra-testdeps" # ray-ml base with test deps
- "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX" # Ray wheel
dockerfile: ci/docker/ray-image.Dockerfile
build_args:
- PYTHON_VERSION
- ARCH_SUFFIX
- BASE_VARIANT=base-extra-testdeps
- IMAGE_TYPE=ray-ml
- PLATFORM=$GPU
- RAY_COMMIT=$BUILDKITE_COMMIT
- RAY_VERSION
+95
View File
@@ -0,0 +1,95 @@
# syntax=docker/dockerfile:1.3-labs
#
# Ray Wheel Builder
# =================
# Builds manylinux2014-compatible ray wheel using pre-built C++ artifacts from wanda cache.
#
# GLIBC Compatibility:
# --------------------
# manylinux2014 requires GLIBC <= 2.17 for broad Linux compatibility.
# The pre-built _raylet.so is compiled inside manylinux2014 with GLIBC 2.17.
#
ARG RAY_CORE_IMAGE
ARG RAY_JAVA_IMAGE=scratch
ARG RAY_DASHBOARD_IMAGE=scratch
ARG MANYLINUX_IMAGE
FROM ${RAY_CORE_IMAGE} AS ray-core
FROM ${RAY_JAVA_IMAGE} AS ray-java
FROM ${RAY_DASHBOARD_IMAGE} AS ray-dashboard
# Main build stage - manylinux2014 provides GLIBC 2.17
FROM ${MANYLINUX_IMAGE} AS builder
ARG PYTHON_VERSION=3.10
ARG BUILDKITE_COMMIT
WORKDIR /home/forge/ray
# Copy artifacts from all stages
COPY --from=ray-core /ray_pkg.zip /tmp/
COPY --from=ray-core /ray_py_proto.zip /tmp/
# Source files needed for wheel build
COPY --chown=forge ci/build/build-manylinux-wheel.sh ci/build/
COPY --chown=forge README.rst pyproject.toml ./
COPY --chown=forge rllib/ rllib/
COPY --chown=forge python/ python/
USER forge
# - BUILDKITE_COMMIT: Used for ray.__commit__. Defaults to "unknown" for local builds.
ENV PYTHON_VERSION=${PYTHON_VERSION} \
BUILDKITE_COMMIT=${BUILDKITE_COMMIT:-unknown}
RUN --mount=from=ray-java,target=/mnt/java \
--mount=from=ray-dashboard,target=/mnt/dashboard \
<<'EOF'
#!/bin/bash
set -euo pipefail
# Clean extraction dirs to avoid stale leftovers
rm -rf /tmp/ray_pkg
mkdir -p /tmp/ray_pkg
# Unpack pre-built artifacts
unzip -o /tmp/ray_pkg.zip -d /tmp/ray_pkg
unzip -o /tmp/ray_py_proto.zip -d python/
# Dashboard (optional)
if [[ -f /mnt/dashboard/dashboard.tar.gz ]]; then
mkdir -p python/ray/dashboard/client/build
tar -xzf /mnt/dashboard/dashboard.tar.gz -C python/ray/dashboard/client/build/
fi
# C++ core artifacts
cp -r /tmp/ray_pkg/ray/* python/ray/
# Java JARs (optional)
if [[ -f /mnt/java/ray_java_pkg.zip ]]; then
mkdir -p /tmp/ray_java_pkg
unzip -o /mnt/java/ray_java_pkg.zip -d /tmp/ray_java_pkg
cp -r /tmp/ray_java_pkg/ray/* python/ray/
fi
# Build ray wheel
PY_VERSION="${PYTHON_VERSION//./}"
PY_BIN="cp${PY_VERSION}-cp${PY_VERSION}"
SKIP_BAZEL_BUILD=1 RAY_DISABLE_EXTRA_CPP=1 \
./ci/build/build-manylinux-wheel.sh "$PY_BIN"
# Sanity check: ensure wheels exist
if [[ ! -d .whl ]]; then
echo "ERROR: .whl directory not created"
exit 1
fi
wheels=($(find .whl -maxdepth 1 -name '*.whl'))
if (( ${#wheels[@]} == 0 )); then
echo "ERROR: No wheels produced in .whl/"
ls -la .whl
exit 1
fi
EOF
FROM scratch
COPY --from=builder /home/forge/ray/.whl/*.whl /opt/artifacts/
+26
View File
@@ -0,0 +1,26 @@
name: "ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX"
disable_caching: true
froms:
- "rayproject/manylinux2014:$MANYLINUX_VERSION$JDK_SUFFIX-$HOSTTYPE"
- "cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX" # C++ binaries (ray_pkg.zip)
- "$RAY_JAVA_IMAGE" # Java JARs (scratch to skip)
- "$RAY_DASHBOARD_IMAGE" # Dashboard (scratch to skip)
dockerfile: ci/docker/ray-wheel.Dockerfile
srcs:
- pyproject.toml
- README.rst
- ci/build/build-manylinux-wheel.sh
- python/
- rllib/
build_args:
- PYTHON_VERSION
- BUILDKITE_COMMIT
- ARCH_SUFFIX
- WHEEL_TYPE=ray
- MANYLINUX_IMAGE=rayproject/manylinux2014:$MANYLINUX_VERSION$JDK_SUFFIX-$HOSTTYPE
- RAY_CORE_IMAGE=cr.ray.io/rayproject/ray-core-py$PYTHON_VERSION$ARCH_SUFFIX
- RAY_JAVA_IMAGE
- RAY_DASHBOARD_IMAGE
artifacts:
- src: "/opt/artifacts/"
dst: "./"
+30
View File
@@ -0,0 +1,30 @@
# syntax=docker/dockerfile:1.3-labs
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_ml-py3.10
FROM $DOCKER_IMAGE_BASE_BUILD
ARG RAYCI_IS_GPU_BUILD
ARG PYTHON
ARG BUILD_VARIANT=build
ARG PYTHON_DEPSET=python/deplocks/ci/rllib-$BUILD_VARIANT-ci_depset_py$PYTHON.lock
SHELL ["/bin/bash", "-ice"]
COPY . .
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
RUN <<EOF
#!/bin/bash
set -euo pipefail
# Install system packages required for gymnasium rendering (MuJoCo, OpenGL, GLFW)
sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends \
libosmesa6-dev libgl1 libglfw3 patchelf
rm -rf /var/lib/apt/lists/*
# Install Python dependencies from depset lock file
uv pip install -r /home/ray/python_depset.lock --no-deps --system --index-strategy unsafe-best-match
EOF
+11
View File
@@ -0,0 +1,11 @@
name: "rllib$BUILD_VARIANT-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON"]
dockerfile: ci/docker/rllib.build.Dockerfile
srcs:
- python/deplocks/ci/rllib-$BUILD_VARIANT-ci_depset_py$PYTHON.lock
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_$BASE_TYPE-py$PYTHON
- RAYCI_IS_GPU_BUILD
- BUILD_VARIANT
tags:
- cr.ray.io/rayproject/rllib$BUILD_VARIANT-py$PYTHON
@@ -0,0 +1,17 @@
ARG BASE_IMAGE
FROM $BASE_IMAGE
COPY python/ray/tests/runtime_env_container/ /home/ray/tests/
RUN pip install --no-cache-dir -c /home/ray/requirements_compiled.txt httpx
# Podman for runtime tests; docker-ce-cli so `docker save | podman load` can talk to
# the mounted host socket (stock podman's docker-daemon: pull uses an old API).
RUN sudo apt-get update -qq \
&& sudo apt-get install -y -qq ca-certificates curl gnupg \
&& sudo mkdir -p /etc/apt/keyrings \
&& curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
&& sudo chmod a+r /etc/apt/keyrings/docker.gpg \
&& echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu jammy stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
&& sudo apt-get update \
&& sudo apt-get install -y docker-ce-cli podman
+31
View File
@@ -0,0 +1,31 @@
# syntax=docker/dockerfile:1.3-labs
ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py3.10
FROM $DOCKER_IMAGE_BASE_BUILD
ARG ENABLE_TRACING
ARG PYDANTIC_VERSION
ARG IMAGE_TYPE="base"
ARG PYTHON
ARG PYTHON_DEPSET="python/deplocks/ci/serve_${IMAGE_TYPE}_depset_py${PYTHON}.lock"
COPY "$PYTHON_DEPSET" /home/ray/python_depset.lock
SHELL ["/bin/bash", "-ice"]
COPY . .
RUN <<EOF
#!/bin/bash
set -euo pipefail
uv pip install --system --no-cache-dir --no-deps --index-strategy unsafe-best-match \
-r /home/ray/python_depset.lock
git clone --branch=4.2.0 --depth=1 https://github.com/wg/wrk.git /tmp/wrk
make -C /tmp/wrk -j
sudo cp /tmp/wrk/wrk /usr/local/bin/wrk
rm -rf /tmp/wrk
EOF
+11
View File
@@ -0,0 +1,11 @@
name: "servebuild-py$PYTHON"
froms: ["cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON"]
dockerfile: ci/docker/serve.build.Dockerfile
srcs:
- python/deplocks/ci/serve_ci_depset_py$PYTHON.lock
build_args:
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py$PYTHON
- PYTHON=$PYTHON
- IMAGE_TYPE=ci
tags:
- cr.ray.io/rayproject/servebuild-py$PYTHON
+12
View File
@@ -0,0 +1,12 @@
name: "servetracingbuild"
froms: ["cr.ray.io/rayproject/oss-ci-base_build-py3.10"]
dockerfile: ci/docker/serve.build.Dockerfile
srcs:
- python/deplocks/ci/serve_tracing_depset_py3.10.lock
build_args:
- ENABLE_TRACING=1
- DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build-py3.10
- PYTHON=3.10
- IMAGE_TYPE=tracing
tags:
- cr.ray.io/rayproject/servetracingbuild
+9
View File
@@ -0,0 +1,9 @@
ARG DOCKER_IMAGE_BASE_BUILD=rayproject/buildenv:windows
FROM $DOCKER_IMAGE_BASE_BUILD
ENV PYTHON=3.10
ENV PYTHON_FULL_VERSION=3.10.19
ENV RAY_BUILD_ENV=win_py$PYTHON_FULL_VERSION
COPY . .
RUN bash ci/ray_ci/windows/build_base.sh
+13
View File
@@ -0,0 +1,13 @@
name: "windowsbuild"
froms: ["@rayproject/buildenv:windows"]
dockerfile: ci/docker/windows.build.Dockerfile
srcs:
- python/requirements.txt
- python/requirements_compiled.txt
- python/requirements/test-requirements.txt
- python/requirements/ml/dl-cpu-requirements.txt
- ci/ray_ci/windows/build_base.sh
- ci/ray_ci/windows/install_bazelisk.ps1
- ci/ray_ci/windows/cleanup.ps1
tags:
- cr.ray.io/rayproject/windowsbuild