chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,132 @@
|
||||
# syntax=docker/dockerfile:1.3-labs
|
||||
|
||||
# The base-deps Docker image installs main libraries needed to run Ray
|
||||
|
||||
# The GPU options are NVIDIA CUDA developer images.
|
||||
ARG BASE_IMAGE="ubuntu:22.04"
|
||||
|
||||
FROM ${BASE_IMAGE}
|
||||
# If this arg is not "autoscaler" then no autoscaler requirements will be included
|
||||
ENV TZ=America/Los_Angeles
|
||||
ENV LC_ALL=C.UTF-8
|
||||
ENV LANG=C.UTF-8
|
||||
|
||||
# TODO(ilr) $HOME seems to point to result in "" instead of "/home/ray"
|
||||
# Q: Why add paths like /usr/local/nvidia/lib64 and /usr/local/nvidia/bin?
|
||||
# A: The NVIDIA GPU operator version used by GKE injects these into the container
|
||||
# after it's mounted to a pod.
|
||||
# Issue is tracked here:
|
||||
# https://github.com/GoogleCloudPlatform/compute-gpu-installation/issues/46
|
||||
# More context here:
|
||||
# https://github.com/NVIDIA/nvidia-container-toolkit/issues/275
|
||||
# and here:
|
||||
# https://gitlab.com/nvidia/container-images/cuda/-/issues/27
|
||||
ENV PATH "/home/ray/anaconda3/bin:$PATH:/usr/local/nvidia/bin"
|
||||
ENV LD_LIBRARY_PATH "$LD_LIBRARY_PATH:/usr/local/nvidia/lib64"
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
ARG PYTHON_VERSION=3.10
|
||||
ARG CONSTRAINTS_FILE="python/requirements_compiled_py${PYTHON_VERSION}.txt"
|
||||
ARG PYTHON_DEPSET="python/deplocks/base_deps/ray_base_deps_py${PYTHON_VERSION}.lock"
|
||||
|
||||
ARG RAY_UID=1000
|
||||
ARG RAY_GID=100
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
apt-get update -y
|
||||
apt-get upgrade -y
|
||||
|
||||
APT_PKGS=(
|
||||
sudo
|
||||
tzdata
|
||||
git
|
||||
libjemalloc-dev
|
||||
wget
|
||||
cmake
|
||||
g++
|
||||
zlib1g-dev
|
||||
|
||||
# For autoscaler
|
||||
tmux
|
||||
screen
|
||||
rsync
|
||||
netbase
|
||||
openssh-client
|
||||
gnupg
|
||||
)
|
||||
|
||||
apt-get install -y "${APT_PKGS[@]}"
|
||||
|
||||
useradd -ms /bin/bash -d /home/ray ray --uid $RAY_UID --gid $RAY_GID
|
||||
usermod -aG sudo ray
|
||||
echo 'ray ALL=NOPASSWD: ALL' >> /etc/sudoers
|
||||
|
||||
EOF
|
||||
|
||||
USER $RAY_UID
|
||||
ENV HOME=/home/ray
|
||||
WORKDIR /home/ray
|
||||
|
||||
COPY --chown=ray "$CONSTRAINTS_FILE" /home/ray/requirements_compiled.txt
|
||||
COPY --chown=ray "$PYTHON_DEPSET" /home/ray/python_depset.lock
|
||||
|
||||
SHELL ["/bin/bash", "-c"]
|
||||
|
||||
RUN <<EOF
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Determine the architecture of the host
|
||||
if [[ "${HOSTTYPE}" =~ ^x86_64 ]]; then
|
||||
ARCH="x86_64"
|
||||
elif [[ "${HOSTTYPE}" =~ ^aarch64 ]]; then
|
||||
ARCH="aarch64"
|
||||
else
|
||||
echo "Unsupported architecture ${HOSTTYPE}" >/dev/stderr
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install miniforge
|
||||
wget --quiet \
|
||||
"https://github.com/conda-forge/miniforge/releases/download/24.11.3-0/Miniforge3-24.11.3-0-Linux-${ARCH}.sh" \
|
||||
-O /tmp/miniforge.sh
|
||||
|
||||
/bin/bash /tmp/miniforge.sh -b -u -p $HOME/anaconda3
|
||||
|
||||
$HOME/anaconda3/bin/conda init
|
||||
echo 'export PATH=$HOME/anaconda3/bin:$PATH' >> $HOME/.bashrc
|
||||
rm /tmp/miniforge.sh
|
||||
$HOME/anaconda3/bin/conda install -y libgcc-ng python=$PYTHON_VERSION
|
||||
$HOME/anaconda3/bin/conda install -y -c conda-forge libffi=3.4.6
|
||||
$HOME/anaconda3/bin/conda clean -y --all
|
||||
|
||||
# Install uv
|
||||
wget -qO- https://astral.sh/uv/install.sh | sudo -E env UV_UNMANAGED_INSTALL="/usr/local/bin" sh
|
||||
|
||||
# Set up Conda as system Python
|
||||
export PATH=$HOME/anaconda3/bin:$PATH
|
||||
|
||||
# Some packages are on PyPI as well as other indices, but the latter
|
||||
# (unhelpfully) take precedence. We use `--index-strategy unsafe-best-match`
|
||||
# to ensure that the best match is chosen from the available indices.
|
||||
uv pip install --system --no-cache-dir --no-deps --index-strategy unsafe-best-match \
|
||||
-r $HOME/python_depset.lock
|
||||
|
||||
# We install cmake temporarily to get psutil
|
||||
sudo apt-get autoremove -y cmake zlib1g-dev
|
||||
|
||||
# We keep g++ on GPU images, because uninstalling removes CUDA Devel tooling
|
||||
if [[ ! -d /usr/local/cuda ]]; then
|
||||
sudo apt-get autoremove -y g++
|
||||
fi
|
||||
|
||||
sudo rm -rf /var/lib/apt/lists/*
|
||||
sudo apt-get clean
|
||||
|
||||
EOF
|
||||
|
||||
WORKDIR $HOME
|
||||
@@ -0,0 +1,42 @@
|
||||
## About
|
||||
This is an internal image, the [`rayproject/ray`](https://hub.docker.com/repository/docker/rayproject/ray) or [`rayproject/ray-ml`](https://hub.docker.com/repository/docker/rayproject/ray-ml) should be used!
|
||||
|
||||
|
||||
This image has the system-level dependencies for `Ray` and the `Ray Autoscaler`. The `ray-deps` image is built on top of this. This image is built periodically or when dependencies are added. [Find the Dockerfile here.](https://github.com/ray-project/ray/blob/master/docker/base-deps/Dockerfile)
|
||||
|
||||
## Tags
|
||||
|
||||
Images are `tagged` with the format `{Ray version}[-{Python version}][-{Platform}][-{Architecture}]`. `Ray version` tag can be one of the following:
|
||||
|
||||
| Ray version tag | Description |
|
||||
| --------------- | ----------- |
|
||||
| `latest` | The most recent Ray release. |
|
||||
| `x.y.z` | A specific Ray release, e.g. 2.9.3 |
|
||||
| `nightly` | The most recent Ray development build (a recent commit from GitHub `master`) |
|
||||
|
||||
The optional `Python version` tag specifies the Python version in the image. All Python versions supported by Ray are available, e.g. `py39`, `py310` and `py311`. If unspecified, the tag points to an image using `Python 3.9`.
|
||||
|
||||
The optional `Platform` tag specifies the platform where the image is intended for:
|
||||
|
||||
| Platform tag | Description |
|
||||
| --------------- | ----------- |
|
||||
| `-cpu` | These are based off of an Ubuntu image. |
|
||||
| `-cuXX` | These are based off of an NVIDIA CUDA image with the specified CUDA version `xx`. They require the NVIDIA Docker Runtime. |
|
||||
| `-gpu` | Aliases to a specific `-cuXX` tagged image. |
|
||||
| no tag | Aliases to `-cpu` tagged images for `ray`, and aliases to ``-gpu`` tagged images for `ray-ml`. |
|
||||
|
||||
The optional `Architecture` tag can be used to specify images for different CPU architectures.
|
||||
Currently, we support the `x86_64` (`amd64`) and `aarch64` (`arm64`) architectures.
|
||||
|
||||
Please note that suffixes are only used to specify `aarch64` images. No suffix means
|
||||
`x86_64`/`amd64`-compatible images.
|
||||
|
||||
| Platform tag | Description |
|
||||
|--------------|-------------------------|
|
||||
| `-aarch64` | arm64-compatible images |
|
||||
| no tag | Defaults to `amd64` |
|
||||
|
||||
|
||||
----
|
||||
|
||||
See [`rayproject/ray`](https://hub.docker.com/repository/docker/rayproject/ray) for Ray and all of its dependencies.
|
||||
@@ -0,0 +1,12 @@
|
||||
name: "ray-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX"
|
||||
froms: ["ubuntu:22.04"]
|
||||
dockerfile: docker/base-deps/Dockerfile
|
||||
srcs:
|
||||
- python/requirements_compiled.txt
|
||||
- python/requirements_compiled_py$PYTHON_VERSION.txt
|
||||
- python/deplocks/base_deps/ray_base_deps_py$PYTHON_VERSION.lock
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- BASE_IMAGE=ubuntu:22.04
|
||||
tags:
|
||||
- cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base$ARCH_SUFFIX
|
||||
@@ -0,0 +1,12 @@
|
||||
name: "ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX"
|
||||
froms: ["nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04"]
|
||||
dockerfile: docker/base-deps/Dockerfile
|
||||
srcs:
|
||||
- python/requirements_compiled.txt
|
||||
- python/requirements_compiled_py$PYTHON_VERSION.txt
|
||||
- python/deplocks/base_deps/ray_base_deps_py$PYTHON_VERSION.lock
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- BASE_IMAGE=nvidia/cuda:$CUDA_VERSION-devel-ubuntu22.04
|
||||
tags:
|
||||
- cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base$ARCH_SUFFIX
|
||||
@@ -0,0 +1,14 @@
|
||||
setuptools==80.9.0
|
||||
flatbuffers
|
||||
cython
|
||||
numpy # Necessary for Dataset to work properly.
|
||||
psutil
|
||||
# For the ease to submit jobs on various cloud providers.
|
||||
smart_open[s3,gcs,azure,http]
|
||||
six
|
||||
boto3
|
||||
pyopenssl
|
||||
cryptography
|
||||
google-api-python-client
|
||||
google-oauth
|
||||
adlfs[abfs]
|
||||
@@ -0,0 +1,18 @@
|
||||
# Pin to satisfy both cupy (needs <2.3) and JAX (needs >2.0)
|
||||
numpy>=2.0
|
||||
|
||||
# Minimum version for tpu7x compatibility
|
||||
jax[tpu]>=0.8.2; python_version >= "3.11"
|
||||
jax[tpu]; python_version < "3.11"
|
||||
|
||||
# Standard JAX ecosystem dependencies
|
||||
flax
|
||||
optax
|
||||
orbax-checkpoint
|
||||
ml-collections
|
||||
|
||||
# TPU profiling & telemetry
|
||||
cloud-tpu-diagnostics
|
||||
tensorboard-plugin-profile
|
||||
ml-goodput-measurement
|
||||
tpu-info
|
||||
@@ -0,0 +1,13 @@
|
||||
name: "ray-py$PYTHON_VERSION-tpu-base$ARCH_SUFFIX"
|
||||
froms: ["ubuntu:22.04"]
|
||||
dockerfile: docker/base-deps/Dockerfile
|
||||
srcs:
|
||||
- python/requirements_compiled.txt
|
||||
- python/requirements_compiled_py$PYTHON_VERSION.txt
|
||||
- python/deplocks/base_deps/ray_base_deps_tpu_py$PYTHON_VERSION.lock
|
||||
build_args:
|
||||
- PYTHON_VERSION
|
||||
- BASE_IMAGE=ubuntu:22.04
|
||||
- PYTHON_DEPSET=python/deplocks/base_deps/ray_base_deps_tpu_py$PYTHON_VERSION.lock
|
||||
tags:
|
||||
- cr.ray.io/rayproject/ray-py$PYTHON_VERSION-tpu-base$ARCH_SUFFIX
|
||||
Reference in New Issue
Block a user