chore: import upstream snapshot with attribution

python/requirements/ml/core-requirements.txt

@@ -0,0 +1,15 @@
+# ML tracking integrations
+comet-ml==3.44.1
+mlflow>=2.22.0
+wandb>=0.23.1
+
+# ML training frameworks
+xgboost==2.1.0
+lightgbm==4.6.0
+
+# Huggingface
+transformers>=5.0
+accelerate>=1.0
+
+# Cloud storage tools
+s3fs==2023.12.1

python/requirements/ml/data-requirements.txt

@@ -0,0 +1,16 @@
+# Used by CI for datasets and docs.
+# https://github.com/ray-project/ray/pull/29448#discussion_r1006256498
+
+daft>=0.7.0
+dask[complete]==2023.6.1; python_version < '3.12'
+distributed==2023.6.1; python_version < '3.12'
+dask[complete]==2025.5.0; python_version >= '3.12'
+distributed==2025.5.0; python_version >= '3.12'
+aioboto3==12.1.0
+crc32c==2.3
+flask_cors
+bokeh==2.4.3; python_version < '3.12'
+modin>=0.31.0
+pandas>=2.2.3
+responses>=0.15.0
+pymars>=0.8.3; python_version < "3.12"

python/requirements/ml/data-test-requirements.txt

@@ -0,0 +1,29 @@
+# Used by CI for datasets tests.
+# https://github.com/ray-project/ray/pull/29448#discussion_r1006256498
+
+python-snappy
+tensorflow-datasets==4.9.3
+datasets>=3.0.2
+pytest-repeat
+soundfile
+fastavro
+google-cloud-bigquery
+google-cloud-core
+google-cloud-bigquery-storage
+google-api-core
+webdataset
+raydp==1.7.0b20250423.dev0
+pylance==1.0.3
+delta-sharing
+deltalake==1.5.0
+pytest-mock
+decord
+snowflake-connector-python>=3.15.0
+pyiceberg[sql-sqlite]==0.11.0
+clickhouse-connect
+confluent-kafka
+pybase64
+hudi==0.4.0
+datasketches
+testcontainers[kafka]
+obstore

python/requirements/ml/dl-cpu-requirements.txt

@@ -0,0 +1,34 @@
+# These requirements are used for the CI and CPU-only Docker images so we install CPU only versions of torch.
+# For GPU Docker images, you should install dl-gpu-requirements.txt afterwards.
+
+tensorflow==2.15.1; python_version < '3.12' and (sys_platform != 'darwin' or platform_machine != 'arm64')
+tensorflow-macos==2.15.1; python_version < '3.12' and sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow-probability==0.23.0; python_version < '3.12'
+tensorflow-io-gcs-filesystem==0.31.0; python_version < '3.12'
+tensorflow-datasets; python_version < '3.12'
+array-record==0.5.1; python_version < '3.12' and sys_platform != 'darwin' and platform_system != 'Windows'
+etils==1.5.2; python_version < '3.12'
+
+# If you make changes below this line, please also make the corresponding changes to `dl-gpu-requirements.txt`
+# and to `install-dependencies.sh`!
+
+--extra-index-url https://download.pytorch.org/whl/cpu  # for CPU versions of torch, torchvision
+--find-links https://data.pyg.org/whl/torch-2.7.0+cpu.html  # for CPU versions of torch-scatter, torch-sparse, torch-cluster, torch-spline-conv
+
+torch==2.7.0
+torchmetrics==0.10.3
+# torchtext 0.18.0 has no cp313 wheels on PyPI; keep for 3.10–3.12 only.
+torchtext==0.18.0; python_version < "3.13"
+torchvision==0.22.0
+
+torch-scatter==2.1.2
+torch-sparse==0.6.18
+torch-cluster==1.6.3
+torch-spline-conv==1.2.2
+torch-geometric==2.5.3
+
+cupy-cuda12x>=13.4.0; sys_platform != 'darwin'
+
+# Keep JAX version consistent with dl-gpu-requirements.txt
+jax==0.4.33; python_version < '3.12' and sys_platform != 'darwin'
+jaxlib==0.4.33; python_version < '3.12' and sys_platform != 'darwin'

python/requirements/ml/dl-gpu-requirements.txt

@@ -0,0 +1,24 @@
+# If you make changes below this line, please also make the corresponding changes to `dl-cpu-requirements.txt`!
+
+tensorflow==2.15.1; python_version < '3.12' and (sys_platform != 'darwin' or platform_machine != 'arm64')
+tensorflow-macos==2.15.1; python_version < '3.12' and sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow-probability==0.23.0; python_version < '3.12'
+tensorflow-datasets; python_version < '3.12'
+
+--extra-index-url https://download.pytorch.org/whl/cu128  # for GPU versions of torch, torchvision
+--find-links https://data.pyg.org/whl/torch-2.7.0+cu128.html  # for GPU versions of torch-scatter, torch-sparse, torch-cluster, torch-spline-conv
+# specifying explicit plus-notation below so pip overwrites the existing cpu verisons
+torch==2.7.0+cu128
+torchvision==0.22.0+cu128
+torch-scatter==2.1.2+pt27cu128
+torch-sparse==0.6.18+pt27cu128
+torch-cluster==1.6.3+pt27cu128
+torch-spline-conv==1.2.2+pt27cu128
+
+cupy-cuda12x>=13.4.0; sys_platform != 'darwin'
+cudf-cu12>=24.12.0; sys_platform != 'darwin'
+nixl==1.2.0; sys_platform != 'darwin'
+
+jax==0.4.33; python_version < '3.12' and sys_platform != 'darwin'
+jaxlib==0.4.33; python_version < '3.12' and sys_platform != 'darwin'
+jax-cuda12-plugin[cuda12]==0.4.33; python_version < '3.12' and sys_platform != 'darwin'

python/requirements/ml/py313/core-requirements.txt

@@ -0,0 +1,15 @@
+# ML tracking integrations
+comet-ml==3.44.1
+mlflow>=3.0.0
+wandb>=0.23.1
+
+# ML training frameworks
+xgboost==2.1.0
+lightgbm==4.6.0
+
+# Huggingface
+transformers>=5.0
+accelerate>=1.0
+
+# Cloud storage tools
+s3fs==2023.12.1

python/requirements/ml/py313/data-requirements.txt

@@ -0,0 +1,15 @@
+# Used by CI for datasets and docs.
+# https://github.com/ray-project/ray/pull/29448#discussion_r1006256498
+
+daft>=0.7.0
+dask[complete]>=2025.5.0
+distributed>=2025.5.0
+aioboto3==12.1.0
+crc32c==2.3
+flask_cors
+bokeh==3.1.0
+modin>=0.26.0
+pandas>=2.2.2
+responses>=0.15.0
+pymars>=0.8.3; python_version < "3.12"
+lance-namespace==0.6.1

python/requirements/ml/py313/data-test-requirements.txt

@@ -0,0 +1,46 @@
+# Used by CI for datasets tests.
+# https://github.com/ray-project/ray/pull/29448#discussion_r1006256498
+
+python-snappy
+tensorflow-datasets==4.9.3
+datasets>=3.0.2
+pytest-repeat
+soundfile
+fastavro
+google-cloud-bigquery
+google-cloud-core
+google-cloud-bigquery-storage
+google-api-core
+webdataset
+raydp==1.7.0b20250423.dev0
+pylance
+delta-sharing
+deltalake==1.5.0
+pytest-mock
+decord
+snowflake-connector-python>=3.15.0
+pyiceberg[sql-sqlite]==0.11.0
+clickhouse-connect
+confluent-kafka
+pybase64
+hudi==0.4.0
+datasketches
+testcontainers[kafka]
+obstore
+pyarrow
+torch
+tensorflow
+jax
+jaxlib
+tensorflow-datasets
+tensorflow-metadata>=1.17.0
+tf-keras
+torchvision==0.24.0
+confluent-kafka
+zarr<3 ; python_version >= '3.11'  # zarr 2.18.4+ requires py3.11+ (v2 API)
+zarr>=2.18,<2.18.4 ; python_version < '3.11'  # 2.18.3: last v2 line supporting py3.10
+# numcodecs is zarr's codec dep; 0.14+ dropped py3.10. Pin per-Python with exact
+# versions so the markers survive pip-compile -- the compiled-constraint pin must
+# stay gated to py3.11+, otherwise the py3.10 data locks can't resolve zarr.
+numcodecs==0.15.1 ; python_version >= '3.11'
+numcodecs==0.13.1 ; python_version < '3.11'

python/requirements/ml/py313/dl-cpu-requirements.txt

@@ -0,0 +1,37 @@
+# These requirements are used for the CI and CPU-only Docker images so we install CPU only versions of torch.
+# For GPU Docker images, you should install dl-gpu-requirements.txt afterwards.
+
+tensorflow==2.20.0; sys_platform != 'darwin' or platform_machine != 'arm64'
+tensorflow-macos==2.20.0; sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow-probability==0.24.0
+tensorflow-io-gcs-filesystem==0.31.0; python_version < '3.12'
+tensorflow-datasets; python_version < '3.12'
+array-record==0.5.1; python_version < '3.12' and sys_platform != 'darwin' and platform_system != 'Windows'
+etils==1.5.2; python_version < '3.12'
+tf-keras==2.20.0
+# If you make changes below this line, please also make the corresponding changes to `dl-gpu-requirements.txt`
+# and to `install-dependencies.sh`!
+
+--extra-index-url https://download.pytorch.org/whl/cpu  # for CPU versions of torch, torchvision
+--find-links https://data.pyg.org/whl/torch-2.9.0+cpu.html  # for CPU versions of torch-scatter, torch-sparse, torch-cluster, torch-spline-conv
+
+torch==2.9.0
+torchmetrics==0.10.3
+torchtext==0.18.0
+torchvision==0.24.0
+# xgboost pulls nvidia-nccl-cu12 transitively even in CPU context. Align the
+# pin with what cu128 torch requires so the compiled lock doesn't clash with
+# GPU depsets that consume it as a constraint.
+nvidia-nccl-cu12==2.27.5; platform_system == 'Linux' and platform_machine != 'aarch64'
+
+torch-scatter==2.1.2
+torch-sparse==0.6.18
+torch-cluster==1.6.3
+torch-spline-conv==1.2.2
+torch-geometric==2.5.3
+
+cupy-cuda12x==13.6.0; sys_platform != 'darwin'
+
+# Keep JAX version consistent with dl-gpu-requirements.txt
+jax==0.4.33; sys_platform != 'darwin'
+jaxlib==0.4.33; sys_platform != 'darwin'

python/requirements/ml/py313/dl-gpu-requirements.txt

@@ -0,0 +1,28 @@
+# If you make changes below this line, please also make the corresponding changes to `dl-cpu-requirements.txt`!
+
+tensorflow==2.20.0; sys_platform != 'darwin' or platform_machine != 'arm64'
+tensorflow-macos==2.20.0; sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow-probability==0.24.0
+tf-keras==2.20.0
+
+--extra-index-url https://download.pytorch.org/whl/cu128  # for GPU versions of torch, torchvision
+--find-links https://data.pyg.org/whl/torch-2.9.0+cu128.html  # for GPU versions of torch-scatter, torch-sparse, torch-cluster, torch-spline-conv
+# specifying explicit plus-notation below so pip overwrites the existing cpu verisons
+torch==2.9.0+cu128
+torchvision==0.24.0+cu128
+torch-scatter==2.1.2+pt29cu128
+torch-sparse==0.6.18+pt29cu128
+torch-cluster==1.6.3+pt29cu128
+torch-spline-conv==1.2.2+pt29cu128
+torch-geometric==2.5.3
+# Declared explicitly so GPU depsets resolve nccl from cu128 torch
+# transitively rather than being pinned by the CPU-built py3.13 lock.
+nvidia-nccl-cu12; platform_system == 'Linux' and platform_machine != 'aarch64'
+
+cupy-cuda12x==13.6.0; sys_platform != 'darwin'
+cudf-cu12>=24.12.0; sys_platform != 'darwin' and python_version >= '3.11'
+nixl==0.4.0; sys_platform != 'darwin'
+
+jax==0.4.33; sys_platform != 'darwin'
+jaxlib==0.4.33; sys_platform != 'darwin'
+jax-cuda12-plugin[cuda12]==0.4.33; sys_platform != 'darwin'

python/requirements/ml/py313/lightning1.txt

@@ -0,0 +1,2 @@
+pytorch-lightning==1.8.6
+numpy==1.26.4

python/requirements/ml/py313/ml-requirements.txt

@@ -0,0 +1,6 @@
+tf-keras
+transformers
+# keras 3.13 dropped py3.10 support; pin explicitly so pip-compile preserves
+# a py_version marker on the lock entry (tensorflow pulls keras transitively).
+keras==3.12.1; python_version < '3.11'
+keras==3.14.0; python_version >= '3.11'

python/requirements/ml/py313/rllib-requirements.txt

@@ -0,0 +1,9 @@
+# For auto-generating an env-rendering Window.
+pyglet==1.5.15
+imageio-ffmpeg==0.4.5
+rich==13.7.1
+# Msgpack checkpoint stuff.
+msgpack
+msgpack-numpy
+ormsgpack
+tf_keras

python/requirements/ml/py313/rllib-test-requirements.txt

@@ -0,0 +1,51 @@
+# Testing framework.
+pytest
+pytest-asyncio
+
+# Environment adapters.
+# ---------------------
+# Atari
+ale_py==0.10.1
+imageio==2.34.2
+opencv-python-headless==4.10.0.84
+
+# For testing MuJoCo envs with gymnasium.
+mujoco==3.2.4
+dm_control==1.0.12; python_version < "3.12"
+
+# For tests on PettingZoo's multi-agent envs.
+pettingzoo==1.24.3
+pymunk==6.2.1
+tinyscaler==1.2.8
+shimmy==2.0.0
+supersuit==3.9.3
+
+# For tests on minigrid.
+minigrid==2.3.1
+tensorflow_estimator
+
+# DeepMind's OpenSpiel
+open-spiel==1.4
+
+# Requires libtorrent which is unavailable for arm64
+h5py==3.12.1
+
+# Requirements for rendering.
+moviepy
+
+# numexpr is an optional pandas dependency that gets imported at runtime.
+# It must be explicitly pinned here to ensure compatibility with numpy 2.x.
+numexpr
+
+# For ONNX export tests (policy_inference_after_training examples, --use-onnx-for-inference).
+# onnxscript 0.5.x has a version-converter bug that breaks every torch>=2.9 dynamo ONNX
+# export; pin >=0.6 directly (bumping onnx alone won't force it -- the resolver keeps the
+# existing onnxscript pin). onnxscript>=0.6 in turn requires onnx>=1.17.
+# Pinned only on this py3.13 track, NOT in the non-py313 rllib-test-requirements.txt: that
+# track's tensorflow 2.15.1 caps ml_dtypes~=0.3.1, which conflicts with onnxscript>=0.6's
+# onnx-ir -> ml_dtypes>=0.5.0. The rllib ONNX tests run from py3.13-derived deplocks
+# (rllib_build_depset), so pinning here is sufficient; don't add this to the non-py313 file.
+onnx>=1.17.0; sys_platform != 'darwin' or platform_machine != 'arm64'
+onnxruntime==1.20.0; (sys_platform != 'darwin' or platform_machine != 'arm64') and python_version == '3.10'
+onnxruntime==1.24.4; (sys_platform != 'darwin' or platform_machine != 'arm64') and python_version > '3.10'
+onnxscript>=0.6.2; sys_platform != 'darwin' or platform_machine != 'arm64'

python/requirements/ml/py313/third_party.txt

@@ -0,0 +1,2 @@
+psutil
+colorama

python/requirements/ml/py313/torchft.txt

@@ -0,0 +1 @@
+torchft-nightly==2026.5.15

python/requirements/ml/py313/train-requirements.txt

@@ -0,0 +1,6 @@
+accelerate>=0.20.1
+deepspeed>=0.12.3
+datasets>=4.0.0,<5.0.0
+huggingface-hub>=1.0,<2.0
+numexpr>=2.8.4
+accelerate

python/requirements/ml/py313/train-test-requirements.txt

@@ -0,0 +1,9 @@
+boto3==1.29.7
+evaluate==0.4.6
+freezegun==1.1.0
+mosaicml; python_version < "3.12"
+sentencepiece==0.2.1
+s3torchconnector==1.4.3
+jupytext
+tblib
+xmltodict

python/requirements/ml/py313/tune-requirements.txt

@@ -0,0 +1,14 @@
+# Searchers
+ax-platform==1.2.1
+# Python 3.13 only: no jaxtyping cap here; default tune-requirements.txt uses jaxtyping<0.3.8 for Python 3.10.
+
+bayesian-optimization>=1.4.0
+
+# BOHB
+ConfigSpace>=0.7.1; python_version < "3.12"
+hpbandster==0.7.4; python_version < "3.12"
+
+hyperopt @ git+https://github.com/hyperopt/hyperopt.git@2504ee61419737e814e2dec2961b15d12775529c
+future
+nevergrad>=0.4.3.post7
+optuna==4.1.0

python/requirements/ml/py313/tune-test-requirements.txt

@@ -0,0 +1,18 @@
+aim==3.23.0; python_version < "3.12"
+boto3==1.29.7
+
+jupyterlab
+matplotlib!=3.4.3
+
+pytest-remotedata==0.3.2
+lightning>2
+fairscale==0.4.6
+shortuuid==1.0.1
+timm==0.9.2
+
+zoopt==0.4.1
+
+# timeseries lib
+statsforecast==1.7.0
+prophet==1.1.5
+holidays==0.39

python/requirements/ml/rllib-requirements.txt

@@ -0,0 +1,8 @@
+# For auto-generating an env-rendering Window.
+pyglet==1.5.15
+imageio-ffmpeg==0.4.5
+rich==13.7.1
+# Msgpack checkpoint stuff.
+msgpack
+msgpack-numpy
+ormsgpack

python/requirements/ml/rllib-test-requirements.txt

@@ -0,0 +1,35 @@
+# Environment adapters.
+# ---------------------
+# Atari
+ale_py==0.10.1
+imageio==2.34.2
+opencv-python-headless==4.9.0.80
+
+# For testing MuJoCo envs with gymnasium.
+mujoco==3.2.4
+dm_control==1.0.12; python_version < "3.12"
+
+# For tests on PettingZoo's multi-agent envs.
+pettingzoo==1.24.3
+pymunk==6.2.1
+tinyscaler==1.2.8
+shimmy==2.0.0
+supersuit==3.9.3
+
+# For tests on minigrid.
+minigrid==2.3.1
+tensorflow_estimator
+
+# DeepMind's OpenSpiel
+open-spiel==1.4
+
+# Requires libtorrent which is unavailable for arm64
+h5py==3.12.1
+
+# Requirements for rendering.
+moviepy
+
+# For ONNX export tests (policy_inference_after_training examples, --use-onnx-for-inference).
+onnx==1.16.0; sys_platform != 'darwin' or platform_machine != 'arm64'
+onnxruntime==1.18.0; sys_platform != 'darwin' or platform_machine != 'arm64'
+onnxscript; sys_platform != 'darwin' or platform_machine != 'arm64'

python/requirements/ml/train-requirements.txt

@@ -0,0 +1,3 @@
+deepspeed==0.17.2
+datasets>=4.0.0,<5.0.0
+huggingface-hub>=1.0,<2.0

python/requirements/ml/train-test-requirements.txt

@@ -0,0 +1,4 @@
+evaluate==0.4.6
+mosaicml; python_version < "3.12"
+sentencepiece==0.1.96
+s3torchconnector==1.4.3

python/requirements/ml/tune-requirements.txt

@@ -0,0 +1,20 @@
+# Searchers
+ax-platform==1.2.1
+# ax-platform -> botorch -> gpytorch -> jaxtyping: jaxtyping>=0.3.8 requires Python>=3.11.
+# botorch itself declares python_requires>=3.10, but that constraint does not cover its
+# transitive dependencies.
+# In our case we need to pin jaxtyping<0.3.8 (i.e. 0.3.7) which supports Python>=3.10.
+# Remove this pin once Ray drops Python 3.10 support.
+jaxtyping<0.3.8
+
+
+bayesian-optimization==1.4.3
+
+# BOHB
+ConfigSpace==0.7.1; python_version < "3.12"
+hpbandster==0.7.4; python_version < "3.12"
+
+hyperopt @ git+https://github.com/hyperopt/hyperopt.git@2504ee61419737e814e2dec2961b15d12775529c
+future
+nevergrad==0.4.3.post7
+optuna==4.1.0

python/requirements/ml/tune-test-requirements.txt

@@ -0,0 +1,17 @@
+aim==3.23.0; python_version < "3.12"
+
+jupyterlab
+matplotlib!=3.4.3
+
+pytest-remotedata==0.3.2
+lightning>2
+fairscale==0.4.6
+shortuuid==1.0.1
+timm==0.9.2
+
+zoopt==0.4.1
+
+# timeseries lib
+statsforecast==1.7.0
+prophet==1.1.5
+holidays==0.39