wehub-resource-sync
4976 lines
125 KiB
YAML
4976 lines
125 KiB
YAML
# Global release test configuration file.
|
|
# All your release test configuration should go here. Adding release tests here
|
|
# will automatically enable them in the Buildkite release testing schedules
|
|
# (unless they have frequency: manual).
|
|
# Here is an example configuration for reference:
|
|
#- name: example_test
|
|
# # Tests with the same group will be grouped in the Buildkite UI
|
|
# group: Example group
|
|
# # Provide the working directory which will be uploaded to the cluster
|
|
# working_dir: example_dir
|
|
#
|
|
# # How often to run the tests.
|
|
# # One of [manual, any, multi, nightly, nightly-3x, weekly].
|
|
# # Descriptions of each frequency (that's not immediately obvious):
|
|
# # - manual: Not run on a schedule, but can be manually run through the buildkite UI.
|
|
# # - nightly-3x: Run 3 times a week (Monday, Wednesday, Friday).
|
|
# frequency: weekly
|
|
# # Owning team. This field will be persisted to the database
|
|
# team: ml
|
|
#
|
|
# # Python version. This optional field determines which Python version to run tests
|
|
# # on. This must be a string!
|
|
# python: "3.7"
|
|
#
|
|
# # Cluster information
|
|
# cluster:
|
|
# # Location of cluster compute, relative to working_dir
|
|
# cluster_compute: cluster_compute.yaml
|
|
# # Autosuspend parameter passed to the cluster.
|
|
# # The cluster will automatically terminate if inactive for this
|
|
# # many minutes. Defaults to 10 if not set.
|
|
# autosuspend_mins: 10
|
|
# # Optional cloud_id to use instead of the default cloud
|
|
# cloud_id: cld_12345678
|
|
#
|
|
# # Run configuration for the test
|
|
# run:
|
|
# # If you want to wait for nodes to be ready, you can specify this here:
|
|
# wait_for_nodes:
|
|
# # Number of nodes
|
|
# num_nodes: 16
|
|
# # Timeout for waiting for nodes. If nodes are not up by then, the
|
|
# # test will fail.
|
|
# timeout: 600
|
|
#
|
|
# # Optional prepare script to be run on the cluster before the test script
|
|
# prepare: python prepare.py
|
|
# # The prepare command can have a separate timeout
|
|
# prepare_timeout: 300
|
|
#
|
|
# # Main script to run as the test script
|
|
# script: python workloads/train_small.py
|
|
# # Timeout in seconds. After this time the test is considered as failed.
|
|
# timeout: 600
|
|
#
|
|
# # You can specify smoke test definitions here. If a smoke test is triggered,
|
|
# # it will deep update the main test configuration with the values provided
|
|
# # here. Smoke tests will automatically run with IS_SMOKE_TEST=1 as en
|
|
# # environment variable and receive the --smoke-test flag as a parameter in the
|
|
# # run script.
|
|
# smoke_test:
|
|
# # Smoke tests can have different frequencies. A smoke test is only triggered
|
|
# # when the regular test is not matched.
|
|
# frequency: nightly
|
|
# # Here we adjust the run timeout down and run on less nodes. The test script
|
|
# # remains the same.
|
|
# run:
|
|
# timeout: 300
|
|
# wait_for_nodes:
|
|
# num_nodes: 4
|
|
# timeout: 600
|
|
#
|
|
# # After the test finished, this handler (in alerts/) will process the results.
|
|
# # It can then let the test fail, e.g. if a metric regression is observed.
|
|
# alert: default
|
|
|
|
#######################
|
|
# Baseline test
|
|
#######################
|
|
- name: hello_world
|
|
python: "3.10"
|
|
team: reef
|
|
group: hello_world
|
|
frequency: nightly
|
|
working_dir: hello_world_tests
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: hello_world_compute_config.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
num_retries: 3
|
|
script: python hello_world.py
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: hello_world_compute_config_gce.yaml
|
|
- __suffix__: azure
|
|
env: azure
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: hello_world_compute_config_azure.yaml
|
|
- __suffix__: released
|
|
cluster:
|
|
ray_version: 2.50.0
|
|
|
|
- name: hello_world_custom_byod
|
|
python: "3.10"
|
|
team: reef
|
|
group: hello_world
|
|
frequency: nightly
|
|
working_dir: hello_world_tests
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
post_build_script: byod_hello_world.sh
|
|
python_depset: emoji.lock
|
|
cluster_compute: hello_world_compute_config.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: python hello_world_emoji.py
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: hello_world_compute_config_gce.yaml
|
|
- __suffix__: released
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
ray_version: 2.50.0
|
|
|
|
- name: hello_world_custom_runtime_env
|
|
python: "3.10"
|
|
team: reef
|
|
group: hello_world
|
|
frequency: nightly
|
|
working_dir: hello_world_tests
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- HELLO_WORLD_TEST=1
|
|
cluster_compute: hello_world_compute_config.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: python hello_world.py
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: hello_world_custom_byod_depset_only
|
|
python: "3.10"
|
|
team: reef
|
|
group: hello_world
|
|
frequency: nightly
|
|
working_dir: hello_world_tests
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
python_depset: emoji.lock
|
|
cluster_compute: hello_world_compute_config.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: bash test.sh
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: hello_world_py313
|
|
python: "3.13"
|
|
team: reef
|
|
group: hello_world
|
|
frequency: nightly
|
|
working_dir: hello_world_tests
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod: {}
|
|
cluster_compute: hello_world_compute_config.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
num_retries: 3
|
|
script: python hello_world.py
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: "hello_world_cu130_py{{python}}"
|
|
python: "{{python}}"
|
|
team: reef
|
|
group: hello_world
|
|
frequency: nightly
|
|
working_dir: hello_world_tests
|
|
matrix:
|
|
setup:
|
|
python: ["3.10", "3.11", "3.12", "3.13"]
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu-cu130
|
|
cluster_compute: hello_world_compute_config.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
num_retries: 3
|
|
script: python hello_world.py
|
|
|
|
#######################
|
|
# SSH sanity checks
|
|
#######################
|
|
|
|
- name: ssh_sanity_check
|
|
python: "3.11"
|
|
team: reef
|
|
group: hello_world
|
|
frequency: nightly
|
|
working_dir: hello_world_tests
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: hello_world_compute_config.yaml
|
|
|
|
run:
|
|
timeout: 600
|
|
script: bash sanity_check_ssh.sh
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: ssh_sanity_check_llm
|
|
python: "3.12"
|
|
team: llm
|
|
group: hello_world
|
|
frequency: nightly
|
|
working_dir: hello_world_tests
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: llm-cu130
|
|
# Doesn't need GPUs to run ssh sanity check
|
|
cluster_compute: hello_world_compute_config.yaml
|
|
|
|
run:
|
|
timeout: 600
|
|
script: bash sanity_check_ssh.sh
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
#######################
|
|
# Cluster scaling tests
|
|
#######################
|
|
- name: cluster_tune_scale_up_down
|
|
python: "3.10"
|
|
group: Cluster tests
|
|
working_dir: cluster_tests
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
cluster_compute: cpt_autoscaling_1-3_aws.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python workloads/tune_scale_up_down.py
|
|
wait_for_nodes:
|
|
num_nodes: 0
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: cpt_autoscaling_1-3_gce.yaml
|
|
- __suffix__: kuberay
|
|
env: kuberay
|
|
frequency: nightly
|
|
cluster:
|
|
cluster_compute: cpt_autoscaling_1-3_kuberay.yaml
|
|
|
|
alert: default
|
|
|
|
|
|
#########################
|
|
# AIR release tests
|
|
#########################
|
|
|
|
# Ray Train distributed Torch benchmarks
|
|
- name: air_benchmark_torch_mnist_gpu_4x4
|
|
python: "3.10"
|
|
group: AIR tests
|
|
working_dir: air_tests/air_benchmarks
|
|
|
|
frequency: weekly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_gpu_4x4_aws.yaml
|
|
|
|
run:
|
|
timeout: 4800
|
|
script: RAY_TRAIN_V2_ENABLED=1 python workloads/torch_benchmark.py run --num-runs 3 --num-epochs 120 --num-workers 16 --cpus-per-worker 4 --batch-size 1024 --use-gpu
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
smoke_test:
|
|
frequency: weekly
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_gpu_2x2_aws.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python workloads/torch_benchmark.py run --num-runs 3 --num-epochs 60 --num-workers 2 --cpus-per-worker 4 --batch-size 512 --use-gpu
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 2
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_gpu_4x4_gce.yaml
|
|
smoke_test:
|
|
frequency: manual
|
|
|
|
alert: default
|
|
|
|
- name: air_benchmark_tune_torch_mnist
|
|
python: "3.10"
|
|
group: AIR tests
|
|
working_dir: air_tests/air_benchmarks
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_cpu_4_aws.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: RAY_TRAIN_V2_ENABLED=1 python workloads/tune_torch_benchmark.py --num-runs 1 --num-trials 2 --num-workers 2
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_cpu_4_gce.yaml
|
|
|
|
alert: default
|
|
|
|
# Ray Train distributed Tensorflow benchmarks
|
|
- name: air_benchmark_tensorflow_mnist_gpu_4x4
|
|
python: "3.10"
|
|
group: AIR tests
|
|
working_dir: air_tests/air_benchmarks
|
|
|
|
# https://github.com/ray-project/ray/issues/46687
|
|
frequency: manual
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_gpu_4x4_aws.yaml
|
|
|
|
run:
|
|
timeout: 5400
|
|
script: RAY_TRAIN_V2_ENABLED=1 python workloads/tensorflow_benchmark.py run --num-runs 3 --num-epochs 200 --num-workers 16 --cpus-per-worker 4 --batch-size 1024 --use-gpu
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
smoke_test:
|
|
frequency: manual
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_gpu_2x2_aws.yaml
|
|
|
|
run:
|
|
script: python workloads/tensorflow_benchmark.py run --num-runs 3 --num-epochs 60 --num-workers 2 --cpus-per-worker 4 --batch-size 512 --use-gpu
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 2
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_gpu_4x4_gce.yaml
|
|
smoke_test:
|
|
frequency: manual
|
|
|
|
alert: default
|
|
|
|
|
|
#######################
|
|
# AIR examples
|
|
#######################
|
|
|
|
|
|
# Test additional CPU nodes for preprocessing.
|
|
- name: air_example_dreambooth_finetuning
|
|
python: "3.10"
|
|
group: AIR examples
|
|
working_dir: air_examples/dreambooth
|
|
|
|
# https://github.com/ray-project/ray/issues/49847
|
|
frequency: manual
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: dreambooth_compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: pip install -Ur dreambooth/requirements.txt && bash dreambooth_run.sh
|
|
artifact_path: /tmp/artifacts/example_out.jpg
|
|
|
|
# variations: A10G not available on GCE, yet.
|
|
|
|
- name: air_example_dreambooth_finetuning_lora
|
|
python: "3.10"
|
|
group: AIR examples
|
|
working_dir: air_examples/dreambooth
|
|
|
|
# https://github.com/ray-project/ray/issues/49846
|
|
frequency: manual
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: dreambooth_compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: pip install -Ur dreambooth/requirements.txt && bash dreambooth_run.sh --lora
|
|
artifact_path: /tmp/artifacts/example_out.jpg
|
|
|
|
- name: air_example_gptj_deepspeed_fine_tuning
|
|
python: "3.10"
|
|
group: AIR examples
|
|
working_dir: air_examples/gptj_deepspeed_finetuning
|
|
frequency: manual
|
|
team: ml
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
post_build_script: byod_gptj_test.sh
|
|
cluster_compute: gptj_deepspeed_compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 4500
|
|
script: RAY_TRAIN_V2_ENABLED=1 python test_myst_doc.py --path gptj_deepspeed_fine_tuning.ipynb
|
|
|
|
- name: air_example_dolly_v2_lightning_fsdp_finetuning
|
|
python: "3.10"
|
|
group: AIR examples
|
|
working_dir: air_examples/dolly_v2_lightning_fsdp_finetuning
|
|
frequency: manual
|
|
team: ml
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
post_build_script: byod_dolly_test.sh
|
|
cluster_compute: dolly_v2_fsdp_compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 4700
|
|
script: RAY_TRAIN_V2_ENABLED=1 python test_myst_doc.py --path lightning-llm-finetuning-7b.ipynb
|
|
|
|
# variations: TODO(jungong): add GCP variation.
|
|
|
|
- name: air_example_vicuna_13b_lightning_deepspeed_finetuning
|
|
python: "3.10"
|
|
group: AIR examples
|
|
working_dir: air_examples/vicuna_13b_lightning_deepspeed_finetuning
|
|
frequency: manual
|
|
team: ml
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
post_build_script: byod_vicuna_test.sh
|
|
cluster_compute: vicuna_13b_deepspeed_compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 4700
|
|
script: python test_myst_doc.py --path vicuna_13b_lightning_deepspeed_finetune.ipynb
|
|
|
|
#######################
|
|
# Tune tests
|
|
#######################
|
|
- name: tune_with_frequent_pausing
|
|
python: "3.10"
|
|
group: Tune tests
|
|
working_dir: tune_tests
|
|
|
|
frequency: nightly-3x
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RAY_memory_usage_threshold=0.5
|
|
- automatic_object_spilling_enabled=0
|
|
cluster_compute: frequent_pausing/compute_config_aws.yaml
|
|
|
|
run:
|
|
timeout: 600 # 10min
|
|
long_running: true
|
|
script: RAY_TRAIN_V2_ENABLED=1 python frequent_pausing/script.py
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: frequent_pausing/compute_config_gce.yaml
|
|
|
|
alert: default
|
|
|
|
- name: tune_cloud_long_running_cloud_storage
|
|
python: "3.10"
|
|
group: Tune cloud tests
|
|
working_dir: tune_tests/cloud_tests
|
|
frequency: weekly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_aws_1x4.yaml
|
|
|
|
run:
|
|
# 14 hours
|
|
timeout: 50400
|
|
long_running: true
|
|
script: RAY_TRAIN_V2_ENABLED=1 python workloads/long_running_cloud_storage.py s3://tune-cloud-tests/long_running_cloud_storage
|
|
|
|
# NOTE: This smoke test is not useful to run because the point of the test
|
|
# is to be long running. This is just for debugging updates to the test quickly.
|
|
smoke_test:
|
|
frequency: manual
|
|
run:
|
|
timeout: 600
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: tpl_gce_1x4.yaml
|
|
run:
|
|
# 14 hours
|
|
timeout: 50400
|
|
long_running: true
|
|
script: RAY_TRAIN_V2_ENABLED=1 python workloads/long_running_cloud_storage.py gs://tune-cloud-tests/long_running_cloud_storage
|
|
wait_for_nodes:
|
|
num_nodes: 1
|
|
|
|
alert: long_running_tests
|
|
|
|
########################
|
|
# Tune scalability tests
|
|
########################
|
|
|
|
- name: tune_scalability_bookkeeping_overhead
|
|
python: "3.10"
|
|
group: Tune scalability tests
|
|
working_dir: tune_tests/scalability_tests
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
cluster_compute: tpl_1x16.yaml
|
|
|
|
run:
|
|
timeout: 1200
|
|
script: python workloads/test_bookkeeping_overhead.py
|
|
|
|
alert: tune_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_gce_1x16.yaml
|
|
- __suffix__: kuberay
|
|
env: kuberay
|
|
frequency: nightly
|
|
cluster:
|
|
cluster_compute: kuberay.yaml
|
|
|
|
- name: tune_scalability_durable_trainable
|
|
python: "3.10"
|
|
group: Tune scalability tests
|
|
working_dir: tune_tests/scalability_tests
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_16x2.yaml
|
|
|
|
run:
|
|
timeout: 900
|
|
script: python workloads/test_durable_trainable.py --bucket s3://tune-cloud-tests/scalability_durable_trainable
|
|
wait_for_nodes:
|
|
num_nodes: 16
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
run:
|
|
timeout: 900
|
|
script: python workloads/test_durable_trainable.py --bucket gs://tune-cloud-tests/scalability_durable_trainable
|
|
wait_for_nodes:
|
|
num_nodes: 16
|
|
cluster:
|
|
cluster_compute: tpl_gce_16x2.yaml
|
|
|
|
alert: tune_tests
|
|
|
|
|
|
- name: tune_scalability_durable_multifile_checkpoints
|
|
python: "3.10"
|
|
group: Tune scalability tests
|
|
working_dir: tune_tests/scalability_tests
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_16x2.yaml
|
|
|
|
run:
|
|
timeout: 900
|
|
script: python workloads/test_durable_multifile_checkpoints.py --bucket s3://tune-cloud-tests/scalability_durable_multifile_checkpoints
|
|
wait_for_nodes:
|
|
num_nodes: 16
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
run:
|
|
timeout: 900
|
|
script: python workloads/test_durable_multifile_checkpoints.py --bucket gs://tune-cloud-tests/scalability_durable_multifile_checkpoints
|
|
wait_for_nodes:
|
|
num_nodes: 16
|
|
cluster:
|
|
cluster_compute: tpl_gce_16x2.yaml
|
|
|
|
alert: tune_tests
|
|
|
|
- name: tune_scalability_long_running_large_checkpoints
|
|
python: "3.10"
|
|
group: Tune scalability tests
|
|
working_dir: tune_tests/scalability_tests
|
|
|
|
frequency: weekly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_1x32_hd.yaml
|
|
|
|
run:
|
|
timeout: 86400
|
|
script: python workloads/test_long_running_large_checkpoints.py
|
|
long_running: true
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
|
|
run:
|
|
timeout: 3600
|
|
|
|
alert: tune_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
smoke_test:
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: tpl_gce_1x32_hd.yaml
|
|
|
|
- name: tune_scalability_network_overhead
|
|
python: "3.10"
|
|
group: Tune scalability tests
|
|
working_dir: tune_tests/scalability_tests
|
|
|
|
frequency: weekly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_100x2.yaml
|
|
|
|
run:
|
|
timeout: 750
|
|
prepare_timeout: 1200
|
|
script: python workloads/test_network_overhead.py
|
|
wait_for_nodes:
|
|
num_nodes: 100
|
|
|
|
alert: tune_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: smoke-test
|
|
frequency: nightly
|
|
cluster:
|
|
cluster_compute: tpl_20x2.yaml
|
|
run:
|
|
timeout: 750
|
|
prepare_timeout: 600
|
|
script: python workloads/test_network_overhead.py --smoke-test
|
|
wait_for_nodes:
|
|
num_nodes: 20
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: tpl_gce_100x2.yaml
|
|
|
|
- name: tune_scalability_result_throughput_cluster
|
|
python: "3.10"
|
|
group: Tune scalability tests
|
|
working_dir: tune_tests/scalability_tests
|
|
|
|
frequency: nightly-3x
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_16x64.yaml
|
|
|
|
run:
|
|
timeout: 600
|
|
script: python workloads/test_result_throughput_cluster.py
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 16
|
|
|
|
alert: tune_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: tpl_gce_16x64.yaml
|
|
|
|
- name: tune_scalability_result_throughput_single_node
|
|
python: "3.10"
|
|
group: Tune scalability tests
|
|
working_dir: tune_tests/scalability_tests
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_1x96.yaml
|
|
|
|
run:
|
|
timeout: 600
|
|
script: python workloads/test_result_throughput_single_node.py
|
|
|
|
alert: tune_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: tpl_gce_1x96.yaml
|
|
|
|
|
|
############################
|
|
# Tune fault tolerance tests
|
|
############################
|
|
- name: tune_worker_fault_tolerance
|
|
python: "3.10"
|
|
group: Tune fault tolerance tests
|
|
working_dir: tune_tests/fault_tolerance_tests
|
|
|
|
frequency: nightly-3x
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_aws_16x1.yaml
|
|
|
|
run:
|
|
timeout: 5400
|
|
script: python workloads/test_tune_worker_fault_tolerance.py --bucket s3://tune-cloud-tests/worker_fault_tolerance
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 16
|
|
|
|
# Disabled until we can kill nodes in GCE
|
|
# variations:
|
|
# - __suffix__: aws
|
|
# - __suffix__: gce
|
|
# env: gce
|
|
# frequency: manual
|
|
# run:
|
|
# timeout: 5400
|
|
# script: python workloads/test_tune_worker_fault_tolerance.py --bucket gs://tune-cloud-tests/worker_fault_tolerance
|
|
#
|
|
# wait_for_nodes:
|
|
# num_nodes: 16
|
|
# cluster:
|
|
# cluster_compute: tpl_gce_16x1.yaml
|
|
|
|
#######################
|
|
# Long running tests
|
|
#######################
|
|
|
|
- name: long_running_actor_deaths
|
|
python: "3.10"
|
|
group: Long running tests
|
|
working_dir: long_running_tests
|
|
|
|
frequency: weekly
|
|
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RLLIB_TEST_NO_JAX_IMPORT=1
|
|
cluster_compute: tpl_cpu_1.yaml
|
|
|
|
run:
|
|
timeout: 86400
|
|
script: python workloads/actor_deaths.py
|
|
long_running: true
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
|
|
run:
|
|
timeout: 3600
|
|
|
|
alert: long_running_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
smoke_test:
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: tpl_cpu_1_gce.yaml
|
|
|
|
- name: long_running_many_actor_tasks
|
|
python: "3.10"
|
|
group: Long running tests
|
|
working_dir: long_running_tests
|
|
|
|
frequency: weekly
|
|
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RLLIB_TEST_NO_JAX_IMPORT=1
|
|
cluster_compute: tpl_cpu_1.yaml
|
|
|
|
run:
|
|
timeout: 86400
|
|
script: python workloads/many_actor_tasks.py
|
|
long_running: true
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
|
|
run:
|
|
timeout: 3600
|
|
|
|
alert: long_running_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
smoke_test:
|
|
frequency: manual
|
|
run:
|
|
timeout: 3600
|
|
cluster:
|
|
cluster_compute: tpl_cpu_1_gce.yaml
|
|
|
|
- name: long_running_many_drivers
|
|
python: "3.10"
|
|
group: Long running tests
|
|
working_dir: long_running_tests
|
|
|
|
frequency: weekly
|
|
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RLLIB_TEST_NO_JAX_IMPORT=1
|
|
cluster_compute: tpl_cpu_4.yaml
|
|
|
|
run:
|
|
timeout: 86400
|
|
script: python workloads/many_drivers.py --iteration-num=4000
|
|
long_running: true
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
|
|
run:
|
|
timeout: 3600
|
|
|
|
alert: long_running_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
smoke_test:
|
|
frequency: manual
|
|
run:
|
|
timeout: 3600
|
|
cluster:
|
|
cluster_compute: tpl_cpu_4_gce.yaml
|
|
|
|
- name: long_running_many_tasks
|
|
python: "3.10"
|
|
group: Long running tests
|
|
working_dir: long_running_tests
|
|
|
|
frequency: weekly
|
|
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RLLIB_TEST_NO_JAX_IMPORT=1
|
|
cluster_compute: tpl_cpu_1.yaml
|
|
|
|
run:
|
|
timeout: 86400
|
|
script: python workloads/many_tasks.py
|
|
long_running: true
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
|
|
run:
|
|
timeout: 3600
|
|
|
|
alert: long_running_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
smoke_test:
|
|
frequency: manual
|
|
run:
|
|
timeout: 3600
|
|
cluster:
|
|
cluster_compute: tpl_cpu_1_gce.yaml
|
|
|
|
- name: long_running_many_tasks_serialized_ids
|
|
python: "3.10"
|
|
group: Long running tests
|
|
working_dir: long_running_tests
|
|
|
|
frequency: weekly
|
|
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RLLIB_TEST_NO_JAX_IMPORT=1
|
|
cluster_compute: tpl_cpu_1.yaml
|
|
|
|
run:
|
|
timeout: 86400
|
|
script: python workloads/many_tasks_serialized_ids.py
|
|
long_running: true
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
|
|
run:
|
|
timeout: 3600
|
|
|
|
alert: long_running_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
smoke_test:
|
|
frequency: manual
|
|
run:
|
|
timeout: 3600
|
|
cluster:
|
|
cluster_compute: tpl_cpu_1_gce.yaml
|
|
|
|
- name: long_running_node_failures
|
|
python: "3.10"
|
|
group: Long running tests
|
|
working_dir: long_running_tests
|
|
|
|
frequency: weekly
|
|
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RLLIB_TEST_NO_JAX_IMPORT=1
|
|
cluster_compute: tpl_cpu_1.yaml
|
|
|
|
run:
|
|
timeout: 86400
|
|
script: python workloads/node_failures.py
|
|
long_running: true
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
|
|
run:
|
|
timeout: 3600
|
|
|
|
alert: long_running_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
smoke_test:
|
|
frequency: manual
|
|
run:
|
|
timeout: 3600
|
|
cluster:
|
|
cluster_compute: tpl_cpu_1_gce.yaml
|
|
|
|
- name: long_running_serve
|
|
python: "3.10"
|
|
group: Long running tests
|
|
working_dir: long_running_tests
|
|
|
|
frequency: weekly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RLLIB_TEST_NO_JAX_IMPORT=1
|
|
cluster_compute: tpl_cpu_1_large.yaml
|
|
|
|
run:
|
|
timeout: 86400
|
|
script: python workloads/serve.py
|
|
long_running: true
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
|
|
run:
|
|
timeout: 3600
|
|
|
|
alert: long_running_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
smoke_test:
|
|
frequency: manual
|
|
run:
|
|
timeout: 3600
|
|
cluster:
|
|
cluster_compute: tpl_cpu_1_gce.yaml
|
|
|
|
- name: long_running_serve_failure
|
|
python: "3.10"
|
|
group: Long running tests
|
|
working_dir: long_running_tests
|
|
|
|
frequency: weekly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RLLIB_TEST_NO_JAX_IMPORT=1
|
|
cluster_compute: tpl_cpu_1_c5.yaml
|
|
|
|
run:
|
|
timeout: 86400
|
|
script: python workloads/serve_failure.py
|
|
long_running: true
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
|
|
run:
|
|
timeout: 600
|
|
|
|
alert: long_running_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
smoke_test:
|
|
frequency: manual
|
|
run:
|
|
timeout: 86400
|
|
cluster:
|
|
cluster_compute: tpl_cpu_1_c5_gce.yaml
|
|
|
|
- name: long_running_many_jobs
|
|
python: "3.10"
|
|
group: Long running tests
|
|
working_dir: long_running_tests
|
|
|
|
frequency: weekly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RLLIB_TEST_NO_JAX_IMPORT=1
|
|
cluster_compute: tpl_cpu_1.yaml
|
|
|
|
run:
|
|
timeout: 86400
|
|
script: python workloads/long_running_many_jobs.py --num-clients=1
|
|
long_running: true
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
|
|
run:
|
|
timeout: 1800
|
|
|
|
alert: long_running_tests
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
smoke_test:
|
|
frequency: manual
|
|
run:
|
|
timeout: 3600
|
|
cluster:
|
|
cluster_compute: tpl_cpu_1_gce.yaml
|
|
|
|
########################
|
|
# Jobs tests
|
|
########################
|
|
|
|
- name: jobs_basic_local_working_dir
|
|
python: "3.10"
|
|
group: Jobs tests
|
|
working_dir: jobs_tests
|
|
|
|
frequency: nightly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_tpl_4_xlarge.yaml
|
|
|
|
run:
|
|
timeout: 600
|
|
script: python workloads/jobs_basic.py --working-dir "workloads"
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: compute_tpl_gce_4_xlarge.yaml
|
|
|
|
- name: jobs_basic_remote_working_dir
|
|
python: "3.10"
|
|
group: Jobs tests
|
|
working_dir: jobs_tests
|
|
|
|
frequency: nightly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_tpl_4_xlarge.yaml
|
|
|
|
run:
|
|
timeout: 600
|
|
script: python workloads/jobs_basic.py --working-dir "https://github.com/anyscale/job-services-cuj-examples/archive/refs/heads/main.zip"
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: compute_tpl_gce_4_xlarge.yaml
|
|
|
|
- name: jobs_remote_multi_node
|
|
python: "3.10"
|
|
group: Jobs tests
|
|
team: serve
|
|
frequency: nightly
|
|
working_dir: jobs_tests
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_tpl_4_xlarge.yaml
|
|
run:
|
|
timeout: 600
|
|
script: python workloads/jobs_remote_multi_node.py
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: compute_tpl_gce_4_xlarge.yaml
|
|
|
|
- name: jobs_check_cuda_available
|
|
python: "3.10"
|
|
group: Jobs tests
|
|
team: serve
|
|
|
|
frequency: nightly
|
|
working_dir: jobs_tests
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_tpl_gpu_node.yaml
|
|
run:
|
|
timeout: 600
|
|
script: python workloads/jobs_check_cuda_available.py
|
|
wait_for_nodes:
|
|
num_nodes: 2
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: compute_tpl_gce_gpu_node.yaml
|
|
- __suffix__: py310_cu130
|
|
python: "3.10"
|
|
cluster:
|
|
byod:
|
|
type: gpu-cu130
|
|
python_depset: gpu_cu130_py3.10.lock
|
|
cluster_compute: compute_tpl_gpu_node.yaml
|
|
- __suffix__: py311_cu130
|
|
python: "3.11"
|
|
cluster:
|
|
byod:
|
|
type: gpu-cu130
|
|
python_depset: gpu_cu130_py3.11.lock
|
|
cluster_compute: compute_tpl_gpu_node.yaml
|
|
- __suffix__: py312_cu130
|
|
python: "3.12"
|
|
cluster:
|
|
byod:
|
|
type: gpu-cu130
|
|
python_depset: gpu_cu130_py3.12.lock
|
|
cluster_compute: compute_tpl_gpu_node.yaml
|
|
- __suffix__: py313_cu130
|
|
python: "3.13"
|
|
cluster:
|
|
byod:
|
|
type: gpu-cu130
|
|
python_depset: gpu_cu130_py3.13.lock
|
|
cluster_compute: compute_tpl_gpu_node.yaml
|
|
|
|
- name: "jobs_check_cuda_version_cu130_py{{python}}"
|
|
python: "{{python}}"
|
|
group: Jobs tests
|
|
team: serve
|
|
frequency: nightly
|
|
working_dir: jobs_tests
|
|
matrix:
|
|
setup:
|
|
python: ["3.10", "3.11", "3.12", "3.13"]
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu-cu130
|
|
python_depset: gpu_cu130_py{{python}}.lock
|
|
cluster_compute: compute_tpl_gpu_node.yaml
|
|
run:
|
|
timeout: 600
|
|
script: python workloads/jobs_check_cuda_version.py --expected-cuda-version 13.0
|
|
wait_for_nodes:
|
|
num_nodes: 2
|
|
|
|
- name: jobs_specify_num_gpus
|
|
python: "3.10"
|
|
group: Jobs tests
|
|
team: serve
|
|
|
|
frequency: nightly
|
|
working_dir: jobs_tests
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_tpl_gpu_worker.yaml
|
|
run:
|
|
timeout: 600
|
|
script: python workloads/jobs_specify_num_gpus.py --working-dir "workloads"
|
|
wait_for_nodes:
|
|
num_nodes: 2
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: compute_tpl_gce_gpu_worker.yaml
|
|
|
|
########################
|
|
# Runtime env tests
|
|
########################
|
|
- name: runtime_env_rte_many_tasks_actors
|
|
python: "3.10"
|
|
group: Runtime env tests
|
|
working_dir: runtime_env_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: rte_small.yaml
|
|
|
|
run:
|
|
timeout: 600
|
|
script: python workloads/rte_many_tasks_actors.py
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: rte_gce_small.yaml
|
|
|
|
|
|
- name: runtime_env_wheel_urls
|
|
python: "3.10"
|
|
group: Runtime env tests
|
|
working_dir: runtime_env_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: rte_minimal.yaml
|
|
|
|
run:
|
|
timeout: 9000
|
|
script: python workloads/wheel_urls.py
|
|
wait_for_nodes:
|
|
num_nodes: 1
|
|
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: rte_gce_minimal.yaml
|
|
|
|
# It seems like the consensus is that this should be tested in CI, and not in a nightly test.
|
|
|
|
# - name: runtime_env_rte_ray_client
|
|
# group: Runtime env tests
|
|
# working_dir: runtime_env_tests
|
|
|
|
# frequency: nightly
|
|
# team: core
|
|
|
|
# cluster:
|
|
# cluster_compute: rte_minimal.yaml
|
|
|
|
# run:
|
|
# timeout: 600
|
|
# script: python workloads/rte_ray_client.py
|
|
# wait_for_nodes:
|
|
# num_nodes: 1
|
|
|
|
# alert: default
|
|
|
|
|
|
########################
|
|
# Serve tests
|
|
########################
|
|
|
|
- name: pytest_serve_scale_replicas
|
|
python: "3.10"
|
|
group: Serve tests
|
|
working_dir: serve_tests
|
|
|
|
frequency: nightly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_tpl_single_node_32_cpu.yaml
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 7200
|
|
long_running: false
|
|
script: python workloads/replica_scalability.py
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: aws.haproxy
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
|
|
- name: pytest_serve_multi_deployment_1k_noop_replica
|
|
python: "3.10"
|
|
group: Serve tests
|
|
working_dir: serve_tests
|
|
|
|
frequency: weekly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_tpl_32_cpu.yaml
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 7200
|
|
long_running: false
|
|
script: python workloads/multi_deployment_1k_noop_replica.py
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: aws.py312
|
|
python: "3.12"
|
|
- __suffix__: aws.haproxy
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
project_id: prj_xqmpk8ps6civt438u1hp5pi88g
|
|
cloud_id: cld_vy7xqacrvddvbuy95auinvuqmt
|
|
cluster_compute: compute_tpl_32_cpu_gce.yaml
|
|
|
|
- name: pytest_serve_autoscaling_load_test
|
|
python: "3.10"
|
|
group: Serve tests
|
|
working_dir: serve_tests
|
|
|
|
frequency: nightly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_tpl_single_node_32_cpu.yaml
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 7200
|
|
long_running: false
|
|
script: python workloads/autoscaling_load_test.py
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: aws.haproxy
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
|
|
- name: serve_controller_benchmark_haproxy
|
|
python: "3.10"
|
|
group: Serve tests
|
|
working_dir: serve_tests
|
|
|
|
frequency: weekly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
# specifically testing haproxy here because with serve
|
|
# proxy the benchmark profile changes significantly.
|
|
# Given that haproxy is going to be the path forward, make sense
|
|
# to use that here.
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
- RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S=0.01
|
|
cluster_compute: compute_tpl_8_cpu_autoscaling.yaml
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 14400
|
|
long_running: false
|
|
script: python workloads/microbenchmarks.py --run-controller
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
project_id: prj_xqmpk8ps6civt438u1hp5pi88g
|
|
cloud_id: cld_vy7xqacrvddvbuy95auinvuqmt
|
|
cluster_compute: compute_tpl_single_node_gce.yaml
|
|
|
|
- name: pytest_serve_microbenchmarks
|
|
python: "3.10"
|
|
group: Serve tests
|
|
working_dir: serve_tests
|
|
|
|
frequency: nightly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_tpl_single_node_16_cpu.yaml
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 7200
|
|
long_running: false
|
|
script: python workloads/microbenchmarks.py --run-all
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
project_id: prj_xqmpk8ps6civt438u1hp5pi88g
|
|
cloud_id: cld_vy7xqacrvddvbuy95auinvuqmt
|
|
cluster_compute: compute_tpl_single_node_gce.yaml
|
|
|
|
- name: pytest_serve_throughput_optimized_microbenchmarks
|
|
python: "3.10"
|
|
group: Serve tests
|
|
working_dir: serve_tests
|
|
|
|
frequency: nightly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RAY_SERVE_THROUGHPUT_OPTIMIZED=1
|
|
- RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S=0.01
|
|
- RAY_SERVE_REPLICA_GRPC_MAX_MESSAGE_LENGTH=10500000
|
|
cluster_compute: compute_tpl_single_node_16_cpu.yaml
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 7200
|
|
long_running: false
|
|
script: python workloads/microbenchmarks.py --run-all
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: aws.haproxy
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
- RAY_SERVE_THROUGHPUT_OPTIMIZED=1
|
|
- RAY_SERVE_DIRECT_INGRESS_MIN_DRAINING_PERIOD_S=0.01
|
|
- RAY_SERVE_REPLICA_GRPC_MAX_MESSAGE_LENGTH=10500000
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
run:
|
|
timeout: 7200
|
|
long_running: false
|
|
# HAProxy is HTTP-only and does not proxy the gRPC server bound by
|
|
# gRPCOptions(port=9000); --run-grpc would crash with connection refused
|
|
# under RAY_SERVE_ENABLE_HA_PROXY=1. Run the HAProxy-proxied transports
|
|
# only.
|
|
script: >-
|
|
python workloads/microbenchmarks.py
|
|
--run-http --run-handle --run-latency --run-throughput
|
|
--run-streaming
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
project_id: prj_xqmpk8ps6civt438u1hp5pi88g
|
|
cloud_id: cld_vy7xqacrvddvbuy95auinvuqmt
|
|
cluster_compute: compute_tpl_single_node_gce.yaml
|
|
|
|
- name: pytest_serve_router_benchmark
|
|
python: "3.10"
|
|
group: Serve tests
|
|
working_dir: serve_tests
|
|
|
|
frequency: weekly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_tpl_8_cpu_autoscaling.yaml
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 7200
|
|
long_running: false
|
|
# Every variation fully overrides this
|
|
script: "false"
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
# Low-latency / high-throughput workload
|
|
- __suffix__: aws.standard
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
- RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD=0
|
|
- RAY_SERVE_RUN_ROUTER_IN_SEPARATE_LOOP=0
|
|
run:
|
|
timeout: 7200
|
|
long_running: false
|
|
script: >-
|
|
python workloads/router_microbenchmark.py
|
|
--simulated-latency-mean-s 0.01
|
|
--simulated-latency-cap-s 0.1
|
|
--max-ongoing-requests-child 5
|
|
--max-ongoing-requests-parent 5
|
|
--max-users-per-task 24
|
|
--router-type pow2
|
|
# Supply-constrained workload
|
|
- __suffix__: aws.supply_constraint
|
|
run:
|
|
timeout: 7200
|
|
long_running: false
|
|
script: >-
|
|
python workloads/router_microbenchmark.py
|
|
--simulated-latency-mean-s 1
|
|
--simulated-latency-cap-s 10
|
|
--max-ongoing-requests-child 2
|
|
--max-ongoing-requests-parent 5
|
|
--router-type pow2
|
|
--router-type capacity_queue
|
|
|
|
- name: pytest_serve_resnet_benchmark
|
|
python: "3.10"
|
|
group: Serve tests
|
|
working_dir: serve_tests
|
|
|
|
frequency: nightly
|
|
team: serve
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_tpl_gpu_node.yaml
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 7200
|
|
long_running: false
|
|
script: python workloads/serve_resnet_benchmark.py --gpu-env
|
|
|
|
alert: default
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: aws.haproxy
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
project_id: prj_xqmpk8ps6civt438u1hp5pi88g
|
|
cloud_id: cld_vy7xqacrvddvbuy95auinvuqmt
|
|
cluster_compute: compute_tpl_gpu_node_gce.yaml
|
|
|
|
########################
|
|
# Train tests
|
|
########################
|
|
|
|
- name: training_ingest_benchmark-task=image_classification
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/benchmark
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
# Enable verbose stats for resource manager
|
|
- RAY_DATA_DEBUG_RESOURCE_MANAGER=1
|
|
# Fail the test if a worker OOMs
|
|
- RAYTEST_FAIL_ON_WORKER_OOM=1
|
|
|
|
# 'type: gpu' means: use the 'ray-ml' image.
|
|
type: gpu
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_configs/compute_gpu_4x4_aws.yaml
|
|
|
|
variations:
|
|
- __suffix__: full_training.parquet
|
|
run:
|
|
timeout: 2000
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --image_classification_data_format=parquet
|
|
|
|
- __suffix__: full_training.parquet.preserve_order
|
|
frequency: weekly
|
|
run:
|
|
timeout: 2000
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --image_classification_data_format=parquet --preserve_order=True
|
|
|
|
- __suffix__: full_training.parquet.torch_dataloader
|
|
frequency: manual
|
|
run:
|
|
timeout: 2000
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=torch --num_workers=16 --image_classification_data_format=parquet
|
|
|
|
- __suffix__: skip_training.parquet
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --skip_train_step=True --skip_validation_at_epoch_end=True --image_classification_data_format=parquet
|
|
|
|
- __suffix__: skip_training.parquet.preserve_order
|
|
frequency: weekly
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --skip_train_step=True --skip_validation_at_epoch_end=True --image_classification_data_format=parquet --preserve_order=True
|
|
|
|
- __suffix__: skip_training.parquet.torch_dataloader
|
|
frequency: manual
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=torch --num_workers=16 --skip_train_step=True --skip_validation_at_epoch_end=True --image_classification_data_format=parquet
|
|
|
|
- __suffix__: skip_training.parquet.slow_consumer
|
|
run:
|
|
timeout: 3000
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --skip_train_step=True --skip_validation_at_epoch_end=True --image_classification_data_format=parquet --train_step_sleep_s=5.0 --max_train_batches=500 --limit_training_rows=-1 --image_classification_use_1t_dataset=True
|
|
|
|
- __suffix__: skip_training.parquet.fault_tolerance
|
|
frequency: weekly
|
|
run:
|
|
timeout: 2700
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --mock_gpu=True --skip_train_step=True --skip_validation_step=True --num_epochs=5 --max_failures=4 --image_classification_data_format=parquet
|
|
prepare: python setup_chaos.py --kill-interval 480 --max-to-kill 2 --kill-delay 360
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
- RAY_DATA_DEBUG_RESOURCE_MANAGER=1
|
|
- RAYTEST_FAIL_ON_WORKER_OOM=0 # Workers are intentionally killed
|
|
cluster_compute: compute_configs/compute_mock_gpu_4x4_aws.yaml
|
|
|
|
- __suffix__: full_training.s3_url
|
|
run:
|
|
timeout: 2000
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --image_classification_data_format=s3_url
|
|
|
|
- __suffix__: skip_training.s3_url
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --skip_train_step True --skip_validation_at_epoch_end True --image_classification_data_format=s3_url
|
|
|
|
- __suffix__: full_training.jpeg
|
|
run:
|
|
timeout: 2000
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --dataloader_type=ray_data --num_workers=16
|
|
|
|
- __suffix__: full_training.jpeg.preserve_order
|
|
frequency: weekly
|
|
run:
|
|
timeout: 2000
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --dataloader_type=ray_data --num_workers=16 --preserve_order=True
|
|
|
|
- __suffix__: full_training.jpeg.torch_dataloader
|
|
frequency: manual
|
|
run:
|
|
timeout: 2000
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --dataloader_type=torch --num_workers=16 --num_torch_workers=16
|
|
|
|
- __suffix__: skip_training.jpeg
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --dataloader_type=ray_data --num_workers=16 --skip_train_step=True --skip_validation_at_epoch_end=True
|
|
|
|
- __suffix__: skip_training.jpeg.preserve_order
|
|
frequency: weekly
|
|
run:
|
|
timeout: 2400
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --dataloader_type=ray_data --num_workers=16 --skip_train_step=True --skip_validation_at_epoch_end=True --preserve_order=True
|
|
|
|
- __suffix__: skip_training.jpeg.torch_dataloader
|
|
frequency: manual
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --dataloader_type=torch --num_workers=16 --num_torch_workers=16 --skip_train_step=True --skip_validation_at_epoch_end=True
|
|
|
|
- __suffix__: skip_training.jpeg.local_fs
|
|
run:
|
|
timeout: 1200
|
|
script: bash image_classification/jpeg/download_input_data_from_s3.sh && RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --image_classification_local_dataset=True --dataloader_type=ray_data --num_workers=1 --skip_train_step=True --skip_validation_at_epoch_end=True
|
|
cluster:
|
|
cluster_compute: compute_configs/compute_gpu_1x1_aws.yaml
|
|
|
|
- __suffix__: skip_training.jpeg.local_fs.preserve_order
|
|
frequency: manual
|
|
run:
|
|
timeout: 1200
|
|
script: bash image_classification/jpeg/download_input_data_from_s3.sh && RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --image_classification_local_dataset=True --dataloader_type=ray_data --num_workers=1 --skip_train_step=True --skip_validation_at_epoch_end=True --preserve_order=True
|
|
cluster:
|
|
cluster_compute: compute_configs/compute_gpu_1x1_aws.yaml
|
|
|
|
- __suffix__: skip_training.jpeg.local_fs.torch_dataloader
|
|
frequency: manual
|
|
run:
|
|
timeout: 1200
|
|
script: bash image_classification/jpeg/download_input_data_from_s3.sh && RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --image_classification_local_dataset=True --dataloader_type=torch --num_workers=1 --num_torch_workers=32 --skip_train_step=True --skip_validation_at_epoch_end=True
|
|
cluster:
|
|
cluster_compute: compute_configs/compute_gpu_1x1_aws.yaml
|
|
|
|
- __suffix__: skip_training.jpeg.local_fs_multi_gpus
|
|
run:
|
|
timeout: 1200
|
|
script: bash image_classification/jpeg/download_input_data_from_s3.sh && RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --image_classification_local_dataset=True --dataloader_type=ray_data --num_workers=4 --skip_train_step=True --skip_validation_at_epoch_end=True
|
|
cluster:
|
|
cluster_compute: compute_configs/compute_gpu_1x4_aws.yaml
|
|
|
|
- __suffix__: skip_training.jpeg.local_fs_multi_gpus.preserve_order
|
|
frequency: manual
|
|
run:
|
|
timeout: 1200
|
|
script: bash image_classification/jpeg/download_input_data_from_s3.sh && RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --image_classification_local_dataset=True --dataloader_type=ray_data --num_workers=4 --skip_train_step=True --skip_validation_at_epoch_end=True --preserve_order=True
|
|
cluster:
|
|
cluster_compute: compute_configs/compute_gpu_1x4_aws.yaml
|
|
|
|
- __suffix__: skip_training.jpeg.local_fs_multi_gpus.torch_dataloader
|
|
frequency: manual
|
|
run:
|
|
timeout: 1200
|
|
script: bash image_classification/jpeg/download_input_data_from_s3.sh && RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --image_classification_data_format=jpeg --image_classification_local_dataset=True --dataloader_type=torch --num_workers=4 --num_torch_workers=9 --skip_train_step=True --skip_validation_at_epoch_end=True
|
|
cluster:
|
|
cluster_compute: compute_configs/compute_gpu_1x4_aws.yaml
|
|
|
|
- name: training_ingest_benchmark-task=image_classification-heterogenous_fixed_size
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/benchmark
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
# Enable verbose stats for resource manager
|
|
- RAY_DATA_DEBUG_RESOURCE_MANAGER=1
|
|
# Fail the test if a worker OOMs
|
|
- RAYTEST_FAIL_ON_WORKER_OOM=1
|
|
# Fail the test if a node dies
|
|
- RAYTEST_FAIL_ON_DEAD_NODES=1
|
|
|
|
# 'type: gpu' means: use the 'ray-ml' image.
|
|
type: gpu
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_configs/heterogenous_fixed_size_gpu_4x4_aws.yaml
|
|
|
|
variations:
|
|
- __suffix__: skip_training.parquet
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --skip_train_step True --skip_validation_at_epoch_end True --image_classification_data_format=parquet
|
|
|
|
- __suffix__: skip_training.s3_url
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --skip_train_step True --skip_validation_at_epoch_end True --image_classification_data_format=s3_url
|
|
|
|
- name: training_ingest_benchmark-task=image_classification-heterogenous_autoscaling
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/benchmark
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
# Enable verbose stats for resource manager
|
|
- RAY_DATA_DEBUG_RESOURCE_MANAGER=1
|
|
# Fail the test if a worker OOMs
|
|
- RAYTEST_FAIL_ON_WORKER_OOM=1
|
|
# Fail the test if a node dies
|
|
- RAYTEST_FAIL_ON_DEAD_NODES=1
|
|
|
|
# 'type: gpu' means: use the 'ray-ml' image.
|
|
type: gpu
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_configs/heterogenous_autoscaling_gpu_4x4_aws.yaml
|
|
|
|
variations:
|
|
- __suffix__: skip_training.parquet
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --skip_train_step True --skip_validation_at_epoch_end True --image_classification_data_format=parquet
|
|
|
|
- __suffix__: skip_training.s3_url
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --skip_train_step True --skip_validation_at_epoch_end True --image_classification_data_format=s3_url
|
|
|
|
- name: training_ingest_benchmark-task=recsys
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/benchmark
|
|
|
|
# TODO: Keep these manual for now -- there are a still a few issues to resolve.
|
|
frequency: manual
|
|
team: ml
|
|
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
# Fail the test if a worker OOMs
|
|
- RAYTEST_FAIL_ON_WORKER_OOM=1
|
|
type: gpu
|
|
post_build_script: byod_install_torchrec.sh
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_configs/compute_gpu_8xA100_aws.yaml
|
|
|
|
variations:
|
|
- __suffix__: skip_training.mock_dataloader
|
|
run:
|
|
timeout: 600
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=recsys --dataloader_type=mock --num_workers=8 --train_batch_size=8192 --num_epochs=1 --skip_train_step=True --skip_validation_at_epoch_end=True
|
|
|
|
- __suffix__: skip_training.ray_data
|
|
run:
|
|
timeout: 600
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=recsys --dataloader_type=ray_data --num_workers=8 --train_batch_size=8192 --num_epochs=1 --skip_train_step=True --skip_validation_at_epoch_end=True
|
|
|
|
- __suffix__: full_training.mock_dataloader
|
|
run:
|
|
timeout: 600
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=recsys --dataloader_type=mock --num_workers=8 --train_batch_size=8192 --validation_batch_size=16384 --num_epochs=1
|
|
|
|
- __suffix__: full_training.ray_data
|
|
run:
|
|
timeout: 1200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=recsys --dataloader_type=ray_data --num_workers=8 --train_batch_size=8192 --validation_batch_size=16384 --num_epochs=1
|
|
|
|
- name: training_ingest_benchmark-soak_test
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/benchmark
|
|
|
|
frequency: weekly
|
|
team: ml
|
|
|
|
cluster:
|
|
byod:
|
|
type: gpu
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_configs/compute_gpu_4x4_cpu_4_aws.yaml
|
|
|
|
run:
|
|
timeout: 43200
|
|
long_running: true
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_benchmark.py --task=image_classification --dataloader_type=ray_data --num_workers=16 --image_classification_data_format=parquet --num_epochs=50
|
|
|
|
- name: train_multinode_persistence
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/multinode_persistence
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
post_build_script: byod_train_persistence_test.sh
|
|
cluster_compute: compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 3000
|
|
script: pytest -v test_persistence.py -s
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
variations:
|
|
- __suffix__: aws_v1
|
|
- __suffix__: aws_v2
|
|
run:
|
|
script: RAY_TRAIN_V2_ENABLED=1 pytest -v test_persistence.py -s
|
|
- __suffix__: gce_v1
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: compute_gce.yaml
|
|
- __suffix__: gce_v2
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: compute_gce.yaml
|
|
run:
|
|
script: RAY_TRAIN_V2_ENABLED=1 pytest -v test_persistence.py -s
|
|
|
|
alert: default
|
|
|
|
- name: train_async_checkpointing_validation_benchmark
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/async_checkpointing_validation_benchmark
|
|
frequency: nightly
|
|
team: ml
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_aws.yaml
|
|
run:
|
|
timeout: 10800
|
|
script: RAY_TRAIN_V2_ENABLED=1 python test_async_checkpointing_validation_benchmark.py
|
|
wait_for_nodes:
|
|
num_nodes: 2
|
|
alert: default
|
|
|
|
- name: train_colocate_trainer
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/colocate_trainer
|
|
|
|
# Ray Train V2 doesn't support colocation.
|
|
# TODO: Decide whether to remove this test or re-enable it if we add support again.
|
|
frequency: manual
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 3000
|
|
script: pytest -v test_colocate_trainer.py -s
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
alert: default
|
|
|
|
|
|
- name: torch_local_mode
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/local_mode
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
byod:
|
|
type: gpu
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_gpu_2x4_aws.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: python torch_local_mode_launcher.py
|
|
|
|
# Reference: how to run a release test on the torchft Ray ML image variant.
|
|
# The custom BYOD image is the core Ray CUDA image (py3.13) with the torchft ML
|
|
# dependency lock (release/ray_release/byod/ml_torchft_py3.13.lock, generated by
|
|
# ci/raydepsets/configs/release_ml_torchft_tests.depsets.yaml) installed on top.
|
|
# Setting `python_depset` is enough: the BYOD build automatically runs
|
|
# `uv pip install --system --no-deps -r python_depset.lock`, so no
|
|
# post_build_script is needed to install the deps.
|
|
- name: torchft_hello_world
|
|
python: "3.13"
|
|
group: Train tests
|
|
team: ml
|
|
frequency: manual
|
|
working_dir: train_tests/torchft
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: cu123
|
|
python_depset: ml_torchft_py3.13.lock
|
|
cluster_compute: compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: python torchft_hello_world.py
|
|
|
|
- name: torch_lightning
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/pytorch_lightning
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
post_build_script: byod_pytorch_lightning_test.sh
|
|
cluster_compute: compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: RAY_TRAIN_V2_ENABLED=1 python test_lightning.py
|
|
|
|
- name: huggingface_transformers
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/huggingface_transformers
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
post_build_script: byod_huggingface_transformers_test.sh
|
|
cluster_compute: compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: RAY_TRAIN_V2_ENABLED=1 python test_huggingface_transformers.py
|
|
|
|
- name: huggingface_accelerate
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/huggingface_accelerate
|
|
|
|
frequency: nightly
|
|
team: ml
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: compute_aws.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: RAY_TRAIN_V2_ENABLED=1 python test_huggingface_accelerate.py
|
|
|
|
- name: "{{framework}}_train_batch_inference_benchmark_{{size}}"
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/xgboost_lightgbm
|
|
|
|
matrix:
|
|
setup:
|
|
framework: [xgboost, lightgbm]
|
|
size: [10G, 100G]
|
|
|
|
frequency: nightly-3x
|
|
team: ml
|
|
cluster:
|
|
byod:
|
|
type: gpu
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_aws_{{size}}.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: RAY_TRAIN_V2_ENABLED=1 python train_batch_inference_benchmark.py "{{framework}}" --size={{size}}
|
|
|
|
smoke_test:
|
|
frequency: manual
|
|
|
|
run:
|
|
timeout: 1800
|
|
|
|
alert: default
|
|
|
|
- name: train_elastic_e2e_with_node_killing
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/elastic_training
|
|
|
|
frequency: weekly
|
|
team: ml
|
|
|
|
cluster:
|
|
byod:
|
|
type: gpu
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_gpu_aws.yaml
|
|
|
|
run:
|
|
timeout: 4800
|
|
script: RAY_TRAIN_V2_ENABLED=1 python torch_elastic_e2e.py run --num-epochs 60
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 1
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
alert: default
|
|
|
|
- name: elastic_benchmark
|
|
python: "3.10"
|
|
group: Train tests
|
|
working_dir: train_tests/benchmark
|
|
|
|
frequency: weekly
|
|
team: ml
|
|
|
|
cluster:
|
|
byod:
|
|
type: gpu
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_configs/compute_gpu_8x4_aws.yaml
|
|
|
|
run:
|
|
timeout: 7200
|
|
script: RAY_TRAIN_V2_ENABLED=1 python -m elastic_training.elastic_e2e --task="image_classification" --image_classification_data_format="parquet" --dataloader_type=ray_data --checkpoint_every_n_steps=100 --min_workers=8 --max_workers=32 --num_epochs=4
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 1
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
alert: default
|
|
|
|
###############
|
|
# RLlib tests #
|
|
###############
|
|
|
|
# -----------------
|
|
# RLlib - APPO test
|
|
# -----------------
|
|
- name: rllib_learning_tests_pong_appo_torch
|
|
python: "3.10"
|
|
group: RLlib tests
|
|
working_dir: rllib_tests
|
|
|
|
frequency: nightly
|
|
team: rllib
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
post_build_script: byod_rllib.sh
|
|
runtime_env:
|
|
- RLLIB_TEST_NO_JAX_IMPORT=1
|
|
cluster_compute: 1gpu_16cpus.yaml
|
|
|
|
run:
|
|
timeout: 1500 # expected 1000 seconds
|
|
script: python example_algorithms/appo/pong_appo.py --num-learners=1 --num-env-runners=12 --as-release-test
|
|
|
|
########################
|
|
# Core Nightly Tests
|
|
########################
|
|
|
|
- name: shuffle_100gb
|
|
python: "3.10"
|
|
group: core-multi-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: shuffle/shuffle_compute_multi.yaml
|
|
|
|
run:
|
|
timeout: 3000
|
|
script: python shuffle/shuffle_test.py --num-partitions=200 --partition-size=500e6
|
|
wait_for_nodes:
|
|
num_nodes: 4
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: shuffle/shuffle_compute_multi_gce.yaml
|
|
|
|
|
|
- name: stress_test_placement_group
|
|
python: "3.10"
|
|
group: core-multi-test
|
|
working_dir: nightly_tests
|
|
env: aws_perf
|
|
|
|
frequency: nightly
|
|
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/placement_group_tests_compute.yaml
|
|
|
|
run:
|
|
timeout: 7200
|
|
script: python stress_tests/test_placement_group.py
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/placement_group_tests_compute_gce.yaml
|
|
|
|
- name: autoscaling_shuffle_1tb_1000_partitions
|
|
python: "3.10"
|
|
group: core-multi-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: shuffle/shuffle_compute_autoscaling.yaml
|
|
|
|
run:
|
|
timeout: 4000
|
|
script: python shuffle/shuffle_test.py --num-partitions=1000 --partition-size=1e9
|
|
--no-streaming
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: shuffle/shuffle_compute_autoscaling_gce.yaml
|
|
|
|
- name: microbenchmark
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
team: core
|
|
frequency: nightly
|
|
env: aws_perf
|
|
working_dir: microbenchmark
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_64.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: OMP_NUM_THREADS=64 RAY_ADDRESS=local python run_microbenchmark.py
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
repeated_run: 5
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_64_gce.yaml
|
|
- __suffix__: aws.py312
|
|
frequency: weekly
|
|
python: "3.12"
|
|
|
|
- name: compiled_graphs
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
team: core
|
|
frequency: manual
|
|
working_dir: microbenchmark
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: tpl_64.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: OMP_NUM_THREADS=64 RAY_ADDRESS=local python run_microbenchmark.py --experimental
|
|
|
|
- name: compiled_graphs_GPU
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
team: core
|
|
frequency: manual
|
|
working_dir: microbenchmark
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: experimental/compute_t4_gpu.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: python experimental/compiled_graph_gpu_microbenchmark.py
|
|
|
|
- name: compiled_graphs_GPU_multinode
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
team: core
|
|
frequency: manual
|
|
working_dir: microbenchmark
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: experimental/compute_gpu_2x1_aws.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: python experimental/compiled_graph_gpu_microbenchmark.py --distributed
|
|
|
|
- name: "compiled_graphs_GPU_cu130_py{{python}}"
|
|
python: "{{python}}"
|
|
group: core-daily-test
|
|
team: core
|
|
frequency: manual
|
|
working_dir: microbenchmark
|
|
matrix:
|
|
setup:
|
|
python: ["3.10", "3.11", "3.12", "3.13"]
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu-cu130
|
|
python_depset: gpu_cu130_py{{python}}.lock
|
|
cluster_compute: experimental/compute_l4_gpu.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: python experimental/compiled_graph_gpu_microbenchmark.py
|
|
|
|
- name: "compiled_graphs_GPU_multinode_cu130_py{{python}}"
|
|
python: "{{python}}"
|
|
group: core-daily-test
|
|
team: core
|
|
frequency: manual
|
|
working_dir: microbenchmark
|
|
matrix:
|
|
setup:
|
|
python: ["3.10", "3.11", "3.12", "3.13"]
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu-cu130
|
|
python_depset: gpu_cu130_py{{python}}.lock
|
|
cluster_compute: experimental/compute_l4_gpu_2x1_aws.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: python experimental/compiled_graph_gpu_microbenchmark.py --distributed
|
|
|
|
- name: rdt_single_node_T4_microbenchmark
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
team: core
|
|
frequency: nightly
|
|
working_dir: microbenchmark
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: experimental/compute_t4_gpu.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
# NIXL currently only works on T4's for <100mb tensors so not enabling.
|
|
script: python experimental/rdt_single_node_microbenchmark.py
|
|
|
|
- name: rdt_single_node_A100_microbenchmark
|
|
python: "3.12"
|
|
group: core-daily-test
|
|
team: core
|
|
frequency: weekly
|
|
working_dir: microbenchmark
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: llm-cu130 # llm image already has nixl
|
|
post_build_script: byod_llm_pin_cupy_cuda12x.sh # match cupy-cuda12x to ray-llm base lock (BYOD testdeps can float major)
|
|
runtime_env: # env vars for nixl
|
|
- UCX_TLS=all
|
|
- UCX_NET_DEVICES=all
|
|
cluster_compute: experimental/compute_a100_gpu.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: python experimental/rdt_single_node_microbenchmark.py --enable_10gb --enable_nixl
|
|
|
|
# TODO(dayshah): Add multi-node benchmarks with actual hardware RDMA support
|
|
# - name: rdt_GPU_multinode
|
|
# python: "3.10"
|
|
# group: core-daily-test
|
|
# team: core
|
|
# frequency: nightly
|
|
# working_dir: microbenchmark
|
|
|
|
# cluster:
|
|
# byod:
|
|
# type: gpu
|
|
# cluster_compute: experimental/compute_gpu_2x1_aws.yaml
|
|
|
|
# run:
|
|
# timeout: 1800
|
|
# script: python experimental/rdt_microbenchmark.py --distributed
|
|
|
|
- name: benchmark_worker_startup
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
team: core
|
|
frequency: nightly
|
|
working_dir: benchmark-worker-startup
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
cluster_compute: only_head_node_1gpu_64cpu.yaml
|
|
|
|
run:
|
|
timeout: 7200
|
|
script: python benchmark_worker_startup.py
|
|
--num_cpus_in_cluster 64
|
|
--num_gpus_in_cluster 64
|
|
--num_tasks_or_actors_per_run 64
|
|
--num_measurements_per_configuration 5
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: only_head_node_1gpu_64cpu_gce.yaml
|
|
|
|
- name: dask_on_ray_100gb_sort
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: manual # was nightly
|
|
team: core
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: dask_on_ray/dask_on_ray_sort_compute_template.yaml
|
|
|
|
run:
|
|
timeout: 7200
|
|
script: python dask_on_ray/dask_on_ray_sort.py --nbytes 100_000_000_000 --npartitions
|
|
200 --num-nodes 1 --ray --data-dir /tmp/ray --file-path /tmp/ray
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: dask_on_ray/dask_on_ray_sort_compute_template_gce.yaml
|
|
|
|
|
|
- name: stress_test_state_api_scale
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RAY_MAX_LIMIT_FROM_API_SERVER=1000000000
|
|
- RAY_MAX_LIMIT_FROM_DATA_SOURCE=1000000000
|
|
cluster_compute: stress_tests/stress_tests_compute_large.yaml
|
|
|
|
run:
|
|
timeout: 4200
|
|
script: python stress_tests/test_state_api_scale.py
|
|
wait_for_nodes:
|
|
num_nodes: 7
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
app_config: stress_tests/state_api_app_config.yaml
|
|
cluster_compute: stress_tests/smoke_test_compute.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 5
|
|
script: python stress_tests/test_state_api_scale.py --smoke-test
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: aws.py312
|
|
frequency: manual
|
|
python: "3.12"
|
|
smoke_test:
|
|
frequency: nightly-3x
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/stress_tests_compute_large_gce.yaml
|
|
smoke_test:
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: stress_tests/smoke_test_compute_gce.yaml
|
|
|
|
|
|
- name: shuffle_20gb_with_state_api
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RAY_MAX_LIMIT_FROM_API_SERVER=1000000000
|
|
- RAY_MAX_LIMIT_FROM_DATA_SOURCE=1000000000
|
|
cluster_compute: shuffle/shuffle_compute_single.yaml
|
|
|
|
run:
|
|
timeout: 1000
|
|
script: python stress_tests/test_state_api_with_other_tests.py
|
|
nightly_tests/shuffle/shuffle_test.py --test-args="--num-partitions=100 --partition-size=200e6"
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: shuffle/shuffle_compute_single_gce.yaml
|
|
|
|
- name: stress_test_many_tasks
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
env: aws_perf
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/stress_tests_compute.yaml
|
|
|
|
run:
|
|
timeout: 14400
|
|
wait_for_nodes:
|
|
num_nodes: 101
|
|
|
|
script: python stress_tests/test_many_tasks.py
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
app_config: stress_tests/stress_tests_app_config.yaml
|
|
cluster_compute: stress_tests/smoke_test_compute.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 5
|
|
script: python stress_tests/test_many_tasks.py --num-nodes=4 --smoke-test
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/stress_tests_compute_gce.yaml
|
|
smoke_test:
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: stress_tests/smoke_test_compute_gce.yaml
|
|
|
|
- name: stress_test_dead_actors
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
env: aws_perf
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/stress_tests_compute.yaml
|
|
|
|
run:
|
|
timeout: 7200
|
|
wait_for_nodes:
|
|
num_nodes: 101
|
|
|
|
script: python stress_tests/test_dead_actors.py
|
|
|
|
smoke_test:
|
|
frequency: nightly
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
app_config: stress_tests/stress_tests_app_config.yaml
|
|
cluster_compute: stress_tests/smoke_test_compute.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 5
|
|
script: python stress_tests/test_dead_actors.py --num-nodes=4 --num-parents=3
|
|
--num-children=3
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/stress_tests_compute_gce.yaml
|
|
smoke_test:
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: stress_tests/smoke_test_compute_gce.yaml
|
|
|
|
# The full test is not stable, so run the smoke test only.
|
|
# See https://github.com/ray-project/ray/issues/23244.
|
|
- name: threaded_actors_stress_test
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/smoke_test_compute.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python stress_tests/test_threaded_actors.py --test-runtime 1800 --kill-interval_s
|
|
30
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 5
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/smoke_test_compute_gce.yaml
|
|
|
|
# - name: threaded_actors_stress_test
|
|
# group: core-daily-test
|
|
# working_dir: nightly_tests
|
|
#
|
|
# frequency: nightly
|
|
# team: core
|
|
# cluster:
|
|
# cluster_compute: stress_tests/stress_test_threaded_actor_compute.yaml
|
|
#
|
|
# run:
|
|
# timeout: 7200
|
|
# script: python stress_tests/test_threaded_actors.py --test-runtime 3600 --kill-interval_s
|
|
# 60
|
|
#
|
|
# wait_for_nodes:
|
|
# num_nodes: 201
|
|
# timeout: 600
|
|
#
|
|
# smoke_test:
|
|
# frequency: nightly
|
|
# cluster:
|
|
# app_config: stress_tests/stress_tests_app_config.yaml
|
|
# cluster_compute: stress_tests/smoke_test_compute.yaml
|
|
#
|
|
# run:
|
|
# timeout: 3600
|
|
# script: python stress_tests/test_threaded_actors.py --test-runtime 1800 --kill-interval_s
|
|
# 30
|
|
#
|
|
# wait_for_nodes:
|
|
# num_nodes: 5
|
|
# timeout: 600
|
|
|
|
- name: stress_test_many_runtime_envs
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/smoke_test_compute.yaml
|
|
|
|
run:
|
|
timeout: 14400
|
|
wait_for_nodes:
|
|
num_nodes: 5
|
|
script: python stress_tests/test_many_runtime_envs.py --num_runtime_envs=100 --num_tasks=10000
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/smoke_test_compute_gce.yaml
|
|
smoke_test:
|
|
frequency: manual
|
|
|
|
- name: single_node_oom
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
env: aws_perf
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
# Lower the memory usage threshold at which the Ray memory monitor will kill
|
|
# running tasks so that it does not compete with the system OOM killer.
|
|
- RAY_memory_usage_threshold=0.7
|
|
cluster_compute: stress_tests/stress_tests_single_node_oom_compute.yaml
|
|
|
|
run:
|
|
# The script parameters are tuned to run for ~30min.
|
|
timeout: 3600
|
|
# The memory allocated per task is tuned along with the `RAY_memory_usage_threshold` setting above
|
|
# to trigger the Ray memory monitor to kick in before the Ray worker cgroup OOMs.
|
|
script: python stress_tests/test_parallel_tasks_memory_pressure.py --num-tasks 64 --mem-pct-per-task .5
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: stress_tests/stress_tests_single_node_oom_compute_gce.yaml
|
|
|
|
|
|
- name: dask_on_ray_1tb_sort
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: manual # was nightly-3x
|
|
team: core
|
|
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: dask_on_ray/1tb_sort_compute.yaml
|
|
|
|
run:
|
|
timeout: 7200
|
|
script: python dask_on_ray/dask_on_ray_sort.py --nbytes 1_000_000_000_000 --npartitions
|
|
1000 --num-nodes 31 --ray --data-dir /tmp/ray --s3-bucket core-nightly-test
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 32
|
|
|
|
|
|
- name: many_nodes_actor_test_on_v2
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly-3x
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: distributed/many_nodes_tests/compute_config.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
# 2cpus per node x 1000 nodes / 0.2 cpus per actor = 10k
|
|
# 2cpus per node x 2000 nodes / 0.2 cpus per actor = 20k
|
|
script: python distributed/many_nodes_tests/actor_test.py --no-wait --cpus-per-actor=0.2 --total-actors 10000 20000
|
|
wait_for_nodes:
|
|
num_nodes: 500
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: distributed/many_nodes_tests/compute_config_gce.yaml
|
|
|
|
#- name: many_nodes_multi_master_test
|
|
# group: core-daily-test
|
|
# working_dir: nightly_tests
|
|
#
|
|
# frequency: nightly-3x
|
|
# team: core
|
|
# cluster:
|
|
# cluster_compute: many_nodes_tests/compute_config.yaml
|
|
#
|
|
# run:
|
|
# timeout: 7200
|
|
# script: python many_nodes_tests/multi_master_test.py
|
|
# wait_for_nodes:
|
|
# num_nodes: 251
|
|
#
|
|
|
|
- name: pg_autoscaling_regression_test
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: placement_group_tests/compute.yaml
|
|
|
|
run:
|
|
timeout: 1200
|
|
script: python placement_group_tests/pg_run.py
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: placement_group_tests/compute_gce.yaml
|
|
|
|
- name: placement_group_performance_test
|
|
python: "3.10"
|
|
group: core-daily-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: placement_group_tests/pg_perf_test_compute.yaml
|
|
|
|
run:
|
|
timeout: 1200
|
|
script: python placement_group_tests/placement_group_performance_test.py
|
|
wait_for_nodes:
|
|
num_nodes: 5
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: placement_group_tests/pg_perf_test_compute_gce.yaml
|
|
|
|
|
|
#########################
|
|
# Core Scalability Tests
|
|
#########################
|
|
|
|
- name: single_node
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly
|
|
team: core
|
|
env: aws_perf
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: single_node.yaml
|
|
anyscale_sdk_2026: true
|
|
|
|
run:
|
|
timeout: 12000
|
|
prepare: sleep 0
|
|
script: python single_node/test_single_node.py
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: single_node_gce.yaml
|
|
|
|
- name: object_store
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly
|
|
team: core
|
|
env: aws_perf
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: object_store.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python object_store/test_object_store.py
|
|
wait_for_nodes:
|
|
num_nodes: 50
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: object_store_gce.yaml
|
|
|
|
- name: callback_throughput
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly
|
|
team: core
|
|
env: aws_perf
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: object_store/callback_throughput.yaml
|
|
|
|
run:
|
|
timeout: 600
|
|
script: python object_store/test_callback_throughput.py
|
|
wait_for_nodes:
|
|
num_nodes: 11
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: small_objects
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly
|
|
team: core
|
|
env: aws_perf
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: object_store/small_objects.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python object_store/test_small_objects.py
|
|
wait_for_nodes:
|
|
num_nodes: 5
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: large_objects
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly
|
|
team: core
|
|
env: aws_perf
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: object_store/large_objects.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python object_store/test_large_objects.py
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: many_actors
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly-3x
|
|
team: core
|
|
env: aws_perf
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: distributed.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python distributed/test_many_actors.py
|
|
wait_for_nodes:
|
|
num_nodes: 65
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: distributed_gce.yaml
|
|
|
|
- name: many_actors_smoke_test
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: distributed_smoke_test.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: SMOKE_TEST=1 python distributed/test_many_actors.py
|
|
wait_for_nodes:
|
|
num_nodes: 2
|
|
|
|
|
|
- name: many_tasks
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly
|
|
team: core
|
|
env: aws_perf
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: distributed.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python distributed/test_many_tasks.py --num-tasks=10000
|
|
wait_for_nodes:
|
|
num_nodes: 65
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: distributed_gce.yaml
|
|
|
|
- name: many_pgs
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly-3x
|
|
team: core
|
|
env: aws_perf
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: distributed.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python distributed/test_many_pgs.py
|
|
wait_for_nodes:
|
|
num_nodes: 65
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: distributed_gce.yaml
|
|
|
|
|
|
- name: many_pgs_smoke_test
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: distributed_smoke_test.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: SMOKE_TEST=1 python distributed/test_many_pgs.py
|
|
wait_for_nodes:
|
|
num_nodes: 2
|
|
|
|
|
|
- name: many_nodes
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly-3x
|
|
team: core
|
|
env: aws_perf
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: many_nodes.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python distributed/test_many_tasks.py --num-tasks=1000
|
|
wait_for_nodes:
|
|
num_nodes: 250
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: many_nodes_gce.yaml
|
|
|
|
- name: scheduling_test_many_0s_tasks_many_nodes
|
|
python: "3.10"
|
|
group: core-scalability-test
|
|
working_dir: benchmarks
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
- LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so
|
|
cluster_compute: scheduling.yaml
|
|
anyscale_sdk_2026: true
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python distributed/test_scheduling.py --total-num-task=1984000 --num-cpu-per-task=1
|
|
--task-duration-s=0 --total-num-actors=32 --num-actors-per-nodes=1
|
|
|
|
wait_for_nodes:
|
|
num_nodes: 32
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: scheduling_gce.yaml
|
|
|
|
|
|
# - name: scheduling_test_many_5s_tasks_single_node
|
|
# group: core-scalability-test
|
|
# working_dir: benchmarks
|
|
|
|
# frequency: nightly
|
|
# team: core
|
|
# cluster:
|
|
# cluster_compute: scheduling.yaml
|
|
|
|
# run:
|
|
# timeout: 3600
|
|
# script: python distributed/test_scheduling.py --total-num-task=1984000 --num-cpu-per-task=1
|
|
# --task-duration-s=5 --total-num-actors=1 --num-actors-per-nodes=1
|
|
|
|
# wait_for_nodes:
|
|
# num_nodes: 32
|
|
# timeout: 600
|
|
|
|
# - name: scheduling_test_many_5s_tasks_many_nodes
|
|
# group: core-scalability-test
|
|
# working_dir: benchmarks
|
|
|
|
# frequency: nightly
|
|
# team: core
|
|
# cluster:
|
|
# cluster_compute: scheduling.yaml
|
|
|
|
# run:
|
|
# timeout: 3600
|
|
# script: python distributed/test_scheduling.py --total-num-task=1984000 --num-cpu-per-task=1
|
|
# --task-duration-s=5 --total-num-actors=32 --num-actors-per-nodes=1
|
|
|
|
# wait_for_nodes:
|
|
# num_nodes: 32
|
|
# timeout: 600
|
|
|
|
|
|
##################
|
|
# Core Chaos tests
|
|
##################
|
|
|
|
- name: chaos_many_tasks_baseline
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
script: python chaos_test/test_chaos.py --workload=tasks --disable-resource-killer
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_many_tasks_kill_raylet
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
prepare: python setup_chaos.py --no-start
|
|
script: python chaos_test/test_chaos.py --workload=tasks
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_many_tasks_terminate_instance
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
prepare: python setup_chaos.py --no-start --chaos TerminateEC2Instance
|
|
script: python chaos_test/test_chaos.py --workload=tasks
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: chaos_many_tasks_iptable_failure_injection
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RAY_health_check_period_ms=10000
|
|
- RAY_health_check_timeout_ms=100000
|
|
- RAY_health_check_failure_threshold=10
|
|
- RAY_gcs_rpc_server_connect_timeout_s=60
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
script: >
|
|
python simulate_cross_az_network_failure.py --network-failure-interval 60 --network-failure-duration 5 --command python chaos_test/test_chaos.py --workload=tasks --disable-resource-killer
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_many_actors_baseline
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 4200
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
script: python chaos_test/test_chaos.py --workload=actors --disable-resource-killer
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_many_actors_kill_raylet
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 4200
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
prepare: python setup_chaos.py --no-start
|
|
script: python chaos_test/test_chaos.py --workload=actors
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_many_actors_terminate_instance
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 4200
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
prepare: python setup_chaos.py --no-start --chaos TerminateEC2Instance
|
|
script: python chaos_test/test_chaos.py --workload=actors
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: chaos_many_actors_iptable_failure_injection
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RAY_health_check_period_ms=10000
|
|
- RAY_health_check_timeout_ms=100000
|
|
- RAY_health_check_failure_threshold=10
|
|
- RAY_gcs_rpc_server_connect_timeout_s=60
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 4200
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
script: >
|
|
python simulate_cross_az_network_failure.py --network-failure-interval 60 --network-failure-duration 5 --command python chaos_test/test_chaos.py --workload=actors --disable-resource-killer
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_streaming_generator_baseline
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
script: python chaos_test/test_chaos.py --workload=streaming --disable-resource-killer
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_streaming_generator_kill_raylet
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
prepare: python setup_chaos.py --no-start --max-to-kill 10
|
|
script: python chaos_test/test_chaos.py --workload=streaming
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_streaming_generator_terminate_instance
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
prepare: python setup_chaos.py --no-start --chaos TerminateEC2Instance --max-to-kill 10
|
|
script: python chaos_test/test_chaos.py --workload=streaming
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: chaos_streaming_generator_iptable_failure_injection
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RAY_health_check_period_ms=10000
|
|
- RAY_health_check_timeout_ms=100000
|
|
- RAY_health_check_failure_threshold=10
|
|
- RAY_gcs_rpc_server_connect_timeout_s=60
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
script: >
|
|
python simulate_cross_az_network_failure.py --network-failure-interval 60 --network-failure-duration 5 --command python chaos_test/test_chaos.py --workload=streaming --disable-resource-killer
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_object_ref_borrowing_baseline
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
script: python chaos_test/test_chaos.py --workload=borrowing --disable-resource-killer
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_object_ref_borrowing_kill_raylet
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
prepare: python setup_chaos.py --no-start
|
|
script: python chaos_test/test_chaos.py --workload=borrowing
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
- name: chaos_object_ref_borrowing_terminate_instance
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
prepare: python setup_chaos.py --no-start --chaos TerminateEC2Instance
|
|
script: python chaos_test/test_chaos.py --workload=borrowing
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
|
|
- name: chaos_object_ref_borrowing_iptable_failure_injection
|
|
python: "3.10"
|
|
group: core-nightly-test
|
|
working_dir: nightly_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
runtime_env:
|
|
- RAY_health_check_period_ms=10000
|
|
- RAY_health_check_timeout_ms=100000
|
|
- RAY_health_check_failure_threshold=10
|
|
- RAY_gcs_rpc_server_connect_timeout_s=60
|
|
cluster_compute: chaos_test/compute_template.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
wait_for_nodes:
|
|
num_nodes: 10
|
|
script: >
|
|
python simulate_cross_az_network_failure.py --network-failure-interval 60 --network-failure-duration 5 --command python chaos_test/test_chaos.py --workload=borrowing --disable-resource-killer
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: chaos_test/compute_template_gce.yaml
|
|
|
|
#####################
|
|
# Observability tests
|
|
#####################
|
|
- name: agent_stress_test
|
|
python: "3.10"
|
|
group: core-observability-test
|
|
working_dir: dashboard
|
|
|
|
frequency: nightly
|
|
team: core
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
byod:
|
|
type: gpu
|
|
runtime_env:
|
|
- RAY_INTERNAL_MEM_PROFILE_COMPONENTS=dashboard_agent
|
|
post_build_script: byod_agent_stress_test.sh
|
|
cluster_compute: agent_stress_compute.yaml
|
|
|
|
run:
|
|
timeout: 14400
|
|
script: python mem_check.py --working-dir .
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: agent_stress_compute_gce.yaml
|
|
|
|
- name: k8s_serve_ha_test
|
|
group: k8s-test
|
|
working_dir: k8s_tests
|
|
|
|
# Failing since Oct 2024.
|
|
# https://github.com/ray-project/ray/issues/36190
|
|
frequency: manual
|
|
team: serve
|
|
cluster:
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: compute_tpl.yaml
|
|
|
|
run:
|
|
timeout: 28800 # 8h
|
|
prepare: bash prepare.sh
|
|
script: python run_gcs_ft_on_k8s.py
|
|
|
|
- name: azure_cluster_launcher
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
# TODO: the test seems to be leaking resources, and will
|
|
# start failing over time; needs more investigation.
|
|
frequency: manual
|
|
team: clusters
|
|
cluster:
|
|
byod:
|
|
post_build_script: byod_azure_cluster_launcher.sh
|
|
cluster_compute: azure/tests/azure_compute.yaml
|
|
run:
|
|
timeout: 2400
|
|
script: bash release/azure_docker_login.sh && python -I launch_and_verify_cluster.py azure/tests/azure-cluster.yaml --num-expected-nodes 3 --retries 10
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 python -I launch_and_verify_cluster.py azure/tests/azure-cluster.yaml --num-expected-nodes 3 --retries 10
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 python -I launch_and_verify_cluster.py azure/tests/azure-cluster.yaml --num-expected-nodes 3 --retries 10
|
|
|
|
- name: aws_cluster_launcher
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
frequency: nightly
|
|
team: clusters
|
|
|
|
cluster:
|
|
cluster_compute: aws/tests/aws_compute.yaml
|
|
|
|
run:
|
|
timeout: 2400
|
|
script: python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10
|
|
|
|
- name: aws_cluster_launcher_nightly_image
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
frequency: manual
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: aws/tests/aws_compute.yaml
|
|
|
|
run:
|
|
timeout: 2400
|
|
script: python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override nightly
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override nightly
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override nightly
|
|
|
|
- name: aws_cluster_launcher_latest_image
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
frequency: manual
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: aws/tests/aws_compute.yaml
|
|
|
|
run:
|
|
timeout: 2400
|
|
script: python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override latest
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override latest
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override latest
|
|
|
|
- name: aws_cluster_launcher_release_image
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
frequency: manual
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: aws/tests/aws_compute.yaml
|
|
|
|
run:
|
|
timeout: 2400
|
|
script: python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override commit
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override commit
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/tests/aws_cluster.yaml --num-expected-nodes 2 --retries 10 --docker-override commit
|
|
|
|
|
|
- name: aws_cluster_launcher_minimal
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
frequency: nightly
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: aws/tests/aws_compute.yaml
|
|
|
|
run:
|
|
timeout: 1200
|
|
script: python launch_and_verify_cluster.py aws/example-minimal.yaml
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/example-minimal.yaml
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/example-minimal.yaml
|
|
|
|
- name: aws_cluster_launcher_full
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
frequency: nightly
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: aws/tests/aws_compute.yaml
|
|
|
|
run:
|
|
timeout: 3000
|
|
script: python launch_and_verify_cluster.py aws/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 python launch_and_verify_cluster.py aws/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 python launch_and_verify_cluster.py aws/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest
|
|
|
|
- name: gcp_cluster_launcher_minimal
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
env: gce
|
|
frequency: nightly
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
|
|
|
|
run:
|
|
timeout: 1200
|
|
script: python launch_and_verify_cluster.py gcp/example-minimal-pinned.yaml
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-minimal-pinned.yaml
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-minimal-pinned.yaml
|
|
|
|
- name: gcp_cluster_launcher_full
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
env: gce
|
|
frequency: nightly
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
|
|
|
|
run:
|
|
timeout: 4800
|
|
script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 30
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 30 --docker-override latest
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 30 --docker-override latest
|
|
|
|
- name: gcp_cluster_launcher_latest_image
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
env: gce
|
|
frequency: manual
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override latest
|
|
|
|
- name: gcp_cluster_launcher_nightly_image
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
env: gce
|
|
frequency: manual
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override nightly
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override nightly
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override nightly
|
|
|
|
- name: gcp_cluster_launcher_release_image
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
env: gce
|
|
frequency: manual
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override commit
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override commit
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-full.yaml --num-expected-nodes 2 --retries 20 --docker-override commit
|
|
|
|
- name: gcp_cluster_launcher_gpu_docker
|
|
python: "3.10"
|
|
group: cluster-launcher-test
|
|
working_dir: ../python/ray/autoscaler/
|
|
|
|
env: gce
|
|
frequency: weekly
|
|
team: clusters
|
|
cluster:
|
|
cluster_compute: gcp/tests/single_node_32_cpu_gce.yaml
|
|
|
|
run:
|
|
timeout: 1200
|
|
script: python launch_and_verify_cluster.py gcp/example-gpu-docker.yaml
|
|
|
|
variations:
|
|
- __suffix__: v1
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=0 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-gpu-docker.yaml
|
|
- __suffix__: v2
|
|
run:
|
|
script: RAY_UP_enable_autoscaler_v2=1 RAY_TRIM_AUTOSCALER_SSH_KEYS=1 python launch_and_verify_cluster.py gcp/example-gpu-docker.yaml
|
|
|
|
- name: autoscaler_aws
|
|
python: "3.10"
|
|
group: autoscaler-test
|
|
working_dir: autoscaling_tests
|
|
|
|
frequency: nightly
|
|
team: core
|
|
|
|
cluster:
|
|
cluster_compute: aws.yaml
|
|
|
|
run:
|
|
timeout: 1800
|
|
script: python run.py
|
|
|
|
|
|
##############
|
|
# LLM Serve
|
|
##############
|
|
|
|
# Runs performance benchmark tests against vLLM service
|
|
- name: llm_serve_llama_3dot1_8B_quantized_tp_1
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1.yaml --run-vllm-profiler --run-serve-llm-profiler
|
|
|
|
- name: llm_serve_llama_3dot1_8B_quantized_tp_1_direct_streaming
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
runtime_env:
|
|
- RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING=1
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
- RAY_SERVE_DEFAULT_HTTP_HOST=0.0.0.0
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1.yaml --run-vllm-profiler --run-serve-llm-profiler
|
|
|
|
# Direct streaming with the prefix-cache-aware request router. Guards #64326,
|
|
# where a body-aware router got no routing key on the direct-streaming path.
|
|
- name: llm_serve_llama_3dot1_8B_quantized_tp_1_direct_streaming_prefix_cache
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
runtime_env:
|
|
- RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING=1
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
- RAY_SERVE_DEFAULT_HTTP_HOST=0.0.0.0
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1_prefix_cache.yaml
|
|
|
|
# Runs performance benchmark tests against vLLM service
|
|
- name: llm_serve_llama_3dot1_8B_tp_2
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_tp2.yaml --run-vllm-profiler --run-serve-llm-profiler
|
|
|
|
- name: llm_serve_llama_3dot1_8B_tp_2_direct_streaming
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
runtime_env:
|
|
- RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING=1
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
- RAY_SERVE_DEFAULT_HTTP_HOST=0.0.0.0
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_tp2.yaml --run-vllm-profiler --run-serve-llm-profiler
|
|
|
|
- name: llm_serve_multi_node
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
runtime_env:
|
|
- RAY_SERVE_USE_COMPACT_SCHEDULING_STRATEGY=1 # For test_llm_serve_prefill_decode_with_data_parallelism
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_2x_4xl4.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: >
|
|
pytest -sv test_llm_serve_multi_node_integration.py
|
|
|
|
- name: llm_serve_multi_node_direct_streaming
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
runtime_env:
|
|
- RAY_SERVE_USE_COMPACT_SCHEDULING_STRATEGY=1 # For test_llm_serve_prefill_decode_with_data_parallelism
|
|
- RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING=1
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
- RAY_SERVE_DEFAULT_HTTP_HOST=0.0.0.0
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_2x_4xl4.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: >
|
|
pytest -sv test_llm_serve_multi_node_integration.py
|
|
|
|
- name: llm_serve_dp_engine_fault
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_2x_4xl4.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: >
|
|
pytest -sv test_llm_serve_dp_engine_fault.py
|
|
|
|
- name: llm_serve_llama_3dot2_1B_no_accelerator
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_g5-4xlarge.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud: serve_release_tests_cloud
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot2_1b_no_accelerator.yaml --run-serve-llm-profiler
|
|
|
|
|
|
- name: llm_serve_llama_3dot1_8B_lora
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_lora.yaml --timeout 900
|
|
|
|
- name: llm_serve_llama_3dot2_1B_s3
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot2_1b_s3.yaml --skip-hf-token true
|
|
|
|
- name: llm_serve_correctness
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_four_L4_gpu_head_node.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud: serve_release_tests_cloud
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: pytest -vs test_llm_serve_correctness.py
|
|
|
|
- name: llm_serve_vllm_integration_tests
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_four_L4_gpu_head_node.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud: serve_release_tests_cloud
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: pytest -vs test_llm_serve_integration.py test_llm_serve_fault_tolerance.py
|
|
|
|
- name: llm_serve_vllm_integration_tests_direct_streaming
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
runtime_env:
|
|
- RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING=1
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_four_L4_gpu_head_node.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud: serve_release_tests_cloud
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: pytest -vs test_llm_serve_integration.py test_llm_serve_fault_tolerance.py
|
|
|
|
- name: llm_serve_sleep_wakeup
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_four_L4_gpu_head_node.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud: serve_release_tests_cloud
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: pytest -vs test_llm_serve_sleep_wakeup.py
|
|
|
|
- name: llm_serve_pause_resume
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_four_L4_gpu_head_node.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud: serve_release_tests_cloud
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: pytest -vs test_llm_serve_pause_resume.py
|
|
|
|
- name: llm_serve_rlhf
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_four_L4_gpu_head_node.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud: serve_release_tests_cloud
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python test_llm_serve_rlhf.py
|
|
|
|
- name: llm_serve_llama_3dot1_8B_quantized_tp1_1p1d
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1_1p1d.yaml --skip-hf-token true
|
|
|
|
- name: llm_serve_llama_3dot1_8B_quantized_tp1_1p1d_direct_streaming
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
runtime_env:
|
|
- RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING=1
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
- RAY_SERVE_DEFAULT_HTTP_HOST=0.0.0.0
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1_1p1d.yaml --skip-hf-token true
|
|
|
|
- name: llm_serve_llama_3dot1_8B_quantized_tp1_2p6d
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1_2p6d.yaml --skip-hf-token true
|
|
|
|
- name: llm_serve_llama_3dot1_8B_quantized_tp1_2p6d_direct_streaming
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
runtime_env:
|
|
- RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING=1
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
- RAY_SERVE_DEFAULT_HTTP_HOST=0.0.0.0
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1_2p6d.yaml --skip-hf-token true
|
|
|
|
- name: llm_serve_llama_3dot1_8B_quantized_tp1_2p6d_lmcache
|
|
frequency: manual # todo(ray-llm): fix this test with new/old lmcache version and new vllm version and re-enable it.
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
post_build_script: byod_llm_lmcache_test.sh
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_auto_select_worker.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud_id: cld_wy5a6nhazplvu32526ams61d98
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
long_running: false
|
|
script: python run_llm_serve_test_and_bms.py --serve-config-file configs/serve_llama_3dot1_8b_quantized_tp1_2p6d_lmcache.yaml --skip-hf-token true
|
|
|
|
- name: llm_serve_sglang_e2e
|
|
frequency: manual
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/serve
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
runtime_env:
|
|
- RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES=1
|
|
post_build_script: byod_llm_sglang_test.sh
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_four_L4_gpu_head_node.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: pytest -vs test_llm_serve_sglang.py
|
|
|
|
- name: llm_serve_kv_router
|
|
frequency: manual
|
|
python: "3.12"
|
|
group: llm-serve
|
|
team: llm
|
|
working_dir: llm_tests/kv_router_test
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
runtime_env:
|
|
- RAY_SERVE_LLM_ENABLE_DIRECT_STREAMING=1
|
|
- RAY_SERVE_ENABLE_HA_PROXY=1
|
|
- RAY_SERVE_DEFAULT_HTTP_HOST=0.0.0.0
|
|
post_build_script: byod_llm_kv_router_test.sh
|
|
anyscale_sdk_2026: true
|
|
# Kept in this working_dir rather than referenced across dirs
|
|
# so it ships in the test's runfiles.
|
|
cluster_compute: llm_4xl4.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: pytest -vs .
|
|
|
|
##############
|
|
# LLM Batch
|
|
##############
|
|
|
|
- name: llm_batch_vllm
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-batch
|
|
team: llm
|
|
working_dir: llm_tests/batch
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_4xl4.yaml
|
|
# NOTE: Important for getting the correct secrets
|
|
cloud: serve_release_tests_cloud
|
|
project_id: prj_lhlrf1u5yv8qz9qg3xzw8fkiiq
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: >
|
|
pytest -sv test_batch_vllm.py
|
|
|
|
- name: llm_batch_sglang_llama
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-batch
|
|
team: llm
|
|
working_dir: llm_tests/batch
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
post_build_script: byod_llm_sglang_test.sh
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_4xl4.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: >
|
|
pytest -sv test_batch_sglang.py
|
|
|
|
- name: llm_batch_vllm_multi_node
|
|
frequency: nightly
|
|
python: "3.12"
|
|
group: llm-batch
|
|
team: llm
|
|
working_dir: llm_tests/batch
|
|
|
|
cluster:
|
|
byod:
|
|
type: llm-cu130
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_2x_4xl4.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: >
|
|
pytest -sv test_batch_multi_node_vllm.py
|
|
|
|
- name: llm_batch_single_node_baseline_benchmark
|
|
python: "3.12"
|
|
group: llm-batch
|
|
working_dir: llm_tests/batch
|
|
frequency: nightly
|
|
team: llm
|
|
|
|
cluster:
|
|
byod:
|
|
runtime_env:
|
|
- VLLM_DISABLE_COMPILE_CACHE=1
|
|
type: llm-cu130
|
|
post_build_script: byod_llm_single_node_baseline_benchmark.sh
|
|
python_depset: llm_batch/llm_batch_single_node_benchmark_py312_cu130.lock
|
|
anyscale_sdk_2026: true
|
|
cluster_compute: llm_single_node_benchmark_l4.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: pytest -v test_batch_single_node_vllm.py::test_single_node_baseline_benchmark
|
|
|
|
|
|
#######################
|
|
# Ray examples tests
|
|
#######################
|
|
|
|
|
|
- name: video_analysis_pipeline # do not use dashes (regex sensitive)
|
|
frequency: weekly
|
|
python: "3.10"
|
|
group: ray-examples
|
|
team: serve
|
|
working_dir: //doc/source/serve/tutorials/video-analysis
|
|
|
|
cluster:
|
|
byod:
|
|
type: cpu
|
|
post_build_script: byod_serve_video_analysis.sh
|
|
cluster_compute: ci/aws.yaml
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: bash ci/tests.sh # relative to working_dir
|
|
|
|
variations:
|
|
- __suffix__: aws # uses default specs above
|
|
# probably wont work on gce as tests depend on s3 access
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: ci/gce.yaml # relative to working_dir
|
|
|
|
- name: transformers_reinforcement_learning # do not use dashes (regex sensitive)
|
|
python: "3.10"
|
|
frequency: weekly
|
|
group: ray-examples
|
|
team: ml
|
|
working_dir: //doc/source/train/examples/transformers/transformer_reinforcement_learning
|
|
|
|
cluster:
|
|
byod:
|
|
type: gpu
|
|
post_build_script: byod_huggingface_trl.sh # release/ray_release/byod/
|
|
cluster_compute: ci/aws.yaml # relative to working_dir
|
|
|
|
run:
|
|
timeout: 3600
|
|
script: bash ci/tests.sh # relative to working_dir
|
|
|
|
variations:
|
|
- __suffix__: aws
|
|
- __suffix__: gce
|
|
env: gce
|
|
frequency: manual
|
|
cluster:
|
|
cluster_compute: ci/gce.yaml # relative to working_dir
|