wehub-resource-sync
380 lines
12 KiB
YAML
380 lines
12 KiB
YAML
group: data tests
|
|
depends_on:
|
|
- forge
|
|
- oss-ci-base_ml-multipy(*)
|
|
- ray-core-build(*)
|
|
- ray-dashboard-build
|
|
steps:
|
|
# builds
|
|
- name: data17build-multipy
|
|
label: "wanda: data17build-py{{array.python}}"
|
|
wanda: ci/docker/data17.build.wanda.yaml
|
|
array:
|
|
python:
|
|
- "3.10"
|
|
env:
|
|
PYTHON: "{{array.python}}"
|
|
tags: cibase
|
|
|
|
- name: datalbuild-multipy
|
|
label: "wanda: datalbuild-py{{array.python}}"
|
|
wanda: ci/docker/datal.build.wanda.yaml
|
|
array:
|
|
python:
|
|
- "3.10"
|
|
- "3.12"
|
|
env:
|
|
PYTHON: "{{array.python}}"
|
|
tags: cibase
|
|
|
|
- name: datanbuild-multipy
|
|
wanda: ci/docker/datan.build.wanda.yaml
|
|
env:
|
|
PYTHON: "3.10"
|
|
tags: cibase
|
|
|
|
- name: datamongobuild-multipy
|
|
label: "wanda: datamongobuild-py3.10"
|
|
wanda: ci/docker/datamongo.build.wanda.yaml
|
|
env:
|
|
PYTHON: "3.10"
|
|
tags: cibase
|
|
|
|
- name: datatfdsbuild-multipy
|
|
label: "wanda: datatfdsbuild-py3.12"
|
|
wanda: ci/docker/datatfds.build.wanda.yaml
|
|
env:
|
|
PYTHON: "3.12"
|
|
tags: cibase
|
|
|
|
- name: docgpubuild-py312
|
|
label: "wanda: docgpubuild-py3.12"
|
|
wanda: ci/docker/docgpu.build.wanda.yaml
|
|
env:
|
|
PYTHON: "3.12"
|
|
depends_on: oss-ci-base_gpu-multipy(python=3.12)
|
|
tags: cibase
|
|
|
|
# type check
|
|
- label: ":python: pyrefly type check"
|
|
tags:
|
|
- data
|
|
instance_type: small
|
|
commands:
|
|
- bash ci/lint/pyrefly-check.sh
|
|
depends_on: data17build-multipy(python=3.10)
|
|
job_env: data17build-py3.10
|
|
|
|
# tests
|
|
- label: ":database: data: arrow v17 tests"
|
|
tags:
|
|
- data
|
|
instance_type: large
|
|
parallelism: 8
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
|
|
--workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
|
|
--worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
|
|
--build-name data17build-py3.10 --python-version 3.10
|
|
--except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials,tensorflow_datasets,cudf
|
|
depends_on: data17build-multipy(python=3.10)
|
|
|
|
- label: ":database: data: tensorflow-datasets tests"
|
|
key: data_tensorflow_datasets_tests
|
|
tags:
|
|
- data
|
|
instance_type: medium
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
|
|
--build-name datatfdsbuild-py3.12 --python-version 3.12
|
|
--only-tags tensorflow_datasets
|
|
depends_on: datatfdsbuild-multipy
|
|
|
|
- label: ":database: data: arrow v17 tests (data_non_parallel)"
|
|
tags:
|
|
- data
|
|
- data_non_parallel
|
|
instance_type: medium
|
|
parallelism: 3
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
|
|
--workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
|
|
--worker-id "$${BUILDKITE_PARALLEL_JOB}"
|
|
--build-name data17build-py3.10 --python-version 3.10
|
|
--only-tags data_non_parallel
|
|
--except-tags data_non_parallel_postmerge_only,cudf
|
|
depends_on: data17build-multipy(python=3.10)
|
|
|
|
- label: ":database: data: arrow v23 tests"
|
|
tags:
|
|
- data
|
|
- skip-on-premerge
|
|
instance_type: medium
|
|
parallelism: 8
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
|
|
--workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
|
|
--worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
|
|
--build-name datalbuild-py3.10 --python-version 3.10
|
|
--except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials,tensorflow_datasets,cudf
|
|
depends_on: datalbuild-multipy(python=3.10)
|
|
|
|
- label: ":database: data: arrow v23 tests (data_non_parallel)"
|
|
tags:
|
|
- data
|
|
- data_non_parallel
|
|
- skip-on-premerge
|
|
instance_type: medium
|
|
parallelism: 3
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
|
|
--workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
|
|
--worker-id "$${BUILDKITE_PARALLEL_JOB}"
|
|
--build-name datalbuild-py3.10 --python-version 3.10
|
|
--only-tags data_non_parallel
|
|
--except-tags data_non_parallel_postmerge_only,cudf
|
|
depends_on: datalbuild-multipy(python=3.10)
|
|
|
|
- label: ":database: data: arrow v23 py{{array.python}} tests ({{array.worker_id}})"
|
|
key: datal_python_tests
|
|
if: build.pull_request.labels includes "continuous-build" || pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
|
|
tags:
|
|
- data
|
|
instance_type: medium
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
|
|
--workers 2 --worker-id {{array.worker_id}} --parallelism-per-worker 3
|
|
--build-name datalbuild-py{{array.python}} --python-version {{array.python}}
|
|
--except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials,tensorflow_datasets,cudf
|
|
depends_on: datalbuild-multipy($)
|
|
array:
|
|
python: ["3.12"]
|
|
worker_id: ["0", "1"]
|
|
|
|
- label: ":database: data: arrow v23 py{{array.python}} tests (data_non_parallel)"
|
|
key: datal_python_non_parallel_tests
|
|
if: build.pull_request.labels includes "continuous-build" || pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
|
|
tags:
|
|
- data
|
|
instance_type: medium
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
|
|
--build-name datalbuild-py{{array.python}} --python-version {{array.python}}
|
|
--only-tags data_non_parallel
|
|
--except-tags data_non_parallel_postmerge_only,cudf
|
|
depends_on: datalbuild-multipy($)
|
|
array:
|
|
python: ["3.12"]
|
|
|
|
- label: ":database: data: arrow nightly tests"
|
|
tags:
|
|
- data
|
|
- skip-on-premerge
|
|
- oss
|
|
- disabled
|
|
instance_type: medium
|
|
parallelism: 2
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
|
|
--workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
|
|
--worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
|
|
--build-name datanbuild-py3.10 --python-version 3.10
|
|
--except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials,tensorflow_datasets,cudf
|
|
depends_on: datanbuild-multipy
|
|
soft_fail: true
|
|
|
|
- label: ":database: data: arrow nightly tests (data_non_parallel)"
|
|
tags:
|
|
- data
|
|
- skip-on-premerge
|
|
- oss
|
|
- disabled
|
|
instance_type: medium
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
|
|
--build-name datanbuild-py3.10 --python-version 3.10
|
|
--only-tags data_non_parallel
|
|
--except-tags data_non_parallel_postmerge_only,cudf
|
|
depends_on: datanbuild-multipy
|
|
soft_fail: true
|
|
|
|
- label: ":database: data: postmerge arrow v17 tests"
|
|
tags:
|
|
- data
|
|
- skip-on-premerge
|
|
instance_type: medium
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
|
|
--build-name data17build-py3.10 --python-version 3.10
|
|
--only-tags data_non_parallel_postmerge_only
|
|
depends_on: data17build-multipy(python=3.10)
|
|
|
|
- label: ":database: data: postmerge arrow v23 tests"
|
|
tags:
|
|
- data
|
|
- skip-on-premerge
|
|
instance_type: medium
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
|
|
--build-name datalbuild-py3.10 --python-version 3.10
|
|
--only-tags data_non_parallel_postmerge_only
|
|
depends_on: datalbuild-multipy(python=3.10)
|
|
|
|
- label: ":database: data: dask tests"
|
|
tags:
|
|
- data
|
|
- dask
|
|
instance_type: medium
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
|
|
--build-name datalbuild-py3.12 --python-version 3.12
|
|
--only-tags dask
|
|
depends_on: datalbuild-multipy(python=3.12)
|
|
|
|
- label: ":database: data: doc tests"
|
|
if: '!(build.pull_request.labels includes "docs-go")'
|
|
tags:
|
|
- data
|
|
- doc
|
|
instance_type: medium
|
|
commands:
|
|
# doc tests
|
|
- bazel run //ci/ray_ci:test_in_docker -- python/ray/... //doc/... data
|
|
--build-name datalbuild-py3.10
|
|
--python-version 3.10
|
|
--except-tags gpu
|
|
--only-tags doctest
|
|
--parallelism-per-worker 2
|
|
# doc examples
|
|
- bazel run //ci/ray_ci:test_in_docker -- //doc/... data
|
|
--build-name datalbuild-py3.10
|
|
--python-version 3.10
|
|
--except-tags gpu,post_wheel_build,doctest,dask
|
|
--parallelism-per-worker 2
|
|
--skip-ray-installation
|
|
depends_on: datalbuild-multipy(python=3.10)
|
|
|
|
- label: ":database: data: dask doc tests"
|
|
if: '!(build.pull_request.labels includes "docs-go")'
|
|
tags:
|
|
- data
|
|
- doc
|
|
instance_type: medium
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //doc/... data
|
|
--build-name datalbuild-py3.12
|
|
--python-version 3.12
|
|
--only-tags dask
|
|
--parallelism-per-worker 2
|
|
depends_on: datalbuild-multipy(python=3.12)
|
|
|
|
- label: ":database: data: doc gpu tests"
|
|
if: '!(build.pull_request.labels includes "docs-go")'
|
|
tags:
|
|
- data
|
|
- doc
|
|
- gpu
|
|
instance_type: gpu-large
|
|
commands:
|
|
# doc tests
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //doc/... data
|
|
--build-name docgpubuild-py3.10
|
|
--only-tags doctest
|
|
--except-tags cpu
|
|
--python-version 3.10
|
|
# doc examples
|
|
- bazel run //ci/ray_ci:test_in_docker -- //doc/... data
|
|
--build-name docgpubuild-py3.10
|
|
--except-tags doctest
|
|
--only-tags gpu
|
|
--skip-ray-installation
|
|
--python-version 3.10
|
|
depends_on: docgpubuild
|
|
|
|
- label: ":database: data: cuDF batch format tests"
|
|
tags:
|
|
- data
|
|
- gpu
|
|
- cudf
|
|
instance_type: gpu-large
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
|
|
--build-name docgpubuild-py3.12
|
|
--only-tags cudf
|
|
--gpus 1
|
|
--python-version 3.12
|
|
depends_on: docgpubuild-py312
|
|
|
|
- label: ":database: data: integration tests"
|
|
tags:
|
|
- data
|
|
instance_type: medium
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
|
|
--build-name datamongobuild-py3.10
|
|
--python-version 3.10
|
|
--build-type java
|
|
--only-tags data_integration
|
|
--except-tags doctest
|
|
depends_on: datamongobuild-multipy
|
|
|
|
- label: ":database: data: dashboard tests"
|
|
tags:
|
|
- data
|
|
- dashboard
|
|
instance_type: small
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- python/ray/dashboard/... data
|
|
--build-name datalbuild-py3.10
|
|
--python-version 3.10
|
|
--parallelism-per-worker 3
|
|
depends_on: datalbuild-multipy(python=3.10)
|
|
|
|
- label: ":database: data: flaky tests"
|
|
key: data_flaky_tests
|
|
tags:
|
|
- data
|
|
- flaky
|
|
- skip-on-premerge
|
|
instance_type: medium
|
|
soft_fail: true
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests
|
|
--parallelism-per-worker 3
|
|
--build-name datalbuild-py3.10
|
|
--python-version 3.10
|
|
--except-tags gpu_only,gpu
|
|
depends_on: datalbuild-multipy(python=3.10)
|
|
|
|
- label: ":database: data: flaky gpu tests"
|
|
key: data_flaky_gpu_tests
|
|
tags:
|
|
- data
|
|
- skip-on-premerge
|
|
instance_type: gpu-large
|
|
soft_fail: true
|
|
commands:
|
|
- bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests
|
|
--build-name docgpubuild-py3.10
|
|
--only-tags gpu,gpu_only
|
|
--python-version 3.10
|
|
depends_on: docgpubuild
|
|
|
|
- label: ":database: data: postmerge authenticated tests"
|
|
key: data_postmerge_authenticated_tests
|
|
tags:
|
|
- data
|
|
- oss
|
|
- skip-on-premerge
|
|
instance_type: medium
|
|
commands:
|
|
- $(python ci/env/setup_credentials.py)
|
|
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
|
|
--build-name datalbuild-py3.10
|
|
--python-version 3.10
|
|
--only-tags needs_credentials
|
|
--test-env=SNOWFLAKE_USER --test-env=SNOWFLAKE_ACCOUNT
|
|
--test-env=SNOWFLAKE_DATABASE --test-env=SNOWFLAKE_SCHEMA
|
|
--test-env=SNOWFLAKE_WAREHOUSE --test-env=SNOWFLAKE_PRIVATE_KEY
|
|
depends_on: datalbuild-multipy(python=3.10)
|