chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,37 @@
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import click
|
||||
|
||||
from ray_release.byod.build import build_anyscale_custom_byod_image
|
||||
from ray_release.byod.build_context import BuildContext
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--image-name", type=str, required=True)
|
||||
@click.option("--base-image", type=str, required=True)
|
||||
@click.option("--post-build-script", type=str)
|
||||
@click.option("--python-depset", type=str)
|
||||
@click.option("--env", "envs", type=str, multiple=True)
|
||||
def main(
|
||||
image_name: str,
|
||||
base_image: str,
|
||||
post_build_script: Optional[str],
|
||||
python_depset: Optional[str],
|
||||
envs: Tuple[str, ...],
|
||||
):
|
||||
if not post_build_script and not python_depset and not envs:
|
||||
raise click.UsageError(
|
||||
"At least one of post_build_script, python_depset, or env must be provided"
|
||||
)
|
||||
build_context: BuildContext = {}
|
||||
if envs:
|
||||
build_context["envs"] = dict(e.split("=", 1) for e in envs)
|
||||
if post_build_script:
|
||||
build_context["post_build_script"] = post_build_script
|
||||
if python_depset:
|
||||
build_context["python_depset"] = python_depset
|
||||
build_anyscale_custom_byod_image(image_name, base_image, build_context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,340 @@
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import click
|
||||
|
||||
from ray_release.buildkite.filter import filter_tests, group_tests
|
||||
from ray_release.buildkite.settings import (
|
||||
get_frequency,
|
||||
get_pipeline_settings,
|
||||
get_test_filters,
|
||||
)
|
||||
from ray_release.buildkite.step import generate_block_step, get_step_for_test_group
|
||||
from ray_release.config import (
|
||||
RELEASE_TEST_CONFIG_FILES,
|
||||
read_and_validate_release_test_collection,
|
||||
)
|
||||
from ray_release.configs.global_config import init_global_config
|
||||
from ray_release.custom_byod_build_init_helper import (
|
||||
build_short_gpu_map,
|
||||
collect_rayci_select_keys,
|
||||
create_custom_build_yaml,
|
||||
)
|
||||
from ray_release.exception import ReleaseTestCLIError, ReleaseTestConfigError
|
||||
from ray_release.logger import logger
|
||||
|
||||
_bazel_workspace_dir = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "")
|
||||
PIPELINE_ARTIFACT_PATH = "/tmp/pipeline_artifacts"
|
||||
|
||||
# Buildkite rejects single pipeline uploads above an organization limit (500
|
||||
# at time of writing). We split below that with headroom so future growth
|
||||
# or step multipliers we don't account for don't trip the limit again.
|
||||
DEFAULT_MAX_JOBS_PER_UPLOAD = 450
|
||||
|
||||
|
||||
def _group_job_count(group: Dict[str, Any]) -> int:
|
||||
"""Count Buildkite jobs for a group step, including the group itself."""
|
||||
total = 1
|
||||
for step in group.get("steps", []):
|
||||
if isinstance(step, dict):
|
||||
parallelism = step.get("parallelism")
|
||||
if isinstance(parallelism, int) and parallelism > 1:
|
||||
total += parallelism
|
||||
continue
|
||||
total += 1
|
||||
return total
|
||||
|
||||
|
||||
def _split_into_batches(
|
||||
steps: List[Dict[str, Any]], max_jobs: int
|
||||
) -> List[List[Dict[str, Any]]]:
|
||||
"""Greedy-pack top-level group steps into batches of <= max_jobs.
|
||||
|
||||
Groups are atomic: we never split a single group across batches, so if
|
||||
any group alone exceeds max_jobs the caller must split that group.
|
||||
"""
|
||||
batches: List[List[Dict[str, Any]]] = [[]]
|
||||
current_count = 0
|
||||
for group in steps:
|
||||
n = _group_job_count(group)
|
||||
if n > max_jobs:
|
||||
raise ValueError(
|
||||
f"group {group.get('group', '<unnamed>')!r} has {n} jobs, "
|
||||
f"exceeds the limit of {max_jobs}; split this group into "
|
||||
f"smaller groups"
|
||||
)
|
||||
if current_count + n > max_jobs and batches[-1]:
|
||||
batches.append([])
|
||||
current_count = 0
|
||||
batches[-1].append(group)
|
||||
current_count += n
|
||||
return batches
|
||||
|
||||
|
||||
@click.command(
|
||||
help="Create a rayci yaml file for building custom BYOD images based on tests."
|
||||
)
|
||||
@click.option(
|
||||
"--test-collection-file",
|
||||
type=str,
|
||||
multiple=True,
|
||||
help="Test collection file, relative path to ray repo.",
|
||||
)
|
||||
@click.option(
|
||||
"--run-jailed-tests",
|
||||
is_flag=True,
|
||||
show_default=True,
|
||||
default=False,
|
||||
help=("Will run jailed tests."),
|
||||
)
|
||||
@click.option(
|
||||
"--run-unstable-tests",
|
||||
is_flag=True,
|
||||
show_default=True,
|
||||
default=False,
|
||||
help=("Will run unstable tests."),
|
||||
)
|
||||
@click.option(
|
||||
"--global-config",
|
||||
default="oss_config.yaml",
|
||||
type=click.Choice(
|
||||
[x.name for x in (Path(__file__).parent.parent / "configs").glob("*.yaml")]
|
||||
),
|
||||
help="Global config to use for test execution.",
|
||||
)
|
||||
@click.option(
|
||||
"--frequency",
|
||||
default=None,
|
||||
type=click.Choice(["manual", "nightly", "nightly-3x", "weekly"]),
|
||||
help="Run frequency of the test",
|
||||
)
|
||||
@click.option(
|
||||
"--test-filters",
|
||||
default=None,
|
||||
type=str,
|
||||
help="Test filters by prefix/regex",
|
||||
)
|
||||
@click.option(
|
||||
"--run-per-test",
|
||||
default=1,
|
||||
type=int,
|
||||
help=("The number of time we run test on the same commit"),
|
||||
)
|
||||
@click.option(
|
||||
"--custom-build-jobs-output-file",
|
||||
type=str,
|
||||
help="The output file for the custom build yaml file",
|
||||
)
|
||||
@click.option(
|
||||
"--test-jobs-output-file",
|
||||
type=str,
|
||||
help=(
|
||||
"Base path for the test jobs JSON output. The actual output files are "
|
||||
"chunked as <stem>_0.json, <stem>_1.json, ... to stay under the "
|
||||
"Buildkite per-upload job limit; callers should upload each chunk in "
|
||||
"order."
|
||||
),
|
||||
)
|
||||
@click.option(
|
||||
"--max-jobs-per-upload",
|
||||
default=DEFAULT_MAX_JOBS_PER_UPLOAD,
|
||||
type=int,
|
||||
help=(
|
||||
"Maximum Buildkite jobs per output chunk file. Counts each group as 1 "
|
||||
"job plus its steps, with `parallelism: N` steps expanding to N jobs."
|
||||
),
|
||||
)
|
||||
@click.option(
|
||||
"--rayci-select-output-file",
|
||||
type=str,
|
||||
help="Output file for RAYCI_SELECT (comma-separated base-image publish step keys).",
|
||||
)
|
||||
@click.option(
|
||||
"--selection-block-threshold",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Number of tests to trigger before asking for confirmation when manually triggered; set to 0 to disable blocking regardless of the number of tests.",
|
||||
)
|
||||
@click.option(
|
||||
"--upload-to-buildkite",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help=(
|
||||
"After writing chunk files, upload each one in order via "
|
||||
"`buildkite-agent pipeline upload`."
|
||||
),
|
||||
)
|
||||
def main(
|
||||
test_collection_file: Tuple[str],
|
||||
run_jailed_tests: bool = False,
|
||||
run_unstable_tests: bool = False,
|
||||
global_config: str = "oss_config.yaml",
|
||||
frequency: str = None,
|
||||
test_filters: str = None,
|
||||
run_per_test: int = 1,
|
||||
custom_build_jobs_output_file: str = None,
|
||||
test_jobs_output_file: str = None,
|
||||
max_jobs_per_upload: int = DEFAULT_MAX_JOBS_PER_UPLOAD,
|
||||
rayci_select_output_file: str = None,
|
||||
selection_block_threshold: int = 0,
|
||||
upload_to_buildkite: bool = False,
|
||||
):
|
||||
global_config_file = os.path.join(
|
||||
os.path.dirname(__file__), "..", "configs", global_config
|
||||
)
|
||||
init_global_config(global_config_file)
|
||||
settings = get_pipeline_settings()
|
||||
env = {}
|
||||
|
||||
frequency = get_frequency(frequency) if frequency else settings["frequency"]
|
||||
prefer_smoke_tests = settings["prefer_smoke_tests"]
|
||||
test_filters = get_test_filters(test_filters) or settings["test_filters"]
|
||||
priority = settings["priority"]
|
||||
|
||||
try:
|
||||
test_collection = read_and_validate_release_test_collection(
|
||||
test_collection_file or RELEASE_TEST_CONFIG_FILES
|
||||
)
|
||||
except ReleaseTestConfigError as e:
|
||||
logger.info("Error: %s", e)
|
||||
raise ReleaseTestConfigError(
|
||||
"Cannot load test yaml file.\nHINT: If you're kicking off tests for a "
|
||||
"specific commit on Buildkite to test Ray wheels, after clicking "
|
||||
"'New build', leave the commit at HEAD, and only specify the commit "
|
||||
"in the dialog that asks for the Ray wheels."
|
||||
) from e
|
||||
|
||||
filtered_tests = filter_tests(
|
||||
test_collection,
|
||||
frequency=frequency,
|
||||
test_filters=test_filters,
|
||||
prefer_smoke_tests=prefer_smoke_tests,
|
||||
run_jailed_tests=run_jailed_tests,
|
||||
run_unstable_tests=run_unstable_tests,
|
||||
)
|
||||
logger.info(f"Found {len(filtered_tests)} tests to run.")
|
||||
if len(filtered_tests) == 0:
|
||||
raise ReleaseTestCLIError(
|
||||
"Empty test collection. The selected frequency or filter did "
|
||||
"not return any tests to run. Adjust your filters."
|
||||
)
|
||||
tests = [test for test, _ in filtered_tests]
|
||||
|
||||
gpu_map = build_short_gpu_map(os.path.join(_bazel_workspace_dir, "ray-images.json"))
|
||||
|
||||
# Generate custom image build steps
|
||||
create_custom_build_yaml(
|
||||
os.path.join(_bazel_workspace_dir, custom_build_jobs_output_file),
|
||||
tests,
|
||||
gpu_map,
|
||||
)
|
||||
|
||||
rayci_select_keys = collect_rayci_select_keys(tests, gpu_map)
|
||||
|
||||
# Generate test job steps
|
||||
grouped_tests = group_tests(filtered_tests)
|
||||
|
||||
group_str = ""
|
||||
for group, tests in grouped_tests.items():
|
||||
group_str += f"\n{group}:\n"
|
||||
for test, smoke in tests:
|
||||
group_str += f" {test['name']}"
|
||||
if smoke:
|
||||
group_str += " [smoke test]"
|
||||
group_str += "\n"
|
||||
logger.info(f"Tests to run:\n{group_str}")
|
||||
|
||||
no_concurrency_limit = settings["no_concurrency_limit"]
|
||||
if no_concurrency_limit:
|
||||
logger.warning("Concurrency is not limited for this run!")
|
||||
|
||||
if os.environ.get("REPORT_TO_RAY_TEST_DB", False):
|
||||
env["REPORT_TO_RAY_TEST_DB"] = "1"
|
||||
|
||||
# Pipe through RAYCI_BUILD_ID from the forge step.
|
||||
# TODO(khluu): convert the steps to rayci steps and stop passing through
|
||||
# RAYCI_BUILD_ID.
|
||||
build_id = os.environ.get("RAYCI_BUILD_ID")
|
||||
if build_id:
|
||||
env["RAYCI_BUILD_ID"] = build_id
|
||||
|
||||
# If the build is manually triggered and there are more than 5 tests
|
||||
# Ask user to confirm before launching the tests.
|
||||
block_step = None
|
||||
|
||||
is_automatic = os.environ.get("AUTOMATIC", "") == "1"
|
||||
if not is_automatic and test_filters and selection_block_threshold > 0:
|
||||
if len(tests) >= selection_block_threshold:
|
||||
block_step = generate_block_step(len(tests))
|
||||
|
||||
steps = get_step_for_test_group(
|
||||
grouped_tests,
|
||||
minimum_run_per_test=run_per_test,
|
||||
test_collection_file=test_collection_file,
|
||||
env=env,
|
||||
priority=priority.value,
|
||||
global_config=global_config,
|
||||
is_concurrency_limit=not no_concurrency_limit,
|
||||
block_step_key=block_step["key"] if block_step else None,
|
||||
gpu_map=gpu_map,
|
||||
)
|
||||
steps = [{"group": "block", "steps": [block_step]}] + steps if block_step else steps
|
||||
|
||||
if "BUILDKITE" in os.environ:
|
||||
if os.path.exists(PIPELINE_ARTIFACT_PATH):
|
||||
shutil.rmtree(PIPELINE_ARTIFACT_PATH)
|
||||
os.makedirs(PIPELINE_ARTIFACT_PATH, exist_ok=True, mode=0o755)
|
||||
|
||||
with open(os.path.join(PIPELINE_ARTIFACT_PATH, "pipeline.json"), "wt") as fp:
|
||||
json.dump(steps, fp)
|
||||
|
||||
batches = _split_into_batches(steps, max_jobs_per_upload)
|
||||
output_stem, output_ext = os.path.splitext(test_jobs_output_file)
|
||||
for stale in glob.glob(
|
||||
os.path.join(_bazel_workspace_dir, f"{output_stem}_*{output_ext}")
|
||||
):
|
||||
os.remove(stale)
|
||||
chunk_paths = []
|
||||
for i, batch in enumerate(batches):
|
||||
chunk_path = os.path.join(
|
||||
_bazel_workspace_dir, f"{output_stem}_{i}{output_ext}"
|
||||
)
|
||||
with open(chunk_path, "wt") as fp:
|
||||
json.dump(batch, fp)
|
||||
chunk_paths.append(chunk_path)
|
||||
logger.info(
|
||||
f"Wrote {len(batches)} chunk file(s) for "
|
||||
f"{sum(_group_job_count(g) for g in steps)} total jobs."
|
||||
)
|
||||
|
||||
# Only emit RAYCI_SELECT when a filter narrows the test set; an unfiltered
|
||||
# run (e.g. full nightly) wants the complete image pipeline.
|
||||
if rayci_select_output_file and test_filters:
|
||||
with open(
|
||||
os.path.join(_bazel_workspace_dir, rayci_select_output_file),
|
||||
"wt",
|
||||
) as fp:
|
||||
fp.write(",".join(sorted(rayci_select_keys)))
|
||||
|
||||
if upload_to_buildkite:
|
||||
for chunk_path in chunk_paths:
|
||||
logger.info(f"Uploading {chunk_path}")
|
||||
subprocess.run(
|
||||
["buildkite-agent", "pipeline", "upload", chunk_path],
|
||||
check=True,
|
||||
)
|
||||
|
||||
settings["frequency"] = settings["frequency"].value
|
||||
settings["priority"] = settings["priority"].value
|
||||
with open(os.path.join(PIPELINE_ARTIFACT_PATH, "settings.json"), "wt") as fp:
|
||||
json.dump(settings, fp)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,44 @@
|
||||
from typing import Tuple
|
||||
|
||||
import click
|
||||
|
||||
from ray_release.config import (
|
||||
RELEASE_TEST_CONFIG_FILES,
|
||||
read_and_validate_release_test_collection,
|
||||
)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option(
|
||||
"--test-collection-file",
|
||||
type=str,
|
||||
multiple=True,
|
||||
help="Test collection file, relative path to ray repo.",
|
||||
)
|
||||
@click.option(
|
||||
"--show-disabled",
|
||||
is_flag=True,
|
||||
default=False,
|
||||
help="Show disabled tests.",
|
||||
)
|
||||
def main(
|
||||
test_collection_file: Tuple[str],
|
||||
show_disabled: bool,
|
||||
):
|
||||
if not test_collection_file:
|
||||
test_collection_file = tuple(RELEASE_TEST_CONFIG_FILES)
|
||||
|
||||
tests = read_and_validate_release_test_collection(test_collection_file)
|
||||
for test in tests:
|
||||
name = test["name"]
|
||||
python_version = test.get("python", "")
|
||||
test_frequency = test.get("frequency", "missing")
|
||||
test_team = test.get("team", "missing")
|
||||
if not show_disabled and test_frequency == "manual":
|
||||
continue
|
||||
|
||||
print(f"{name} python={python_version} team={test_team}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,294 @@
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Set, Tuple
|
||||
|
||||
import click
|
||||
|
||||
from ray_release.bazel import bazel_runfile
|
||||
from ray_release.buildkite.step import get_step
|
||||
from ray_release.byod.build import (
|
||||
build_anyscale_base_byod_images,
|
||||
build_anyscale_custom_byod_image,
|
||||
)
|
||||
from ray_release.byod.build_context import BuildContext
|
||||
from ray_release.config import (
|
||||
RELEASE_TEST_CONFIG_FILES,
|
||||
read_and_validate_release_test_collection,
|
||||
)
|
||||
from ray_release.configs.global_config import init_global_config
|
||||
from ray_release.logger import logger
|
||||
from ray_release.test import Test
|
||||
from ray_release.test_automation.release_state_machine import ReleaseTestStateMachine
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("test_name", required=True, type=str)
|
||||
@click.argument("passing_commit", required=True, type=str)
|
||||
@click.argument("failing_commit", required=True, type=str)
|
||||
@click.option(
|
||||
"--test-collection-file",
|
||||
type=str,
|
||||
multiple=True,
|
||||
help="Test collection file, relative path to ray repo.",
|
||||
)
|
||||
@click.option(
|
||||
"--concurrency",
|
||||
default=3,
|
||||
type=int,
|
||||
help=(
|
||||
"Maximum number of concurrent test jobs to run. Higher number uses more "
|
||||
"capacity, but reduce the bisect duration"
|
||||
),
|
||||
)
|
||||
@click.option(
|
||||
"--run-per-commit",
|
||||
default=1,
|
||||
type=int,
|
||||
help=(
|
||||
"The number of time we run test on the same commit, to account for test "
|
||||
"flakiness. Commit passes only when it passes on all runs"
|
||||
),
|
||||
)
|
||||
@click.option(
|
||||
"--is-full-test",
|
||||
is_flag=True,
|
||||
show_default=True,
|
||||
default=False,
|
||||
help=("Use the full, non-smoke version of the test"),
|
||||
)
|
||||
@click.option(
|
||||
"--global-config",
|
||||
default="oss_config.yaml",
|
||||
type=click.Choice(
|
||||
[x.name for x in (Path(__file__).parent.parent / "configs").glob("*.yaml")]
|
||||
),
|
||||
help="Global config to use for test execution.",
|
||||
)
|
||||
def main(
|
||||
test_name: str,
|
||||
passing_commit: str,
|
||||
failing_commit: str,
|
||||
test_collection_file: Tuple[str],
|
||||
concurrency: int = 1,
|
||||
run_per_commit: int = 1,
|
||||
is_full_test: bool = False,
|
||||
global_config: str = "oss_config.yaml",
|
||||
) -> None:
|
||||
init_global_config(
|
||||
bazel_runfile("release/ray_release/configs", global_config),
|
||||
)
|
||||
if concurrency <= 0:
|
||||
raise ValueError(
|
||||
f"Concurrency input need to be a positive number, received: {concurrency}"
|
||||
)
|
||||
test = _get_test(test_name, test_collection_file)
|
||||
pre_sanity_check = _sanity_check(
|
||||
test, passing_commit, failing_commit, run_per_commit, is_full_test
|
||||
)
|
||||
if not pre_sanity_check:
|
||||
logger.info(
|
||||
"Failed pre-saniy check, the test might be flaky or fail due to"
|
||||
" an external (not a code change) factors"
|
||||
)
|
||||
return
|
||||
commit_lists = _get_commit_lists(passing_commit, failing_commit)
|
||||
blamed_commit = _bisect(
|
||||
test, commit_lists, concurrency, run_per_commit, is_full_test
|
||||
)
|
||||
logger.info(f"Blamed commit found for test {test_name}: {blamed_commit}")
|
||||
# TODO(can): this env var is used as a feature flag, in case we need to turn this
|
||||
# off quickly. We should remove this when the new db reporter is stable.
|
||||
if os.environ.get("UPDATE_TEST_STATE_MACHINE", False):
|
||||
logger.info(f"Updating test state for test {test_name} to CONSISTENTLY_FAILING")
|
||||
_update_test_state(test, blamed_commit)
|
||||
|
||||
|
||||
def _bisect(
|
||||
test: Test,
|
||||
commit_list: List[str],
|
||||
concurrency: int,
|
||||
run_per_commit: int,
|
||||
is_full_test: bool = False,
|
||||
) -> str:
|
||||
while len(commit_list) > 2:
|
||||
logger.info(
|
||||
f"Bisecting between {len(commit_list)} commits: "
|
||||
f"{commit_list[0]} to {commit_list[-1]} with concurrency {concurrency}"
|
||||
)
|
||||
idx_to_commit = {}
|
||||
for i in range(concurrency):
|
||||
idx = len(commit_list) * (i + 1) // (concurrency + 1)
|
||||
# make sure that idx is not at the boundary; this avoids rerun bisect
|
||||
# on the previously run revision
|
||||
idx = min(max(idx, 1), len(commit_list) - 2)
|
||||
idx_to_commit[idx] = commit_list[idx]
|
||||
outcomes = _run_test(
|
||||
test, set(idx_to_commit.values()), run_per_commit, is_full_test
|
||||
)
|
||||
passing_idx = 0
|
||||
failing_idx = len(commit_list) - 1
|
||||
for idx, commit in idx_to_commit.items():
|
||||
is_passing = all(
|
||||
outcome == "passed" for outcome in outcomes[commit].values()
|
||||
)
|
||||
if is_passing and idx > passing_idx:
|
||||
passing_idx = idx
|
||||
if not is_passing and idx < failing_idx:
|
||||
failing_idx = idx
|
||||
commit_list = commit_list[passing_idx : failing_idx + 1]
|
||||
return commit_list[-1]
|
||||
|
||||
|
||||
def _sanity_check(
|
||||
test: Test,
|
||||
passing_revision: str,
|
||||
failing_revision: str,
|
||||
run_per_commit: int,
|
||||
is_full_test: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
Sanity check that the test indeed passes on the passing revision, and fails on the
|
||||
failing revision
|
||||
"""
|
||||
logger.info(
|
||||
f"Sanity check passing revision: {passing_revision}"
|
||||
f" and failing revision: {failing_revision}"
|
||||
)
|
||||
outcomes = _run_test(
|
||||
test, [passing_revision, failing_revision], run_per_commit, is_full_test
|
||||
)
|
||||
if any(map(lambda x: x != "passed", outcomes[passing_revision].values())):
|
||||
return False
|
||||
return any(map(lambda x: x != "passed", outcomes[failing_revision].values()))
|
||||
|
||||
|
||||
def _run_test(
|
||||
test: Test, commits: Set[str], run_per_commit: int, is_full_test: bool
|
||||
) -> Dict[str, Dict[int, str]]:
|
||||
logger.info(f'Running test {test["name"]} on commits {commits}')
|
||||
for commit in commits:
|
||||
_trigger_test_run(test, commit, run_per_commit, is_full_test)
|
||||
return _obtain_test_result(commits, run_per_commit)
|
||||
|
||||
|
||||
def _trigger_test_run(
|
||||
test: Test, commit: str, run_per_commit: int, is_full_test: bool
|
||||
) -> None:
|
||||
os.environ["COMMIT_TO_TEST"] = commit
|
||||
build_anyscale_base_byod_images([test])
|
||||
if test.require_custom_byod_image():
|
||||
build_context: BuildContext = {}
|
||||
post_build_script = test.get_byod_post_build_script()
|
||||
python_depset = test.get_byod_python_depset()
|
||||
if post_build_script:
|
||||
build_context["post_build_script"] = post_build_script
|
||||
if python_depset:
|
||||
build_context["python_depset"] = python_depset
|
||||
build_anyscale_custom_byod_image(
|
||||
test.get_anyscale_byod_image(),
|
||||
test.get_anyscale_base_byod_image(),
|
||||
build_context,
|
||||
)
|
||||
for run in range(run_per_commit):
|
||||
step = get_step(
|
||||
copy.deepcopy(test), # avoid mutating the original test
|
||||
smoke_test=test.get("smoke_test", False) and not is_full_test,
|
||||
env={
|
||||
"RAY_COMMIT_OF_WHEEL": commit,
|
||||
"COMMIT_TO_TEST": commit,
|
||||
},
|
||||
)
|
||||
step["label"] = f'{test["name"]}:{commit[:7]}-{run}'
|
||||
step["key"] = f"{commit}-{run}"
|
||||
pipeline = subprocess.Popen(
|
||||
["echo", json.dumps({"steps": [step]})], stdout=subprocess.PIPE
|
||||
)
|
||||
subprocess.check_output(
|
||||
["buildkite-agent", "pipeline", "upload"], stdin=pipeline.stdout
|
||||
)
|
||||
pipeline.stdout.close()
|
||||
|
||||
|
||||
def _obtain_test_result(
|
||||
commits: Set[str], run_per_commit: int
|
||||
) -> Dict[str, Dict[int, str]]:
|
||||
outcomes = {}
|
||||
wait = 5
|
||||
total_wait = 0
|
||||
while True:
|
||||
logger.info(f"... waiting for test result ...({total_wait} seconds)")
|
||||
for commit in commits:
|
||||
if commit in outcomes and len(outcomes[commit]) == run_per_commit:
|
||||
continue
|
||||
for run in range(run_per_commit):
|
||||
outcome = (
|
||||
subprocess.check_output(
|
||||
[
|
||||
"buildkite-agent",
|
||||
"step",
|
||||
"get",
|
||||
"outcome",
|
||||
"--step",
|
||||
f"{commit}-{run}",
|
||||
]
|
||||
)
|
||||
.decode("utf-8")
|
||||
.strip()
|
||||
)
|
||||
if not outcome:
|
||||
continue
|
||||
if commit not in outcomes:
|
||||
outcomes[commit] = {}
|
||||
outcomes[commit][run] = outcome
|
||||
all_commit_finished = len(outcomes) == len(commits)
|
||||
per_commit_finished = all(
|
||||
len(outcome) == run_per_commit for outcome in outcomes.values()
|
||||
)
|
||||
if all_commit_finished and per_commit_finished:
|
||||
break
|
||||
time.sleep(wait)
|
||||
total_wait = total_wait + wait
|
||||
logger.info(f"Final test outcomes: {outcomes}")
|
||||
return outcomes
|
||||
|
||||
|
||||
def _get_test(test_name: str, test_collection_file: Tuple[str]) -> Test:
|
||||
test_collection = read_and_validate_release_test_collection(
|
||||
test_collection_file or RELEASE_TEST_CONFIG_FILES,
|
||||
)
|
||||
return [test for test in test_collection if test["name"] == test_name][0]
|
||||
|
||||
|
||||
def _get_commit_lists(passing_commit: str, failing_commit: str) -> List[str]:
|
||||
# This command obtains all commits between inclusively
|
||||
return (
|
||||
subprocess.check_output(
|
||||
f"git rev-list --reverse ^{passing_commit}~ {failing_commit}",
|
||||
shell=True,
|
||||
)
|
||||
.decode("utf-8")
|
||||
.strip()
|
||||
.split("\n")
|
||||
)
|
||||
|
||||
|
||||
def _update_test_state(test: Test, blamed_commit: str) -> None:
|
||||
test.update_from_s3()
|
||||
logger.info(f"Test object: {json.dumps(test)}")
|
||||
test[Test.KEY_BISECT_BLAMED_COMMIT] = blamed_commit
|
||||
|
||||
# Compute and update the next test state, then comment blamed commit on github issue
|
||||
sm = ReleaseTestStateMachine(test)
|
||||
sm.move()
|
||||
sm.comment_blamed_commit_on_github_issue()
|
||||
|
||||
logger.info(f"Test object: {json.dumps(test)}")
|
||||
test.persist_to_s3()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,155 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import click
|
||||
|
||||
from ray_release.aws import maybe_fetch_api_token
|
||||
from ray_release.config import (
|
||||
RELEASE_TEST_CONFIG_FILES,
|
||||
as_smoke_test,
|
||||
find_test,
|
||||
read_and_validate_release_test_collection,
|
||||
)
|
||||
from ray_release.configs.global_config import init_global_config
|
||||
from ray_release.env import DEFAULT_ENVIRONMENT, load_environment, populate_os_env
|
||||
from ray_release.exception import ReleaseTestCLIError, ReleaseTestError
|
||||
from ray_release.glue import run_release_test
|
||||
from ray_release.logger import logger
|
||||
from ray_release.reporter.artifacts import ArtifactsReporter
|
||||
from ray_release.reporter.db import DBReporter
|
||||
from ray_release.reporter.log import LogReporter
|
||||
from ray_release.reporter.ray_test_db import RayTestDBReporter
|
||||
from ray_release.result import Result
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument("test_name", required=True, type=str)
|
||||
@click.option(
|
||||
"--test-collection-file",
|
||||
multiple=True,
|
||||
type=str,
|
||||
help="Test collection file, relative path to ray repo.",
|
||||
)
|
||||
@click.option(
|
||||
"--smoke-test",
|
||||
default=False,
|
||||
type=bool,
|
||||
is_flag=True,
|
||||
help="Finish quickly for testing",
|
||||
)
|
||||
@click.option(
|
||||
"--report",
|
||||
default=False,
|
||||
type=bool,
|
||||
is_flag=True,
|
||||
help="Report results to database",
|
||||
)
|
||||
@click.option(
|
||||
"--env",
|
||||
default=None,
|
||||
# Get the names without suffixes of all files in "../environments"
|
||||
type=click.Choice(
|
||||
[x.stem for x in (Path(__file__).parent.parent / "environments").glob("*.env")]
|
||||
),
|
||||
help="Environment to use. Will overwrite environment used in test config.",
|
||||
)
|
||||
@click.option(
|
||||
"--global-config",
|
||||
default="oss_config.yaml",
|
||||
type=click.Choice(
|
||||
[x.name for x in (Path(__file__).parent.parent / "configs").glob("*.yaml")]
|
||||
),
|
||||
help="Global config to use for test execution.",
|
||||
)
|
||||
@click.option(
|
||||
"--test-definition-root",
|
||||
default=None,
|
||||
type=str,
|
||||
help="Root of the test definition files. Default is the root of the repo.",
|
||||
)
|
||||
@click.option(
|
||||
"--image",
|
||||
default=None,
|
||||
type=str,
|
||||
help="Image to use for the test.",
|
||||
)
|
||||
def main(
|
||||
test_name: str,
|
||||
test_collection_file: Tuple[str],
|
||||
smoke_test: bool = False,
|
||||
report: bool = False,
|
||||
env: Optional[str] = None,
|
||||
global_config: str = "oss_config.yaml",
|
||||
test_definition_root: Optional[str] = None,
|
||||
image: Optional[str] = None,
|
||||
):
|
||||
global_config_file = os.path.join(
|
||||
os.path.dirname(__file__), "..", "configs", global_config
|
||||
)
|
||||
init_global_config(global_config_file)
|
||||
test_collection = read_and_validate_release_test_collection(
|
||||
test_collection_file or RELEASE_TEST_CONFIG_FILES,
|
||||
test_definition_root,
|
||||
)
|
||||
test = find_test(test_collection, test_name)
|
||||
|
||||
if not test:
|
||||
raise ReleaseTestCLIError(
|
||||
f"Test `{test_name}` not found in collection file: "
|
||||
f"{test_collection_file}"
|
||||
)
|
||||
|
||||
if smoke_test:
|
||||
test = as_smoke_test(test)
|
||||
|
||||
env_to_use = env or test.get("env", DEFAULT_ENVIRONMENT)
|
||||
env_dict = load_environment(env_to_use)
|
||||
populate_os_env(env_dict)
|
||||
anyscale_project = os.environ.get("ANYSCALE_PROJECT", None)
|
||||
if not test.is_kuberay() and not anyscale_project:
|
||||
raise ReleaseTestCLIError(
|
||||
"You have to set the ANYSCALE_PROJECT environment variable!"
|
||||
)
|
||||
|
||||
maybe_fetch_api_token()
|
||||
|
||||
result = Result()
|
||||
|
||||
reporters = [LogReporter()]
|
||||
|
||||
if "BUILDKITE" in os.environ:
|
||||
reporters.append(ArtifactsReporter())
|
||||
|
||||
if report:
|
||||
reporters.append(DBReporter())
|
||||
|
||||
# TODO(can): this env var is used as a feature flag, in case we need to turn this
|
||||
# off quickly. We should remove this when the new db reporter is stable.
|
||||
if os.environ.get("REPORT_TO_RAY_TEST_DB", False):
|
||||
reporters.append(RayTestDBReporter())
|
||||
|
||||
try:
|
||||
result = run_release_test(
|
||||
test=test,
|
||||
anyscale_project=anyscale_project,
|
||||
result=result,
|
||||
reporters=reporters,
|
||||
smoke_test=smoke_test,
|
||||
test_definition_root=test_definition_root,
|
||||
image=image,
|
||||
)
|
||||
return_code = result.return_code
|
||||
except ReleaseTestError as e:
|
||||
logger.exception(e)
|
||||
return_code = e.exit_code.value
|
||||
logger.info(
|
||||
f"Release test pipeline for test {test['name']} completed. "
|
||||
f"Returning with exit code = {return_code}"
|
||||
)
|
||||
sys.exit(return_code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user