chore: import upstream snapshot with attribution

release/ml_user_tests/horovod/app_config_master.yaml

@@ -0,0 +1,17 @@
+base_image: {{ env["RAY_IMAGE_ML_NIGHTLY_GPU"] }}
+env_vars: {"HOROVOD_GLOO_TIMEOUT_SECONDS": "120"}
+debian_packages:
+  - curl
+
+python:
+  pip_packages:
+    - pytest
+    - awscli
+  conda_packages: []
+
+post_build_cmds:
+  - pip3 uninstall ray -y || true  && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
+  - pip3 install 'ray[tune]'
+  - pip3 install torch torchvision
+  - HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_PYTORCH=1 pip3 install -U git+https://github.com/horovod/horovod.git
+  - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }}

release/ml_user_tests/horovod/compute_tpl_aws.yaml

@@ -0,0 +1,22 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+max_workers: 3
+
+head_node_type:
+    name: head_node
+    instance_type: m5.xlarge
+
+worker_node_types:
+    - name: worker_node
+      instance_type: g6.12xlarge
+      max_workers: 3
+      min_workers: 3
+      use_spot: false
+
+advanced_configurations_json:
+  TagSpecifications:
+    - ResourceType: "instance"
+      Tags:
+        - Key: ttl-hours
+          Value: '24'

release/ml_user_tests/horovod/compute_tpl_gce.yaml

@@ -0,0 +1,24 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west1
+allowed_azs:
+    - us-west1-b
+
+max_workers: 3
+
+head_node_type:
+    name: head_node
+    instance_type: n1-standard-4
+
+worker_node_types:
+    - name: worker_node
+      instance_type: n1-standard-32-nvidia-tesla-t4-2
+      max_workers: 3
+      min_workers: 3
+      use_spot: false
+
+#advanced_configurations_json:
+#  TagSpecifications:
+#    - ResourceType: "instance"
+#      Tags:
+#        - Key: ttl-hours
+#          Value: '24'

release/ml_user_tests/horovod/driver_requirements.txt

@@ -0,0 +1,8 @@
+# Make sure the driver versions are the same as cluster versions.
+# The cluster uses ray-ml Docker image.
+# ray-ml Docker image installs dependencies from ray/python/requirements/ml/ directory.
+# We constrain on these requirements file so that the same versions are installed.
+-c ../../../python/requirements/ml/dl-cpu-requirements.txt
+
+torch
+torchvision

release/ml_user_tests/horovod/driver_setup_latest.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+cd "${0%/*}" || exit 1
+
+sudo apt update
+sudo apt -y install build-essential
+pip install cmake
+
+pip install -U -r ./driver_requirements.txt
+
+
+HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_PYTORCH=1 pip install horovod

release/ml_user_tests/horovod/driver_setup_master.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+cd "${0%/*}" || exit 1
+
+sudo apt update
+sudo apt -y install build-essential
+pip install cmake
+
+pip install -U -r ./driver_requirements.txt
+
+HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_PYTORCH=1 pip install -U git+https://github.com/horovod/horovod.git

release/ml_user_tests/horovod/horovod_example.py

@@ -0,0 +1,242 @@
+# This file is duplicated in ray/tests/horovod
+import argparse
+import os
+from filelock import FileLock
+
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from torchvision import datasets, transforms
+import torch.utils.data.distributed
+
+import horovod.torch as hvd
+from horovod.ray import RayExecutor
+
+
+def metric_average(val, name):
+    tensor = torch.tensor(val)
+    avg_tensor = hvd.allreduce(tensor, name=name)
+    return avg_tensor.item()
+
+
+class Net(nn.Module):
+    def __init__(self):
+        super(Net, self).__init__()
+        self.conv1 = nn.Conv2d(1, 10, kernel_size=5)
+        self.conv2 = nn.Conv2d(10, 20, kernel_size=5)
+        self.conv2_drop = nn.Dropout2d()
+        self.fc1 = nn.Linear(320, 50)
+        self.fc2 = nn.Linear(50, 10)
+
+    def forward(self, x):
+        x = F.relu(F.max_pool2d(self.conv1(x), 2))
+        x = F.relu(F.max_pool2d(self.conv2_drop(self.conv2(x)), 2))
+        x = x.view(-1, 320)
+        x = F.relu(self.fc1(x))
+        x = F.dropout(x, training=self.training)
+        x = self.fc2(x)
+        return F.log_softmax(x)
+
+
+def train_fn(
+    data_dir=None,
+    seed=42,
+    use_cuda=False,
+    batch_size=64,
+    use_adasum=False,
+    lr=0.01,
+    momentum=0.5,
+    num_epochs=10,
+    log_interval=10,
+):
+    # Horovod: initialize library.
+    hvd.init()
+    torch.manual_seed(seed)
+
+    if use_cuda:
+        # Horovod: pin GPU to local rank.
+        torch.cuda.set_device(hvd.local_rank())
+        torch.cuda.manual_seed(seed)
+
+    # Horovod: limit # of CPU threads to be used per worker.
+    torch.set_num_threads(1)
+
+    kwargs = {"num_workers": 1, "pin_memory": True} if use_cuda else {}
+    data_dir = data_dir or "./data"
+    with FileLock(os.path.expanduser("~/.horovod_lock")):
+        train_dataset = datasets.MNIST(
+            data_dir,
+            train=True,
+            download=True,
+            transform=transforms.Compose(
+                [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]
+            ),
+        )
+    # Horovod: use DistributedSampler to partition the training data.
+    train_sampler = torch.utils.data.distributed.DistributedSampler(
+        train_dataset, num_replicas=hvd.size(), rank=hvd.rank()
+    )
+    train_loader = torch.utils.data.DataLoader(
+        train_dataset, batch_size=batch_size, sampler=train_sampler, **kwargs
+    )
+
+    model = Net()
+
+    # By default, Adasum doesn't need scaling up learning rate.
+    lr_scaler = hvd.size() if not use_adasum else 1
+
+    if use_cuda:
+        # Move model to GPU.
+        model.cuda()
+        # If using GPU Adasum allreduce, scale learning rate by local_size.
+        if use_adasum and hvd.nccl_built():
+            lr_scaler = hvd.local_size()
+
+    # Horovod: scale learning rate by lr_scaler.
+    optimizer = optim.SGD(model.parameters(), lr=lr * lr_scaler, momentum=momentum)
+
+    # Horovod: wrap optimizer with DistributedOptimizer.
+    optimizer = hvd.DistributedOptimizer(
+        optimizer,
+        named_parameters=model.named_parameters(),
+        op=hvd.Adasum if use_adasum else hvd.Average,
+    )
+
+    for epoch in range(1, num_epochs + 1):
+        model.train()
+        # Horovod: set epoch to sampler for shuffling.
+        train_sampler.set_epoch(epoch)
+        for batch_idx, (data, target) in enumerate(train_loader):
+            if use_cuda:
+                data, target = data.cuda(), target.cuda()
+            optimizer.zero_grad()
+            output = model(data)
+            loss = F.nll_loss(output, target)
+            loss.backward()
+            optimizer.step()
+            if batch_idx % log_interval == 0:
+                # Horovod: use train_sampler to determine the number of
+                # examples in this worker's partition.
+                print(
+                    "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format(
+                        epoch,
+                        batch_idx * len(data),
+                        len(train_sampler),
+                        100.0 * batch_idx / len(train_loader),
+                        loss.item(),
+                    )
+                )
+
+
+def main(
+    num_workers, use_gpu, timeout_s=30, placement_group_timeout_s=100, kwargs=None
+):
+    kwargs = kwargs or {}
+    if use_gpu:
+        kwargs["use_cuda"] = True
+    settings = RayExecutor.create_settings(
+        timeout_s=timeout_s, placement_group_timeout_s=placement_group_timeout_s
+    )
+    executor = RayExecutor(settings, use_gpu=use_gpu, num_workers=num_workers)
+    executor.start()
+    executor.run(train_fn, kwargs=kwargs)
+
+
+if __name__ == "__main__":
+    # Training settings
+    parser = argparse.ArgumentParser(
+        description="PyTorch MNIST Example",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--batch-size",
+        type=int,
+        default=64,
+        metavar="N",
+        help="input batch size for training (default: 64)",
+    )
+    parser.add_argument(
+        "--num-epochs",
+        type=int,
+        default=5,
+        metavar="N",
+        help="number of epochs to train (default: 10)",
+    )
+    parser.add_argument(
+        "--lr",
+        type=float,
+        default=0.01,
+        metavar="LR",
+        help="learning rate (default: 0.01)",
+    )
+    parser.add_argument(
+        "--momentum",
+        type=float,
+        default=0.5,
+        metavar="M",
+        help="SGD momentum (default: 0.5)",
+    )
+    parser.add_argument(
+        "--use-cuda", action="store_true", default=False, help="enables CUDA training"
+    )
+    parser.add_argument(
+        "--seed", type=int, default=42, metavar="S", help="random seed (default: 42)"
+    )
+    parser.add_argument(
+        "--log-interval",
+        type=int,
+        default=10,
+        metavar="N",
+        help="how many batches to wait before logging training status",
+    )
+    parser.add_argument(
+        "--use-adasum",
+        action="store_true",
+        default=False,
+        help="use adasum algorithm to do reduction",
+    )
+    parser.add_argument(
+        "--num-workers",
+        type=int,
+        default=4,
+        help="Number of Ray workers to use for training.",
+    )
+    parser.add_argument(
+        "--data-dir",
+        help="location of the training dataset in the local filesystem ("
+        "will be downloaded if needed)",
+    )
+    parser.add_argument(
+        "--address",
+        required=False,
+        type=str,
+        default=None,
+        help="Address of Ray cluster.",
+    )
+
+    args = parser.parse_args()
+
+    import ray
+
+    if args.address:
+        ray.init(args.address)
+    else:
+        ray.init()
+
+    kwargs = {
+        "data_dir": args.data_dir,
+        "seed": args.seed,
+        "use_cuda": args.use_cuda if args.use_cuda else False,
+        "batch_size": args.batch_size,
+        "use_adasum": args.use_adasum if args.use_adasum else False,
+        "lr": args.lr,
+        "momentum": args.momentum,
+        "num_epochs": args.num_epochs,
+        "log_interval": args.log_interval,
+    }
+
+    main(
+        num_workers=args.num_workers,
+        use_gpu=args.use_cuda if args.use_cuda else False,
+        kwargs=kwargs,
+    )

release/ml_user_tests/horovod/horovod_user_test.py

@@ -0,0 +1,37 @@
+import json
+import os
+import time
+
+import ray
+from horovod_example import main
+
+if __name__ == "__main__":
+    start = time.time()
+
+    addr = os.environ.get("RAY_ADDRESS")
+    job_name = os.environ.get("RAY_JOB_NAME", "horovod_user_test")
+
+    runtime_env = {"working_dir": os.path.dirname(__file__)}
+
+    if addr.startswith("anyscale://"):
+        ray.init(address=addr, job_name=job_name, runtime_env=runtime_env)
+    else:
+        ray.init(address="auto", runtime_env=runtime_env)
+
+    main(
+        num_workers=6,
+        use_gpu=True,
+        placement_group_timeout_s=2000,
+        timeout_s=120,
+        kwargs={"num_epochs": 20},
+    )
+
+    taken = time.time() - start
+    result = {
+        "time_taken": taken,
+    }
+    test_output_json = os.environ.get("TEST_OUTPUT_JSON", "/tmp/horovod_user_test.json")
+    with open(test_output_json, "wt") as f:
+        json.dump(result, f)
+
+    print("Test Successful!")

release/ml_user_tests/train/compute_tpl_aws.yaml

@@ -0,0 +1,15 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west-2
+
+max_workers: 2
+
+head_node_type:
+    name: head_node
+    instance_type: g6.12xlarge
+
+worker_node_types:
+    -   name: worker_node
+        instance_type: g6.12xlarge
+        min_workers: 2
+        max_workers: 2
+        use_spot: false

release/ml_user_tests/train/compute_tpl_gce.yaml

@@ -0,0 +1,17 @@
+cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
+region: us-west1
+allowed_azs:
+    - us-west1-b
+
+max_workers: 2
+
+head_node_type:
+    name: head_node
+    instance_type: n1-standard-32-nvidia-tesla-t4-2
+
+worker_node_types:
+    -   name: worker_node
+        instance_type: n1-standard-32-nvidia-tesla-t4-2
+        min_workers: 2
+        max_workers: 2
+        use_spot: false

release/ml_user_tests/train/driver_requirements.txt

@@ -0,0 +1,8 @@
+# Make sure the driver versions are the same as cluster versions.
+# The cluster uses ray-ml Docker image.
+# ray-ml Docker image installs dependencies from ray/python/requirements/ml/ directory.
+# We constrain on these requirements file so that the same versions are installed.
+-c ../../../python/requirements/ml/dl-cpu-requirements.txt
+
+torch
+tensorflow

release/ml_user_tests/train/driver_setup.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd "${0%/*}" || exit 1
+
+pip install -U -r ./driver_requirements.txt

release/ml_user_tests/train/train_tensorflow_mnist_test.py

@@ -0,0 +1,34 @@
+import json
+import os
+import time
+
+import ray
+from ray.train.examples.tf.tensorflow_mnist_example import train_tensorflow_mnist
+
+if __name__ == "__main__":
+    start = time.time()
+
+    addr = os.environ.get("RAY_ADDRESS")
+    job_name = os.environ.get("RAY_JOB_NAME", "train_tensorflow_mnist_test")
+
+    if addr is not None and addr.startswith("anyscale://"):
+        ray.init(address=addr, job_name=job_name)
+    else:
+        ray.init(address="auto")
+
+    train_tensorflow_mnist(
+        num_workers=6, use_gpu=True, epochs=20, storage_path="/mnt/cluster_storage"
+    )
+
+    taken = time.time() - start
+    result = {
+        "time_taken": taken,
+    }
+    test_output_json = os.environ.get(
+        "TEST_OUTPUT_JSON", "/tmp/train_tensorflow_mnist_test.json"
+    )
+
+    with open(test_output_json, "wt") as f:
+        json.dump(result, f)
+
+    print("Test Successful!")

release/ml_user_tests/train/train_torch_linear_test.py

@@ -0,0 +1,33 @@
+import json
+import os
+import time
+
+import ray
+
+from ray.train.examples.pytorch.torch_linear_example import train_linear
+
+if __name__ == "__main__":
+    start = time.time()
+
+    addr = os.environ.get("RAY_ADDRESS")
+    job_name = os.environ.get("RAY_JOB_NAME", "train_torch_linear_test")
+
+    if addr is not None and addr.startswith("anyscale://"):
+        ray.init(address=addr, job_name=job_name)
+    else:
+        ray.init(address="auto")
+
+    results = train_linear(
+        num_workers=6, use_gpu=True, epochs=20, storage_path="/mnt/cluster_storage"
+    )
+
+    taken = time.time() - start
+    result = {"time_taken": taken}
+    test_output_json = os.environ.get(
+        "TEST_OUTPUT_JSON", "/tmp/train_torch_linear_test.json"
+    )
+
+    with open(test_output_json, "wt") as f:
+        json.dump(result, f)
+
+    print("Test Successful!")