chore: import upstream snapshot with attribution

doc/source/cluster/doc_code/yarn/dashboard.py

@@ -0,0 +1,18 @@
+import skein
+import sys
+from urllib.parse import urlparse
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python dashboard.py <dashboard-address>")
+        sys.exit(1)
+    address = sys.argv[1]
+    # Check if the address is a valid URL
+    result = urlparse(address)
+    if not all([result.scheme, result.netloc]):
+        print("Error: Invalid dashboard address. Please provide a valid URL.")
+        sys.exit(1)
+
+    print("Registering dashboard " + address + " on skein.")
+    app = skein.ApplicationClient.from_current()
+    app.ui.add_page("ray-dashboard", address, "Ray Dashboard")

doc/source/cluster/doc_code/yarn/example.py

@@ -0,0 +1,50 @@
+import sys
+import time
+from collections import Counter
+
+import ray
+
+
+@ray.remote
+def get_host_name(x):
+    import platform
+    import time
+
+    time.sleep(0.01)
+    return x + (platform.node(),)
+
+
+def wait_for_nodes(expected):
+    # Wait for all nodes to join the cluster.
+    while True:
+        num_nodes = len(ray.nodes())
+        if num_nodes < expected:
+            print(
+                "{} nodes have joined so far, waiting for {} more.".format(
+                    num_nodes, expected - num_nodes
+                )
+            )
+            sys.stdout.flush()
+            time.sleep(1)
+        else:
+            break
+
+
+def main():
+    wait_for_nodes(4)
+
+    # Check that objects can be transferred from each node to each other node.
+    for i in range(10):
+        print("Iteration {}".format(i))
+        results = [get_host_name.remote(get_host_name.remote(())) for _ in range(100)]
+        print(Counter(ray.get(results)))
+        sys.stdout.flush()
+
+    print("Success!")
+    sys.stdout.flush()
+    time.sleep(20)
+
+
+if __name__ == "__main__":
+    ray.init(address="localhost:6379")
+    main()

doc/source/cluster/doc_code/yarn/ray-skein.yaml

@@ -0,0 +1,71 @@
+name: ray
+
+services:
+    # Head service.
+    ray-head:
+        # There should only be one instance of the head node per cluster.
+        instances: 1
+        resources:
+            # The resources for the head node.
+            vcores: 1
+            memory: 2048
+        files:
+            # ray/doc/source/cluster/doc_code/yarn/example.py
+            example.py: example.py
+            # ray/doc/source/cluster/doc_code/yarn/dashboard.py
+            dashboard.py: dashboard.py
+        #     # A packaged python environment using `conda-pack`. Note that Skein
+        #     # doesn't require any specific way of distributing files, but this
+        #     # is a good one for python projects. This is optional.
+        #     # See https://jcrist.github.io/skein/distributing-files.html
+        #     environment: environment.tar.gz
+        script: |
+            # Activate the packaged conda environment
+            #  - source environment/bin/activate
+
+            # This gets the IP address of the head node.
+            RAY_HEAD_ADDRESS=$(hostname -i)
+
+            # This stores the Ray head address in the Skein key-value store so that the workers can retrieve it later.
+            skein kv put current --key=RAY_HEAD_ADDRESS --value=$RAY_HEAD_ADDRESS
+
+            # This command starts all the processes needed on the ray head node.
+            # By default, we set object store memory and heap memory to roughly 200 MB. This is conservative
+            # and should be set according to application needs.
+            #
+            ray start --head --port=6379 --object-store-memory=200000000 --memory 200000000 --num-cpus=1 --dashboard-host=$RAY_HEAD_ADDRESS
+
+            # This registers the Ray dashboard on Skein, which can be accessed on Skein's web UI.
+            python dashboard.py "http://$RAY_HEAD_ADDRESS:8265"
+
+            # This executes the user script.
+            python example.py
+
+            # After the user script has executed, all started processes should also die.
+            ray stop
+            skein application shutdown current
+    # Worker service.
+    ray-worker:
+        # The number of instances to start initially. This can be scaled
+        # dynamically later.
+        instances: 4
+        resources:
+            # The resources for the worker node
+            vcores: 1
+            memory: 2048
+        # files:
+        #     environment: environment.tar.gz
+        depends:
+            # Don't start any worker nodes until the head node is started
+            - ray-head
+        script: |
+            # Activate the packaged conda environment
+            #  - source environment/bin/activate
+
+            # This command gets any addresses it needs (e.g. the head node) from
+            # the skein key-value store.
+            RAY_HEAD_ADDRESS=$(skein kv get --key=RAY_HEAD_ADDRESS current)
+
+            # The below command starts all the processes needed on a ray worker node, blocking until killed with sigterm.
+            # After sigterm, all started processes should also die (ray stop).
+            ray start --object-store-memory=200000000 --memory 200000000 --num-cpus=1 --address=$RAY_HEAD_ADDRESS:6379 --block; ray stop