chore: import upstream snapshot with attribution

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/check_code_search_path_is_propagated.py

@@ -0,0 +1,31 @@
+"""
+A dummy ray driver script that executes in subprocess.
+Prints global worker's `load_code_from_local` property that ought to be set
+whenever `JobConfig.code_search_path` is specified
+"""
+
+
+def run():
+    import ray
+    from ray.job_config import JobConfig
+
+    ray.init(job_config=JobConfig(code_search_path=["/home/code/"]))
+
+    @ray.remote
+    def foo() -> bool:
+        return ray._private.worker.global_worker.load_code_from_local
+
+    load_code_from_local = ray.get(foo.remote())
+
+    statement = "propagated" if load_code_from_local else "NOT propagated"
+
+    # Step 1: Print the statement indicating that the code_search_path have been
+    #         properly respected
+    print(f"Code search path is {statement}")
+    # Step 2: Print the whole runtime_env to validate that it's been passed
+    #         appropriately from submit_job API
+    print(ray.get_runtime_context().runtime_env)
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/check_cuda_devices.py

@@ -0,0 +1,22 @@
+import os
+
+import ray
+
+cuda_env = ray._private.accelerators.nvidia_gpu.NOSET_CUDA_VISIBLE_DEVICES_ENV_VAR
+if os.environ.get("RAY_TEST_RESOURCES_SPECIFIED") == "1":
+    assert cuda_env not in os.environ
+    if os.environ.get("RAY_TEST_GPUS_SPECIFIED") == "1":
+        assert "CUDA_VISIBLE_DEVICES" in os.environ
+    else:
+        assert "CUDA_VISIBLE_DEVICES" not in os.environ
+else:
+    assert os.environ[cuda_env] == "1"
+
+
+@ray.remote
+def f():
+    assert cuda_env not in os.environ
+
+
+# Will raise if task fails.
+ray.get(f.remote())

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/check_niceness.py

@@ -0,0 +1,23 @@
+"""
+A dummy ray driver script that executes in subprocess.
+Checks that job manager's environment variable is different.
+"""
+
+import os
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote
+    def foo():
+        print("worker", os.nice(0))
+
+    ray.get(foo.remote())
+
+
+if __name__ == "__main__":
+    print("driver", os.nice(0))
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/consume_one_cpu.py

@@ -0,0 +1,13 @@
+import ray
+
+ray.init()
+
+
+@ray.remote(num_cpus=1)
+def f():
+    pass
+
+
+print("Hanging...")
+ray.get(f.remote())
+print("Success!")

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/driver_runtime_env_inheritance.py

@@ -0,0 +1,58 @@
+import argparse
+import sys
+import time
+
+import ray
+
+# This prefix is used to identify the output log line that contains the runtime env.
+RUNTIME_ENV_LOG_LINE_PREFIX = "ray_job_test_runtime_env_output:"
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Dashboard agent.")
+    parser.add_argument(
+        "--conflict",
+        type=str,
+    )
+    parser.add_argument(
+        "--worker-process-setup-hook",
+        type=str,
+    )
+
+    args = parser.parse_args()
+
+    if args.worker_process_setup_hook:
+        ray.init(
+            runtime_env={
+                "worker_process_setup_hook": lambda: print(
+                    args.worker_process_setup_hook
+                )
+            }
+        )
+
+        @ray.remote
+        def f():
+            pass
+
+        ray.get(f.remote())
+        time.sleep(5)
+        sys.exit(0)
+
+    if args.conflict == "pip":
+        ray.init(runtime_env={"pip": ["numpy"]})
+        print(
+            RUNTIME_ENV_LOG_LINE_PREFIX + ray._private.worker.global_worker.runtime_env
+        )
+    elif args.conflict == "env_vars":
+        ray.init(runtime_env={"env_vars": {"A": "1"}})
+        print(
+            RUNTIME_ENV_LOG_LINE_PREFIX + ray._private.worker.global_worker.runtime_env
+        )
+    else:
+        ray.init(
+            runtime_env={
+                "env_vars": {"C": "1"},
+            }
+        )
+        print(
+            RUNTIME_ENV_LOG_LINE_PREFIX + ray._private.worker.global_worker.runtime_env
+        )

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/override_env_var.py

@@ -0,0 +1,27 @@
+"""
+Test script that attempts to set its own runtime_env, but we should ensure
+we ended up using job submission API call's runtime_env instead of scripts
+"""
+
+
+def run():
+    import os
+
+    import ray
+
+    ray.init(
+        runtime_env={
+            "env_vars": {"TEST_SUBPROCESS_JOB_CONFIG_ENV_VAR": "SHOULD_BE_OVERRIDEN"}
+        },
+    )
+
+    @ray.remote
+    def foo():
+        return "bar"
+
+    ray.get(foo.remote())
+    print(os.environ.get("TEST_SUBPROCESS_JOB_CONFIG_ENV_VAR", None))
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/per_task_runtime_env.py

@@ -0,0 +1,26 @@
+import os
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote(runtime_env={"env_vars": {"FOO": "bar"}})
+    def get_task_working_dir():
+        # Check behavior of working_dir: The cwd should contain the
+        # current file, which is being used as a job entrypoint script.
+        assert os.path.exists("per_task_runtime_env.py")
+
+        return ray.get_runtime_context().runtime_env.working_dir()
+
+    driver_working_dir = ray.get_runtime_context().runtime_env.working_dir()
+    task_working_dir = ray.get(get_task_working_dir.remote())
+    assert driver_working_dir == task_working_dir, (
+        driver_working_dir,
+        task_working_dir,
+    )
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/print_namespace.py

@@ -0,0 +1,22 @@
+"""
+A dummy ray driver script that executes in subprocess. Prints namespace
+from ray's runtime context for job submission API testing.
+"""
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote
+    def foo():
+        return "bar"
+
+    ray.get(foo.remote())
+
+    print(ray.get_runtime_context().namespace)
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/print_runtime_env.py

@@ -0,0 +1,22 @@
+"""
+A dummy ray driver script that executes in subprocess. Prints runtime_env
+from ray's runtime context for job submission API testing.
+"""
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote
+    def foo():
+        return "bar"
+
+    ray.get(foo.remote())
+
+    print(ray.get_runtime_context().runtime_env)
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/ray_tune_basic.py

@@ -0,0 +1,15 @@
+"""Tests that Ray Tune works with the working_dir set in Jobs.
+
+Ray Tune internally sets environment variables using runtime_env.
+If the inherited internal runtime environment overwrites the working_dir
+from jobs with an empty working_dir, this test will fail.  See #25484"""
+from ray_tune_dependency import foo
+
+from ray import tune
+
+
+def objective(*args):
+    foo()
+
+
+tune.run(objective)

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/ray_tune_dependency.py

@@ -0,0 +1,5 @@
+"""A file dependency for testing working_dir behavior with Ray Tune."""
+
+
+def foo():
+    pass

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/script_with_exception.py

@@ -0,0 +1,6 @@
+def run():
+    raise Exception("Script failed with exception !")
+
+
+if __name__ == "__main__":
+    run()