chore: import upstream snapshot with attribution

python/ray/dashboard/modules/job/tests/backwards_compatibility_scripts/script.py

@@ -0,0 +1,26 @@
+import requests
+
+import ray
+
+ray.init()
+
+
+@ray.remote
+class Counter:
+    def __init__(self):
+        self.counter = 0
+
+    def inc(self):
+        self.counter += 1
+
+    def get_counter(self):
+        return self.counter
+
+
+counter = Counter.remote()
+
+for _ in range(5):
+    ray.get(counter.inc.remote())
+    print(ray.get(counter.get_counter.remote()))
+
+print(requests.__version__)

python/ray/dashboard/modules/job/tests/backwards_compatibility_scripts/test_backwards_compatibility.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+set -ex
+
+unset RAY_ADDRESS
+
+if ! [ -x "$(command -v conda)" ]; then
+    echo "conda doesn't exist. Please download conda for this machine"
+    exit 1
+else
+    echo "conda exists"
+fi
+
+# This is required to use conda activate
+source "$(conda info --base)/etc/profile.d/conda.sh"
+
+PYTHON_VERSION=$(python -c"from platform import python_version; print(python_version())")
+
+RAY_VERSIONS=("2.0.1")
+
+for RAY_VERSION in "${RAY_VERSIONS[@]}"
+do
+    env_name=${JOB_COMPATIBILITY_TEST_TEMP_ENV}
+
+    # Check if the conda env exists
+    if conda env list | grep -q "${env_name}"; then
+        # Clean up if env name is already taken from previous leaking runs
+        conda env remove --name="${env_name}"
+    fi
+
+    printf "\n\n\n"
+    echo "========================================================================================="
+    printf "Creating new conda environment with python %s for ray %s \n" "${PYTHON_VERSION}" "${RAY_VERSION}"
+    echo "========================================================================================="
+    printf "\n\n\n"
+
+    # Include `pip` explicitly: conda-forge's `python` package stopped
+    # bundling pip as a dep, and without it `conda activate` puts us in
+    # an env with python but no pip, so subsequent `pip install` falls
+    # back to the base miniforge env's pip. That clobbers the editable
+    # ray 3.0.0.dev0 in base with ray 2.0.1, and every subsequent
+    # dashboard test that imports `ray._common` fails because 2.0.1
+    # predates that module.
+    conda create -y -n "${env_name}" python="${PYTHON_VERSION}" pip=25.2
+    conda activate "${env_name}"
+    python -m pip install --upgrade pip
+
+    # Pin pydantic version due to: https://github.com/ray-project/ray/issues/36990.
+    # ray<2.9 is only compatible with pydantic<2 and setuptools < 70.
+    python -m pip install -U "pydantic<2" ray=="${RAY_VERSION}" ray[default]=="${RAY_VERSION}" setuptools==69.5.1
+
+    printf "\n\n\n"
+    echo "========================================================="
+    printf "Installed ray job server version: "
+    SERVER_RAY_VERSION=$(python -c "import ray; print(ray.__version__)")
+    printf "%s \n" "${SERVER_RAY_VERSION}"
+    echo "========================================================="
+    printf "\n\n\n"
+    ray stop --force
+    ray start --head
+
+    conda deactivate
+
+    CLIENT_RAY_VERSION=$(python -c "import ray; print(ray.__version__)")
+    CLIENT_RAY_COMMIT=$(python -c "import ray; print(ray.__commit__)")
+    printf "\n\n\n"
+    echo "========================================================================================="
+    printf "Using Ray %s on %s as job client \n" "${CLIENT_RAY_VERSION}" "${CLIENT_RAY_COMMIT}"
+    echo "========================================================================================="
+    printf "\n\n\n"
+
+    export RAY_ADDRESS="http://127.0.0.1:8265"
+
+    cleanup () {
+        unset RAY_ADDRESS
+        ray stop --force
+        conda remove -y --name "${env_name}" --all
+    }
+
+    JOB_ID=$(python -c "import uuid; print(uuid.uuid4().hex)")
+
+    # Get directory of current file. https://stackoverflow.com/questions/59895/
+    DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+    if ! ray job submit --job-id="${JOB_ID}" --working-dir="${DIR}" --runtime-env-json='{"pip": ["requests==2.26.0", "setuptools==69.5.1"]}' -- python script.py; then
+        cleanup
+        exit 1
+    fi
+
+    if ! ray job status "${JOB_ID}"; then
+        cleanup
+        exit 1
+    fi
+
+    if ! ray job logs "${JOB_ID}"; then
+        cleanup
+        exit 1
+    fi
+
+    if ! pytest -vs "${DIR}"/../test_backwards_compatibility.py::test_error_message; then
+        cleanup
+        exit 1
+    fi
+
+    cleanup
+done

python/ray/dashboard/modules/job/tests/conftest.py

@@ -0,0 +1,30 @@
+import os
+
+import ray
+from ray._raylet import GcsClient
+from ray.dashboard.modules.job.job_manager import JobManager
+
+TEST_NAMESPACE = "jobs_test_namespace"
+
+
+def create_ray_cluster(_tracing_startup_hook=None):
+    return ray.init(
+        num_cpus=16,
+        num_gpus=1,
+        resources={"Custom": 1},
+        namespace=TEST_NAMESPACE,
+        log_to_driver=True,
+        _tracing_startup_hook=_tracing_startup_hook,
+    )
+
+
+def create_job_manager(ray_cluster, tmp_path):
+    address_info = ray_cluster
+    gcs_client = GcsClient(address=address_info["gcs_address"])
+    return JobManager(gcs_client, tmp_path)
+
+
+def _driver_script_path(file_name: str) -> str:
+    return os.path.join(
+        os.path.dirname(__file__), "subprocess_driver_scripts", file_name
+    )

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/check_code_search_path_is_propagated.py

@@ -0,0 +1,31 @@
+"""
+A dummy ray driver script that executes in subprocess.
+Prints global worker's `load_code_from_local` property that ought to be set
+whenever `JobConfig.code_search_path` is specified
+"""
+
+
+def run():
+    import ray
+    from ray.job_config import JobConfig
+
+    ray.init(job_config=JobConfig(code_search_path=["/home/code/"]))
+
+    @ray.remote
+    def foo() -> bool:
+        return ray._private.worker.global_worker.load_code_from_local
+
+    load_code_from_local = ray.get(foo.remote())
+
+    statement = "propagated" if load_code_from_local else "NOT propagated"
+
+    # Step 1: Print the statement indicating that the code_search_path have been
+    #         properly respected
+    print(f"Code search path is {statement}")
+    # Step 2: Print the whole runtime_env to validate that it's been passed
+    #         appropriately from submit_job API
+    print(ray.get_runtime_context().runtime_env)
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/check_cuda_devices.py

@@ -0,0 +1,22 @@
+import os
+
+import ray
+
+cuda_env = ray._private.accelerators.nvidia_gpu.NOSET_CUDA_VISIBLE_DEVICES_ENV_VAR
+if os.environ.get("RAY_TEST_RESOURCES_SPECIFIED") == "1":
+    assert cuda_env not in os.environ
+    if os.environ.get("RAY_TEST_GPUS_SPECIFIED") == "1":
+        assert "CUDA_VISIBLE_DEVICES" in os.environ
+    else:
+        assert "CUDA_VISIBLE_DEVICES" not in os.environ
+else:
+    assert os.environ[cuda_env] == "1"
+
+
+@ray.remote
+def f():
+    assert cuda_env not in os.environ
+
+
+# Will raise if task fails.
+ray.get(f.remote())

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/check_niceness.py

@@ -0,0 +1,23 @@
+"""
+A dummy ray driver script that executes in subprocess.
+Checks that job manager's environment variable is different.
+"""
+
+import os
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote
+    def foo():
+        print("worker", os.nice(0))
+
+    ray.get(foo.remote())
+
+
+if __name__ == "__main__":
+    print("driver", os.nice(0))
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/consume_one_cpu.py

@@ -0,0 +1,13 @@
+import ray
+
+ray.init()
+
+
+@ray.remote(num_cpus=1)
+def f():
+    pass
+
+
+print("Hanging...")
+ray.get(f.remote())
+print("Success!")

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/driver_runtime_env_inheritance.py

@@ -0,0 +1,58 @@
+import argparse
+import sys
+import time
+
+import ray
+
+# This prefix is used to identify the output log line that contains the runtime env.
+RUNTIME_ENV_LOG_LINE_PREFIX = "ray_job_test_runtime_env_output:"
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Dashboard agent.")
+    parser.add_argument(
+        "--conflict",
+        type=str,
+    )
+    parser.add_argument(
+        "--worker-process-setup-hook",
+        type=str,
+    )
+
+    args = parser.parse_args()
+
+    if args.worker_process_setup_hook:
+        ray.init(
+            runtime_env={
+                "worker_process_setup_hook": lambda: print(
+                    args.worker_process_setup_hook
+                )
+            }
+        )
+
+        @ray.remote
+        def f():
+            pass
+
+        ray.get(f.remote())
+        time.sleep(5)
+        sys.exit(0)
+
+    if args.conflict == "pip":
+        ray.init(runtime_env={"pip": ["numpy"]})
+        print(
+            RUNTIME_ENV_LOG_LINE_PREFIX + ray._private.worker.global_worker.runtime_env
+        )
+    elif args.conflict == "env_vars":
+        ray.init(runtime_env={"env_vars": {"A": "1"}})
+        print(
+            RUNTIME_ENV_LOG_LINE_PREFIX + ray._private.worker.global_worker.runtime_env
+        )
+    else:
+        ray.init(
+            runtime_env={
+                "env_vars": {"C": "1"},
+            }
+        )
+        print(
+            RUNTIME_ENV_LOG_LINE_PREFIX + ray._private.worker.global_worker.runtime_env
+        )

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/override_env_var.py

@@ -0,0 +1,27 @@
+"""
+Test script that attempts to set its own runtime_env, but we should ensure
+we ended up using job submission API call's runtime_env instead of scripts
+"""
+
+
+def run():
+    import os
+
+    import ray
+
+    ray.init(
+        runtime_env={
+            "env_vars": {"TEST_SUBPROCESS_JOB_CONFIG_ENV_VAR": "SHOULD_BE_OVERRIDEN"}
+        },
+    )
+
+    @ray.remote
+    def foo():
+        return "bar"
+
+    ray.get(foo.remote())
+    print(os.environ.get("TEST_SUBPROCESS_JOB_CONFIG_ENV_VAR", None))
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/per_task_runtime_env.py

@@ -0,0 +1,26 @@
+import os
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote(runtime_env={"env_vars": {"FOO": "bar"}})
+    def get_task_working_dir():
+        # Check behavior of working_dir: The cwd should contain the
+        # current file, which is being used as a job entrypoint script.
+        assert os.path.exists("per_task_runtime_env.py")
+
+        return ray.get_runtime_context().runtime_env.working_dir()
+
+    driver_working_dir = ray.get_runtime_context().runtime_env.working_dir()
+    task_working_dir = ray.get(get_task_working_dir.remote())
+    assert driver_working_dir == task_working_dir, (
+        driver_working_dir,
+        task_working_dir,
+    )
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/print_namespace.py

@@ -0,0 +1,22 @@
+"""
+A dummy ray driver script that executes in subprocess. Prints namespace
+from ray's runtime context for job submission API testing.
+"""
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote
+    def foo():
+        return "bar"
+
+    ray.get(foo.remote())
+
+    print(ray.get_runtime_context().namespace)
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/print_runtime_env.py

@@ -0,0 +1,22 @@
+"""
+A dummy ray driver script that executes in subprocess. Prints runtime_env
+from ray's runtime context for job submission API testing.
+"""
+
+import ray
+
+
+def run():
+    ray.init()
+
+    @ray.remote
+    def foo():
+        return "bar"
+
+    ray.get(foo.remote())
+
+    print(ray.get_runtime_context().runtime_env)
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/ray_tune_basic.py

@@ -0,0 +1,15 @@
+"""Tests that Ray Tune works with the working_dir set in Jobs.
+
+Ray Tune internally sets environment variables using runtime_env.
+If the inherited internal runtime environment overwrites the working_dir
+from jobs with an empty working_dir, this test will fail.  See #25484"""
+from ray_tune_dependency import foo
+
+from ray import tune
+
+
+def objective(*args):
+    foo()
+
+
+tune.run(objective)

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/ray_tune_dependency.py

@@ -0,0 +1,5 @@
+"""A file dependency for testing working_dir behavior with Ray Tune."""
+
+
+def foo():
+    pass

python/ray/dashboard/modules/job/tests/subprocess_driver_scripts/script_with_exception.py

@@ -0,0 +1,6 @@
+def run():
+    raise Exception("Script failed with exception !")
+
+
+if __name__ == "__main__":
+    run()

python/ray/dashboard/modules/job/tests/test_backwards_compatibility.py

@@ -0,0 +1,124 @@
+import logging
+import os
+import subprocess
+import sys
+import uuid
+from contextlib import contextmanager
+
+import pytest
+
+from ray.job_submission import JobStatus, JobSubmissionClient
+
+logger = logging.getLogger(__name__)
+
+
+@contextmanager
+def conda_env(env_name):
+    # Set env name for shell script
+    os.environ["JOB_COMPATIBILITY_TEST_TEMP_ENV"] = env_name
+    # Delete conda env if it already exists
+    try:
+        yield
+    finally:
+        # Clean up created conda env upon test exit to prevent leaking
+        del os.environ["JOB_COMPATIBILITY_TEST_TEMP_ENV"]
+        subprocess.run(
+            f"conda env remove -y --name {env_name}", shell=True, stdout=subprocess.PIPE
+        )
+
+
+def _compatibility_script_path(file_name: str) -> str:
+    return os.path.join(
+        os.path.dirname(__file__), "backwards_compatibility_scripts", file_name
+    )
+
+
+class TestBackwardsCompatibility:
+    @pytest.mark.skipif(
+        sys.platform == "darwin",
+        reason="ray 2.0.1 runs differently on apple silicon than today's.",
+    )
+    def test_cli(self):
+        """
+        Test that the current commit's CLI works with old server-side Ray versions.
+
+        1) Create a new conda environment with old ray version X installed;
+            inherits same env as current conda envionment except ray version
+        2) (Server) Start head node and dashboard with old ray version X
+        3) (Client) Use current commit's CLI code to do sample job submission flow
+        4) Deactivate the new conda environment and back to original place
+        """
+        # Shell script creates and cleans up tmp conda environment regardless
+        # of the outcome
+        env_name = f"jobs-backwards-compatibility-{uuid.uuid4().hex}"
+        with conda_env(env_name):
+            shell_cmd = f"{_compatibility_script_path('test_backwards_compatibility.sh')}"  # noqa: E501
+
+            try:
+                subprocess.check_output(shell_cmd, shell=True, stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                logger.error(str(e))
+                logger.error(e.stdout.decode())
+                raise e
+
+
+@pytest.mark.skipif(
+    os.environ.get("JOB_COMPATIBILITY_TEST_TEMP_ENV") is None,
+    reason="This test is only meant to be run from the "
+    "test_backwards_compatibility.sh shell script.",
+)
+def test_error_message():
+    """
+    Check that we get a good error message when running against an old server version.
+    """
+    # Import lazily so the module still loads when the compatibility script
+    # installs an older Ray that does not expose `ray._common`.
+    from ray._common.test_utils import wait_for_condition
+
+    client = JobSubmissionClient("http://127.0.0.1:8265")
+
+    # Check that a basic job successfully runs.
+    job_id = client.submit_job(
+        entrypoint="echo 'hello world'",
+    )
+    wait_for_condition(lambda: client.get_job_status(job_id) == JobStatus.SUCCEEDED)
+
+    # `entrypoint_num_cpus`, `entrypoint_num_gpus`, `entrypoint_resources`, and
+    # `entrypoint_label_selector`
+    # are not supported in ray<2.2.0.
+    for unsupported_submit_kwargs in [
+        {"entrypoint_num_cpus": 1},
+        {"entrypoint_num_gpus": 1},
+        {"entrypoint_resources": {"custom": 1}},
+        {"entrypoint_label_selector": {"fragile_node": "!1"}},
+    ]:
+        with pytest.raises(
+            Exception,
+            match="Ray version 2.0.1 is running on the cluster. "
+            "`entrypoint_num_cpus`, `entrypoint_num_gpus`, "
+            "`entrypoint_resources`, and `entrypoint_label_selector` kwargs"
+            " are not supported on the Ray cluster. Please ensure the cluster is "
+            "running Ray 2.2 or higher.",
+        ):
+            client.submit_job(
+                entrypoint="echo hello",
+                **unsupported_submit_kwargs,
+            )
+
+    with pytest.raises(
+        Exception,
+        match="Ray version 2.0.1 is running on the cluster. "
+        "`entrypoint_memory` kwarg"
+        " is not supported on the Ray cluster. Please ensure the cluster is "
+        "running Ray 2.8 or higher.",
+    ):
+        client.submit_job(
+            entrypoint="echo hello",
+            entrypoint_memory=4,
+        )
+
+    assert True
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_cli.py

@@ -0,0 +1,695 @@
+import json
+import logging
+import os
+import shlex
+import sys
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from subprocess import list2cmdline
+from typing import Optional
+from unittest import mock
+
+import pytest
+import yaml
+from click.testing import CliRunner
+
+from ray.dashboard.modules.job.cli import job_cli_group
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture
+def mock_sdk_client():
+    class AsyncIterator:
+        def __init__(self, seq):
+            self._seq = seq
+            self.iter = iter(self._seq)
+
+        def __aiter__(self):
+            return self
+
+        async def __anext__(self):
+            try:
+                return next(self.iter)
+            except StopIteration:
+                self.iter = iter(self._seq)
+                raise StopAsyncIteration
+
+    if "RAY_ADDRESS" in os.environ:
+        del os.environ["RAY_ADDRESS"]
+    with mock.patch("ray.dashboard.modules.job.cli.JobSubmissionClient") as mock_client:
+        # In python 3.6 it will fail with error
+        # 'async for' requires an object with __aiter__ method, got MagicMock"
+        mock_client().tail_job_logs.return_value = AsyncIterator(range(10))
+
+        # We need to return a string for the address and not a MagicMock
+        mock_client().get_address.return_value = ""
+
+        yield mock_client
+
+
+@pytest.fixture
+def runtime_env_formats():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+
+        test_env = {
+            "working_dir": "s3://bogus.zip",
+            "conda": "conda_env",
+            "pip": ["pip-install-test"],
+            "env_vars": {"hi": "hi2"},
+        }
+
+        yaml_file = path / "env.yaml"
+        with yaml_file.open(mode="w") as f:
+            yaml.dump(test_env, f)
+
+        yield test_env, json.dumps(test_env), yaml_file
+
+
+@contextmanager
+def set_env_var(key: str, val: Optional[str] = None):
+    old_val = os.environ.get(key, None)
+    if val is not None:
+        os.environ[key] = val
+    elif key in os.environ:
+        del os.environ[key]
+
+    yield
+
+    if key in os.environ:
+        del os.environ[key]
+    if old_val is not None:
+        os.environ[key] = old_val
+
+
+def check_exit_code(result, exit_code):
+    assert result.exit_code == exit_code, result.output
+
+
+def _expected_entrypoint(*args):
+    """Return the expected entrypoint string for the current platform.
+
+    On Windows, the CLI uses subprocess.list2cmdline (double quotes).
+    On POSIX, it uses shlex.join (single quotes).
+    """
+    if sys.platform == "win32":
+        return list2cmdline(args)
+    return shlex.join(args)
+
+
+def _job_cli_group_test_address(mock_sdk_client, cmd, *args):
+    runner = CliRunner()
+
+    create_cluster_if_needed = True if cmd == "submit" else False
+    # Test passing address via command line.
+    result = runner.invoke(job_cli_group, [cmd, "--address=arg_addr", *args])
+    mock_sdk_client.assert_called_with(
+        "arg_addr", create_cluster_if_needed, headers=None, verify=True
+    )
+    with pytest.raises(AssertionError):
+        mock_sdk_client.assert_called_with(
+            "some_other_addr", True, headers=None, verify=True
+        )
+    check_exit_code(result, 0)
+    # Test passing address via env var.
+    with set_env_var("RAY_ADDRESS", "env_addr"):
+        result = runner.invoke(job_cli_group, [cmd, *args])
+        check_exit_code(result, 0)
+        # RAY_ADDRESS is read inside the SDK client.
+        mock_sdk_client.assert_called_with(
+            None, create_cluster_if_needed, headers=None, verify=True
+        )
+    # Test passing no address.
+    result = runner.invoke(job_cli_group, [cmd, *args])
+    check_exit_code(result, 0)
+    mock_sdk_client.assert_called_with(
+        None, create_cluster_if_needed, headers=None, verify=True
+    )
+
+
+class TestList:
+    def test_address(self, mock_sdk_client):
+        _job_cli_group_test_address(mock_sdk_client, "list")
+
+    def test_list(self, mock_sdk_client):
+        runner = CliRunner()
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["list"],
+            )
+            check_exit_code(result, 0)
+
+            result = runner.invoke(job_cli_group, ["submit", "--", "echo hello"])
+            check_exit_code(result, 0)
+
+            result = runner.invoke(
+                job_cli_group,
+                ["list"],
+            )
+            check_exit_code(result, 0)
+
+
+class TestSubmit:
+    def test_address(self, mock_sdk_client):
+        _job_cli_group_test_address(mock_sdk_client, "submit", "--", "echo", "hello")
+
+    def test_working_dir(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(job_cli_group, ["submit", "--", "echo hello"])
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--working-dir", "blah", "--", "echo hello"],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={"working_dir": "blah"},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            result = runner.invoke(
+                job_cli_group, ["submit", "--working-dir='.'", "--", "echo hello"]
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={"working_dir": "'.'"},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    def test_runtime_env(self, mock_sdk_client, runtime_env_formats):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+        env_dict, env_json, env_yaml = runtime_env_formats
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            # Test passing via file.
+            result = runner.invoke(
+                job_cli_group, ["submit", "--runtime-env", env_yaml, "--", "echo hello"]
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env=env_dict,
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            # Test passing via json.
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--runtime-env-json", env_json, "--", "echo hello"],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env=env_dict,
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            # Test passing both throws an error.
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    "--runtime-env",
+                    env_yaml,
+                    "--runtime-env-json",
+                    env_json,
+                    "--",
+                    "echo hello",
+                ],
+            )
+            check_exit_code(result, 1)
+            assert "Only one of" in str(result.exception)
+
+            # Test overriding working_dir.
+            env_dict.update(working_dir=".")
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    "--runtime-env",
+                    env_yaml,
+                    "--working-dir",
+                    ".",
+                    "--",
+                    "echo hello",
+                ],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env=env_dict,
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    "--runtime-env-json",
+                    env_json,
+                    "--working-dir",
+                    ".",
+                    "--",
+                    "echo hello",
+                ],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env=env_dict,
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    def test_job_id(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(job_cli_group, ["submit", "--", "echo hello"])
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--submission-id=my_job_id", "--", "echo hello"],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id="my_job_id",
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    def test_entrypoint_num_cpus(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--entrypoint-num-cpus=2", "--", "echo hello"],
+            )
+            assert result.exit_code == 0
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=2,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    def test_entrypoint_num_gpus(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--entrypoint-num-gpus=2", "--", "echo hello"],
+            )
+            assert result.exit_code == 0
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=2,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    def test_entrypoint_memory(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--entrypoint-memory=4", "--", "echo hello"],
+            )
+            assert result.exit_code == 0
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=4,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+            )
+
+    @pytest.mark.parametrize(
+        "resources",
+        [
+            ("--entrypoint-num-cpus=2", {"entrypoint_num_cpus": 2}),
+            ("--entrypoint-num-gpus=2", {"entrypoint_num_gpus": 2}),
+            (
+                """--entrypoint-resources={"Custom":3}""",
+                {"entrypoint_resources": {"Custom": 3}},
+            ),
+        ],
+    )
+    def test_entrypoint_resources(self, mock_sdk_client, resources):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", resources[0], "--", "echo hello"],
+            )
+            print(result.output)
+            assert result.exit_code == 0
+            expected_kwargs = {
+                "entrypoint": _expected_entrypoint("echo hello"),
+                "submission_id": None,
+                "runtime_env": {},
+                "metadata": None,
+                "entrypoint_num_cpus": None,
+                "entrypoint_num_gpus": None,
+                "entrypoint_memory": None,
+                "entrypoint_resources": None,
+                "entrypoint_label_selector": None,
+            }
+            expected_kwargs.update(resources[1])
+            mock_client_instance.submit_job.assert_called_with(**expected_kwargs)
+
+    def test_entrypoint_resources_invalid_json(self, mock_sdk_client):
+        runner = CliRunner()
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    """--entrypoint-resources={"Custom":3""",
+                    "--",
+                    "echo hello world",
+                ],
+            )
+            print(result.output)
+            assert result.exit_code == 1
+            assert "not a valid JSON string" in result.output
+
+    def test_entrypoint_label_selector(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    """--entrypoint-label-selector={"fragile_node":"!1"}""",
+                    "--",
+                    "echo hello",
+                ],
+            )
+            assert result.exit_code == 0
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                metadata=None,
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector={"fragile_node": "!1"},
+            )
+
+    def test_metadata(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    "--metadata-json",
+                    '{"key": "value"}',
+                    "--",
+                    "echo hello",
+                ],
+            )
+            check_exit_code(result, 0)
+            mock_client_instance.submit_job.assert_called_with(
+                entrypoint=_expected_entrypoint("echo hello"),
+                submission_id=None,
+                runtime_env={},
+                entrypoint_num_cpus=None,
+                entrypoint_num_gpus=None,
+                entrypoint_memory=None,
+                entrypoint_resources=None,
+                entrypoint_label_selector=None,
+                metadata={"key": "value"},
+            )
+
+    def test_metadata_invalid_json(self, mock_sdk_client):
+        runner = CliRunner()
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                [
+                    "submit",
+                    "--metadata-json",
+                    '{"key": "value"',
+                    "--",
+                    "echo hello",
+                ],
+            )
+            print(result.output)
+            check_exit_code(result, 1)
+            assert "not a valid JSON string" in result.output
+
+    @pytest.mark.parametrize(
+        "cli_val, verify_param",
+        [
+            ("True", True),
+            ("true", True),
+            ("1", True),
+            ("False", False),
+            ("false", False),
+            ("0", False),
+            ("a/rel/path", "a/rel/path"),
+            ("/an/abs/path", "/an/abs/path"),
+        ],
+    )
+    def test_entrypoint_verify(self, mock_sdk_client, cli_val, verify_param):
+        runner = CliRunner()
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", f"--verify={cli_val}", "--", "echo hello"],
+            )
+            assert result.exit_code == 0
+            mock_sdk_client.assert_called_with(
+                None, True, headers=None, verify=verify_param
+            )
+
+
+class TestDelete:
+    def test_address(self, mock_sdk_client):
+        _job_cli_group_test_address(mock_sdk_client, "delete", "fake_job_id")
+
+    def test_delete(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(job_cli_group, ["delete", "job_id"])
+            check_exit_code(result, 0)
+            mock_client_instance.delete_job.assert_called_with("job_id")
+
+
+class TestStatus:
+    def test_address(self, mock_sdk_client):
+        _job_cli_group_test_address(mock_sdk_client, "status", "fake_job_id")
+
+    def test_status(self, mock_sdk_client):
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(job_cli_group, ["status", "job_id"])
+            check_exit_code(result, 0)
+            mock_client_instance.get_job_info.assert_called_with("job_id")
+
+
+class TestEntrypointShellQuoting:
+    """Regression test for https://github.com/ray-project/ray/issues/56232.
+
+    `ray job submit` previously used `subprocess.list2cmdline` unconditionally
+    to join entrypoint arguments. That function wraps arguments in double
+    quotes, which causes POSIX shells on the server to expand $VAR references.
+
+    The fix uses `shlex.join` on POSIX platforms (which single-quotes
+    arguments to prevent expansion) and `list2cmdline` on Windows (which
+    double-quotes arguments as expected by cmd.exe).
+    """
+
+    def test_entrypoint_preserves_shell_variables(self, mock_sdk_client):
+        """Ensure $VAR in entrypoint is single-quoted on POSIX, not double-quoted."""
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            with mock.patch("ray.dashboard.modules.job.cli.sys") as mock_sys:
+                mock_sys.platform = "linux"
+                result = runner.invoke(
+                    job_cli_group,
+                    [
+                        "submit",
+                        "--",
+                        "python",
+                        "-m",
+                        "launcher",
+                        "--config",
+                        "$CONFIG_PATH",
+                    ],
+                )
+                check_exit_code(result, 0)
+                call_kwargs = mock_client_instance.submit_job.call_args
+                entrypoint = call_kwargs.kwargs["entrypoint"]
+
+                # shlex.join must single-quote the $VAR argument so that
+                # the server-side POSIX shell does NOT expand it.
+                assert "'$CONFIG_PATH'" in entrypoint, (
+                    f"Expected single-quoted $CONFIG_PATH in entrypoint, "
+                    f"got: {entrypoint!r}"
+                )
+                # Double quotes around $CONFIG_PATH would cause expansion.
+                assert '"$CONFIG_PATH"' not in entrypoint, (
+                    f"Double-quoted $CONFIG_PATH would be expanded by the "
+                    f"server shell, got: {entrypoint!r}"
+                )
+
+    def test_entrypoint_uses_list2cmdline_on_windows(self, mock_sdk_client):
+        """On Windows, entrypoint should use list2cmdline (double quotes)."""
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            with mock.patch("ray.dashboard.modules.job.cli.sys") as mock_sys:
+                mock_sys.platform = "win32"
+                result = runner.invoke(
+                    job_cli_group,
+                    [
+                        "submit",
+                        "--",
+                        "echo",
+                        "hello world",
+                    ],
+                )
+                check_exit_code(result, 0)
+                call_kwargs = mock_client_instance.submit_job.call_args
+                entrypoint = call_kwargs.kwargs["entrypoint"]
+
+                # list2cmdline wraps args with spaces in double quotes
+                assert (
+                    entrypoint == 'echo "hello world"'
+                ), f"Expected list2cmdline output on Windows, got: {entrypoint!r}"
+
+    def test_entrypoint_simple_args_not_over_quoted(self, mock_sdk_client):
+        """Simple arguments without special chars should not be quoted."""
+        runner = CliRunner()
+        mock_client_instance = mock_sdk_client.return_value
+
+        with set_env_var("RAY_ADDRESS", "env_addr"):
+            result = runner.invoke(
+                job_cli_group,
+                ["submit", "--", "echo", "hello"],
+            )
+            check_exit_code(result, 0)
+            call_kwargs = mock_client_instance.submit_job.call_args
+            entrypoint = call_kwargs.kwargs["entrypoint"]
+            assert entrypoint == "echo hello"
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_cli_integration.py

@@ -0,0 +1,356 @@
+import json
+import logging
+import os
+import subprocess
+import sys
+from contextlib import contextmanager
+from typing import Optional, Tuple
+
+import pytest
+
+import ray
+
+logger = logging.getLogger(__name__)
+
+
+@pytest.fixture
+def shutdown_only():
+    yield None
+    # The code after the yield will run as teardown code.
+    ray.shutdown()
+    # Delete the cluster address just in case.
+    ray._common.utils.reset_ray_address()
+
+
+@contextmanager
+def set_env_var(key: str, val: Optional[str] = None):
+    old_val = os.environ.get(key, None)
+    if val is not None:
+        os.environ[key] = val
+    elif key in os.environ:
+        del os.environ[key]
+
+    try:
+        yield
+    finally:
+        if key in os.environ:
+            del os.environ[key]
+        if old_val is not None:
+            os.environ[key] = old_val
+
+
+@pytest.fixture
+def ray_start_stop():
+    subprocess.check_output(["ray", "start", "--head"])
+    try:
+        with set_env_var("RAY_ADDRESS", "http://127.0.0.1:8265"):
+            yield
+    finally:
+        subprocess.check_output(["ray", "stop", "--force"])
+
+
+@contextmanager
+def ray_cluster_manager():
+    """
+    Used not as fixture in case we want to set RAY_ADDRESS first.
+    """
+    subprocess.check_output(["ray", "start", "--head"])
+    try:
+        yield
+    finally:
+        subprocess.check_output(["ray", "stop", "--force"])
+
+
+def _run_cmd(cmd: str, should_fail=False) -> Tuple[str, str]:
+    """Convenience wrapper for subprocess.run.
+
+    We always run with shell=True to simulate the CLI.
+
+    Asserts that the process succeeds/fails depending on should_fail.
+
+    Returns (stdout, stderr).
+    """
+    print(f"Running command: '{cmd}'")
+    p: subprocess.CompletedProcess = subprocess.run(
+        cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+    )
+    if p.returncode == 0:
+        print("Command succeeded.")
+        if should_fail:
+            raise RuntimeError(
+                f"Expected command to fail, but got exit code: {p.returncode}."
+            )
+    else:
+        print(f"Command failed with exit code: {p.returncode}.")
+        if not should_fail:
+            raise RuntimeError(
+                f"Expected command to succeed, but got exit code: {p.returncode}."
+            )
+
+    return p.stdout.decode("utf-8"), p.stderr.decode("utf-8")
+
+
+class TestJobSubmitHook:
+    """Tests the RAY_JOB_SUBMIT_HOOK env var."""
+
+    def test_hook(self, ray_start_stop):
+        with set_env_var("RAY_JOB_SUBMIT_HOOK", "ray._private.test_utils.job_hook"):
+            stdout, _ = _run_cmd("ray job submit -- echo hello")
+            assert "hook intercepted: echo hello" in stdout
+
+
+class TestRayJobHeaders:
+    """
+    Integration version of job CLI test that ensures interaction with the
+    following components are working as expected:
+    1) Ray client: use of RAY_JOB_HEADERS and ray.init() in job_head.py
+    2) Ray dashboard: `ray start --head`
+    """
+
+    def test_empty_ray_job_headers(self, ray_start_stop):
+        with set_env_var("RAY_JOB_HEADERS", None):
+            stdout, _ = _run_cmd("ray job submit -- echo hello")
+            assert "hello" in stdout
+            assert "succeeded" in stdout
+
+    @pytest.mark.parametrize("ray_job_headers", ['{"key": "value"}'])
+    def test_ray_job_headers(self, ray_start_stop, ray_job_headers: str):
+        with set_env_var("RAY_JOB_HEADERS", ray_job_headers):
+            _run_cmd("ray job submit -- echo hello", should_fail=False)
+
+    @pytest.mark.parametrize("ray_job_headers", ["{key value}"])
+    def test_ray_incorrectly_formatted_job_headers(
+        self, ray_start_stop, ray_job_headers: str
+    ):
+        with set_env_var("RAY_JOB_HEADERS", ray_job_headers):
+            _run_cmd("ray job submit -- echo hello", should_fail=True)
+
+
+class TestRayAddress:
+    """
+    Integration version of job CLI test that ensures interaction with the
+    following components are working as expected:
+
+    1) Ray client: use of RAY_ADDRESS and ray.init() in job_head.py
+    2) Ray dashboard: `ray start --head`
+    """
+
+    def test_empty_ray_address(self, ray_start_stop):
+        with set_env_var("RAY_ADDRESS", None):
+            stdout, _ = _run_cmd("ray job submit -- echo hello")
+            assert "hello" in stdout
+            assert "succeeded" in stdout
+
+    @pytest.mark.parametrize(
+        "ray_api_server_address,should_fail",
+        [
+            ("http://127.0.0.1:8265", False),  # correct API server
+            ("127.0.0.1:8265", True),  # wrong format without http
+            ("http://127.0.0.1:9999", True),  # wrong port
+        ],
+    )
+    def test_ray_api_server_address(
+        self,
+        ray_start_stop,
+        ray_api_server_address: str,
+        should_fail: bool,
+    ):
+        # Set a `RAY_ADDRESS` that would not work with the `ray job submit` CLI because it uses the `ray://` prefix.
+        # This verifies that the `RAY_API_SERVER_ADDRESS` env var takes precedence.
+        with set_env_var("RAY_ADDRESS", "ray://127.0.0.1:8265"):
+            with set_env_var("RAY_API_SERVER_ADDRESS", ray_api_server_address):
+                _run_cmd("ray job submit -- echo hello", should_fail=should_fail)
+
+    @pytest.mark.parametrize(
+        "ray_client_address,should_fail",
+        [
+            ("127.0.0.1:8265", True),
+            ("ray://127.0.0.1:8265", True),
+            ("http://127.0.0.1:8265", False),
+        ],
+    )
+    def test_ray_client_address(
+        self, ray_start_stop, ray_client_address: str, should_fail: bool
+    ):
+        with set_env_var("RAY_ADDRESS", ray_client_address):
+            _run_cmd("ray job submit -- echo hello", should_fail=should_fail)
+
+    def test_valid_http_ray_address(self, ray_start_stop):
+        stdout, _ = _run_cmd("ray job submit -- echo hello")
+        assert "hello" in stdout
+        assert "succeeded" in stdout
+
+
+class TestJobSubmit:
+    def test_basic_submit(self, ray_start_stop):
+        """Should tail logs and wait for process to exit."""
+        cmd = "sleep 1 && echo hello && sleep 1 && echo hello"
+        stdout, _ = _run_cmd(f"ray job submit -- bash -c '{cmd}'")
+
+        # 'hello' should appear four times: twice when we print the entrypoint, then
+        # two more times in the logs from the `echo`.
+        assert stdout.count("hello") == 4
+        assert "succeeded" in stdout
+
+    def test_submit_no_wait(self, ray_start_stop):
+        """Should exit immediately w/o printing logs."""
+        cmd = "echo hello && sleep 1000"
+        stdout, _ = _run_cmd(f"ray job submit --no-wait -- bash -c '{cmd}'")
+        assert "hello" not in stdout
+        assert "Tailing logs until the job exits" not in stdout
+
+    def test_submit_with_logs_instant_job(self, ray_start_stop):
+        """Should exit immediately and print logs even if job returns instantly."""
+        cmd = "echo hello"
+        stdout, _ = _run_cmd(f"ray job submit -- bash -c '{cmd}'")
+
+        # 'hello' should appear twice: once when we print the entrypoint, then
+        # again from the `echo`.
+        assert stdout.count("hello") == 2
+
+    def test_multiple_ray_init(self, ray_start_stop):
+        cmd = (
+            "python -c 'import ray; ray.init(); ray.shutdown(); "
+            "ray.init(); ray.shutdown();'"
+        )
+        stdout, _ = _run_cmd(f"ray job submit -- {cmd}")
+        assert "succeeded" in stdout
+
+    def test_metadata(self, ray_start_stop):
+        cmd = "echo hello"
+        stdout, _ = _run_cmd(
+            f'ray job submit --metadata-json=\'{{"key": "value"}}\' -- {cmd}'
+        )
+        assert "hello" in stdout
+        assert "succeeded" in stdout
+
+    def test_job_failed(self, ray_start_stop):
+        cmd = "python -c 'import ray; ray.init(); assert 1 == 2;'"
+        _run_cmd(f"ray job submit -- {cmd}", should_fail=True)
+
+
+class TestRuntimeEnv:
+    def test_bad_runtime_env(self, ray_start_stop):
+        """Should fail with helpful error if runtime env setup fails."""
+        stdout, _ = _run_cmd(
+            'ray job submit --runtime-env-json=\'{"pip": '
+            '["does-not-exist"]}\' -- echo hi',
+            should_fail=True,
+        )
+        assert "Tailing logs until the job exits" in stdout
+        assert "runtime_env setup failed" in stdout
+        assert "No matching distribution found for does-not-exist" in stdout
+
+
+class TestJobStop:
+    def test_basic_stop(self, ray_start_stop):
+        """Should wait until the job is stopped."""
+        cmd = "sleep 1000"
+        job_id = "test_basic_stop"
+        _run_cmd(f"ray job submit --no-wait --job-id={job_id} -- {cmd}")
+
+        stdout, _ = _run_cmd(f"ray job stop {job_id}")
+        assert "Waiting for job" in stdout
+        assert f"Job '{job_id}' was stopped" in stdout
+
+    def test_stop_no_wait(self, ray_start_stop):
+        """Should not wait until the job is stopped."""
+        cmd = "echo hello && sleep 1000"
+        job_id = "test_stop_no_wait"
+        _run_cmd(f"ray job submit --no-wait --job-id={job_id} -- bash -c '{cmd}'")
+
+        stdout, _ = _run_cmd(f"ray job stop --no-wait {job_id}")
+        assert "Waiting for job" not in stdout
+        assert f"Job '{job_id}' was stopped" not in stdout
+
+
+class TestJobList:
+    def test_empty(self, ray_start_stop):
+        stdout, _ = _run_cmd("ray job list")
+        assert "[]" in stdout
+
+    def test_list(self, ray_start_stop):
+        _run_cmd("ray job submit --job-id='hello_id' -- echo hello")
+
+        runtime_env = {"env_vars": {"TEST": "123"}}
+        _run_cmd(
+            "ray job submit --job-id='hi_id' "
+            f"--runtime-env-json='{json.dumps(runtime_env)}' -- echo hi"
+        )
+        stdout, _ = _run_cmd("ray job list")
+        assert "123" in stdout
+        assert "hello_id" in stdout
+        assert "hi_id" in stdout
+
+
+class TestJobDelete:
+    def test_basic_delete(self, ray_start_stop):
+        cmd = "sleep 1000"
+        job_id = "test_basic_delete"
+        _run_cmd(f"ray job submit --no-wait --submission-id={job_id} -- {cmd}")
+
+        # Job shouldn't be able to be deleted because it is not in a terminal state.
+        stdout, stderr = _run_cmd(f"ray job delete {job_id}", should_fail=True)
+        assert "it is in a non-terminal state" in stderr
+
+        # Submit a job that finishes quickly.
+        cmd = "echo hello"
+        job_id = "test_basic_delete_quick"
+        _run_cmd(f"ray job submit --submission-id={job_id} -- bash -c '{cmd}'")
+
+        # Job should be able to be deleted because it is finished.
+        stdout, _ = _run_cmd(f"ray job delete {job_id}")
+        assert f"Job '{job_id}' deleted successfully" in stdout
+
+
+class TestJobStatus:
+    # `ray job status` should exit with 0 if the job exists and non-zero if it doesn't.
+    # This is the contract between Ray and KubRay v1.3.0.
+    def test_status_job_exists(self, ray_start_stop):
+        cmd = "echo hello"
+        job_id = "test_job_id"
+        _run_cmd(
+            f"ray job submit --submission-id={job_id} -- bash -c '{cmd}'",
+            should_fail=False,
+        )
+        _run_cmd(f"ray job status {job_id}", should_fail=False)
+
+    def test_status_job_does_not_exist(self, ray_start_stop):
+        job_id = "test_job_id"
+        _run_cmd(f"ray job status {job_id}", should_fail=True)
+
+
+def test_quote_escaping(ray_start_stop):
+    cmd = "echo \"hello 'world'\""
+    job_id = "test_quote_escaping"
+    stdout, _ = _run_cmd(
+        f"ray job submit --job-id={job_id} -- {cmd}",
+    )
+    assert "hello 'world'" in stdout
+
+
+def test_resources(shutdown_only):
+    ray.init(num_cpus=1, num_gpus=1, resources={"Custom": 1}, _memory=4)
+
+    # Check the case of too many resources.
+    for id, arg in [
+        ("entrypoint_num_cpus", "--entrypoint-num-cpus=2"),
+        ("entrypoint_num_gpus", "--entrypoint-num-gpus=2"),
+        ("entrypoint_memory", "--entrypoint-memory=5"),
+        ("entrypoint_resources", "--entrypoint-resources='{\"Custom\": 2}'"),
+    ]:
+        _run_cmd(f"ray job submit --submission-id={id} --no-wait {arg} -- echo hi")
+        stdout, _ = _run_cmd(f"ray job status {id}")
+        assert "waiting for resources" in stdout
+
+    # Check the case of sufficient resources.
+    stdout, _ = _run_cmd(
+        "ray job submit --entrypoint-num-cpus=1 "
+        "--entrypoint-num-gpus=1 --entrypoint-memory=4 --entrypoint-resources='{"
+        '"Custom": 1}\' -- echo hello',
+    )
+    assert "hello" in stdout
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_common.py

@@ -0,0 +1,302 @@
+import asyncio
+import json
+from dataclasses import asdict
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from google.protobuf.json_format import Parse
+
+from ray.core.generated.gcs_pb2 import JobsAPIInfo
+from ray.dashboard.modules.job.common import (
+    JobErrorType,
+    JobInfo,
+    JobInfoStorageClient,
+    JobStatus,
+    JobSubmitRequest,
+    http_uri_components_to_uri,
+    uri_to_http_components,
+    validate_request_type,
+)
+
+
+class TestJobSubmitRequestValidation:
+    def test_validate_entrypoint(self):
+        r = validate_request_type({"entrypoint": "abc"}, JobSubmitRequest)
+        assert r.entrypoint == "abc"
+
+        with pytest.raises(TypeError, match="required positional argument"):
+            validate_request_type({}, JobSubmitRequest)
+
+        with pytest.raises(TypeError, match="must be a string"):
+            validate_request_type({"entrypoint": 123}, JobSubmitRequest)
+
+    def test_validate_submission_id(self):
+        r = validate_request_type({"entrypoint": "abc"}, JobSubmitRequest)
+        assert r.entrypoint == "abc"
+        assert r.submission_id is None
+
+        r = validate_request_type(
+            {"entrypoint": "abc", "submission_id": "123"}, JobSubmitRequest
+        )
+        assert r.entrypoint == "abc"
+        assert r.submission_id == "123"
+
+        with pytest.raises(TypeError, match="must be a string"):
+            validate_request_type(
+                {"entrypoint": 123, "submission_id": 1}, JobSubmitRequest
+            )
+
+    def test_validate_runtime_env(self):
+        r = validate_request_type({"entrypoint": "abc"}, JobSubmitRequest)
+        assert r.entrypoint == "abc"
+        assert r.runtime_env is None
+
+        r = validate_request_type(
+            {"entrypoint": "abc", "runtime_env": {"hi": "hi2"}}, JobSubmitRequest
+        )
+        assert r.entrypoint == "abc"
+        assert r.runtime_env == {"hi": "hi2"}
+
+        with pytest.raises(TypeError, match="must be a dict"):
+            validate_request_type(
+                {"entrypoint": "abc", "runtime_env": 123}, JobSubmitRequest
+            )
+
+        with pytest.raises(TypeError, match="keys must be strings"):
+            validate_request_type(
+                {"entrypoint": "abc", "runtime_env": {1: "hi"}}, JobSubmitRequest
+            )
+
+    def test_validate_metadata(self):
+        r = validate_request_type({"entrypoint": "abc"}, JobSubmitRequest)
+        assert r.entrypoint == "abc"
+        assert r.metadata is None
+
+        r = validate_request_type(
+            {"entrypoint": "abc", "metadata": {"hi": "hi2"}}, JobSubmitRequest
+        )
+        assert r.entrypoint == "abc"
+        assert r.metadata == {"hi": "hi2"}
+
+        with pytest.raises(TypeError, match="must be a dict"):
+            validate_request_type(
+                {"entrypoint": "abc", "metadata": 123}, JobSubmitRequest
+            )
+
+        with pytest.raises(TypeError, match="keys must be strings"):
+            validate_request_type(
+                {"entrypoint": "abc", "metadata": {1: "hi"}}, JobSubmitRequest
+            )
+
+        with pytest.raises(TypeError, match="values must be strings"):
+            validate_request_type(
+                {"entrypoint": "abc", "metadata": {"hi": 1}}, JobSubmitRequest
+            )
+
+    def test_validate_entrypoint_label_selector(self):
+        r = validate_request_type(
+            {
+                "entrypoint": "abc",
+                "entrypoint_label_selector": {"fragile_node": "!1"},
+            },
+            JobSubmitRequest,
+        )
+        assert r.entrypoint_label_selector == {"fragile_node": "!1"}
+
+        with pytest.raises(TypeError, match="must be a dict"):
+            validate_request_type(
+                {"entrypoint": "abc", "entrypoint_label_selector": "bad"},
+                JobSubmitRequest,
+            )
+
+        with pytest.raises(TypeError, match="keys must be strings"):
+            validate_request_type(
+                {"entrypoint": "abc", "entrypoint_label_selector": {1: "bad"}},
+                JobSubmitRequest,
+            )
+
+        with pytest.raises(TypeError, match="values must be strings"):
+            validate_request_type(
+                {"entrypoint": "abc", "entrypoint_label_selector": {"k": 1}},
+                JobSubmitRequest,
+            )
+
+    def test_entrypoint_resources_disallow_strings(self):
+        with pytest.raises(TypeError, match="values must be numbers"):
+            validate_request_type(
+                {"entrypoint": "abc", "entrypoint_resources": {"Custom": "1"}},
+                JobSubmitRequest,
+            )
+
+
+def test_uri_to_http_and_back():
+    assert uri_to_http_components("gcs://hello.zip") == ("gcs", "hello.zip")
+    assert uri_to_http_components("gcs://hello.whl") == ("gcs", "hello.whl")
+
+    with pytest.raises(ValueError, match="'blah' is not a valid Protocol"):
+        uri_to_http_components("blah://halb.zip")
+
+    with pytest.raises(ValueError, match="does not end in .zip or .whl"):
+        assert uri_to_http_components("gcs://hello.not_zip")
+
+    with pytest.raises(ValueError, match="does not end in .zip or .whl"):
+        assert uri_to_http_components("gcs://hello")
+
+    assert http_uri_components_to_uri("gcs", "hello.zip") == "gcs://hello.zip"
+    assert http_uri_components_to_uri("blah", "halb.zip") == "blah://halb.zip"
+    assert http_uri_components_to_uri("blah", "halb.whl") == "blah://halb.whl"
+
+    for original_uri in ["gcs://hello.zip", "gcs://fasdf.whl"]:
+        new_uri = http_uri_components_to_uri(*uri_to_http_components(original_uri))
+        assert new_uri == original_uri
+
+
+def test_dynamic_status_message():
+    info = JobInfo(
+        status=JobStatus.PENDING, entrypoint="echo hi", entrypoint_num_cpus=1
+    )
+    assert "may be waiting for resources" in info.message
+
+    info = JobInfo(
+        status=JobStatus.PENDING, entrypoint="echo hi", entrypoint_num_gpus=1
+    )
+    assert "may be waiting for resources" in info.message
+
+    info = JobInfo(status=JobStatus.PENDING, entrypoint="echo hi", entrypoint_memory=4)
+    assert "may be waiting for resources" in info.message
+
+    info = JobInfo(
+        status=JobStatus.PENDING,
+        entrypoint="echo hi",
+        entrypoint_resources={"Custom": 1},
+    )
+    assert "may be waiting for resources" in info.message
+
+    info = JobInfo(
+        status=JobStatus.PENDING, entrypoint="echo hi", runtime_env={"conda": "env"}
+    )
+    assert "may be waiting for the runtime environment" in info.message
+
+
+def test_job_info_to_json():
+    info = JobInfo(
+        status=JobStatus.PENDING,
+        entrypoint="echo hi",
+        entrypoint_num_cpus=1,
+        entrypoint_num_gpus=1,
+        entrypoint_memory=4,
+        entrypoint_resources={"Custom": 1},
+        runtime_env={"pip": ["pkg"]},
+    )
+    expected_items = {
+        "status": "PENDING",
+        "message": (
+            "Job has not started yet. It may be waiting for resources "
+            "(CPUs, GPUs, memory, custom resources) to become available. "
+            "It may be waiting for the runtime environment to be set up."
+        ),
+        "entrypoint": "echo hi",
+        "entrypoint_num_cpus": 1,
+        "entrypoint_num_gpus": 1,
+        "entrypoint_memory": 4,
+        "entrypoint_resources": {"Custom": 1},
+        "runtime_env_json": '{"pip": ["pkg"]}',
+    }
+
+    # Check that the expected items are in the JSON.
+    assert expected_items.items() <= info.to_json().items()
+
+    new_job_info = JobInfo.from_json(info.to_json())
+    assert new_job_info == info
+
+    # If `status` is just a string, then operations like status.is_terminal()
+    # would fail, so we should make sure that it's a JobStatus.
+    assert isinstance(new_job_info.status, JobStatus)
+
+
+def test_job_info_json_to_proto():
+    """Test that JobInfo JSON can be converted to JobsAPIInfo protobuf."""
+    info = JobInfo(
+        status=JobStatus.PENDING,
+        entrypoint="echo hi",
+        error_type=JobErrorType.JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE,
+        start_time=123,
+        end_time=456,
+        metadata={"hi": "hi2"},
+        entrypoint_num_cpus=1,
+        entrypoint_num_gpus=1,
+        entrypoint_memory=4,
+        entrypoint_resources={"Custom": 1},
+        runtime_env={"pip": ["pkg"]},
+        driver_agent_http_address="http://localhost:1234",
+        driver_node_id="node_id",
+    )
+    info_json = json.dumps(info.to_json())
+    info_proto = Parse(info_json, JobsAPIInfo())
+    assert info_proto.status == "PENDING"
+    assert info_proto.entrypoint == "echo hi"
+    assert info_proto.start_time == 123
+    assert info_proto.end_time == 456
+    assert info_proto.metadata == {"hi": "hi2"}
+    assert info_proto.entrypoint_num_cpus == 1
+    assert info_proto.entrypoint_num_gpus == 1
+    assert info_proto.entrypoint_memory == 4
+    assert info_proto.entrypoint_resources == {"Custom": 1}
+    assert info_proto.runtime_env_json == '{"pip": ["pkg"]}'
+    assert info_proto.message == (
+        "Job has not started yet. It may be waiting for resources "
+        "(CPUs, GPUs, memory, custom resources) to become available. "
+        "It may be waiting for the runtime environment to be set up."
+    )
+    assert info_proto.error_type == "JOB_SUPERVISOR_ACTOR_UNSCHEDULABLE"
+    assert info_proto.driver_agent_http_address == "http://localhost:1234"
+    assert info_proto.driver_node_id == "node_id"
+
+    minimal_info = JobInfo(status=JobStatus.PENDING, entrypoint="echo hi")
+    minimal_info_json = json.dumps(minimal_info.to_json())
+    minimal_info_proto = Parse(minimal_info_json, JobsAPIInfo())
+    assert minimal_info_proto.status == "PENDING"
+    assert minimal_info_proto.entrypoint == "echo hi"
+    for unset_optional_field in [
+        "entrypoint_num_cpus",
+        "entrypoint_num_gpus",
+        "entrypoint_memory",
+        "runtime_env_json",
+        "error_type",
+        "driver_agent_http_address",
+        "driver_node_id",
+    ]:
+        assert not minimal_info_proto.HasField(unset_optional_field)
+
+
+def test_get_all_jobs_filters_out_none_job_info():
+    prefix = JobInfoStorageClient.JOB_DATA_KEY_PREFIX
+    mock_gcs = MagicMock()
+    mock_gcs.async_internal_kv_keys = AsyncMock(
+        return_value=[
+            (prefix + "job1").encode(),
+            (prefix + "job2").encode(),
+        ]
+    )
+
+    storage = JobInfoStorageClient(mock_gcs)
+    job_info_1 = JobInfo(status=JobStatus.RUNNING, entrypoint="echo 1")
+
+    async def mock_get_info(job_id, timeout=30):
+        if job_id == "job1":
+            return job_info_1
+        return None
+
+    storage.get_info = mock_get_info
+
+    result = asyncio.run(storage.get_all_jobs())
+
+    assert result == {"job1": job_info_1}
+    for job_id, job_info in result.items():
+        asdict(job_info)  # This should not raise an exception
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_component_activities.py

@@ -0,0 +1,177 @@
+import json
+import os
+import pprint
+import sys
+
+import jsonschema
+import pytest
+import requests
+
+from ray._common.test_utils import (
+    run_string_as_driver,
+    wait_for_condition,
+)
+from ray._private.test_utils import (
+    format_web_url,
+    run_string_as_driver_nonblocking,
+)
+from ray.dashboard import dashboard
+from ray.dashboard.consts import RAY_CLUSTER_ACTIVITY_HOOK
+from ray.dashboard.modules.job.job_head import RayActivityResponse
+from ray.dashboard.tests.conftest import *  # noqa
+
+
+@pytest.fixture
+def set_ray_cluster_activity_hook(request):
+    """
+    Fixture that sets RAY_CLUSTER_ACTIVITY_HOOK environment variable
+    for test_e2e_component_activities_hook.
+    """
+    external_hook = request.param
+    assert (
+        external_hook
+    ), "Please pass value of RAY_CLUSTER_ACTIVITY_HOOK env var to this fixture"
+    old_hook = os.environ.get(RAY_CLUSTER_ACTIVITY_HOOK)
+    os.environ[RAY_CLUSTER_ACTIVITY_HOOK] = external_hook
+
+    yield external_hook
+
+    if old_hook is not None:
+        os.environ[RAY_CLUSTER_ACTIVITY_HOOK] = old_hook
+    else:
+        del os.environ[RAY_CLUSTER_ACTIVITY_HOOK]
+
+
+@pytest.mark.parametrize(
+    "set_ray_cluster_activity_hook",
+    [
+        "ray._private.test_utils.external_ray_cluster_activity_hook1",
+        "ray._private.test_utils.external_ray_cluster_activity_hook2",
+        "ray._private.test_utils.external_ray_cluster_activity_hook3",
+        "ray._private.test_utils.external_ray_cluster_activity_hook4",
+        "ray._private.test_utils.external_ray_cluster_activity_hook5",
+    ],
+    indirect=True,
+)
+def test_component_activities_hook(set_ray_cluster_activity_hook, call_ray_start):
+    """
+    Tests /api/component_activities returns correctly for various
+    responses of RAY_CLUSTER_ACTIVITY_HOOK.
+
+    Verify no active drivers are correctly reflected in response.
+    """
+    external_hook = set_ray_cluster_activity_hook
+
+    response = requests.get("http://127.0.0.1:8265/api/component_activities")
+    response.raise_for_status()
+
+    # Validate schema of response
+    data = response.json()
+    schema_path = os.path.join(
+        os.path.dirname(dashboard.__file__),
+        "modules/job/component_activities_schema.json",
+    )
+    pprint.pprint(data)
+    jsonschema.validate(instance=data, schema=json.load(open(schema_path)))
+
+    # Validate driver response can be cast to RayActivityResponse object
+    # and that there are no active drivers.
+    driver_ray_activity_response = RayActivityResponse(**data["driver"])
+    assert driver_ray_activity_response.is_active == "INACTIVE"
+    assert driver_ray_activity_response.reason is None
+
+    # Validate external component response can be cast to RayActivityResponse object
+    if external_hook[-1] == "5":
+        external_activity_response = RayActivityResponse(**data["test_component5"])
+        assert external_activity_response.is_active == "ACTIVE"
+        assert external_activity_response.reason == "Counter: 1"
+    elif external_hook[-1] == "4":
+        external_activity_response = RayActivityResponse(**data["external_component"])
+        assert external_activity_response.is_active == "ERROR"
+        assert (
+            "'Error in external cluster activity hook'"
+            in external_activity_response.reason
+        )
+    elif external_hook[-1] == "3":
+        external_activity_response = RayActivityResponse(**data["external_component"])
+        assert external_activity_response.is_active == "ERROR"
+    elif external_hook[-1] == "2":
+        external_activity_response = RayActivityResponse(**data["test_component2"])
+        assert external_activity_response.is_active == "ERROR"
+    elif external_hook[-1] == "1":
+        external_activity_response = RayActivityResponse(**data["test_component1"])
+        assert external_activity_response.is_active == "ACTIVE"
+        assert external_activity_response.reason == "Counter: 1"
+
+        # Call endpoint again to validate different response
+        response = requests.get("http://127.0.0.1:8265/api/component_activities")
+        response.raise_for_status()
+        data = response.json()
+        jsonschema.validate(instance=data, schema=json.load(open(schema_path)))
+
+        external_activity_response = RayActivityResponse(**data["test_component1"])
+        assert external_activity_response.is_active == "ACTIVE"
+        assert external_activity_response.reason == "Counter: 2"
+
+
+def test_active_component_activities(ray_start_with_dashboard):
+    # Verify drivers which don't have namespace starting with _ray_internal_
+    # are considered active.
+
+    webui_url = ray_start_with_dashboard["webui_url"]
+    webui_url = format_web_url(webui_url)
+
+    driver_template = """
+import ray
+
+ray.init(address="auto", namespace="{namespace}")
+import time
+time.sleep({sleep_time_s})
+    """
+    run_string_as_driver(
+        driver_template.format(namespace="my_namespace", sleep_time_s=0)
+    )
+
+    run_string_as_driver_nonblocking(
+        driver_template.format(namespace="my_namespace", sleep_time_s=5)
+    )
+    run_string_as_driver_nonblocking(
+        driver_template.format(namespace="_ray_internal_job_info_id1", sleep_time_s=5)
+    )
+    # Simulate the default driver that gets created by dashboard
+    run_string_as_driver_nonblocking(
+        driver_template.format(namespace="_ray_internal_dashboard", sleep_time_s=5)
+    )
+
+    def verify_driver_response():
+        # Verify drivers are considered active after running script
+        response = requests.get(f"{webui_url}/api/component_activities")
+        response.raise_for_status()
+
+        # Validate schema of response
+        data = response.json()
+        schema_path = os.path.join(
+            os.path.dirname(dashboard.__file__),
+            "modules/job/component_activities_schema.json",
+        )
+
+        jsonschema.validate(instance=data, schema=json.load(open(schema_path)))
+
+        # Validate ray_activity_response field can be cast to RayActivityResponse object
+        driver_ray_activity_response = RayActivityResponse(**data["driver"])
+        print(driver_ray_activity_response)
+
+        assert driver_ray_activity_response.is_active == "ACTIVE"
+        # Drivers with namespace starting with "_ray_internal" are not
+        # considered active drivers. Two active drivers are the second one
+        # run with namespace "my_namespace" and the one started
+        # from ray_start_with_dashboard
+        assert driver_ray_activity_response.reason == "Number of active drivers: 2"
+
+        return True
+
+    wait_for_condition(verify_driver_response)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_http_job_server.py

@@ -0,0 +1,777 @@
+import json
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+from typing import Optional
+from unittest.mock import patch
+
+import pytest
+import requests
+import yaml
+
+import ray
+from ray._common.test_utils import wait_for_condition
+from ray._private.runtime_env.packaging import (
+    create_package,
+    download_and_unpack_package,
+    get_uri_for_file,
+)
+from ray._private.test_utils import (
+    chdir,
+    format_web_url,
+    wait_until_server_available,
+)
+from ray.dashboard.modules.dashboard_sdk import ClusterInfo, parse_cluster_info
+from ray.dashboard.modules.job.common import uri_to_http_components
+from ray.dashboard.modules.job.pydantic_models import JobDetails
+from ray.dashboard.modules.job.tests.test_cli_integration import set_env_var
+from ray.dashboard.modules.version import CURRENT_VERSION
+from ray.dashboard.tests.conftest import *  # noqa
+from ray.job_submission import JobStatus, JobSubmissionClient
+from ray.runtime_env.runtime_env import RuntimeEnv, RuntimeEnvConfig
+from ray.tests.conftest import _ray_start
+
+# This test requires you have AWS credentials set up (any AWS credentials will
+# do, this test only accesses a public bucket).
+
+logger = logging.getLogger(__name__)
+
+DRIVER_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "subprocess_driver_scripts")
+
+
+@pytest.fixture(scope="module")
+def headers():
+    return {"Connection": "keep-alive", "Authorization": "TOK:<MY_TOKEN>"}
+
+
+@pytest.fixture(scope="module")
+def ray_start_context():
+    with _ray_start(include_dashboard=True, num_cpus=1) as ctx:
+        yield ctx
+
+
+@pytest.fixture(scope="module")
+def job_sdk_client(headers, ray_start_context) -> JobSubmissionClient:
+    address = ray_start_context.address_info["webui_url"]
+    assert wait_until_server_available(address)
+    yield JobSubmissionClient(format_web_url(address), headers=headers)
+
+
+@pytest.fixture
+def shutdown_only():
+    yield None
+    # The code after the yield will run as teardown code.
+    ray.shutdown()
+
+
+def test_submit_job_with_resources(shutdown_only):
+    ctx = ray.init(
+        include_dashboard=True,
+        num_cpus=1,
+        num_gpus=1,
+        resources={"Custom": 1},
+        dashboard_port=8269,
+        _memory=4,
+    )
+    address = ctx.address_info["webui_url"]
+    client = JobSubmissionClient(format_web_url(address))
+    # Check the case of too many resources.
+    for kwargs in [
+        {"entrypoint_num_cpus": 2},
+        {"entrypoint_num_gpus": 2},
+        {"entrypoint_memory": 4},
+        {"entrypoint_resources": {"Custom": 2}},
+    ]:
+        job_id = client.submit_job(entrypoint="echo hello", **kwargs)
+        data = client.get_job_info(job_id)
+        assert "waiting for resources" in data.message
+
+    # Check the case of sufficient resources.
+    job_id = client.submit_job(
+        entrypoint="echo hello",
+        entrypoint_num_cpus=1,
+        entrypoint_num_gpus=1,
+        entrypoint_memory=4,
+        entrypoint_resources={"Custom": 1},
+    )
+    wait_for_condition(_check_job_succeeded, client=client, job_id=job_id, timeout=10)
+
+
+@pytest.mark.parametrize("use_sdk", [True, False])
+def test_list_jobs_empty(headers, use_sdk: bool):
+    # Create a cluster using `ray start` instead of `ray.init` to avoid creating a job
+    subprocess.check_output(["ray", "start", "--head"])
+    address = "http://127.0.0.1:8265"
+    try:
+        with set_env_var("RAY_ADDRESS", address):
+            client = JobSubmissionClient(format_web_url(address), headers=headers)
+
+            if use_sdk:
+                assert client.list_jobs() == []
+            else:
+                r = client._do_request(
+                    "GET",
+                    "/api/jobs/",
+                )
+
+                assert r.status_code == 200
+                assert json.loads(r.text) == []
+
+    finally:
+        subprocess.check_output(["ray", "stop", "--force"])
+
+
+@pytest.mark.parametrize("use_sdk", [True, False])
+def test_list_jobs(job_sdk_client: JobSubmissionClient, use_sdk: bool):
+    client = job_sdk_client
+
+    runtime_env = {"env_vars": {"TEST": "123"}}
+    metadata = {"foo": "bar"}
+    entrypoint = "echo hello"
+    submission_id = client.submit_job(
+        entrypoint=entrypoint, runtime_env=runtime_env, metadata=metadata
+    )
+
+    wait_for_condition(_check_job_succeeded, client=client, job_id=submission_id)
+    if use_sdk:
+        info: JobDetails = next(
+            job_info
+            for job_info in client.list_jobs()
+            if job_info.submission_id == submission_id
+        )
+    else:
+        r = client._do_request(
+            "GET",
+            "/api/jobs/",
+        )
+
+        assert r.status_code == 200
+        jobs_info_json = json.loads(r.text)
+        info_json = next(
+            job_info
+            for job_info in jobs_info_json
+            if job_info["submission_id"] == submission_id
+        )
+        info = JobDetails(**info_json)
+
+    assert info.entrypoint == entrypoint
+    assert info.status == JobStatus.SUCCEEDED
+    assert info.message is not None
+    assert info.end_time >= info.start_time
+    assert info.runtime_env == runtime_env
+    assert info.metadata == metadata
+
+    # Test get job status by job / driver id
+    status = client.get_job_status(info.submission_id)
+    assert status == JobStatus.SUCCEEDED
+
+
+def _check_job_succeeded(client: JobSubmissionClient, job_id: str) -> bool:
+    status = client.get_job_status(job_id)
+    if status == JobStatus.FAILED:
+        logs = client.get_job_logs(job_id)
+        raise RuntimeError(
+            f"Job failed\nlogs:\n{logs}, info: {client.get_job_info(job_id)}"
+        )
+    assert status == JobStatus.SUCCEEDED
+    return True
+
+
+def _check_job_failed(client: JobSubmissionClient, job_id: str) -> bool:
+    status = client.get_job_status(job_id)
+    return status == JobStatus.FAILED
+
+
+def _check_job_stopped(client: JobSubmissionClient, job_id: str) -> bool:
+    status = client.get_job_status(job_id)
+    return status == JobStatus.STOPPED
+
+
+@pytest.fixture(
+    scope="module",
+    params=[
+        "no_working_dir",
+        "local_working_dir",
+        "s3_working_dir",
+        "local_py_modules",
+        "working_dir_and_local_py_modules_whl",
+        "local_working_dir_zip",
+        "pip_txt",
+        "conda_yaml",
+        "local_py_modules",
+    ],
+)
+def runtime_env_option(request):
+    import_in_task_script = """
+import ray
+ray.init(address="auto")
+
+@ray.remote
+def f():
+    import pip_install_test
+
+ray.get(f.remote())
+"""
+    if request.param == "no_working_dir":
+        yield {
+            "runtime_env": {},
+            "entrypoint": "echo hello",
+            "expected_logs": "hello\n",
+        }
+    elif request.param in {
+        "local_working_dir",
+        "local_working_dir_zip",
+        "local_py_modules",
+        "working_dir_and_local_py_modules_whl",
+    }:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            path = Path(tmp_dir)
+
+            hello_file = path / "test.py"
+            with hello_file.open(mode="w") as f:
+                f.write("from test_module import run_test\n")
+                f.write("print(run_test())")
+
+            module_path = path / "test_module"
+            module_path.mkdir(parents=True)
+
+            test_file = module_path / "test.py"
+            with test_file.open(mode="w") as f:
+                f.write("def run_test():\n")
+                f.write("    return 'Hello from test_module!'\n")  # noqa: Q000
+
+            init_file = module_path / "__init__.py"
+            with init_file.open(mode="w") as f:
+                f.write("from test_module.test import run_test\n")
+
+            if request.param == "local_working_dir":
+                yield {
+                    "runtime_env": {"working_dir": tmp_dir},
+                    "entrypoint": "python test.py",
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "local_working_dir_zip":
+                local_zipped_dir = shutil.make_archive(
+                    os.path.join(tmp_dir, "test"), "zip", tmp_dir
+                )
+                yield {
+                    "runtime_env": {"working_dir": local_zipped_dir},
+                    "entrypoint": "python test.py",
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "local_py_modules":
+                yield {
+                    "runtime_env": {"py_modules": [str(Path(tmp_dir) / "test_module")]},
+                    "entrypoint": (
+                        "python -c 'import test_module;print(test_module.run_test())'"
+                    ),
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "working_dir_and_local_py_modules_whl":
+                yield {
+                    "runtime_env": {
+                        "working_dir": "s3://runtime-env-test/script_runtime_env.zip",
+                        "py_modules": [
+                            Path(os.path.dirname(__file__))
+                            / "pip_install_test-0.5-py3-none-any.whl"
+                        ],
+                    },
+                    "entrypoint": (
+                        "python script.py && python -c 'import pip_install_test'"
+                    ),
+                    "expected_logs": (
+                        "Executing main() from script.py !!\n"
+                        "Good job!  You installed a pip module."
+                    ),
+                }
+            else:
+                raise ValueError(f"Unexpected pytest fixture option {request.param}")
+    elif request.param == "s3_working_dir":
+        yield {
+            "runtime_env": {
+                "working_dir": "s3://runtime-env-test/script_runtime_env.zip",
+            },
+            "entrypoint": "python script.py",
+            "expected_logs": "Executing main() from script.py !!\n",
+        }
+    elif request.param == "pip_txt":
+        with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
+            pip_list = ["pip-install-test==0.5"]
+            relative_filepath = "requirements.txt"
+            pip_file = Path(relative_filepath)
+            pip_file.write_text("\n".join(pip_list))
+            runtime_env = {"pip": {"packages": relative_filepath, "pip_check": False}}
+            yield {
+                "runtime_env": runtime_env,
+                "entrypoint": (
+                    f"python -c 'import pip_install_test' && "
+                    f"python -c '{import_in_task_script}'"
+                ),
+                "expected_logs": "Good job!  You installed a pip module.",
+            }
+    elif request.param == "conda_yaml":
+        with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
+            conda_dict = {"dependencies": ["pip", {"pip": ["pip-install-test==0.5"]}]}
+            relative_filepath = "environment.yml"
+            conda_file = Path(relative_filepath)
+            conda_file.write_text(yaml.dump(conda_dict))
+            runtime_env = {"conda": relative_filepath}
+
+            yield {
+                "runtime_env": runtime_env,
+                "entrypoint": f"python -c '{import_in_task_script}'",
+                # TODO(architkulkarni): Uncomment after #22968 is fixed.
+                # "entrypoint": "python -c 'import pip_install_test'",
+                "expected_logs": "Good job!  You installed a pip module.",
+            }
+    else:
+        assert False, f"Unrecognized option: {request.param}."
+
+
+def test_submit_job(job_sdk_client, runtime_env_option, monkeypatch):
+    # This flag allows for local testing of runtime env conda functionality
+    # without needing a built Ray wheel.  Rather than insert the link to the
+    # wheel into the conda spec, it links to the current Python site.
+    monkeypatch.setenv("RAY_RUNTIME_ENV_LOCAL_DEV_MODE", "1")
+
+    client = job_sdk_client
+
+    job_id = client.submit_job(
+        entrypoint=runtime_env_option["entrypoint"],
+        runtime_env=runtime_env_option["runtime_env"],
+    )
+
+    try:
+        job_start_time = time.time()
+        wait_for_condition(
+            _check_job_succeeded, client=client, job_id=job_id, timeout=300
+        )
+        job_duration = time.time() - job_start_time
+        print(f"The job took {job_duration}s to succeed.")
+    except RuntimeError as e:
+        # If the job is still pending, include job logs and info in error.
+        if client.get_job_status(job_id) == JobStatus.PENDING:
+            logs = client.get_job_logs(job_id)
+            info = client.get_job_info(job_id)
+            raise RuntimeError(
+                f"Job was stuck in PENDING.\nLogs: {logs}\nInfo: {info}"
+            ) from e
+
+    logs = client.get_job_logs(job_id)
+    assert runtime_env_option["expected_logs"] in logs
+
+
+def test_timeout(job_sdk_client):
+    client = job_sdk_client
+
+    job_id = client.submit_job(
+        entrypoint="echo hello",
+        # Assume pip packages take > 1s to download, or this test will spuriously fail.
+        runtime_env=RuntimeEnv(
+            pip={
+                "packages": ["tensorflow", "requests", "botocore", "torch"],
+                "pip_check": False,
+                "pip_version": "==23.3.2;python_version=='3.9.16'",
+            },
+            config=RuntimeEnvConfig(setup_timeout_seconds=1),
+        ),
+    )
+
+    wait_for_condition(_check_job_failed, client=client, job_id=job_id, timeout=10)
+    data = client.get_job_info(job_id)
+    assert "Failed to set up runtime environment" in data.message
+    assert "Timeout" in data.message
+    assert "setup_timeout_seconds" in data.message
+
+
+def test_per_task_runtime_env(job_sdk_client: JobSubmissionClient):
+    run_cmd = "python per_task_runtime_env.py"
+    job_id = job_sdk_client.submit_job(
+        entrypoint=run_cmd,
+        runtime_env={"working_dir": DRIVER_SCRIPT_DIR},
+    )
+
+    wait_for_condition(_check_job_succeeded, client=job_sdk_client, job_id=job_id)
+
+
+def test_ray_tune_basic(job_sdk_client: JobSubmissionClient):
+    run_cmd = "python ray_tune_basic.py"
+    job_id = job_sdk_client.submit_job(
+        entrypoint=run_cmd,
+        runtime_env={"working_dir": DRIVER_SCRIPT_DIR},
+    )
+    wait_for_condition(
+        _check_job_succeeded, timeout=30, client=job_sdk_client, job_id=job_id
+    )
+
+
+def test_http_bad_request(job_sdk_client):
+    """
+    Send bad requests to job http server and ensure right return code and
+    error message is returned via http.
+    """
+    client = job_sdk_client
+
+    # 400 - HTTPBadRequest
+    r = client._do_request(
+        "POST",
+        "/api/jobs/",
+        json_data={"key": "baaaad request"},
+    )
+
+    assert r.status_code == 400
+    assert "__init__() got an unexpected keyword argument" in r.text
+
+
+def test_invalid_runtime_env(job_sdk_client):
+    client = job_sdk_client
+    with pytest.raises(ValueError, match="Only .zip, .tar.gz, and .tgz files"):
+        client.submit_job(
+            entrypoint="echo hello", runtime_env={"working_dir": "s3://not_a_zip"}
+        )
+
+
+def test_runtime_env_setup_failure(job_sdk_client):
+    client = job_sdk_client
+    job_id = client.submit_job(
+        entrypoint="echo hello", runtime_env={"working_dir": "s3://does_not_exist.zip"}
+    )
+
+    wait_for_condition(_check_job_failed, client=client, job_id=job_id)
+    data = client.get_job_info(job_id)
+    assert "Failed to set up runtime environment" in data.message
+
+
+def test_submit_job_with_exception_in_driver(job_sdk_client):
+    """
+    Submit a job that's expected to throw exception while executing.
+    """
+    client = job_sdk_client
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+        driver_script = """
+print('Hello !')
+raise RuntimeError('Intentionally failed.')
+        """
+        test_script_file = path / "test_script.py"
+        with open(test_script_file, "w+") as file:
+            file.write(driver_script)
+
+        job_id = client.submit_job(
+            entrypoint="python test_script.py", runtime_env={"working_dir": tmp_dir}
+        )
+
+        wait_for_condition(_check_job_failed, client=client, job_id=job_id)
+        logs = client.get_job_logs(job_id)
+        assert "Hello !" in logs
+        assert "RuntimeError: Intentionally failed." in logs
+
+
+def test_stop_long_running_job(job_sdk_client):
+    """
+    Submit a job that runs for a while and stop it in the middle.
+    """
+    client = job_sdk_client
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+        driver_script = """
+print('Hello !')
+import time
+time.sleep(300) # This should never finish
+raise RuntimeError('Intentionally failed.')
+        """
+        test_script_file = path / "test_script.py"
+        with open(test_script_file, "w+") as file:
+            file.write(driver_script)
+
+        job_id = client.submit_job(
+            entrypoint="python test_script.py", runtime_env={"working_dir": tmp_dir}
+        )
+        assert client.stop_job(job_id) is True
+        wait_for_condition(_check_job_stopped, client=client, job_id=job_id)
+
+
+def test_delete_job(job_sdk_client, capsys):
+    """
+    Submit a job and delete it.
+    """
+    client: JobSubmissionClient = job_sdk_client
+
+    job_id = client.submit_job(entrypoint="sleep 300 && echo hello")
+    with pytest.raises(Exception, match="but it is in a non-terminal state"):
+        # This should fail because the job is not in a terminal state.
+        client.delete_job(job_id)
+
+    # Check that the job appears in list_jobs
+    jobs = client.list_jobs()
+    assert job_id in [job.submission_id for job in jobs]
+
+    finished_job_id = client.submit_job(entrypoint="echo hello")
+    wait_for_condition(_check_job_succeeded, client=client, job_id=finished_job_id)
+    deleted = client.delete_job(finished_job_id)
+    assert deleted is True
+
+    # Check that the job no longer appears in list_jobs
+    jobs = client.list_jobs()
+    assert finished_job_id not in [job.submission_id for job in jobs]
+
+
+def test_job_metadata(job_sdk_client):
+    client = job_sdk_client
+
+    print_metadata_cmd = (
+        'python -c"'
+        "import ray;"
+        "ray.init();"
+        "job_config=ray._private.worker.global_worker.core_worker.get_job_config();"
+        "print(dict(sorted(job_config.metadata.items())))"
+        '"'
+    )
+
+    job_id = client.submit_job(
+        entrypoint=print_metadata_cmd, metadata={"key1": "val1", "key2": "val2"}
+    )
+
+    wait_for_condition(_check_job_succeeded, client=client, job_id=job_id)
+
+    assert str(
+        {
+            "job_name": job_id,
+            "job_submission_id": job_id,
+            "key1": "val1",
+            "key2": "val2",
+        }
+    ) in client.get_job_logs(job_id)
+
+
+def test_pass_job_id(job_sdk_client):
+    client = job_sdk_client
+
+    job_id = "my_custom_id"
+    returned_id = client.submit_job(entrypoint="echo hello", job_id=job_id)
+
+    assert returned_id == job_id
+    wait_for_condition(_check_job_succeeded, client=client, job_id=returned_id)
+
+    # Test that a duplicate job_id is rejected.
+    with pytest.raises(Exception, match=f"{job_id} already exists"):
+        returned_id = client.submit_job(entrypoint="echo hello", job_id=job_id)
+
+
+def test_nonexistent_job(job_sdk_client):
+    client = job_sdk_client
+
+    with pytest.raises(RuntimeError, match="nonexistent_job does not exist"):
+        client.get_job_status("nonexistent_job")
+
+
+def test_submit_optional_args(job_sdk_client):
+    """Check that job_id, runtime_env, and metadata are optional."""
+    client = job_sdk_client
+
+    r = client._do_request(
+        "POST",
+        "/api/jobs/",
+        json_data={"entrypoint": "ls"},
+    )
+
+    wait_for_condition(
+        _check_job_succeeded, client=client, job_id=r.json()["submission_id"]
+    )
+
+
+def test_submit_still_accepts_job_id_or_submission_id(job_sdk_client):
+    """Check that job_id, runtime_env, and metadata are optional."""
+    client = job_sdk_client
+
+    client._do_request(
+        "POST",
+        "/api/jobs/",
+        json_data={"entrypoint": "ls", "job_id": "raysubmit_12345"},
+    )
+
+    wait_for_condition(_check_job_succeeded, client=client, job_id="raysubmit_12345")
+
+    client._do_request(
+        "POST",
+        "/api/jobs/",
+        json_data={"entrypoint": "ls", "submission_id": "raysubmit_23456"},
+    )
+
+    wait_for_condition(_check_job_succeeded, client=client, job_id="raysubmit_23456")
+
+
+def test_missing_resources(job_sdk_client):
+    """Check that 404s are raised for resources that don't exist."""
+    client = job_sdk_client
+
+    conditions = [
+        ("GET", "/api/jobs/fake_job_id"),
+        ("GET", "/api/jobs/fake_job_id/logs"),
+        ("POST", "/api/jobs/fake_job_id/stop"),
+        ("GET", "/api/packages/fake_package_uri"),
+    ]
+
+    for method, route in conditions:
+        assert client._do_request(method, route).status_code == 404
+
+
+def test_version_endpoint(job_sdk_client):
+    client = job_sdk_client
+
+    r = client._do_request("GET", "/api/version")
+    assert r.status_code == 200
+    body = r.json()
+    assert body == {
+        "version": CURRENT_VERSION,
+        "ray_version": ray.__version__,
+        "ray_commit": ray.__commit__,
+        "session_name": body["session_name"],
+    }
+
+
+def test_request_headers(job_sdk_client):
+    client = job_sdk_client
+    with patch("requests.request") as mock_request:
+        _ = client._do_request(
+            "POST",
+            "/api/jobs/",
+            json_data={"entrypoint": "ls"},
+        )
+        mock_request.assert_called_with(
+            "POST",
+            "http://127.0.0.1:8265/api/jobs/",
+            cookies=None,
+            data=None,
+            json={"entrypoint": "ls"},
+            headers={"Connection": "keep-alive", "Authorization": "TOK:<MY_TOKEN>"},
+            verify=True,
+        )
+
+
+@pytest.mark.parametrize("scheme", ["http", "https", "fake_module"])
+@pytest.mark.parametrize("host", ["127.0.0.1", "localhost", "fake.dns.name"])
+@pytest.mark.parametrize("port", [None, 8265, 10000])
+def test_parse_cluster_info(scheme: str, host: str, port: Optional[int]):
+    address = f"{scheme}://{host}"
+    if port is not None:
+        address += f":{port}"
+
+    if scheme in {"http", "https"}:
+        assert parse_cluster_info(address, False) == ClusterInfo(
+            address=address,
+            cookies=None,
+            metadata=None,
+            headers=None,
+        )
+    else:
+        with pytest.raises(RuntimeError):
+            parse_cluster_info(address, False)
+
+
+@pytest.mark.asyncio
+async def test_tail_job_logs(job_sdk_client):
+    client = job_sdk_client
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+        driver_script = """
+import time
+for i in range(100):
+    print("Hello", i)
+    time.sleep(0.1)
+"""
+        test_script_file = path / "test_script.py"
+        with open(test_script_file, "w+") as f:
+            f.write(driver_script)
+
+        job_id = client.submit_job(
+            entrypoint="python test_script.py", runtime_env={"working_dir": tmp_dir}
+        )
+
+        st = time.time()
+        while time.time() - st <= 10:
+            try:
+                i = 0
+                async for lines in client.tail_job_logs(job_id):
+                    print(lines, end="")
+                    for line in lines.strip().split("\n"):
+                        assert line.split(" ") == ["Hello", str(i)]
+                        i += 1
+            except Exception as ex:
+                print("Exception:", ex)
+
+        wait_for_condition(_check_job_succeeded, client=client, job_id=job_id)
+
+
+def _hook(env):
+    with open(env["env_vars"]["TEMPPATH"], "w+") as f:
+        f.write(env["env_vars"]["TOKEN"])
+    return env
+
+
+def test_jobs_env_hook(job_sdk_client: JobSubmissionClient):
+    client = job_sdk_client
+
+    _, path = tempfile.mkstemp()
+    runtime_env = {"env_vars": {"TEMPPATH": path, "TOKEN": "Ray rocks!"}}
+    run_job_script = """
+import os
+import ray
+os.environ["RAY_RUNTIME_ENV_HOOK"] =\
+    "ray.dashboard.modules.job.tests.test_http_job_server._hook"
+ray.init(address="auto")
+"""
+    entrypoint = f"python -c '{run_job_script}'"
+    job_id = client.submit_job(entrypoint=entrypoint, runtime_env=runtime_env)
+
+    wait_for_condition(_check_job_succeeded, client=client, job_id=job_id)
+
+    with open(path) as f:
+        assert f.read().strip() == "Ray rocks!"
+
+
+@pytest.mark.asyncio
+async def test_get_upload_package(ray_start_context, tmp_path):
+    assert wait_until_server_available(ray_start_context["webui_url"])
+    webui_url = format_web_url(ray_start_context["webui_url"])
+    gcs_client = ray._private.worker.global_worker.gcs_client
+    url = webui_url + "/api/packages/{protocol}/{package_name}"
+
+    pkg_dir = tmp_path / "pkg"
+    pkg_dir.mkdir()
+    filename = "task.py"
+
+    file_content = b"Hello world"
+    with (pkg_dir / filename).open("wb") as f:
+        f.write(file_content)
+
+    package_uri = get_uri_for_file(str(pkg_dir / filename))
+    protocol, package_name = uri_to_http_components(package_uri)
+    package_file = tmp_path / package_name
+    create_package(str(pkg_dir), package_file, include_gitignore=True)
+
+    resp = requests.get(url.format(protocol=protocol, package_name=package_name))
+    assert resp.status_code == 404
+
+    resp = requests.put(
+        url.format(protocol=protocol, package_name=package_name),
+        data=package_file.read_bytes(),
+    )
+    assert resp.status_code == 200
+
+    resp = requests.get(url.format(protocol=protocol, package_name=package_name))
+    assert resp.status_code == 200
+
+    await download_and_unpack_package(package_uri, str(tmp_path), gcs_client)
+    assert (package_file.with_suffix("") / filename).read_bytes() == file_content
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_https_connection.py

@@ -0,0 +1,47 @@
+import ssl
+import sys
+
+import pytest
+import trustme
+
+import ray
+from ray.job_submission import JobSubmissionClient
+
+
+@pytest.fixture(scope="session")
+def ca():
+    return trustme.CA()
+
+
+@pytest.fixture(scope="session")
+def httpserver_ssl_context(ca):
+    context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+    localhost_cert = ca.issue_cert("localhost")
+    localhost_cert.configure_cert(context)
+    return context
+
+
+@pytest.fixture(scope="session")
+def httpclient_ssl_context(ca):
+    with ca.cert_pem.tempfile() as ca_temp_path:
+        return ssl.create_default_context(cafile=ca_temp_path)
+
+
+def test_mock_https_connection(httpserver, ca):
+    """Test connections to a mock HTTPS job submission server."""
+    httpserver.expect_request("/api/version").respond_with_json(
+        {"ray_version": ray.__version__}
+    )
+    mock_url = httpserver.url_for("/")
+    # Connection without SSL certificate should fail
+    with pytest.raises(ConnectionError):
+        JobSubmissionClient(mock_url)
+    # Connecton with SSL verification skipped should succeed
+    JobSubmissionClient(mock_url, verify=False)
+    # Connection with SSL verification should succeed
+    with ca.cert_pem.tempfile() as ca_temp_path:
+        JobSubmissionClient(mock_url, verify=ca_temp_path)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_job_agent.py

@@ -0,0 +1,692 @@
+import logging
+import os
+import shutil
+import sys
+import tempfile
+import time
+from functools import partial
+from pathlib import Path
+
+import pytest
+import pytest_asyncio
+import requests
+import yaml
+
+import ray
+from ray._common.network_utils import build_address
+from ray._common.test_utils import async_wait_for_condition, wait_for_condition
+from ray._common.utils import get_or_create_event_loop
+from ray._private.ray_constants import DEFAULT_DASHBOARD_AGENT_LISTEN_PORT
+from ray._private.runtime_env.py_modules import upload_py_modules_if_needed
+from ray._private.runtime_env.working_dir import upload_working_dir_if_needed
+from ray._private.test_utils import (
+    chdir,
+    format_web_url,
+    get_current_unused_port,
+    run_string_as_driver_nonblocking,
+    wait_until_server_available,
+)
+from ray.dashboard.modules.job.common import (
+    JOB_ACTOR_NAME_TEMPLATE,
+    SUPERVISOR_ACTOR_RAY_NAMESPACE,
+    JobSubmitRequest,
+    validate_request_type,
+)
+from ray.dashboard.modules.job.job_head import JobAgentSubmissionClient
+from ray.dashboard.tests.conftest import *  # noqa
+from ray.job_submission import JobStatus, JobSubmissionClient
+from ray.runtime_env.runtime_env import RuntimeEnv, RuntimeEnvConfig
+from ray.tests.conftest import _ray_start
+from ray.util.state import get_node, list_actors, list_nodes
+
+# This test requires you have AWS credentials set up (any AWS credentials will
+# do, this test only accesses a public bucket).
+
+logger = logging.getLogger(__name__)
+
+DRIVER_SCRIPT_DIR = os.path.join(os.path.dirname(__file__), "subprocess_driver_scripts")
+EVENT_LOOP = get_or_create_event_loop()
+
+
+def get_node_id_for_supervisor_actor_for_job(
+    address: str, job_submission_id: str
+) -> str:
+    actors = list_actors(
+        address=address,
+        filters=[("ray_namespace", "=", SUPERVISOR_ACTOR_RAY_NAMESPACE)],
+    )
+    for actor in actors:
+        if actor.name == JOB_ACTOR_NAME_TEMPLATE.format(job_id=job_submission_id):
+            return actor.node_id
+    raise ValueError(f"actor not found for job_submission_id {job_submission_id}")
+
+
+def get_node_ip_by_id(node_id: str) -> str:
+    node = get_node(id=node_id)
+    return node.node_ip
+
+
+class JobAgentSubmissionBrowserClient(JobAgentSubmissionClient):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._session.headers[
+            "User-Agent"
+        ] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"  # noqa: E501
+
+
+@pytest_asyncio.fixture
+async def job_sdk_client(make_sure_dashboard_http_port_unused):
+    with _ray_start(include_dashboard=True, num_cpus=1) as ctx:
+        node_ip = ctx.address_info["node_ip_address"]
+        agent_address = build_address(node_ip, DEFAULT_DASHBOARD_AGENT_LISTEN_PORT)
+        assert wait_until_server_available(agent_address)
+        head_address = ctx.address_info["webui_url"]
+        assert wait_until_server_available(head_address)
+        yield (
+            JobAgentSubmissionClient(format_web_url(agent_address)),
+            JobSubmissionClient(format_web_url(head_address)),
+        )
+
+
+def _check_job(
+    client: JobSubmissionClient, job_id: str, status: JobStatus, timeout: int = 10
+) -> bool:
+    res_status = client.get_job_status(job_id)
+    assert res_status == status
+    return True
+
+
+@pytest.fixture(
+    scope="module",
+    params=[
+        "no_working_dir",
+        "local_working_dir",
+        "s3_working_dir",
+        "local_py_modules",
+        "working_dir_and_local_py_modules_whl",
+        "local_working_dir_zip",
+        "pip_txt",
+        "conda_yaml",
+        "local_py_modules",
+    ],
+)
+def runtime_env_option(request):
+    import_in_task_script = """
+import ray
+ray.init(address="auto")
+
+@ray.remote
+def f():
+    import pip_install_test
+
+ray.get(f.remote())
+"""
+    if request.param == "no_working_dir":
+        yield {
+            "runtime_env": {},
+            "entrypoint": "echo hello",
+            "expected_logs": "hello\n",
+        }
+    elif request.param in {
+        "local_working_dir",
+        "local_working_dir_zip",
+        "local_py_modules",
+        "working_dir_and_local_py_modules_whl",
+    }:
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            path = Path(tmp_dir)
+
+            hello_file = path / "test.py"
+            with hello_file.open(mode="w") as f:
+                f.write("from test_module import run_test\n")
+                f.write("print(run_test())")
+
+            module_path = path / "test_module"
+            module_path.mkdir(parents=True)
+
+            test_file = module_path / "test.py"
+            with test_file.open(mode="w") as f:
+                f.write("def run_test():\n")
+                f.write("    return 'Hello from test_module!'\n")  # noqa: Q000
+
+            init_file = module_path / "__init__.py"
+            with init_file.open(mode="w") as f:
+                f.write("from test_module.test import run_test\n")
+
+            if request.param == "local_working_dir":
+                yield {
+                    "runtime_env": {"working_dir": tmp_dir},
+                    "entrypoint": "python test.py",
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "local_working_dir_zip":
+                local_zipped_dir = shutil.make_archive(
+                    os.path.join(tmp_dir, "test"), "zip", tmp_dir
+                )
+                yield {
+                    "runtime_env": {"working_dir": local_zipped_dir},
+                    "entrypoint": "python test.py",
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "local_py_modules":
+                yield {
+                    "runtime_env": {"py_modules": [str(Path(tmp_dir) / "test_module")]},
+                    "entrypoint": (
+                        "python -c 'import test_module;print(test_module.run_test())'"
+                    ),
+                    "expected_logs": "Hello from test_module!\n",
+                }
+            elif request.param == "working_dir_and_local_py_modules_whl":
+                yield {
+                    "runtime_env": {
+                        "working_dir": "s3://runtime-env-test/script_runtime_env.zip",
+                        "py_modules": [
+                            Path(os.path.dirname(__file__))
+                            / "pip_install_test-0.5-py3-none-any.whl"
+                        ],
+                    },
+                    "entrypoint": (
+                        "python script.py && python -c 'import pip_install_test'"
+                    ),
+                    "expected_logs": (
+                        "Executing main() from script.py !!\n"
+                        "Good job!  You installed a pip module."
+                    ),
+                }
+            else:
+                raise ValueError(f"Unexpected pytest fixture option {request.param}")
+    elif request.param == "s3_working_dir":
+        yield {
+            "runtime_env": {
+                "working_dir": "s3://runtime-env-test/script_runtime_env.zip",
+            },
+            "entrypoint": "python script.py",
+            "expected_logs": "Executing main() from script.py !!\n",
+        }
+    elif request.param == "pip_txt":
+        with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
+            pip_list = ["pip-install-test==0.5"]
+            relative_filepath = "requirements.txt"
+            pip_file = Path(relative_filepath)
+            pip_file.write_text("\n".join(pip_list))
+            runtime_env = {"pip": {"packages": relative_filepath, "pip_check": False}}
+            yield {
+                "runtime_env": runtime_env,
+                "entrypoint": (
+                    f"python -c 'import pip_install_test' && "
+                    f"python -c '{import_in_task_script}'"
+                ),
+                "expected_logs": "Good job!  You installed a pip module.",
+            }
+    elif request.param == "conda_yaml":
+        with tempfile.TemporaryDirectory() as tmpdir, chdir(tmpdir):
+            conda_dict = {"dependencies": ["pip", {"pip": ["pip-install-test==0.5"]}]}
+            relative_filepath = "environment.yml"
+            conda_file = Path(relative_filepath)
+            conda_file.write_text(yaml.dump(conda_dict))
+            runtime_env = {"conda": relative_filepath}
+
+            yield {
+                "runtime_env": runtime_env,
+                "entrypoint": f"python -c '{import_in_task_script}'",
+                # TODO(architkulkarni): Uncomment after #22968 is fixed.
+                # "entrypoint": "python -c 'import pip_install_test'",
+                "expected_logs": "Good job!  You installed a pip module.",
+            }
+    else:
+        assert False, f"Unrecognized option: {request.param}."
+
+
+@pytest.mark.asyncio
+async def test_submit_job(job_sdk_client, runtime_env_option, monkeypatch):
+    # This flag allows for local testing of runtime env conda functionality
+    # without needing a built Ray wheel.  Rather than insert the link to the
+    # wheel into the conda spec, it links to the current Python site.
+    monkeypatch.setenv("RAY_RUNTIME_ENV_LOCAL_DEV_MODE", "1")
+
+    agent_client, head_client = job_sdk_client
+
+    runtime_env = runtime_env_option["runtime_env"]
+    runtime_env = upload_working_dir_if_needed(
+        runtime_env, include_gitignore=True, logger=logger
+    )
+    runtime_env = upload_py_modules_if_needed(
+        runtime_env, include_gitignore=True, logger=logger
+    )
+    runtime_env = RuntimeEnv(**runtime_env_option["runtime_env"]).to_dict()
+    request = validate_request_type(
+        {"runtime_env": runtime_env, "entrypoint": runtime_env_option["entrypoint"]},
+        JobSubmitRequest,
+    )
+    submit_result = await agent_client.submit_job_internal(request)
+    job_id = submit_result.submission_id
+
+    try:
+        job_start_time = time.time()
+        wait_for_condition(
+            partial(
+                _check_job,
+                client=head_client,
+                job_id=job_id,
+                status=JobStatus.SUCCEEDED,
+            ),
+            timeout=300,
+        )
+        job_duration = time.time() - job_start_time
+        print(f"The job took {job_duration}s to succeed.")
+    except RuntimeError as e:
+        # If the job is still pending, include job logs and info in error.
+        if head_client.get_job_status(job_id) == JobStatus.PENDING:
+            logs = head_client.get_job_logs(job_id)
+            info = head_client.get_job_info(job_id)
+            raise RuntimeError(
+                f"Job was stuck in PENDING.\nLogs: {logs}\nInfo: {info}"
+            ) from e
+
+    # There is only one node, so there is no need to replace the client of the JobAgent
+    resp = await agent_client.get_job_logs_internal(job_id)
+    assert runtime_env_option["expected_logs"] in resp.logs
+
+
+@pytest.mark.asyncio
+async def test_submit_job_rejects_browsers(
+    job_sdk_client, runtime_env_option, monkeypatch
+):
+    # This flag allows for local testing of runtime env conda functionality
+    # without needing a built Ray wheel.  Rather than insert the link to the
+    # wheel into the conda spec, it links to the current Python site.
+    monkeypatch.setenv("RAY_RUNTIME_ENV_LOCAL_DEV_MODE", "1")
+
+    agent_client, head_client = job_sdk_client
+    agent_address = agent_client._agent_address
+    agent_client = JobAgentSubmissionBrowserClient(agent_address)
+
+    runtime_env = runtime_env_option["runtime_env"]
+    runtime_env = upload_working_dir_if_needed(
+        runtime_env, include_gitignore=True, logger=logger
+    )
+    runtime_env = upload_py_modules_if_needed(
+        runtime_env, include_gitignore=True, logger=logger
+    )
+    runtime_env = RuntimeEnv(**runtime_env_option["runtime_env"]).to_dict()
+    request = validate_request_type(
+        {"runtime_env": runtime_env, "entrypoint": runtime_env_option["entrypoint"]},
+        JobSubmitRequest,
+    )
+
+    with pytest.raises(RuntimeError) as exc:
+        _ = await agent_client.submit_job_internal(request)
+
+    assert "status code 403" in str(exc.value)
+
+
+@pytest.mark.asyncio
+async def test_delete_job_rejects_browsers(job_sdk_client, monkeypatch):
+    """Test that DELETE job requests from browsers are rejected."""
+    monkeypatch.setenv("RAY_RUNTIME_ENV_LOCAL_DEV_MODE", "1")
+
+    agent_client, head_client = job_sdk_client
+
+    # First, submit a job using the normal client
+    runtime_env = RuntimeEnv().to_dict()
+    request = validate_request_type(
+        {"runtime_env": runtime_env, "entrypoint": "echo hello"},
+        JobSubmitRequest,
+    )
+    submit_result = await agent_client.submit_job_internal(request)
+    job_id = submit_result.submission_id
+
+    # Now try to delete the job using browser-like headers
+    agent_address = agent_client._agent_address
+    browser_client = JobAgentSubmissionBrowserClient(agent_address)
+
+    with pytest.raises(RuntimeError) as exc:
+        _ = await browser_client.delete_job_internal(job_id)
+
+    assert "status code 403" in str(exc.value)
+
+    await browser_client.close()
+
+
+@pytest.mark.asyncio
+async def test_timeout(job_sdk_client):
+    agent_client, head_client = job_sdk_client
+
+    runtime_env = RuntimeEnv(
+        pip={
+            "packages": ["tensorflow", "requests", "botocore", "torch"],
+            "pip_check": False,
+            "pip_version": "==23.3.2;python_version=='3.9.16'",
+        },
+        config=RuntimeEnvConfig(setup_timeout_seconds=1),
+    ).to_dict()
+    request = validate_request_type(
+        {"runtime_env": runtime_env, "entrypoint": "echo hello"},
+        JobSubmitRequest,
+    )
+
+    submit_result = await agent_client.submit_job_internal(request)
+    job_id = submit_result.submission_id
+
+    wait_for_condition(
+        partial(_check_job, client=head_client, job_id=job_id, status=JobStatus.FAILED),
+        timeout=10,
+    )
+
+    data = head_client.get_job_info(job_id)
+    assert "Failed to set up runtime environment" in data.message
+    assert "Timeout" in data.message
+    assert "setup_timeout_seconds" in data.message
+
+
+@pytest.mark.asyncio
+async def test_runtime_env_setup_failure(job_sdk_client):
+    agent_client, head_client = job_sdk_client
+
+    runtime_env = RuntimeEnv(working_dir="s3://does_not_exist.zip").to_dict()
+    request = validate_request_type(
+        {"runtime_env": runtime_env, "entrypoint": "echo hello"},
+        JobSubmitRequest,
+    )
+
+    submit_result = await agent_client.submit_job_internal(request)
+    job_id = submit_result.submission_id
+
+    wait_for_condition(
+        partial(_check_job, client=head_client, job_id=job_id, status=JobStatus.FAILED),
+        timeout=10,
+    )
+
+    data = head_client.get_job_info(job_id)
+    assert "Failed to set up runtime environment" in data.message
+
+
+@pytest.mark.asyncio
+async def test_stop_long_running_job(job_sdk_client):
+    """
+    Submit a job that runs for a while and stop it in the middle.
+    """
+    agent_client, head_client = job_sdk_client
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+        driver_script = """
+print('Hello !')
+import time
+time.sleep(300) # This should never finish
+raise RuntimeError('Intentionally failed.')
+        """
+        test_script_file = path / "test_script.py"
+        with open(test_script_file, "w+") as file:
+            file.write(driver_script)
+
+        runtime_env = {"working_dir": tmp_dir}
+        runtime_env = upload_working_dir_if_needed(
+            runtime_env, include_gitignore=True, scratch_dir=tmp_dir, logger=logger
+        )
+        runtime_env = RuntimeEnv(**runtime_env).to_dict()
+
+        request = validate_request_type(
+            {"runtime_env": runtime_env, "entrypoint": "python test_script.py"},
+            JobSubmitRequest,
+        )
+        submit_result = await agent_client.submit_job_internal(request)
+        job_id = submit_result.submission_id
+
+        resp = await agent_client.stop_job_internal(job_id)
+        assert resp.stopped is True
+
+        wait_for_condition(
+            partial(
+                _check_job, client=head_client, job_id=job_id, status=JobStatus.STOPPED
+            ),
+            timeout=10,
+        )
+
+
+@pytest.mark.asyncio
+async def test_tail_job_logs_with_echo(job_sdk_client):
+    agent_client, head_client = job_sdk_client
+
+    runtime_env = RuntimeEnv().to_dict()
+    entrypoint = "python -c \"import time; [(print('Hello', i), time.sleep(0.1)) for i in range(100)]\""  # noqa: E501
+    request = validate_request_type(
+        {
+            "runtime_env": runtime_env,
+            "entrypoint": entrypoint,
+        },
+        JobSubmitRequest,
+    )
+
+    submit_result = await agent_client.submit_job_internal(request)
+    job_id = submit_result.submission_id
+
+    i = 0
+    async for lines in agent_client.tail_job_logs(job_id):
+        print(lines, end="")
+        for line in lines.strip().split("\n"):
+            if (
+                "Runtime env is setting up." in line
+                or "Running entrypoint for job" in line
+            ):
+                continue
+            assert line.split(" ") == ["Hello", str(i)]
+            i += 1
+
+    wait_for_condition(
+        partial(
+            _check_job, client=head_client, job_id=job_id, status=JobStatus.SUCCEEDED
+        ),
+        timeout=10,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "ray_start_cluster_head",
+    [
+        {
+            "include_dashboard": True,
+            "dashboard_agent_listen_port": DEFAULT_DASHBOARD_AGENT_LISTEN_PORT,
+        }
+    ],
+    indirect=True,
+)
+async def test_job_log_in_multiple_node(
+    make_sure_dashboard_http_port_unused,
+    enable_test_module,
+    disable_aiohttp_cache,
+    ray_start_cluster_head,
+):
+    cluster = ray_start_cluster_head
+    assert wait_until_server_available(cluster.webui_url) is True
+    webui_url = cluster.webui_url
+    webui_url = format_web_url(webui_url)
+    cluster.add_node(
+        dashboard_agent_listen_port=DEFAULT_DASHBOARD_AGENT_LISTEN_PORT + 1
+    )
+    cluster.add_node(
+        dashboard_agent_listen_port=DEFAULT_DASHBOARD_AGENT_LISTEN_PORT + 2
+    )
+
+    node_ip = cluster.head_node.node_ip_address
+    agent_address = build_address(node_ip, DEFAULT_DASHBOARD_AGENT_LISTEN_PORT)
+    assert wait_until_server_available(agent_address)
+    client = JobAgentSubmissionClient(format_web_url(agent_address))
+
+    def _check_nodes():
+        try:
+            assert len(list_nodes()) == 3
+            return True
+        except Exception as ex:
+            logger.info(ex)
+            return False
+
+    wait_for_condition(_check_nodes, timeout=15)
+
+    job_ids = []
+    job_check_status = []
+    JOB_NUM = 10
+    job_agent_ports = [
+        DEFAULT_DASHBOARD_AGENT_LISTEN_PORT,
+        DEFAULT_DASHBOARD_AGENT_LISTEN_PORT + 1,
+        DEFAULT_DASHBOARD_AGENT_LISTEN_PORT + 2,
+    ]
+    for index in range(JOB_NUM):
+        runtime_env = RuntimeEnv().to_dict()
+        request = validate_request_type(
+            {
+                "runtime_env": runtime_env,
+                "entrypoint": f"while true; do echo hello index-{index}"
+                " && sleep 3600; done",
+            },
+            JobSubmitRequest,
+        )
+
+        submit_result = await client.submit_job_internal(request)
+        job_ids.append(submit_result.submission_id)
+        job_check_status.append(False)
+
+    async def _check_all_jobs_log():
+        response = requests.get(webui_url + "/nodes?view=summary")
+        response.raise_for_status()
+        summary = response.json()
+        assert summary["result"] is True, summary["msg"]
+        summary = summary["data"]["summary"]
+
+        for index, job_id in enumerate(job_ids):
+            if job_check_status[index]:
+                continue
+            result_log = f"hello index-{index}"
+            # Try to get the node id which supervisor actor running in.
+            node_id = get_node_id_for_supervisor_actor_for_job(cluster.address, job_id)
+            for node_info in summary:
+                if node_info["raylet"]["nodeId"] == node_id:
+                    break
+            assert node_info["raylet"]["nodeId"] == node_id, f"node id: {node_id}"
+
+            # Try to get the agent HTTP port by node id.
+            for agent_port in job_agent_ports:
+                if f"--listen-port={agent_port}" in " ".join(node_info["cmdline"]):
+                    break
+            assert f"--listen-port={agent_port}" in " ".join(
+                node_info["cmdline"]
+            ), f"port: {agent_port}"
+
+            # Finally, we got the whole agent address, and try to get the job log.
+            ip = get_node_ip_by_id(node_id)
+            agent_address = f"{ip}:{agent_port}"
+            assert wait_until_server_available(agent_address)
+            client = JobAgentSubmissionClient(format_web_url(agent_address))
+            resp = await client.get_job_logs_internal(job_id)
+            assert result_log in resp.logs, f"logs: {resp.logs}"
+
+            job_check_status[index] = True
+        return True
+
+    st = time.time()
+    while time.time() - st <= 30:
+        try:
+            await _check_all_jobs_log()
+            break
+        except Exception as ex:
+            print("error:", ex)
+            time.sleep(1)
+    assert all(job_check_status), job_check_status
+
+
+def test_agent_logs_not_streamed_to_drivers():
+    """Ensure when the job submission is used,
+    (ray.init is called from an agent), the agent logs are
+    not streamed to drivers.
+
+    Related: https://github.com/ray-project/ray/issues/29944
+    """
+    script = """
+import ray
+from ray.job_submission import JobSubmissionClient, JobStatus
+from ray._private.test_utils import format_web_url
+from ray._common.test_utils import wait_for_condition
+
+ray.init()
+address = ray._private.worker._global_node.webui_url
+address = format_web_url(address)
+client = JobSubmissionClient(address)
+submission_id = client.submit_job(entrypoint="ls")
+wait_for_condition(
+    lambda: client.get_job_status(submission_id) == JobStatus.SUCCEEDED
+)
+    """
+
+    proc = run_string_as_driver_nonblocking(script)
+    out_str = proc.stdout.read().decode("ascii")
+    err_str = proc.stderr.read().decode("ascii")
+
+    print(out_str, err_str)
+
+    assert "(raylet)" not in out_str
+    assert "(raylet)" not in err_str
+
+
+@pytest.mark.asyncio
+async def test_non_default_dashboard_agent_http_port(tmp_path):
+    """Test that we can connect to the dashboard agent with a non-default
+    http port.
+    """
+    import subprocess
+
+    dashboard_agent_port = get_current_unused_port()
+    cmd = f"ray start --head --dashboard-agent-listen-port {dashboard_agent_port}"
+    subprocess.check_output(cmd, shell=True)
+
+    try:
+        # We will need to wait for the ray to be started in the subprocess.
+        address_info = ray.init("auto", ignore_reinit_error=True).address_info
+
+        node_ip = address_info["node_ip_address"]
+
+        dashboard_agent_listen_port = address_info["dashboard_agent_listen_port"]
+        agent_address = build_address(node_ip, dashboard_agent_listen_port)
+        print("agent address = ", agent_address)
+
+        agent_client = JobAgentSubmissionClient(format_web_url(agent_address))
+        head_client = JobSubmissionClient(format_web_url(address_info["webui_url"]))
+
+        assert wait_until_server_available(agent_address)
+
+        # Submit a job through the agent.
+        runtime_env = RuntimeEnv().to_dict()
+        request = validate_request_type(
+            {
+                "runtime_env": runtime_env,
+                "entrypoint": "echo hello",
+            },
+            JobSubmitRequest,
+        )
+        submit_result = await agent_client.submit_job_internal(request)
+        job_id = submit_result.submission_id
+
+        async def verify():
+            # Wait for job finished.
+            wait_for_condition(
+                partial(
+                    _check_job,
+                    client=head_client,
+                    job_id=job_id,
+                    status=JobStatus.SUCCEEDED,
+                ),
+                timeout=10,
+            )
+
+            resp = await agent_client.get_job_logs_internal(job_id)
+            assert "hello" in resp.logs, resp.logs
+
+            return True
+
+        await async_wait_for_condition(verify, retry_interval_ms=2000)
+    finally:
+        subprocess.check_output("ray stop --force", shell=True)
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_job_inheritance.py

@@ -0,0 +1,182 @@
+import json
+import sys
+import time
+
+import pytest
+
+from ray.cluster_utils import Cluster
+from ray.dashboard.consts import RAY_STREAM_RUNTIME_ENV_LOG_TO_JOB_DRIVER_LOG_ENV_VAR
+from ray.dashboard.modules.job.tests.conftest import _driver_script_path
+from ray.dashboard.modules.job.tests.subprocess_driver_scripts.driver_runtime_env_inheritance import (  # noqa: E501
+    RUNTIME_ENV_LOG_LINE_PREFIX,
+)
+from ray.job_submission import JobStatus, JobSubmissionClient
+
+
+def wait_until_status(client, job_id, status_to_wait_for, timeout_seconds=20):
+    start = time.time()
+    while time.time() - start <= timeout_seconds:
+        status = client.get_job_status(job_id)
+        print(f"status: {status}")
+        if status in status_to_wait_for:
+            return
+        time.sleep(1)
+    raise Exception
+
+
+def wait(client, job_id):
+    wait_until_status(
+        client,
+        job_id,
+        {JobStatus.SUCCEEDED, JobStatus.STOPPED, JobStatus.FAILED},
+        timeout_seconds=60,
+    )
+
+
+def get_runtime_env_from_logs(client, job_id):
+    wait(client, job_id)
+    logs = client.get_job_logs(job_id)
+    print(logs)
+    assert client.get_job_status(job_id) == JobStatus.SUCCEEDED
+    # Split logs by line, find the unique line that starts with
+    # RUNTIME_ENV_LOG_LINE_PREFIX, strip it and parse it as JSON.
+    lines = logs.strip().split("\n")
+    assert len(lines) > 0
+    for line in lines:
+        if line.startswith(RUNTIME_ENV_LOG_LINE_PREFIX):
+            return json.loads(line[len(RUNTIME_ENV_LOG_LINE_PREFIX) :])
+
+
+def test_job_driver_inheritance():
+    try:
+        c = Cluster()
+        c.add_node(num_cpus=1)
+        # If using a remote cluster, replace 127.0.0.1 with the head node's IP address.
+        client = JobSubmissionClient("http://127.0.0.1:8265")
+        driver_script_path = _driver_script_path("driver_runtime_env_inheritance.py")
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path}",
+            runtime_env={
+                "env_vars": {"A": "1", "B": "2"},
+                "pip": ["requests"],
+            },
+        )
+
+        # Test key is merged
+        print("Test key merged")
+        runtime_env = get_runtime_env_from_logs(client, job_id)
+        assert runtime_env["env_vars"] == {"A": "1", "B": "2", "C": "1"}
+        assert runtime_env["pip"] == {"packages": ["requests"], "pip_check": False}
+
+        # Test worker process setuphook works.
+        print("Test key setup hook")
+        expected_str = "HELLOWORLD"
+        job_id = client.submit_job(
+            entrypoint=(
+                f"python {driver_script_path} "
+                f"--worker-process-setup-hook {expected_str}"
+            ),
+            runtime_env={
+                "env_vars": {"A": "1", "B": "2"},
+            },
+        )
+        wait(client, job_id)
+        logs = client.get_job_logs(job_id)
+        assert expected_str in logs
+
+        # Test raise an exception upon key conflict
+        print("Test conflicting pip")
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path} --conflict=pip",
+            runtime_env={"pip": ["numpy"]},
+        )
+        wait(client, job_id)
+        status = client.get_job_status(job_id)
+        logs = client.get_job_logs(job_id)
+        assert status == JobStatus.FAILED
+        assert "Failed to merge the Job's runtime env" in logs
+
+        # Test raise an exception upon env var conflict
+        print("Test conflicting env vars")
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path} --conflict=env_vars",
+            runtime_env={
+                "env_vars": {"A": "1"},
+            },
+        )
+        wait(client, job_id)
+        status = client.get_job_status(job_id)
+        logs = client.get_job_logs(job_id)
+        assert status == JobStatus.FAILED
+        assert "Failed to merge the Job's runtime env" in logs
+    finally:
+        c.shutdown()
+
+
+@pytest.mark.parametrize("stream_runtime_env_log", ["1", "0"])
+def test_runtime_env_logs_streamed_to_job_driver_log(
+    monkeypatch, stream_runtime_env_log
+):
+    monkeypatch.setenv(
+        RAY_STREAM_RUNTIME_ENV_LOG_TO_JOB_DRIVER_LOG_ENV_VAR, stream_runtime_env_log
+    )
+    try:
+        c = Cluster()
+        c.add_node(num_cpus=1)
+        client = JobSubmissionClient("http://127.0.0.1:8265")
+        job_id = client.submit_job(
+            entrypoint="echo hello world",
+            runtime_env={"pip": ["requests==2.25.1"]},
+        )
+        wait(client, job_id)
+        logs = client.get_job_logs(job_id)
+        if stream_runtime_env_log == "0":
+            assert "Creating virtualenv at" not in logs
+        else:
+            assert "Creating virtualenv at" in logs
+    finally:
+        c.shutdown()
+
+
+def test_job_driver_inheritance_override(monkeypatch):
+    monkeypatch.setenv("RAY_OVERRIDE_JOB_RUNTIME_ENV", "1")
+
+    try:
+        c = Cluster()
+        c.add_node(num_cpus=1)
+        # If using a remote cluster, replace 127.0.0.1 with the head node's IP address.
+        client = JobSubmissionClient("http://127.0.0.1:8265")
+        driver_script_path = _driver_script_path("driver_runtime_env_inheritance.py")
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path}",
+            runtime_env={
+                "env_vars": {"A": "1", "B": "2"},
+                "pip": ["requests"],
+            },
+        )
+
+        # Test conflict resolution regular field
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path} --conflict=pip",
+            runtime_env={"pip": ["pip-install-test==0.5"]},
+        )
+        runtime_env = get_runtime_env_from_logs(client, job_id)
+        print(runtime_env)
+        assert runtime_env["pip"] == {"packages": ["numpy"], "pip_check": False}
+
+        # Test raise an exception upon env var conflict
+        job_id = client.submit_job(
+            entrypoint=f"python {driver_script_path} --conflict=env_vars",
+            runtime_env={
+                "env_vars": {"A": "2"},
+            },
+        )
+        runtime_env = get_runtime_env_from_logs(client, job_id)
+        print(runtime_env)
+        assert runtime_env["env_vars"]["A"] == "1"
+    finally:
+        c.shutdown()
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_job_manager_standalone.py

@@ -0,0 +1,73 @@
+import sys
+
+import pytest
+
+from ray._common.test_utils import async_wait_for_condition
+from ray.dashboard.modules.job.tests.conftest import (
+    _driver_script_path,
+    create_job_manager,
+    create_ray_cluster,
+)
+from ray.dashboard.modules.job.tests.test_job_manager import check_job_succeeded
+
+
+@pytest.mark.asyncio
+class TestRuntimeEnvStandalone:
+    """NOTE: PLEASE READ CAREFULLY BEFORE MODIFYING
+    This test is extracted into a standalone module such that it can bootstrap its own
+    (standalone) Ray cluster while avoiding affecting the shared one used by other
+    JobManager tests
+    """
+
+    @pytest.mark.parametrize(
+        "tracing_enabled",
+        [
+            False,
+            # TODO(issues/38633): local code loading is broken when tracing is enabled
+            # True,
+        ],
+    )
+    async def test_user_provided_job_config_honored_by_worker(
+        self, tracing_enabled, tmp_path
+    ):
+        """Ensures that the JobConfig instance injected into ray.init in the driver
+        script is honored even in case when job is submitted via JobManager.submit_job
+        API (involving RAY_JOB_CONFIG_JSON_ENV_VAR being set in child process env)
+
+        """
+
+        if tracing_enabled:
+            tracing_startup_hook = (
+                "ray.util.tracing.setup_local_tmp_tracing:setup_tracing"
+            )
+        else:
+            tracing_startup_hook = None
+
+        with create_ray_cluster(_tracing_startup_hook=tracing_startup_hook) as cluster:
+            job_manager = create_job_manager(cluster, tmp_path)
+
+            driver_script_path = _driver_script_path(
+                "check_code_search_path_is_propagated.py"
+            )
+
+            job_id = await job_manager.submit_job(
+                entrypoint=f"python {driver_script_path}",
+                # NOTE: We inject runtime_env in here, but also specify the JobConfig in
+                #       the driver script: settings to JobConfig (other than the
+                #       runtime_env) passed in via ray.init(...) have to be respected
+                #       along with the runtime_env passed from submit_job API
+                runtime_env={"env_vars": {"TEST_SUBPROCESS_RANDOM_VAR": "0xDEEDDEED"}},
+            )
+
+            await async_wait_for_condition(
+                check_job_succeeded, job_manager=job_manager, job_id=job_id
+            )
+
+            logs = job_manager.get_job_logs(job_id)
+
+            assert "Code search path is propagated" in logs, logs
+            assert "0xDEEDDEED" in logs, logs
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_sdk.py

@@ -0,0 +1,435 @@
+import os
+import sys
+import tempfile
+import time
+from pathlib import Path
+from typing import Dict, Optional, Tuple
+from unittest.mock import AsyncMock, MagicMock, Mock, patch
+
+import aiohttp
+import pytest
+
+import ray.experimental.internal_kv as kv
+from ray._common.test_utils import wait_for_condition
+from ray._private import worker
+from ray._private.ray_constants import (
+    KV_NAMESPACE_DASHBOARD,
+    PROCESS_TYPE_DASHBOARD,
+)
+from ray._private.test_utils import (
+    format_web_url,
+    wait_until_server_available,
+)
+from ray._raylet import GcsClient
+from ray.dashboard.consts import (
+    DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX,
+    GCS_RPC_TIMEOUT_SECONDS,
+    RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR,
+)
+from ray.dashboard.modules.dashboard_sdk import (
+    DEFAULT_DASHBOARD_ADDRESS,
+    ClusterInfo,
+    parse_cluster_info,
+)
+from ray.dashboard.modules.job.pydantic_models import JobType
+from ray.dashboard.modules.job.sdk import JobStatus, JobSubmissionClient
+from ray.dashboard.tests.conftest import *  # noqa
+from ray.runtime_env.runtime_env import RuntimeEnv
+from ray.tests.conftest import _ray_start
+from ray.util.state import list_nodes
+
+import psutil
+
+
+def _check_job_succeeded(client: JobSubmissionClient, job_id: str) -> bool:
+    status = client.get_job_status(job_id)
+    if status == JobStatus.FAILED:
+        logs = client.get_job_logs(job_id)
+        raise RuntimeError(f"Job failed\nlogs:\n{logs}")
+    return status == JobStatus.SUCCEEDED
+
+
+def check_internal_kv_gced():
+    return len(kv._internal_kv_list("gcs://")) == 0
+
+
+@pytest.mark.parametrize(
+    "address_param",
+    [
+        ("ray://1.2.3.4:10001", "ray", "1.2.3.4:10001"),
+        ("other_module://", "other_module", ""),
+        ("other_module://address", "other_module", "address"),
+    ],
+)
+@pytest.mark.parametrize("create_cluster_if_needed", [True, False])
+@pytest.mark.parametrize("cookies", [None, {"test_cookie_key": "test_cookie_val"}])
+@pytest.mark.parametrize("metadata", [None, {"test_metadata_key": "test_metadata_val"}])
+@pytest.mark.parametrize("headers", [None, {"test_headers_key": "test_headers_val"}])
+@pytest.mark.parametrize("extra_kwargs", [{}, {"cloud": "my-cloud"}])
+def test_parse_cluster_info(
+    address_param: Tuple[str, str, str],
+    create_cluster_if_needed: bool,
+    cookies: Optional[Dict[str, str]],
+    metadata: Optional[Dict[str, str]],
+    headers: Optional[Dict[str, str]],
+    extra_kwargs: Dict[str, str],
+):
+    """
+    Test ray.dashboard.modules.dashboard_sdk.parse_cluster_info for different
+    format of addresses.
+    """
+    mock_get_job_submission_client_cluster = Mock(return_value="Ray ClusterInfo")
+    mock_module = Mock()
+    mock_module.get_job_submission_client_cluster_info = Mock(
+        return_value="Other module ClusterInfo"
+    )
+    mock_import_module = Mock(return_value=mock_module)
+
+    address, module_string, inner_address = address_param
+
+    with (
+        patch.multiple(
+            "ray.dashboard.modules.dashboard_sdk",
+            get_job_submission_client_cluster_info=mock_get_job_submission_client_cluster,
+        ),
+        patch.multiple("importlib", import_module=mock_import_module),
+    ):
+        if module_string == "ray":
+            with pytest.raises(ValueError, match="ray://"):
+                parse_cluster_info(
+                    address,
+                    create_cluster_if_needed=create_cluster_if_needed,
+                    cookies=cookies,
+                    metadata=metadata,
+                    headers=headers,
+                    **extra_kwargs,
+                )
+        elif module_string == "other_module":
+            assert (
+                parse_cluster_info(
+                    address,
+                    create_cluster_if_needed=create_cluster_if_needed,
+                    cookies=cookies,
+                    metadata=metadata,
+                    headers=headers,
+                    **extra_kwargs,
+                )
+                == "Other module ClusterInfo"
+            )
+            mock_import_module.assert_called_once_with(module_string)
+            mock_module.get_job_submission_client_cluster_info.assert_called_once_with(
+                inner_address,
+                create_cluster_if_needed=create_cluster_if_needed,
+                cookies=cookies,
+                metadata=metadata,
+                headers=headers,
+                **extra_kwargs,
+            )
+
+
+def test_parse_cluster_info_default_address():
+    assert parse_cluster_info(
+        address=None,
+    ) == ClusterInfo(address=DEFAULT_DASHBOARD_ADDRESS)
+
+
+def test_submit_job_does_not_mutate_runtime_env():
+    class TestClient(JobSubmissionClient):
+        def __init__(self):
+            self._default_metadata = {}
+
+        def _upload_working_dir_if_needed(self, runtime_env):
+            runtime_env["working_dir"] = "gcs://test.zip"
+
+        def _upload_py_modules_if_needed(self, runtime_env):
+            runtime_env["py_modules"] = ["gcs://test_module.zip"]
+
+        def _do_request(self, method, endpoint, **kwargs):
+            return MagicMock(
+                status_code=200,
+                json=lambda: {"job_id": "test_job", "submission_id": "test_job"},
+            )
+
+    runtime_env = {"working_dir": "/tmp/test", "py_modules": ["/tmp/test_module"]}
+    original_runtime_env = {
+        "working_dir": runtime_env["working_dir"],
+        "py_modules": list(runtime_env["py_modules"]),
+    }
+
+    assert (
+        TestClient().submit_job(entrypoint="echo hi", runtime_env=runtime_env)
+        == "test_job"
+    )
+    assert runtime_env == original_runtime_env
+
+
+@pytest.mark.parametrize("expiration_s", [0, 10])
+def test_temporary_uri_reference(monkeypatch, expiration_s):
+    """Test that temporary GCS URI references are deleted after expiration_s."""
+    monkeypatch.setenv(
+        "RAY_RUNTIME_ENV_TEMPORARY_REFERENCE_EXPIRATION_S", str(expiration_s)
+    )
+    # We can't use a fixture with a shared Ray runtime because we need to set the
+    # expiration_s env var before Ray starts.
+    with _ray_start(include_dashboard=True, num_cpus=1) as ctx:
+        headers = {"Connection": "keep-alive", "Authorization": "TOK:<MY_TOKEN>"}
+        address = ctx.address_info["webui_url"]
+        assert wait_until_server_available(address)
+        client = JobSubmissionClient(format_web_url(address), headers=headers)
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            path = Path(tmp_dir)
+
+            hello_file = path / "hi.txt"
+            with hello_file.open(mode="w") as f:
+                f.write("hi\n")
+
+            start = time.time()
+
+            runtime_env = {"working_dir": tmp_dir}
+            job_id = client.submit_job(entrypoint="echo hi", runtime_env=runtime_env)
+            assert runtime_env == {"working_dir": tmp_dir}
+            wait_for_condition(
+                _check_job_succeeded, client=client, job_id=job_id, timeout=30
+            )
+
+            # Give time for deletion to occur if expiration_s is 0.
+            time.sleep(2)
+            # Need to connect to Ray to check internal_kv.
+            # ray.init(address="auto")
+
+            print("Starting Internal KV checks at time ", time.time() - start)
+            if expiration_s > 0:
+                assert not check_internal_kv_gced()
+                wait_for_condition(check_internal_kv_gced, timeout=2 * expiration_s)
+                assert expiration_s < time.time() - start < 2 * expiration_s
+                print("Internal KV was GC'ed at time ", time.time() - start)
+            else:
+                wait_for_condition(check_internal_kv_gced)
+                print("Internal KV was GC'ed at time ", time.time() - start)
+
+                # Regression test for #46625: reusing the same runtime_env after
+                # the package has been GC'ed should re-upload the local working_dir.
+                job_id = client.submit_job(
+                    entrypoint="echo hi", runtime_env=runtime_env
+                )
+                wait_for_condition(
+                    _check_job_succeeded, client=client, job_id=job_id, timeout=30
+                )
+
+
+def get_register_agents_number(gcs_client):
+    keys = gcs_client.internal_kv_keys(
+        prefix=DASHBOARD_AGENT_ADDR_NODE_ID_PREFIX,
+        namespace=KV_NAMESPACE_DASHBOARD,
+        timeout=GCS_RPC_TIMEOUT_SECONDS,
+    )
+    return len(keys)
+
+
+@pytest.mark.parametrize(
+    "ray_start_cluster_head_with_env_vars",
+    [
+        {
+            "include_dashboard": True,
+            "env_vars": {RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR: "1"},
+        },
+        {
+            "include_dashboard": True,
+            "env_vars": {RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR: "0"},
+        },
+    ],
+    indirect=True,
+)
+def test_jobs_run_on_head_by_default_E2E(ray_start_cluster_head_with_env_vars):
+    allow_driver_on_worker_nodes = (
+        os.environ.get(RAY_JOB_ALLOW_DRIVER_ON_WORKER_NODES_ENV_VAR) == "1"
+    )
+    # Cluster setup
+    cluster = ray_start_cluster_head_with_env_vars
+    cluster.add_node(dashboard_agent_listen_port=52366)
+    cluster.add_node(dashboard_agent_listen_port=52367)
+    assert wait_until_server_available(cluster.webui_url) is True
+    webui_url = cluster.webui_url
+    webui_url = format_web_url(webui_url)
+    client = JobSubmissionClient(webui_url)
+    gcs_client = GcsClient(address=cluster.gcs_address)
+
+    def _check_nodes(num_nodes):
+        try:
+            assert len(list_nodes()) == num_nodes
+            return True
+        except Exception as ex:
+            print(ex)
+            return False
+
+    wait_for_condition(lambda: _check_nodes(num_nodes=3), timeout=15)
+    wait_for_condition(lambda: get_register_agents_number(gcs_client) == 3, timeout=20)
+
+    # Submit 20 simple jobs.
+    for i in range(20):
+        client.submit_job(entrypoint="echo hi", submission_id=f"job_{i}")
+    import pprint
+
+    def check_all_jobs_succeeded():
+        submission_jobs = [
+            job for job in client.list_jobs() if job.type == JobType.SUBMISSION
+        ]
+        for job in submission_jobs:
+            pprint.pprint(job)
+            if job.status != JobStatus.SUCCEEDED:
+                return False
+        return True
+
+    # Wait until all jobs have finished.
+    wait_for_condition(check_all_jobs_succeeded, timeout=60, retry_interval_ms=1000)
+
+    # Check driver_node_id of all jobs.
+    submission_jobs = [
+        job for job in client.list_jobs() if job.type == JobType.SUBMISSION
+    ]
+    driver_node_ids = [job.driver_node_id for job in submission_jobs]
+
+    # Spuriously fails with probability (1/3)^20.
+    pprint.pprint(driver_node_ids)
+    num_ids = len(set(driver_node_ids))
+    assert (num_ids > 1) if allow_driver_on_worker_nodes else (num_ids == 1), [
+        id[:5] for id in driver_node_ids
+    ]
+
+
+@pytest.fixture
+def runtime_env_working_dir():
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        path = Path(tmp_dir)
+        working_dir = path / "working_dir"
+        working_dir.mkdir(parents=True)
+        yield working_dir
+
+
+@pytest.fixture
+def py_module_whl():
+    with tempfile.NamedTemporaryFile(suffix=".whl") as tmp_file:
+        yield tmp_file.name
+
+
+def test_job_submission_with_runtime_env_as_dict(
+    runtime_env_working_dir, py_module_whl
+):
+    working_dir_str = str(runtime_env_working_dir)
+    with _ray_start(num_cpus=1):
+        client = JobSubmissionClient()
+        runtime_env = {"working_dir": working_dir_str, "py_modules": [py_module_whl]}
+        job_id = client.submit_job(entrypoint="echo hi", runtime_env=runtime_env)
+        job_details = client.get_job_info(job_id)
+        parsed_runtime_env = job_details.runtime_env
+        assert "gcs://" in parsed_runtime_env["working_dir"]
+        assert len(parsed_runtime_env["py_modules"]) == 1
+        assert "gcs://" in parsed_runtime_env["py_modules"][0]
+
+
+def test_job_submission_with_runtime_env_as_object(
+    runtime_env_working_dir, py_module_whl
+):
+    working_dir_str = str(runtime_env_working_dir)
+    with _ray_start(num_cpus=1):
+        client = JobSubmissionClient()
+        runtime_env = RuntimeEnv(
+            working_dir=working_dir_str, py_modules=[py_module_whl]
+        )
+        job_id = client.submit_job(entrypoint="echo hi", runtime_env=runtime_env)
+        job_details = client.get_job_info(job_id)
+        parsed_runtime_env = job_details.runtime_env
+        assert "gcs://" in parsed_runtime_env["working_dir"]
+        assert len(parsed_runtime_env["py_modules"]) == 1
+        assert "gcs://" in parsed_runtime_env["py_modules"][0]
+
+
+@pytest.mark.asyncio
+async def test_tail_job_logs_passes_headers_to_websocket(ray_start_regular):
+    """
+    Test that authentication headers are passed to WebSocket connections.
+
+    This test verifies that headers provided to JobSubmissionClient are
+    explicitly passed to the ws_connect() method, not just to the ClientSession.
+    This is required because aiohttp's ClientSession does not automatically
+    include session headers in WebSocket upgrade requests.
+    """
+    dashboard_url = ray_start_regular.dashboard_url
+    test_headers = {"Authorization": "Bearer test-token"}
+    client = JobSubmissionClient(format_web_url(dashboard_url), headers=test_headers)
+
+    # Submit a simple job
+    job_id = client.submit_job(entrypoint="echo hello")
+
+    # Mock the aiohttp ClientSession and WebSocket
+    mock_ws = AsyncMock()
+    mock_ws.receive = AsyncMock()
+    mock_ws.receive.side_effect = [
+        # First call returns a text message
+        MagicMock(type=aiohttp.WSMsgType.TEXT, data="test log line\n"),
+        # Second call indicates WebSocket is closed
+        MagicMock(type=aiohttp.WSMsgType.CLOSED),
+    ]
+    mock_ws.close_code = 1000  # Normal closure
+
+    mock_session = AsyncMock()
+    mock_session.ws_connect = AsyncMock(return_value=mock_ws)
+    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
+    mock_session.__aexit__ = AsyncMock(return_value=None)
+
+    # Patch ClientSession to use our mock
+    with patch("aiohttp.ClientSession", return_value=mock_session):
+        # Tail logs
+        log_lines = []
+        async for lines in client.tail_job_logs(job_id):
+            log_lines.append(lines)
+
+        # Verify ws_connect was called with headers
+        mock_session.ws_connect.assert_called_once()
+        call_args = mock_session.ws_connect.call_args
+
+        assert "headers" in call_args.kwargs
+        assert call_args.kwargs["headers"] == test_headers
+
+
+@pytest.mark.asyncio
+async def test_tail_job_logs_websocket_abnormal_closure(ray_start_regular):
+    """
+    Test that ABNORMAL_CLOSURE raises RuntimeError when tailing logs.
+
+    This test uses its own Ray cluster and kills the dashboard while tailing logs
+    to simulate an abnormal WebSocket closure.
+    """
+    dashboard_url = ray_start_regular.dashboard_url
+    client = JobSubmissionClient(format_web_url(dashboard_url))
+
+    # Submit a long-running job
+    driver_script = """
+import time
+for i in range(100):
+    print("Hello", i)
+    time.sleep(0.5)
+"""
+    entrypoint = f"python -c '{driver_script}'"
+    job_id = client.submit_job(entrypoint=entrypoint)
+
+    # Start tailing logs and stop Ray while tailing
+    # Expect RuntimeError when WebSocket closes abnormally
+    with pytest.raises(
+        RuntimeError,
+        match="WebSocket connection closed unexpectedly with close code",
+    ):
+        i = 0
+        async for lines in client.tail_job_logs(job_id):
+            print(lines, end="")
+            i += 1
+
+            # Kill the dashboard after receiving a few log lines
+            if i == 3:
+                print("\nKilling the dashboard to close websocket abnormally...")
+                dash_info = worker._global_node.all_processes[PROCESS_TYPE_DASHBOARD][0]
+                psutil.Process(dash_info.process.pid).kill()
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/dashboard/modules/job/tests/test_utils.py

@@ -0,0 +1,303 @@
+import os
+import sys
+from tempfile import NamedTemporaryFile
+
+import pytest
+
+from ray.dashboard.modules.job.common import JobSubmitRequest
+from ray.dashboard.modules.job.utils import (
+    fast_tail_last_n_lines,
+    file_tail_iterator,
+    parse_and_validate_request,
+    redact_url_password,
+    strip_keys_with_value_none,
+)
+
+
+# Polyfill anext() function for Python 3.9 compatibility
+# May raise StopAsyncIteration.
+async def anext_polyfill(iterator):
+    return await iterator.__anext__()
+
+
+# Use the built-in anext() for Python 3.10+, otherwise use our polyfilled function
+if sys.version_info < (3, 10):
+    anext = anext_polyfill
+
+
+@pytest.fixture
+def tmp():
+    with NamedTemporaryFile() as f:
+        yield f.name
+
+
+def test_strip_keys_with_value_none():
+    d = {"a": 1, "b": None, "c": 3}
+    assert strip_keys_with_value_none(d) == {"a": 1, "c": 3}
+    d = {"a": 1, "b": 2, "c": 3}
+    assert strip_keys_with_value_none(d) == d
+    d = {"a": 1, "b": None, "c": None}
+    assert strip_keys_with_value_none(d) == {"a": 1}
+
+
+def test_redact_url_password():
+    url = "http://user:password@host:port"
+    assert redact_url_password(url) == "http://user:<redacted>@host:port"
+    url = "http://user:password@host:port?query=1"
+    assert redact_url_password(url) == "http://user:<redacted>@host:port?query=1"
+    url = "http://user:password@host:port?query=1&password=2"
+    assert (
+        redact_url_password(url)
+        == "http://user:<redacted>@host:port?query=1&password=2"
+    )
+    url = "https://user:password@127.0.0.1:8080"
+    assert redact_url_password(url) == "https://user:<redacted>@127.0.0.1:8080"
+    url = "https://user:password@host:port?query=1"
+    assert redact_url_password(url) == "https://user:<redacted>@host:port?query=1"
+    url = "https://user:password@host:port?query=1&password=2"
+    assert (
+        redact_url_password(url)
+        == "https://user:<redacted>@host:port?query=1&password=2"
+    )
+
+
+# Mock for aiohttp.web.Request, which should not be constructed directly.
+class MockRequest:
+    def __init__(self, **kwargs):
+        self._json = kwargs
+
+    async def json(self):
+        return self._json
+
+
+@pytest.mark.asyncio
+async def test_mock_request():
+    request = MockRequest(a=1, b=2)
+    assert await request.json() == {"a": 1, "b": 2}
+    request = MockRequest(a=1, b=None)
+    assert await request.json() == {"a": 1, "b": None}
+
+
+# async test
+@pytest.mark.asyncio
+class TestParseAndValidateRequest:
+    async def test_basic(self):
+        request = MockRequest(entrypoint="echo hi")
+        expected = JobSubmitRequest(entrypoint="echo hi")
+        assert await parse_and_validate_request(request, JobSubmitRequest) == expected
+
+    async def test_forward_compatibility(self):
+        request = MockRequest(entrypoint="echo hi", new_client_field=None)
+        expected = JobSubmitRequest(entrypoint="echo hi")
+        assert await parse_and_validate_request(request, JobSubmitRequest) == expected
+
+
+class TestIterLine:
+    @pytest.mark.asyncio
+    async def test_invalid_type(self):
+        with pytest.raises(TypeError, match="path must be a string"):
+            await anext(file_tail_iterator(1))
+
+    @pytest.mark.asyncio
+    async def test_file_not_created(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+        f = open(tmp, "w")
+        f.write("hi\n")
+        f.flush()
+        assert await anext(it) is not None
+
+    @pytest.mark.asyncio
+    async def test_wait_for_newline(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+
+        f = open(tmp, "w")
+        f.write("no_newline_yet")
+        assert await anext(it) is None
+        f.write("\n")
+        f.flush()
+        assert await anext(it) == ["no_newline_yet\n"]
+
+    @pytest.mark.asyncio
+    async def test_multiple_lines(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+
+        f = open(tmp, "w")
+
+        num_lines = 10
+        for i in range(num_lines):
+            s = f"{i}\n"
+            f.write(s)
+            f.flush()
+            assert await anext(it) == [s]
+
+        assert await anext(it) is None
+
+    @pytest.mark.asyncio
+    async def test_batching(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+
+        f = open(tmp, "w")
+
+        # Write lines in batches of 10, check that we get them back in batches.
+        for _ in range(100):
+            num_lines = 10
+            for i in range(num_lines):
+                f.write(f"{i}\n")
+            f.flush()
+
+            assert await anext(it) == [f"{i}\n" for i in range(10)]
+
+        assert await anext(it) is None
+
+    @pytest.mark.asyncio
+    async def test_max_line_batching(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+
+        f = open(tmp, "w")
+
+        # Write lines in batches of 50, check that we get them back in batches of 10.
+        for _ in range(100):
+            num_lines = 50
+            for i in range(num_lines):
+                f.write(f"{i}\n")
+            f.flush()
+
+            assert await anext(it) == [f"{i}\n" for i in range(10)]
+            assert await anext(it) == [f"{i}\n" for i in range(10, 20)]
+            assert await anext(it) == [f"{i}\n" for i in range(20, 30)]
+            assert await anext(it) == [f"{i}\n" for i in range(30, 40)]
+            assert await anext(it) == [f"{i}\n" for i in range(40, 50)]
+
+        assert await anext(it) is None
+
+    @pytest.mark.asyncio
+    async def test_max_char_batching(self, tmp):
+        it = file_tail_iterator(tmp)
+        assert await anext(it) is None
+
+        f = open(tmp, "w")
+
+        # Write a single line that is 60k characters
+        f.write(f"{'1234567890' * 6000}\n")
+        # Write a 4 lines that are 10k characters each
+        for _ in range(4):
+            f.write(f"{'1234567890' * 500}\n")
+        f.flush()
+
+        # First line will come in a batch of its own
+        assert await anext(it) == [f"{'1234567890' * 6000}\n"]
+        # Other 4 lines will be batched together
+        assert (
+            await anext(it)
+            == [
+                f"{'1234567890' * 500}\n",
+            ]
+            * 4
+        )
+        assert await anext(it) is None
+
+    @pytest.mark.asyncio
+    async def test_delete_file(self):
+        with NamedTemporaryFile() as tmp:
+            it = file_tail_iterator(tmp.name)
+            f = open(tmp.name, "w")
+
+            assert await anext(it) is None
+
+            f.write("hi\n")
+            f.flush()
+
+            assert await anext(it) == ["hi\n"]
+
+        # Calls should continue returning None after file deleted.
+        assert await anext(it) is None
+
+
+class TestFastTailLastNLines:
+    def test_nonexistent_path(self, tmp):
+        missing = tmp + ".missing"
+        assert not os.path.exists(missing)
+        with pytest.raises(FileNotFoundError):
+            fast_tail_last_n_lines(missing, num_lines=10, max_chars=1000)
+
+    def test_basic_last_n(self, tmp):
+        # Write 100 lines, check that we get the last 10 lines.
+        with open(tmp, "w") as f:
+            for i in range(100):
+                f.write(f"line-{i}\n")
+        out = fast_tail_last_n_lines(tmp, num_lines=10, max_chars=1000)
+        expected = "".join([f"line-{i}\n" for i in range(90, 100)])
+        assert out == expected
+
+    def test_truncate_max_chars(self, tmp):
+        # Construct a log file with two lines, each over max_chars,
+        # check that we truncate to max_chars.
+        with open(tmp, "w") as f:
+            f.write("x" * 5000 + "\n")
+            f.write("y" * 5000 + "\n")
+        out = fast_tail_last_n_lines(tmp, num_lines=2, max_chars=3000)
+        assert len(out) == 3000
+        # Check that we truncate to max_chars, and include the last line.
+        assert out.endswith("\n")
+
+    def test_partial_last_line(self, tmp):
+        # Write a log file with a partial last line, check that we include it.
+        with open(tmp, "w") as f:
+            f.write("a\n")
+            f.write("b\n")
+            f.write("partial_last_line")  # No newline at end
+        out = fast_tail_last_n_lines(tmp, num_lines=3, max_chars=1000)
+        assert out == "a\nb\npartial_last_line"
+
+    def test_small_block_size(self, tmp):
+        # Write 30 lines, check that we can read a small block size and get the last N lines.
+        with open(tmp, "w") as f:
+            for i in range(30):
+                f.write(f"{i}\n")
+        out = fast_tail_last_n_lines(tmp, num_lines=5, max_chars=1000, block_size=16)
+        expected = "".join([f"{i}\n" for i in range(25, 30)])
+        assert out == expected
+
+    def test_mixed_long_lines(self, tmp):
+        # Write a log file with a mix of short and long lines, check that we get the last N lines.
+        with open(tmp, "w") as f:
+            f.write("short-1\n")
+            f.write("short-2\n")
+            f.write("long-" + ("Z" * 10000) + "\n")
+            f.write("short-3\n")
+            f.write("short-4\n")
+        out = fast_tail_last_n_lines(tmp, num_lines=3, max_chars=20000)
+        # Check that we get the last 3 lines, including the long line.
+        assert out.splitlines()[-1] == "short-4"
+        assert out.splitlines()[-2] == "short-3"
+        assert out.splitlines()[-3].startswith("long-Z")
+
+    def test_sparse_large_file_tail_max_chars(self, tmp):
+        """Simulate ~8 GiB sparse file tail and verify max_chars=20000 truncation."""
+        size_8g = 8 * 1024 * 1024 * 1024
+        # Build tail of two extremely long lines
+        tail = "\n" + ("Q" * 25000 + "\n") + ("R" * 25000 + "\n")
+        tail_bytes = tail.encode("utf-8")
+
+        print("Start writing sparse file tail...")
+        # Create a sparse file: seek to near EOF then write only the tail.
+        with open(tmp, "wb") as f:
+            f.seek(size_8g - len(tail_bytes))
+            f.write(tail_bytes)
+            f.flush()
+
+        print("Finish writing sparse file tail.")
+        out = fast_tail_last_n_lines(tmp, num_lines=2, max_chars=20000)
+        print("Finish reading sparse file tail.")
+        assert len(out) == 20000
+        assert out.endswith("\n")
+        assert "R" * 100 in out  # sampling check for last line content
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))