ray-project--ray/python/ray/tests/test_client_builder.py

import os
import subprocess
import sys
import warnings
from unittest.mock import Mock, patch

import pytest

import ray
import ray.client_builder as client_builder
import ray.util.client.server.server as ray_client_server
from ray._common.test_utils import (
    run_string_as_driver,
    wait_for_condition,
)
from ray._private.test_utils import run_string_as_driver_nonblocking
from ray.util.state import list_workers


@pytest.mark.parametrize(
    "address",
    [
        "localhost:1234",
        "localhost:1234/url?params",
        "1.2.3.4/cluster-1?test_param=param1?",
        "",
    ],
)
def test_split_address(address):
    assert client_builder._split_address(address) == ("ray", address)

    specified_module = f"ray://{address}"
    assert client_builder._split_address(specified_module) == ("ray", address)

    specified_other_module = f"module://{address}"
    assert client_builder._split_address(specified_other_module) == ("module", address)
    non_url_compliant_module = f"module_test://{address}"
    assert client_builder._split_address(non_url_compliant_module) == (
        "module_test",
        address,
    )


@pytest.mark.parametrize(
    "address", ["localhost", "1.2.3.4:1200", "ray://1.2.3.4:1200", "local", None]
)
def test_client(address):
    builder = client_builder.client(address)
    assert isinstance(builder, client_builder.ClientBuilder)
    if address in ("local", None):
        assert isinstance(builder, client_builder._LocalClientBuilder)
    else:
        assert type(builder) is client_builder.ClientBuilder
        assert builder.address == address.replace("ray://", "")


def test_namespace(ray_start_cluster):
    """
    Most of the "checks" in this test case rely on the fact that
    `run_string_as_driver` will throw an exception if the driver string exits
    with a non-zero exit code (e.g. when the driver scripts throws an
    exception). Since all of these drivers start named, detached actors, the
    most likely failure case would be a collision of named actors if they're
    put in the same namespace.

    This test checks that:
    * When two drivers don't specify a namespace, they are placed in different
      anonymous namespaces.
    * When two drivers specify a namespace, they collide.
    * The namespace name (as provided by the runtime context) is correct.
    """
    cluster = ray_start_cluster
    cluster.add_node(num_cpus=4, ray_client_server_port=50055)
    cluster.wait_for_nodes(1)

    template = """
import ray
ray.client("localhost:50055").namespace({namespace}).connect()

@ray.remote
class Foo:
    def ping(self):
        return "pong"

a = Foo.options(lifetime="detached", name="abc").remote()
ray.get(a.ping.remote())
print("Current namespace:", ray.get_runtime_context().namespace)
    """

    anon_driver = template.format(namespace="None")
    run_string_as_driver(anon_driver)
    # This second run will fail if the actors don't run in separate anonymous
    # namespaces.
    run_string_as_driver(anon_driver)

    run_in_namespace = template.format(namespace="'namespace'")
    script_output = run_string_as_driver(run_in_namespace)
    # The second run fails because the actors are run in the same namespace.
    with pytest.raises(subprocess.CalledProcessError):
        run_string_as_driver(run_in_namespace)

    assert "Current namespace: namespace" in script_output
    subprocess.check_output("ray stop --force", shell=True)


@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.")
def test_start_local_cluster():
    """This tests that ray.client() starts a new local cluster when appropriate.

    * Using `ray.client("local").connect() ` should always create a new
      cluster.
    * Using `ray.client().connect()` should create a new cluster if it doesn't
      connect to an existing one that was started via `ray start --head`..
    """
    driver_template = """
import ray
info = ray.client({address}).connect()
print("NODE_ID:", ray.get_runtime_context().get_node_id())

# Block.
while True:
    time.sleep(1)
"""

    def _get_node_id(p: subprocess.Popen) -> str:
        l = p.stdout.readline().decode("ascii").strip()
        assert "NODE_ID" in l
        return l[len("NODE_ID: ") :]

    p1, p2, p3 = None, None, None
    unbuffered = {"PYTHONUNBUFFERED": "1"}
    try:
        # ray.client() should start a cluster if none is running.
        p1 = run_string_as_driver_nonblocking(
            driver_template.format(address=""), env=unbuffered
        )
        p1_node_id = _get_node_id(p1)

        # ray.client("local") should always start a cluster.
        p2 = run_string_as_driver_nonblocking(driver_template.format(address="'local'"))
        p2_node_id = _get_node_id(p2)

        # ray.client() shouldn't connect to a cluster started by ray.client() or
        # ray.client("local").
        p3 = run_string_as_driver_nonblocking(driver_template.format(address=""))
        p3_node_id = _get_node_id(p3)

        # Check that all three drivers started their own local clusters.
        assert len({p1_node_id, p2_node_id, p3_node_id}) == 3
    finally:
        # Kill processes concurrently.
        if p1 is not None:
            p1.kill()
        if p2 is not None:
            p2.kill()
        if p3 is not None:
            p3.kill()

        # Wait for processes to exit.
        if p1 is not None:
            p1.wait()
        if p2 is not None:
            p2.wait()
        if p3 is not None:
            p3.wait()


@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.")
def test_connect_to_local_cluster(call_ray_start):
    """This tests that ray.client connects to a local cluster when appropriate.

    * Using `ray.client("local").connect() ` should always create a new
      cluster even if one is running.
    * Using `ray.client().connect()` should connect to a local cluster that was
      started with `ray start --head`.
    """
    driver_template = """
import ray
info = ray.client({address}).connect()
print("NODE_ID:", ray.get_runtime_context().get_node_id())
"""

    def _get_node_id(p: subprocess.Popen) -> str:
        l = p.stdout.readline().decode("ascii").strip()
        assert "NODE_ID" in l
        return l[len("NODE_ID: ") :]

    existing_node_id = ray.get_runtime_context().get_node_id()

    p1, p2 = None, None
    unbuffered = {"PYTHONUNBUFFERED": "1"}
    try:
        # ray.client() should connect to the running cluster.
        p1 = run_string_as_driver_nonblocking(
            driver_template.format(address=""), env=unbuffered
        )
        assert _get_node_id(p1) == existing_node_id

        # ray.client("local") should always start a cluster.
        p2 = run_string_as_driver_nonblocking(driver_template.format(address="'local'"))
        assert _get_node_id(p2) != existing_node_id
    finally:
        # Kill processes concurrently.
        if p1 is not None:
            p1.kill()
        if p2 is not None:
            p2.kill()

        # Wait for processes to exit.
        if p1 is not None:
            p1.wait()
        if p2 is not None:
            p2.wait()


def test_non_existent_modules():
    exception = None
    try:
        ray.client("badmodule://address")
    except RuntimeError as e:
        exception = e

    assert exception is not None, "Bad Module did not raise RuntimeException"
    assert "does not exist" in str(exception)


def test_module_lacks_client_builder():
    mock_importlib = Mock()

    def mock_import_module(module_string):
        if module_string == "ray":
            return ray
        else:
            # Mock() does not have a `ClientBuilder` in its scope
            return Mock()

    mock_importlib.import_module = mock_import_module
    with patch("ray.client_builder.importlib", mock_importlib):
        assert isinstance(ray.client(""), ray.ClientBuilder)
        assert isinstance(ray.client("ray://"), ray.ClientBuilder)
        exception = None
        try:
            ray.client("othermodule://")
        except RuntimeError as e:
            exception = e
        assert (
            exception is not None
        ), "Module without ClientBuilder did not raise RuntimeError"
        assert "does not have ClientBuilder" in str(exception)


@pytest.mark.parametrize("address", ["http://127.0.0.1:8265", "https://127.0.0.1:8265"])
def test_http_address_error_message(address):
    """Test that a helpful error is raised when RAY_ADDRESS is set to HTTP/HTTPS."""
    exception = None
    try:
        ray.client(address)
    except ValueError as e:
        exception = e

    assert exception is not None, f"Address {address} did not raise ValueError"
    assert "RAY_ADDRESS" in str(exception)
    assert "GCS address" in str(exception)


@pytest.mark.skipif(sys.platform == "win32", reason="RC Proxy is Flaky on Windows.")
def test_disconnect(call_ray_stop_only, set_enable_auto_connect):
    subprocess.check_output(
        "ray start --head --ray-client-server-port=25555", shell=True
    )
    with ray.client("localhost:25555").namespace("n1").connect():
        # Connect via Ray Client
        namespace = ray.get_runtime_context().namespace
        assert namespace == "n1"
        assert ray.util.client.ray.is_connected()

    with pytest.raises(ray.exceptions.RaySystemError):
        ray.put(300)

    with ray.client(None).namespace("n1").connect():
        # Connect Directly via Driver
        namespace = ray.get_runtime_context().namespace
        assert namespace == "n1"
        assert not ray.util.client.ray.is_connected()

    with pytest.raises(ray.exceptions.RaySystemError):
        ray.put(300)

    ctx = ray.client("localhost:25555").namespace("n1").connect()
    # Connect via Ray Client
    namespace = ray.get_runtime_context().namespace
    assert namespace == "n1"
    assert ray.util.client.ray.is_connected()
    ctx.disconnect()
    # Check idempotency
    ctx.disconnect()
    with pytest.raises(ray.exceptions.RaySystemError):
        ray.put(300)


@pytest.mark.skipif(sys.platform == "win32", reason="RC Proxy is Flaky on Windows.")
def test_address_resolution(call_ray_stop_only):
    subprocess.check_output(
        "ray start --head --ray-client-server-port=50055", shell=True
    )

    with ray.client("localhost:50055").connect():
        assert ray.util.client.ray.is_connected()

    try:
        os.environ["RAY_ADDRESS"] = "local"
        with ray.client("localhost:50055").connect():
            # client(...) takes precedence of RAY_ADDRESS=local
            assert ray.util.client.ray.is_connected()

        # This tries to call `ray.init(address="local") which creates a new Ray
        # instance.
        with ray.client(None).connect():
            wait_for_condition(
                lambda: len(ray._private.services.find_gcs_addresses()) == 2,
                retry_interval_ms=1000,
            )

    finally:
        if os.environ.get("RAY_ADDRESS"):
            del os.environ["RAY_ADDRESS"]
        ray.shutdown()


def mock_connect(*args, **kwargs):
    """
    Force exit instead of actually attempting to connect
    """
    raise ConnectionError


def has_client_deprecation_warn(warning: Warning, expected_replacement: str) -> bool:
    """
    Returns true if expected_replacement is in the message of the passed
    warning, and that the warning mentions deprecation.
    """
    start = "Starting a connection through `ray.client` will be deprecated"
    message = str(warning.message)
    if start not in message:
        return False
    if expected_replacement not in message:
        return False
    return True


@pytest.mark.skipif(
    sys.platform == "win32", reason="pip not supported in Windows runtime envs."
)
@pytest.mark.filterwarnings(
    "default:Starting a connection through `ray.client` will be deprecated"
)
def test_client_deprecation_warn():
    """
    Tests that calling ray.client directly raises a deprecation warning with
    a copy pasteable replacement for the client().connect() call converted
    to ray.init style.
    """
    # Test warning when local client mode is used
    with warnings.catch_warnings(record=True) as w:
        ray.client().connect()
        assert any(has_client_deprecation_warn(warning, "ray.init()") for warning in w)
        ray.shutdown()

    with warnings.catch_warnings(record=True) as w:
        ray.client().namespace("nmspc").env({"pip": ["requests"]}).connect()
    expected = (
        'ray.init(namespace="nmspc", runtime_env=<your_runtime_env>)'  # noqa E501
    )
    assert any(
        has_client_deprecation_warn(warning, expected) for warning in w  # noqa E501
    )
    ray.shutdown()

    server = ray_client_server.serve("localhost", 50055)

    # Test warning when namespace and runtime env aren't specified
    with warnings.catch_warnings(record=True) as w:
        with ray.client("localhost:50055").connect():
            pass
    assert any(
        has_client_deprecation_warn(warning, 'ray.init("ray://localhost:50055")')
        for warning in w
    )

    # Test warning when just namespace specified
    with warnings.catch_warnings(record=True) as w:
        with ray.client("localhost:50055").namespace("nmspc").connect():
            pass
    assert any(
        has_client_deprecation_warn(
            warning, 'ray.init("ray://localhost:50055", namespace="nmspc")'
        )
        for warning in w
    )

    # Test that passing namespace through env doesn't add namespace to the
    # replacement
    with (
        warnings.catch_warnings(record=True) as w,
        patch.dict(os.environ, {"RAY_NAMESPACE": "aksdj"}),
    ):
        with ray.client("localhost:50055").connect():
            pass
    assert any(
        has_client_deprecation_warn(warning, 'ray.init("ray://localhost:50055")')
        for warning in w
    )

    # Skip actually connecting on these, since updating the runtime env is
    # time consuming
    with patch("ray.util.client_connect.connect", mock_connect):
        # Test warning when just runtime_env specified
        with warnings.catch_warnings(record=True) as w:
            try:
                ray.client("localhost:50055").env({"pip": ["requests"]}).connect()
            except ConnectionError:
                pass
        expected = 'ray.init("ray://localhost:50055", runtime_env=<your_runtime_env>)'  # noqa E501
        assert any(has_client_deprecation_warn(warning, expected) for warning in w)

        # Test warning works if both runtime env and namespace specified
        with warnings.catch_warnings(record=True) as w:
            try:
                ray.client("localhost:50055").namespace("nmspc").env(
                    {"pip": ["requests"]}
                ).connect()
            except ConnectionError:
                pass
        expected = 'ray.init("ray://localhost:50055", namespace="nmspc", runtime_env=<your_runtime_env>)'  # noqa E501
        assert any(has_client_deprecation_warn(warning, expected) for warning in w)

        # We don't expect namespace to appear in the warning message, since
        # it was configured through an env var
        with (
            warnings.catch_warnings(record=True) as w,
            patch.dict(os.environ, {"RAY_NAMESPACE": "abcdef"}),
        ):
            try:
                ray.client("localhost:50055").env({"pip": ["requests"]}).connect()
            except ConnectionError:
                pass
        expected = 'ray.init("ray://localhost:50055", runtime_env=<your_runtime_env>)'  # noqa E501
        assert any(has_client_deprecation_warn(warning, expected) for warning in w)

    # cleanup
    server.stop(0)
    subprocess.check_output("ray stop --force", shell=True)


@pytest.mark.parametrize(
    "call_ray_start",
    [
        "ray start --head --num-cpus=2 --min-worker-port=0 --max-worker-port=0 "
        "--port 0 --ray-client-server-port=50056"
    ],
    indirect=True,
)
def test_task_use_prestarted_worker(call_ray_start):
    ray.init("ray://localhost:50056")

    assert (
        len(
            list_workers(
                filters=[("worker_type", "!=", "DRIVER")], raise_on_missing_output=False
            )
        )
        == 2
    )

    @ray.remote(num_cpus=2)
    def f():
        return 42

    assert ray.get(f.remote()) == 42

    assert (
        len(
            list_workers(
                filters=[("worker_type", "!=", "DRIVER")], raise_on_missing_output=False
            )
        )
        == 2
    )


if __name__ == "__main__":
    sys.exit(pytest.main(["-sv", __file__]))