chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,492 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import warnings
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import ray
|
||||
import ray.client_builder as client_builder
|
||||
import ray.util.client.server.server as ray_client_server
|
||||
from ray._common.test_utils import (
|
||||
run_string_as_driver,
|
||||
wait_for_condition,
|
||||
)
|
||||
from ray._private.test_utils import run_string_as_driver_nonblocking
|
||||
from ray.util.state import list_workers
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"address",
|
||||
[
|
||||
"localhost:1234",
|
||||
"localhost:1234/url?params",
|
||||
"1.2.3.4/cluster-1?test_param=param1?",
|
||||
"",
|
||||
],
|
||||
)
|
||||
def test_split_address(address):
|
||||
assert client_builder._split_address(address) == ("ray", address)
|
||||
|
||||
specified_module = f"ray://{address}"
|
||||
assert client_builder._split_address(specified_module) == ("ray", address)
|
||||
|
||||
specified_other_module = f"module://{address}"
|
||||
assert client_builder._split_address(specified_other_module) == ("module", address)
|
||||
non_url_compliant_module = f"module_test://{address}"
|
||||
assert client_builder._split_address(non_url_compliant_module) == (
|
||||
"module_test",
|
||||
address,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"address", ["localhost", "1.2.3.4:1200", "ray://1.2.3.4:1200", "local", None]
|
||||
)
|
||||
def test_client(address):
|
||||
builder = client_builder.client(address)
|
||||
assert isinstance(builder, client_builder.ClientBuilder)
|
||||
if address in ("local", None):
|
||||
assert isinstance(builder, client_builder._LocalClientBuilder)
|
||||
else:
|
||||
assert type(builder) is client_builder.ClientBuilder
|
||||
assert builder.address == address.replace("ray://", "")
|
||||
|
||||
|
||||
def test_namespace(ray_start_cluster):
|
||||
"""
|
||||
Most of the "checks" in this test case rely on the fact that
|
||||
`run_string_as_driver` will throw an exception if the driver string exits
|
||||
with a non-zero exit code (e.g. when the driver scripts throws an
|
||||
exception). Since all of these drivers start named, detached actors, the
|
||||
most likely failure case would be a collision of named actors if they're
|
||||
put in the same namespace.
|
||||
|
||||
This test checks that:
|
||||
* When two drivers don't specify a namespace, they are placed in different
|
||||
anonymous namespaces.
|
||||
* When two drivers specify a namespace, they collide.
|
||||
* The namespace name (as provided by the runtime context) is correct.
|
||||
"""
|
||||
cluster = ray_start_cluster
|
||||
cluster.add_node(num_cpus=4, ray_client_server_port=50055)
|
||||
cluster.wait_for_nodes(1)
|
||||
|
||||
template = """
|
||||
import ray
|
||||
ray.client("localhost:50055").namespace({namespace}).connect()
|
||||
|
||||
@ray.remote
|
||||
class Foo:
|
||||
def ping(self):
|
||||
return "pong"
|
||||
|
||||
a = Foo.options(lifetime="detached", name="abc").remote()
|
||||
ray.get(a.ping.remote())
|
||||
print("Current namespace:", ray.get_runtime_context().namespace)
|
||||
"""
|
||||
|
||||
anon_driver = template.format(namespace="None")
|
||||
run_string_as_driver(anon_driver)
|
||||
# This second run will fail if the actors don't run in separate anonymous
|
||||
# namespaces.
|
||||
run_string_as_driver(anon_driver)
|
||||
|
||||
run_in_namespace = template.format(namespace="'namespace'")
|
||||
script_output = run_string_as_driver(run_in_namespace)
|
||||
# The second run fails because the actors are run in the same namespace.
|
||||
with pytest.raises(subprocess.CalledProcessError):
|
||||
run_string_as_driver(run_in_namespace)
|
||||
|
||||
assert "Current namespace: namespace" in script_output
|
||||
subprocess.check_output("ray stop --force", shell=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.")
|
||||
def test_start_local_cluster():
|
||||
"""This tests that ray.client() starts a new local cluster when appropriate.
|
||||
|
||||
* Using `ray.client("local").connect() ` should always create a new
|
||||
cluster.
|
||||
* Using `ray.client().connect()` should create a new cluster if it doesn't
|
||||
connect to an existing one that was started via `ray start --head`..
|
||||
"""
|
||||
driver_template = """
|
||||
import ray
|
||||
info = ray.client({address}).connect()
|
||||
print("NODE_ID:", ray.get_runtime_context().get_node_id())
|
||||
|
||||
# Block.
|
||||
while True:
|
||||
time.sleep(1)
|
||||
"""
|
||||
|
||||
def _get_node_id(p: subprocess.Popen) -> str:
|
||||
l = p.stdout.readline().decode("ascii").strip()
|
||||
assert "NODE_ID" in l
|
||||
return l[len("NODE_ID: ") :]
|
||||
|
||||
p1, p2, p3 = None, None, None
|
||||
unbuffered = {"PYTHONUNBUFFERED": "1"}
|
||||
try:
|
||||
# ray.client() should start a cluster if none is running.
|
||||
p1 = run_string_as_driver_nonblocking(
|
||||
driver_template.format(address=""), env=unbuffered
|
||||
)
|
||||
p1_node_id = _get_node_id(p1)
|
||||
|
||||
# ray.client("local") should always start a cluster.
|
||||
p2 = run_string_as_driver_nonblocking(driver_template.format(address="'local'"))
|
||||
p2_node_id = _get_node_id(p2)
|
||||
|
||||
# ray.client() shouldn't connect to a cluster started by ray.client() or
|
||||
# ray.client("local").
|
||||
p3 = run_string_as_driver_nonblocking(driver_template.format(address=""))
|
||||
p3_node_id = _get_node_id(p3)
|
||||
|
||||
# Check that all three drivers started their own local clusters.
|
||||
assert len({p1_node_id, p2_node_id, p3_node_id}) == 3
|
||||
finally:
|
||||
# Kill processes concurrently.
|
||||
if p1 is not None:
|
||||
p1.kill()
|
||||
if p2 is not None:
|
||||
p2.kill()
|
||||
if p3 is not None:
|
||||
p3.kill()
|
||||
|
||||
# Wait for processes to exit.
|
||||
if p1 is not None:
|
||||
p1.wait()
|
||||
if p2 is not None:
|
||||
p2.wait()
|
||||
if p3 is not None:
|
||||
p3.wait()
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Flaky on Windows.")
|
||||
def test_connect_to_local_cluster(call_ray_start):
|
||||
"""This tests that ray.client connects to a local cluster when appropriate.
|
||||
|
||||
* Using `ray.client("local").connect() ` should always create a new
|
||||
cluster even if one is running.
|
||||
* Using `ray.client().connect()` should connect to a local cluster that was
|
||||
started with `ray start --head`.
|
||||
"""
|
||||
driver_template = """
|
||||
import ray
|
||||
info = ray.client({address}).connect()
|
||||
print("NODE_ID:", ray.get_runtime_context().get_node_id())
|
||||
"""
|
||||
|
||||
def _get_node_id(p: subprocess.Popen) -> str:
|
||||
l = p.stdout.readline().decode("ascii").strip()
|
||||
assert "NODE_ID" in l
|
||||
return l[len("NODE_ID: ") :]
|
||||
|
||||
existing_node_id = ray.get_runtime_context().get_node_id()
|
||||
|
||||
p1, p2 = None, None
|
||||
unbuffered = {"PYTHONUNBUFFERED": "1"}
|
||||
try:
|
||||
# ray.client() should connect to the running cluster.
|
||||
p1 = run_string_as_driver_nonblocking(
|
||||
driver_template.format(address=""), env=unbuffered
|
||||
)
|
||||
assert _get_node_id(p1) == existing_node_id
|
||||
|
||||
# ray.client("local") should always start a cluster.
|
||||
p2 = run_string_as_driver_nonblocking(driver_template.format(address="'local'"))
|
||||
assert _get_node_id(p2) != existing_node_id
|
||||
finally:
|
||||
# Kill processes concurrently.
|
||||
if p1 is not None:
|
||||
p1.kill()
|
||||
if p2 is not None:
|
||||
p2.kill()
|
||||
|
||||
# Wait for processes to exit.
|
||||
if p1 is not None:
|
||||
p1.wait()
|
||||
if p2 is not None:
|
||||
p2.wait()
|
||||
|
||||
|
||||
def test_non_existent_modules():
|
||||
exception = None
|
||||
try:
|
||||
ray.client("badmodule://address")
|
||||
except RuntimeError as e:
|
||||
exception = e
|
||||
|
||||
assert exception is not None, "Bad Module did not raise RuntimeException"
|
||||
assert "does not exist" in str(exception)
|
||||
|
||||
|
||||
def test_module_lacks_client_builder():
|
||||
mock_importlib = Mock()
|
||||
|
||||
def mock_import_module(module_string):
|
||||
if module_string == "ray":
|
||||
return ray
|
||||
else:
|
||||
# Mock() does not have a `ClientBuilder` in its scope
|
||||
return Mock()
|
||||
|
||||
mock_importlib.import_module = mock_import_module
|
||||
with patch("ray.client_builder.importlib", mock_importlib):
|
||||
assert isinstance(ray.client(""), ray.ClientBuilder)
|
||||
assert isinstance(ray.client("ray://"), ray.ClientBuilder)
|
||||
exception = None
|
||||
try:
|
||||
ray.client("othermodule://")
|
||||
except RuntimeError as e:
|
||||
exception = e
|
||||
assert (
|
||||
exception is not None
|
||||
), "Module without ClientBuilder did not raise RuntimeError"
|
||||
assert "does not have ClientBuilder" in str(exception)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("address", ["http://127.0.0.1:8265", "https://127.0.0.1:8265"])
|
||||
def test_http_address_error_message(address):
|
||||
"""Test that a helpful error is raised when RAY_ADDRESS is set to HTTP/HTTPS."""
|
||||
exception = None
|
||||
try:
|
||||
ray.client(address)
|
||||
except ValueError as e:
|
||||
exception = e
|
||||
|
||||
assert exception is not None, f"Address {address} did not raise ValueError"
|
||||
assert "RAY_ADDRESS" in str(exception)
|
||||
assert "GCS address" in str(exception)
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="RC Proxy is Flaky on Windows.")
|
||||
def test_disconnect(call_ray_stop_only, set_enable_auto_connect):
|
||||
subprocess.check_output(
|
||||
"ray start --head --ray-client-server-port=25555", shell=True
|
||||
)
|
||||
with ray.client("localhost:25555").namespace("n1").connect():
|
||||
# Connect via Ray Client
|
||||
namespace = ray.get_runtime_context().namespace
|
||||
assert namespace == "n1"
|
||||
assert ray.util.client.ray.is_connected()
|
||||
|
||||
with pytest.raises(ray.exceptions.RaySystemError):
|
||||
ray.put(300)
|
||||
|
||||
with ray.client(None).namespace("n1").connect():
|
||||
# Connect Directly via Driver
|
||||
namespace = ray.get_runtime_context().namespace
|
||||
assert namespace == "n1"
|
||||
assert not ray.util.client.ray.is_connected()
|
||||
|
||||
with pytest.raises(ray.exceptions.RaySystemError):
|
||||
ray.put(300)
|
||||
|
||||
ctx = ray.client("localhost:25555").namespace("n1").connect()
|
||||
# Connect via Ray Client
|
||||
namespace = ray.get_runtime_context().namespace
|
||||
assert namespace == "n1"
|
||||
assert ray.util.client.ray.is_connected()
|
||||
ctx.disconnect()
|
||||
# Check idempotency
|
||||
ctx.disconnect()
|
||||
with pytest.raises(ray.exceptions.RaySystemError):
|
||||
ray.put(300)
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="RC Proxy is Flaky on Windows.")
|
||||
def test_address_resolution(call_ray_stop_only):
|
||||
subprocess.check_output(
|
||||
"ray start --head --ray-client-server-port=50055", shell=True
|
||||
)
|
||||
|
||||
with ray.client("localhost:50055").connect():
|
||||
assert ray.util.client.ray.is_connected()
|
||||
|
||||
try:
|
||||
os.environ["RAY_ADDRESS"] = "local"
|
||||
with ray.client("localhost:50055").connect():
|
||||
# client(...) takes precedence of RAY_ADDRESS=local
|
||||
assert ray.util.client.ray.is_connected()
|
||||
|
||||
# This tries to call `ray.init(address="local") which creates a new Ray
|
||||
# instance.
|
||||
with ray.client(None).connect():
|
||||
wait_for_condition(
|
||||
lambda: len(ray._private.services.find_gcs_addresses()) == 2,
|
||||
retry_interval_ms=1000,
|
||||
)
|
||||
|
||||
finally:
|
||||
if os.environ.get("RAY_ADDRESS"):
|
||||
del os.environ["RAY_ADDRESS"]
|
||||
ray.shutdown()
|
||||
|
||||
|
||||
def mock_connect(*args, **kwargs):
|
||||
"""
|
||||
Force exit instead of actually attempting to connect
|
||||
"""
|
||||
raise ConnectionError
|
||||
|
||||
|
||||
def has_client_deprecation_warn(warning: Warning, expected_replacement: str) -> bool:
|
||||
"""
|
||||
Returns true if expected_replacement is in the message of the passed
|
||||
warning, and that the warning mentions deprecation.
|
||||
"""
|
||||
start = "Starting a connection through `ray.client` will be deprecated"
|
||||
message = str(warning.message)
|
||||
if start not in message:
|
||||
return False
|
||||
if expected_replacement not in message:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32", reason="pip not supported in Windows runtime envs."
|
||||
)
|
||||
@pytest.mark.filterwarnings(
|
||||
"default:Starting a connection through `ray.client` will be deprecated"
|
||||
)
|
||||
def test_client_deprecation_warn():
|
||||
"""
|
||||
Tests that calling ray.client directly raises a deprecation warning with
|
||||
a copy pasteable replacement for the client().connect() call converted
|
||||
to ray.init style.
|
||||
"""
|
||||
# Test warning when local client mode is used
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
ray.client().connect()
|
||||
assert any(has_client_deprecation_warn(warning, "ray.init()") for warning in w)
|
||||
ray.shutdown()
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
ray.client().namespace("nmspc").env({"pip": ["requests"]}).connect()
|
||||
expected = (
|
||||
'ray.init(namespace="nmspc", runtime_env=<your_runtime_env>)' # noqa E501
|
||||
)
|
||||
assert any(
|
||||
has_client_deprecation_warn(warning, expected) for warning in w # noqa E501
|
||||
)
|
||||
ray.shutdown()
|
||||
|
||||
server = ray_client_server.serve("localhost", 50055)
|
||||
|
||||
# Test warning when namespace and runtime env aren't specified
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
with ray.client("localhost:50055").connect():
|
||||
pass
|
||||
assert any(
|
||||
has_client_deprecation_warn(warning, 'ray.init("ray://localhost:50055")')
|
||||
for warning in w
|
||||
)
|
||||
|
||||
# Test warning when just namespace specified
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
with ray.client("localhost:50055").namespace("nmspc").connect():
|
||||
pass
|
||||
assert any(
|
||||
has_client_deprecation_warn(
|
||||
warning, 'ray.init("ray://localhost:50055", namespace="nmspc")'
|
||||
)
|
||||
for warning in w
|
||||
)
|
||||
|
||||
# Test that passing namespace through env doesn't add namespace to the
|
||||
# replacement
|
||||
with (
|
||||
warnings.catch_warnings(record=True) as w,
|
||||
patch.dict(os.environ, {"RAY_NAMESPACE": "aksdj"}),
|
||||
):
|
||||
with ray.client("localhost:50055").connect():
|
||||
pass
|
||||
assert any(
|
||||
has_client_deprecation_warn(warning, 'ray.init("ray://localhost:50055")')
|
||||
for warning in w
|
||||
)
|
||||
|
||||
# Skip actually connecting on these, since updating the runtime env is
|
||||
# time consuming
|
||||
with patch("ray.util.client_connect.connect", mock_connect):
|
||||
# Test warning when just runtime_env specified
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
try:
|
||||
ray.client("localhost:50055").env({"pip": ["requests"]}).connect()
|
||||
except ConnectionError:
|
||||
pass
|
||||
expected = 'ray.init("ray://localhost:50055", runtime_env=<your_runtime_env>)' # noqa E501
|
||||
assert any(has_client_deprecation_warn(warning, expected) for warning in w)
|
||||
|
||||
# Test warning works if both runtime env and namespace specified
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
try:
|
||||
ray.client("localhost:50055").namespace("nmspc").env(
|
||||
{"pip": ["requests"]}
|
||||
).connect()
|
||||
except ConnectionError:
|
||||
pass
|
||||
expected = 'ray.init("ray://localhost:50055", namespace="nmspc", runtime_env=<your_runtime_env>)' # noqa E501
|
||||
assert any(has_client_deprecation_warn(warning, expected) for warning in w)
|
||||
|
||||
# We don't expect namespace to appear in the warning message, since
|
||||
# it was configured through an env var
|
||||
with (
|
||||
warnings.catch_warnings(record=True) as w,
|
||||
patch.dict(os.environ, {"RAY_NAMESPACE": "abcdef"}),
|
||||
):
|
||||
try:
|
||||
ray.client("localhost:50055").env({"pip": ["requests"]}).connect()
|
||||
except ConnectionError:
|
||||
pass
|
||||
expected = 'ray.init("ray://localhost:50055", runtime_env=<your_runtime_env>)' # noqa E501
|
||||
assert any(has_client_deprecation_warn(warning, expected) for warning in w)
|
||||
|
||||
# cleanup
|
||||
server.stop(0)
|
||||
subprocess.check_output("ray stop --force", shell=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call_ray_start",
|
||||
[
|
||||
"ray start --head --num-cpus=2 --min-worker-port=0 --max-worker-port=0 "
|
||||
"--port 0 --ray-client-server-port=50056"
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_task_use_prestarted_worker(call_ray_start):
|
||||
ray.init("ray://localhost:50056")
|
||||
|
||||
assert (
|
||||
len(
|
||||
list_workers(
|
||||
filters=[("worker_type", "!=", "DRIVER")], raise_on_missing_output=False
|
||||
)
|
||||
)
|
||||
== 2
|
||||
)
|
||||
|
||||
@ray.remote(num_cpus=2)
|
||||
def f():
|
||||
return 42
|
||||
|
||||
assert ray.get(f.remote()) == 42
|
||||
|
||||
assert (
|
||||
len(
|
||||
list_workers(
|
||||
filters=[("worker_type", "!=", "DRIVER")], raise_on_missing_output=False
|
||||
)
|
||||
)
|
||||
== 2
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
Reference in New Issue
Block a user