chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,399 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
from telnetlib import Telnet
|
||||
from typing import Union
|
||||
|
||||
import pexpect
|
||||
import pytest
|
||||
from pexpect.popen_spawn import PopenSpawn
|
||||
|
||||
import ray
|
||||
from ray._common.network_utils import parse_address
|
||||
from ray._common.test_utils import (
|
||||
run_string_as_driver,
|
||||
wait_for_condition,
|
||||
)
|
||||
from ray._private import ray_constants, services
|
||||
from ray.cluster_utils import Cluster, cluster_not_supported
|
||||
|
||||
|
||||
def _pexpect_spawn(cmd: str) -> Union["pexpect.spawn", PopenSpawn]:
|
||||
"""Handles compatibility for pexpect on Windows."""
|
||||
if sys.platform == "win32":
|
||||
p = PopenSpawn(cmd, encoding="utf-8")
|
||||
else:
|
||||
p = pexpect.spawn(cmd)
|
||||
|
||||
return p
|
||||
|
||||
|
||||
def test_ray_debugger_breakpoint(shutdown_only):
|
||||
ray.init(num_cpus=1, runtime_env={"env_vars": {"RAY_DEBUG": "legacy"}})
|
||||
|
||||
@ray.remote
|
||||
def f():
|
||||
ray.util.pdb.set_trace()
|
||||
return 1
|
||||
|
||||
result = f.remote()
|
||||
|
||||
wait_for_condition(
|
||||
lambda: len(
|
||||
ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
> 0
|
||||
)
|
||||
active_sessions = ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
assert len(active_sessions) == 1
|
||||
|
||||
# Now continue execution:
|
||||
session = json.loads(
|
||||
ray.experimental.internal_kv._internal_kv_get(
|
||||
active_sessions[0], namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
host, port = parse_address(session["pdb_address"])
|
||||
assert host == "localhost" # Should be private by default.
|
||||
|
||||
tn = Telnet(host, int(port))
|
||||
tn.write(b"c\n")
|
||||
|
||||
# The message above should cause this to return now.
|
||||
ray.get(result)
|
||||
|
||||
|
||||
def test_ray_debugger_commands(shutdown_only):
|
||||
ray.init(num_cpus=2, runtime_env={"env_vars": {"RAY_DEBUG": "legacy"}})
|
||||
|
||||
@ray.remote
|
||||
def f():
|
||||
"""We support unicode too: 🐛"""
|
||||
ray.util.pdb.set_trace()
|
||||
|
||||
result1 = f.remote()
|
||||
result2 = f.remote()
|
||||
|
||||
wait_for_condition(
|
||||
lambda: len(
|
||||
ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
> 0
|
||||
)
|
||||
|
||||
# Make sure that calling "continue" in the debugger
|
||||
# gives back control to the debugger loop:
|
||||
p = _pexpect_spawn("ray debug")
|
||||
p.expect("Enter breakpoint index or press enter to refresh: ")
|
||||
p.sendline("0")
|
||||
p.expect("-> ray.util.pdb.set_trace()")
|
||||
p.sendline("ll")
|
||||
# Cannot use the 🐛 symbol here because pexpect doesn't support
|
||||
# unicode, but this test also does nicely:
|
||||
p.expect("unicode")
|
||||
p.sendline("c")
|
||||
p.expect("Enter breakpoint index or press enter to refresh: ")
|
||||
p.sendline("0")
|
||||
p.expect("-> ray.util.pdb.set_trace()")
|
||||
p.sendline("c")
|
||||
|
||||
ray.get([result1, result2])
|
||||
|
||||
|
||||
def test_ray_debugger_stepping(shutdown_only):
|
||||
os.environ["RAY_DEBUG"] = "legacy"
|
||||
ray.init(num_cpus=1, runtime_env={"env_vars": {"RAY_DEBUG": "legacy"}})
|
||||
|
||||
@ray.remote
|
||||
def g():
|
||||
return None
|
||||
|
||||
@ray.remote
|
||||
def f():
|
||||
ray.util.pdb.set_trace()
|
||||
x = g.remote()
|
||||
return ray.get(x)
|
||||
|
||||
result = f.remote()
|
||||
|
||||
wait_for_condition(
|
||||
lambda: len(
|
||||
ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
> 0
|
||||
)
|
||||
|
||||
p = _pexpect_spawn("ray debug")
|
||||
p.expect("Enter breakpoint index or press enter to refresh: ")
|
||||
p.sendline("0")
|
||||
p.expect("-> x = g.remote()")
|
||||
p.sendline("remote")
|
||||
p.expect("(Pdb)")
|
||||
p.sendline("get")
|
||||
p.expect("(Pdb)")
|
||||
p.sendline("continue")
|
||||
|
||||
# This should succeed now!
|
||||
ray.get(result)
|
||||
|
||||
|
||||
def test_ray_debugger_recursive(shutdown_only):
|
||||
os.environ["RAY_DEBUG"] = "legacy"
|
||||
ray.init(num_cpus=1, runtime_env={"env_vars": {"RAY_DEBUG": "legacy"}})
|
||||
|
||||
@ray.remote
|
||||
def fact(n):
|
||||
if n < 1:
|
||||
return n
|
||||
ray.util.pdb.set_trace()
|
||||
n_id = fact.remote(n - 1)
|
||||
return n * ray.get(n_id)
|
||||
|
||||
result = fact.remote(5)
|
||||
|
||||
wait_for_condition(
|
||||
lambda: len(
|
||||
ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
> 0
|
||||
)
|
||||
|
||||
p = _pexpect_spawn("ray debug")
|
||||
p.expect("Enter breakpoint index or press enter to refresh: ")
|
||||
p.sendline("0")
|
||||
p.expect("(Pdb)")
|
||||
p.sendline("remote")
|
||||
p.expect("(Pdb)")
|
||||
p.sendline("remote")
|
||||
p.expect("(Pdb)")
|
||||
p.sendline("remote")
|
||||
p.expect("(Pdb)")
|
||||
p.sendline("remote")
|
||||
p.expect("(Pdb)")
|
||||
p.sendline("remote")
|
||||
p.expect("(Pdb)")
|
||||
p.sendline("remote")
|
||||
|
||||
ray.get(result)
|
||||
|
||||
|
||||
def test_job_exit_cleanup(ray_start_regular):
|
||||
address = ray_start_regular["address"]
|
||||
|
||||
driver_script = """
|
||||
import time
|
||||
|
||||
import ray
|
||||
ray.init(address="{}", runtime_env={{"env_vars": {{"RAY_DEBUG": "legacy"}}}})
|
||||
|
||||
@ray.remote
|
||||
def f():
|
||||
ray.util.rpdb.set_trace()
|
||||
|
||||
f.remote()
|
||||
# Give the remote function long enough to actually run.
|
||||
time.sleep(5)
|
||||
""".format(
|
||||
address
|
||||
)
|
||||
|
||||
assert not len(
|
||||
ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
|
||||
run_string_as_driver(driver_script)
|
||||
|
||||
def one_active_session():
|
||||
return len(
|
||||
ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
|
||||
wait_for_condition(one_active_session)
|
||||
|
||||
# Start the debugger. This should clean up any existing sessions that
|
||||
# belong to dead jobs.
|
||||
_ = _pexpect_spawn("ray debug")
|
||||
|
||||
def no_active_sessions():
|
||||
return not len(
|
||||
ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
|
||||
wait_for_condition(no_active_sessions)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32", reason="Windows does not print '--address' on init"
|
||||
)
|
||||
@pytest.mark.parametrize("ray_debugger_external", [False, True])
|
||||
def test_ray_debugger_public(shutdown_only, call_ray_stop_only, ray_debugger_external):
|
||||
redis_substring_prefix = "--address='"
|
||||
cmd = ["ray", "start", "--head", "--num-cpus=1"]
|
||||
if ray_debugger_external:
|
||||
cmd.append("--ray-debugger-external")
|
||||
out = ray._common.utils.decode(
|
||||
subprocess.check_output(cmd, stderr=subprocess.STDOUT)
|
||||
)
|
||||
# Get the redis address from the output.
|
||||
redis_substring_prefix = "--address='"
|
||||
address_location = out.find(redis_substring_prefix) + len(redis_substring_prefix)
|
||||
address = out[address_location:]
|
||||
address = address.split("'")[0]
|
||||
|
||||
ray.init(address=address, runtime_env={"env_vars": {"RAY_DEBUG": "legacy"}})
|
||||
|
||||
@ray.remote
|
||||
def f():
|
||||
ray.util.pdb.set_trace()
|
||||
return 1
|
||||
|
||||
result = f.remote()
|
||||
|
||||
wait_for_condition(
|
||||
lambda: len(
|
||||
ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
> 0
|
||||
)
|
||||
|
||||
active_sessions = ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
assert len(active_sessions) == 1
|
||||
session = json.loads(
|
||||
ray.experimental.internal_kv._internal_kv_get(
|
||||
active_sessions[0], namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
|
||||
host, port = parse_address(session["pdb_address"])
|
||||
if ray_debugger_external:
|
||||
assert host == services.get_node_ip_address(), host
|
||||
else:
|
||||
assert host == "localhost", host
|
||||
|
||||
# Check that we can successfully connect to both breakpoints.
|
||||
tn = Telnet(host, int(port))
|
||||
tn.write(b"c\n")
|
||||
|
||||
# The message above should cause this to return now.
|
||||
ray.get(result)
|
||||
|
||||
|
||||
@pytest.mark.xfail(cluster_not_supported, reason="cluster not supported")
|
||||
@pytest.mark.parametrize("ray_debugger_external", [False, True])
|
||||
def test_ray_debugger_public_multi_node(shutdown_only, ray_debugger_external):
|
||||
c = Cluster(
|
||||
initialize_head=True,
|
||||
connect=False,
|
||||
head_node_args={
|
||||
"num_cpus": 0,
|
||||
"num_gpus": 1,
|
||||
"ray_debugger_external": ray_debugger_external,
|
||||
},
|
||||
)
|
||||
c.add_node(num_cpus=1, ray_debugger_external=ray_debugger_external)
|
||||
|
||||
ray.init(runtime_env={"env_vars": {"RAY_DEBUG": "legacy"}})
|
||||
|
||||
@ray.remote
|
||||
def f():
|
||||
ray.util.pdb.set_trace()
|
||||
return 1
|
||||
|
||||
# num_gpus=1 forces the task onto the head node.
|
||||
head_node_result = f.options(num_cpus=0, num_gpus=1).remote()
|
||||
|
||||
# num_cpus=1 forces the task onto the worker node.
|
||||
worker_node_result = f.options(num_cpus=1).remote()
|
||||
|
||||
wait_for_condition(
|
||||
lambda: len(
|
||||
ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
== 2
|
||||
)
|
||||
|
||||
active_sessions = ray.experimental.internal_kv._internal_kv_list(
|
||||
"RAY_PDB_", namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
assert len(active_sessions) == 2
|
||||
session1 = json.loads(
|
||||
ray.experimental.internal_kv._internal_kv_get(
|
||||
active_sessions[0], namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
session2 = json.loads(
|
||||
ray.experimental.internal_kv._internal_kv_get(
|
||||
active_sessions[1], namespace=ray_constants.KV_NAMESPACE_PDB
|
||||
)
|
||||
)
|
||||
|
||||
host1, port1 = parse_address(session1["pdb_address"])
|
||||
if ray_debugger_external:
|
||||
assert host1 == services.get_node_ip_address(), host1
|
||||
else:
|
||||
assert host1 == "localhost", host1
|
||||
|
||||
host2, port2 = parse_address(session2["pdb_address"])
|
||||
if ray_debugger_external:
|
||||
assert host2 == services.get_node_ip_address(), host2
|
||||
else:
|
||||
assert host2 == "localhost", host2
|
||||
|
||||
# Check that we can successfully connect to both breakpoints.
|
||||
tn1 = Telnet(host1, int(port1))
|
||||
tn1.write(b"c\n")
|
||||
|
||||
tn2 = Telnet(host2, int(port2))
|
||||
tn2.write(b"c\n")
|
||||
|
||||
# The messages above should cause these to return now.
|
||||
ray.get([head_node_result, worker_node_result])
|
||||
|
||||
|
||||
def test_env_var_enables_ray_debugger():
|
||||
with unittest.mock.patch.dict(os.environ):
|
||||
os.environ["RAY_DEBUG_POST_MORTEM"] = "1"
|
||||
assert ray.util.pdb._is_ray_debugger_post_mortem_enabled(), (
|
||||
"Expected post-mortem Debugger to be enabled when "
|
||||
"RAY_DEBUG_POST_MORTEM env var is present."
|
||||
)
|
||||
|
||||
with unittest.mock.patch.dict(os.environ):
|
||||
if "RAY_DEBUG_POST_MORTEM" in os.environ:
|
||||
del os.environ["RAY_DEBUG_POST_MORTEM"]
|
||||
|
||||
assert not ray.util.pdb._is_ray_debugger_post_mortem_enabled(), (
|
||||
"Expected post-mortem Debugger to be disabled when "
|
||||
"RAY_DEBUG_POST_MORTEM env var is absent."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Make subprocess happy in bazel.
|
||||
os.environ["LC_ALL"] = "en_US.UTF-8"
|
||||
os.environ["LANG"] = "en_US.UTF-8"
|
||||
sys.exit(pytest.main(["-sv", __file__]))
|
||||
Reference in New Issue
Block a user