import os
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from typing import Optional

import pytest

import ray
import ray._private.gcs_utils as gcs_utils
import ray.cluster_utils
from ray._common.test_utils import (
    SignalActor,
    run_string_as_driver,
    wait_for_condition,
)
from ray._private.test_utils import (
    kill_actor_and_wait_for_failure,
    make_global_state_accessor,
    wait_for_pid_to_exit,
)
from ray.experimental.internal_kv import _internal_kv_get, _internal_kv_put
from ray.util.state import list_actors


def test_actors_on_nodes_with_no_cpus(ray_start_no_cpu):
    @ray.remote
    class Foo:
        def method(self):
            pass

    f = Foo.remote()
    ready_ids, _ = ray.wait([f.method.remote()], timeout=0.1)
    assert ready_ids == []


def test_actor_load_balancing(ray_start_cluster):
    """Check that actor scheduling is load balanced across worker nodes."""
    cluster = ray_start_cluster
    worker_node_ids = set()
    for i in range(2):
        worker_node_ids.add(cluster.add_node(num_cpus=1).node_id)

    ray.init(address=cluster.address)

    @ray.remote
    class Actor:
        def get_node_id(self) -> str:
            return ray.get_runtime_context().get_node_id()

    # Schedule a group of actors, ensure that the actors are spread between all nodes.
    node_ids = ray.get([Actor.remote().get_node_id.remote() for _ in range(10)])
    assert set(node_ids) == worker_node_ids


@pytest.mark.parametrize(
    "ray_start_regular",
    [
        {
            "resources": {"actor": 1},
            "num_cpus": 2,
        }
    ],
    indirect=True,
)
def test_deleted_actor_no_restart(ray_start_regular):
    @ray.remote(resources={"actor": 1}, max_restarts=3)
    class Actor:
        def method(self):
            return 1

        def getpid(self):
            return os.getpid()

    @ray.remote
    def f(actor, signal):
        ray.get(signal.wait.remote())
        return ray.get(actor.method.remote())

    signal = SignalActor.remote()
    a = Actor.remote()
    pid = ray.get(a.getpid.remote())
    # Pass the handle to another task that cannot run yet.
    x_id = f.remote(a, signal)
    # Delete the original handle. The actor should not get killed yet.
    del a

    # Once the task finishes, the actor process should get killed.
    ray.get(signal.send.remote())
    assert ray.get(x_id) == 1
    wait_for_pid_to_exit(pid)

    # Create another actor with the same resource requirement to make sure the
    # old one was not restarted.
    a = Actor.remote()
    pid = ray.get(a.getpid.remote())


def test_exception_raised_when_actor_node_dies(ray_start_cluster_head):
    cluster = ray_start_cluster_head
    remote_node = cluster.add_node()

    @ray.remote(max_restarts=0, scheduling_strategy="SPREAD")
    class Counter:
        def __init__(self):
            self.x = 0

        def node_id(self):
            return ray._private.worker.global_worker.node.unique_id

        def inc(self):
            self.x += 1
            return self.x

    # Create an actor that is not on the raylet.
    actor = Counter.remote()
    while ray.get(actor.node_id.remote()) != remote_node.unique_id:
        actor = Counter.remote()

    # Kill the second node.
    cluster.remove_node(remote_node)

    # Submit some new actor tasks both before and after the node failure is
    # detected. Make sure that getting the result raises an exception.
    for _ in range(10):
        # Submit some new actor tasks.
        x_ids = [actor.inc.remote() for _ in range(5)]
        for x_id in x_ids:
            with pytest.raises(ray.exceptions.RayActorError):
                # There is some small chance that ray.get will actually
                # succeed (if the object is transferred before the raylet
                # dies).
                ray.get(x_id)


def test_actor_fail_during_constructor_restart(ray_start_cluster_head):
    cluster = ray_start_cluster_head
    worker_nodes = {
        node.node_id: node for node in [cluster.add_node() for _ in range(2)]
    }

    @ray.remote
    class ReportNodeIDActor:
        def __init__(self):
            self._reported_node_id = None

        def report(self, node_id: str):
            self._reported_node_id = node_id

        def get(self) -> Optional[str]:
            return self._reported_node_id

    # Pin these actors to the head node so they don't crash.
    # Occupy the 1 CPU on the head node so the actor below is forced to a worker node.
    pin_head_resources = {"node:__internal_head__": 0.1}
    report_node_id_actor = ReportNodeIDActor.options(
        num_cpus=0.5, resources=pin_head_resources
    ).remote()
    signal = SignalActor.options(
        num_cpus=0.5,
        resources=pin_head_resources,
    ).remote()

    @ray.remote(max_restarts=1, max_task_retries=-1)
    class Actor:
        def __init__(self):
            ray.get(
                report_node_id_actor.report.remote(
                    ray.get_runtime_context().get_node_id()
                )
            )
            ray.get(signal.wait.remote())

    # Create the actor and wait for it to start initializing.
    actor = Actor.remote()
    wait_for_condition(lambda: ray.get(signal.cur_num_waiters.remote()) == 1)
    actor_node_id = ray.get(report_node_id_actor.get.remote())
    assert actor_node_id is not None

    # Kill the worker node.
    cluster.remove_node(worker_nodes[actor_node_id])

    # Verify that the actor was restarted on the other node.
    ray.get(signal.send.remote())
    ray.get(actor.__ray_ready__.remote())
    assert ray.get(report_node_id_actor.get.remote()) != actor_node_id


def test_actor_restart_multiple_callers(ray_start_cluster):
    cluster = ray_start_cluster
    _ = cluster.add_node(num_cpus=4)
    ray.init(address=cluster.address)

    _ = cluster.add_node(num_cpus=4)
    actor_worker_node = cluster.add_node(num_cpus=0, resources={"actor": 1})
    cluster.wait_for_nodes()

    @ray.remote(
        num_cpus=0,
        # Only one of the callers should successfully restart the actor.
        max_restarts=1,
        # Retry transient ActorUnavailableErrors.
        max_task_retries=-1,
        # Schedule the actor on actor_worker_node.
        resources={"actor": 1},
    )
    class A:
        def get_node_id(self) -> str:
            return ray.get_runtime_context().get_node_id()

    a = A.remote()

    @ray.remote
    def call_a() -> str:
        return ray.get(a.get_node_id.remote())

    # Run caller tasks in parallel across the other two nodes.
    results = ray.get([call_a.remote() for _ in range(8)])
    assert all(r == actor_worker_node.node_id for r in results), results

    # Kill the node that the actor is running on.
    cluster.remove_node(actor_worker_node)

    # Run caller tasks in parallel again.
    refs = [call_a.remote() for _ in range(8)]
    ready, _ = ray.wait(refs, timeout=0.1)
    assert len(ready) == 0

    # The actor should be restarted once the node becomes available.
    new_actor_worker_node = cluster.add_node(num_cpus=0, resources={"actor": 1})
    results = ray.get(refs)
    assert all(r == new_actor_worker_node.node_id for r in results), results


@pytest.fixture
def setup_queue_actor():
    ray.init(num_cpus=1, object_store_memory=int(150 * 1024 * 1024))

    @ray.remote
    class Queue:
        def __init__(self):
            self.queue = []

        def enqueue(self, key, item):
            self.queue.append((key, item))

        def read(self):
            return self.queue

    queue = Queue.remote()
    # Make sure queue actor is initialized.
    ray.get(queue.read.remote())

    yield queue

    # The code after the yield will run as teardown code.
    ray.shutdown()


def test_fork(setup_queue_actor):
    queue = setup_queue_actor

    @ray.remote
    def fork(queue, key, item):
        # ray.get here could be blocked and cause ray to start
        # a lot of python workers.
        return ray.get(queue.enqueue.remote(key, item))

    # Fork num_iters times.
    num_iters = 100
    ray.get([fork.remote(queue, i, 0) for i in range(num_iters)])
    items = ray.get(queue.read.remote())
    for i in range(num_iters):
        filtered_items = [item[1] for item in items if item[0] == i]
        assert filtered_items == list(range(1))


def test_fork_consistency(setup_queue_actor):
    queue = setup_queue_actor

    @ray.remote
    def fork(queue, key, num_items):
        x = None
        for item in range(num_items):
            x = queue.enqueue.remote(key, item)
        return ray.get(x)

    # Fork num_iters times.
    num_forks = 5
    num_items_per_fork = 100

    # Submit some tasks on new actor handles.
    forks = [fork.remote(queue, i, num_items_per_fork) for i in range(num_forks)]
    # Submit some more tasks on the original actor handle.
    for item in range(num_items_per_fork):
        local_fork = queue.enqueue.remote(num_forks, item)
    forks.append(local_fork)
    # Wait for tasks from all handles to complete.
    ray.get(forks)
    # Check that all tasks from all handles have completed.
    items = ray.get(queue.read.remote())
    for i in range(num_forks + 1):
        filtered_items = [item[1] for item in items if item[0] == i]
        assert filtered_items == list(range(num_items_per_fork))


def test_pickled_handle_consistency(setup_queue_actor):
    queue = setup_queue_actor

    @ray.remote
    def fork(pickled_queue, key, num_items):
        queue = ray._private.worker.pickle.loads(pickled_queue)
        x = None
        for item in range(num_items):
            x = queue.enqueue.remote(key, item)
        return ray.get(x)

    # Fork num_iters times.
    num_forks = 10
    num_items_per_fork = 100

    # Submit some tasks on the pickled actor handle.
    new_queue = ray._private.worker.pickle.dumps(queue)
    forks = [fork.remote(new_queue, i, num_items_per_fork) for i in range(num_forks)]
    # Submit some more tasks on the original actor handle.
    for item in range(num_items_per_fork):
        local_fork = queue.enqueue.remote(num_forks, item)
    forks.append(local_fork)
    # Wait for tasks from all handles to complete.
    ray.get(forks)
    # Check that all tasks from all handles have completed.
    items = ray.get(queue.read.remote())
    for i in range(num_forks + 1):
        filtered_items = [item[1] for item in items if item[0] == i]
        assert filtered_items == list(range(num_items_per_fork))


def test_nested_fork(setup_queue_actor):
    queue = setup_queue_actor

    @ray.remote
    def fork(queue, key, num_items):
        x = None
        for item in range(num_items):
            x = queue.enqueue.remote(key, item)
        return ray.get(x)

    @ray.remote
    def nested_fork(queue, key, num_items):
        # Pass the actor into a nested task.
        ray.get(fork.remote(queue, key + 1, num_items))
        x = None
        for item in range(num_items):
            x = queue.enqueue.remote(key, item)
        return ray.get(x)

    # Fork num_iters times.
    num_forks = 10
    num_items_per_fork = 100

    # Submit some tasks on new actor handles.
    forks = [
        nested_fork.remote(queue, i, num_items_per_fork) for i in range(0, num_forks, 2)
    ]
    ray.get(forks)
    # Check that all tasks from all handles have completed.
    items = ray.get(queue.read.remote())
    for i in range(num_forks):
        filtered_items = [item[1] for item in items if item[0] == i]
        assert filtered_items == list(range(num_items_per_fork))


def test_calling_put_on_actor_handle(ray_start_regular):
    @ray.remote
    class Counter:
        def __init__(self):
            self.x = 0

        def inc(self):
            self.x += 1
            return self.x

    @ray.remote
    def f():
        return Counter.remote()

    # Currently, calling ray.put on an actor handle is allowed, but is
    # there a good use case?
    counter = Counter.remote()
    counter_id = ray.put(counter)
    new_counter = ray.get(counter_id)
    assert ray.get(new_counter.inc.remote()) == 1
    assert ray.get(counter.inc.remote()) == 2
    assert ray.get(new_counter.inc.remote()) == 3

    ray.get(f.remote())


def test_named_but_not_detached(ray_start_regular):
    address = ray_start_regular["address"]

    driver_script = """
import ray
ray.init(address="{}")

@ray.remote
class NotDetached:
    def ping(self):
        return "pong"

actor = NotDetached.options(name="actor").remote()
assert ray.get(actor.ping.remote()) == "pong"
handle = ray.get_actor("actor")
assert ray.util.list_named_actors() == ["actor"]
assert ray.get(handle.ping.remote()) == "pong"
""".format(
        address
    )

    # Creates and kills actor once the driver exits.
    run_string_as_driver(driver_script)

    # Must raise an exception since lifetime is not detached.
    with pytest.raises(Exception):
        assert not ray.util.list_named_actors()
        detached_actor = ray.get_actor("actor")
        ray.get(detached_actor.ping.remote())

    # Check that the names are reclaimed after actors die.

    def check_name_available(name):
        try:
            ray.get_actor(name)
            return False
        except ValueError:
            return True

    @ray.remote
    class A:
        pass

    a = A.options(name="my_actor_1").remote()
    ray.kill(a, no_restart=True)
    wait_for_condition(lambda: check_name_available("my_actor_1"))

    b = A.options(name="my_actor_2").remote()
    del b
    wait_for_condition(lambda: check_name_available("my_actor_2"))


def test_detached_actor(ray_start_regular):
    @ray.remote
    class DetachedActor:
        def ping(self):
            return "pong"

    with pytest.raises(TypeError):
        DetachedActor._remote(lifetime="detached", name=1)

    with pytest.raises(ValueError, match="Actor name cannot be an empty string"):
        DetachedActor._remote(lifetime="detached", name="")

    with pytest.raises(ValueError):
        DetachedActor._remote(lifetime="detached", name="hi", namespace="")

    with pytest.raises(TypeError):
        DetachedActor._remote(lifetime="detached", name="hi", namespace=2)

    d = DetachedActor._remote(lifetime="detached", name="d_actor")
    assert ray.get(d.ping.remote()) == "pong"

    with pytest.raises(ValueError, match="Please use a different name"):
        DetachedActor._remote(lifetime="detached", name="d_actor")

    address = ray_start_regular["address"]

    get_actor_name = "d_actor"
    create_actor_name = "DetachedActor"
    driver_script = """
import ray
ray.init(address="{}", namespace="default_test_namespace")

name = "{}"
assert ray.util.list_named_actors() == [name]
existing_actor = ray.get_actor(name)
assert ray.get(existing_actor.ping.remote()) == "pong"

@ray.remote
def foo():
    return "bar"

@ray.remote
class NonDetachedActor:
    def foo(self):
        return "bar"

@ray.remote
class DetachedActor:
    def ping(self):
        return "pong"

    def foobar(self):
        actor = NonDetachedActor.remote()
        return ray.get([foo.remote(), actor.foo.remote()])

actor = DetachedActor._remote(lifetime="detached", name="{}")
ray.get(actor.ping.remote())
""".format(
        address, get_actor_name, create_actor_name
    )

    run_string_as_driver(driver_script)
    assert len(ray.util.list_named_actors()) == 2
    assert get_actor_name in ray.util.list_named_actors()
    assert create_actor_name in ray.util.list_named_actors()
    detached_actor = ray.get_actor(create_actor_name)
    assert ray.get(detached_actor.ping.remote()) == "pong"
    # Verify that a detached actor is able to create tasks/actors
    # even if the driver of the detached actor has exited.
    assert ray.get(detached_actor.foobar.remote()) == ["bar", "bar"]


@pytest.mark.parametrize(
    "ray_start_regular",
    [{"include_dashboard": True}],
    indirect=True,
)
def test_detached_actor_cleanup(ray_start_regular):
    @ray.remote
    class DetachedActor:
        def ping(self):
            return "pong"

    dup_actor_name = "actor"

    def create_and_kill_actor(actor_name):
        # Make sure same name is creatable after killing it.
        detached_actor = DetachedActor.options(
            lifetime="detached", name=actor_name
        ).remote()
        # Wait for detached actor creation.
        assert ray.get(detached_actor.ping.remote()) == "pong"
        del detached_actor
        assert ray.util.list_named_actors() == [dup_actor_name]
        detached_actor = ray.get_actor(dup_actor_name)
        ray.kill(detached_actor)
        # Wait until actor dies.
        actor_status = ray.util.state.get_actor(id=detached_actor._actor_id.hex())
        max_wait_time = 10
        wait_time = 0
        while actor_status.state != "DEAD":
            actor_status = ray.util.state.get_actor(id=detached_actor._actor_id.hex())
            print(f"actor status is {actor_status}")
            time.sleep(1.0)
            wait_time += 1
            if wait_time >= max_wait_time:
                assert None, "It took too much time to kill an actor: {}".format(
                    detached_actor._actor_id
                )

    create_and_kill_actor(dup_actor_name)

    # This shouldn't be broken because actor
    # name should have been cleaned up from GCS.
    create_and_kill_actor(dup_actor_name)

    address = ray_start_regular["address"]
    driver_script = """
import ray
import ray._private.gcs_utils as gcs_utils
import time
from ray._private.test_utils import convert_actor_state
import traceback


try:

    def _load_state_api():
        try:
            from ray.util import state as state_api
            return state_api
        except Exception:
            pass

        raise ImportError("No usable Ray State API found")

    ray.init(address="{}", namespace="default_test_namespace")

    @ray.remote
    class DetachedActor:
        def ping(self):
            return "pong"

    # Make sure same name is creatable after killing it.
    detached_actor = DetachedActor.options(lifetime="detached", name="{}").remote()
    assert ray.get(detached_actor.ping.remote()) == "pong"
    ray.kill(detached_actor)
    # Wait until actor dies.
    actor_status = _load_state_api().get_actor(id=detached_actor._actor_id.hex())
    max_wait_time = 10
    wait_time = 0
    while actor_status.state != "DEAD": # noqa
        actor_status = _load_state_api().get_actor(id=detached_actor._actor_id.hex())
        time.sleep(1.0)
        wait_time += 1
        if wait_time >= max_wait_time:
            assert None, (
                "It took too much time to kill an actor")
except Exception:
    traceback.print_exc()
    raise
""".format(
        address, dup_actor_name
    )

    run_string_as_driver(driver_script)
    # Make sure we can create a detached actor created/killed
    # at other scripts.
    create_and_kill_actor(dup_actor_name)


@pytest.mark.parametrize(
    "ray_start_cluster",
    [
        {
            "num_cpus": 3,
            "num_nodes": 1,
            "resources": {"first_node": 5},
            "include_dashboard": True,
        }
    ],
    indirect=True,
)
def test_detached_actor_cleanup_due_to_failure(ray_start_cluster):
    cluster = ray_start_cluster
    node = cluster.add_node(resources={"second_node": 1})
    cluster.wait_for_nodes()

    @ray.remote
    class DetachedActor:
        def ping(self):
            return "pong"

        def kill_itself(self):
            # kill itself.
            os._exit(0)

    worker_failure_actor_name = "worker_failure_actor_name"
    node_failure_actor_name = "node_failure_actor_name"

    def wait_until_actor_dead(handle):
        actor_status = ray.util.state.get_actor(id=handle._actor_id.hex())
        max_wait_time = 10
        wait_time = 0
        while actor_status.state != "DEAD":
            actor_status = ray.util.state.get_actor(id=handle._actor_id.hex())
            time.sleep(1.0)
            wait_time += 1
            if wait_time >= max_wait_time:
                assert None, "It took too much time to kill an actor: {}".format(
                    handle._actor_id
                )

    def create_detached_actor_blocking(actor_name, schedule_in_second_node=False):
        resources = {"second_node": 1} if schedule_in_second_node else {"first_node": 1}
        actor_handle = DetachedActor.options(
            lifetime="detached", name=actor_name, resources=resources
        ).remote()
        # Wait for detached actor creation.
        assert ray.get(actor_handle.ping.remote()) == "pong"
        return actor_handle

    # Name should be cleaned when workers fail
    deatched_actor = create_detached_actor_blocking(worker_failure_actor_name)
    deatched_actor.kill_itself.remote()
    wait_until_actor_dead(deatched_actor)
    # Name should be available now.
    deatched_actor = create_detached_actor_blocking(worker_failure_actor_name)
    assert ray.get(deatched_actor.ping.remote()) == "pong"

    # Name should be cleaned when nodes fail.
    deatched_actor = create_detached_actor_blocking(
        node_failure_actor_name, schedule_in_second_node=True
    )
    cluster.remove_node(node)
    wait_until_actor_dead(deatched_actor)
    # Name should be available now.
    deatched_actor = create_detached_actor_blocking(node_failure_actor_name)
    assert ray.get(deatched_actor.ping.remote()) == "pong"


# This test verifies actor creation task failure will not
# hang the caller.
def test_actor_creation_task_crash(ray_start_regular):
    # Test actor death in constructor.
    @ray.remote(max_restarts=0)
    class Actor:
        def __init__(self):
            print("crash")
            os._exit(0)

        def f(self):
            return "ACTOR OK"

    # Verify an exception is thrown.
    a = Actor.remote()
    with pytest.raises(ray.exceptions.RayActorError) as excinfo:
        ray.get(a.f.remote())
    assert excinfo.value.actor_id == a._actor_id.hex()

    # Test an actor can be restarted successfully
    # afte it dies in its constructor.
    @ray.remote(max_restarts=3)
    class RestartableActor:
        def __init__(self):
            count = self.get_count()
            count += 1
            # Make it die for the first 2 times.
            if count < 3:
                self.set_count(count)
                print("crash: " + str(count))
                os._exit(0)
            else:
                print("no crash")

        def f(self):
            return "ACTOR OK"

        def get_count(self):
            value = _internal_kv_get("count")
            if value is None:
                count = 0
            else:
                count = int(value)
            return count

        def set_count(self, count):
            _internal_kv_put("count", str(count), True)

    # Verify we can get the object successfully.
    ra = RestartableActor.remote()
    ray.get(ra.f.remote())


@pytest.mark.parametrize(
    "ray_start_regular", [{"num_cpus": 2, "resources": {"a": 1}}], indirect=True
)
def test_pending_actor_removed_by_owner(ray_start_regular):
    # Verify when an owner of pending actors is killed, the actor resources
    # are correctly returned.

    @ray.remote(num_cpus=1, resources={"a": 1})
    class A:
        def __init__(self):
            self.actors = []

        def create_actors(self):
            self.actors = [B.remote() for _ in range(2)]

    @ray.remote(resources={"a": 1})
    class B:
        def ping(self):
            return True

    @ray.remote(resources={"a": 1})
    def f():
        return True

    a = A.remote()
    # Create pending actors
    ray.get(a.create_actors.remote())

    # Owner is dead. pending actors should be killed
    # and raylet should return workers correctly.
    del a
    a = B.remote()
    assert ray.get(a.ping.remote())
    ray.kill(a)
    assert ray.get(f.remote())


def test_pickling_actor_handle(ray_start_regular_shared):
    @ray.remote
    class Foo:
        def method(self):
            pass

    f = Foo.remote()
    new_f = ray._private.worker.pickle.loads(ray._private.worker.pickle.dumps(f))
    # Verify that we can call a method on the unpickled handle. TODO(rkn):
    # we should also test this from a different driver.
    ray.get(new_f.method.remote())


def test_pickled_actor_handle_call_in_method_twice(ray_start_regular_shared):
    @ray.remote
    class Actor1:
        def f(self):
            return 1

    @ray.remote
    class Actor2:
        def __init__(self, constructor):
            self.actor = constructor()

        def step(self):
            ray.get(self.actor.f.remote())

    a = Actor1.remote()

    b = Actor2.remote(lambda: a)

    ray.get(b.step.remote())
    ray.get(b.step.remote())


def test_kill(ray_start_regular_shared):
    @ray.remote
    class Actor:
        def hang(self):
            while True:
                time.sleep(1)

    actor = Actor.remote()
    result = actor.hang.remote()
    ready, _ = ray.wait([result], timeout=0.5)
    assert len(ready) == 0
    kill_actor_and_wait_for_failure(actor)

    with pytest.raises(ray.exceptions.RayActorError):
        ray.get(result)

    with pytest.raises(ValueError):
        ray.kill("not_an_actor_handle")


def test_get_actor_no_input(ray_start_regular_shared):
    for bad_name in [None, "", "    "]:
        with pytest.raises(ValueError):
            ray.get_actor(bad_name)


def test_actor_resource_demand(shutdown_only):
    ray.shutdown()
    cluster = ray.init(num_cpus=3)
    global_state_accessor = make_global_state_accessor(cluster)

    @ray.remote(num_cpus=2)
    class Actor:
        def foo(self):
            return "ok"

    a = Actor.remote()
    ray.get(a.foo.remote())
    time.sleep(1)

    message = global_state_accessor.get_all_resource_usage()
    resource_usages = gcs_utils.ResourceUsageBatchData.FromString(message)

    # The actor is scheduled so there should be no more demands left.
    assert len(resource_usages.resource_load_by_shape.resource_demands) == 0

    @ray.remote(num_cpus=80)
    class Actor2:
        pass

    actors = []
    actors.append(Actor2.remote())
    time.sleep(1)

    # This actor cannot be scheduled.
    message = global_state_accessor.get_all_resource_usage()
    resource_usages = gcs_utils.ResourceUsageBatchData.FromString(message)
    assert len(resource_usages.resource_load_by_shape.resource_demands) == 1
    assert resource_usages.resource_load_by_shape.resource_demands[0].shape == {
        "CPU": 80.0
    }
    assert (
        resource_usages.resource_load_by_shape.resource_demands[
            0
        ].num_infeasible_requests_queued
        == 1
    )

    actors.append(Actor2.remote())
    time.sleep(1)

    # Two actors cannot be scheduled.
    message = global_state_accessor.get_all_resource_usage()
    resource_usages = gcs_utils.ResourceUsageBatchData.FromString(message)
    assert len(resource_usages.resource_load_by_shape.resource_demands) == 1
    assert (
        resource_usages.resource_load_by_shape.resource_demands[
            0
        ].num_infeasible_requests_queued
        == 2
    )


def test_kill_pending_actor_with_no_restart_true():
    cluster = ray.init()
    global_state_accessor = make_global_state_accessor(cluster)

    @ray.remote(resources={"WORKER": 1.0})
    class PendingActor:
        pass

    # Kill actor with `no_restart=True`.
    actor = PendingActor.remote()
    # TODO(ffbin): The raylet doesn't guarantee the order when dealing with
    # RequestWorkerLease and CancelWorkerLease. If we kill the actor
    # immediately after creating the actor, we may not be able to clean up
    # the request cached by the raylet.
    # See https://github.com/ray-project/ray/issues/13545 for details.
    time.sleep(1)
    ray.kill(actor, no_restart=True)

    def condition1():
        message = global_state_accessor.get_all_resource_usage()
        resource_usages = gcs_utils.ResourceUsageBatchData.FromString(message)
        if len(resource_usages.resource_load_by_shape.resource_demands) == 0:
            return True
        return False

    # Actor is dead, so the infeasible task queue length is 0.
    wait_for_condition(condition1, timeout=10)

    ray.shutdown()


def test_actor_timestamps(ray_start_regular):
    @ray.remote
    class Foo:
        def get_id(self):
            return ray.get_runtime_context().get_actor_id()

        def kill_self(self):
            sys.exit(1)

    def graceful_exit():
        actor = Foo.remote()
        actor_id = ray.get(actor.get_id.remote())

        state_after_starting = ray._private.state.actors()[actor_id]
        time.sleep(1)
        del actor
        time.sleep(1)
        state_after_ending = ray._private.state.actors()[actor_id]

        assert state_after_starting["StartTime"] == state_after_ending["StartTime"]
        start_time = state_after_ending["StartTime"]
        end_time = state_after_ending["EndTime"]
        assert end_time > start_time > 0, f"Start: {start_time}, End: {end_time}"

    def not_graceful_exit():
        actor = Foo.remote()
        actor_id = ray.get(actor.get_id.remote())

        state_after_starting = ray._private.state.actors()[actor_id]
        time.sleep(1)
        actor.kill_self.remote()
        time.sleep(1)
        state_after_ending = ray._private.state.actors()[actor_id]

        assert state_after_starting["StartTime"] == state_after_ending["StartTime"]

        start_time = state_after_ending["StartTime"]
        end_time = state_after_ending["EndTime"]
        assert end_time > start_time > 0, f"Start: {start_time}, End: {end_time}"

    def restarted():
        actor = Foo.options(max_restarts=1, max_task_retries=-1).remote()
        actor_id = ray.get(actor.get_id.remote())

        state_after_starting = ray._private.state.actors()[actor_id]
        time.sleep(1)
        actor.kill_self.remote()
        time.sleep(1)
        actor.kill_self.remote()
        time.sleep(1)
        state_after_ending = ray._private.state.actors()[actor_id]

        assert state_after_starting["StartTime"] == state_after_ending["StartTime"]

        start_time = state_after_ending["StartTime"]
        end_time = state_after_ending["EndTime"]
        assert end_time > start_time > 0, f"Start: {start_time}, End: {end_time}"

    graceful_exit()
    not_graceful_exit()
    restarted()


def test_kill_pending_actor_with_no_restart_false():
    cluster = ray.init()
    global_state_accessor = make_global_state_accessor(cluster)

    @ray.remote(resources={"WORKER": 1.0}, max_restarts=1)
    class PendingActor:
        pass

    # Kill actor with `no_restart=False`.
    actor = PendingActor.remote()
    # TODO(ffbin): The raylet doesn't guarantee the order when dealing with
    # RequestWorkerLease and CancelWorkerLease. If we kill the actor
    # immediately after creating the actor, we may not be able to clean up
    # the request cached by the raylet.
    # See https://github.com/ray-project/ray/issues/13545 for details.
    time.sleep(1)
    ray.kill(actor, no_restart=False)

    def condition1():
        message = global_state_accessor.get_all_resource_usage()
        resource_usages = gcs_utils.ResourceUsageBatchData.FromString(message)
        if len(resource_usages.resource_load_by_shape.resource_demands) == 0:
            return False
        return True

    # Actor restarts, so the infeasible task queue length is 1.
    wait_for_condition(condition1, timeout=10)

    # Kill actor again and actor is dead,
    # so the infeasible task queue length is 0.
    ray.kill(actor, no_restart=False)

    def condition2():
        message = global_state_accessor.get_all_resource_usage()
        resource_usages = gcs_utils.ResourceUsageBatchData.FromString(message)
        if len(resource_usages.resource_load_by_shape.resource_demands) == 0:
            return True
        return False

    wait_for_condition(condition2, timeout=10)

    ray.shutdown()


def test_actor_namespace_access(ray_start_regular):
    @ray.remote
    class A:
        def hi(self):
            return "hi"

    A.options(name="actor_in_current_namespace", lifetime="detached").remote()
    A.options(name="actor_name", namespace="namespace", lifetime="detached").remote()
    ray.get_actor("actor_in_current_namespace")  # => works
    ray.get_actor("actor_name", namespace="namespace")  # => works
    match_str = r"Failed to look up actor with name.*"
    with pytest.raises(ValueError, match=match_str):
        ray.get_actor("actor_name")  # => errors


def test_get_actor_after_killed(shutdown_only):
    ray.init(num_cpus=2, include_dashboard=True)

    @ray.remote
    class A:
        def ready(self):
            return True

    actor = A.options(name="actor", namespace="namespace").remote()
    ray.kill(actor)
    with pytest.raises(ValueError):
        ray.get_actor("actor", namespace="namespace")

    actor = A.options(
        name="actor_2",
        namespace="namespace",
        max_restarts=1,
        max_task_retries=-1,
    ).remote()
    ray.kill(actor, no_restart=False)
    assert ray.get(ray.get_actor("actor_2", namespace="namespace").ready.remote())


def test_get_actor_from_concurrent_tasks(shutdown_only):
    @ray.remote
    class Actor:
        def get_actor_id(self) -> str:
            return ray.get_runtime_context().get_actor_id()

    actor_name = "test_actor"

    @ray.remote(num_cpus=0)
    def get_or_create_actor():
        try:
            # The first task will try to get the actor but fail (doesn't exist).
            try:
                actor = ray.get_actor(actor_name)
            except Exception:
                print("Get failed, trying to create")
                # Actor must be detached so it outlives this task and other tasks can
                # get a handle to it.
                actor = Actor.options(name=actor_name, lifetime="detached").remote()
        except Exception:
            # Multiple tasks may have reached the creation block above.
            # Only one will succeed and the others will get an error, in which case
            # they fall here and should be able to get the actor handle.
            print("Someone else created it, trying to get")
            actor = ray.get_actor(actor_name)

        return ray.get(actor.get_actor_id.remote())

    # Run 10 concurrent tasks to get or create the same actor.
    # Only one task should succeed at creating it, and all the others should get it.
    assert len(set(ray.get([get_or_create_actor.remote() for _ in range(10)]))) == 1


def test_get_or_create_actor_from_multiple_threads(shutdown_only):
    """Make sure we can create actors in multiple threads without
    race conditions.

    Check https://github.com/ray-project/ray/issues/41324
    """

    @ray.remote
    class Counter:
        def __init__(self):
            self._count = 0

        def inc(self):
            self._count += 1

        def get(self) -> int:
            return self._count

    counter = Counter.remote()

    @ray.remote
    class Actor:
        def __init__(self):
            ray.get(counter.inc.remote())

        def get_actor_id(self) -> str:
            return ray.get_runtime_context().get_actor_id()

    def _create_or_get_actor(*args):
        a = Actor.options(
            name="test_actor",
            get_if_exists=True,
            # Actor must be detached so it outlives this function and other threads
            # can get a handle to it.
            lifetime="detached",
        ).remote()

        return ray.get(a.get_actor_id.remote())

    # Concurrently submit 100 calls to create or get the actor from 10 threads.
    # Ensure that exactly one call actually creates the actor and the other 99 get it.
    with ThreadPoolExecutor(max_workers=10) as tp:
        assert len(set(tp.map(_create_or_get_actor, range(100)))) == 1
        assert ray.get(counter.get.remote()) == 1


def test_get_actor_in_remote_workers(ray_start_cluster):
    """Make sure we can get and create actors without
    race condition in a remote worker.

    Check https://github.com/ray-project/ray/issues/20092. # noqa
    """
    cluster = ray_start_cluster
    cluster.add_node(num_cpus=0)
    cluster.add_node(num_cpus=1)
    ray.init(address=cluster.address, namespace="xxx")

    @ray.remote(num_cpus=0)
    class RemoteProc:
        def __init__(self):
            pass

        def procTask(self, a, b):
            print("[%s]-> %s" % (a, b))
            return a, b

    @ray.remote
    def submit_named_actors():
        RemoteProc.options(
            name="test", lifetime="detached", max_concurrency=10, namespace="xxx"
        ).remote()
        proc = ray.get_actor("test", namespace="xxx")
        ray.get(proc.procTask.remote(1, 2))
        # Should be able to create an actor with the same name
        # immediately after killing it.
        ray.kill(proc)
        RemoteProc.options(
            name="test", lifetime="detached", max_concurrency=10, namespace="xxx"
        ).remote()
        proc = ray.get_actor("test", namespace="xxx")
        return ray.get(proc.procTask.remote(1, 2))

    assert (1, 2) == ray.get(submit_named_actors.remote())


def test_resource_leak_when_cancel_actor_in_phase_of_creating(ray_start_cluster):
    """Make sure there is no resource leak when cancel an actor in phase of
    creating.

    Check https://github.com/ray-project/ray/issues/27743. # noqa
    """
    cluster = ray_start_cluster
    cluster.add_node(num_cpus=2)
    ray.init(address=cluster.address)
    cluster.wait_for_nodes()

    @ray.remote(num_cpus=1)
    class Actor:
        def __init__(self, signal_1, signal_2):
            signal_1.send.remote()
            ray.get(signal_2.wait.remote())
            pass

    signal_1 = SignalActor.remote()
    signal_2 = SignalActor.remote()
    actor = Actor.remote(signal_1, signal_2)

    wait_for_condition(lambda: ray.available_resources()["CPU"] != 2)

    # Checking that the constructor of `Actor`` is invoked.
    ready_ids, _ = ray.wait([signal_1.wait.remote()], timeout=3.0)
    assert len(ready_ids) == 1

    # Kill the actor which is in the phase of creating.
    ray.kill(actor)

    # Ensure there is no resource leak.
    wait_for_condition(lambda: ray.available_resources()["CPU"] == 2)


def test_actor_gc(monkeypatch, shutdown_only):
    MAX_DEAD_ACTOR_CNT = 5
    with monkeypatch.context() as m:
        m.setenv("RAY_maximum_gcs_destroyed_actor_cached_count", MAX_DEAD_ACTOR_CNT)
        ray.init()

        @ray.remote
        class Actor:
            def ready(self):
                pass

        actors = [Actor.remote() for _ in range(10)]
        ray.get([actor.ready.remote() for actor in actors])
        alive_actors = 0
        for a in list_actors():
            if a["state"] == "ALIVE":
                alive_actors += 1
        assert alive_actors == 10
        # Kill actors
        del actors

        def verify_cached_dead_actor_cleaned():
            return len(list_actors()) == MAX_DEAD_ACTOR_CNT  # noqa

        wait_for_condition(verify_cached_dead_actor_cleaned)

        # Test detached actors
        actors = [Actor.options(lifetime="detached").remote() for _ in range(10)]
        ray.get([actor.ready.remote() for actor in actors])
        alive_actors = 0
        for a in list_actors():
            if a["state"] == "ALIVE":
                alive_actors += 1
        assert alive_actors == 10
        # Kill actors
        for actor in actors:
            ray.kill(actor)

        wait_for_condition(verify_cached_dead_actor_cleaned)

        # Test actors created by a driver.

        driver = """
import ray
from ray.util.state import list_actors
ray.init("auto")

@ray.remote
class A:
    def ready(self):
        pass

actors = [A.remote() for _ in range(10)]
ray.get([actor.ready.remote() for actor in actors])
alive_actors = 0
for a in list_actors():
    if a.state == "ALIVE":
        alive_actors += 1
assert alive_actors == 10
"""

        run_string_as_driver(driver)
        # Driver exits, so dead actors must be cleaned.
        wait_for_condition(verify_cached_dead_actor_cleaned)
        print(list_actors())


if __name__ == "__main__":
    sys.exit(pytest.main(["-sv", __file__]))