chore: import upstream snapshot with attribution

python/ray/tune/tests/execution/test_actor_caching.py

@@ -0,0 +1,131 @@
+import sys
+
+import pytest
+
+import ray
+from ray.tune import PlacementGroupFactory
+from ray.tune.tests.execution.utils import TestingTrial, create_execution_test_objects
+
+
+@pytest.fixture
+def ray_start_2_cpus():
+    address_info = ray.init(num_cpus=2)
+    yield address_info
+    ray.shutdown()
+
+
+def test_actor_cached(tmpdir, ray_start_2_cpus):
+    tune_controller, actor_manger, resource_manager = create_execution_test_objects(
+        max_pending_trials=8
+    )
+
+    assert not actor_manger.added_actors
+
+    tune_controller.add_trial(TestingTrial("trainable1", stub=True, trial_id="trial1"))
+    tune_controller.step()
+
+    tracked_actor, cls_name, kwargs = actor_manger.added_actors[0]
+    assert cls_name == "trainable1"
+
+
+def test_actor_reuse_unstaged(tmpdir, ray_start_2_cpus):
+    """A trial that hasn't been staged can re-use an actor.
+
+    In specific circumstances, this can lead to errors. Notably, when an
+    external source (e.g. a scheduler) directly calls TuneController APIs,
+    we can be in a situation where a trial has not been staged, but there is
+    still an actor available for it to use (because it hasn't been evicted from
+    the cache, yet).
+
+    This test constructs such a situation an asserts that actor re-use does not
+    lead to errors in those cases.
+    """
+    tune_controller, actor_manger, resource_manager = create_execution_test_objects(
+        max_pending_trials=1
+    )
+    tune_controller._reuse_actors = True
+
+    assert not actor_manger.added_actors
+
+    trialA1 = TestingTrial(
+        "trainable1",
+        stub=True,
+        trial_id="trialA1",
+        placement_group_factory=PlacementGroupFactory([{"CPU": 1}]),
+    )
+    tune_controller.add_trial(trialA1)
+    trialB1 = TestingTrial(
+        "trainable1",
+        stub=True,
+        trial_id="trialB1",
+        placement_group_factory=PlacementGroupFactory([{"CPU": 5}]),
+    )
+    tune_controller.add_trial(trialB1)
+    trialA2 = TestingTrial(
+        "trainable1",
+        stub=True,
+        trial_id="trialA2",
+        placement_group_factory=PlacementGroupFactory([{"CPU": 1}]),
+    )
+    tune_controller.add_trial(trialA2)
+    tune_controller.step()
+
+    # Prevent trial A3 from being staged by setting the number
+    # of pending actors to the maximum allowed
+    actor_manger.set_num_pending(2)
+
+    trialA3 = TestingTrial(
+        "trainable1",
+        stub=True,
+        trial_id="trialA3",
+        placement_group_factory=PlacementGroupFactory([{"CPU": 1}]),
+    )
+    tune_controller.add_trial(trialA3)
+    tune_controller.step()
+
+    tracked_actorA1, _, _ = actor_manger.added_actors[0]
+    tracked_actorB1, _, _ = actor_manger.added_actors[1]
+    tracked_actorA2, _, _ = actor_manger.added_actors[2]
+
+    # Start trial A1, report that it's done training.
+    # This will cache the actor for A1 as A2 is already scheduled.
+    tune_controller._actor_started(tracked_actorA1)
+    tune_controller._on_training_result(trialA1, {"done": True})
+
+    # Trial A2 should be in the staged trials. A3 should still not be staged.
+    assert trialA2 in tune_controller._staged_trials
+    assert trialA3 not in tune_controller._staged_trials
+
+    # The actor of A1 should be cached for re-use now.
+    assert tune_controller._actor_cache.num_cached_objects == 1
+
+    # In the meantime, actor A2 started. This will unstage it.
+    tune_controller._actor_started(tracked_actorA2)
+
+    # Now, an external source (e.g. the BOHB scheduler) wants to prematurely
+    # stop trial A2. This will leave the cached actor intact, but trial A3
+    # is still not scheduled.
+    tune_controller._schedule_trial_stop(trialA2)
+    assert tune_controller._actor_cache.num_cached_objects == 1
+
+    # Process events. This will invoke "path 3" in TuneController._maybe_add_actors
+    # and re-use the cached actor
+    tune_controller.step()
+
+    # Reset future scheduled
+    assert actor_manger.scheduled_futures[-1][2] == "reset"
+
+    # Prior to https://github.com/ray-project/ray/pull/36951, there was a bug here:
+    # Because trial A3 was never staged, the unstage ran into an error.
+    # This fails without the line: self._staged_trials.add(start_trial)
+    tune_controller._on_trial_reset(trialA3, True)
+
+    # When the actor finally stops, the cache size is adjusted and the actor is
+    # evicted. This test failed without the line:
+    # self._actor_cache.increase_max(start_trial.placement_group_factory)
+    tune_controller._actor_stopped(tracked_actorA1)
+    tune_controller.step()
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))