ray-project--ray/python/ray/train/v2/_internal/execution/controller/state.py

from enum import Enum

from ray.train.v2._internal.execution.scaling_policy.scaling_policy import (
    ScalingDecision,
)
from ray.train.v2.api.exceptions import TrainingFailedError


class TrainControllerStateType(Enum):
    """Enum representing different states of the train controller.

    States:
       INITIALIZING: The train controller is starting up. This is always the initial
           state of the controller.
       SCHEDULING: The train controller is in the process of scheduling a new worker
           group.
       RESCHEDULING: The train controller is in the process of rescheduling the worker
           group.
       RUNNING: The train controller is actively running training tasks.
       RESTARTING: The train controller is in the process of recovering from an error.
       RESIZING: The train controller is in the process of resizing a running worker
           group.
       SHUTTING_DOWN: The train controller has already shut down the worker group and
           and is in the process of shutting itself down.
       ERRORED: A terminal state indicating that training has encountered an error and
           cannot continue.
       FINISHED: A terminal state indicating that training has completed.
       ABORTED: A terminal state indicating that training has been aborted.

    Args:
        state_name: The name of the state.
        is_terminal: Whether this is a terminal state that should not be further processed.
        needs_new_run_attempt: Whether this state requires starting a new run attempt, where
            a run attempt is a logical unit that encompasses both scheduling workers and
            executing training on those workers.
    """

    INITIALIZING = ("INITIALIZING", False, True)
    SCHEDULING = ("SCHEDULING", False, False)
    RESCHEDULING = ("RESCHEDULING", False, False)
    RUNNING = ("RUNNING", False, False)
    RESTARTING = ("RESTARTING", False, True)
    RESIZING = ("RESIZING", False, True)
    SHUTTING_DOWN = ("SHUTTING_DOWN", False, False)
    ERRORED = ("ERRORED", True, False)
    FINISHED = ("FINISHED", True, False)
    ABORTED = ("ABORTED", True, False)

    def __init__(
        self,
        state_name: str,
        is_terminal: bool,
        needs_new_run_attempt: bool,
    ):
        self.state_name = state_name
        self.is_terminal = is_terminal
        self.needs_new_run_attempt = needs_new_run_attempt


class TrainControllerState:
    """Base class for all train controller states.

    Methods:
        get_type() -> TrainControllerStateType: Returns the type of the state.
        is_terminal() -> bool: Returns whether the state is terminal.
        needs_new_run_attempt() -> bool: Returns whether a new run attempt is needed.
    """

    def __init__(self, state_type: TrainControllerStateType):
        self._state_type = state_type

    def __repr__(self) -> str:
        attrs = {
            "type": self._state_type.name,
            "is_terminal": self._state_type.is_terminal,
            "needs_new_run_attempt": self._state_type.needs_new_run_attempt,
            **{k: v for k, v in vars(self).items() if not k.startswith("_")},
        }
        attrs_str = "\n    ".join(f"{k}={v}" for k, v in attrs.items())
        return f"{self.__class__.__name__}(\n    {attrs_str}\n)"

    def is_terminal(self) -> bool:
        return self._state_type.is_terminal

    def needs_new_run_attempt(self) -> bool:
        return self._state_type.needs_new_run_attempt


class InitializingState(TrainControllerState):
    def __init__(self):
        super().__init__(state_type=TrainControllerStateType.INITIALIZING)


class SchedulingState(TrainControllerState):
    def __init__(self, scaling_decision: ScalingDecision):
        super().__init__(state_type=TrainControllerStateType.SCHEDULING)
        self.scaling_decision = scaling_decision


class ReschedulingState(TrainControllerState):
    def __init__(
        self,
        training_failed_error: TrainingFailedError,
    ):
        super().__init__(state_type=TrainControllerStateType.RESCHEDULING)
        self.training_failed_error = training_failed_error


class RunningState(TrainControllerState):
    # TODO: Split into multiple more granular states, or add more fields.
    # For example, we may want to indicate if any health checks failed.
    def __init__(self):
        super().__init__(state_type=TrainControllerStateType.RUNNING)


class RestartingState(TrainControllerState):
    def __init__(
        self,
        training_failed_error: TrainingFailedError,
    ):
        super().__init__(state_type=TrainControllerStateType.RESTARTING)
        self.training_failed_error = training_failed_error


class ResizingState(TrainControllerState):
    def __init__(
        self,
        scaling_decision: ScalingDecision,
    ):
        super().__init__(state_type=TrainControllerStateType.RESIZING)
        self.scaling_decision = scaling_decision


class ShuttingDownState(TrainControllerState):
    def __init__(self, next_state: "TrainControllerState"):
        super().__init__(state_type=TrainControllerStateType.SHUTTING_DOWN)
        self.next_state = next_state


class ErroredState(TrainControllerState):
    def __init__(
        self,
        training_failed_error: TrainingFailedError,
    ):
        super().__init__(state_type=TrainControllerStateType.ERRORED)
        self.training_failed_error = training_failed_error


class FinishedState(TrainControllerState):
    def __init__(self):
        super().__init__(state_type=TrainControllerStateType.FINISHED)


class AbortedState(TrainControllerState):
    def __init__(self):
        super().__init__(state_type=TrainControllerStateType.ABORTED)