wehub-resource-sync
354 lines
13 KiB
Python
354 lines
13 KiB
Python
import logging
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import Any, Dict
|
|
|
|
import torch
|
|
from packaging.version import Version
|
|
|
|
import ray
|
|
import ray.train
|
|
from ray._common.usage.usage_lib import TagKey, record_extra_usage_tag
|
|
from ray.train import Checkpoint
|
|
from ray.train.v2._internal.constants import is_v2_enabled
|
|
from ray.util import PublicAPI
|
|
|
|
|
|
def import_lightning(): # noqa: F402
|
|
try:
|
|
import lightning.pytorch as pl
|
|
except ModuleNotFoundError:
|
|
import pytorch_lightning as pl
|
|
return pl
|
|
|
|
|
|
pl = import_lightning()
|
|
|
|
if is_v2_enabled():
|
|
from ray.train.v2.api.report_config import CheckpointUploadMode
|
|
|
|
_LIGHTNING_GREATER_EQUAL_2_0 = Version(pl.__version__) >= Version("2.0.0")
|
|
_LIGHTNING_LESS_THAN_2_1 = Version(pl.__version__) < Version("2.1.0")
|
|
_TORCH_GREATER_EQUAL_1_12 = Version(torch.__version__) >= Version("1.12.0")
|
|
_TORCH_FSDP_AVAILABLE = _TORCH_GREATER_EQUAL_1_12 and torch.distributed.is_available()
|
|
|
|
try:
|
|
from lightning.pytorch.plugins.environments import LightningEnvironment
|
|
except ModuleNotFoundError:
|
|
from pytorch_lightning.plugins.environments import LightningEnvironment
|
|
|
|
if _LIGHTNING_GREATER_EQUAL_2_0:
|
|
FSDPStrategy = pl.strategies.FSDPStrategy
|
|
else:
|
|
FSDPStrategy = pl.strategies.DDPFullyShardedStrategy
|
|
|
|
if _TORCH_FSDP_AVAILABLE:
|
|
from torch.distributed.fsdp import (
|
|
FullStateDictConfig,
|
|
FullyShardedDataParallel,
|
|
StateDictType,
|
|
)
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
LIGHTNING_REPORT_STAGE_KEY = "_report_on"
|
|
|
|
|
|
@PublicAPI(stability="beta")
|
|
class RayDDPStrategy(pl.strategies.DDPStrategy):
|
|
"""Subclass of DDPStrategy to ensure compatibility with Ray orchestration.
|
|
|
|
For a full list of initialization arguments, please refer to:
|
|
https://lightning.ai/docs/pytorch/stable/api/lightning.pytorch.strategies.DDPStrategy.html
|
|
|
|
Note that `process_group_backend`, `timeout`, and `start_method` are disabled here,
|
|
please specify these arguments in :class:`~ray.train.torch.TorchConfig` instead.
|
|
"""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
record_extra_usage_tag(TagKey.TRAIN_LIGHTNING_RAYDDPSTRATEGY, "1")
|
|
|
|
@property
|
|
def root_device(self) -> torch.device:
|
|
return ray.train.torch.get_device()
|
|
|
|
@property
|
|
def distributed_sampler_kwargs(self) -> Dict[str, Any]:
|
|
return dict(
|
|
num_replicas=self.world_size,
|
|
rank=self.global_rank,
|
|
)
|
|
|
|
|
|
@PublicAPI(stability="beta")
|
|
class RayFSDPStrategy(FSDPStrategy): # noqa: F821
|
|
"""Subclass of FSDPStrategy to ensure compatibility with Ray orchestration.
|
|
|
|
For a full list of initialization arguments, please refer to:
|
|
https://lightning.ai/docs/pytorch/stable/api/lightning.pytorch.strategies.FSDPStrategy.html
|
|
|
|
.. note::
|
|
It is recommended to upgrade `lightning>=2.1` or above when using FSDP
|
|
with Lightning, since Lightning starts to natively support `state_dict_type`,
|
|
`sharding_strategy`, `auto_wrap_policy` and other FSDP configurations from 2.1.
|
|
"""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
record_extra_usage_tag(TagKey.TRAIN_LIGHTNING_RAYFSDPSTRATEGY, "1")
|
|
|
|
@property
|
|
def root_device(self) -> torch.device:
|
|
return ray.train.torch.get_device()
|
|
|
|
@property
|
|
def distributed_sampler_kwargs(self) -> Dict[str, Any]:
|
|
return dict(
|
|
num_replicas=self.world_size,
|
|
rank=self.global_rank,
|
|
)
|
|
|
|
def lightning_module_state_dict(self) -> Dict[str, Any]:
|
|
"""Gathers the full state dict to rank 0 on CPU.
|
|
|
|
FSDP checkpointing is broken in Lightning 2.0.x. This subclass patches the
|
|
behavior to perform a full state dict checkpointing, gathering the checkpoint
|
|
shards on rank 0 CPU. Upgrade to `lightning>=2.1` to do sharded state dict
|
|
checkpointing.
|
|
|
|
See the note in the class docstring for more details.
|
|
"""
|
|
|
|
assert self.model is not None, "Failed to get the state dict for a None model!"
|
|
|
|
if (
|
|
_TORCH_FSDP_AVAILABLE
|
|
and _LIGHTNING_GREATER_EQUAL_2_0
|
|
and _LIGHTNING_LESS_THAN_2_1
|
|
):
|
|
with FullyShardedDataParallel.state_dict_type(
|
|
module=self.model,
|
|
state_dict_type=StateDictType.FULL_STATE_DICT,
|
|
state_dict_config=FullStateDictConfig(
|
|
offload_to_cpu=True, rank0_only=True
|
|
),
|
|
):
|
|
state_dict = self.model.state_dict()
|
|
|
|
ckpt_state_dict = {}
|
|
prefix_len = len("_forward_module.")
|
|
for k, v in state_dict.items():
|
|
if k.startswith("_forward_module."):
|
|
non_prefixed_key = k[prefix_len:]
|
|
ckpt_state_dict[non_prefixed_key] = v
|
|
else:
|
|
ckpt_state_dict[k] = v
|
|
return ckpt_state_dict
|
|
else:
|
|
# Otherwise Lightning uses Fairscale FSDP, no need to unshard by ourself.
|
|
return super().lightning_module_state_dict()
|
|
|
|
|
|
@PublicAPI(stability="beta")
|
|
class RayDeepSpeedStrategy(pl.strategies.DeepSpeedStrategy):
|
|
"""Subclass of DeepSpeedStrategy to ensure compatibility with Ray orchestration.
|
|
|
|
For a full list of initialization arguments, please refer to:
|
|
https://lightning.ai/docs/pytorch/stable/api/lightning.pytorch.strategies.DeepSpeedStrategy.html
|
|
"""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
record_extra_usage_tag(TagKey.TRAIN_LIGHTNING_RAYDEEPSPEEDSTRATEGY, "1")
|
|
|
|
@property
|
|
def root_device(self) -> torch.device:
|
|
return ray.train.torch.get_device()
|
|
|
|
@property
|
|
def distributed_sampler_kwargs(self) -> Dict[str, Any]:
|
|
return dict(
|
|
num_replicas=self.world_size,
|
|
rank=self.global_rank,
|
|
)
|
|
|
|
|
|
@PublicAPI(stability="beta")
|
|
class RayLightningEnvironment(LightningEnvironment): # noqa: F821
|
|
"""Setup Lightning DDP training environment for Ray cluster."""
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
record_extra_usage_tag(TagKey.TRAIN_LIGHTNING_RAYLIGHTNINGENVIRONMENT, "1")
|
|
|
|
def world_size(self) -> int:
|
|
return ray.train.get_context().get_world_size()
|
|
|
|
def global_rank(self) -> int:
|
|
return ray.train.get_context().get_world_rank()
|
|
|
|
def local_rank(self) -> int:
|
|
return ray.train.get_context().get_local_rank()
|
|
|
|
def node_rank(self) -> int:
|
|
return ray.train.get_context().get_node_rank()
|
|
|
|
def set_world_size(self, size: int) -> None:
|
|
# Disable it since `world_size()` directly returns data from Train context.
|
|
pass
|
|
|
|
def set_global_rank(self, rank: int) -> None:
|
|
# Disable it since `global_rank()` directly returns data from Train.
|
|
pass
|
|
|
|
def teardown(self):
|
|
pass
|
|
|
|
|
|
@PublicAPI(stability="beta")
|
|
def prepare_trainer(trainer: pl.Trainer) -> pl.Trainer:
|
|
"""Prepare the PyTorch Lightning Trainer for distributed execution."""
|
|
|
|
# Check strategy class
|
|
valid_strategy_class = [RayDDPStrategy, RayFSDPStrategy, RayDeepSpeedStrategy]
|
|
|
|
if not any(isinstance(trainer.strategy, cls) for cls in valid_strategy_class):
|
|
raise RuntimeError(
|
|
f"Invalid strategy class: {type(trainer.strategy)}. To use "
|
|
"PyTorch Lightning with Ray, the strategy object should be one of "
|
|
f"{[cls.__name__ for cls in valid_strategy_class]} class "
|
|
"or its subclass."
|
|
)
|
|
|
|
# Check cluster environment
|
|
cluster_environment = getattr(trainer.strategy, "cluster_environment", None)
|
|
if cluster_environment and not isinstance(
|
|
cluster_environment, RayLightningEnvironment
|
|
):
|
|
raise RuntimeError(
|
|
"Invalid cluster environment plugin. The expected class is"
|
|
"`ray.train.lightning.RayLightningEnvironment` "
|
|
f"but got {type(cluster_environment)}!"
|
|
)
|
|
|
|
record_extra_usage_tag(TagKey.TRAIN_LIGHTNING_PREPARE_TRAINER, "1")
|
|
return trainer
|
|
|
|
|
|
@PublicAPI(stability="beta")
|
|
class RayTrainReportCallback(pl.callbacks.Callback):
|
|
"""A simple callback that reports checkpoints to Ray on train epoch end.
|
|
|
|
This callback is a subclass of `lightning.pytorch.callbacks.Callback
|
|
<https://lightning.ai/docs/pytorch/stable/api/lightning.pytorch.callbacks.Callback.html#lightning.pytorch.callbacks.Callback>`_.
|
|
|
|
It fetches the latest `trainer.callback_metrics` and reports together with
|
|
the checkpoint on each training epoch end.
|
|
|
|
Checkpoints will be saved in the following structure::
|
|
|
|
checkpoint_00000*/ Ray Train Checkpoint
|
|
└─ checkpoint.ckpt PyTorch Lightning Checkpoint
|
|
|
|
You can also provide the following arguments to the callback:
|
|
|
|
- checkpoint_upload_mode: The manner in which to upload the checkpoint.
|
|
See :ref:`Checkpoint upload modes <train-checkpoint-upload-modes>` for more details.
|
|
- validation: Whether to asynchronously validate the checkpoint.
|
|
See :ref:`Validating checkpoints asynchronously <train-validating-checkpoints>` for more details.
|
|
|
|
For customized reporting and checkpointing logic, implement your own
|
|
`lightning.pytorch.callbacks.Callback` following this user
|
|
guide: :ref:`Saving and Loading Checkpoints <train-dl-saving-checkpoints>`.
|
|
"""
|
|
|
|
CHECKPOINT_NAME = "checkpoint.ckpt"
|
|
|
|
def __init__(
|
|
self,
|
|
# v2-only arguments
|
|
checkpoint_upload_mode=None,
|
|
validation=False,
|
|
) -> None:
|
|
super().__init__()
|
|
self.checkpoint_upload_mode = checkpoint_upload_mode
|
|
self.validation = validation
|
|
|
|
if is_v2_enabled():
|
|
if checkpoint_upload_mode is None:
|
|
self.checkpoint_upload_mode = CheckpointUploadMode.SYNC
|
|
else:
|
|
if checkpoint_upload_mode is not None:
|
|
raise ValueError(
|
|
"`checkpoint_upload_mode` is only supported in Ray Train v2. "
|
|
"To enable it, please set `RAY_TRAIN_V2_ENABLED=1`."
|
|
)
|
|
if validation:
|
|
raise ValueError(
|
|
"`validation` is only supported in Ray Train v2. "
|
|
"To enable it, please set `RAY_TRAIN_V2_ENABLED=1`."
|
|
)
|
|
|
|
job_id = ray.get_runtime_context().get_job_id()
|
|
experiment_name = ray.train.get_context().get_experiment_name()
|
|
|
|
self.tmpdir_prefix = Path(
|
|
tempfile.gettempdir(),
|
|
f"lightning_checkpoints-job_id={job_id}-name={experiment_name}-world_rank={ray.train.get_context().get_world_rank()}",
|
|
).as_posix()
|
|
if os.path.isdir(self.tmpdir_prefix):
|
|
shutil.rmtree(self.tmpdir_prefix)
|
|
|
|
record_extra_usage_tag(TagKey.TRAIN_LIGHTNING_RAYTRAINREPORTCALLBACK, "1")
|
|
|
|
def on_train_epoch_end(self, trainer, pl_module) -> None:
|
|
# Creates a checkpoint dir with fixed name
|
|
tmpdir = Path(self.tmpdir_prefix, str(trainer.current_epoch)).as_posix()
|
|
os.makedirs(tmpdir, exist_ok=True)
|
|
|
|
# Fetch metrics
|
|
metrics = trainer.callback_metrics
|
|
metrics = {k: v.item() for k, v in metrics.items()}
|
|
|
|
# (Optional) Add customized metrics
|
|
metrics["epoch"] = trainer.current_epoch
|
|
metrics["step"] = trainer.global_step
|
|
|
|
# Save checkpoint to local
|
|
ckpt_path = Path(tmpdir, self.CHECKPOINT_NAME).as_posix()
|
|
# TODO: with CheckpointUploadMode.ASYNC, this does cpu -> disk synchronously
|
|
# and disk -> remote asynchronously. We can add a new CheckpointIO class to do
|
|
# cpu -> remote asynchronously and a checkpoint_upload_fn to wait for it.
|
|
trainer.save_checkpoint(ckpt_path, weights_only=False)
|
|
|
|
# Report to train session
|
|
checkpoint = Checkpoint.from_directory(tmpdir)
|
|
if is_v2_enabled():
|
|
ray.train.report(
|
|
metrics=metrics,
|
|
checkpoint=checkpoint,
|
|
checkpoint_upload_mode=self.checkpoint_upload_mode,
|
|
validation=self.validation,
|
|
)
|
|
else:
|
|
ray.train.report(metrics=metrics, checkpoint=checkpoint)
|
|
|
|
# Add a barrier to ensure all workers finished reporting here
|
|
trainer.strategy.barrier()
|
|
|
|
# With CheckpointUploadMode.ASYNC, the upload may still be in progress
|
|
# after report() returns. Let ray.train.report delete_local_checkpoint_after_upload
|
|
# handle cleanup instead.
|
|
if is_v2_enabled() and self.checkpoint_upload_mode is not None:
|
|
# Check here because CheckpointUploadMode is only imported when is_v2_enabled() is True
|
|
if (
|
|
self.checkpoint_upload_mode.default_delete_local_checkpoint_after_upload()
|
|
):
|
|
return
|
|
|
|
shutil.rmtree(tmpdir)
|