# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations import contextlib import logging import paddle from ..utils.log_utils import get_logger from .process_mesh import retrieve_unique_id_for_process_mesh from .static.utils import _get_idx_in_axis _logger = get_logger(logging.INFO) _rng_name_to_seed = {} _rng_name_to_states = {} _inited_rng_name_to_seed = {} _enable_random_control = False _basic_seed = 42 _basic_name = "" # use Prime number as offset to avoid conflict _mesh_offset = 173 _dim_offsets = [11, 23, 37, 73] def is_enable_auto_rand_ctrl(): global _enable_random_control return _enable_random_control def enable_auto_rand_ctrl(): global _enable_random_control _enable_random_control = True def parallel_manual_seed(seed: int, name: str = "") -> None: """Enable auto parallel random control. Random control maintain the randomness when tensor is distributed across devices on a Mesh(any order). * Independency: If tensor is **Sharded** on a Mesh dimension, Devices along that Mesh dimension should have Different randomness. * Consistency: Meanwhile if the tensor is **Replicated** on another Mesh dimension, randomness of Devices along that Mesh dimension should be Consistent. For instance: rank0 ~ rank7 consist a Mesh of shape of [2, 4]; A 2D tensor is distributed in that Mesh using dims_mapping [-1, 1]. Randomness for rank0-rank1-rank2-rank3 (rank4-rank5-rank6-rank7) should be Independent; Randomness for rank0 and rank4 (rank1 and rank5, ...) should be Consistent. This function should be called only once before auto parallel compiles the computation graph (e.g. auto_parallel.engine.prepare() or fit()). This seed only affects how randomness-relative **operators** (dropout, fuse op with dropout inside, etc) are execute among mesh, and would NOT affect other process like Parameter initialization. Examples: # seed relative to training step auto_parallel_random_seed((step + 13) * 257) ... engine.prepare() """ enable_auto_rand_ctrl() global _basic_seed _basic_seed = seed global _basic_name _basic_name = name def determinate_rng( rank, dims_mapping=None, process_mesh=None, placements=None ): assert process_mesh is not None, "Must provide process mesh" assert dims_mapping is not None or placements is not None, ( "Must provide one of dims mapping or placements." ) assert not (dims_mapping is not None and placements is not None), ( "Cannot provide dims mapping and placements at same time." ) # TODO(JZ-LIANG) Support Mesh with any high rank # use a string to unique integer hashing algorithm for seed computation. # instead of using offsets to coordinate seed across devices. if len(process_mesh.shape) > 4: raise NotImplementedError( f"Auto Parallel Random Control for Mesh's rank > 4 is NOT supported! Got {process_mesh}" ) global _basic_seed seed_ = _basic_seed global _basic_name name_ = _basic_name if name_: name_ += "_" # FIXME # unique_id = process_mesh.unique_id unique_id = retrieve_unique_id_for_process_mesh( process_mesh.shape, process_mesh.process_ids ) sharding_expr = name_ + f'mesh:{unique_id}' seed_ += _mesh_offset * (unique_id + 1) for i in range(len(process_mesh.shape)): if (dims_mapping is not None and i not in dims_mapping) or ( placements is not None and not placements[i].is_shard() ): relative_idx = -1 else: relative_idx = _get_idx_in_axis( process_mesh.process_ids, process_mesh.shape, i, rank, ) sharding_expr += f"_dim{i}:{relative_idx}" seed_ += _dim_offsets[i] * (relative_idx + 1) global _rng_name_to_seed global _rng_name_to_states if sharding_expr in _rng_name_to_seed: assert _rng_name_to_seed[sharding_expr] == seed_ else: assert seed_ not in _rng_name_to_seed.values(), ( f"Seed Conflict! current seed: {seed_}, current sharding expr: {sharding_expr}, generated seed: {_rng_name_to_seed}" ) _rng_name_to_seed[sharding_expr] = seed_ if paddle.in_dynamic_mode(): # for dygraph, just init the seed when meeting a new seed orig_rng_state = paddle.get_rng_state() paddle.seed(seed_) _rng_name_to_states[sharding_expr] = paddle.get_rng_state() paddle.set_rng_state(orig_rng_state) return sharding_expr @contextlib.contextmanager def rng_state(name): global _rng_name_to_states assert name in _rng_name_to_states, ( f"The rng state name {name} haven't been init. " ) orig_rng_state = paddle.get_rng_state() paddle.set_rng_state(_rng_name_to_states[name]) try: yield finally: _rng_name_to_states[name] = paddle.get_rng_state() paddle.set_rng_state(orig_rng_state) def init_auto_parallel_rng(): if not is_enable_auto_rand_ctrl(): return global _rng_name_to_seed # NOTE init rng maybe call multiple times, avoid init same rng twice global _inited_rng_name_to_seed for rng_name, seed in _rng_name_to_seed.items(): if rng_name in _inited_rng_name_to_seed: assert _inited_rng_name_to_seed[rng_name] == seed else: _logger.info( f"Init Auto Parallel RNG: {rng_name}, with seed {seed}" ) paddle.framework.random.set_random_seed_generator(rng_name, seed) _inited_rng_name_to_seed[rng_name] = seed