wehub-resource-sync
211 lines
7.6 KiB
Python
211 lines
7.6 KiB
Python
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# import yaml
|
|
import unittest
|
|
|
|
from paddle.distributed.fleet import auto
|
|
|
|
|
|
class TestStrategy(unittest.TestCase):
|
|
def test_default_config(self):
|
|
strategy = auto.Strategy()
|
|
|
|
recompute = strategy.recompute
|
|
self.assertEqual(recompute.enable, False)
|
|
self.assertEqual(recompute.checkpoints, [])
|
|
|
|
amp = strategy.amp
|
|
self.assertEqual(amp.enable, False)
|
|
self.assertEqual(amp.dtype, "float16")
|
|
self.assertEqual(amp.level, "o1")
|
|
self.assertAlmostEqual(amp.init_loss_scaling, 32768.0)
|
|
self.assertEqual(amp.incr_every_n_steps, 1000)
|
|
self.assertEqual(amp.decr_every_n_nan_or_inf, 2)
|
|
self.assertAlmostEqual(amp.incr_ratio, 2.0)
|
|
self.assertAlmostEqual(amp.decr_ratio, 0.8)
|
|
self.assertEqual(amp.use_dynamic_loss_scaling, True)
|
|
self.assertEqual(amp.custom_black_list, [])
|
|
self.assertEqual(amp.custom_white_list, [])
|
|
self.assertEqual(amp.custom_black_varnames, [])
|
|
self.assertEqual(amp.use_fp16_guard, False)
|
|
self.assertEqual(amp.use_bf16_guard, False)
|
|
|
|
sharding = strategy.sharding
|
|
self.assertEqual(sharding.enable, False)
|
|
self.assertEqual(sharding.stage, 1)
|
|
self.assertEqual(sharding.degree, 8)
|
|
self.assertAlmostEqual(sharding.enable_overlap, False)
|
|
self.assertAlmostEqual(sharding.param_comm_stream_num, 1)
|
|
self.assertAlmostEqual(sharding.grad_comm_stream_num, 1)
|
|
self.assertAlmostEqual(sharding.partition_algor, "greedy_even")
|
|
self.assertAlmostEqual(sharding.param_bucket_size_numel, 1)
|
|
self.assertAlmostEqual(sharding.grad_bucket_size_numel, 1)
|
|
self.assertAlmostEqual(sharding.enable_hierarchical_comm, False)
|
|
self.assertEqual(sharding.enable_tuning, False)
|
|
self.assertEqual(sharding.tuning_range, [])
|
|
|
|
gradient_merge = strategy.gradient_merge
|
|
self.assertEqual(gradient_merge.enable, False)
|
|
self.assertEqual(gradient_merge.k_steps, 1)
|
|
self.assertEqual(gradient_merge.avg, True)
|
|
|
|
qat = strategy.qat
|
|
self.assertEqual(qat.enable, False)
|
|
self.assertEqual(qat.channel_wise_abs_max, True)
|
|
self.assertEqual(qat.weight_bits, 8)
|
|
self.assertEqual(qat.activation_bits, 8)
|
|
self.assertEqual(qat.not_quant_pattern, ['skip_quant'])
|
|
self.assertIsNone(qat.algo)
|
|
|
|
tuning = strategy.tuning
|
|
self.assertEqual(tuning.enable, False)
|
|
self.assertEqual(tuning.profile_start_step, 1)
|
|
self.assertEqual(tuning.profile_end_step, 1)
|
|
self.assertEqual(tuning.run_after_tuning, True)
|
|
self.assertEqual(tuning.debug, False)
|
|
|
|
def test_modify_config(self):
|
|
strategy = auto.Strategy()
|
|
|
|
recompute = strategy.recompute
|
|
recompute.enable = True
|
|
recompute.checkpoints = ["x"]
|
|
self.assertEqual(recompute.enable, True)
|
|
self.assertEqual(recompute.checkpoints, ["x"])
|
|
|
|
amp = strategy.amp
|
|
amp.enable = True
|
|
amp.dtype = "float16"
|
|
amp.level = "o2"
|
|
amp.init_loss_scaling = 16384.0
|
|
amp.incr_every_n_steps = 2000
|
|
amp.decr_every_n_nan_or_inf = 4
|
|
amp.incr_ratio = 4.0
|
|
amp.decr_ratio = 0.4
|
|
amp.use_dynamic_loss_scaling = False
|
|
amp.custom_white_list = ["x"]
|
|
amp.custom_black_list = ["y"]
|
|
amp.custom_black_varnames = ["z"]
|
|
amp.use_fp16_guard = False
|
|
self.assertEqual(amp.enable, True)
|
|
self.assertEqual(amp.dtype, "float16")
|
|
self.assertEqual(amp.level, "o2")
|
|
self.assertAlmostEqual(amp.init_loss_scaling, 16384.0)
|
|
self.assertEqual(amp.incr_every_n_steps, 2000)
|
|
self.assertEqual(amp.decr_every_n_nan_or_inf, 4)
|
|
self.assertAlmostEqual(amp.incr_ratio, 4.0)
|
|
self.assertAlmostEqual(amp.decr_ratio, 0.4)
|
|
self.assertEqual(amp.use_dynamic_loss_scaling, False)
|
|
self.assertEqual(amp.custom_white_list, ["x"])
|
|
self.assertEqual(amp.custom_black_list, ["y"])
|
|
self.assertEqual(amp.custom_black_varnames, ["z"])
|
|
self.assertEqual(amp.use_fp16_guard, False)
|
|
|
|
sharding = strategy.sharding
|
|
sharding.enable = True
|
|
sharding.stage = 2
|
|
sharding.degree = 2
|
|
sharding.segment_broadcast_MB = 64.0
|
|
sharding.enable_tuning = True
|
|
sharding.tuning_range = [1, 2, 3]
|
|
self.assertEqual(sharding.enable, True)
|
|
self.assertEqual(sharding.stage, 2)
|
|
self.assertEqual(sharding.degree, 2)
|
|
self.assertAlmostEqual(sharding.segment_broadcast_MB, 64.0)
|
|
self.assertEqual(sharding.enable_tuning, True)
|
|
self.assertEqual(sharding.tuning_range, [1, 2, 3])
|
|
|
|
gradient_merge = strategy.gradient_merge
|
|
gradient_merge.enable = True
|
|
gradient_merge.k_steps = 4
|
|
gradient_merge.avg = False
|
|
self.assertEqual(gradient_merge.enable, True)
|
|
self.assertEqual(gradient_merge.k_steps, 4)
|
|
self.assertEqual(gradient_merge.avg, False)
|
|
|
|
# def test_file_config(self):
|
|
# yaml_data = """
|
|
# all_ranks: false
|
|
# amp:
|
|
# custom_black_list:
|
|
# - y
|
|
# custom_black_varnames:
|
|
# - z
|
|
# custom_white_list:
|
|
# - x
|
|
# decr_every_n_nan_or_inf: 4
|
|
# decr_ratio: 0.4
|
|
# enable: false
|
|
# incr_every_n_steps: 2000
|
|
# incr_ratio: 4.0
|
|
# init_loss_scaling: 16384.0
|
|
# use_dynamic_loss_scaling: false
|
|
# use_fp16_guard: false
|
|
# use_optimizer_fp16: true
|
|
# use_pure_fp16: true
|
|
# auto_mode: semi
|
|
# gradient_merge:
|
|
# avg: false
|
|
# enable: false
|
|
# k_steps: 4
|
|
# gradient_scale: true
|
|
# qat:
|
|
# activation_bits: 8
|
|
# algo: null
|
|
# channel_wise_abs_max: true
|
|
# enable: false
|
|
# not_quant_pattern:
|
|
# - skip_quant
|
|
# weight_bits: 8
|
|
# recompute:
|
|
# checkpoints: null
|
|
# enable: false
|
|
# enable_tuning: false
|
|
# return_numpy: true
|
|
# seed: null
|
|
# sharding:
|
|
# enable: false
|
|
# enable_tuning: true
|
|
# segment_broadcast_MB: 64.0
|
|
# degree: 8
|
|
# stage: 2
|
|
# tuning_range: None
|
|
# split_data: false
|
|
# tuning:
|
|
# batch_size: 1
|
|
# dataset: null
|
|
# enable: false
|
|
# profile_end_step: 1
|
|
# profile_start_step: 1
|
|
# run_after_tuning: true
|
|
# verbose: true
|
|
# use_cache: true
|
|
# """
|
|
# yaml_path = "./strategy.yml"
|
|
# yaml_dict = yaml.load(yaml_data, Loader=yaml.Loader)
|
|
# with open(yaml_path, 'w') as outfile:
|
|
# yaml.dump(yaml_dict, outfile, default_flow_style=False)
|
|
|
|
# strategy = auto.Strategy(yaml_path)
|
|
# self.assertEqual(yaml_dict, strategy.to_dict())
|
|
|
|
# # Remove the created file
|
|
# if os.path.exists(yaml_path):
|
|
# os.remove(yaml_path)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|