wehub-resource-sync
18 lines
485 B
JSON
18 lines
485 B
JSON
{
|
|
"zero_optimization": {
|
|
"stage": 3,
|
|
"allgather_partitions": true,
|
|
"allgather_bucket_size": 5e8,
|
|
"overlap_comm": false,
|
|
"reduce_scatter": true,
|
|
"reduce_bucket_size": 5e8,
|
|
"contiguous_gradients": false,
|
|
"stage3_max_live_parameters": 1e9,
|
|
"stage3_max_reuse_distance": 1e9,
|
|
"stage3_prefetch_bucket_size": 5e8,
|
|
"stage3_param_persistence_threshold": 1e6,
|
|
"stage3_gather_16bit_weights_on_model_save": false,
|
|
"sub_group_size": 1e12
|
|
}
|
|
}
|