wehub-resource-sync
118 lines
3.6 KiB
Python
118 lines
3.6 KiB
Python
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
'''
|
|
python3.8 -m paddle.distributed.launch \
|
|
--devices=128 \
|
|
ipu \
|
|
--hosts=host1,host2 \
|
|
--ipus_per_host=2 \
|
|
--nproc_per_host=1 \
|
|
--ipu_partition=pod128 \
|
|
--vipu_server=lr17-1-ctrl \
|
|
test/ipu/disabled/test_dist_pod128_ipu.py
|
|
Equal to:
|
|
poprun \
|
|
--host=localhost,host2 \
|
|
--num-instances=2 \
|
|
--num-replicas=64 \
|
|
--ipus-per-replica=2 \
|
|
--print-topology=yes \
|
|
--vipu-partition=pod128_bert \
|
|
--vipu-server-host=lr17-1-ctrl \
|
|
--update-partition=yes \
|
|
python3.8 test/ipu/disabled/test_dist_pod128_ipu.py
|
|
'''
|
|
|
|
import os
|
|
|
|
import numpy as np
|
|
|
|
import paddle
|
|
|
|
|
|
def TestDistTraining():
|
|
paddle.enable_static()
|
|
|
|
attrs = {"size": [128, 16], "padding_idx": -1, "dtype": 'float32'}
|
|
|
|
scope = paddle.base.core.Scope()
|
|
main_prog = paddle.static.Program()
|
|
startup_prog = paddle.static.Program()
|
|
paddle.seed(42)
|
|
|
|
np.random.seed(42)
|
|
input_data = np.random.uniform(0, 127, size=[128, 3, 2, 1]).astype(np.int32)
|
|
|
|
with (
|
|
paddle.base.scope_guard(scope),
|
|
paddle.static.program_guard(main_prog, startup_prog),
|
|
):
|
|
x = paddle.static.data(name="x", shape=[3, 2, 1], dtype='int64')
|
|
with paddle.static.ipu_shard_guard(index=0, stage=0):
|
|
out = paddle.static.nn.embedding(x, **attrs)
|
|
with paddle.static.ipu_shard_guard(index=1, stage=1):
|
|
loss = paddle.mean(out)
|
|
opt = paddle.optimizer.Adam(learning_rate=1e-1)
|
|
opt.minimize(loss)
|
|
|
|
feed_list = ["x"]
|
|
fetch_list = [loss.name]
|
|
|
|
place = paddle.IPUPlace()
|
|
exe = paddle.static.Executor(place)
|
|
exe.run(startup_prog)
|
|
|
|
ipu_strategy = paddle.static.IpuStrategy()
|
|
ipu_strategy.set_graph_config(
|
|
num_ipus=64, is_training=True, enable_manual_shard=True
|
|
)
|
|
ipu_strategy.set_pipelining_config(
|
|
enable_pipelining=True,
|
|
batches_per_step=1,
|
|
enable_gradient_accumulation=True,
|
|
accumulation_factor=4,
|
|
)
|
|
ipu_strategy.set_options(
|
|
{
|
|
"enable_distribution": True,
|
|
"enable_replicated_graphs": True,
|
|
"replicated_graph_count": 32,
|
|
"enable_distributed_replicated_graphs": True,
|
|
"global_replica_offset":
|
|
# Paddle : int(os.environ.get("PADDLE_TRAINER_ID")) * 32
|
|
# PopRun : int(os.environ.get("POPDIST_REPLICA_INDEX_OFFSET"))
|
|
int(os.environ.get("PADDLE_TRAINER_ID")) * 32,
|
|
"global_replication_factor": 64,
|
|
"location_optimizer": {
|
|
"on_chip": False,
|
|
"use_replicated_tensor_sharding": True,
|
|
},
|
|
}
|
|
)
|
|
|
|
ipu_program = paddle.static.IpuCompiledProgram(
|
|
main_prog, ipu_strategy=ipu_strategy
|
|
)
|
|
program = ipu_program.compile(feed_list, fetch_list)
|
|
|
|
for i in range(10):
|
|
res = exe.run(
|
|
program, feed={"x": input_data}, fetch_list=fetch_list
|
|
)
|
|
print(f"index: {i}, result: {res}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
TestDistTraining()
|