chore: import upstream snapshot with attribution

test/ipu/distributed/run_dist_ipu.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+partition_name=pod64
+vipu_server=10.137.96.62
+allclose_script="
+import sys
+import numpy as np
+data1 = np.loadtxt(\"ipu_res.txt\")
+data2 = np.loadtxt(\"cpu_res.txt\")
+if np.allclose(data1[::16], data2, atol=1e-6):
+    sys.exit(0)
+else:
+    sys.exit(1)
+"
+
+for opt in lamb sgd adam ;
+do
+    for onchip in False True ;
+    do
+        for rts in False True ;
+        do
+            echo "Testcase: opt: ${opt}, onchip: ${onchip}, rts: ${rts}"
+            echo "paddle.distributed.fleet.launch test with IPUs..."
+            python3.8 -m paddle.distributed.launch \
+            --devices=8 \
+            ipu \
+            --hosts=localhost \
+            --nproc_per_host=2 \
+            --ipus_per_replica=2 \
+            --ipu_partition=${partition_name} \
+            --vipu_server=${vipu_server} \
+            test_dist_data_parallel_ipu.py ${opt} ipu_res.txt ${onchip} ${rts} > ipu.log
+            echo "paddle.distributed.fleet.launch test with IPUs...Done"
+
+            echo "paddle normal test with CPU..."
+            export POPLAR_IPUMODEL=1
+            python3.8 test_dist_data_parallel_ipu.py ${opt} cpu_res.txt > cpu.log
+            unset POPLAR_IPUMODEL
+            echo "paddle normal test with CPU...Done"
+
+            echo "Compare results..."
+            python3.8 -c """${allclose_script}"""
+            if [ $? -eq 0 ];then
+            echo "Compare results...Done"
+            else
+            echo "Error occurs. Please check ipu.log, cpu.log, ipu_res.txt and cpu_res.txt"
+            exit 0
+            fi
+        done
+    done
+done
+
+if [ -f "ipu.log" ]; then
+    rm "ipu.log"
+fi
+if [ -f "cpu.log" ]; then
+    rm "cpu.log"
+fi
+if [ -f "ipu_res.txt" ]; then
+    rm "ipu_res.txt"
+fi
+if [ -f "cpu_res.txt" ]; then
+    rm "cpu_res.txt"
+fi

test/ipu/distributed/test_dist_data_parallel_ipu.py

@@ -0,0 +1,210 @@
+#  Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import random
+import sys
+import unittest
+
+import numpy as np
+
+import paddle
+import paddle.static
+
+from ..op_test_ipu import IPUOpTest
+
+mpi_comm = None
+
+
+@unittest.skip('Disable distributed tests on auto CI.')
+class TestBase(IPUOpTest):
+    def set_attrs(self, enable_ipu, optimizer, log, onchip=False, rts=False):
+        self.ipu_options = {
+            "enable_pipelining": True,
+            "batches_per_step": 1,
+            "enable_gradient_accumulation": True,
+            "accumulation_factor": 4,
+            "enable_replicated_graphs": True,
+            "replicated_graph_count": 2,
+            "location_optimizer": {
+                "on_chip": onchip,
+                "use_replicated_tensor_sharding": rts,
+            },
+        }
+
+        self.cpu_bs = 16
+        self.ipu_bs = 1
+        self.optimizer = optimizer
+        self.log = log
+        self.enable_ipu = enable_ipu
+
+    def test(self):
+        seed = 2021
+        np.random.seed(seed)
+        random.seed(seed)
+        scope = paddle.static.Scope()
+        main_prog = paddle.static.Program()
+        startup_prog = paddle.static.Program()
+        paddle.seed(seed)
+
+        bs = self.ipu_bs if self.enable_ipu else self.cpu_bs
+        data = np.random.rand(1, 3, 10, 10).astype(np.float32)
+
+        with (
+            paddle.static.scope_guard(scope),
+            paddle.static.program_guard(main_prog, startup_prog),
+        ):
+            image = paddle.static.data(
+                name='image', shape=[bs, 3, 10, 10], dtype='float32'
+            )
+            with paddle.static.ipu_shard_guard(index=0, stage=0):
+                conv1 = paddle.nn.Conv2D(
+                    in_channels=image.shape[1],
+                    out_channels=3,
+                    kernel_size=3,
+                    bias_attr=False,
+                )(image)
+
+            with paddle.static.ipu_shard_guard(index=1, stage=1):
+                conv2 = paddle.nn.Conv2D(
+                    in_channels=conv1.shape[1],
+                    out_channels=3,
+                    kernel_size=3,
+                    bias_attr=False,
+                )(conv1)
+
+                # should consider influence of bs
+                loss = paddle.mean(conv2)
+
+            if self.optimizer == 'sgd':
+                opt = paddle.optimizer.SGD(learning_rate=1e-2)
+            elif self.optimizer == 'adam':
+                opt = paddle.optimizer.Adam(learning_rate=1e-2)
+            elif self.optimizer == 'lamb':
+                opt = paddle.optimizer.Lamb(learning_rate=1e-2)
+            else:
+                raise Exception('optimizer must be sgd, adam or lamb')
+
+            opt.minimize(loss)
+
+            if self.enable_ipu:
+                place = paddle.IPUPlace()
+            else:
+                place = paddle.CPUPlace()
+            executor = paddle.static.Executor(place)
+            executor.run(startup_prog)
+
+            if self.enable_ipu:
+                feed_list = [image.name]
+                fetch_list = [loss.name]
+                ipu_strategy = paddle.static.IpuStrategy()
+                ipu_strategy.set_graph_config(
+                    num_ipus=2 * self.ipu_options['replicated_graph_count'],
+                    is_training=True,
+                    enable_manual_shard=True,
+                )
+                ipu_strategy.set_options(self.ipu_options)
+                ipu_strategy.set_options(
+                    {
+                        "enable_distribution": True,
+                        "enable_distributed_replicated_graphs": True,
+                        "global_replica_offset": int(
+                            os.environ.get("PADDLE_TRAINER_ID")
+                        )
+                        * 2,
+                        "global_replication_factor": 4,
+                    }
+                )
+                program = paddle.static.IpuCompiledProgram(
+                    main_prog, ipu_strategy=ipu_strategy
+                ).compile(feed_list, fetch_list)
+                feed = {
+                    "image": np.tile(
+                        data,
+                        [
+                            self.ipu_options['replicated_graph_count']
+                            * self.ipu_options['batches_per_step']
+                            * self.ipu_options['accumulation_factor'],
+                            1,
+                            1,
+                            1,
+                        ],
+                    )
+                }
+
+            else:
+                program = main_prog
+                feed = {"image": np.tile(data, [self.cpu_bs, 1, 1, 1])}
+
+            epoch = 10
+            if not self.enable_ipu:
+                # global replication factor
+                epoch *= 4
+                epoch *= self.ipu_options['batches_per_step']
+                epoch *= self.ipu_options['accumulation_factor']
+                epoch = epoch / (self.cpu_bs / self.ipu_bs)
+
+            results = []
+            for i in range(int(epoch)):
+                res = executor.run(program, feed=feed, fetch_list=[loss])
+                if self.enable_ipu:
+                    res = mpi_comm.gather(res, root=0)
+                results.append(res)
+            if self.enable_ipu:
+                if int(os.environ.get("PADDLE_TRAINER_ID")) == 0:
+                    np.savetxt(self.log, np.array(results).flatten())
+            else:
+                np.savetxt(self.log, np.array(results).flatten())
+
+
+if __name__ == "__main__":
+    paddle.enable_static()
+    # Run distributed tests
+    if len(sys.argv) == 5:
+        from mpi4py import MPI
+
+        DISTRIBUTED_COMM = MPI.COMM_WORLD
+
+        def _get_comm():
+            global DISTRIBUTED_COMM
+            if DISTRIBUTED_COMM is None:
+                raise RuntimeError(
+                    "Distributed Commumication not setup. Please run setup_comm(MPI.COMM_WORLD) first."
+                )
+            return DISTRIBUTED_COMM
+
+        mpi_comm = _get_comm()
+
+        optimizer = sys.argv[1]
+        log = sys.argv[2]
+        onchip = True if sys.argv[3] == "True" else False
+        rts = True if sys.argv[4] == "True" else False
+        test = TestBase()
+        test.set_attrs(
+            enable_ipu=True,
+            optimizer=optimizer,
+            log=log,
+            onchip=onchip,
+            rts=rts,
+        )
+        test.test()
+    # Run cpu tests for compare
+    elif len(sys.argv) == 3:
+        test = TestBase()
+        test.set_attrs(enable_ipu=False, optimizer=sys.argv[1], log=sys.argv[2])
+        test.test()
+    else:
+        raise ValueError(
+            "Only support 3 or 5 args. 3 for cpu test, 5 for ipu distributed test"
+        )

test/ipu/distributed/test_dist_pod128_sample.py

@@ -0,0 +1,117 @@
+#  Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+python3.8 -m paddle.distributed.launch \
+--devices=128 \
+ipu \
+--hosts=host1,host2 \
+--ipus_per_host=2 \
+--nproc_per_host=1 \
+--ipu_partition=pod128 \
+--vipu_server=lr17-1-ctrl \
+test/ipu/disabled/test_dist_pod128_ipu.py
+Equal to:
+poprun \
+--host=localhost,host2 \
+--num-instances=2 \
+--num-replicas=64 \
+--ipus-per-replica=2 \
+--print-topology=yes \
+--vipu-partition=pod128_bert \
+--vipu-server-host=lr17-1-ctrl \
+--update-partition=yes \
+python3.8 test/ipu/disabled/test_dist_pod128_ipu.py
+'''
+
+import os
+
+import numpy as np
+
+import paddle
+
+
+def TestDistTraining():
+    paddle.enable_static()
+
+    attrs = {"size": [128, 16], "padding_idx": -1, "dtype": 'float32'}
+
+    scope = paddle.base.core.Scope()
+    main_prog = paddle.static.Program()
+    startup_prog = paddle.static.Program()
+    paddle.seed(42)
+
+    np.random.seed(42)
+    input_data = np.random.uniform(0, 127, size=[128, 3, 2, 1]).astype(np.int32)
+
+    with (
+        paddle.base.scope_guard(scope),
+        paddle.static.program_guard(main_prog, startup_prog),
+    ):
+        x = paddle.static.data(name="x", shape=[3, 2, 1], dtype='int64')
+        with paddle.static.ipu_shard_guard(index=0, stage=0):
+            out = paddle.static.nn.embedding(x, **attrs)
+        with paddle.static.ipu_shard_guard(index=1, stage=1):
+            loss = paddle.mean(out)
+        opt = paddle.optimizer.Adam(learning_rate=1e-1)
+        opt.minimize(loss)
+
+        feed_list = ["x"]
+        fetch_list = [loss.name]
+
+        place = paddle.IPUPlace()
+        exe = paddle.static.Executor(place)
+        exe.run(startup_prog)
+
+        ipu_strategy = paddle.static.IpuStrategy()
+        ipu_strategy.set_graph_config(
+            num_ipus=64, is_training=True, enable_manual_shard=True
+        )
+        ipu_strategy.set_pipelining_config(
+            enable_pipelining=True,
+            batches_per_step=1,
+            enable_gradient_accumulation=True,
+            accumulation_factor=4,
+        )
+        ipu_strategy.set_options(
+            {
+                "enable_distribution": True,
+                "enable_replicated_graphs": True,
+                "replicated_graph_count": 32,
+                "enable_distributed_replicated_graphs": True,
+                "global_replica_offset":
+                # Paddle : int(os.environ.get("PADDLE_TRAINER_ID")) * 32
+                # PopRun : int(os.environ.get("POPDIST_REPLICA_INDEX_OFFSET"))
+                int(os.environ.get("PADDLE_TRAINER_ID")) * 32,
+                "global_replication_factor": 64,
+                "location_optimizer": {
+                    "on_chip": False,
+                    "use_replicated_tensor_sharding": True,
+                },
+            }
+        )
+
+        ipu_program = paddle.static.IpuCompiledProgram(
+            main_prog, ipu_strategy=ipu_strategy
+        )
+        program = ipu_program.compile(feed_list, fetch_list)
+
+        for i in range(10):
+            res = exe.run(
+                program, feed={"x": input_data}, fetch_list=fetch_list
+            )
+            print(f"index: {i}, result: {res}")
+
+
+if __name__ == "__main__":
+    TestDistTraining()

test/ipu/distributed/test_dist_sample.py

@@ -0,0 +1,198 @@
+#  Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+Single host:
+python3.8 -m paddle.distributed.launch \
+--devices=4 \
+ipu \
+--hosts=localhost \
+--nproc_per_host=2 \
+--ipus_per_replica=1 \
+--ipu_partition=pod64 \
+--vipu_server=10.137.96.62 \
+test/ipu/distributed/test_dist_sample.py
+Equal to:
+poprun \
+--host=localhost \
+--num-instances=2 \
+--num-replicas=4 \
+--ipus-per-replica=1 \
+--print-topology=yes \
+python3.8 test/ipu/distributed/test_dist_sample.py
+'''
+
+'''
+Multi hosts:
+python3.8 -m paddle.distributed.launch \
+--devices=4 \
+ipu \
+--hosts=host1,host2 \
+--nproc_per_host=1 \
+--ipus_per_replica=1 \
+--ipu_partition=pod64 \
+--vipu_server=10.137.96.62 \
+test/ipu/distributed/test_dist_sample.py
+Equal to:
+poprun \
+--host=host1,host2 \
+--num-instances=2 \
+--num-replicas=4 \
+--ipus-per-replica=1 \
+--print-topology=yes \
+python3.8 test/ipu/distributed/test_dist_sample.py
+'''
+
+import os
+import sys
+
+import numpy as np
+
+import paddle
+
+mpi_comm = None
+
+
+def Test(use_dist, file_name):
+    paddle.enable_static()
+
+    attrs = {"size": [128, 16], "padding_idx": -1, "dtype": 'float32'}
+
+    scope = paddle.base.core.Scope()
+    main_prog = paddle.static.Program()
+    startup_prog = paddle.static.Program()
+    paddle.seed(42)
+
+    with (
+        paddle.base.scope_guard(scope),
+        paddle.static.program_guard(main_prog, startup_prog),
+    ):
+        x = paddle.static.data(name="x", shape=[3, 2, 1], dtype='int64')
+
+        out = paddle.static.nn.embedding(x, **attrs)
+        loss = paddle.mean(out)
+        opt = paddle.optimizer.Adam(learning_rate=1e-1)
+        opt.minimize(loss)
+
+        feed_list = ["x"]
+        fetch_list = [loss.name]
+
+        place = paddle.IPUPlace()
+        exe = paddle.static.Executor(place)
+        exe.run(startup_prog)
+
+        ipu_strategy = paddle.static.IpuStrategy()
+        if use_dist:
+            ipu_strategy.set_graph_config(num_ipus=2, is_training=True)
+            # Set distributed envs
+            ipu_strategy.set_options(
+                {
+                    "enable_distribution": True,
+                    "enable_replicated_graphs": True,
+                    "replicated_graph_count": 2,
+                    "enable_distributed_replicated_graphs": True,
+                    "global_replica_offset": int(
+                        os.environ.get("PADDLE_TRAINER_ID")
+                    )
+                    * 2,
+                    "global_replication_factor": 4,
+                }
+            )
+        else:
+            ipu_strategy.set_graph_config(num_ipus=4, is_training=True)
+            ipu_strategy.set_options(
+                {
+                    "enable_replicated_graphs": True,
+                    "replicated_graph_count": 4,
+                }
+            )
+
+        ipu_program = paddle.static.IpuCompiledProgram(
+            main_prog, ipu_strategy=ipu_strategy
+        )
+        program = ipu_program.compile(feed_list, fetch_list)
+
+        if use_dist:
+            if os.environ.get("PADDLE_TRAINER_ID") == "0":
+                input_data = np.concatenate(
+                    [
+                        np.array([[[1], [3]], [[2], [4]], [[4], [127]]]).astype(
+                            np.int32
+                        ),
+                        np.array([[[1], [3]], [[2], [4]], [[4], [127]]]).astype(
+                            np.int32
+                        ),
+                    ]
+                )
+            else:
+                input_data = np.concatenate(
+                    [
+                        np.array(
+                            [[[8], [60]], [[50], [77]], [[90], [13]]]
+                        ).astype(np.int32),
+                        np.array(
+                            [[[8], [60]], [[50], [77]], [[90], [13]]]
+                        ).astype(np.int32),
+                    ]
+                )
+        else:
+            input_data = np.concatenate(
+                [
+                    np.array([[[1], [3]], [[2], [4]], [[4], [127]]]).astype(
+                        np.int32
+                    ),
+                    np.array([[[1], [3]], [[2], [4]], [[4], [127]]]).astype(
+                        np.int32
+                    ),
+                    np.array([[[8], [60]], [[50], [77]], [[90], [13]]]).astype(
+                        np.int32
+                    ),
+                    np.array([[[8], [60]], [[50], [77]], [[90], [13]]]).astype(
+                        np.int32
+                    ),
+                ]
+            )
+        feed_data = {"x": input_data}
+
+        for step in range(10):
+            res = exe.run(program, feed=feed_data, fetch_list=fetch_list)
+
+        if use_dist:
+            res = mpi_comm.gather(res)
+            if os.getenv("PADDLE_TRAINER_ID") == "0":
+                np.savetxt(file_name, np.array(res).flatten())
+        else:
+            np.savetxt(file_name, np.array(res).flatten())
+
+
+if __name__ == "__main__":
+    file_name = sys.argv[1]
+
+    use_dist = False
+    if 'PADDLE_TRAINER_ID' in os.environ:
+        from mpi4py import MPI
+
+        DISTRIBUTED_COMM = MPI.COMM_WORLD
+
+        def _get_comm():
+            global DISTRIBUTED_COMM
+            if DISTRIBUTED_COMM is None:
+                raise RuntimeError(
+                    "Distributed Communication not setup. Please run setup_comm(MPI.COMM_WORLD) first."
+                )
+            return DISTRIBUTED_COMM
+
+        mpi_comm = _get_comm()
+        use_dist = True
+
+    Test(use_dist, file_name)