chore: import upstream snapshot with attribution

python/paddle/distributed/launch/controllers/rpc.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+from .controller import Controller, ControllerMode
+
+
+class RpcController(Controller):
+    @classmethod
+    def enable(cls, ctx):
+        if ctx.args.run_mode == ControllerMode.RPC:
+            ctx.logger.debug(f"{cls.__name__} enabled")
+            return True
+        else:
+            return False
+
+    def build_pod(self):
+        assert self.ctx.args.master is not None, (
+            "Master is None, Please set master address!"
+        )
+        self._build_pod_with_master()
+
+    def _build_pod_with_master(self):
+        # nproc_per_node
+        self.pod.replicas = self.pod_replicas()
+
+        # rank will be reset when restart
+        self.pod.rank = int(self.ctx.args.rank)
+
+        port = self.ctx.node.get_free_port()
+
+        # compatible
+        endpoints = [
+            f"{self.ctx.node.ip}:{p}"
+            for p in self.ctx.node.get_free_ports(self.pod.replicas)
+        ]
+
+        data = json.dumps(
+            {
+                "name": self.pod.name,
+                "rank": self.pod.rank,
+                "replicas": self.pod.replicas,
+                "dtype": self.ctx.node.device.dtype,
+                "candidate": f"{self.ctx.node.ip}:{port}",
+                "endpoints": ",".join(endpoints),
+            }
+        )
+        peer_list, rank = self.master.sync_peers(
+            f"/{self.job.id}/info",
+            self.pod.name,
+            data,
+            self.job.replicas,
+            self.pod.rank,
+        )
+        self.pod.rank = rank
+
+        if len(peer_list) < 1:
+            return False
+
+        peer_list = [json.loads(i) for i in peer_list]
+
+        self.ctx.logger.debug(f"sync peers done {peer_list}")
+        self.save_pod_log(peer_list)
+
+        global_size = sum([i["replicas"] for i in peer_list])
+        rank_offset = sum([i["replicas"] for i in peer_list[:rank]])
+
+        rpc_master = peer_list[0]["candidate"]
+        self.pod.reset()
+        for i in range(self.pod.replicas):
+            e = {
+                "PADDLE_MASTER_ENDPOINT": rpc_master,
+                "PADDLE_WORKER_ENDPOINT": endpoints[i],
+                "PADDLE_TRAINER_ID": f"{i + rank_offset}",
+                "PADDLE_TRAINERS_NUM": f"{global_size}",
+            }
+            log_file = f"workerlog.{i + rank_offset}"
+            self.add_container(envs=e, log_file=log_file)
+        return True