chore: import upstream snapshot with attribution

python/paddle/distributed/launch/context/__init__.py

@@ -0,0 +1,107 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from paddle.distributed.launch import plugins
+
+from .args_envs import env_args_mapping, fetch_envs, parse_args
+from .node import Node
+from .status import Status
+
+
+class Context:
+    def __init__(self, enable_plugin=True):
+        self.args, self.unknown_args = parse_args()
+        self.envs = fetch_envs()
+
+        self.set_env_in_args()
+
+        self.node = Node()
+        self.status = Status()
+
+        self.logger = self.get_logger()
+
+        # design for event queue, later
+        self.events = []
+
+        if enable_plugin:
+            self._enable_plugin()
+        self.max_time_per_task = -1
+        self.run_best = False
+
+    def print(self):
+        self.logger.info("-----------  Configuration  ----------------------")
+        for arg, value in sorted(vars(self.args).items()):
+            self.logger.info(f"{arg}: {value}")
+        self.logger.info("--------------------------------------------------")
+
+    def is_legacy_mode(self):
+        if self.args.legacy:
+            return True
+
+        if self.args.master:
+            return False
+
+        if len(self.unknown_args) > 0:
+            self.logger.warning(
+                f"Compatible mode enable with args {self.unknown_args}"
+            )
+            return True
+
+        return False
+
+    def is_auto_tuner_mode(self):
+        if self.args.auto_tuner_json:
+            return True
+        return False
+
+    def get_envs(self):
+        return self.envs.copy()
+
+    def set_envs(self, env={}):
+        env = {k: v for k, v in env.items() if isinstance(v, str)}
+        self.envs.update(env)
+
+    def _enable_plugin(self):
+        for pl in plugins.enabled_plugins:
+            pl(self)
+
+    def get_logger(self, level=logging.INFO):
+        logger = logging.getLogger("LAUNCH")
+        # forbid the child logger pass on to its parent
+        logger.propagate = False
+        logger.setLevel(self.args.log_level.upper() or level)
+        formatter = logging.Formatter(
+            fmt='%(name)s %(levelname)s %(asctime)s %(message)s'
+        )
+        ch = logging.StreamHandler()
+        ch.setFormatter(formatter)
+        logger.addHandler(ch)
+        return logger
+
+    def continuous_log(self) -> bool:
+        if self.args.log_level.upper() in ['DEBUG', 'ERROR']:
+            return True
+        else:
+            return False
+
+    def set_env_in_args(self):
+        for k, v in env_args_mapping.items():
+            attr, attr_type = v
+            if k in self.envs:
+                print(
+                    f"LAUNCH WARNING args {attr} will be overridden by env: {k} value: {self.envs[k]}"
+                )
+                setattr(self.args, attr, attr_type(self.envs[k]))

python/paddle/distributed/launch/context/args_envs.py

@@ -0,0 +1,246 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import warnings
+from argparse import REMAINDER, ArgumentParser
+
+from paddle.utils import strtobool
+
+env_args_mapping = {
+    'POD_IP': ('host', str),
+    'PADDLE_MASTER': ('master', str),
+    'PADDLE_DEVICES': ('devices', str),
+    'PADDLE_NNODES': ('nnodes', str),
+    'PADDLE_RUN_MODE': ('run_mode', str),
+    'PADDLE_LOG_LEVEL': ('log_level', str),
+    'PADDLE_LOG_OVERWRITE': ('log_overwrite', strtobool),
+    'PADDLE_SORT_IP': ('sort_ip', strtobool),
+    'PADDLE_NPROC_PER_NODE': ('nproc_per_node', int),
+    'PADDLE_JOB_ID': ('job_id', str),
+    'PADDLE_RANK': ('rank', int),
+    'PADDLE_LOG_DIR': ('log_dir', str),
+    'PADDLE_MAX_RESTART': ('max_restart', int),
+    'PADDLE_ELASTIC_LEVEL': ('elastic_level', int),
+    'PADDLE_ELASTIC_TIMEOUT': ('elastic_timeout', int),
+    'PADDLE_SERVER_NUM': ('server_num', int),
+    'PADDLE_TRAINER_NUM': ('trainer_num', int),
+    'PADDLE_SERVERS_ENDPOINTS': ('servers', str),
+    'PADDLE_TRAINERS_ENDPOINTS': ('trainers', str),
+    'PADDLE_GLOO_PORT': ('gloo_port', int),
+    'PADDLE_WITH_GLOO': ('with_gloo', str),
+    'PADDLE_START_PORT': ('start_port', int),
+    'PADDLE_IPS': ('ips', str),
+    "PADDLE_AUTO_PARALLEL_CONFIG": ('auto_parallel_config', str),
+    'PADDLE_AUTO_CLUSTER': ('auto_cluster_config', strtobool),
+}
+
+
+def fetch_envs():
+    for proxy_key in ("http_proxy", "https_proxy"):
+        if os.environ.get(proxy_key) is not None:
+            os.environ[f"{proxy_key}_original"] = os.environ.pop(proxy_key)
+            warnings.warn(
+                f"Unset '{proxy_key}' to ensure stable NCCL communication in distributed training "
+                f"(backed up as '{proxy_key}_original').",
+                category=UserWarning,
+            )
+
+    return os.environ.copy()
+
+
+def parse_args():
+    parser = ArgumentParser()
+
+    base_group = parser.add_argument_group("Base Parameters")
+
+    base_group.add_argument(
+        "--master",
+        type=str,
+        default=None,
+        help="the master/rendezvous server, ip:port",
+    )
+
+    base_group.add_argument(
+        "--legacy", type=strtobool, default=False, help="use legacy launch"
+    )
+
+    base_group.add_argument(
+        "--rank", type=int, default=-1, help="the node rank"
+    )
+
+    base_group.add_argument(
+        "--log_level", type=str, default="INFO", help="log level. Default INFO"
+    )
+
+    base_group.add_argument(
+        "--log_overwrite",
+        type=strtobool,
+        default=False,
+        help="overwrite exits logfiles. Default False",
+    )
+
+    base_group.add_argument(
+        "--sort_ip",
+        type=strtobool,
+        default=False,
+        help="rank node by ip. Default False",
+    )
+
+    base_group.add_argument(
+        "--enable_gpu_log",
+        type=strtobool,
+        default=True,
+        help="enable capture gpu log while running. Default True",
+    )
+
+    base_group.add_argument(
+        "--nnodes",
+        type=str,
+        default="1",
+        help="the number of nodes, i.e. pod/node number",
+    )
+
+    base_group.add_argument(
+        "--nproc_per_node",
+        type=int,
+        default=None,
+        help="the number of processes in a pod",
+    )
+
+    base_group.add_argument(
+        "--log_dir",
+        type=str,
+        default="log",
+        help="the path for each process's log. Default ./log",
+    )
+    base_group.add_argument(
+        "--run_mode",
+        type=str,
+        default=None,
+        help="run mode of the job, collective/ps/ps-heter",
+    )
+
+    base_group.add_argument(
+        "--job_id",
+        type=str,
+        default="default",
+        help="unique id of the job. Default default",
+    )
+
+    base_group.add_argument(
+        "--devices",
+        "--gpus",
+        "--npus",
+        "--xpus",
+        type=str,
+        default=None,
+        help="accelerate devices. as --gpus,npus,xpus",
+    )
+
+    base_group.add_argument("--host", type=str, default=None, help="host ip")
+
+    base_group.add_argument(
+        "--ips",
+        type=str,
+        default=None,
+        help="nodes ips, e.g. 10.10.1.1,10.10.1.2",
+    )
+
+    base_group.add_argument(
+        "--start_port", type=int, default=6070, help="fix port start with"
+    )
+
+    base_group.add_argument(
+        "--auto_parallel_config",
+        type=str,
+        default=None,
+        help="auto parallel config file absolute path, the file should be json format",
+    )
+
+    base_group.add_argument(
+        "--auto_cluster_config",
+        type=strtobool,
+        default=0,
+        help="auto parallel auto cluster config switch",
+    )
+
+    base_group.add_argument(
+        "training_script",
+        type=str,
+        help="the full path of py script,"
+        "followed by arguments for the "
+        "training script",
+    )
+
+    base_group.add_argument(
+        "--auto_tuner_json",
+        type=str,
+        default=None,
+        help="auto tuner json file path",
+    )
+
+    base_group.add_argument('training_script_args', nargs=REMAINDER)
+
+    ps_group = parser.add_argument_group("Parameter-Server Parameters")
+    # for parameter server
+    ps_group.add_argument(
+        "--servers", type=str, default='', help="servers endpoints full list"
+    )
+    ps_group.add_argument(
+        "--trainers", type=str, default='', help="trainers endpoints full list"
+    )
+
+    ps_group.add_argument(
+        "--trainer_num", type=int, default=None, help="number of trainers"
+    )
+    ps_group.add_argument(
+        "--server_num", type=int, default=None, help="number of servers"
+    )
+    ps_group.add_argument(
+        "--gloo_port", type=int, default=6767, help="gloo http port"
+    )
+    ps_group.add_argument(
+        "--with_gloo", type=str, default="1", help="use gloo or not"
+    )
+
+    # parameter elastic mode
+    elastic_group = parser.add_argument_group("Elastic Parameters")
+    elastic_group.add_argument(
+        "--max_restart",
+        type=int,
+        default=3,
+        help="the times can restart. Default 3",
+    )
+
+    elastic_group.add_argument(
+        "--elastic_level",
+        type=int,
+        default=-1,
+        help="elastic level: -1 disable, 0 failed exit, peers hold, 1 internal restart",
+    )
+
+    elastic_group.add_argument(
+        "--elastic_timeout",
+        type=int,
+        default=30,
+        help="seconds to wait before elastic job begin to train",
+    )
+
+    args = parser.parse_known_args()
+    env_rank = int(os.getenv('PADDLE_TRAINER_ID', -1))
+    if env_rank >= 0:
+        assert hasattr(args[0], "rank")
+        args[0].rank = env_rank
+    return args

python/paddle/distributed/launch/context/device.py

@@ -0,0 +1,181 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+# (TODO: GhostScreaming) It will be removed later.
+from paddle.base import core
+from paddle.base.core import get_all_custom_device_type
+from paddle.device import get_available_custom_device
+
+
+class DeviceType:
+    CPU = 'cpu'
+    GPU = 'gpu'
+    XPU = 'xpu'
+    IPU = 'ipu'
+    CUSTOM_DEVICE = 'custom_device'
+
+
+class Device:
+    def __init__(self, dtype=None, memory="", labels=""):
+        self._dtype = dtype
+        self._memory = memory
+        self._labels = labels
+
+    def __str__(self):
+        return ",".join(self._labels)
+
+    @property
+    def dtype(self):
+        return self._dtype
+
+    @property
+    def count(self):
+        return len(self._labels) or 1
+
+    @property
+    def memory(self):
+        return self._memory
+
+    @property
+    def labels(self):
+        return self._labels
+
+    @labels.setter
+    def labels(self, lbs):
+        if isinstance(lbs, str):
+            self._labels = lbs.split(',')
+        elif isinstance(lbs, list):
+            self._labels = lbs
+        else:
+            self._labels = []
+
+    def get_selected_device_key(self):
+        if self._dtype == DeviceType.CPU:
+            return 'FLAGS_selected_cpus'
+        if self._dtype == DeviceType.GPU:
+            return 'FLAGS_selected_gpus'
+        if self._dtype == DeviceType.XPU:
+            return 'FLAGS_selected_xpus'
+        if self._dtype == DeviceType.IPU:
+            return 'FLAGS_selected_ipus'
+        if self._dtype == DeviceType.CUSTOM_DEVICE:
+            custom_device_types = get_all_custom_device_type()
+            device_type = (
+                str(custom_device_types[0]) if custom_device_types else ""
+            )
+            return f'FLAGS_selected_{device_type}s'
+        return 'FLAGS_selected_devices'
+
+    def get_selected_devices(self, devices=''):
+        '''
+        return the device label/id relative to the visible devices
+        '''
+        if not devices:
+            return [str(x) for x in range(0, len(self._labels))]
+        else:
+            devs = [x.strip() for x in devices.split(',')]
+            return [str(self._labels.index(d)) for d in devs]
+
+    def get_custom_device_envs(self):
+        custom_device_types = get_all_custom_device_type()
+        device_type = str(custom_device_types[0]) if custom_device_types else ""
+        return {
+            'PADDLE_DISTRI_BACKEND': 'xccl',
+            'PADDLE_XCCL_BACKEND': device_type,
+        }
+
+    @classmethod
+    def parse_device(self):
+        dev = Device()
+        visible_devices = None
+        custom_device_types = get_all_custom_device_type()
+        if custom_device_types:
+            dev._dtype = DeviceType.CUSTOM_DEVICE
+            device_type = str(custom_device_types[0])
+            visible_devices_str = f'{device_type.upper()}_VISIBLE_DEVICES'
+            if visible_devices_str in os.environ:
+                visible_devices = os.getenv(visible_devices_str)
+        elif 'XPULINK_VISIBLE_DEVICES' in os.environ:
+            dev._dtype = DeviceType.XPU
+            visible_devices = os.getenv("XPULINK_VISIBLE_DEVICES")
+        elif 'XPU_VISIBLE_DEVICES' in os.environ:
+            dev._dtype = DeviceType.XPU
+            visible_devices = os.getenv("XPU_VISIBLE_DEVICES")
+        elif 'CUDA_VISIBLE_DEVICES' in os.environ:
+            if core.is_compiled_with_xpu():
+                dev._dtype = DeviceType.XPU
+            else:
+                dev._dtype = DeviceType.GPU
+            visible_devices = os.getenv("CUDA_VISIBLE_DEVICES")
+
+        if visible_devices is not None and visible_devices != 'all':
+            dev._labels = visible_devices.split(',')
+        else:
+            return self.detect_device()
+
+        return dev
+
+    @classmethod
+    def detect_device(self):
+        def get_custom_devices_count(device_type):
+            all_custom_devices = get_available_custom_device()
+            all_custom_devices = [
+                device.split(':')[0] for device in all_custom_devices
+            ]
+            custom_devices_count = all_custom_devices.count(device_type)
+            return custom_devices_count
+
+        dev = Device()
+        num = 0
+        visible_devices = None
+        custom_device_types = get_all_custom_device_type()
+        if custom_device_types:
+            custom_device_type = str(custom_device_types[0])
+            dev._dtype = DeviceType.CUSTOM_DEVICE
+            num = get_custom_devices_count(custom_device_type)
+            visible_devices_str = (
+                f'{custom_device_type.upper()}_VISIBLE_DEVICES'
+            )
+            if visible_devices_str in os.environ:
+                visible_devices = os.getenv(visible_devices_str)
+        elif core.is_compiled_with_cuda():
+            dev._dtype = DeviceType.GPU
+            num = core.get_cuda_device_count()
+            visible_devices = os.getenv("CUDA_VISIBLE_DEVICES")
+        elif core.is_compiled_with_xpu():
+            dev._dtype = DeviceType.XPU
+            num = core.get_xpu_device_count()
+            visible_devices = os.getenv("XPU_VISIBLE_DEVICES")
+        elif core.is_compiled_with_ipu():
+            dev._dtype = DeviceType.IPU
+            num = core.get_ipu_device_count()
+            # For IPUs, 'labels' is a list which contains the available numbers of IPU devices.
+            dev._labels = [str(x) for x in range(0, num + 1)]
+            return dev
+
+        if num == 0:
+            dev._dtype = DeviceType.CPU
+        elif visible_devices is None or visible_devices == "all":
+            dev._labels = [str(x) for x in range(0, num)]
+        else:
+            dev._labels = visible_devices.split(',')
+
+        return dev
+
+
+if __name__ == '__main__':
+    d = Device.parse_device()
+    print(d.get_selected_devices())

python/paddle/distributed/launch/context/event.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class Event:
+    def __init__(self, kind="status", message="", fatal=False):
+        self.kind = kind
+        self.message = message
+        self.fatal = fatal

python/paddle/distributed/launch/context/node.py

@@ -0,0 +1,99 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import random
+import socket
+import struct
+from contextlib import closing
+
+from .device import Device
+
+
+class Node:
+    def __init__(self):
+        # self.device = Device.detect_device()
+        self.device = Device.parse_device()
+        self.ip = self.get_host_ip()
+        self.free_ports = []
+        self._allocated_ports = []
+
+        port_range = os.getenv('PORT_RANGE', '35100:64000')
+        port_range = port_range.split(':')
+        self._port_start = int(port_range[0])
+        self._port_end = int(port_range[1])
+        self._port_cur = random.randint(self._port_start, self._port_end)
+
+    def get_host_ip(self):
+        try:
+            self.hostname = socket.gethostname()
+            self.ip = socket.gethostbyname(socket.getfqdn(self.hostname))
+            return self.ip
+        except:
+            return '127.0.0.1'
+
+    def get_free_ports(self, n=1, rank=0):
+        if os.environ.get('FLAGS_FIXED_PORT') is None:
+            free_ports = [self.get_free_port() for i in range(n)]
+            self.free_ports += free_ports
+        else:
+            start_port = int(os.environ.get('FLAGS_FIXED_PORT'))
+            free_ports = list(
+                range(start_port + rank, start_port + rank + n, 1)
+            )
+            self.free_ports += free_ports
+        return free_ports
+
+    def get_ports_occupied(self):
+        return self.free_ports
+
+    def _get_free_port(self, port=0):
+        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+            s.setsockopt(
+                socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)
+            )
+            try:
+                s.bind(('', port))
+                return s.getsockname()[1]
+            except:
+                return -1
+
+    def _update_port_cur(self):
+        self._port_cur += 1
+        if self._port_cur > self._port_end:
+            self._port_cur = self._port_start
+
+    def get_free_port(self):
+        for _ in range(100):
+            ret = self._get_free_port(self._port_cur)
+            if ret > 0:
+                self._update_port_cur()
+                return ret
+            else:
+                self._update_port_cur()
+
+        return self._port_cur
+
+    @classmethod
+    def is_server_ready(self, ip, port):
+        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
+            # sock.settimeout(0.01)
+            # sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            if hasattr(socket, 'SO_REUSEPORT'):
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+            result = sock.connect_ex((ip, int(port)))
+            if result == 0:
+                return True
+            else:
+                return False

python/paddle/distributed/launch/context/resource.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class Resource:
+    def __init__(self):
+        self.devices = []

python/paddle/distributed/launch/context/status.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class Status:
+    UNINIT = "uninit"
+    READY = "ready"
+    RUNNING = "running"
+    FAILED = "failed"
+    TERMINATING = "terminating"
+    RESTARTING = "restarting"
+    UNKNOWN = "unknown"
+    COMPLETED = "completed"
+    DONE = "done"  # should exit whatever status
+
+    def __init__(self):
+        self._current_status = None
+
+    def current(self):
+        return self._current_status
+
+    def is_running(self):
+        return self._current_status == self.RUNNING
+
+    def is_restarting(self):
+        return self._current_status == self.RESTARTING
+
+    def is_done(self):
+        if self._current_status in [self.DONE, self.COMPLETED, self.FAILED]:
+            return True
+        else:
+            return False
+
+    def run(self):
+        self._current_status = self.RUNNING
+
+    def fail(self):
+        self._current_status = self.FAILED
+
+    def complete(self):
+        self._current_status = self.COMPLETED
+
+    def restart(self):
+        self._current_status = self.RESTARTING
+
+    def done(self):
+        self._current_status = self.DONE