chore: import upstream snapshot with attribution

python/paddle/distributed/launch/utils/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

python/paddle/distributed/launch/utils/etcd_client.py

@@ -0,0 +1,180 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import time
+
+import etcd3
+
+
+class ETCDClient:
+    def __init__(self, host, port, retry_times=20):
+        self.retry_times = retry_times
+        times = 0
+        while times < self.retry_times:
+            try:
+                self.client = etcd3.client(host=host, port=port)
+                break
+            except Exception as e:
+                times += 1
+                logging.info(
+                    f"Initialize etcd client failed with exception {e}, retry after 1 second."
+                )
+                time.sleep(1)
+
+        if times >= self.retry_times:
+            raise ValueError(
+                f"Initialize etcd client failed failed after {self.retry_times} times."
+            )
+
+    def put(self, key, value, lease=None, prev_kv=False):
+        times = 0
+        while times < self.retry_times:
+            try:
+                return self.client.put(key, value, lease, prev_kv)
+            except Exception as e:
+                times += 1
+                logging.info(
+                    f"Put failed with exception {e}, retry after 1 second."
+                )
+                time.sleep(1)
+
+        if times >= self.retry_times:
+            raise ValueError(f"Put failed after {self.retry_times} times.")
+
+    def get(self, key):
+        times = 0
+        while times < self.retry_times:
+            try:
+                return self.client.get(key)
+
+            except Exception as e:
+                times += 1
+                logging.info(
+                    f"Get {key} failed with exception {e}, retry after 1 second."
+                )
+                time.sleep(1)
+
+        if times >= self.retry_times:
+            raise ValueError(
+                f"Get {key} failed after {self.retry_times} times."
+            )
+
+    def delete(self, key, prev_kv=False, return_response=False):
+        times = 0
+        while times < self.retry_times:
+            try:
+                return self.client.delete(key, prev_kv, return_response)
+                break
+            except Exception as e:
+                times += 1
+                logging.info(
+                    f"Delete {key} failed with exception {e}, retry after 1 second."
+                )
+                time.sleep(1)
+
+        if times >= self.retry_times:
+            raise ValueError(
+                f"Delete {key} failed after {self.retry_times} times."
+            )
+
+    def get_prefix(self, key_prefix, sort_order=None, sort_target='key'):
+        times = 0
+        while times < self.retry_times:
+            try:
+                return self.client.get_prefix(key_prefix)
+                break
+            except Exception as e:
+                times += 1
+                logging.info(
+                    f"Get prefix {key_prefix} failed with exception {e}, retry after 1 second."
+                )
+                time.sleep(1)
+
+        if times >= self.retry_times:
+            raise ValueError(
+                f"Get prefix {key_prefix} failed after {self.retry_times} times."
+            )
+
+    def delete_prefix(self, prefix):
+        times = 0
+        while times < self.retry_times:
+            try:
+                return self.client.delete_prefix(prefix)
+                break
+            except Exception as e:
+                times += 1
+                logging.info(
+                    f"Delete prefix {prefix} failed with exception {e}, retry after 1 second."
+                )
+                time.sleep(1)
+
+        if times >= self.retry_times:
+            raise ValueError(
+                f"Delete prefix {prefix} failed after {self.retry_times} times."
+            )
+
+    def lease(self, ttl, lease_id=None):
+        times = 0
+        while times < self.retry_times:
+            try:
+                return self.client.lease(ttl, lease_id)
+                break
+            except Exception as e:
+                times += 1
+                logging.info(
+                    f"Lease failed with exception {e}, retry after 1 second."
+                )
+                time.sleep(1)
+
+        if times >= self.retry_times:
+            raise ValueError(f"Lease failed after {self.retry_times} times.")
+
+    def add_watch_prefix_callback(self, key_prefix, callback, **kwargs):
+        times = 0
+        while times < self.retry_times:
+            try:
+                return self.client.add_watch_prefix_callback(
+                    key_prefix, callback, **kwargs
+                )
+                break
+            except Exception as e:
+                times += 1
+                logging.info(
+                    f"Add watch prefix callback failed with exception {e}, retry after 1 second."
+                )
+                time.sleep(1)
+
+        if times >= self.retry_times:
+            raise ValueError(
+                f"Add watch prefix callback failed after {self.retry_times} times."
+            )
+
+    def cancel_watch(self, watch_id):
+        times = 0
+        while times < self.retry_times:
+            try:
+                return self.client.cancel_watch(watch_id)
+                break
+            except Exception as e:
+                times += 1
+                logging.info(
+                    f"Cancel watch failed with exception {e}, retry after 1 second."
+                )
+                time.sleep(1)
+
+        if times >= self.retry_times:
+            raise ValueError(
+                f"Cancel watch failed after {self.retry_times} times."
+            )

python/paddle/distributed/launch/utils/kv_client.py

@@ -0,0 +1,96 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+
+import httpx
+
+
+class KVClient:
+    def __init__(self, endpoint='localhost:2379'):
+        self.endpoint = (
+            endpoint if endpoint.startswith("http://") else f"http://{endpoint}"
+        )
+
+    def put(self, key, value):
+        key = key if key.startswith('/') else f"/{key}"
+        u = f"{self.endpoint}{key}"
+        try:
+            r = httpx.post(u, data=value, timeout=None, follow_redirects=True)
+            if r.status_code == 200:
+                return True
+            else:
+                return False
+        except:
+            return False
+
+    def get(self, key):
+        key = key if key.startswith('/') else f"/{key}"
+        u = f"{self.endpoint}{key}"
+        try:
+            r = httpx.get(u, timeout=None, follow_redirects=True)
+            if r.status_code == 200:
+                ret = r.json()
+                return ret.get(key, '')
+            else:
+                return "error"
+        except:
+            return ""
+
+    def get_prefix(self, key):
+        key = key if key.startswith('/') else f"/{key}"
+        u = f"{self.endpoint}{key}"
+        try:
+            r = httpx.get(u, timeout=None, follow_redirects=True)
+            if r.status_code == 200:
+                return r.json()
+        except:
+            return ""
+
+    def delete(self, key):
+        key = key if key.startswith('/') else f"/{key}"
+        u = f"{self.endpoint}{key}"
+        try:
+            r = httpx.delete(u, timeout=None, follow_redirects=True)
+            if r.status_code == 200:
+                return True
+            else:
+                return False
+        except:
+            return False
+
+    def wait_server_ready(self, timeout=3):
+        end = time.time() + timeout
+        while time.time() < end:
+            if self.get("/healthy") == "ok":
+                return True
+
+
+if __name__ == '__main__':
+    cli = KVClient("http://localhost:8090")
+    data = {"/workers/1": "rank1", "/workers/2": "rank2"}
+    for k, v in data.items():
+        cli.put(k, v)
+    x = cli.get_prefix("/workers")
+    print(x)
+    for k, v in data.items():
+        assert x[k] == v
+
+    cli.put("key", "value")
+    print(cli.get("key"))
+    assert cli.get("key") == "value"
+    cli.delete("key")
+    print(cli.get("/key"))
+    print(cli.get("/healthy"))
+    assert cli.get("/healthy") == "ok"

python/paddle/distributed/launch/utils/kv_server.py

@@ -0,0 +1,128 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import http.server as SimpleHTTPServer
+import json
+import threading
+from http.server import HTTPServer
+from multiprocessing import Process
+
+from .topology import SingleNodeTopology
+
+
+class KVHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+    def do_GET(self):
+        with self.server.kv_lock:
+            ret = {}
+            for k, v in self.server.kv.items():
+                if k.startswith(self.path):
+                    ret[k] = v.decode(encoding="utf-8")
+            if ret:
+                self.output(200, json.dumps(ret).encode("utf-8"))
+            else:
+                self.output(404)
+
+    def do_PUT(self):
+        self.do_POST()
+
+    def do_POST(self):
+        content_length = int(self.headers['Content-Length'] or 0)
+        try:
+            value = self.rfile.read(content_length)
+            with self.server.kv_lock:
+                self.server.kv[self.path] = value
+                self.output(200)
+                return
+        except:
+            self.output(500)
+
+    def do_DELETE(self):
+        with self.server.kv_lock:
+            if self.path in self.server.kv:
+                del self.server.kv[self.path]
+                self.output(200)
+            else:
+                self.output(404)
+
+    def output(self, code, value=''):
+        self.send_response(code)
+        self.send_header("Content-Length", len(value))
+        self.send_header("Content-Type", "application/json; charset=utf8")
+        self.end_headers()
+        if value:
+            self.wfile.write(value)
+
+    def log_message(self, format, *args):
+        return
+
+
+class KVServer(HTTPServer):
+    def __init__(self, port):
+        super().__init__(('', port), KVHandler)
+        self.kv_lock = threading.Lock()
+        self.kv = {'/healthy': b'ok'}
+        self.port = port
+        self.stopped = False
+        self.started = False
+        self.node_topo = None
+
+    def start(self):
+        self.listen_thread = threading.Thread(target=self.serve_forever)
+        self.listen_thread.start()
+        self.started = True
+
+    def stop(self):
+        self.shutdown()
+        self.listen_thread.join()
+        self.server_close()
+        self.stopped = True
+
+    def get_topology(self):
+        if self.node_topo is None:
+            self.node_topo = SingleNodeTopology()
+        self.node_topo.detect()
+        return self.node_topo.json_object
+
+
+class PKVServer:
+    def __init__(self, port):
+        self._server = KVServer(port)
+
+    def start(self):
+        self.proc = Process(target=self._server.start)
+        self.proc.daemon = True
+        self.proc.start()
+
+    def stop(self):
+        self._server.stop()
+        self.proc.join()
+
+    @property
+    def started(self):
+        return self._server.started
+
+    @property
+    def stopped(self):
+        return self._server.stopped
+
+
+if __name__ == '__main__':
+    # kv = PKVServer(8090)
+    kv = KVServer(8090)
+    kv.start()
+    import time
+
+    # print("serve at 8090 for 600 s")
+
+    time.sleep(600)

python/paddle/distributed/launch/utils/nvsmi.py

@@ -0,0 +1,256 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import re
+import shutil
+import subprocess
+import time
+
+import paddle
+from paddle.base import core
+
+
+class Info:
+    def __repr__(self):
+        return str(self.__dict__)
+
+    def json(self):
+        return json.dumps(self.__dict__)
+
+    def dict(self):
+        return self.__dict__
+
+    def str(self, keys=None):
+        if keys is None:
+            keys = self.__dict__.keys()
+
+        if isinstance(keys, str):
+            keys = keys.split(',')
+
+        values = [str(self.__dict__.get(k, '')) for k in keys]
+        return ",".join(values)
+
+
+def query_smi(query=None, query_type="gpu", index=None, dtype=None):
+    """
+    query_type: gpu/compute
+    """
+
+    if not has_nvidia_smi():
+        return []
+
+    cmd = ["nvidia-smi", "--format=csv,noheader,nounits"]
+    if isinstance(query, list) and query_type == "gpu":
+        cmd.extend(["--query-gpu={}".format(",".join(query))])
+    elif isinstance(query, list) and query_type.startswith("compute"):
+        cmd.extend(["--query-compute-apps={}".format(",".join(query))])
+    else:
+        return
+
+    if isinstance(index, list) and len(index) > 0:
+        cmd.extend(["--id={}".format(",".join(index))])
+    if not isinstance(dtype, list) or len(dtype) != len(query):
+        dtype = [str] * len(query)
+
+    output = subprocess.check_output(cmd, timeout=3)
+    lines = output.decode("utf-8").split(os.linesep)
+    ret = []
+    for line in lines:
+        if not line:
+            continue
+        info = Info()
+        for k, v, d in zip(query, line.split(", "), dtype):
+            setattr(info, k.replace(".", "_"), d(v))
+        ret.append(info)
+    return ret
+
+
+def query_rocm_smi(query=None, index=None, dtype=None, mem=32150):
+    if not has_rocm_smi():
+        return []
+
+    cmd = ["rocm-smi"]
+
+    if not isinstance(dtype, list) or len(dtype) != len(query):
+        dtype = [str] * len(query)
+
+    output = subprocess.check_output(cmd, timeout=3)
+    lines = output.decode("utf-8").split(os.linesep)
+    ret = []
+    for line in lines:
+        if not line:
+            continue
+        if len(line.split()) != 8 or "DCU" in line.split():
+            continue
+        info = Info()
+        line = line.split()
+        line = [
+            line[0],
+            line[7][: len(line[7]) - 1],
+            mem,
+            mem * float(line[6][: len(line[6]) - 1]) / 100,
+            mem - mem * float(line[6][: len(line[6]) - 1]) / 100,
+            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
+        ]
+        for k, v, d in zip(query, line, dtype):
+            setattr(info, k.replace(".", "_"), d(v))
+        ret.append(info)
+    return ret
+
+
+def query_npu_smi(query=None, index=None, dtype=None):
+    if not has_npu_smi():
+        return []
+
+    cmd = ["npu-smi", "info"]
+
+    if not isinstance(dtype, list) or len(dtype) != len(query):
+        dtype = [str] * len(query)
+
+    output = subprocess.check_output(cmd, timeout=3)
+    lines = output.decode("utf-8").split(os.linesep)
+    ret = []
+    i = 0
+
+    for line in lines:
+        if not line:
+            continue
+        result = re.split(r',|/|\s+|\|', line)
+        # result = [item for item in result if item]
+        length = len(result)
+        if length not in [18, 19] or "NPU" in result:
+            continue
+        result = [item for item in result if item]
+        info = Info()
+        result = [
+            i,
+            result[2],
+            result[6],
+            float(result[5]),
+            (float(result[6]) - float(result[5])),
+            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
+        ]
+        i += 1
+        for k, v, d in zip(query, result, dtype):
+            setattr(info, k.replace(".", "_"), d(v))
+        ret.append(info)
+    return ret
+
+
+def query_xpu_smi(query=None, index=None, dtype=None):
+    if (
+        not hasattr(core, "get_xpu_device_count")
+        or core.get_xpu_device_count() == 0
+    ):
+        return []
+    if not isinstance(dtype, list) or len(dtype) != len(query):
+        dtype = [str] * len(query)
+    if not isinstance(index, list) or len(index) == 0:
+        index = list(range(core.get_xpu_device_count()))
+    ret = []
+    for dev_id in index:
+        dev_id = int(dev_id)
+        utilization_xpu = core.get_xpu_device_utilization_rate(dev_id)
+        mem_total = (
+            core.get_xpu_device_total_memory(dev_id) / 1024 / 1024
+        )  # with MB
+        mem_used = (
+            core.get_xpu_device_used_memory(dev_id) / 1024 / 1024
+        )  # with MB
+        result = [
+            dev_id,
+            utilization_xpu,
+            mem_total,
+            mem_used,
+            (mem_total - mem_used),
+            time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
+        ]
+        info = Info()
+        for k, v, d in zip(query, result, dtype):
+            setattr(info, k.replace(".", "_"), d(v))
+        ret.append(info)
+    return ret
+
+
+def get_gpu_info(index=None):
+    q = "index,uuid,driver_version,name,gpu_serial,display_active,display_mode".split(
+        ","
+    )
+    d = [int, str, str, str, str, str, str]
+    index = (
+        index
+        if index is None or isinstance(index, list)
+        else str(index).split(",")
+    )
+
+    return query_smi(q, index=index, dtype=d)
+
+
+def get_gpu_util(index=None):
+    q = "index,utilization.gpu,memory.total,memory.used,memory.free,timestamp".split(
+        ","
+    )
+    d = [int, int, int, int, int, str]
+    index = (
+        index
+        if index is None or isinstance(index, list)
+        else str(index).split(",")
+    )
+    if paddle.device.is_compiled_with_rocm():
+        return query_rocm_smi(q, index=index, dtype=d)
+    elif paddle.device.is_compiled_with_custom_device('npu'):
+        return query_npu_smi(q, index=index, dtype=d)
+    elif paddle.is_compiled_with_xpu():
+        return query_xpu_smi(q, index=index, dtype=d)
+    return query_smi(q, index=index, dtype=d)
+
+
+def get_gpu_process(index=None):
+    q = "pid,process_name,gpu_uuid,gpu_name,used_memory".split(",")
+    d = [int, str, str, str, int]
+    index = (
+        index
+        if index is None or isinstance(index, list)
+        else str(index).split(",")
+    )
+
+    return query_smi(q, index=index, query_type="compute", dtype=d)
+
+
+def has_nvidia_smi():
+    return shutil.which("nvidia-smi")
+
+
+def has_rocm_smi():
+    return shutil.which("rocm-smi")
+
+
+def has_npu_smi():
+    return shutil.which("npu-smi")
+
+
+def has_xpu_smi():
+    return shutil.which("xpu-smi")
+
+
+if __name__ == '__main__':
+    print(get_gpu_info(0))
+    print(get_gpu_util(0))
+    print(get_gpu_process(0))
+
+    u = get_gpu_util()
+    for i in u:
+        print(i.str())

python/paddle/distributed/launch/utils/process_context.py

@@ -0,0 +1,116 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import signal
+import subprocess
+import sys
+import time
+
+LIMIT_LEN_ENVS = ["TRAINER_IP_PORT_LIST", "PADDLE_TRAINER_ENDPOINTS"]
+
+
+class ProcessContext:
+    def __init__(
+        self,
+        cmd,
+        env=os.environ,
+        out=sys.stdout,
+        err=sys.stderr,
+        group=True,
+        preexec_fn=None,
+        shell=False,
+    ):
+        self._cmd = cmd
+        self._env = env
+        self._preexec_fn = preexec_fn
+        self._stdout = out
+        self._stderr = err
+        self._group = group if os.name != 'nt' else False
+        self._proc = None
+        self._code = None
+        self._shell = shell
+
+    def _start(self):
+        pre_fn = os.setsid if self._group else None
+        log_dir = self._env["PADDLE_LOG_DIR"]
+        os.makedirs(log_dir, exist_ok=True)
+
+        rank = self._env.get("PADDLE_TRAINER_ID")
+        if rank is not None:
+            rank = int(rank)
+            backup_env_path = str(
+                os.path.join(log_dir, f'backup_env.{rank}.json')
+            )
+            envs = {"PADDLE_BACKUP_ENV_PATH": backup_env_path}
+
+            max_len = int(os.getenv('PADDLE_ENV_LIMIT_LEN', 48000))
+            for k, v in self._env.items():
+                if k not in LIMIT_LEN_ENVS or len(v) < max_len:
+                    envs[k] = v
+
+            with open(backup_env_path, 'w') as f:
+                json.dump(dict(self._env), f, indent=4, sort_keys=True)
+        else:
+            envs = self._env
+
+        self._proc = subprocess.Popen(
+            self._cmd,
+            env=envs,
+            stdout=self._stdout,
+            stderr=self._stderr,
+            preexec_fn=self._preexec_fn or pre_fn,
+            shell=self._shell,
+        )
+
+    def _close_std(self):
+        try:
+            if not self._stdout.isatty():
+                self._stdout.close()
+
+            if not self._stderr.isatty():
+                self._stderr.close()
+        except:
+            pass
+
+    def alive(self):
+        return self._proc and self._proc.poll() is None
+
+    def exit_code(self):
+        return self._proc.poll() if self._proc else None
+
+    def start(self):
+        self._start()
+
+    def terminate(self, force=False, max_retry=3):
+        for i in range(max_retry):
+            if self.alive():
+                if self._group:
+                    os.killpg(os.getpgid(self._proc.pid), signal.SIGTERM)
+                else:
+                    self._proc.terminate()
+                time.sleep(0.2)
+            else:
+                break
+
+        if force and self.alive():
+            self._proc.kill()
+
+        self._close_std()
+
+        return self.alive()
+
+    def wait(self, timeout=None):
+        self._proc.wait(timeout)

python/paddle/distributed/launch/utils/topology.py

@@ -0,0 +1,362 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import subprocess
+import warnings
+
+import paddle
+
+
+def call_cmd(cmd, err_msg, default_value):
+    process = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        universal_newlines=True,
+        shell=True,
+    )
+    stdout, stderr = process.communicate()
+    if stderr:
+        warnings.warn(err_msg)
+        stdout = default_value
+
+    return stdout
+
+
+class SingleNodeTopology:
+    def __init__(self):
+        self.pcie_latency = 0.0
+        self.pcie_bandwidth = float('inf')
+        self.nvlink_bandwidth = -1.0
+        self.nb_devices = 8
+
+        self.machine = {}
+        self.devices = []
+        self.links = []
+        self.json_object = None
+
+    def calculate_cpu_flops(self):
+        # Get number sockets
+        cmd = "lscpu | grep 'Socket(s)' | awk '{print $NF}'"
+        err_msg = "Failed to get number of sockets"
+        default_value = 4
+        nb_sockets = call_cmd(cmd, err_msg, default_value)
+
+        # Get number of cores per socket
+        cmd = "lscpu | grep 'Core(s) per socket' | awk '{print $NF}'"
+        err_msg = "Failed to get number of cores per socket"
+        default_value = 20
+        nb_cores_per_socket = call_cmd(cmd, err_msg, default_value)
+
+        # Get clock speed
+        cmd = "lscpu | grep GHz | awk -F '@' '{print $NF}' | awk -F 'G' '{print $1}'"
+        err_msg = "Failed to get cpu clock rate"
+        default_value = 2.4
+        clock_rate = call_cmd(cmd, err_msg, default_value)
+
+        # Get number of FMA units
+        # TODO(changtao02): find a way to detect this value
+        nb_fmas = 2
+
+        # Get SIMD width
+        simd_width_sp = 0
+        simd_width_dp = 0
+
+        cmd = "lscpu | grep sse"
+        err_msg = "Failed to get cpu vector size"
+        default_value = "sse"
+        vector_size = call_cmd(cmd, err_msg, default_value)
+
+        if vector_size:
+            simd_width_sp = 4  # 128 / 32
+            simd_width_dp = 2  # 128 / 64
+
+        cmd = "lscpu | grep avx2"
+        err_msg = "Failed to get cpu vector size"
+        default_value = "avx2"
+        vector_size = call_cmd(cmd, err_msg, default_value)
+
+        if vector_size:
+            simd_width_sp = 8  # 256 / 32
+            simd_width_dp = 4  # 256 / 64
+
+        cmd = "lscpu | grep avx512"
+        err_msg = "Failed to get cpu vector size"
+        default_value = "avx512"
+        vector_size = call_cmd(cmd, err_msg, default_value)
+
+        if vector_size:
+            simd_width_sp = 16  # 512 / 32
+            simd_width_dp = 8  # 512 / 64
+
+        gflops_per_element = (
+            int(nb_sockets)
+            * int(nb_cores_per_socket)
+            * float(clock_rate)
+            * nb_fmas
+        )
+        sp_gflops = gflops_per_element * simd_width_sp
+        dp_gflops = gflops_per_element * simd_width_dp
+
+        self.machine['sp_gflops'] = sp_gflops
+        self.machine['dp_gflops'] = dp_gflops
+
+    def pcie_gen2bandwidth(self, pcie_generation):
+        if pcie_generation == 1:
+            return 0.25
+        elif pcie_generation == 2:
+            return 0.5
+        elif pcie_generation == 3:
+            return 1.0
+        elif pcie_generation == 4:
+            return 2.0
+        elif pcie_generation == 5:
+            return 4.0
+        elif pcie_generation == 6:
+            return 8.0
+
+    def model2gflops(self, model):
+        if "H100" in model and "SXM5" in model:
+            return 60000, 30000
+        elif "H100" in model and "PCIe" in model:
+            return 48000, 24000
+        elif "A100" in model:
+            return 19500, 9700
+        elif "A800" in model:
+            return 19500, 9700
+        elif "V100" in model:
+            return 15700, 7800
+        elif "P100" in model:
+            return 10600, 5300
+
+    def get_link_bandwidth(self, source_id, target_id):
+        # Get link type
+        row_id = 2 + source_id
+        column_id = 2 + target_id
+
+        cmd = (
+            "cat /tmp/matrix.txt | awk 'FNR=="
+            + str(row_id)
+            + " {print $"
+            + str(column_id)
+            + "}'"
+        )
+        err_msg = "Failed to get topo matrix"
+        default_value = "NVL"
+        link_type = call_cmd(cmd, err_msg, default_value)
+
+        link_bandwidth = self.pcie_bandwidth
+
+        if "NV" in link_type:
+            if self.nvlink_bandwidth == -1.0:
+                cmd = "nvidia-smi nvlink -s -i 0 | tail -n 1 | awk '{print $3}'"
+                err_msg = "Failed to get nvlink bandwidth"
+                default_value = "25"
+                self.nvlink_bandwidth = float(
+                    call_cmd(cmd, err_msg, default_value)
+                )
+
+            link_bandwidth = int(link_type[2:]) * self.nvlink_bandwidth
+            link_type = "NVL"
+
+        return link_type, link_bandwidth
+
+    def get_host_info(self):
+        # Get hostname
+        cmd = "hostname -s"
+        err_msg = "Failed to get hostname"
+        default_value = "localhost"
+        hostname = call_cmd(cmd, err_msg, default_value).strip()
+
+        # Get ip address
+        cmd = "hostname -i"
+        err_msg = "Failed to get host ip address"
+        default_value = "127.0.0.1"
+        ip_addr = call_cmd(cmd, err_msg, default_value).strip()
+
+        # Get CPU memory (GB)
+        cmd = "cat /proc/meminfo | grep 'MemAvailable' | awk -F ':' '{print $NF}' | awk '{print $1}'"
+        err_msg = "Failed to get cpu memory"
+        default_value = "41366484"
+        cpu_memory = int(call_cmd(cmd, err_msg, default_value)) // 1e6
+
+        # Get single-point flops and double-point flops (GFLOPs)
+        self.calculate_cpu_flops()
+
+        self.machine['hostname'] = hostname
+        self.machine['addr'] = ip_addr
+        self.machine['memory'] = cpu_memory
+
+    def get_device_info(self):
+        # Get device count
+        cmd = "nvidia-smi -L | wc -l"
+        err_msg = "Failed to get device count"
+        default_value = "8"
+        self.nb_devices = int(call_cmd(cmd, err_msg, default_value))
+
+        local_size = int(os.getenv("PADDLE_LOCAL_SIZE"))
+        if local_size < self.nb_devices:
+            self.nb_devices = local_size
+        # Get PCIe latency and bandwidth (ms, GB/s)
+        for i in range(self.nb_devices):
+            cmd = (
+                "nvidia-smi --id="
+                + str(i)
+                + " --query-gpu=pcie.link.gen.max --format=csv,noheader"
+            )
+            err_msg = "Failed to get max pcie link generation"
+            default_value = "4"
+            pcie_generation = int(call_cmd(cmd, err_msg, default_value))
+
+            cmd = (
+                "nvidia-smi --id="
+                + str(i)
+                + " --query-gpu=pcie.link.width.max --format=csv,noheader"
+            )
+            err_msg = "Failed to get max pcie link width"
+            default_value = "16"
+            pcie_width = int(call_cmd(cmd, err_msg, default_value))
+
+            self.pcie_bandwidth = min(
+                self.pcie_bandwidth,
+                self.pcie_gen2bandwidth(pcie_generation) * pcie_width,
+            )
+
+        dev_global_ids = []
+        dev_local_ids = []
+        dev_types = []
+        dev_models = []
+        dev_memories = []  # GiB
+        dev_sp_gflops = []  # GB/s
+        dev_dp_gflops = []  # GB/s
+
+        # Get device info
+        rank_first = paddle.paddle.distributed.get_rank()
+        for i in range(self.nb_devices):
+            dev_global_ids.append(i + rank_first)
+            dev_local_ids.append(i)
+            dev_types.append("GPU")
+
+            cmd = (
+                "nvidia-smi --id="
+                + str(i)
+                + " --query-gpu=name --format=csv,noheader"
+            )
+            err_msg = "Failed to get device name"
+            default_value = "NVIDIA A100-SXM4-40GB"
+            dev_models.append(call_cmd(cmd, err_msg, default_value).strip())
+
+            cmd = (
+                "nvidia-smi --id="
+                + str(i)
+                + " --query-gpu=memory.free --format=csv,noheader | awk '{print $1}'"
+            )
+            err_msg = "Failed to get device available memory"
+            default_value = "40536"
+            dev_memories.append(
+                int(call_cmd(cmd, err_msg, default_value)) // 1e3
+            )
+
+            sp_gflops, dp_gflops = self.model2gflops(dev_models[i])
+            dev_sp_gflops.append(sp_gflops)
+            dev_dp_gflops.append(dp_gflops)
+
+        for i in range(len(dev_global_ids)):
+            device = {}
+            device['global_id'] = dev_global_ids[i]
+            device['local_id'] = dev_local_ids[i]
+            device['type'] = dev_types[i]
+            device['model'] = dev_models[i]
+            device['memory'] = dev_memories[i]
+            device['sp_gflops'] = dev_sp_gflops[i]
+            device['dp_gflops'] = dev_dp_gflops[i]
+            self.devices.append(device)
+
+        self.machine['latency'] = self.pcie_latency
+        self.machine['bandwidth'] = self.pcie_bandwidth
+        self.machine['device_type'] = dev_types[0]
+        self.machine['device_type_full'] = f"{dev_types[0]}-{dev_models[0]}"
+        self.machine['devices'] = self.devices
+
+    def get_link_info(self):
+        link_source_global_ids = []
+        link_target_global_ids = []
+        link_types = []
+        link_latencies = []  # ms
+        link_bandwidths = []  # GB/s
+
+        cmd = "nvidia-smi topo -m > /tmp/matrix.txt"
+        err_msg = "Failed to get topo matrix"
+        default_value = ""
+        call_cmd(cmd, err_msg, default_value)
+
+        rank_first = paddle.paddle.distributed.get_rank()
+        # Get link info between devices
+        for i in range(self.nb_devices):
+            for j in range(self.nb_devices):
+                if i == j:
+                    link_types.append("X")
+                    link_bandwidths.append(-1.0)
+                else:
+                    link_source_global_ids.append(i + rank_first)
+                    link_target_global_ids.append(j + rank_first)
+                    link_latencies.append(0.0)
+                    if i > j:
+                        index = j * self.nb_devices + i
+                        link_types.append(link_types[index])
+                        link_bandwidths.append(link_bandwidths[index])
+                    elif i < j:
+                        link_type, link_bandwidth = self.get_link_bandwidth(
+                            i, j
+                        )
+                        link_types.append(link_type)
+                        link_bandwidths.append(link_bandwidth)
+
+        for i in reversed(range(self.nb_devices)):
+            link_types.pop(i * self.nb_devices + i)
+            link_bandwidths.pop(i * self.nb_devices + i)
+
+        cmd = "rm /tmp/matrix.txt"
+        err_msg = "Failed to delete matrix.txt"
+        default_value = ""
+        # call_cmd(cmd, err_msg, default_value)
+
+        for i in range(len(link_types)):
+            link = {}
+            link['source_global_id'] = link_source_global_ids[i]
+            link['target_global_id'] = link_target_global_ids[i]
+            link['type'] = link_types[i]
+            link['latency'] = link_latencies[i]
+            link['bandwidth'] = link_bandwidths[i]
+            self.links.append(link)
+
+        self.machine['links'] = self.links
+
+    def detect(self):
+        # Get host info
+        self.get_host_info()
+
+        # Get device info
+        self.get_device_info()
+
+        # Get link info between devices
+        self.get_link_info()
+
+        self.json_object = json.dumps(self.machine, indent=4)
+
+    def dump(self, output_path):
+        with open(output_path, "w") as outfile:
+            json.dump(self.machine, outfile, indent=4)