chore: import upstream snapshot with attribution

python/paddle/amp/__init__.py

@@ -0,0 +1,113 @@
+#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from paddle.base import core
+from paddle.base.framework import (
+    _current_expected_place,
+    _get_paddle_place,
+)
+
+from . import (  # noqa: F401
+    accuracy_compare,
+    debugging,
+    grad_scaler,
+)
+from .amp_lists import (  # noqa: F401
+    black_list,
+    white_list,
+)
+from .auto_cast import (  # noqa: F401
+    amp_decorate,
+    amp_guard,
+    auto_cast,
+    autocast,
+    decorate,
+    get_autocast_dtype,
+    is_autocast_enabled,
+)
+from .grad_scaler import (  # noqa: F401
+    AmpScaler,
+    GradScaler,
+    OptimizerState,
+)
+
+__all__ = [
+    'auto_cast',
+    'GradScaler',
+    'decorate',
+    'is_float16_supported',
+    'is_bfloat16_supported',
+    'is_autocast_enabled',
+    'get_autocast_dtype',
+    'get_autocast_cpu_dtype',
+    'get_autocast_gpu_dtype',
+]
+
+get_autocast_cpu_dtype = get_autocast_dtype
+get_autocast_gpu_dtype = get_autocast_dtype
+
+
+def is_float16_supported(device: str | None = None) -> bool:
+    """
+    Determine whether the place supports float16 in the auto-mixed-precision training.
+
+    Args:
+        device (str|None, optional): Specify the running device.
+            It can be ``cpu``, ``gpu``, ``xpu``, ``gpu:x`` and ``xpu:x``,
+            where ``x`` is the index of the GPUs or XPUs. if device is None, the device is the current device. Default: None.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> paddle.amp.is_float16_supported()  # True or False
+            False
+    """
+
+    device = (
+        _current_expected_place()
+        if device is None
+        else _get_paddle_place(device)
+    )
+
+    return core.is_float16_supported(device)
+
+
+def is_bfloat16_supported(device: str | None = None) -> bool:
+    """
+    Determine whether the place supports bfloat16 in the auto-mixed-precision training.
+
+    Args:
+        device (str|None, optional): Specify the running device.
+            It can be ``cpu``, ``gpu``, ``xpu``, ``gpu:x`` and ``xpu:x``,
+            where ``x`` is the index of the GPUs or XPUs. if device is None, the device is the current device. Default: None.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> paddle.amp.is_bfloat16_supported()  # True or False
+            True
+    """
+
+    device = (
+        _current_expected_place()
+        if device is None
+        else _get_paddle_place(device)
+    )
+
+    return core.is_bfloat16_supported(device)

python/paddle/amp/accuracy_compare.py

@@ -0,0 +1,700 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+import numpy as np
+
+
+# Judge whether the value is within the range indicated by fp16
+def is_infinite(value, dtype=np.float16):
+    # return value > np.finfo(np.float16).max or value < np.finfo(np.float16).min
+    array = np.array([value]).astype(dtype)
+    return np.isinf(array) or np.isnan(array)
+
+
+# Judge whether the value of fp32 is equal to that of fp16
+def is_allclose(actual, expected, atol=1e-2, rtol=1e-2):
+    return np.allclose(
+        np.array([actual]), np.array([expected]), atol=atol, rtol=rtol
+    )
+
+
+class TensorInfo:
+    def __init__(self):
+        self.device = None
+        self.op_type = None
+        self.tensor_name = None
+        self.dtype = None
+        self.numel = None
+        self.max_value = None
+        self.min_value = None
+        self.mean_value = None
+        self.has_inf = None
+        self.has_nan = None
+        self.num_zero = None
+
+    def __str__(self):
+        return f"[TensorInfo] device={self.device}, op_type={self.op_type}, tensor_name={self.tensor_name}, dtype={self.dtype}, numel={self.numel}, num_inf={self.has_inf}, num_nan={self.has_nan}, num_zero={self.num_zero}, max_value={self.max_value:.6f}, min_value={self.min_value:.6f}, mean_value={self.mean_value:.6f}"
+
+    def key(
+        self,
+    ):
+        return self.op_type + "/" + self.tensor_name
+
+    def init_from_string(self, line):
+        try:
+            line_frags = line.strip().split(" ")
+            for frag in line_frags:
+                word_str = (
+                    frag.replace("[", "").replace("]", "").replace(",", "")
+                )
+                words = word_str.split("=")
+                if words[0] == "op":
+                    self.op_type = words[1]
+                elif words[0] == "device":
+                    self.device = words[1]
+                elif words[0] == "tensor":
+                    self.tensor_name = words[1]
+                elif words[0] == "dtype":
+                    self.dtype = words[1]
+                elif words[0] == "numel":
+                    self.numel = np.int64(words[1])
+                elif words[0] == "max":
+                    self.max_value = np.float32(words[1])
+                elif words[0] == "min":
+                    self.min_value = np.float32(words[1])
+                elif words[0] == "mean":
+                    self.mean_value = np.float32(words[1])
+                elif words[0] == "num_inf":
+                    self.has_inf = int(words[1])
+                elif words[0] == "num_nan":
+                    self.has_nan = int(words[1])
+                elif words[0] == "num_zero":
+                    self.num_zero = np.int64(words[1])
+        except Exception as e:
+            print(f"!! Error parsing {line}")
+        return self
+
+
+class MixedPrecisionTensorInfo:
+    def __init__(
+        self, fp32_tensor_info, fp16_tensor_info, fp32_idx=0, grad_scale=1.0
+    ):
+        self.is_normal = True
+        self.fp32_idx = fp32_idx
+
+        self.fp32_tensor_name = None
+        self.fp32_dtype = None
+        self.fp32_max_value = None
+        self.fp32_min_value = None
+        self.fp32_mean_value = None
+        self.fp32_num_zero = None
+        self.scaled_fp32_max_value = None
+        self.scaled_fp32_min_value = None
+
+        self.fp16_tensor_name = None
+        self.fp16_dtype = None
+        self.fp16_max_value = None
+        self.fp16_min_value = None
+        self.fp16_mean_value = None
+        self.fp16_num_zero = None
+        self.fp16_has_inf = None
+        self.fp16_has_nan = None
+
+        self.fp32_div_fp16_max_value = None
+        self.fp32_div_fp16_min_value = None
+        self.fp32_div_fp16_mean_value = None
+
+        if fp32_tensor_info is not None:
+            self.op_type = fp32_tensor_info.op_type
+            self.numel = fp32_tensor_info.numel
+            self.fp32_num_zero = fp32_tensor_info.num_zero
+            self.fp32_tensor_name = fp32_tensor_info.tensor_name
+            self.fp32_dtype = fp32_tensor_info.dtype
+            self.fp32_max_value = fp32_tensor_info.max_value
+            self.fp32_min_value = fp32_tensor_info.min_value
+            self.fp32_mean_value = fp32_tensor_info.mean_value
+            if "GRAD" in self.fp32_tensor_name:
+                self.scaled_fp32_max_value = (
+                    grad_scale * fp32_tensor_info.max_value
+                )
+                self.scaled_fp32_min_value = (
+                    grad_scale * fp32_tensor_info.min_value
+                )
+
+        if fp16_tensor_info is not None:
+            self.op_type = fp16_tensor_info.op_type
+            self.numel = fp16_tensor_info.numel
+            self.fp16_num_zero = fp16_tensor_info.num_zero
+            self.fp16_tensor_name = fp16_tensor_info.tensor_name
+            self.fp16_dtype = fp16_tensor_info.dtype
+            self.fp16_max_value = fp16_tensor_info.max_value
+            self.fp16_min_value = fp16_tensor_info.min_value
+            self.fp16_mean_value = fp16_tensor_info.mean_value
+            self.fp16_has_inf = fp16_tensor_info.has_inf
+            self.fp16_has_nan = fp16_tensor_info.has_nan
+
+        if fp32_tensor_info is not None and fp16_tensor_info is not None:
+            # Check whether the op name and data are equal
+            assert fp32_tensor_info.op_type == fp16_tensor_info.op_type
+            assert fp32_tensor_info.numel == fp16_tensor_info.numel, (
+                f"Error:\n\tFP32 Tensor Info:{fp32_tensor_info}\n\tFP16 Tensor Info:{fp16_tensor_info}"
+            )
+            # Fp16 divided by fp32
+            self.fp32_div_fp16_max_value = self._div(
+                self.fp16_max_value, self.fp32_max_value
+            )
+            self.fp32_div_fp16_min_value = self._div(
+                self.fp16_min_value, self.fp32_min_value
+            )
+            self.fp32_div_fp16_mean_value = self._div(
+                self.fp16_mean_value, self.fp32_mean_value
+            )
+
+        self._check_normal()
+
+    def __str__(self):
+        def _float_str(value):
+            return f"{value:.6f}" if value is not None else value
+
+        debug_str = f"[MixedPrecisionTensorInfo] op_type={self.op_type}, numel={self.numel}"
+        debug_str += f"\n  FP32: tensor_name={self.fp32_tensor_name}, dtype={self.fp32_dtype}, max_value={_float_str(self.fp32_max_value)}, min_value={_float_str(self.fp32_min_value)}, mean_value={_float_str(self.fp32_mean_value)}"
+        debug_str += f"\n  FP16: tensor_name={self.fp16_tensor_name}, dtype={self.fp16_dtype}, max_value={_float_str(self.fp16_max_value)}, min_value={_float_str(self.fp16_min_value)}, mean_value={_float_str(self.fp16_mean_value)}, has_inf={self.fp16_has_inf}, has_nan={self.fp16_has_nan}"
+        return debug_str
+
+    def _div(self, a, b):
+        if a is not None and b is not None:
+            return a / b if b != 0 else 1
+        return None
+
+    def get_tensor_name(self):
+        if self.fp32_tensor_name is None:
+            return self.fp16_tensor_name  # + "#" + str(self.idx)
+        elif self.fp16_tensor_name is None:
+            return self.fp32_tensor_name + "#" + str(self.fp32_idx)
+        else:
+            return (
+                self.fp16_tensor_name.replace(".cast_fp16", "/.cast_fp16/")
+                + "#"
+                + str(self.fp32_idx)
+            )
+
+    def _check_normal(self):
+        # When the OP meets the following conditions, it is abnormal data, and use --skip_normal_tensors to retain the data in Excel:
+        # 1. The number of OP outputs exceeds the indication range of int32
+        # 2. The output data exceeds the representation range of fp16
+        # 3. Nan or inf appears in fp16 output data
+        # 4. The maximum value of fp32 is not equal to the maximum value of fp16
+        # 5. The minimum value of fp32 is not equal to the minimum value of fp16
+        if self.numel is not None and self.numel > np.iinfo(np.int32).max:
+            self.is_normal = False
+            return
+
+        check_list = [
+            self.fp32_max_value,
+            self.fp32_min_value,
+            self.scaled_fp32_max_value,
+            self.scaled_fp32_min_value,
+            self.fp16_max_value,
+            self.fp16_min_value,
+        ]
+
+        for value in check_list:
+            if value is not None and is_infinite(value):
+                self.is_normal = False
+                return
+
+        if self.fp16_has_inf is not None and self.fp16_has_inf:
+            self.is_normal = False
+            return
+        if self.fp16_has_nan is not None and self.fp16_has_nan:
+            self.is_normal = False
+            return
+
+        if (
+            self.scaled_fp32_max_value is not None
+            and self.fp16_max_value is not None
+            and not is_allclose(self.fp16_max_value, self.scaled_fp32_max_value)
+        ):
+            self.is_normal = False
+            return
+        if (
+            self.scaled_fp32_min_value is not None
+            and self.fp16_min_value is not None
+            and not is_allclose(self.fp16_min_value, self.scaled_fp32_min_value)
+        ):
+            self.is_normal = False
+            return
+
+
+class ExcelWriter:
+    def __init__(self, log_fp32_dir, log_fp16_dir, output_path):
+        self.log_fp32_dir = log_fp32_dir
+        self.log_fp16_dir = log_fp16_dir
+
+        try:
+            import xlsxwriter as xlw
+        except ImportError:
+            print(
+                "import xlsxwriter failed. please run 'pip install xlsxwriter==3.0.9' to install it"
+            )
+
+        self.workbook = xlw.Workbook(output_path)
+        self.title_format = self.workbook.add_format(
+            {
+                'bold': True,
+                'border': 1,
+                'font_color': 'black',
+                'bg_color': '#6495ED',
+                'align': 'center',
+            }
+        )
+        self.tensor_name_format = self.workbook.add_format(
+            {'bold': True, 'bg_color': '#F5F5F5'}
+        )
+        self.red_bg_cell_format = self.workbook.add_format(
+            {'bold': True, 'bg_color': 'red'}
+        )
+        self.yellow_bg_cell_format = self.workbook.add_format(
+            {'bold': True, 'bg_color': 'yellow'}
+        )
+        self.orange_bg_cell_format = self.workbook.add_format(
+            {'bold': True, 'bg_color': 'orange'}
+        )
+
+    def close(self):
+        self.workbook.close()
+        self.workbook = None
+
+    def _write_dtype(self, worksheet, value, row, col):
+        if value is None:
+            worksheet.write(row, col, "--")
+        else:
+            if value == "fp16":
+                worksheet.write(row, col, value, self.yellow_bg_cell_format)
+            else:
+                worksheet.write(row, col, value)
+
+    def _write_tensor_name(self, worksheet, mp_tensor_info, row, col):
+        tensor_name = mp_tensor_info.get_tensor_name()
+        if (
+            mp_tensor_info.fp32_tensor_name is not None
+            and mp_tensor_info.fp16_tensor_name
+        ):
+            worksheet.write(row, col, tensor_name, self.tensor_name_format)
+        else:
+            worksheet.write(row, col, tensor_name)
+
+    def _write_maxmin_value(
+        self, worksheet, value, row, col, check_finite=True
+    ):
+        if value is None:
+            worksheet.write(row, col, "--")
+        else:
+            if abs(value) < 1e-5:
+                value_str = f"{value:.6E}"
+            else:
+                value_str = f"{value:.6f}"
+            if check_finite and is_infinite(value, np.float16):
+                worksheet.write(row, col, value_str, self.red_bg_cell_format)
+            else:
+                worksheet.write(row, col, value_str)
+
+    def _write_tensor_num_zero(
+        self, worksheet, value, row, col, check_finite=True
+    ):
+        if value is None:
+            worksheet.write(row, col, "--")
+        else:
+            value_str = f"{value:>10d}"
+            worksheet.write(row, col, value_str)
+
+    def _write_infinite_status(self, worksheet, value, row, col):
+        if value is None:
+            worksheet.write(row, col, "--")
+        else:
+            if value == 1:
+                worksheet.write(row, col, value, self.red_bg_cell_format)
+            else:
+                worksheet.write(row, col, value)
+
+    def _write_fp32divfp16_value(self, worksheet, value, row, col, loss_scale):
+        def _in_range(value, scale=1):
+            return value > scale * 0.95 and value < scale * 1.05
+
+        if value is None:
+            worksheet.write(row, col, "--")
+        else:
+            value_str = f"{value:.6f}"
+            if _in_range(value, scale=1) or _in_range(value, loss_scale):
+                worksheet.write(row, col, value_str)
+            else:
+                worksheet.write(row, col, value_str, self.orange_bg_cell_format)
+
+    def _write_titles(self, worksheet, loss_scale, row):
+        column_width_dict = {
+            "op_type": 24,
+            "tensor_name": 60,
+            "numel": 10,
+            "num_zero": 10,
+            "infinite": 8,
+            "dtype": 8,
+            "max_value": 16,
+            "min_value": 16,
+            "mean_value": 16,
+            "num_inf": 8,
+            "num_nan": 8,
+        }
+        title_names = ["op_type", "tensor_name", "numel", "infinite"]
+        if self.log_fp16_dir is None:
+            # only fp32 values
+            worksheet.merge_range("E1:H1", "fp32", self.title_format)
+            worksheet.merge_range(
+                "I1:J1", f"fp32 (scale={loss_scale})", self.title_format
+            )
+            title_names.extend(
+                [
+                    "dtype",
+                    "max_value",
+                    "min_value",
+                    "mean_value",
+                    "max_value",
+                    "min_value",
+                ]
+            )
+        elif self.log_fp32_dir is None:
+            # only fp16 values
+            worksheet.merge_range(
+                "E1:J1", f"fp16 (scale={loss_scale})", self.title_format
+            )
+            title_names.extend(
+                [
+                    "dtype",
+                    "max_value",
+                    "min_value",
+                    "mean_value",
+                    "num_zero",
+                    "num_inf",
+                    "num_nan",
+                ]
+            )
+        else:
+            # fp32 and fp16 values
+            worksheet.merge_range("E1:H1", "fp32", self.title_format)
+            worksheet.merge_range(
+                "I1:N1", f"fp16 (scale={loss_scale})", self.title_format
+            )
+            worksheet.merge_range("O1:Q1", "fp16 / fp32", self.title_format)
+            title_names.extend(
+                [
+                    "dtype",
+                    "max_value",
+                    "min_value",
+                    "mean_value",
+                    "num_zero",
+                    "dtype",
+                    "max_value",
+                    "min_value",
+                    "mean_value",
+                    "num_zero",
+                    "num_inf",
+                    "num_nan",
+                    "max_value",
+                    "min_value",
+                    "mean_value",
+                ]
+            )
+
+        for col in range(len(title_names)):
+            col_char = chr(ord("A") + col)
+            worksheet.set_column(
+                col_char + ":" + col_char, column_width_dict[title_names[col]]
+            )
+        for col in range(len(title_names)):
+            worksheet.write(row, col, title_names[col], self.title_format)
+
+    def add_worksheet(
+        self, mp_tensor_info_list, sheetname, loss_scale, skip_normal_tensors
+    ):
+        assert self.workbook is not None
+
+        worksheet = self.workbook.add_worksheet(sheetname)
+        row = 1
+
+        self._write_titles(worksheet, loss_scale, row)
+        row += 1
+
+        infinite_op_types = []
+        for tensor_info in mp_tensor_info_list:
+            if (
+                not tensor_info.is_normal
+                and tensor_info.op_type not in infinite_op_types
+            ):
+                infinite_op_types.append(tensor_info.op_type)
+
+            if skip_normal_tensors and tensor_info.is_normal:
+                continue
+
+            worksheet.write(row, 0, tensor_info.op_type)
+            self._write_tensor_name(worksheet, tensor_info, row, 1)
+
+            if tensor_info.numel > np.iinfo(np.int32).max:
+                worksheet.write(
+                    row, 2, tensor_info.numel, self.bad_value_format
+                )
+            else:
+                worksheet.write(row, 2, tensor_info.numel)
+
+            if tensor_info.is_normal:
+                worksheet.write(row, 3, "0")
+            else:
+                worksheet.write(row, 3, "1", self.red_bg_cell_format)
+
+            col = 4
+
+            if self.log_fp32_dir is not None:
+                self._write_dtype(worksheet, tensor_info.fp32_dtype, row, col)
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp32_max_value, row, col + 1
+                )
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp32_min_value, row, col + 2
+                )
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp32_mean_value, row, col + 3
+                )
+                self._write_tensor_num_zero(
+                    worksheet, tensor_info.fp32_num_zero, row, col + 4
+                )
+                col += 5
+
+                if self.log_fp16_dir is None:
+                    self._write_maxmin_value(
+                        worksheet, tensor_info.scaled_fp32_max_value, row, col
+                    )
+                    self._write_maxmin_value(
+                        worksheet,
+                        tensor_info.scaled_fp32_min_value,
+                        row,
+                        col + 1,
+                    )
+                    col += 2
+
+            if self.log_fp16_dir is not None:
+                self._write_dtype(worksheet, tensor_info.fp16_dtype, row, col)
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp16_max_value, row, col + 1
+                )
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp16_min_value, row, col + 2
+                )
+                self._write_maxmin_value(
+                    worksheet, tensor_info.fp16_mean_value, row, col + 3
+                )
+                self._write_tensor_num_zero(
+                    worksheet, tensor_info.fp32_num_zero, row, col + 4
+                )
+                col += 5
+
+                self._write_infinite_status(
+                    worksheet, tensor_info.fp16_has_inf, row, col
+                )
+                self._write_infinite_status(
+                    worksheet, tensor_info.fp16_has_nan, row, col + 1
+                )
+                col += 2
+
+            if self.log_fp32_dir is not None and self.log_fp16_dir is not None:
+                self._write_fp32divfp16_value(
+                    worksheet,
+                    tensor_info.fp32_div_fp16_max_value,
+                    row,
+                    col,
+                    loss_scale,
+                )
+                self._write_fp32divfp16_value(
+                    worksheet,
+                    tensor_info.fp32_div_fp16_min_value,
+                    row,
+                    col + 1,
+                    loss_scale,
+                )
+                self._write_fp32divfp16_value(
+                    worksheet,
+                    tensor_info.fp32_div_fp16_mean_value,
+                    row,
+                    col + 2,
+                    loss_scale,
+                )
+                col += 3
+
+            row += 1
+
+        print(f"-- OP Types produce infinite outputs: {infinite_op_types}")
+
+
+def parse_lines(lines, specified_op_list=None):
+    tensor_info_list = []
+
+    for i in range(len(lines)):
+        if i % 10 == 0:
+            print(
+                f"-- Processing {i:-8d} / {len(lines):-8d} line",
+                end="\r",
+            )
+        line = lines[i]
+        if "[PRECISION]" in line:
+            tensor_info = TensorInfo()
+            tensor_info.init_from_string(line)
+            if (
+                tensor_info.tensor_name is not None
+                and tensor_info.tensor_name != ""
+            ):
+                has_tensor_name = True
+            if (
+                specified_op_list is None
+                or tensor_info.op_type in specified_op_list
+            ):
+                tensor_info_list.append(tensor_info)
+            # print(tensor_info)
+    return tensor_info_list
+
+
+def parse_log(log_dir, filename, specified_op_list=None):
+    if log_dir is None or filename is None:
+        return None
+
+    complete_filename = log_dir + "/" + filename
+    tensor_info_list = None
+    has_tensor_name = False
+
+    try:
+        with open(complete_filename, 'r') as f:
+            lines = f.readlines()
+            tensor_info_list = parse_lines(lines, specified_op_list)
+    except FileNotFoundError:
+        print("the file ", complete_filename, "is not found")
+        return None, has_tensor_name
+    return tensor_info_list, has_tensor_name
+
+
+def merge_tensor_info_list(
+    fp32_tensor_info_list, fp16_tensor_info_list, grad_scale
+):
+    mp_tensor_info_list = []
+    if fp16_tensor_info_list is not None:
+        fp32_tensor_info_dict = {}
+        fp32_write_count = {}
+        if fp32_tensor_info_list is not None:
+            for tensor_info in fp32_tensor_info_list:
+                tensor_info_key = tensor_info.key()
+                count = fp32_write_count.get(tensor_info_key, 0)
+                fp32_write_count[tensor_info_key] = count + 1
+                fp32_tensor_info_dict[tensor_info_key + "#" + str(count)] = (
+                    tensor_info
+                )
+
+        fp32_read_count = {}
+        for i in range(len(fp16_tensor_info_list)):
+            if i % 10 == 0:
+                print(
+                    f"-- Processing {i:-8d} / {len(fp16_tensor_info_list):-8d} FP16 Tensor Info",
+                    end="\r",
+                )
+            fp16_tensor_info = fp16_tensor_info_list[i]
+            fp32_tensor_info_key = (
+                fp16_tensor_info.key()
+                .replace(".cast_fp16", "")
+                .replace(".cast_fp32", "")
+            )
+            count = fp32_read_count.get(fp32_tensor_info_key, 0)
+            fp32_tensor_info = fp32_tensor_info_dict.get(
+                fp32_tensor_info_key + "#" + str(count), None
+            )
+            if fp32_tensor_info is not None:
+                fp32_read_count[fp32_tensor_info_key] = count + 1
+            mp_tensor_info = MixedPrecisionTensorInfo(
+                fp32_tensor_info, fp16_tensor_info, count, grad_scale
+            )
+            mp_tensor_info_list.append(mp_tensor_info)
+            # print(mp_tensor_info)
+    elif fp32_tensor_info_list is not None:
+        fp32_count = {}
+        for i in range(len(fp32_tensor_info_list)):
+            if i % 10 == 0:
+                print(
+                    f"-- Processing {i:-8d} / {len(fp32_tensor_info_list):-8d} FP32 Tensor Info",
+                    end="\r",
+                )
+            tensor_info = fp32_tensor_info_list[i]
+            tensor_info_key = tensor_info.key()
+            count = fp32_count.get(tensor_info_key, 0)
+            fp32_count[tensor_info_key] = count + 1
+            mp_tensor_info = MixedPrecisionTensorInfo(
+                tensor_info, None, count, grad_scale
+            )
+            mp_tensor_info_list.append(mp_tensor_info)
+
+    return mp_tensor_info_list
+
+
+def compare_accuracy(
+    dump_path,
+    another_dump_path,
+    output_filename,
+    loss_scale=1,
+    dump_all_tensors=False,
+):
+    excel_writer = ExcelWriter(dump_path, another_dump_path, output_filename)
+    grad_scale = loss_scale
+    workerlog_filenames = []
+    filenames = os.listdir(dump_path)
+    for name in filenames:
+        if "worker_" in name:
+            workerlog_filenames.append(name)
+    print(
+        f"-- There are {len(workerlog_filenames)} workerlogs under {dump_path}: {workerlog_filenames}"
+    )
+
+    for filename in sorted(workerlog_filenames):
+        print(f"-- [Step 1/4] Parsing FP32 logs under {dump_path}/{filename}")
+        fp32_tensor_info_list, fp32_has_tensor_name = parse_log(
+            dump_path, filename, None
+        )
+        print(
+            f"-- [Step 2/4] Parsing FP16 logs under {another_dump_path}/{filename}"
+        )
+        fp16_tensor_info_list, fp16_has_tensor_name = parse_log(
+            another_dump_path, filename, None
+        )
+
+        print(f"-- [Step 3/4] Merge FP32 and FP16 tensor info for {filename}")
+        mp_tensor_info_list = merge_tensor_info_list(
+            fp32_tensor_info_list, fp16_tensor_info_list, grad_scale
+        )
+        print(
+            f"-- [Step 4/4] Add worksheet for mixed precision tensor info of {filename}"
+        )
+        excel_writer.add_worksheet(
+            mp_tensor_info_list,
+            filename,
+            loss_scale,
+            False,
+        )
+
+    print(f"-- Write to {output_filename}")
+
+    print()
+    excel_writer.close()

python/paddle/amp/amp_lists.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The set of ops that support fp16 and bf16 calculation and are considered numerically-
+# safe and performance-critical. These ops are always converted to fp16 or bf16.
+
+from __future__ import annotations
+
+WHITE_LIST = {
+    'conv2d',
+    'einsum',
+    'matmul',
+    'matmul_v2',
+    'linear_v2',
+    'max_pool2d_with_index',
+    'mul',
+    'fused_gemm_epilogue',
+    "fused_rotary_position_embedding",
+    "flash_attn",
+}
+
+# The set of ops that support fp16, and bf16 was unsupported.
+ONLY_FP16_WHITE_LIST = {
+    'fake_quantize_dequantize_abs_max',
+    'fake_quantize_dequantize_moving_average_abs_max',
+    'fused_attention',
+    'fused_feedforward',
+}
+
+FP16_WHITE_LIST = WHITE_LIST | ONLY_FP16_WHITE_LIST
+
+# The set of ops that support fp16 calculation and are considered numerically-
+# dangerous and whose effects may also be observed in downstream ops.
+FP16_BLACK_LIST = {
+    'tan',
+    'acos',
+    'asin',
+    'sinh',
+    'cosh',
+    'atanh',
+    'tanh_shrink',
+    'erfinv',
+    'exp',
+    'expm1',
+    'log',
+    'log10',
+    'log2',
+    'reciprocal',
+    'rsqrt',
+    'pow',
+    'square',
+    'reduce_sum',
+    'mean',
+    'reduce_mean',
+    'reduce_prod',
+    'cumprod',
+    'cumsum',
+    'dist',
+    'pnorm',
+    'frobenius_norm',
+    'renorm',
+    'group_norm',
+    'layer_norm',
+    'softmax',
+    'softmin',
+    'softplus',
+    'log_softmax',
+    'softmax_with_cross_entropy',
+    'sigmoid_cross_entropy_with_logits',
+    'c_softmax_with_cross_entropy',
+    'c_softmax_with_multi_label_cross_entropy',
+    'cross_entropy',
+    'cross_entropy2',
+    'nll_loss',
+    'huber_loss',
+    'triplet_margin_loss',
+    'log_loss',
+    'hsigmoid_loss',
+    'margin_cross_entropy',
+}
+
+# FP16/BF16 performance of grad op is worse than that of FP32. Use FP32 by default.
+EXTRA_BLACK_LIST = {
+    'linear_interp_v2',
+    'nearest_interp_v2',
+    'bilinear_interp_v2',
+    'bicubic_interp_v2',
+    'trilinear_interp_v2',
+    'lookup_table',
+    'lookup_table_v2',
+    'scatter',
+}
+
+BF16_WHITE_LIST = WHITE_LIST
+BF16_BLACK_LIST = FP16_BLACK_LIST
+
+
+# At OD level, ops in WHITE_LIST will use FP16/BF16 and the others will use FP32.
+def white_list() -> dict[str, dict[str, set[str]]]:
+    white_list = {
+        "float16": {
+            "OD": FP16_WHITE_LIST,
+            "O1": FP16_WHITE_LIST,
+            "O2": FP16_WHITE_LIST,
+        },
+        "bfloat16": {
+            "OD": BF16_WHITE_LIST,
+            "O1": BF16_WHITE_LIST,
+            "O2": BF16_WHITE_LIST,
+        },
+    }
+    return white_list
+
+
+def black_list() -> dict[str, dict[str, set[str]]]:
+    black_list = {
+        "float16": {
+            "OD": set(),
+            "O1": FP16_BLACK_LIST | EXTRA_BLACK_LIST,
+            "O2": EXTRA_BLACK_LIST,
+        },
+        "bfloat16": {
+            "OD": set(),
+            "O1": BF16_BLACK_LIST | EXTRA_BLACK_LIST,
+            "O2": EXTRA_BLACK_LIST,
+        },
+    }
+    return black_list

python/paddle/amp/debugging.py

@@ -0,0 +1,731 @@
+#   Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import contextlib
+import random
+from enum import Enum
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    TypeVar,
+)
+
+import numpy as np
+from typing_extensions import ParamSpec
+
+import paddle
+from paddle import _C_ops
+from paddle.base import core
+
+from ..framework import LayerHelper, in_dynamic_or_pir_mode
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Generator, Sequence
+
+    from paddle import Tensor
+
+_InputT = ParamSpec("_InputT")
+_RetT = TypeVar("_RetT")
+__all__ = [
+    "DebugMode",
+    "TensorCheckerConfig",
+    "check_numerics",
+    "enable_operator_stats_collection",
+    "disable_operator_stats_collection",
+    "collect_operator_stats",
+    "enable_tensor_checker",
+    "disable_tensor_checker",
+    "compare_accuracy",
+    "check_layer_numerics",
+]
+
+
+class DebugMode(Enum):
+    """
+    The DebugMode is a feature that helps to present the state of the TensorCheckerConfig. Each DebugMode has a specific meaning, which is explained below:
+
+    - DebugMode.CHECK_NAN_INF_AND_ABORT: This mode prints or saves information about Tensors that contain NaN/Inf and interrupts the program.
+
+    - DebugMode.CHECK_NAN_INF: This mode prints or saves critical information about Tensors that contain NaN/Inf but allows the program to continue running.
+
+    - DebugMode.CHECK_ALL_FOR_OVERFLOW: This mode checks the output of the FP32 operator and prints or saves information about key Tensors that exceed the FP16 representation range, such as overflow or underflow.
+
+    - DebugMode.CHECK_ALL: This mode prints or saves output Tensor key information for all operators.
+
+    """
+
+    CHECK_NAN_INF_AND_ABORT = 0
+    CHECK_NAN_INF = 1
+    CHECK_ALL_FOR_OVERFLOW = 2
+    CHECK_ALL = 3
+    # CHECK_ALL_AND_ABORT = 4
+    # DUMP_ALL = 5
+
+
+def check_layer_numerics(
+    func: Callable[_InputT, _RetT],
+) -> Callable[_InputT, _RetT]:
+    """
+    This decorator is used to check the numerical values of the layer's input and output data.
+
+    Args:
+        func (callable): The function to be decorated.
+
+    Returns:
+        callable: The decorated function.
+
+    Raises:
+        None.
+
+    Example:
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> class MyLayer(paddle.nn.Layer):
+            ...     def __init__(self, dtype):
+            ...         super().__init__()
+            ...         self._w = self.create_parameter([2, 3], dtype=dtype)
+            ...         self._b = self.create_parameter([2, 3], dtype=dtype)
+            ...
+            ...     @paddle.amp.debugging.check_layer_numerics
+            ...     def forward(self, x):
+            ...         # return 1/x * self._w + self._b   open it you will see the error log
+            ...         return x @ self._w + self._b
+            >>> dtype = 'float32'
+            >>> x = paddle.rand([10, 2, 2], dtype=dtype)  # type: ignore[call-overload]
+            >>> model = MyLayer(dtype)
+            >>> x[0] = float(0)
+            >>> loss = model(x)
+            >>> adam = paddle.optimizer.Adam(parameters=model.parameters())
+            >>> loss.backward()
+            >>> adam.step()
+
+            >>> # error log
+            >>> # [PRECISION] [ERROR] in [device=gpu:0, op=divide, tensor=, dtype=fp32], numel=40, num_nan=0, num_inf=4, num_zero=0, max=inf, min=1.048930e+00, mean=inf
+            >>> # Traceback (most recent call last):
+            >>> #   File "tmp.py", line 16, in <module>
+            >>> #     loss = model(x)
+            >>> #   File "/paddle/nn/layer/layers.py", line 1254, in __call__
+            >>> #     return self.forward(*inputs, **kwargs)
+            >>> #   File "/paddle/amp/debugging.py", line 116, in wrapper
+            >>> #     out_data = func(self, *modified_args, **kwargs)
+            >>> #   File "test.py", line 10, in forward
+            >>> #     return 1/x *  self._w+ self._b
+            >>> # RuntimeError: (PreconditionNotMet) There are NAN or INF (num_nan=0, num_inf=4, num_zero=0) in [device=gpu:0, op=divide, tensor=, dtype=fp32].
+    """
+
+    def wrapper(self, *args: _InputT.args, **kwargs: _InputT.kwargs) -> _RetT:
+        if args:
+            # Set temp data and temp.gradient = False
+            start_data = args[0]
+            if not isinstance(start_data, paddle.Tensor):
+                raise RuntimeError("First input of this layer must be tensor.")
+            start_data.stop_gradient = False
+            modified_args = list(args)  # Convert args to a mutable list
+            # Set FLAGS_check_nan_inf = 1
+            modified_args[0] = _C_ops.enable_check_model_nan_inf(start_data, 1)
+            # Call the forward function
+            out_data = func(self, *modified_args, **kwargs)
+            # Set FLAGS_check_nan_inf = 0
+            out = _C_ops.disable_check_model_nan_inf(out_data, 0)
+            return out
+        else:
+            raise RuntimeError("No elements found in args.")
+        out = func(self, *args, **kwargs)
+        return out
+
+    return wrapper
+
+
+def set_checked_op_list(checked_op_list: Sequence[str] | None) -> None:
+    # check checked_op_list
+    if checked_op_list is not None:
+        if isinstance(checked_op_list, (list, tuple)):
+            check_op_list = ",".join(value for value in checked_op_list)
+            paddle.base.core.set_checked_op_list(check_op_list)
+        else:
+            raise ValueError("checked_op_list must be list or tuple")
+
+
+def set_skipped_op_list(skipped_op_list: Sequence[str] | None) -> None:
+    # check skipped_op_list
+    if skipped_op_list is not None:
+        if isinstance(skipped_op_list, (list, tuple)):
+            skip_op_list = ",".join(value for value in skipped_op_list)
+            paddle.base.core.set_skipped_op_list(skip_op_list)
+        else:
+            raise ValueError("skipped_op_list must be list or tuple")
+
+
+class TensorCheckerConfig:
+    """
+    The purpose of this class is to collect the configuration for checking NaN and Inf values in the tensors of a module or operator. It takes the following arguments:
+
+    Args:
+        enable(bool): Indicating whether to enable the detection of NaN and Inf values in tensors. The default value is False, which means that these tools will not be used.
+
+        debug_mode(DebugMode, optional): A parameter that determines the type of debugging to be used. Default is DebugMode.CHECK_NAN_INF_AND_ABORT.
+
+        output_dir(string|None, optional): The path to store collected data. If this parameter is set to None, the data will be printed to the terminal. Default is None.
+
+        checked_op_list(list|tuple|None, optional): Specifies a list of operators that need to be checked during program execution, for example, checked_op_list=['elementwise_add', 'conv2d'], indicating that the output results of elementwise_add and conv2d should be checked for nan/inf during program execution. Default is None.
+
+        skipped_op_list(list|tuple|None, optional): Specifies a list of operators that do not need to be checked during program execution, for example, skipped_op_list=['elementwise_add', 'conv2d'], indicating that the output results of elementwise_add and conv2d should not be checked for nan/inf during program execution. None is None.
+
+        debug_step(list|tuple|None, optional): A list or tuple used primarily for nan/inf checking during model training. For example, debug_step=[1,5] indicates that nan/inf checking should only be performed on model training iterations 1 to 5. Default is None.
+
+        stack_height_limit(int, optional): An integer value specifying the maximum depth of the call stack. This feature supports printing the call stack at the error location. Currently, only enabling or disabling call stack printing is supported. If you want to print the corresponding C++ call stack when NaN is detected in GPU Kernel, set stack_height_limit to 1, otherwise set it to 0. Default is 1.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> import paddle
+
+            >>> checker_config = paddle.amp.debugging.TensorCheckerConfig(
+            ...     enable=True, debug_mode=paddle.amp.debugging.DebugMode.CHECK_NAN_INF
+            ... )
+            >>> paddle.amp.debugging.enable_tensor_checker(checker_config)
+
+            >>> x = paddle.to_tensor([1, 0, 3], place=paddle.CPUPlace(), dtype='float32', stop_gradient=False)
+            >>> y = paddle.to_tensor([0.2, 0, 0.5], place=paddle.CPUPlace(), dtype='float32')
+            >>> res = paddle.pow(x, y)
+            >>> paddle.autograd.backward(res, retain_graph=True)
+            >>> paddle.amp.debugging.disable_tensor_checker()
+
+            >>> # [PRECISION] [ERROR] in [device=cpu, op=elementwise_pow_grad, tensor=, dtype=fp32], numel=3, num_nan=1, num_inf=0, num_zero=0, max=2.886751e-01, min=2.000000e-01, mean=-nan
+
+            >>> # when DebugMode.CHECK_NAN_INF_AND_ABORT and stack_height_limit = 1
+            >>> # Traceback (most recent call last):
+            >>> #     res = paddle.pow(x, y)
+            >>> #   File "/usr/local/lib/python3.8/dist-packages/paddle/tensor/math.py", line 447, in pow
+            >>> #     return _C_ops.elementwise_pow(x, y)
+
+    """
+
+    # For module debugging
+    current_step_id: int = 0
+
+    enable: bool
+    debug_mode: DebugMode
+    output_dir: str | None
+    checked_op_list: Sequence[str] | None
+    skipped_op_list: Sequence[str] | None
+    debug_step: Sequence[int] | None
+    stack_height_limit: int
+    start_step: int | None
+    end_step: int | None
+    seed: int
+    initial_seed: int
+
+    def __init__(
+        self,
+        enable: bool,
+        debug_mode: DebugMode = DebugMode.CHECK_NAN_INF_AND_ABORT,
+        output_dir: str | None = None,
+        checked_op_list: Sequence[str] | None = None,
+        skipped_op_list: Sequence[str] | None = None,
+        debug_step: Sequence[int] | None = None,
+        stack_height_limit: int = 1,
+    ):
+        self.enable = enable
+        self.debug_mode = debug_mode
+        self.output_dir = output_dir
+
+        self.checked_op_list = checked_op_list
+        self.skipped_op_list = skipped_op_list
+
+        self.debug_step = debug_step
+        self.stack_height_limit = stack_height_limit
+
+        self.start_step = None
+        self.end_step = None
+
+        self.seed = 123
+        self.initial_seed = 123
+
+        # check debug_step
+        if debug_step is not None:
+            if isinstance(debug_step, (tuple, list)):
+                assert (
+                    len(self.debug_step) == 2
+                    and self.debug_step[1] > self.debug_step[0]
+                )
+                self.start_step, self.end_step = self.debug_step
+                self.start_step = max(self.start_step, 0)
+            else:
+                raise ValueError("debug_step must be list or tuple")
+
+        if core.is_compiled_with_cuda():
+            for i in range(core.get_cuda_device_count()):
+                self.initial_seed = core.default_cuda_generator(
+                    i
+                ).initial_seed()
+        elif core.is_compiled_with_xpu():
+            for i in range(core.get_xpu_device_count()):
+                self.initial_seed = core.default_xpu_generator(i).initial_seed()
+
+        self.initial_seed = core.default_cpu_generator().initial_seed()
+
+        # check debug_mode
+        if self.debug_mode.name not in DebugMode.__members__:
+            raise ValueError(
+                "debug_mode in DebugMode",
+                self.debug_mode,
+                DebugMode.__members__,
+            )
+
+        set_checked_op_list(self.checked_op_list)
+
+        set_skipped_op_list(self.skipped_op_list)
+
+        if self.enable:
+            self._set_seed(self.enable)
+
+    def _set_seed(self, flag: int) -> None:
+        if self.initial_seed != self.seed:
+            self.seed = self.initial_seed
+
+        if self.seed > np.iinfo(np.uint32).max or self.seed < 0:
+            print("[Warning: Seed must be between 0 and 2**32 - 1")
+            self.seed = 123
+
+        # get random seed
+        paddle.seed(self.seed)
+        np.random.seed(self.seed)
+        random.seed(self.seed)
+
+        # info
+        print("AMP Debugging TensorCheckerConfig: seed ", self.seed)
+
+        # set cudnn and cpu
+        if core.is_compiled_with_cuda():
+            paddle.set_flags({"FLAGS_cudnn_deterministic": flag})
+            print(
+                "AMP Debugging TensorCheckerConfig: FLAGS_cudnn_deterministic is ",
+                flag,
+            )
+
+    def _set_env(self, check_flag: int) -> None:
+        paddle.set_flags({"FLAGS_check_nan_inf": check_flag})
+        if check_flag:
+            # set debug level
+            paddle.set_flags(
+                {"FLAGS_check_nan_inf_level": self.debug_mode.value}
+            )
+
+            # set output_dir
+            if self.output_dir is not None:
+                paddle.base.core.set_nan_inf_debug_path(self.output_dir)
+
+            # set stack_height_limit
+            if isinstance(self.stack_height_limit, (int)):
+                paddle.base.core.set_nan_inf_stack_limit(
+                    self.stack_height_limit
+                )
+            else:
+                raise ValueError("stack_height_limit must be int")
+
+    def update_and_check_step_id(self) -> bool:
+        if self.enable:
+            if self.start_step is not None and self.end_step is not None:
+                if (
+                    self.start_step > TensorCheckerConfig.current_step_id
+                    or TensorCheckerConfig.current_step_id >= self.end_step
+                ):
+                    return False
+                else:
+                    TensorCheckerConfig.current_step_id += 1
+            return True
+        return False
+
+    def start_check_nan_inf(self) -> None:
+        if self.enable:
+            self._set_env(self.enable)
+
+    def stop_check_nan_inf(self) -> None:
+        self._set_env(False)
+
+
+def check_numerics(
+    tensor: Tensor,
+    op_type: str,
+    var_name: str,
+    debug_mode: DebugMode = DebugMode.CHECK_NAN_INF_AND_ABORT,
+) -> tuple[Tensor, Tensor]:
+    """
+    This function is used to debugging a tensor, finding the number of NaNs, Infs and zeros in the tensor.
+
+    Args:
+        tensor(Tensor): The target tensor to check.
+        op_type(str): The OP or API name which produce the target tensor.
+        var_name(str): The name of target tensor.
+        debug_mode(paddle.amp.debugging.DebugMode, optional): The mode of debugging to be used. Default is DebugMode.CHECK_NAN_INF_AND_ABORT.
+
+    Returns:
+        (Tensor, Tensor): A tuple of tensors containing
+
+        - **stats** (Tensor): Returns the number of NaNs, Infs and zeros of input tensor. The shape is [3] and dtype is int64.
+        - **values** (Tensor): Returns the maximum, minimum and mean value of input tensor. The shape is [3] and dtype is float.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> import paddle
+
+            >>> checker_config = paddle.amp.debugging.TensorCheckerConfig(
+            ...     enable=True, debug_mode=paddle.amp.debugging.DebugMode.CHECK_NAN_INF
+            ... )
+
+            >>> x = paddle.to_tensor([1, 0, 3], place=paddle.CPUPlace(), dtype='float32')
+            >>> y = paddle.to_tensor([0.2, 0, 0.5], place=paddle.CPUPlace(), dtype='float32')
+            >>> res = paddle.pow(x, y)
+            >>> paddle.amp.debugging.check_numerics(res, "pow", "res")
+
+    """
+    stack_height_limit = -1
+    output_dir = ""
+
+    if in_dynamic_or_pir_mode():
+        return _C_ops.check_numerics(
+            tensor,
+            op_type,
+            var_name,
+            debug_mode.value,
+            stack_height_limit,
+            output_dir,
+        )
+
+    helper = LayerHelper("check_numerics", **locals())
+
+    stats = helper.create_variable_for_type_inference(dtype="int64")
+    values = helper.create_variable_for_type_inference(dtype="float")
+
+    helper.append_op(
+        type='check_numerics',
+        inputs={
+            'tensor': tensor,
+        },
+        attrs={
+            'op_type': op_type,
+            'var_name': var_name,
+            'check_nan_inf_level': debug_mode.value,
+            'stack_height_limit': stack_height_limit,
+            'output_dir': output_dir,
+        },
+        outputs={'stats': [stats], 'values': [values]},
+    )
+    return stats, values
+
+
+def _get_operator_stats_flag() -> Any:
+    flags = paddle.get_flags(["FLAGS_low_precision_op_list"])
+    return flags["FLAGS_low_precision_op_list"]
+
+
+def _print_operator_stats(op_count_dict: dict[str, str | list[int]]) -> None:
+    """
+    Parse and print the stats of operators, mainly including the calls of
+    dtypes such as different fp32, fp16, bf16 and others.
+
+    Args:
+        op_count_dict(dict): a dict to record the number of calls for different
+            operator and dtype. An example is
+            {'conv2d': '1,0,0,0', 'elementwise_add': '1,0,0,0'} or
+            {'conv2d': [1, 0, 0, 0], 'elementwise_add': [1, 0, 0, 0]}.
+    """
+    print("<{:-^120}>".format(" op list "))
+    total_ops = 0
+    print(
+        "<{:-^40}".format(" Op Name "),
+        "|",
+        "{:-^17}".format(" FP16 Calls "),
+        "|",
+        "{:-^17}".format(" BF16 Calls "),
+        "|",
+        "{:-^17}".format(" FP32 Calls"),
+        "|",
+        "{:-^17}>".format(" Other Calls "),
+    )
+    if op_count_dict is not None and isinstance(op_count_dict, dict):
+        for op_type in sorted(op_count_dict):
+            # fp16, bf16, fp32, other
+            value = op_count_dict[op_type]
+            if isinstance(value, list):
+                called = value
+            elif isinstance(value, str):
+                called = value.split(",")
+            else:
+                raise ValueError(
+                    f"Input {value} is expected to be a list of str, but received {type(value)}."
+                )
+            print(
+                f"  {op_type:<40}|  {called[0]:<17}|  {called[1]:<17}|  {called[2]:<17}|  {called[3]:<17}"
+            )
+            total_ops += 1
+    print("<{:-^120}>\n".format(" op count: " + str(total_ops) + " "))
+
+
+def enable_operator_stats_collection() -> None:
+    """
+    Enable to collect the number of operators for different data types.
+    The statistical data are categorized according to four data types, namely
+    float32, float16, bfloat16 and others. This function is used in pair with
+    the corresponding disable function.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> # doctest: +REQUIRES(env:GPU)
+            >>> import paddle
+            >>> paddle.device.set_device('gpu')
+
+            >>> conv = paddle.nn.Conv2D(3, 2, 3)
+            >>> x = paddle.rand([10, 3, 32, 32])
+
+            >>> paddle.amp.debugging.enable_operator_stats_collection()
+            >>> # AMP list including cast, conv2d, elementwise_add, reshape
+            >>> with paddle.amp.auto_cast(enable=True, level='O2'):
+            ...     out = conv(x)
+            >>> # Print to the standard output.
+            >>> paddle.amp.debugging.disable_operator_stats_collection()
+            >>> # <------------------------------------------------------- op list -------------------------------------------------------->
+            >>> # <--------------- Op Name ---------------- | -- FP16 Calls --- | -- BF16 Calls --- | --- FP32 Calls--- | -- Other Calls -->
+            >>> #   cast                                    |  1                |  0                |  2                |  0
+            >>> #   conv2d                                  |  1                |  0                |  0                |  0
+            >>> #   elementwise_add                         |  0                |  0                |  1                |  0
+            >>> #   reshape                                 |  0                |  0                |  1                |  0
+            >>> # <----------------------------------------------------- op count: 4 ------------------------------------------------------>
+
+    """
+    # Clear the previous stats.
+    paddle.base.core.clear_low_precision_op_list()
+    paddle.set_flags({'FLAGS_low_precision_op_list': 1})
+
+
+def disable_operator_stats_collection() -> None:
+    """
+    Disable the collection the number of operators for different data types.
+    This function is used in pair with the corresponding enable function.
+    The statistical data are categorized according to four data types, namely
+    float32, float16, bfloat16 and others, and will be printed after the
+    function call.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> import paddle
+
+            >>> conv = paddle.nn.Conv2D(3, 2, 3)
+            >>> x = paddle.rand([10, 3, 32, 32])
+
+            >>> paddle.amp.debugging.enable_operator_stats_collection()
+            >>> # AMP list including cast, conv2d, elementwise_add, reshape
+            >>> with paddle.amp.auto_cast(enable=True, level='O2'):
+            ...     out = conv(x)
+            >>> # Print to the standard output.
+            >>> paddle.amp.debugging.disable_operator_stats_collection()
+            >>> # <------------------------------------------------------- op list -------------------------------------------------------->
+            >>> # <--------------- Op Name ---------------- | -- FP16 Calls --- | -- BF16 Calls --- | --- FP32 Calls--- | -- Other Calls -->
+            >>> #   cast                                    |  1                |  0                |  2                |  0
+            >>> #   conv2d                                  |  1                |  0                |  0                |  0
+            >>> #   elementwise_add                         |  0                |  0                |  1                |  0
+            >>> #   reshape                                 |  0                |  0                |  1                |  0
+            >>> # <----------------------------------------------------- op count: 4 ------------------------------------------------------>
+
+    """
+    if not _get_operator_stats_flag():
+        return
+
+    op_count_dict = paddle.base.core.get_low_precision_op_list()
+    _print_operator_stats(op_count_dict)
+    paddle.set_flags({'FLAGS_low_precision_op_list': 0})
+
+
+@contextlib.contextmanager
+def collect_operator_stats() -> Generator[None, None, None]:
+    """
+    The context switcher to enable to collect the number of operators for
+    different data types. The statistical data are categorized according
+    to four data types, namely float32, float16, bfloat16 and others, and
+    will be printed when exiting the context.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> import paddle
+
+            >>> conv = paddle.nn.Conv2D(3, 2, 3)
+            >>> x = paddle.rand([10, 3, 32, 32])
+
+            >>> with paddle.amp.debugging.collect_operator_stats():
+            ...     # AMP list including cast, conv2d, elementwise_add, reshape
+            ...     with paddle.amp.auto_cast(enable=True, level='O2'):
+            ...         out = conv(x)
+            >>> # Print to the standard output.
+            >>> # <------------------------------------------------------- op list -------------------------------------------------------->
+            >>> # <--------------- Op Name ---------------- | -- FP16 Calls --- | -- BF16 Calls --- | --- FP32 Calls--- | -- Other Calls -->
+            >>> #   cast                                    |  1                |  0                |  2                |  0
+            >>> #   conv2d                                  |  1                |  0                |  0                |  0
+            >>> #   elementwise_add                         |  0                |  0                |  1                |  0
+            >>> #   reshape                                 |  0                |  0                |  1                |  0
+            >>> # <----------------------------------------------------- op count: 4 ------------------------------------------------------>
+
+    """
+    enable_operator_stats_collection()
+    yield
+    disable_operator_stats_collection()
+
+
+def compare_accuracy(
+    dump_path: str,
+    another_dump_path: str,
+    output_filename: str,
+    loss_scale: float = 1,
+    dump_all_tensors: bool = False,
+) -> None:
+    r"""
+    This is a precision comparison tool that can be used to compare log data of float16 and float32.
+
+    Args:
+        dump_path(str): The path of the running log, such as the log for execution using the float32 data type.
+        another_dump_path(str): the path of another running log ,such as the log for execution using the float16 data type.
+        output_filename(str): the excel file name of compare output.
+        loss_scale(float, optional): the loss_scale during the training phase. Default is 1.
+        dump_all_tensors(bool, optional): dump all tensor, It is currently not support. Default is False.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> import paddle
+            >>> from paddle.base import core
+            >>> try:
+            ...     import xlsxwriter as xlw
+            ... except ImportError:
+            ...     import subprocess
+            ...
+            ...     subprocess.check_call(
+            ...         ['python', '-m', 'pip', 'install', 'xlsxwriter==3.0.9'],
+            ...     )
+            ...     import xlsxwriter as xlw
+            ...
+            ...     if core.is_compiled_with_cuda():
+            ...         paddle.set_flags({"FLAGS_check_nan_inf": 1, "FLAGS_check_nan_inf_level": 3})
+            ...         path = "workerlog_log_dir"
+            ...         paddle.base.core.set_nan_inf_debug_path(path)
+            ...         x = paddle.to_tensor([2, 3, 4, 0], dtype="float32")
+            ...         y = paddle.to_tensor([1, 5, 2, 0], dtype="float32")
+            ...         z1 = x + y
+            ...         out_excel = "compare_accuracy_out_excel.csv"
+            ...         paddle.amp.debugging.compare_accuracy(
+            ...             path,
+            ...             path,
+            ...             out_excel,
+            ...             loss_scale=1,
+            ...             dump_all_tensors=False,
+            ...         )
+    """
+    assert dump_all_tensors is False, "It is currently not supported."
+    paddle.amp.accuracy_compare.compare_accuracy(
+        dump_path,
+        another_dump_path,
+        output_filename,
+        loss_scale,
+        dump_all_tensors=False,
+    )
+
+
+def enable_tensor_checker(checker_config: TensorCheckerConfig) -> None:
+    """
+    The enable_tensor_checker(checker_config) function enables model-level accuracy checking and is used in combination with disables_tensor_checker() to achieve model-level precision checking by checking the output Tensors of all operators within the specified range.
+
+    Args:
+        checker_config(TensorCheckerConfig): Checker_config is to collect the configuration for checking NaN and Inf values in the tensors of a module or operator.
+
+    Note:
+        If disable_tensor_checker() is called before backward(), the gradient operator will not be checked.
+        If disable_tensor_checker() is called before optimizer.step(), the optimizer and other weight update related operators will not be checked.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> import paddle
+
+            >>> checker_config = paddle.amp.debugging.TensorCheckerConfig(
+            ...     enable=True, debug_mode=paddle.amp.debugging.DebugMode.CHECK_NAN_INF
+            ... )
+            >>> paddle.amp.debugging.enable_tensor_checker(checker_config)
+
+            >>> x = paddle.to_tensor([1, 0, 3], place=paddle.CPUPlace(), dtype='float32', stop_gradient=False)
+            >>> y = paddle.to_tensor([0.2, 0, 0.5], place=paddle.CPUPlace(), dtype='float32')
+            >>> res = paddle.pow(x, y)
+            >>> paddle.autograd.backward(res, retain_graph=True)
+            >>> paddle.amp.debugging.disable_tensor_checker()
+            >>> # [PRECISION] [ERROR] in [device=cpu, op=elementwise_pow_grad, tensor=, dtype=fp32], numel=3, num_nan=1, num_inf=0, num_zero=0, max=2.886751e-01, min=2.000000e-01, mean=-nan
+
+            >>> # when DebugMode.CHECK_NAN_INF_AND_ABORT and stack_height_limit = 1
+            >>> # Traceback (most recent call last):
+            >>> #   File "tp.py", line 8, in <module>
+            >>> #     res = paddle.pow(x, y)
+            >>> #   File "/usr/local/lib/python3.8/dist-packages/paddle/tensor/math.py", line 447, in pow
+            >>> #     return _C_ops.elementwise_pow(x, y)
+
+    """
+    if checker_config.update_and_check_step_id():
+        checker_config.start_check_nan_inf()
+    else:
+        checker_config.stop_check_nan_inf()
+
+
+def disable_tensor_checker() -> None:
+    """
+    disable_tensor_checker() is used to disable accuracy checking, and is used together with enable_tensor_checker(config) to achieve model-level precision checking by checking the output Tensors of all operators within the specified range.
+
+    Note:
+        If disable_tensor_checker() is called before backward(), the gradient operator will not be checked;
+        If disable_tensor_checker() is called before optimizer.step(), the optimizer and other weight update related operators will not be checked.
+
+    Examples:
+
+        .. code-block:: pycon
+
+            >>> import paddle
+
+            >>> checker_config = paddle.amp.debugging.TensorCheckerConfig(
+            ...     enable=True, debug_mode=paddle.amp.debugging.DebugMode.CHECK_NAN_INF
+            ... )
+            >>> paddle.amp.debugging.enable_tensor_checker(checker_config)
+
+            >>> x = paddle.to_tensor([1, 0, 3], place=paddle.CPUPlace(), dtype='float32', stop_gradient=False)
+            >>> y = paddle.to_tensor([0.2, 0, 0.5], place=paddle.CPUPlace(), dtype='float32')
+            >>> res = paddle.pow(x, y)
+            >>> paddle.autograd.backward(res, retain_graph=True)
+            >>> paddle.amp.debugging.disable_tensor_checker()
+            >>> # [PRECISION] [ERROR] in [device=cpu, op=elementwise_pow_grad, tensor=, dtype=fp32], numel=3, num_nan=1, num_inf=0, num_zero=0, max=2.886751e-01, min=2.000000e-01, mean=-nan
+
+            >>> # when DebugMode.CHECK_NAN_INF_AND_ABORT and stack_height_limit = 1
+            >>> # Traceback (most recent call last):
+            >>> #      res = paddle.pow(x, y)
+            >>> #    File "/usr/local/lib/python3.8/dist-packages/paddle/tensor/math.py", line 447, in pow
+            >>> #      return _C_ops.elementwise_pow(x, y)
+
+    """
+    paddle.set_flags({"FLAGS_check_nan_inf": 0})