chore: import upstream snapshot with attribution

tools/CheckPRTemplate.py

@@ -0,0 +1,339 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import sys
+
+import httpx
+import requests
+
+PR_checkTemplate = ['Paddle']
+
+BRANCH = os.environ['BRANCH']
+if BRANCH.startswith("develop"):
+    REPO_TEMPLATE = {
+        "Paddle": r'''### PR Category(.*[^\s].*)### PR Types(.*[^\s].*)### Description(.*[^\s].*)### 是否引起精度变化(.*[^\s].*)'''
+    }
+elif BRANCH.startswith("release"):
+    REPO_TEMPLATE = {
+        "Paddle": r'''### PR Category(.*[^\s].*)### PR Types(.*[^\s].*)### Description(.*?devPR:https://github\.com/PaddlePaddle/Paddle/pull/.*?)(?:\n###|\Z)'''
+    }
+else:
+    REPO_TEMPLATE = {
+        "Paddle": r'''### PR Category(.*[^\s].*)### PR Types(.*[^\s].*)### Description(.*[^\s].*)'''
+    }
+
+
+def re_rule(body, CHECK_TEMPLATE):
+    PR_RE = re.compile(CHECK_TEMPLATE, re.DOTALL)
+    result = PR_RE.search(body)
+    return result
+
+
+def extract_pr_links(description_text):
+    pattern = r'(?:https://github\.com/PaddlePaddle/Paddle/pull/|#)(\d+)'
+    return re.findall(pattern, description_text)
+
+
+def check_link_accessible(url):
+    try:
+        response = requests.head(url, allow_redirects=True, timeout=5)
+        return response.status_code == 200
+    except requests.RequestException:
+        return False
+
+
+def parameter_accuracy(body):
+    PR_dic = {}
+    PR_Category = [
+        'User Experience',
+        'Execute Infrastructure',
+        'Operator Mechanism',
+        'CINN',
+        'Custom Device',
+        'Performance Optimization',
+        'Distributed Strategy',
+        'Parameter Server',
+        'Communication Library',
+        'Auto Parallel',
+        'Inference',
+        'Environment Adaptation',
+    ]
+    PR_Types = [
+        'New features',
+        'Bug fixes',
+        'Improvements',
+        'Performance',
+        'BC Breaking',
+        'Deprecations',
+        'Docs',
+        'Devs',
+        'Not User Facing',
+        'Security',
+        'Others',
+    ]
+
+    Accuracy_Change = [
+        '是',
+        '否',
+    ]
+    body = re.sub("\r\n", "", body)
+    type_end = body.find('### PR Types')
+    changes_end = body.find('### Description')
+    PR_dic['PR Category'] = body[len('### PR Category') : type_end]
+    PR_dic['PR Types'] = body[type_end + len('### PR Types') : changes_end]
+    message = ''
+    for key in PR_dic:
+        test_list = PR_Category if key == 'PR Category' else PR_Types
+        test_list_lower = [l.lower() for l in test_list]
+        value = PR_dic[key].strip().split(',')
+        single_mess = ''
+        if len(value) == 1 and value[0] == '':
+            message += f'{key} should be in {test_list}. but now is None.'
+        else:
+            for i in value:
+                i = i.strip().lower()
+                if i not in test_list_lower:
+                    single_mess += f'{i}.'
+            if len(single_mess) != 0:
+                message += f'{key} should be in {test_list}. but now is [{single_mess}].'
+    if BRANCH.startswith("release"):
+        PR_dic['Description'] = body[changes_end + len('### Description') :]
+        des_pr_id = extract_pr_links(PR_dic['Description'])
+        if len(des_pr_id) == 0 or not check_link_accessible(
+            "https://github.com/PaddlePaddle/Paddle/pull/" + str(des_pr_id[0])
+        ):
+            message += 'The PR link does not exist. To merge into the release branch, you need to merge into the develop branch first and then cherry-pick it to the release branch. Please merge into develop first and fill in the PR link in the Description. Use this format: devPR:https://github.com/PaddlePaddle/Paddle/pull/xxxx'
+
+    if BRANCH.startswith("develop"):
+        accuracy_start = body.find('### 是否引起精度变化')
+        if accuracy_start != -1:
+            # 确保description_start在accuracy_start之后
+            PR_dic['Precision Change Impact'] = body[
+                accuracy_start + len('### 是否引起精度变化') :
+            ]
+        else:
+            PR_dic['Precision Change Impact'] = ''
+        accuracy_value = PR_dic.get('Precision Change Impact', '').strip()
+        print(f'Extracted Precision Change Impact: "{accuracy_value}"')
+        if not accuracy_value:
+            message += '必须填写是否引起精度变化'
+        else:
+            found_valid = False
+            for option in Accuracy_Change:
+                if option in accuracy_value:
+                    found_valid = True
+                    break
+            if not found_valid:
+                message += f'精度变化必须填写为：{Accuracy_Change}. 现在是 {accuracy_value}.'
+    return message
+
+
+def check_precision_change_approval(body, pr_number, pr_user):
+    """Check if PR with precision change has sufficient approval"""
+    # Only check for develop branch
+    if not BRANCH.startswith("develop"):
+        return True, "Not develop branch, skip precision change approval check"
+
+    # Check if involves precision change
+    body_without_comments = re.sub(r'<!--.*?-->', '', body, flags=re.DOTALL)
+    accuracy_start = body_without_comments.find('### 是否引起精度变化')
+    if accuracy_start != -1:
+        precision_text = body_without_comments[
+            accuracy_start + len('### 是否引起精度变化') :
+        ]
+    else:
+        precision_text = ''
+        return False, '未匹配到是否引起精度变化字段，无法判断是否涉及精度变化'
+    precision_text = precision_text.strip()
+
+    print(f'Extracted precision text: "{precision_text}"')
+    has_precision_change = '是' in precision_text
+
+    if not has_precision_change:
+        return True, "不涉及精度变化，无需检查"
+    REQUIRED_APPROVERS = ['sneaxiy']
+    REQUIRED_APPROVERS_LOWER = [user.lower() for user in REQUIRED_APPROVERS]
+    print(
+        f"PR {pr_number} involves precision change, checking approvals from required approvers: {REQUIRED_APPROVERS}"
+    )
+
+    # If has precision change, check approval status
+    reviews_url = f"https://api.github.com/repos/PaddlePaddle/Paddle/pulls/{pr_number}/reviews"
+    headers = {
+        'Authorization': 'token ' + GITHUB_API_TOKEN,
+        'Accept': 'application/vnd.github+json',
+    }
+
+    try:
+        response = httpx.get(reviews_url, headers=headers, timeout=10)
+        if response.status_code != 200:
+            return (
+                False,
+                f"Cannot get review information: {response.status_code}",
+            )
+
+        reviews = response.json()
+
+        # Check for approved reviews
+        approved_by = {}
+        for approver in REQUIRED_APPROVERS:
+            approved_by[approver] = False
+
+        if pr_user.lower() in REQUIRED_APPROVERS_LOWER:
+            approved_by[pr_user] = True
+            print(f"  ✓ Approved by PR author {pr_user}")
+
+        # Check all reviews
+        for review in reviews:
+            if review['state'] == 'APPROVED':
+                reviewer = review['user']['login']
+                if reviewer.lower() in REQUIRED_APPROVERS_LOWER:
+                    # Find the correct casing for this reviewer
+                    for approver in REQUIRED_APPROVERS:
+                        if approver.lower() == reviewer.lower():
+                            approved_by[approver] = True
+                            print(f"  ✓ Approved by {approver}")
+                            break
+        # Check if all required approvers have approved
+        missing_approvers = [
+            approver
+            for approver, has_approved in approved_by.items()
+            if not has_approved
+        ]
+        if len(missing_approvers) == 0:
+            approvers_list = ", ".join(REQUIRED_APPROVERS)
+            return (
+                True,
+                f"✅ All required approvers have approved: {approvers_list}",
+            )
+        else:
+            approved_list = [a for a in REQUIRED_APPROVERS if approved_by[a]]
+            missing_list = ", ".join(missing_approvers)
+
+            if len(approved_list) > 0:
+                approved_str = ", ".join(approved_list)
+                return (
+                    False,
+                    f"❌ Missing approvals from: {missing_list}. Approved by: {approved_str}",
+                )
+            else:
+                return (
+                    False,
+                    f"❌ No approvals from required approvers. Missing: {missing_list}",
+                )
+    except Exception as e:
+        return False, f"Error checking approval: {e!s}"
+
+
+def checkComments(url):
+    headers = {
+        'Authorization': 'token ' + GITHUB_API_TOKEN,
+    }
+    response = httpx.get(
+        url, headers=headers, timeout=None, follow_redirects=True
+    ).json()
+    return response
+
+
+def checkPRTemplate(repo, body, CHECK_TEMPLATE):
+    """
+    Check if PR's description meet the standard of template
+    Args:
+        body: PR's Body.
+        CHECK_TEMPLATE: check template str.
+    Returns:
+        res: True or False
+    """
+    res = False
+    comment_pattern = re.compile(r'<!--.*?-->', re.DOTALL)
+    if body is None:
+        body = ''
+    body = comment_pattern.sub('', body)
+    result = re_rule(body, CHECK_TEMPLATE)
+    message = ''
+    if len(CHECK_TEMPLATE) == 0 and len(body) == 0:
+        res = False
+    elif result is not None:
+        message = parameter_accuracy(body)
+        res = True if message == '' else False
+    elif result is None:
+        res = False
+        message = parameter_accuracy(body)
+        if BRANCH.startswith("release") and len(message) == 0:
+            message = 'The PR link does not exist. To merge into the release branch, you need to merge into the develop branch first and then cherry-pick it to the release branch. Please merge into develop first and fill in the PR link in the Description. Use this format: devPR:https://github.com/PaddlePaddle/Paddle/pull/xxxx'
+    return res, message
+
+
+def pull_request_event_template(event, repo, *args, **kwargs):
+    pr_effect_repos = PR_checkTemplate
+    pr_num = event['number']
+    url = event["comments_url"]
+    BODY = event["body"]
+    sha = event["head"]["sha"]
+    title = event["title"]
+    pr_user = event["user"]["login"]
+    print(f'receive data : pr_num: {pr_num}, title: {title}, user: {pr_user}')
+    if repo in pr_effect_repos:
+        CHECK_TEMPLATE = REPO_TEMPLATE[repo]
+        global check_pr_template
+        global check_pr_template_message
+        check_pr_template, check_pr_template_message = checkPRTemplate(
+            repo, BODY, CHECK_TEMPLATE
+        )
+        print(f"check_pr_template: {check_pr_template} pr: {pr_num}")
+        if check_pr_template is False:
+            print("ERROR MESSAGE:", check_pr_template_message)
+            sys.exit(7)
+        else:
+            print("check approve")
+            approval_ok, approval_msg = check_precision_change_approval(
+                BODY, pr_num, pr_user
+            )
+            print(
+                f"Approval check result: {approval_ok}, message: {approval_msg}"
+            )
+            if not approval_ok:
+                check_pr_template_message = approval_msg
+                print("ERROR MESSAGE:", check_pr_template_message)
+                sys.exit(8)
+            sys.exit(0)
+
+
+def get_a_pull(pull_id):
+    url = "https://api.github.com/repos/PaddlePaddle/Paddle/pulls/ " + str(
+        pull_id
+    )
+
+    payload = {}
+    headers = {
+        'Authorization': 'token ' + GITHUB_API_TOKEN,
+        'Accept': 'application/vnd.github+json',
+    }
+    response = httpx.request(
+        "GET", url, headers=headers, data=payload, follow_redirects=True
+    )
+    return response.json()
+
+
+def main(org, repo, pull_id):
+    pull_info = get_a_pull(pull_id)
+    pull_request_event_template(pull_info, repo)
+
+
+if __name__ == "__main__":
+    AGILE_PULL_ID = os.getenv("AGILE_PULL_ID")
+    GITHUB_API_TOKEN = os.getenv("GITHUB_API_TOKEN")
+    main("PaddlePaddle", "Paddle", AGILE_PULL_ID)

tools/CrossStackProfiler/CspChromeTraceFormatter.py

@@ -0,0 +1,118 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+
+
+class ChromeTraceFormatter:
+    def __init__(self):
+        self._events = []
+        self._metadata = []
+
+    def _create_event(self, ph, category, name, pid, tid, timestamp):
+        """Creates a new Chrome Trace event.
+
+        For details of the file format, see:
+        https://github.com/catapult-project/catapult/blob/master/tracing/README.md
+
+        Args:
+          ph:  The type of event - usually a single character.
+          category: The event category as a string.
+          name:  The event name as a string.
+          pid:  Identifier of the process generating this event as an integer.
+          tid:  Identifier of the thread generating this event as an integer.
+          timestamp:  The timestamp of this event as a long integer.
+
+        Returns:
+          A JSON compatible event object.
+        """
+        event = {}
+        event['ph'] = ph
+        event['cat'] = category
+        event['name'] = name
+        event['pid'] = pid
+        event['tid'] = tid
+        event['ts'] = timestamp
+        return event
+
+    def emit_pid(self, name, pid):
+        """Adds a process metadata event to the trace.
+
+        Args:
+          name:  The process name as a string.
+          pid:  Identifier of the process as an integer.
+        """
+        event = {}
+        event['name'] = 'process_name'
+        event['ph'] = 'M'
+        event['pid'] = pid
+        event['args'] = {'name': name}
+        self._metadata.append(event)
+
+    def emit_region(self, timestamp, duration, pid, tid, category, name, args):
+        """Adds a region event to the trace.
+
+        Args:
+          timestamp:  The start timestamp of this region as a long integer.
+          duration:  The duration of this region as a long integer.
+          pid:  Identifier of the process generating this event as an integer.
+          tid:  Identifier of the thread generating this event as an integer.
+          category: The event category as a string.
+          name:  The event name as a string.
+          args:  A JSON-compatible dictionary of event arguments.
+        """
+        event = self._create_event('X', category, name, pid, tid, timestamp)
+        event['dur'] = duration
+        event['args'] = args
+        self._events.append(event)
+
+    def emit_counter(self, category, name, pid, timestamp, counter, value):
+        """Emits a record for a single counter.
+
+        Args:
+            category: The event category as string
+            name: The event name as string
+            pid: Identifier of the process generating this event as integer
+            timestamp: The timestamps of this event as long integer
+            counter: Name of the counter as string
+            value: Value of the counter as integer
+            tid: Thread id of the allocation as integer
+        """
+        event = self._create_event('C', category, name, pid, 0, timestamp)
+        event['args'] = {counter: value}
+        self._events.append(event)
+
+    def format_to_string(self, pretty=False):
+        """Formats the chrome trace to a string.
+
+        Args:
+          pretty: (Optional.)  If True, produce human-readable JSON output.
+
+        Returns:
+          A JSON-formatted string in Chrome Trace format.
+        """
+        trace = {}
+        trace['traceEvents'] = self._metadata + self._events
+        if pretty:
+            return json.dumps(trace, indent=4, separators=(',', ': '))
+        else:
+            return json.dumps(trace, separators=(',', ':'))
+
+    def clear(self):
+        self._events = []
+        self._metadata = []
+
+
+if __name__ == "__main__":
+    pass

tools/CrossStackProfiler/CspFileReader.py

@@ -0,0 +1,410 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import glob
+import json
+import logging
+import os
+import time
+from multiprocessing import Lock
+
+""" Some terms to clarify the code
+    in most case, one or more parameters may be set as input args for a class or a function
+    in form of single variable or k-v dict
+
+    1.  trainerId
+    2.  gpuId
+    3.  rankId
+    4.  gpuPerTrainer
+    5.  groupSize
+    6.  groupId
+    7.  groupNum
+    8.  displaySize
+    9.  dataPath
+    10. resultPath
+    11. fileOrganizeForm -- "byRank" OR "byTrainer" or "other"
+
+"""
+
+PIPELINEINFO_TRACE_NUM = 1
+
+dcgmMetricParameterMap = {
+    "02_gpuUtility": [("GPUTL", "GPUTL"), ("GRACT", "GRACT")],
+    "03_smUtility": [("SMACT", "SMACT"), ("SMOCC", "SMOCC")],
+    "04_memUtility": [("FB_USED_RATIO", "FB_USED_RATIO"), ("DRAMA", "DRAMA")],
+    "05_txUtility": [
+        ("NVLTX", "NVLTX"),
+        ("NVLRX", "NVLRX"),
+        ("PCITX", "PCITX"),
+        ("PCIRX", "PCIRX"),
+    ],
+    "06_calUtility": [
+        ("FP32A", "FP32A"),
+        ("FP16A", "FP16A"),
+        ("TENSO", "TENSO"),
+    ],
+}
+DCGMINFO_TRACE_NUM = len(dcgmMetricParameterMap.keys())
+NETINFO_TRACE_NUM = 2
+
+DCGM_PATH = "dcgm"
+NET_PATH = "net"
+TIME_PATH = "time"
+PROFILE_PATH = "profile"
+
+FILEORGANIZEFORM_BYRANK = "byRank"
+FILEORGANIZEFORM_BYTRAINER = "byTrainer"
+FILEORGANIZEFORM_BYOTHER = "other"
+FILEORGANIZEFORM = [
+    FILEORGANIZEFORM_BYRANK,
+    FILEORGANIZEFORM_BYTRAINER,
+    FILEORGANIZEFORM_BYOTHER,
+]
+
+
+class FileReader:
+    def __init__(self, logger, args):
+        self._logger = logger
+        self._args = args
+
+        self._fileList = []
+        self._fileNum = 0
+
+        self._dataPath = ""
+        self._groupSize = 0
+        self._displaySize = 0
+        self._organizeForm = FILEORGANIZEFORM_BYOTHER
+        self._gpuPerTrainer = 0
+
+        self._checkArgs()
+        self._getFileList()
+
+        self._lock = Lock()
+
+    def printArgs(self):
+        self._logger.info("dataPath:")
+        self._logger.info(self._dataPath)
+        self._logger.info("groupSize:")
+        self._logger.info(self._groupSize)
+        self._logger.info("displaySize:")
+        self._logger.info(self._displaySize)
+        self._logger.info("organizeForm:")
+        self._logger.info(self._organizeForm)
+        self._logger.info("gpuPerTrainer:")
+        self._logger.info(self._gpuPerTrainer)
+        self._logger.info("minTimeStamp:")
+        self._logger.info(self._minTimeStamp)
+
+    def _checkArgsKey(self, key, type):
+        if key not in self._args:
+            raise KeyError(f"args should has key [{key}]!")
+
+        if not isinstance(self._args[key], type):
+            raise TypeError(
+                f"Invalid type of key [{key}] in args dict, it should be a {type}!"
+            )
+
+        exec(f'self._{key} = self._args["{key}"]')
+
+    def _align_ts(self, ts):
+        return ts - self._minTimeStamp
+
+    def _checkArgs(self):
+        if not isinstance(self._args, dict):
+            raise TypeError("Invalid type of args, it should be a dict!")
+
+        self._checkArgsKey("organizeForm", str)
+        if (
+            self._organizeForm not in FILEORGANIZEFORM
+            or self._organizeForm == FILEORGANIZEFORM_BYOTHER
+        ):
+            raise NotImplementedError(
+                f"we have not known how to process this form of file [{self._organizeForm}]!"
+            )
+
+        self._checkArgsKey("gpuPerTrainer", int)
+
+        self._checkArgsKey("dataPath", str)
+        if not os.path.exists(self._dataPath):
+            raise OSError(f"input data path [{self._dataPath}] not existed!")
+
+        self._checkArgsKey("groupSize", int)
+        self._checkArgsKey("displaySize", int)
+        self._checkArgsKey("minTimeStamp", int)
+
+    def getFileListByGroup(self, groupId):
+        lIndex = 0
+        rIndex = 0
+
+        if self._organizeForm == FILEORGANIZEFORM_BYTRAINER:
+            lIndex = groupId * self._groupSize
+            rIndex = (groupId + 1) * self._groupSize
+        elif self._organizeForm == FILEORGANIZEFORM_BYRANK:
+            lIndex = groupId * self._groupSize * self._gpuPerTrainer
+            rIndex = (groupId + 1) * self._groupSize * self._gpuPerTrainer
+
+        try:
+            return self._fileList[lIndex:rIndex]
+        except IndexError:
+            raise IndexError("invalid index of file list")
+
+    def getFileList(self):
+        return self._getFileList
+
+    def _cmp(self, x, y):
+        return self._getId(x, self._organizeForm) - self._getId(
+            y, self._organizeForm
+        )
+
+    def _getFileList(self):
+        self._fileList = glob.glob(os.path.join(self._dataPath, "*.*"))
+
+        # check unique
+        idList = []
+        newFileList = []
+        for file in self._fileList:
+            id = self._getId(file, self._organizeForm)
+            if id not in idList:
+                idList.append(id)
+                newFileList.append(file)
+            else:
+                raise NotImplementedError(
+                    f"[{file}] is repeated by id, we do not know how to process it!"
+                )
+
+        if not self._fileList:
+            if (
+                self._getId(self._fileList[-1]) - self._getId(self._fileList[0])
+            ) != len(self._fileList) - 1:
+                raise Exception("The file id should be continuous!")
+
+        # sort
+        def _sortBySuffix(elem):
+            return int(elem.split(".")[-1])
+
+        self._fileList.sort(key=_sortBySuffix)
+
+        if not self._fileList:
+            self._logger.warning(
+                f"we can not find any file in dir [{self._dataPath}]!"
+            )
+        else:
+            self._logger.info(
+                "file list in dir [{}] is : {} !".format(
+                    self._dataPath, ',  '.join(self._fileList)
+                )
+            )
+
+        return self._fileList
+
+    def _getId(self, fileName, organizeForm, sed="."):
+        if self._organizeForm != organizeForm:
+            raise TypeError(
+                f"Can not get rank id when organize form is not {organizeForm}!"
+            )
+
+        if not os.path.isfile(fileName):
+            raise OSError(f"[{fileName}] is not a valid file!")
+
+        try:
+            prefix_str = fileName.split(sed)[-1]
+            try:
+                return int(prefix_str)
+            except ValueError as e:
+                print(e)
+                raise TypeError(f"invalid fileName [{fileName}]")
+
+        except IndexError as e:
+            print(e)
+            raise TypeError(
+                f"invalid fileName [{fileName}], the prefix should be a number!"
+            )
+
+    def getRankId(self, fileName, sed="."):
+        return self._getId(fileName, FILEORGANIZEFORM_BYRANK, sed)
+
+    def getRankNum(self):
+        if self._organizeForm == FILEORGANIZEFORM_BYRANK:
+            return len(self._fileList)
+
+        elif self._organizeForm == FILEORGANIZEFORM_BYTRAINER:
+            return len(self._fileList) * self._gpuPerTrainer
+
+    def getTrainerNum(self):
+        if self._organizeForm == FILEORGANIZEFORM_BYRANK:
+            return len(self._fileList) / self._gpuPerTrainer
+
+        elif self._organizeForm == FILEORGANIZEFORM_BYTRAINER:
+            return len(self._fileList)
+
+    def getTrainerId(self, fileName, sed="."):
+        return self._getId(fileName, FILEORGANIZEFORM_BYTRAINER, sed)
+
+    def _splitTaskListForMultiProcess(self, ls, n):
+        if not isinstance(ls, list) or not isinstance(n, int):
+            return []
+        ls_len = len(ls)
+        if n <= 0 or 0 == ls_len:
+            return []
+        if n >= ls_len:
+            return [[i] for i in ls]
+        else:
+            j = int((ls_len + n - 1) / n)
+            k = ls_len % n
+            ls_return = []
+            end = 0
+            for i in range(0, (n) * j, j):
+                if i < len(ls) and (i + j) < len(ls):
+                    ls_return.append(ls[i : i + j])
+                    end = i + j
+            ls_return.append(ls[end:])
+            return ls_return
+
+    def getOpInfoFileName(self, groupId, gpuId, tmpPath="./tmp"):
+        return self.getFileName("opinfo", groupId, gpuId, tmpPath)
+
+    def getPipeLineInfoFileName(self, groupId, gpuId, tmpPath="./tmp"):
+        return self.getFileName("pipelineinfo", groupId, gpuId, tmpPath)
+
+    def getDCGMInfoFileName(self, groupId, gpuId, tmpPath="./tmp"):
+        return self.getFileName("dcgm", groupId, gpuId, tmpPath)
+
+    def getFileName(self, name, groupId, gpuId, tmpPath="./tmp"):
+        return os.path.join(tmpPath, f"{name}_{groupId}_{gpuId}.json")
+
+    def getOpInfoDict(self, groupId, gpuId, tmpPath="./tmp"):
+        return self.getDict("opinfo", groupId, gpuId, tmpPath)
+
+    def getDcgmInfoDict(self, groupId, gpuId, tmpPath="./tmp"):
+        return self.getDict("dcgm", groupId, gpuId, tmpPath)
+
+    def getDict(self, name, groupId, gpuId, tmpPath="./tmp"):
+        fileName = self.getFileName(name, groupId, gpuId, tmpPath)
+        if not os.path.isfile(fileName):
+            raise OSError(f"[{fileName}] does not existed!")
+
+        data = {}
+        with open(fileName, "r") as rf:
+            try:
+                data = json.load(rf)
+            except Exception:
+                self._logger.error(f"read [{fileName}] error. not a json file!")
+                raise TypeError(f"read [{fileName}] error. not a json file!")
+        return data
+
+    def dumpOpInfoDict(
+        self, data, groupId, gpuId, pretty=False, tmpPath="./tmp"
+    ):
+        return self.dumpDict(
+            data, "opinfo", groupId, gpuId, pretty=False, tmpPath="./tmp"
+        )
+
+    def dumpDCGMDict(self, data, groupId, gpuId, pretty=False, tmpPath="./tmp"):
+        return self.dumpDict(
+            data, "dcgm", groupId, gpuId, pretty=False, tmpPath="./tmp"
+        )
+
+    def dumpDict(
+        self, data, name, groupId, gpuId, pretty=False, tmpPath="./tmp"
+    ):
+        self._lock.acquire()
+        if not os.path.exists(tmpPath):
+            os.makedirs(tmpPath)
+        self._lock.release()
+        if pretty:
+            jsObj = json.dumps(data, indent=4, separators=(',', ': '))
+        else:
+            jsObj = json.dumps(data, separators=(',', ':'))
+
+        fileName = self.getFileName(name, groupId, gpuId, tmpPath)
+        if os.path.isfile(fileName):
+            os.remove(fileName)
+
+        fileObject = open(fileName, 'w')
+        fileObject.write(jsObj)
+        fileObject.close()
+        self._logger.info(f"dump [{fileName}] successfully!")
+
+
+def getLogger():
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+
+    rq = time.strftime('%Y%m%d%H%M.%s', time.localtime(time.time()))
+    log_path = os.path.dirname(os.getcwd()) + '/Logs/'
+    if not os.path.exists(log_path):
+        os.makedirs(log_path)
+
+    log_name = log_path + rq + '.log'
+    logfile = log_name
+    fh = logging.FileHandler(logfile, mode='w')
+    fh.setLevel(logging.DEBUG)
+
+    formatter = logging.Formatter(
+        "%(asctime)s - %(filename)s[line:%(lineno)d] - %(process)d - %(levelname)s: %(message)s"
+    )
+    fh.setFormatter(formatter)
+
+    logger.addHandler(fh)
+    return logger
+
+
+def test_FileReader(args):
+    try:
+        testReader = FileReader(None, args)
+    except Exception as e:
+        print(e)
+    else:
+        testReader.printArgs()
+
+
+if __name__ == "__main__":
+    args = 0
+    test_FileReader(args)
+
+    args = {
+        "dataPath": ".",
+        "groupSize": 1,
+        "displaySize": 1,
+        "gpuPerTrainer": 8,
+        "organizeForm": FILEORGANIZEFORM_BYOTHER,
+    }
+    test_FileReader(args)
+
+    args = {
+        "dataPath": ".",
+        "groupSize": 1,
+        "displaySize": 1,
+        "gpuPerTrainer": 8,
+        "organizeForm": FILEORGANIZEFORM_BYTRAINER,
+    }
+    test_FileReader(args)
+
+    args = {
+        "dataPath": "./res",
+        "groupSize": 1,
+        "displaySize": 1,
+        "gpuPerTrainer": 8,
+        "organizeForm": FILEORGANIZEFORM_BYTRAINER,
+    }
+    test_FileReader(args)
+
+    args = {
+        "dataPath": ".",
+        "groupSize": "",
+        "displaySize": 1,
+        "gpuPerTrainer": 8,
+        "organizeForm": FILEORGANIZEFORM_BYTRAINER,
+    }
+    test_FileReader(args)

tools/CrossStackProfiler/CspReporter.py

@@ -0,0 +1,257 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import glob
+import os
+from multiprocessing import Process
+
+from CspFileReader import (
+    DCGM_PATH,
+    FILEORGANIZEFORM_BYRANK,
+    FILEORGANIZEFORM_BYTRAINER,
+    NET_PATH,
+    PROFILE_PATH,
+    TIME_PATH,
+    getLogger,
+)
+from DCGMFileReader import dcgmFileReader
+from ProfileFileReader import profileFileReader
+
+
+def get_argparse():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        '--profile_path',
+        type=str,
+        default='.',
+        help='Working path that store the monitor data.',
+    )
+
+    parser.add_argument(
+        '--timeline_path',
+        type=str,
+        default='.',
+        help='Output timeline file name.',
+    )
+
+    parser.add_argument(
+        '--gpuPerTrainer', type=int, default=8, help='Gpus per trainer.'
+    )
+
+    parser.add_argument(
+        '--trainerNum', type=int, default=4, help='Num of trainer.'
+    )
+
+    parser.add_argument(
+        '--groupSize', type=int, default=8, help='Num of trainer in a group.'
+    )
+
+    parser.add_argument(
+        '--displaySize',
+        type=int,
+        default=2,
+        help='Num of line need to display in a group.',
+    )
+
+    return parser.parse_args()
+
+
+class CspReporter:
+    def __init__(self, args):
+        self._args = args
+        print(self._args)
+
+        self._workPath = self._args.profile_path
+        self._saveFilePath = self._args.timeline_path
+        self._gpuPerTrainer = self._args.gpuPerTrainer
+        self._groupSize = self._args.groupSize
+        self._displaySize = self._args.displaySize
+        self._trainerNum = self._args.trainerNum
+
+        self._checkArgs()
+
+        self._init_logger()
+        self._init_timeInfo()
+        self._init_reader()
+
+    def _checkArgs(self):
+        if self._trainerNum % self._groupSize != 0:
+            raise Exception(
+                f"Input args error: trainerNum[{self._trainerNum}] %% groupSize[{self._groupSize}] != 0"
+            )
+
+    def _init_logger(self):
+        self._logger = getLogger()
+
+    def _init_reader(self):
+        self._dcgmPath = os.path.join(self._workPath, DCGM_PATH)
+        self._netPath = os.path.join(self._workPath, NET_PATH)
+        self._profilePath = os.path.join(self._workPath, PROFILE_PATH)
+
+        self._netFileReaderArgs = {
+            "dataPath": self._netPath,
+            "groupSize": self._groupSize,
+            "displaySize": self._displaySize,
+            "gpuPerTrainer": self._gpuPerTrainer,
+            "minTimeStamp": self._minTimeStamp,
+            "organizeForm": FILEORGANIZEFORM_BYTRAINER,
+        }
+
+        self._dcgmFileReaderArgs = {
+            "dataPath": self._dcgmPath,
+            "groupSize": self._groupSize,
+            "displaySize": self._displaySize,
+            "gpuPerTrainer": self._gpuPerTrainer,
+            "minTimeStamp": self._minTimeStamp,
+            "organizeForm": FILEORGANIZEFORM_BYTRAINER,
+        }
+
+        self._profileFileReaderArgs = {
+            "dataPath": self._profilePath,
+            "groupSize": self._groupSize,
+            "displaySize": self._displaySize,
+            "gpuPerTrainer": self._gpuPerTrainer,
+            "minTimeStamp": self._minTimeStamp,
+            "organizeForm": FILEORGANIZEFORM_BYRANK,
+        }
+
+        self._dcgmFileReader = dcgmFileReader(
+            self._logger, self._dcgmFileReaderArgs
+        )
+        self._profileFileReader = profileFileReader(
+            self._logger, self._profileFileReaderArgs
+        )
+
+    def _init_timeInfo(self):
+        self._timePath = os.path.join(self._workPath, TIME_PATH)
+        self._timeInfo = {}
+        self._minTimeStamp = 0
+        self._set_timeInfo()
+
+    def _set_timeInfo(self, timeFileNamePrefix="time.txt", sed="."):
+        timeFileNameList = glob.glob(
+            os.path.join(self._timePath, timeFileNamePrefix, sed, "*")
+        )
+        for timeFileName in timeFileNameList:
+            trainerId = int(timeFileName.split(sed)[-1])
+            gpuId = int(timeFileName.split(sed)[-2])
+            info = {}
+            with open(timeFileName, "r") as rf:
+                for line in rf:
+                    if line.startswith("start time:"):
+                        info["start_time"] = int(
+                            float(line.split(":")[-1]) * 1e9
+                        )
+
+                        self._minTimeStamp = min(
+                            self._minTimeStamp, info["start_time"]
+                        )
+
+                    if line.startswith("end time:"):
+                        info["end_time"] = int(float(line.split(":")[-1]) * 1e9)
+            if not info:
+                self._timeInfo[gpuId * trainerId] = info
+
+    def _generateTraceFileByGroupAndGpuId(
+        self, pipeLineInfo, netInfo, groupId, gpuId
+    ):
+        dcgmInfoDict = self._dcgmFileReader.getDcgmInfoDict(groupId, gpuId)
+        opInfoDict = self._profileFileReader.getOpInfoDict(groupId, gpuId)
+
+        traceObj = {}
+        traceObj["traceEvents"] = (
+            pipeLineInfo[str(gpuId)]
+            + opInfoDict["traceEvents"]
+            + dcgmInfoDict["traceEvents"]
+            + netInfo["traceEvents"]
+        )
+
+        self._profileFileReader.dumpDict(
+            traceObj, "traceFile", groupId, gpuId, False, self._saveFilePath
+        )
+
+    def _generateTraceFileByGroup(self, groupId, processNum):
+        # first we need to generate pipeline info
+        pipeLineInfo = self._profileFileReader.getPipeLineInfo(
+            groupId, processNum
+        )
+        # second we need to generate dcgm info
+        dcgmInfo = self._dcgmFileReader.getDCGMTraceInfo(groupId, processNum)
+
+        # third we need to generate net info
+        netInfo = {}
+        netInfo["traceEvents"] = []
+        # netInfo = self._netFileReader.parseFileByGroup(groupId, processNum)
+
+        # forth we need to generate op info
+        opInfo = self._profileFileReader.getOPTraceInfo(groupId)
+
+        # finally we need dump this information into disk
+        processPool = []
+        pidList = []
+
+        for gpuId in range(self._gpuPerTrainer):
+            subproc = Process(
+                target=self._generateTraceFileByGroupAndGpuId,
+                args=(
+                    pipeLineInfo,
+                    netInfo,
+                    groupId,
+                    gpuId,
+                ),
+            )
+            processPool.append(subproc)
+            subproc.start()
+            pidList.append(subproc.pid)
+            self._logger.info(
+                f"[traceFile]: process [{subproc.pid}] has been started, total task num is {1} ..."
+            )
+
+        for t in processPool:
+            t.join()
+            pidList.remove(t.pid)
+            self._logger.info(
+                f"[traceFile]: process [{t.pid}] has exited! remained {len(pidList)} process!"
+            )
+
+    def generateTraceFile(self, processNum=8):
+        processPool = []
+        pidList = []
+        for groupId in range(self._trainerNum / self._groupSize):
+            subproc = Process(
+                target=self._generateTraceFileByGroup,
+                args=(
+                    groupId,
+                    processNum,
+                ),
+            )
+            processPool.append(subproc)
+            subproc.start()
+            pidList.append(subproc.pid)
+            self._logger.info(
+                f"[GroupTraceFile]: process [{subproc.pid}] has been started, total task num is {1} ..."
+            )
+        for t in processPool:
+            t.join()
+            pidList.remove(t.pid)
+            self._logger.info(
+                f"[GroupTraceFile]: process [{t.pid}] has exited! remained {len(pidList)} process!"
+            )
+
+
+if __name__ == '__main__':
+    args = get_argparse()
+    tl = CspReporter(args)
+    tl.generateTraceFile()

tools/CrossStackProfiler/DCGMFileReader.py

@@ -0,0 +1,273 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import multiprocessing
+import os
+import re
+import tempfile
+from multiprocessing import Process
+
+import pandas as pd
+from CspFileReader import (
+    FILEORGANIZEFORM_BYTRAINER,
+    PIPELINEINFO_TRACE_NUM,
+    FileReader,
+    dcgmMetricParameterMap,
+    getLogger,
+)
+
+
+class dcgmFileReader(FileReader):
+    def parseFileByGroup(self, groupId, processNum=8):
+        fileFist = self.getFileListByGroup(groupId)
+        displaySize = min(self._displaySize, len(fileFist))
+        fileFist = fileFist[:displaySize]
+
+        if processNum == 0:
+            return self._parseTask(fileFist)
+
+        else:
+            self._logger.info(f"using [{processNum}] process to do this work!")
+            processPool = []
+            pidList = []
+
+            manager = multiprocessing.Manager()
+            q = manager.Queue()
+
+            taskList = self._splitTaskListForMultiProcess(fileFist, processNum)
+            for task in taskList:
+                subproc = Process(
+                    target=self._parseTask,
+                    args=(
+                        task,
+                        q,
+                    ),
+                )
+                processPool.append(subproc)
+                subproc.start()
+                pidList.append(subproc.pid)
+                self._logger.info(
+                    f"[DCGM reader]: process [{subproc.pid}] has been started, total task num is {len(processPool)} ..."
+                )
+
+            for t in processPool:
+                t.join()
+                pidList.remove(t.pid)
+                self._logger.info(
+                    f"[DCGM reader]: process [{t.pid}] has exited! remained {len(pidList)} process!"
+                )
+
+            isFistProcess = True
+            for t in processPool:
+                if isFistProcess:
+                    isFistProcess = False
+                    dcgm_data = q.get()
+                else:
+                    dcgm_data = pd.concat(
+                        [dcgm_data, q.get()], axis=0, join='outer'
+                    )
+
+            return dcgm_data
+
+    def _parseTask(self, taskList, q=None):
+        is_first = True
+        for fileName in taskList:
+            self._logger.info(f"I am processing {fileName}!")
+            tmp_data = self._parseSingleFile(fileName)
+            if tmp_data is None:
+                continue
+
+            if is_first:
+                is_first = False
+                dcgm_data = tmp_data
+            else:
+                dcgm_data = pd.concat(
+                    [dcgm_data, tmp_data], axis=0, join='outer'
+                )
+        dcgm_data = dcgm_data.dropna()
+        if q is not None:
+            q.put(dcgm_data)
+        self._logger.info(f"I finish processing {fileName}!")
+        return dcgm_data
+
+    def _parseSingleFile(self, fileName):
+        trainerId = self.getTrainerId(fileName)
+
+        if not os.path.exists(fileName):
+            logging.warning(fileName + ' not found')
+            return
+
+        regex_list = [
+            (re.compile(r' +'), ','),
+            (re.compile(r'^,'), ''),
+        ]
+
+        csv_tempfile = tempfile.TemporaryFile()
+        with open(fileName, 'r') as fp:
+            has_header = False
+
+            for line in fp:
+                # skip `nvidia-dcgm-dmon.sh` init and fini info lines
+                if (
+                    'nv-hostengine' in line
+                    or 'dmon' in line
+                    or 'Host Engine Listener Started' in line
+                ):
+                    continue
+
+                if not line.strip().startswith(
+                    "GPU"
+                ) and not line.strip().startswith("# Entity"):
+                    continue
+
+                # skip non-needed headers (only the header in 1st line was needed)
+                if line.strip().startswith("# Entity"):
+                    line = line.strip()[2:]
+
+                if 'Entity' == line[0 : len('Entity')]:
+                    if has_header:
+                        continue
+                    else:
+                        has_header = True
+
+                if line.strip().startswith("GPU"):
+                    line = line.strip()[3:]
+
+                for r in regex_list:
+                    line = r[0].sub(r[1], line)
+
+                csv_tempfile.write(bytes(line + "\n"))
+
+        csv_tempfile.seek(0)
+
+        dcgm = pd.read_csv(csv_tempfile, header=0, delimiter=',')
+        # dcgm.info()
+        dcgm['FB_USED_RATIO'] = dcgm['FBUSD'] / dcgm['FBTTL']
+        dcgm['GPUTL'] = dcgm['GPUTL'] / 100.0
+        dcgm['ts'] = dcgm['TIMESTAMP'] * 1e9
+        dcgm['trainerId'] = trainerId
+
+        return dcgm
+
+    def _getDCGMTraceInfoByGpuId(
+        self, groupId, gpuId, dcgm_data, pid_map, q=None
+    ):
+        self._logger.info(
+            f"Begin to generate dcgm info, groupId = {groupId}, gpuID = {gpuId} ..."
+        )
+
+        gpuDcgmData = dcgm_data[dcgm_data['Entity'].isin([gpuId])]
+
+        traceEventList = []
+        for metric, parameterList in dcgmMetricParameterMap.items():
+            metaInfo = {}
+            metaInfo['name'] = 'process_name'
+            metaInfo['ph'] = 'M'
+            metaInfo['pid'] = pid_map[metric]
+            metaInfo['args'] = {'name': metric}
+            traceEventList.append(metaInfo)
+
+        for index, row in gpuDcgmData.iterrows():
+            for metric, parameterList in dcgmMetricParameterMap.items():
+                trainerId = int(row['trainerId']) % self._groupSize
+                if trainerId >= self._displaySize:
+                    continue
+
+                di = {}
+                # name = "%s_%d" % (metric, trainerId)
+                name = f"{metric}"
+                di['name'] = name
+                di['pid'] = pid_map[metric]
+                di['ts'] = self._align_ts(int(row['ts']))
+                # di['ts'] = int(row['ts'])
+                di['cat'] = metric
+                di['tid'] = f"{groupId}_{trainerId}"
+                di['ph'] = "C"
+                di['id'] = trainerId
+
+                args = {}
+                for p in parameterList:
+                    args[p[0]] = row[p[1]]
+                di['args'] = args
+
+                traceEventList.append(di)
+        trace = {}
+        trace['traceEvents'] = traceEventList
+        self.dumpDCGMDict(trace, groupId, gpuId, True)
+
+        return trace
+
+    def getDCGMTraceInfo(self, groupId, processNum=8):
+        dcgm_data = self.parseFileByGroup(groupId, processNum)
+
+        pid_map = {}
+        init_pid = PIPELINEINFO_TRACE_NUM
+
+        for metric in dcgmMetricParameterMap.keys():
+            pid_map[metric] = init_pid
+            init_pid = init_pid + 1
+
+        manager = multiprocessing.Manager()
+        q = manager.Queue()
+        processPool = []
+        pidList = []
+
+        for gpuId in range(self._gpuPerTrainer):
+            subproc = Process(
+                target=self._getDCGMTraceInfoByGpuId,
+                args=(
+                    groupId,
+                    gpuId,
+                    dcgm_data,
+                    pid_map,
+                    q,
+                ),
+            )
+            processPool.append(subproc)
+            subproc.start()
+            pidList.append(subproc.pid)
+            self._logger.info(
+                f"[DCGM info]: process [{subproc.pid}] has been started, total task num is {1} ..."
+            )
+
+        for t in processPool:
+            t.join()
+            pidList.remove(t.pid)
+            self._logger.info(
+                f"[DCGM info]: process [{t.pid}] has exited! remained {len(pidList)} process!"
+            )
+
+        dcgmInfo = {}
+
+        return dcgmInfo
+
+
+def test_dcgmFileReader():
+    args = {
+        "dataPath": "data/newdata/dcgm",
+        "groupSize": 4,
+        "displaySize": 8,
+        "gpuPerTrainer": 8,
+        "minTimeStamp": 0,
+        "organizeForm": FILEORGANIZEFORM_BYTRAINER,
+    }
+
+    testReader = dcgmFileReader(getLogger(), args)
+    testReader.printArgs()
+    data = testReader.getDCGMTraceInfo(0, 8)
+
+
+if __name__ == "__main__":
+    test_dcgmFileReader()

tools/CrossStackProfiler/NetFileReader.py

@@ -0,0 +1,144 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import multiprocessing
+from multiprocessing import Process
+
+from CspFileReader import (
+    FILEORGANIZEFORM_BYTRAINER,
+    PIPELINEINFO_TRACE_NUM,
+    FileReader,
+    getLogger,
+)
+
+
+class netFileReader(FileReader):
+    def _parseSingleFile(self, fileNameList, tx_pid, rx_pid, q=None):
+        traceInfo = {}
+        traceEventList = []
+
+        metaInfo = {}
+        metaInfo['name'] = 'process_name'
+        metaInfo['ph'] = 'M'
+        metaInfo['pid'] = tx_pid
+        metaInfo['args'] = {'name': f"{tx_pid:02}_tx"}
+
+        traceEventList.append(metaInfo)
+        metaInfo = {}
+        metaInfo['name'] = 'process_name'
+        metaInfo['ph'] = 'M'
+        metaInfo['pid'] = rx_pid
+        metaInfo['args'] = {'name': f"{rx_pid:02}_rx"}
+
+        traceEventList.append(metaInfo)
+
+        trainerIdList = []
+        for fileName in fileNameList:
+            trainerId = self.getTrainerId(fileName)
+            trainerIdList.append(trainerId)
+            with open(fileName, "r") as rf:
+                for line in rf:
+                    try:
+                        event_str = json.loads(line.strip())
+                        event_str["pid"] = (
+                            tx_pid if event_str["name"] == "tx" else rx_pid
+                        )
+                        # the unit of net is ms, we need ns
+                        event_str["ts"] = self._align_ts(event_str["ts"] * 1e6)
+                        event_str["id"] = trainerId
+                        traceEventList.append(event_str)
+
+                    except Exception:
+                        self._logger.warning(
+                            f"invalid record [{line[:-1]}] in [{fileName}]. skip it!"
+                        )
+        traceInfo["traceEvents"] = traceEventList
+
+        if q is not None:
+            q.put(traceInfo)
+        else:
+            return traceInfo
+
+    def parseFileByGroup(self, groupId, processNum=8):
+        fileFist = self.getFileListByGroup(groupId)
+        fileFist = fileFist[: min(self._displaySize, len(fileFist))]
+
+        manager = multiprocessing.Manager()
+        q = manager.Queue()
+
+        processPool = []
+        pidList = []
+        tx_pid = PIPELINEINFO_TRACE_NUM
+        rx_pid = PIPELINEINFO_TRACE_NUM + 1
+
+        taskList = self._splitTaskListForMultiProcess(fileFist, processNum)
+        for task in taskList:
+            subproc = Process(
+                target=self._parseSingleFile,
+                args=(
+                    task,
+                    tx_pid,
+                    rx_pid,
+                    q,
+                ),
+            )
+            processPool.append(subproc)
+            subproc.start()
+            pidList.append(subproc.pid)
+            self._logger.info(
+                f"[Net info]: process [{subproc.pid}] has been started, total task num is {len(processPool)} ..."
+            )
+
+        for t in processPool:
+            t.join()
+            pidList.remove(t.pid)
+            self._logger.info(
+                f"[Net info]: process [{t.pid}] has exited! remained {len(pidList)} process!"
+            )
+
+        traceInfo = {}
+        isFirstProcess = True
+        for t in processPool:
+            if isFirstProcess:
+                isFirstProcess = False
+                traceInfo["traceEvents"] = q.get()["traceEvents"]
+            else:
+                traceInfo["traceEvents"].extend(q.get()["traceEvents"])
+
+        return traceInfo
+
+
+def test_netFileReader():
+    args = {
+        "dataPath": "data/newdata/net",
+        "groupSize": 4,
+        "displaySize": 2,
+        "gpuPerTrainer": 8,
+        "minTimeStamp": 0,
+        "organizeForm": FILEORGANIZEFORM_BYTRAINER,
+    }
+
+    testReader = netFileReader(getLogger(), args)
+    testReader.printArgs()
+    data = testReader.parseFileByGroup(0, 8)
+
+    jsObj = json.dumps(data, indent=4, separators=(',', ': '))
+    fileObject = open('jsonFile.json', 'w')
+    fileObject.write(jsObj)
+    fileObject.close()
+
+
+if __name__ == "__main__":
+    test_netFileReader()

tools/CrossStackProfiler/ProfileFileReader.py

@@ -0,0 +1,516 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import multiprocessing
+from multiprocessing import Process
+
+from CspChromeTraceFormatter import ChromeTraceFormatter
+from CspFileReader import (
+    DCGMINFO_TRACE_NUM,
+    FILEORGANIZEFORM_BYRANK,
+    NETINFO_TRACE_NUM,
+    PIPELINEINFO_TRACE_NUM,
+    FileReader,
+    getLogger,
+)
+
+from paddle.base.proto.profiler import profiler_pb2
+
+
+class profileFileReader(FileReader):
+    def _parseSingleFile(self, profile):
+        with open(profile, 'rb') as f:
+            profile_s = f.read()
+            profile_pb = profiler_pb2.Profile()
+            profile_pb.ParseFromString(profile_s)
+
+            return profile_pb
+
+    def _parseTask(self, taskList, q=None):
+        profile_dict = {}
+
+        for fileName in taskList:
+            rankId = self.getRankId(fileName)
+            profile_dict[f"trainerRank.{rankId:03}"] = self._parseSingleFile(
+                fileName
+            )
+            self._logger.info(f"I finish processing {fileName}!")
+
+        if q is not None:
+            q.put(profile_dict)
+
+        return profile_dict
+
+    def _is_forwardBackwardInfo(self, items):
+        if items["name"] == "marker/compute/MarkerCUDA":
+            if "args" in items:
+                if isinstance(items["args"], dict):
+                    args = items["args"]
+                    if "detail_info" in args:
+                        if (
+                            args["detail_info"] == "marker_forward_B"
+                            or args["detail_info"] == "marker_forward_E"
+                            or args["detail_info"] == "marker_backward_B"
+                            or args["detail_info"] == "marker_backward_E"
+                        ):
+                            return True
+        return False
+
+    def _allocate_forwardBackwardInfo(self, restList, pid, tid):
+        def _cmp_ele(items):
+            return items["ts"]
+
+        restList.sort(key=_cmp_ele)
+        newList = []
+
+        lastEle = {}
+        for items in restList:
+            if items["args"]["detail_info"].endswith("E"):
+                if not lastEle:
+                    continue
+                else:
+                    lastEle["dur"] = items["ts"] - lastEle["ts"]
+                    name = lastEle["args"]["detail_info"]
+                    name = name[: name.rfind('_')]
+                    name = name.split('_')[1]
+                    lastEle["name"] = name
+                    lastEle["args"]["detail_info"] = name
+                    lastEle["args"]["name"] = name
+                    if name == "backward":
+                        lastEle["cname"] = "good"
+                    else:
+                        lastEle["cname"] = "bad"
+
+                    lastEle["tid"] = tid
+                    lastEle["pid"] = pid
+
+                    newList.append(lastEle)
+            else:
+                lastEle = items
+
+        return newList
+
+    def _getPipeLineInfo(self, profileList, q=None):
+        res = {}
+        for profile in profileList:
+            rankId = self.getRankId(profile)
+
+            profile_pb = self._parseSingleFile(profile)
+            traceEventList = []
+            pid = 0
+            tid = rankId
+
+            for event in profile_pb.events:
+                args = {'name': event.name}
+                if event.memcopy.bytes > 0:
+                    args['mem_bytes'] = event.memcopy.bytes
+                if hasattr(event, "detail_info") and event.detail_info:
+                    args['detail_info'] = event.detail_info
+
+                traceEvent = {}
+                traceEvent['ph'] = 'X'
+                traceEvent['cat'] = 'Op'
+                traceEvent['name'] = event.name
+                traceEvent['pid'] = pid
+                traceEvent['tid'] = tid
+                traceEvent['ts'] = self._align_ts(event.start_ns)
+                traceEvent['dur'] = (event.end_ns - event.start_ns) / 1.0
+                traceEvent['args'] = args
+
+                if self._is_forwardBackwardInfo(traceEvent):
+                    traceEventList.append(traceEvent)
+
+            pipeLineList = self._allocate_forwardBackwardInfo(
+                traceEventList, pid, tid
+            )
+
+            res[str(rankId)] = pipeLineList
+
+        if q is not None:
+            q.put(res)
+
+        return res
+
+    def getPipeLineInfo(self, groupId, processNum=8):
+        fileFist = self.getFileListByGroup(groupId)
+
+        self._logger.info(
+            f"using [{processNum}] process to do this work, total task num is {len(fileFist)}!"
+        )
+        processPool = []
+        pidList = []
+
+        manager = multiprocessing.Manager()
+        q = manager.Queue()
+
+        taskList = self._splitTaskListForMultiProcess(fileFist, processNum)
+        for task in taskList:
+            subproc = Process(
+                target=self._getPipeLineInfo,
+                args=(
+                    task,
+                    q,
+                ),
+            )
+            processPool.append(subproc)
+            subproc.start()
+            pidList.append(subproc.pid)
+            self._logger.info(
+                f"[pipeline info]: process [{subproc.pid}] has been started, total task num is {len(task)} ..."
+            )
+
+        for t in processPool:
+            t.join()
+            pidList.remove(t.pid)
+            self._logger.info(
+                f"[pipeline info]: process [{t.pid}] has exited! remained {len(pidList)} process!"
+            )
+
+        pipeLineInfo = {}
+
+        metaInfo = {}
+        metaInfo['name'] = 'process_name'
+        metaInfo['ph'] = 'M'
+        metaInfo['pid'] = 0
+        metaInfo['args'] = {'name': f"{PIPELINEINFO_TRACE_NUM:02}_pipeLineInfo"}
+
+        for t in processPool:
+            for k, v in q.get().items():
+                rankId = int(k)
+                gpuId = rankId % self._gpuPerTrainer
+                if str(gpuId) not in pipeLineInfo.keys():
+                    pipeLineInfo[str(gpuId)] = [metaInfo]
+                pipeLineInfo[str(gpuId)].extend(v)
+
+        return pipeLineInfo
+
+    def _allocate_pids(self, profile_dict, gpuId, initPid):
+        chrome_trace = ChromeTraceFormatter()
+        devices = {}
+        mem_devices = {}
+
+        initLineNum = initPid + 1
+        lineDelta = len(profile_dict.keys())
+        i = 0
+        for k, profile_pb in profile_dict.items():
+            lineNum = initLineNum
+            for event in profile_pb.events:
+                if event.type == profiler_pb2.Event.CPU:
+                    if (k, event.device_id, "CPU") not in devices:
+                        pid = initPid
+                        initPid = initPid + 1
+                        devices[(k, event.device_id, "CPU")] = pid
+                        # -1 device id represents CUDA API(RunTime) call.(e.g. cudaLaunch, cudaMemcpy)
+                        if event.device_id == -1:
+                            chrome_trace.emit_pid(
+                                f"{lineNum:02}_{k}:cuda_api", pid
+                            )
+                            lineNum = lineNum + 1
+                        else:
+                            chrome_trace.emit_pid(
+                                f"{lineNum:02}_{k}:cpu:block:{event.device_id}",
+                                pid,
+                            )
+                            lineNum = lineNum + 1
+                elif event.type == profiler_pb2.Event.GPUKernel:
+                    if (k, event.device_id, "GPUKernel") not in devices:
+                        if gpuId == event.device_id:
+                            pid = initPid
+                            initPid = initPid + 1
+
+                            devices[(k, event.device_id, "GPUKernel")] = pid
+                            chrome_trace.emit_pid(
+                                f"{lineNum:02}_{k}:gpu:{event.device_id}",
+                                pid,
+                            )
+                            lineNum = lineNum + 1
+
+            if not hasattr(profile_pb, "mem_events"):
+                continue
+            for mevent in profile_pb.mem_events:
+                if mevent.place == profiler_pb2.MemEvent.CUDAPlace:
+                    if (k, mevent.device_id, "GPU") not in mem_devices:
+                        if gpuId == mevent.device_id:
+                            pid = initPid
+                            initPid = initPid + 1
+
+                            mem_devices[(k, mevent.device_id, "GPU")] = pid
+                            chrome_trace.emit_pid(
+                                f"{lineNum:02}_memory usage on {k}:gpu:{mevent.device_id}",
+                                pid,
+                            )
+                            lineNum = lineNum + 1
+                elif mevent.place == profiler_pb2.MemEvent.CPUPlace:
+                    if (k, mevent.device_id, "CPU") not in mem_devices:
+                        pid = initPid
+                        initPid = initPid + 1
+
+                        mem_devices[(k, mevent.device_id, "CPU")] = pid
+                        chrome_trace.emit_pid(
+                            f"{lineNum:02}_memory usage on {k}:cpu:{mevent.device_id}",
+                            pid,
+                        )
+                        lineNum = lineNum + 1
+                elif mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace:
+                    if (
+                        k,
+                        mevent.device_id,
+                        "CUDAPinnedPlace",
+                    ) not in mem_devices:
+                        if gpuId == mevent.device_id:
+                            pid = initPid
+                            initPid = initPid + 1
+
+                            mem_devices[
+                                (k, mevent.device_id, "CUDAPinnedPlace")
+                            ] = pid
+                            chrome_trace.emit_pid(
+                                f"{lineNum:02}_memory usage on {k}:cudapinnedplace:{mevent.device_id}",
+                                pid,
+                            )
+                            lineNum = lineNum + 1
+                if (k, 0, "CPU") not in mem_devices:
+                    pid = initPid
+                    initPid = initPid + 1
+
+                    mem_devices[(k, 0, "CPU")] = pid
+                    chrome_trace.emit_pid(
+                        f"{lineNum:02}_memory usage on {k}:cpu:{0}", pid
+                    )
+                    lineNum = lineNum + 1
+                if (k, 0, "GPU") not in mem_devices:
+                    # if gpuId == mevent.device_id:
+                    pid = initPid
+                    initPid = initPid + 1
+
+                    mem_devices[(k, 0, "GPU")] = pid
+                    chrome_trace.emit_pid(
+                        f"{lineNum:02}_memory usage on {k}:gpu:{0}", pid
+                    )
+                    lineNum = lineNum + 1
+                if (k, 0, "CUDAPinnedPlace") not in mem_devices:
+                    pid = initPid
+                    initPid = initPid + 1
+
+                    mem_devices[(k, 0, "CUDAPinnedPlace")] = pid
+                    chrome_trace.emit_pid(
+                        f"{lineNum:02}_memory usage on {k}:cudapinnedplace:{0}",
+                        pid,
+                    )
+                    lineNum = lineNum + 1
+            i = i + 1
+        return chrome_trace, devices, mem_devices
+
+    def _allocate_events(self, profile_dict, devices, gpuId):
+        chrome_trace = ChromeTraceFormatter()
+        for k, profile_pb in profile_dict.items():
+            rankId = int(k.split(".")[-1])
+
+            for event in profile_pb.events:
+                if event.type == profiler_pb2.Event.CPU:
+                    type = "CPU"
+                elif event.type == profiler_pb2.Event.GPUKernel:
+                    type = "GPUKernel"
+
+                if (
+                    event.type == profiler_pb2.Event.GPUKernel
+                    and event.device_id != gpuId
+                    and rankId % self._gpuPerTrainer != gpuId
+                ):
+                    continue
+
+                pid = devices[(k, event.device_id, type)]
+                args = {'name': event.name}
+                if event.memcopy.bytes > 0:
+                    args['mem_bytes'] = event.memcopy.bytes
+                if hasattr(event, "detail_info") and event.detail_info:
+                    args['detail_info'] = event.detail_info
+                # TODO(panyx0718): Chrome tracing only handles ms. However, some
+                # ops takes micro-seconds. Hence, we keep the ns here.
+                chrome_trace.emit_region(
+                    self._align_ts(event.start_ns),
+                    (event.end_ns - event.start_ns) / 1.0,
+                    pid,
+                    event.sub_device_id,
+                    'Op',
+                    event.name,
+                    args,
+                )
+        return chrome_trace
+
+    def _allocate_memory_event(self, profile_dict, mem_devices, gpuId):
+        chrome_trace = ChromeTraceFormatter()
+        if not hasattr(profiler_pb2, "MemEvent"):
+            return
+        place_to_str = {
+            profiler_pb2.MemEvent.CPUPlace: "CPU",
+            profiler_pb2.MemEvent.CUDAPlace: "GPU",
+            profiler_pb2.MemEvent.CUDAPinnedPlace: "CUDAPinnedPlace",
+        }
+        for k, profile_pb in profile_dict.items():
+            rankId = int(k.split(".")[-1])
+
+            trainerId = rankId / self._gpuPerTrainer
+
+            if trainerId >= self._displaySize:
+                continue
+
+            mem_list = []
+            end_profiler = 0
+            for mevent in profile_pb.mem_events:
+                crt_info = {}
+                crt_info['time'] = mevent.start_ns
+                crt_info['size'] = mevent.bytes
+                if mevent.place in place_to_str:
+                    place = place_to_str[mevent.place]
+                else:
+                    place = "UnDefine"
+
+                if (
+                    mevent.place == profiler_pb2.MemEvent.CUDAPlace
+                    or mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace
+                ) and mevent.device_id != gpuId:
+                    continue
+
+                crt_info['place'] = place
+                pid = mem_devices[(k, mevent.device_id, place)]
+                crt_info['pid'] = pid
+                crt_info['thread_id'] = mevent.thread_id
+                crt_info['device_id'] = mevent.device_id
+                mem_list.append(crt_info)
+                crt_info = {}
+                crt_info['place'] = place
+                crt_info['pid'] = pid
+                crt_info['thread_id'] = mevent.thread_id
+                crt_info['device_id'] = mevent.device_id
+                crt_info['time'] = mevent.end_ns
+                crt_info['size'] = -mevent.bytes
+                mem_list.append(crt_info)
+                end_profiler = max(end_profiler, crt_info['time'])
+            mem_list.sort(key=lambda tmp: tmp.get('time', 0))
+            i = 0
+            total_size = 0
+            while i < len(mem_list):
+                total_size += mem_list[i]['size']
+                while (
+                    i < len(mem_list) - 1
+                    and mem_list[i]['time'] == mem_list[i + 1]['time']
+                ):
+                    total_size += mem_list[i + 1]['size']
+                    i += 1
+
+                chrome_trace.emit_counter(
+                    "Memory",
+                    "Memory",
+                    mem_list[i]['pid'],
+                    self._align_ts(mem_list[i]['time']),
+                    0,
+                    total_size,
+                )
+                i += 1
+        return chrome_trace
+
+    def _getOPTraceInfoByGpuId(self, groupId, gpuId):
+        fileFist = self.getFileListByGroup(groupId)
+        newFileList = []
+        for file in fileFist:
+            rankId = self.getRankId(file)
+            localRank = rankId % self._gpuPerTrainer
+            if (
+                localRank == gpuId
+                and (rankId / self._gpuPerTrainer) % self._groupSize
+                < self._displaySize
+            ):
+                newFileList.append(file)
+
+        profile_dict = self._parseTask(newFileList)
+        initPid = (
+            PIPELINEINFO_TRACE_NUM + DCGMINFO_TRACE_NUM + NETINFO_TRACE_NUM
+        )
+        metaTrace, devicesPid, mem_devicesPid = self._allocate_pids(
+            profile_dict, gpuId, initPid
+        )
+        eventsTrace = self._allocate_events(profile_dict, devicesPid, gpuId)
+        memEventsTrace = self._allocate_memory_event(
+            profile_dict, mem_devicesPid, gpuId
+        )
+
+        trace = {}
+        trace['traceEvents'] = (
+            metaTrace._metadata + eventsTrace._events + memEventsTrace._events
+        )
+        self.dumpOpInfoDict(trace, groupId, gpuId, True)
+
+        return trace
+
+    def getOPTraceInfo(self, groupId):
+        manager = multiprocessing.Manager()
+        q = manager.Queue()
+        processPool = []
+        pidList = []
+
+        for gpuId in range(self._gpuPerTrainer):
+            subproc = Process(
+                target=self._getOPTraceInfoByGpuId,
+                args=(
+                    groupId,
+                    gpuId,
+                ),
+            )
+            processPool.append(subproc)
+            subproc.start()
+            pidList.append(subproc.pid)
+            self._logger.info(
+                f"[op info]: process [{subproc.pid}] has been started, total task num is {1} ..."
+            )
+
+        for t in processPool:
+            t.join()
+            pidList.remove(t.pid)
+            self._logger.info(
+                f"[op info]: process [{t.pid}] has exited! remained {len(pidList)} process!"
+            )
+
+        opInfo = {}
+
+        return opInfo
+
+    def parseFileByGroup(self, groupId, processNum=8):
+        fileFist = self.getFileListByGroup(groupId)
+        return self._parseTask(fileFist)
+
+
+def test_profileFileReader():
+    args = {
+        "dataPath": "data/newdata/profile",
+        "groupSize": 4,
+        "displaySize": 8,
+        "gpuPerTrainer": 8,
+        "minTimeStamp": 0,
+        "organizeForm": FILEORGANIZEFORM_BYRANK,
+    }
+
+    testReader = profileFileReader(getLogger(), args)
+    testReader.printArgs()
+    data = testReader.getOPTraceInfo(0)
+
+    jsObj = json.dumps(data)
+    fileObject = open('jsonFile.json', 'w')
+    fileObject.write(jsObj)
+    fileObject.close()
+
+
+if __name__ == "__main__":
+    test_profileFileReader()

tools/CrossStackProfiler/__init__.py

@@ -0,0 +1,13 @@
+#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tools/__init__.py

@@ -0,0 +1,13 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tools/analysisPyXml.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import sys
+from xml.etree import ElementTree
+
+import commands
+
+
+def analysisPyXml(rootPath, ut):
+    xml_path = f'{rootPath}/build/pytest/{ut}/python-coverage.xml'
+    related_ut_map_file = f'{rootPath}/build/ut_map/{ut}/related_{ut}.txt'
+    notrelated_ut_map_file = f'{rootPath}/build/ut_map/{ut}/notrelated_{ut}.txt'
+    tree = ElementTree.parse(xml_path)
+    root = tree.getroot()
+    error_files = []
+    pyCov_file = []
+    for clazz in root.findall('packages/package/classes/class'):
+        clazz_filename = clazz.attrib.get('filename')
+        if not clazz_filename.startswith('/paddle'):
+            clazz_filename = f'/paddle/{clazz_filename}'
+        for line in clazz.findall('lines/line'):
+            line_hits = int(line.attrib.get('hits'))
+            if line_hits != 0:
+                line_number = int(line.attrib.get('number'))
+                command = f'sed -n {line_number}p {clazz_filename}'
+                _code, output = commands.getstatusoutput(command)
+                if _code == 0:
+                    if not output.strip().startswith(
+                        (
+                            'from',
+                            'import',
+                            '__all__',
+                            'def',
+                            'class',
+                            '"""',
+                            '@',
+                            '\'\'\'',
+                            'logger',
+                            '_logger',
+                            'logging',
+                            'r"""',
+                            'pass',
+                            'try',
+                            'except',
+                            'if __name__ == "__main__"',
+                        )
+                    ):
+                        pattern = r"""(.*) = ('*')|(.*) = ("*")|(.*) = (\d)|(.*) = (-\d)|(.*) = (None)|(.*) = (True)|(.*) = (False)|(.*) = (URL_PREFIX*)|(.*) = (\[)|(.*) = (\{)|(.*) = (\()"""  # a='b'/a="b"/a=0
+                        if re.match(pattern, output.strip()) is None:
+                            pyCov_file.append(clazz_filename)
+                            coverageMessage = 'RELATED'
+                            break
+                        else:
+                            coverageMessage = 'FILTER'  # hit filter logic
+                    else:
+                        coverageMessage = 'FILTER'
+                else:
+                    coverageMessage = 'ERROR'
+                    error_files.append(clazz_filename)
+                    break
+            else:
+                coverageMessage = 'NOT_RELATED'
+        if coverageMessage in ['NOT_RELATED', 'ERROR', 'FILTER']:
+            os.system(f'echo {clazz_filename} >> {notrelated_ut_map_file}')
+        elif coverageMessage == 'RELATED':
+            os.system(f'echo {clazz_filename} >> {related_ut_map_file}')
+
+    print("============len(pyCov_file)")
+    print(len(pyCov_file))
+    print("============error")
+    print(error_files)
+
+
+if __name__ == "__main__":
+    rootPath = sys.argv[1]
+    ut = sys.argv[2]
+    analysisPyXml(rootPath, ut)

tools/analysis_build_time.py

@@ -0,0 +1,65 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+root_path = os.path.abspath(os.path.join(os.getcwd(), ".."))
+
+
+def strToSecond(strTime):
+    minute = int(strTime.split(':')[0])
+    second = int(strTime.split(':')[1].split('.')[0]) + 1
+    return minute * 60 + second
+
+
+def getUsefulBuildTimeFile(filename):
+    os.system(
+        f"grep -Po -- '-o .*' {filename} | grep ' elapsed' | grep -P -v '0:00.* elapse' > {root_path}/tools/analysis_build_time"
+    )
+    os.system(
+        f"grep -v  -- '-o .*' {filename} |grep ' elapse' |  grep -P -v '0:00.* elapse' >> {root_path}/tools/analysis_build_time"
+    )
+
+
+def analysisBuildTime():
+    filename = f'{root_path}/build/build-time'
+    getUsefulBuildTimeFile(filename)
+    os.system(f'rm -rf {root_path}/tools/tempbuildTime.txt')
+    with open(f'{root_path}/tools/analysis_build_time', 'r') as f:
+        lines = f.readlines()
+        for line in lines:
+            try:
+                line = line.strip()
+                if '-o ' in line:
+                    buildFile = line.split(', ')[0].split(' ')[1]
+                    buildTime = line.split(', ')[1].split('elapsed')[0].strip()
+                    secondTime = strToSecond(buildTime)
+                    os.system(
+                        f"echo {buildFile}, {secondTime} >> {root_path}/tools/tempbuildTime.txt"
+                    )
+                else:
+                    buildTime = line.split(', ')[1].split('elapsed')[0].strip()
+                    secondTime = strToSecond(buildTime)
+                    if secondTime > 30:
+                        os.system(
+                            f"echo {line}, {secondTime} >> {root_path}/tools/tempbuildTime.txt"
+                        )
+            except ValueError:
+                print(line)
+    os.system(
+        f'sort -n -k 2 -r {root_path}/tools/tempbuildTime.txt > {root_path}/tools/buildTime.txt'
+    )
+
+
+analysisBuildTime()

tools/auto_parallel/ci_auto_parallel.sh

@@ -0,0 +1,292 @@
+#!/usr/bin/env bash
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source /workspace/Paddle/tools/auto_parallel/target_path_lists.sh
+
+export paddle=$1
+export paddle_dir=/workspace/Paddle
+mkdir -p /workspace/case_logs
+export log_path=/workspace/case_logs
+export case_list=()
+
+global_total_count=0
+global_success_count=0
+global_exit_250_arr=()
+global_runtime_fail_arr=()
+global_verification_fail_arr=()
+
+install_paddle(){
+    echo -e "\033[31m ---- Install paddlepaddle-gpu  \033"
+    if [ -n "$paddle" ];then
+      python -m pip install --user --no-cache-dir ${paddle} --force-reinstall --no-dependencies;
+    fi
+    python -c "import paddle; print('paddle version:',paddle.__version__,'\npaddle commit:',paddle.version.commit)";
+}
+
+install_external_ops(){
+    echo -e "\033[31m ---- Install extern_ops  \033"
+    export PYTHONPATH=/workspace/PaddleNLP:$PYTHONPATH
+    cd /workspace/PaddleNLP/slm/model_zoo/gpt-3/external_ops
+    python setup.py install
+    python -c "import fused_ln;";
+}
+
+get_diff_TO_case(){
+cd ${paddle_dir}
+# get the location of "test/auto_parallel" in target_lists_for_semi_auto_ci
+count=0
+for element in "${target_lists_for_semi_auto_ci[@]}";do
+  if [[ "$element" == "test/auto_parallel" ]]; then
+    test_auto_num=$count
+    break
+  fi
+  count=$((count+1))
+done
+# get the location of "test/collective/hybrid_strategy" in target_lists_for_dygraph_ci
+count=0
+for element in "${target_lists_for_dygraph_ci[@]}";do
+  if [[ "$element" == "test/collective/hybrid_strategy" ]]; then
+    test_dygraph_num=$count
+    break
+  fi
+  count=$((count+1))
+done
+
+# There are two types of tests included here:
+# 1. The auto-parallel unit testing in Paddle repository. CI will immediately end with
+# an error when a test fails.
+# 2. The auto-parallel testing of large language models in `PaddleNLP` repository. The execution
+# status of each test will be recorded through global variables. When a test fails, it does not
+# affect the execution of subsequent tests. They will be summarized and output after the CI is completed.
+# Therefore, it is required to perform paddle unit testing first, followed by testing in `PaddleNLP`.
+case_list[${#case_list[*]}]="llama_auto_unit_test"
+case_list[${#case_list[*]}]=llama_auto
+case_list[${#case_list[*]}]=gpt-3_auto
+case_list[${#case_list[*]}]=gpt-3_dygraph
+}
+
+print_info(){
+#解决异常退出-6的问题，CI中的偶现问题，无法发现
+if [[ $1 -ne 0 ]] && [[ $1 -ne 250 ]];then
+    EXCODE=2
+    if [ ! -f ${log_path}/$2 ];then
+        echo -e "\033[31m run $2 CI FAIL \033"
+else
+    mv ${log_path}/$2 ${log_path}/$2_FAIL.log
+    echo -e "\033[31m ${log_path}/$2_FAIL \033"
+    tail -70 ${log_path}/$2_FAIL.log
+    fi
+    exit $EXCODE
+else
+    echo -e "\033[32m The $3 CI has completed \033"
+fi
+}
+
+function execute_func_list(){
+    cd ${log_path} || { echo "Failed to enter log_path: $log_path"; return 1; }
+    total_count=0
+    success_count=0
+    runtime_fail_count=0
+    verification_fail_count=0
+    exit_250_count=0
+    while IFS= read -r func_name; do
+        let total_count++
+        let global_total_count++
+        execute_num=1
+        while true; do
+            timeout 10m bash $1 exec_case $func_name $FLAGS_install_deps $FLAGS_download_data
+            result=$?
+            if [ $result -eq 0 ]; then
+                echo -e "\033[32m test success!"
+                let success_count++
+                let global_success_count++
+            elif [ $result -eq 1 ]; then
+                if [ $execute_num -eq 1 ]; then
+                    echo -e "\033[31m first time execute failed, try again!"
+                    let execute_num++
+                    continue
+                else
+                    echo -e "\033[31m second time execute failed, exit!"
+                    mv ${log_path}/$func_name ${log_path}/${func_name}_FAIL.log
+                    echo -e "\033[31m ${log_path}/$func_name_FAIL \033"
+                    tail -15 ${log_path}/${func_name}_FAIL.log
+                    let runtime_fail_count++
+                    global_runtime_fail_arr+=("$func_name")
+                fi
+            elif [ $result -eq 2 ]; then
+                echo -e "\033[31m verification failed!"
+                let verification_fail_count++
+                global_verification_fail_arr+=("$func_name")
+            elif [ $result -eq 250 ]; then
+                if [ $execute_num -eq 1 ]; then
+                    echo -e "\033[31m first time execute failed, try again!"
+                    let execute_num++
+                    continue
+                else
+                    echo -e "\033[31m second time execute failed, exit!"
+                    mv ${log_path}/$func_name ${log_path}/${func_name}_FAIL.log
+                    echo -e "\033[31m ${log_path}/$func_name_FAIL \033"
+                    tail -15 ${log_path}/${func_name}_FAIL.log
+                    let exit_250_count++
+                    global_exit_250_arr+=("$func_name")
+                fi
+            elif [ $result -eq 124 ]; then
+                echo "\033[31m [failed-timeout] Test case execution was terminated after exceeding the 10m limit."
+                mv ${log_path}/$func_name ${log_path}/${func_name}_FAIL.log
+                echo -e "\033[31m ${log_path}/$func_name_FAIL \033"
+                tail -15 ${log_path}/${func_name}_FAIL.log
+                let runtime_fail_count++
+                global_runtime_fail_arr+=("$func_name")
+            else
+                echo "test failed!"
+                mv ${log_path}/$func_name ${log_path}/${func_name}_FAIL.log
+                echo -e "\033[31m ${log_path}/$func_name_FAIL \033"
+                tail -15 ${log_path}/${func_name}_FAIL.log
+                let runtime_fail_count++
+                global_runtime_fail_arr+=("$func_name")
+            fi
+            break
+        done
+    done < functions.txt
+    echo -e "\033[31m $2 test case has complicated \033"
+    echo -e "\033[31m $(printf '\t')  total tests :  $total_count \033"
+    echo -e "\033[31m $(printf '\t')  success tests :  $success_count \033"
+    echo -e "\033[31m $(printf '\t')  runtime fail tests :  $runtime_fail_count \033"
+    echo -e "\033[31m $(printf '\t')  verification fail tests :  $verification_fail_count \033"
+    echo -e "\033[31m $(printf '\t')  exit 250 tests(intermittent issue) :  $exit_250_count \033"
+}
+
+function clean_file(){
+    target_path=$1
+    matching_data_dirs=$(find "$target_path" -maxdepth 1 -type d -name "*data*")
+    if [ -n "$matching_data_dirs" ]; then
+        echo "cleaning data dirs:"
+        echo $matching_data_dirs
+        for dir in $matching_data_dirs; do
+            rm -rf "$dir"
+            echo "deleted $dir"
+        done
+    else
+        echo "$target_path no data dirs found"
+    fi
+
+    matching_output_dirs=$(find "$target_path" -maxdepth 1 -type d -name "*output*")
+    if [ -n "$matching_output_dirs" ]; then
+        echo "cleaning output dirs:"
+        echo $matching_output_dirs
+        for dir in $matching_output_dirs; do
+            rm -rf "$dir"
+            echo "deleted $dir"
+        done
+    else
+        echo "$target_path no output dirs found"
+    fi
+}
+
+# Get the list of pending cases
+get_diff_TO_case
+# Remove duplicates and store the results back to the original list
+
+####################
+case_list=($(awk -v RS=' ' '!a[$1]++' <<< ${case_list[*]}))
+if [[ ${#case_list[*]} -ne 0 ]];then
+    echo -e "\033[31m =======CI Check case========= \033"
+    echo -e "\033[31m ---- case_list length: ${#case_list[*]}, cases: ${case_list[*]} \033"
+    echo -e "\033[31m ============================= \033"
+    set +e
+
+    # Install paddle
+    install_paddle
+    # Install external_ops
+    install_external_ops
+    case_num=1
+    # `FLAGS_install_deps` defaults to 0, indicating that certain required packages must be installed
+    # via `install requirements.txt` prior to running `PaddleNLP` tests.
+    # By setting `FLAGS_install_deps` to 1, indicating that there is no need for reinstallation.
+    export FLAGS_install_deps=0
+    for case in ${case_list[*]};do
+        echo -e "\033[31m ---- running case $case_num/${#case_list[*]}: ${case} \033"
+        # Suggest that the logical order here be consistent with the `case_stst` order.
+        if [[ ${case} == "auto_unit_test" ]];then
+            bash /workspace/Paddle/tools/auto_parallel/ci_case_unit.sh auto_unit_test
+            print_info $? `ls -lt ${log_path} | grep "test" | head -n 1 | awk '{print $9}'` ${case}
+            let case_num++
+        elif [[ ${case} == "dygraph_unit_test" ]];then
+            bash /workspace/Paddle/tools/auto_parallel/ci_case_unit.sh dygraph_unit_test
+            print_info $? `ls -lt ${log_path} | grep "test" | head -n 1 | awk '{print $9}'` ${case}
+            let case_num++
+        elif [[ ${case} == "llama_auto_unit_test" ]];then
+            bash /workspace/Paddle/tools/auto_parallel/ci_case_unit.sh llama_auto_unit_test
+            print_info $? `ls -lt ${log_path} | grep "test" | head -n 1 | awk '{print $9}'` ${case}
+            let case_num++
+        elif [[ ${case} == "llama_auto" ]];then
+            cmd=/workspace/PaddleNLP/scripts/distribute/ci_case_auto.sh
+            timeout 5m bash $cmd prepare_case llama_case_list_auto $FLAGS_install_deps $FLAGS_download_data
+            execute_func_list $cmd llama_auto
+            # There is no need to reinstall the related packages of `PaddleNLP` afterward.
+            export FLAGS_install_deps=1
+            # The `llama` test data has been downloaded, and the `FLAGS_download_data` flag indicates
+            # that there is no need to repeat the download process later.
+            export FLAGS_download_data="llama ""$FLAGS_download_data"
+            let case_num++
+            clean_file /workspace/PaddleNLP/llm/auto_parallel/llama
+        elif [[ ${case} == "gpt-3_auto" ]];then
+            cmd=/workspace/PaddleNLP/scripts/distribute/ci_case_auto.sh
+            timeout 5m bash $cmd prepare_case llm_gpt_case_list_auto $FLAGS_install_deps $FLAGS_download_data
+            execute_func_list $cmd gpt-3_auto
+            # there is no need to repeat the `gpt` download process later.
+            export FLAGS_download_data="gpt ""$FLAGS_download_data"
+            let case_num++
+            clean_file /workspace/PaddleNLP/llm/auto_parallel/gpt-3
+        elif [[ ${case} == "gpt-3_dygraph" ]];then
+            cmd=/workspace/PaddleNLP/scripts/distribute/ci_case_dy.sh
+            timeout 5m bash $cmd prepare_case llm_gpt_case_list_dygraph $FLAGS_install_deps $FLAGS_download_data
+            execute_func_list $cmd gpt-3_dygraph
+            let case_num++
+            clean_file /workspace/PaddleNLP/llm
+        else
+            echo -e "\033[31m ---- no ${case} \033"
+            let case_num++
+        fi
+    done
+    echo -e "\033[31m ---- end run case  \033"
+
+    echo -e "\033[31m ---- total tests :  $global_total_count \033"
+    if [ ${#global_exit_250_arr[@]} -ne 0 ]; then
+        echo -e "\033[32m ---- exit 250 test  :  ${#global_exit_250_arr[@]} \033"
+        for case in "${global_exit_250_arr[@]}"; do
+            echo -e "\t$case(exit 250)"
+        done
+    fi
+
+    if [ ${#global_runtime_fail_arr[@]} -eq 0 ] && [ ${#global_verification_fail_arr[@]} -eq 0 ]; then
+        echo -e "\033[32m ---- all cases Success  \033"
+        EXCODE=0
+    else
+        echo -e "\033[32m ---- runtime failed test  :  ${#global_runtime_fail_arr[@]} \033"
+        for case in "${global_runtime_fail_arr[@]}"; do
+            echo -e "\t$case(failed)"
+        done
+        echo -e "\033[32m ---- verification failed test  :  ${#global_verification_fail_arr[@]} \033"
+        for case in "${global_verification_fail_arr[@]}"; do
+            echo -e "\t$case(failed)"
+        done
+        EXCODE=1
+    fi
+else
+    echo -e "\033[32m Changed Not CI case, Skips \033"
+    EXCODE=0
+fi
+exit $EXCODE

tools/auto_parallel/ci_case_unit.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+export log_path=/workspace/case_logs
+export auto_case_path=/workspace/Paddle/test/auto_parallel/hybrid_strategy
+export dygraph_case_path=/workspace/Paddle/test/collective/hybrid_strategy
+
+function case_list_unit() {
+    if [ ! -f "testslist.csv" ]; then
+        echo "Error: testslist.csv not found in current directory: $(pwd)"
+        exit -1
+    fi
+    if [ ! -f "${log_path}/blacklist.csv" ]; then
+        wget -P ${log_path}/ https://paddle-qa.bj.bcebos.com/Auto-Parallel/blacklist.csv --no-proxy || exit 101
+        echo -e "\033[31m ---- wget blacklist.csv \033[0m"
+    fi
+    blacklist_file=${log_path}/blacklist.csv
+    mapfile -t blacklist < "$blacklist_file"
+
+    target_key=${1:-"all"}
+    for ((i=2; i<=`awk -F, 'END {print NR}' testslist.csv`; i++)); do
+        item=`awk -F, 'NR=='$i' {print}' testslist.csv`
+        case_name=`awk -F, 'NR=='$i' {print $1}' testslist.csv`
+        if [[ ${target_key} != "all" ]] && [[ ! ${case_name} =~ ${target_key} ]]; then
+            echo "=========== skip $case_name run  ==========="
+            continue
+        elif [[ " ${blacklist[@]} " == *" ${case_name} "* ]]; then
+            echo "======= skip blacklist case: $case_name run  ======="
+            continue
+        else
+            echo "=========== $case_name run  begin ==========="
+        fi
+        if [[ $item =~ PYTHONPATH=([^,;]*)([,;]|$) ]]; then
+            substring="${BASH_REMATCH[1]}"
+            echo "PYTHONPATH=$substring"
+            export PYTHONPATH=$substring:$PYTHONPATH
+        fi
+        python $case_name.py >>${log_path}/$case_name 2>&1
+        if [ $? -eq 0 ]; then
+            tail -n 10 ${log_path}/$case_name
+        fi
+        echo "=========== $case_name run  end ==========="
+    done
+}
+
+main() {
+    export exec_case=$1
+    echo -e "\033[31m ---- Start executing $exec_case case \033[0m"
+
+    if [[ $exec_case == "auto_unit_test" ]];then
+        cd ${auto_case_path}
+        case_list_unit
+    elif [[ $exec_case == "dygraph_unit_test" ]];then
+        cd ${dygraph_case_path}
+        case_list_unit
+    elif [[ $exec_case == "llama_auto_unit_test" ]];then
+        cd ${auto_case_path}
+        case_list_unit llama
+        case_list_unit pir_reshard
+    else
+        echo -e "\033[31m ---- Invalid exec_case $exec_case \033[0m"
+    fi
+}
+
+main $@

tools/auto_parallel/ci_distributed_stable.sh

@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source /workspace/Paddle/tools/auto_parallel/target_path_lists.sh
+
+export paddle=$1
+export paddle_dir=/workspace/Paddle
+mkdir -p /workspace/case_logs
+export log_path=/workspace/case_logs
+export case_list=()
+
+install_paddle(){
+    echo -e "\033[31m ---- Install paddlepaddle-gpu  \033"
+    if [ -n "$paddle" ];then
+      python -m pip install --user ${paddle} --no-dependencies;
+    fi
+    python -c "import paddle; print('paddle version:',paddle.__version__,'\npaddle commit:',paddle.version.commit)";
+}
+
+get_diff_TO_case(){
+cd ${paddle_dir}
+# get the location of "test/auto_parallel" in target_lists_for_semi_auto_ci
+count=0
+for element in "${target_lists_for_semi_auto_ci[@]}";do
+  if [[ "$element" == "test/auto_parallel" ]]; then
+    test_auto_num=$count
+    break
+  fi
+  count=$((count+1))
+done
+# get the location of "test/collective/hybrid_strategy" in target_lists_for_dygraph_ci
+count=0
+for element in "${target_lists_for_dygraph_ci[@]}";do
+  if [[ "$element" == "test/collective/hybrid_strategy" ]]; then
+    test_dygraph_num=$count
+    break
+  fi
+  count=$((count+1))
+done
+for file_name in `git diff --numstat upstream/${AGILE_COMPILE_BRANCH} |awk '{print $NF}'`;do
+    arr_file_name=(${file_name//// })
+    dir1=${arr_file_name[0]}
+    dir2=${arr_file_name[1]}
+    dir3=${arr_file_name[2]}
+    dir4=${arr_file_name[3]}
+    dir5=${arr_file_name[4]}
+    dir6=${arr_file_name[5]}
+    file_item=$dir1/$dir2/$dir3/$dir4/$dir5/$dir6
+    echo "file_name:"${file_name}, "path:"${file_item}
+    if [ ! -f ${file_name} ];then # deleting files for PR
+        continue
+    elif [[ ${file_name##*.} == "md" ]] || [[ ${file_name##*.} == "rst" ]] || [[ ${dir1} == "docs" ]];then
+        continue
+    else
+        # The most auto unittests have been monitored in PR-CI-Distribute-stable,
+        # while the other tests of llama model will be executed in PR-CI-Auto-Parallel.
+        for ((i=0; i<${#target_lists_for_semi_auto_ci[@]}; i++)); do
+            if [[ ${file_item} == *${target_lists_for_semi_auto_ci[i]}* ]];then
+                case_list[${#case_list[*]}]="auto_unit_test"
+                break
+            else
+                continue
+            fi
+        done
+        # The dynamic unittests have been monitored in PR-CI-Distribute-stable
+        # and will be no longer redundantly executed in PR-CI-Auto-Parallel.
+        for ((i=0; i<${#target_lists_for_dygraph_ci[@]}; i++)); do
+            if [[ ${file_item} == *${target_lists_for_dygraph_ci[i]}* ]];then
+                case_list[${#case_list[*]}]="dygraph_unit_test"
+                break
+            else
+                continue
+            fi
+        done
+    fi
+done
+}
+
+print_info(){
+#解决异常退出-6的问题，CI中的偶现问题，无法发现
+if [[ $1 -ne 0 ]] && [[ $1 -ne 250 ]];then
+    EXCODE=2
+    if [ ! -f ${log_path}/$2 ];then
+        echo -e "\033[31m run $2 CI FAIL \033"
+    else
+        mv ${log_path}/$2 ${log_path}/$2_FAIL.log
+        echo -e "\033[31m ${log_path}/$2_FAIL \033"
+        tail -70 ${log_path}/$2_FAIL.log
+    fi
+    exit $EXCODE
+else
+    echo -e "\033[32m run $3 CI SUCCESS \033"
+fi
+}
+
+# Get the list of pending cases
+get_diff_TO_case
+# Remove duplicates and store the results back to the original list
+
+####################
+case_list=($(awk -v RS=' ' '!a[$1]++' <<< ${case_list[*]}))
+if [[ ${#case_list[*]} -ne 0 ]];then
+    echo -e "\033[31m =======CI Check case========= \033"
+    echo -e "\033[31m ---- case_list length: ${#case_list[*]}, cases: ${case_list[*]} \033"
+    echo -e "\033[31m ============================= \033"
+    set +e
+
+    # Install paddle
+    install_paddle
+    case_num=1
+    export FLAGS_install_deps=0
+    for case in ${case_list[*]};do
+        echo -e "\033[31m ---- running case $case_num/${#case_list[*]}: ${case} \033"
+        if [[ ${case} == "llama_auto" ]];then
+            bash /workspace/PaddleNLP/scripts/distribute/ci_case_auto.sh llama_case_list_auto $FLAGS_install_deps $FLAGS_download_data
+            print_info $? `ls -lt ${log_path} | grep "llama" | head -n 1 | awk '{print $9}'` ${case}
+            export FLAGS_install_deps=1
+            export FLAGS_download_data="llama ""$FLAGS_download_data"
+            let case_num++
+        elif [[ ${case} == "auto_unit_test" ]];then
+            bash /workspace/Paddle/tools/auto_parallel/ci_case_unit.sh auto_unit_test
+            print_info $? `ls -lt ${log_path} | grep "test" | head -n 1 | awk '{print $9}'` ${case}
+            let case_num++
+        elif [[ ${case} == "gpt-3_dygraph" ]];then
+            bash /workspace/PaddleNLP/scripts/distribute/ci_case_dy.sh llm_gpt_case_list_dygraph $FLAGS_install_deps $FLAGS_download_data
+            print_info $? `ls -lt ${log_path} | grep "llm_gpt" | head -n 1 | awk '{print $9}'` ${case}
+            let case_num++
+        elif [[ ${case} == "dygraph_unit_test" ]];then
+            bash /workspace/Paddle/tools/auto_parallel/ci_case_unit.sh dygraph_unit_test
+            print_info $? `ls -lt ${log_path} | grep "test" | head -n 1 | awk '{print $9}'` ${case}
+            let case_num++
+        elif [[ ${case} == "llama_auto_unit_test" ]];then
+            bash /workspace/Paddle/tools/auto_parallel/ci_case_unit.sh llama_auto_unit_test
+            print_info $? `ls -lt ${log_path} | grep "test" | head -n 1 | awk '{print $9}'` ${case}
+            let case_num++
+        else
+            echo -e "\033[31m ---- no ${case} \033"
+            let case_num++
+        fi
+    done
+    echo -e "\033[31m ---- end run case  \033"
+    cd ${log_path}
+    if [ ! -f *FAIL* ];then
+        FF=0
+        EXCODE=0
+        echo -e "\033[32m ---- all case Success \033"
+    else
+        FF=`ls *FAIL*|wc -l`
+        EXCODE=2
+        echo -e "\033[31m ---- case Failed number: ${FF} \033"
+        ls *_FAIL*
+    fi
+else
+    echo -e "\033[32m Changed Not CI case, Skips \033"
+    EXCODE=0
+fi
+exit $EXCODE

tools/auto_parallel/target_path_lists.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+target_lists_for_semi_auto_ci=(
+    "python/paddle/distributed/auto_parallel"
+    "python/paddle/distributed/checkpoint"
+    "paddle/fluid/distributed/auto_parallel"
+    "paddle/fluid/framework/new_executor"
+    "paddle/fluid/pybind/auto_parallel_py.cc"
+    "paddle/fluid/pybind/auto_parallel_py.h"
+    "paddle/phi/infermeta/spmd_rules"
+    "paddle/phi/core/distributed"
+    "paddle/phi/api/generator/dist_api_gen.py"
+    "paddle/phi/api/generator/dist_bw_api_gen.py"
+    "tools/auto_parallel/target_path_lists.sh"
+    "test/auto_parallel"
+    "paddle/fluid/ir_adaptor/"
+    "paddle/fluid/pir/dialect"
+    "paddle/fluid/pir/transforms"
+    "paddle/fluid/pir/serialize_deserialize"
+    "test/auto_parallel/hybrid_strategy/semi_auto_llama_save_load.py"
+    "python/paddle/base/executor.py"
+)
+
+target_lists_for_dygraph_ci=(
+    "python/paddle/distributed/fleet"
+    "python/paddle/distributed/communication"
+    "python/paddle/distributed/sharding"
+    "paddle/fluid/distributed/collective"
+    "paddle/phi/core/distributed"
+    "tools/auto_parallel/target_path_lists.sh"
+    "test/collective/hybrid_strategy"
+)

tools/check_abi_compatibility.py

@@ -0,0 +1,520 @@
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Check Linux wheel ABI compatibility by comparing protected ELF symbols.
+
+The check is intentionally one-way: symbols added by a PR are allowed, while
+protected symbols present in the base wheel must still exist in the PR wheel.
+"""
+
+from __future__ import annotations
+
+import argparse
+import glob
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import urllib.error
+import urllib.request
+import zipfile
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+WHEEL_LIBRARY_PATHS = (
+    "paddle/base/libpaddle.so",
+    "paddle/libs/libphi.so",
+    "paddle/libs/libphi_core.so",
+    "paddle/libs/libphi_gpu.so",
+)
+
+DEFINED_DYNAMIC_SYMBOL_TYPES = {"FUNC", "OBJECT"}
+
+PROTECTED_COMPAT_CXX_PREFIXES = (
+    "c10::",
+    "at::",
+    "torch::",
+    "caffe2::",
+)
+
+PROTECTED_COMPAT_MANGLED_CXX_PREFIXES = (
+    "_ZN2at",
+    "_ZNK2at",
+    "_ZN3c10",
+    "_ZNK3c10",
+    "_ZN5torch",
+    "_ZNK5torch",
+    "_ZN6caffe2",
+    "_ZNK6caffe2",
+)
+
+REQUIRED_ABI_APPROVERS = ("SigureMo", "BingooYang")
+DEFAULT_GITHUB_REPOSITORY = "PaddlePaddle/Paddle"
+GITHUB_API_URL = "https://api.github.com"
+
+
+@dataclass(frozen=True)
+class DynamicSymbol:
+    name: str
+    symbol_type: str
+    bind: str
+    section: str
+    demangled_name: str
+
+
+@dataclass(frozen=True)
+class RemovedSymbol:
+    library: str
+    name: str
+    demangled_name: str
+
+
+@dataclass(frozen=True)
+class MissingLibrary:
+    library: str
+
+
+@dataclass(frozen=True)
+class ApprovalCheckResult:
+    approved: bool
+    reviewer: str | None = None
+    reason: str | None = None
+
+
+def strip_elf_symbol_version(symbol_name: str) -> str:
+    if "@@" in symbol_name:
+        return symbol_name.split("@@", 1)[0]
+    if "@" in symbol_name:
+        return symbol_name.split("@", 1)[0]
+    return symbol_name
+
+
+def parse_readelf_dynamic_symbols(readelf_output: str) -> list[DynamicSymbol]:
+    symbols = []
+    for line in readelf_output.splitlines():
+        fields = line.split()
+        if len(fields) < 8 or not fields[0].endswith(":"):
+            continue
+        symbol_type = fields[3]
+        bind = fields[4]
+        section = fields[6]
+        name = fields[7]
+        if (
+            bind != "GLOBAL"
+            or section == "UND"
+            or symbol_type not in DEFINED_DYNAMIC_SYMBOL_TYPES
+        ):
+            continue
+        symbols.append(
+            DynamicSymbol(
+                name=name,
+                symbol_type=symbol_type,
+                bind=bind,
+                section=section,
+                demangled_name=strip_elf_symbol_version(name),
+            )
+        )
+    return symbols
+
+
+def demangle_symbol_names(symbol_names: Iterable[str]) -> dict[str, str]:
+    unique_names = sorted(
+        {strip_elf_symbol_version(name) for name in symbol_names}
+    )
+    if not unique_names:
+        return {}
+    cxxfilt = shutil.which("c++filt")
+    if cxxfilt is None:
+        return {name: name for name in unique_names}
+
+    try:
+        result = subprocess.run(
+            [cxxfilt],
+            input="\n".join(unique_names),
+            text=True,
+            capture_output=True,
+            check=True,
+        )
+    except (OSError, subprocess.CalledProcessError):
+        return {name: name for name in unique_names}
+
+    demangled = result.stdout.splitlines()
+    if len(demangled) != len(unique_names):
+        return {name: name for name in unique_names}
+    return dict(zip(unique_names, demangled))
+
+
+def attach_demangled_names(
+    symbols: Iterable[DynamicSymbol],
+) -> list[DynamicSymbol]:
+    symbol_list = list(symbols)
+    demangled_names = demangle_symbol_names(
+        symbol.name for symbol in symbol_list
+    )
+    return [
+        DynamicSymbol(
+            name=symbol.name,
+            symbol_type=symbol.symbol_type,
+            bind=symbol.bind,
+            section=symbol.section,
+            demangled_name=demangled_names.get(
+                strip_elf_symbol_version(symbol.name), symbol.demangled_name
+            ),
+        )
+        for symbol in symbol_list
+    ]
+
+
+def is_protected_paddle_abi_symbol(symbol: DynamicSymbol) -> bool:
+    demangled = symbol.demangled_name
+    if demangled.startswith(PROTECTED_COMPAT_CXX_PREFIXES):
+        return True
+
+    raw_name = strip_elf_symbol_version(symbol.name)
+    return raw_name.startswith(PROTECTED_COMPAT_MANGLED_CXX_PREFIXES)
+
+
+def protected_symbols_by_name(
+    symbols: Iterable[DynamicSymbol],
+) -> dict[str, DynamicSymbol]:
+    return {
+        symbol.name: symbol
+        for symbol in symbols
+        if is_protected_paddle_abi_symbol(symbol)
+    }
+
+
+def read_dynamic_symbols(library_path: str) -> list[DynamicSymbol]:
+    try:
+        result = subprocess.run(
+            ["readelf", "--dyn-syms", "-W", library_path],
+            text=True,
+            capture_output=True,
+            check=True,
+        )
+    except FileNotFoundError as exc:
+        raise RuntimeError("readelf is required to check ABI symbols") from exc
+    except subprocess.CalledProcessError as exc:
+        raise RuntimeError(
+            f"Failed to read dynamic symbols from {library_path}:\n{exc.stderr}"
+        ) from exc
+
+    return attach_demangled_names(parse_readelf_dynamic_symbols(result.stdout))
+
+
+def extract_wheel_libraries(
+    wheel_path: str, library_paths: Iterable[str], output_dir: str
+) -> dict[str, str]:
+    extracted_libraries = {}
+    with zipfile.ZipFile(wheel_path) as wheel:
+        wheel_entries = set(wheel.namelist())
+        for library_path in library_paths:
+            if library_path not in wheel_entries:
+                continue
+            extracted_path = wheel.extract(library_path, output_dir)
+            extracted_libraries[library_path] = extracted_path
+    return extracted_libraries
+
+
+def compare_library_symbols(
+    library: str,
+    base_symbols: Iterable[DynamicSymbol] | None,
+    pr_symbols: Iterable[DynamicSymbol] | None,
+) -> list[RemovedSymbol | MissingLibrary]:
+    if base_symbols is None:
+        return []
+    if pr_symbols is None:
+        return [MissingLibrary(library=library)]
+
+    base_protected_symbols = protected_symbols_by_name(base_symbols)
+    pr_protected_symbols = protected_symbols_by_name(pr_symbols)
+    removed_names = sorted(
+        set(base_protected_symbols) - set(pr_protected_symbols)
+    )
+    return [
+        RemovedSymbol(
+            library=library,
+            name=name,
+            demangled_name=base_protected_symbols[name].demangled_name,
+        )
+        for name in removed_names
+    ]
+
+
+def resolve_wheel_path(pattern: str, label: str) -> str:
+    matches = sorted(glob.glob(pattern))
+    if len(matches) != 1:
+        raise RuntimeError(
+            f"Expected exactly one {label} wheel matching {pattern}, "
+            f"but found {len(matches)}: {matches}"
+        )
+    return matches[0]
+
+
+def compare_wheel_abi(
+    base_wheel: str, pr_wheel: str, library_paths: Iterable[str]
+) -> list[RemovedSymbol | MissingLibrary]:
+    with tempfile.TemporaryDirectory(prefix="paddle_abi_check_") as temp_dir:
+        base_dir = os.path.join(temp_dir, "base")
+        pr_dir = os.path.join(temp_dir, "pr")
+        base_libraries = extract_wheel_libraries(
+            base_wheel, library_paths, base_dir
+        )
+        pr_libraries = extract_wheel_libraries(pr_wheel, library_paths, pr_dir)
+
+        issues: list[RemovedSymbol | MissingLibrary] = []
+        for library in library_paths:
+            base_path = base_libraries.get(library)
+            pr_path = pr_libraries.get(library)
+            base_symbols = (
+                read_dynamic_symbols(base_path)
+                if base_path is not None
+                else None
+            )
+            pr_symbols = (
+                read_dynamic_symbols(pr_path) if pr_path is not None else None
+            )
+            issues.extend(
+                compare_library_symbols(library, base_symbols, pr_symbols)
+            )
+        return issues
+
+
+def format_issues(
+    issues: Iterable[RemovedSymbol | MissingLibrary],
+    max_report: int,
+    title: str = "ABI compatibility check failed.",
+) -> str:
+    issue_list = list(issues)
+    lines = [
+        title,
+        "The PR wheel removed protected dynamic symbols that exist in the base "
+        "wheel. Removing these symbols can break downstream wheels or shared "
+        "libraries compiled against the base branch.",
+        "",
+    ]
+    for issue in issue_list[:max_report]:
+        if isinstance(issue, MissingLibrary):
+            lines.extend(
+                [
+                    f"Library: {issue.library}",
+                    "  PR wheel is missing this library, but the base wheel "
+                    "contains it.",
+                    "",
+                ]
+            )
+        else:
+            lines.extend(
+                [
+                    f"Library: {issue.library}",
+                    f"  Raw symbol: {issue.name}",
+                    f"  Demangled: {issue.demangled_name}",
+                    "",
+                ]
+            )
+
+    omitted_count = len(issue_list) - max_report
+    if omitted_count > 0:
+        lines.append(f"... omitted {omitted_count} additional removed symbols.")
+    return "\n".join(lines)
+
+
+def find_required_abi_approver(
+    reviews: Iterable[dict],
+    required_approvers: Iterable[str] = REQUIRED_ABI_APPROVERS,
+) -> str | None:
+    required_logins = {login.lower() for login in required_approvers}
+    for review in reviews:
+        if review.get("state") != "APPROVED":
+            continue
+        user = review.get("user")
+        if not isinstance(user, dict):
+            continue
+        login = user.get("login")
+        if isinstance(login, str) and login.lower() in required_logins:
+            return login
+    return None
+
+
+def fetch_pr_reviews(
+    pr_id: str,
+    token: str,
+    repository: str = DEFAULT_GITHUB_REPOSITORY,
+    api_url: str = GITHUB_API_URL,
+) -> list[dict]:
+    reviews: list[dict] = []
+    page = 1
+    while True:
+        url = (
+            f"{api_url}/repos/{repository}/pulls/{pr_id}/reviews"
+            f"?per_page=100&page={page}"
+        )
+        request = urllib.request.Request(
+            url,
+            headers={
+                "Accept": "application/vnd.github+json",
+                "Authorization": f"token {token}",
+                "User-Agent": "Paddle-ABI-Compatibility-Check",
+            },
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=30) as response:
+                page_reviews = json.loads(response.read().decode("utf-8"))
+        except urllib.error.HTTPError as exc:
+            raise RuntimeError(
+                f"failed to fetch PR reviews from GitHub: HTTP {exc.code}"
+            ) from exc
+        except (OSError, json.JSONDecodeError) as exc:
+            raise RuntimeError(
+                f"failed to fetch PR reviews from GitHub: {exc}"
+            ) from exc
+
+        if not isinstance(page_reviews, list):
+            raise RuntimeError("GitHub PR reviews response is not a list")
+
+        reviews.extend(page_reviews)
+        if len(page_reviews) < 100:
+            break
+        page += 1
+    return reviews
+
+
+def check_abi_removal_approval(
+    env: dict[str, str] | None = None,
+    fetch_reviews=fetch_pr_reviews,
+) -> ApprovalCheckResult:
+    env = os.environ if env is None else env
+    pr_id = env.get("GIT_PR_ID") or env.get("PR_ID")
+    token = (
+        env.get("GITHUB_API_TOKEN")
+        or env.get("GITHUB_TOKEN")
+        or env.get("GH_TOKEN")
+    )
+    repository = env.get("GITHUB_REPOSITORY", DEFAULT_GITHUB_REPOSITORY)
+
+    if not pr_id:
+        return ApprovalCheckResult(
+            approved=False, reason="GIT_PR_ID or PR_ID is not set"
+        )
+    if not token:
+        return ApprovalCheckResult(
+            approved=False,
+            reason="GITHUB_API_TOKEN, GITHUB_TOKEN, or GH_TOKEN is not set",
+        )
+
+    try:
+        reviews = fetch_reviews(pr_id, token, repository)
+    except RuntimeError as exc:
+        return ApprovalCheckResult(approved=False, reason=str(exc))
+
+    reviewer = find_required_abi_approver(reviews)
+    if reviewer is not None:
+        return ApprovalCheckResult(approved=True, reviewer=reviewer)
+    return ApprovalCheckResult(
+        approved=False,
+        reason=(
+            "no APPROVED review from "
+            + " or ".join(REQUIRED_ABI_APPROVERS)
+            + " was found"
+        ),
+    )
+
+
+def check_abi_issues_approval(
+    issues: Iterable[RemovedSymbol | MissingLibrary],
+    env: dict[str, str] | None = None,
+    fetch_reviews=fetch_pr_reviews,
+) -> ApprovalCheckResult:
+    if not list(issues):
+        return ApprovalCheckResult(approved=True, reason="no ABI issues")
+    return check_abi_removal_approval(env=env, fetch_reviews=fetch_reviews)
+
+
+def format_approval_failure(approval: ApprovalCheckResult) -> str:
+    lines = [
+        "You must have one RD (SigureMo or BingooYang) approval for protected "
+        "ABI symbol removals.",
+    ]
+    if approval.reason:
+        lines.append(f"Approval check failed: {approval.reason}.")
+    return "\n".join(lines)
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    paddle_root = os.environ.get("PADDLE_ROOT", os.getcwd())
+    parser = argparse.ArgumentParser(
+        description="Check Linux wheel ABI compatibility for Paddle symbols."
+    )
+    parser.add_argument(
+        "--base-wheel",
+        default=os.path.join(paddle_root, "build/dev_whl/*.whl"),
+        help="Base branch wheel path or glob pattern.",
+    )
+    parser.add_argument(
+        "--pr-wheel",
+        default=os.path.join(paddle_root, "build/pr_whl/*.whl"),
+        help="PR wheel path or glob pattern.",
+    )
+    parser.add_argument(
+        "--max-report",
+        type=int,
+        default=200,
+        help="Maximum number of ABI issues to print.",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(sys.argv[1:] if argv is None else argv)
+    try:
+        base_wheel = resolve_wheel_path(args.base_wheel, "base")
+        pr_wheel = resolve_wheel_path(args.pr_wheel, "PR")
+        issues = compare_wheel_abi(base_wheel, pr_wheel, WHEEL_LIBRARY_PATHS)
+    except RuntimeError as exc:
+        print(f"ABI compatibility check failed: {exc}", file=sys.stderr)
+        return 1
+
+    if issues:
+        approval = check_abi_issues_approval(issues)
+        if approval.approved:
+            reviewer = approval.reviewer or "a required reviewer"
+            print(
+                format_issues(
+                    issues,
+                    args.max_report,
+                    title="ABI compatibility issues found.",
+                ),
+                file=sys.stderr,
+            )
+            print(
+                f"Protected ABI symbol removals were approved by {reviewer}; "
+                "continuing.",
+                file=sys.stderr,
+            )
+            return 0
+        print(format_issues(issues, args.max_report), file=sys.stderr)
+        print(format_approval_failure(approval), file=sys.stderr)
+        return 1
+
+    print("ABI compatibility check passed.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/check_added_ut.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set +e
+set -x
+SYSTEM=`uname -s`
+if [ -z ${BRANCH} ]; then
+    BRANCH="develop"
+fi
+
+if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then
+    PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../" && pwd )"
+elif [[ "$SYSTEM" == "Windows_NT" ]];then
+    PADDLE_ROOT="$(cd "$PWD/../" && pwd )"
+fi
+CURDIR=`pwd`
+cd $PADDLE_ROOT
+if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then
+    cp $PADDLE_ROOT/paddle/scripts/paddle_build.sh $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh
+elif [[ "$SYSTEM" == "Windows_NT" ]];then
+    git remote | grep upstream
+    if [ $? != 0 ]; then
+        git remote add upstream https://github.com/PaddlePaddle/Paddle.git
+    fi
+    git fetch upstream ${BRANCH}
+fi
+CURBRANCH=`git rev-parse --abbrev-ref HEAD`
+echo $CURBRANCH
+if [ `git branch | grep 'prec_added_ut'` ];then
+    git branch -D 'prec_added_ut'
+fi
+git checkout -b prec_added_ut upstream/${BRANCH}
+git branch
+mkdir prec_build
+cd prec_build
+if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then
+    bash $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh cmake_gen_in_current_dir >prebuild.log 2>&1
+elif [[ "$SYSTEM" == "Windows_NT" ]];then
+    bash $PADDLE_ROOT/win_cmake.sh >prec_build.log 2>&1
+fi
+# remove line ended with .exe to get correct deleted_ut list
+ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' | sed '/\.exe$/d' | grep 'test' > $PADDLE_ROOT/br-ut
+#UNITTEST_DEV.spec is used for checking changes of unittests between pr and paddle_develop in the later step
+spec_path_dev=${PADDLE_ROOT}/paddle/fluid/UNITTEST_DEV.spec
+ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' > ${spec_path_dev}
+cd $PADDLE_ROOT/build
+spec_path_pr=${PADDLE_ROOT}/paddle/fluid/UNITTEST_PR.spec
+ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' > ${spec_path_pr}
+ctest -N | awk -F ':' '{print $2}' | sed '/^$/d' | sed '$d' | sed 's/ //g' | sed '/\.exe$/d' | grep 'test' > $PADDLE_ROOT/pr-ut
+cd $PADDLE_ROOT
+grep -F -x -v -f br-ut pr-ut > $PADDLE_ROOT/added_ut
+if [[ "$SYSTEM" == 'Linux' ]];then
+    sort pr-ut |uniq -d > $PADDLE_ROOT/duplicate_ut
+fi
+echo "New-UT:"
+cat $PADDLE_ROOT/added_ut
+rm -rf prec_build
+if [[ "$SYSTEM" == "Linux" ]] || [[ "$SYSTEM" == "Darwin" ]];then
+    rm $PADDLE_ROOT/br-ut $PADDLE_ROOT/pr-ut $PADDLE_ROOT/paddle/scripts/paddle_build_pre.sh
+elif [[ "$SYSTEM" == "Windows_NT" ]];then
+    # get the deleted ut list in windows, will be used in check_change_of_unittest.sh
+    grep -F -x -v -f pr-ut br-ut > $PADDLE_ROOT/deleted_ut
+    rm $PADDLE_ROOT/br-ut $PADDLE_ROOT/pr-ut $PADDLE_ROOT/win_cmake.sh
+fi
+git checkout -f $CURBRANCH
+echo $CURBRANCH
+git branch -D prec_added_ut
+cd $CURDIR

tools/check_api_approvals.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+if [ -z ${BRANCH} ]; then
+    BRANCH="develop"
+fi
+
+PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../" && pwd )"
+approval_line=`curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000`
+failed_num=0
+echo_list=()
+
+function check_approval(){
+    person_num=`echo $@|awk '{for (i=2;i<=NF;i++)print $i}'`
+    APPROVALS=`echo ${approval_line}|python ${PADDLE_ROOT}/tools/check_pr_approval.py $1 $person_num`
+    if [[ "${APPROVALS}" == "FALSE" && "${echo_line}" != "" ]]; then
+        add_failed "${failed_num}. ${echo_line}"
+    fi
+}
+
+function add_failed(){
+    failed_num=`expr $failed_num + 1`
+    echo_list="${echo_list[@]}$1"
+}
+
+
+api_params_diff=`python ${PADDLE_ROOT}/tools/check_api_compatible.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.spec  ${PADDLE_ROOT}/paddle/fluid/API_PR.spec`
+api_spec_diff=`python ${PADDLE_ROOT}/tools/diff_api.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.spec.api  ${PADDLE_ROOT}/paddle/fluid/API_PR.spec.api`
+api_annotation_diff=`python ${PADDLE_ROOT}/tools/diff_api.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.spec.annotations  ${PADDLE_ROOT}/paddle/fluid/API_PR.spec.annotations`
+if [ "$api_spec_diff" != "" -o "${api_params_diff}" != "" ]; then
+    echo_line="You must have one RD (XiaoguangHu01, jeff41404, qingqing01, zhwesky2010) approval for API change.\n"
+
+    check_approval 1 XiaoguangHu01 jeff41404 qingqing01 zhwesky2010
+fi
+
+if [ "$api_annotation_diff" != "" ]; then
+    echo_line="You must have one member of Typing group (SigureMo, zrr1999, megemini, sunzhongkai588) approval for API annotation change.\n"
+    check_approval 1 SigureMo zrr1999 megemini sunzhongkai588
+fi
+
+api_yaml_diff=`python ${PADDLE_ROOT}/tools/check_api_yaml_same.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.spec  ${PADDLE_ROOT}/paddle/fluid/API_PR.spec ${BRANCH} ${PADDLE_ROOT}`
+if [ "$api_yaml_diff" != "" ]; then
+    echo_line="API's name and params should be consistent with op's name and params in yaml.
+                The API or Yaml file you changed may cause inconsistent.\n"
+    echo_line="${echo_line} please request one of the RD (jeff41404, zhwesky2010, zhangbo9674) review and approve.\n"
+    echo_line="${echo_line}\r\n ${api_yaml_diff}\n"
+    check_approval 1 jeff41404 zhwesky2010 zhangbo9674 XiaoguangHu01
+fi
+
+op_type_spec_diff=`python ${PADDLE_ROOT}/tools/check_op_register_type.py ${PADDLE_ROOT}/paddle/fluid/OP_TYPE_DEV.spec  ${PADDLE_ROOT}/paddle/fluid/OP_TYPE_PR.spec`
+if [ "$op_type_spec_diff" != "" ]; then
+    echo_line="You must have one RD (wanghuancoder or SigureMo) approval for the data_type registration of new operator. More data_type of new operator should be registered in your PR. Please make sure that both float/double (or int/int64_t) have been registered.\n For more details, please click [https://github.com/PaddlePaddle/Paddle/wiki/Data-types-of-generic-Op-must-be-fully-registered].\n"
+    check_approval 1 wanghuancoder SigureMo
+fi
+
+op_kernel_dtype_spec_diff=`python ${PADDLE_ROOT}/tools/check_op_kernel_same_dtypes.py ${PADDLE_ROOT}/paddle/fluid/OP_KERNEL_DTYPE_DEV.spec  ${PADDLE_ROOT}/paddle/fluid/OP_KERNEL_DTYPE_PR.spec`
+if [ "$op_kernel_dtype_spec_diff" != "" ]; then
+    echo_line="You have added or modified Op Kernel, resulting in inconsistent data types supported by the forward and backward kernels of the same op, such modifications are not allowed in principle. If it is a mismatch, please request one RD (wanghuancoder or SigureMo) review and approve. Including the following kernels:\n${op_kernel_dtype_spec_diff}\n"
+    check_approval 1 wanghuancoder SigureMo
+fi
+
+op_desc_diff=`python ${PADDLE_ROOT}/tools/check_op_desc.py ${PADDLE_ROOT}/paddle/fluid/OP_DESC_DEV.spec  ${PADDLE_ROOT}/paddle/fluid/OP_DESC_PR.spec`
+inference_approve=`echo "$op_desc_diff" | grep "need inference to review" -`
+slim_approve=`echo "$op_desc_diff" | grep "need slim to review" -`
+if [ "$op_desc_diff" != "" ]; then
+    echo_line="You must have one RD (inference[ vivienfanghuagood(Recommend), yuanlehome, qingqing01 ] or slim[ wanghaoshuang(Recommend), qingqing01 ] or train[ zhangbo9674 ]) approval for the changes of Inputs/Output/Attrs of OPs. The changes of OPs will cause that the new version inference fails to load model trained by the old version. Please modify your code. \n For more details, please click [https://github.com/PaddlePaddle/Paddle/wiki/OP-Input-Output-Attribute-Compatibility-Modification].\n${op_desc_diff}\n"
+    check_approval 1 vivienfanghuagood yuanlehome qingqing01 wanghaoshuang zhangbo9674
+fi
+
+if [ "$slim_approve" != "" ]; then
+    echo_line="You must have one RD (wanghaoshuang(Recommend), qingqing01) approval for the changes of `quant` Inputs/Output/Attrs of OPs. \n For more details, please click [https://github.com/PaddlePaddle/Paddle/wiki/OP-Input-Output-Attribute-Compatibility-Modification].\n${slim_approve}\n"
+    check_approval 1 wanghaoshuang qingqing01
+fi
+
+if [ "$inference_approve" != "" ]; then
+    echo_line="You must have one RD (qingqing01(Recommend), heavengate) approval for the changes of `def` Inputs/Output/Attrs of OPs. \n For more details, please click [https://github.com/PaddlePaddle/Paddle/wiki/OP-Input-Output-Attribute-Compatibility-Modification].\n${inference_approve}\n"
+    check_approval 1 qingqing01 heavengate
+fi
+
+
+DEV_OP_USE_DEFAULT_GRAD_MAKER_SPEC=${PADDLE_ROOT}/paddle/fluid/op_use_default_grad_maker_DEV.spec
+PR_OP_USE_DEFAULT_GRAD_MAKER_SPEC=${PADDLE_ROOT}/paddle/fluid/op_use_default_grad_maker_PR.spec
+ADDED_OP_USE_DEFAULT_GRAD_MAKER=`python ${PADDLE_ROOT}/tools/diff_use_default_grad_op_maker.py ${DEV_OP_USE_DEFAULT_GRAD_MAKER_SPEC} ${PR_OP_USE_DEFAULT_GRAD_MAKER_SPEC}`
+if [ "${ADDED_OP_USE_DEFAULT_GRAD_MAKER}" != "" ]; then
+  echo_line="You must have one RD (xiaoguoguo626807 (Recommend) or zhhsplendid) approval because you use DefaultGradOpMaker for ${ADDED_OP_USE_DEFAULT_GRAD_MAKER}, which manages the grad_op memory optimization.\n"
+  check_approval 1 xiaoguoguo626807 zhhsplendid
+fi
+
+if [ -n "${echo_list}" ];then
+  echo "**************************************************************"
+  echo "Please find RD for approval first, and then find TPM for approval."
+  echo -e "${echo_list[@]}"
+  echo "There are ${failed_num} approved errors."
+  echo "**************************************************************"
+
+  # L40 L48 L62 has fetch the result out, but there are split.
+  if [ "${api_spec_diff}" != "" -o "${api_annotation_diff}" != "" ] ; then
+    python ${PADDLE_ROOT}/tools/diff_api.py ${PADDLE_ROOT}/paddle/fluid/API_DEV.spec  ${PADDLE_ROOT}/paddle/fluid/API_PR.spec
+  fi
+  if [ "${api_params_diff}" != "" ] ; then
+    echo "api_params_diff: ${api_params_diff}"
+  fi
+  if [ "${op_type_spec_diff}" != "" ] ; then
+    echo "op_type_spec_diff: ${op_type_spec_diff}"
+  fi
+  exit 6
+fi

tools/check_api_compatible.py

@@ -0,0 +1,196 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import inspect
+import logging
+import re
+import sys
+
+logger = logging.getLogger()
+if logger.handlers:
+    # we assume the first handler is the one we want to configure
+    console = logger.handlers[0]
+else:
+    console = logging.StreamHandler(sys.stderr)
+    logger.addHandler(console)
+console.setFormatter(
+    logging.Formatter(
+        "%(asctime)s - %(funcName)s:%(lineno)d - %(levelname)s - %(message)s"
+    )
+)
+
+
+def _check_compatible(args_o, args_n, defaults_o, defaults_n):
+    # 如果参数减少了，需要提醒关注
+    if len(args_o) > len(args_n):
+        logger.debug("args num less then previous: %s vs %s", args_o, args_n)
+        return False
+    # 参数改名了，也要提醒关注
+    for idx in range(min(len(args_o), len(args_n))):
+        if args_o[idx] != args_n[idx]:
+            logger.debug(
+                "args's %d parameter diff with previous: %s vs %s",
+                idx,
+                args_o,
+                args_n,
+            )
+            return False
+    # 新增加了参数，必须提供默认值。以及不能减少默认值数量
+    if (len(args_n) - len(defaults_n)) > (len(args_o) - len(defaults_o)):
+        logger.debug(
+            "defaults num less then previous: %s vs %s", defaults_o, defaults_n
+        )
+        return False
+    # 默认值必须相等
+    for idx in range(min(len(defaults_o), len(defaults_n))):
+        nidx_o = -1 - idx
+        nidx_n = -1 - idx - (len(args_n) - len(args_o))
+        if defaults_o[nidx_o] != defaults_n[nidx_n]:
+            logger.debug(
+                "defaults's %d value diff with previous: %s vs %s",
+                nidx_n,
+                defaults_o,
+                defaults_n,
+            )
+            return False
+    return True
+
+
+def check_compatible(old_api_spec, new_api_spec):
+    """
+    check compatible, FullArgSpec
+    """
+    if not (
+        isinstance(old_api_spec, inspect.FullArgSpec)
+        and isinstance(new_api_spec, inspect.FullArgSpec)
+    ):
+        logger.warning(
+            "new_api_spec or old_api_spec is not instance of inspect.FullArgSpec"
+        )
+        return False
+    return _check_compatible(
+        old_api_spec.args,
+        new_api_spec.args,
+        [] if old_api_spec.defaults is None else old_api_spec.defaults,
+        [] if new_api_spec.defaults is None else new_api_spec.defaults,
+    )
+
+
+def check_compatible_str(old_api_spec_str, new_api_spec_str):
+    patArgSpec = re.compile(
+        r'args=(.*), varargs=.*defaults=(None|\((.*)\)), kwonlyargs=.*'
+    )
+    mo_o = patArgSpec.search(old_api_spec_str)
+    mo_n = patArgSpec.search(new_api_spec_str)
+    if not (mo_o and mo_n):
+        # error
+        logger.warning("old_api_spec_str: %s", old_api_spec_str)
+        logger.warning("new_api_spec_str: %s", new_api_spec_str)
+        return False
+
+    args_o = eval(mo_o.group(1))
+    args_n = eval(mo_n.group(1))
+    defaults_o = mo_o.group(2) if mo_o.group(3) is None else mo_o.group(3)
+    defaults_n = mo_n.group(2) if mo_n.group(3) is None else mo_n.group(3)
+    defaults_o = defaults_o.split(', ') if defaults_o else []
+    defaults_n = defaults_n.split(', ') if defaults_n else []
+    return _check_compatible(args_o, args_n, defaults_o, defaults_n)
+
+
+def read_argspec_from_file(specfile):
+    """
+    read FullArgSpec from spec file
+    """
+    res_dict = {}
+    patArgSpec = re.compile(
+        r'^(paddle[^,]+)\s+\((ArgSpec.*),\s\(\'document\W*([0-9a-z]{32})'
+    )
+    fullargspec_prefix = 'inspect.Full'
+    for line in specfile.readlines():
+        mo = patArgSpec.search(line)
+        if mo and mo.group(2) != 'ArgSpec()':
+            logger.debug("%s argspec: %s", mo.group(1), mo.group(2))
+            try:
+                res_dict[mo.group(1)] = eval(fullargspec_prefix + mo.group(2))
+            except:  # SyntaxError, NameError:
+                res_dict[mo.group(1)] = fullargspec_prefix + mo.group(2)
+    return res_dict
+
+
+arguments = [
+    # flags, dest, type, default, help
+]
+
+
+def parse_args():
+    """
+    Parse input arguments
+    """
+    global arguments
+    parser = argparse.ArgumentParser(
+        description='check api compatible across versions'
+    )
+    parser.add_argument('--debug', dest='debug', action="store_true")
+    parser.add_argument(
+        'prev',
+        type=argparse.FileType('r'),
+        help='the previous version (the version from develop branch)',
+    )
+    parser.add_argument(
+        'post',
+        type=argparse.FileType('r'),
+        help='the post version (the version from PullRequest)',
+    )
+    for item in arguments:
+        parser.add_argument(
+            item[0], dest=item[1], help=item[4], type=item[2], default=item[3]
+        )
+
+    if len(sys.argv) < 2:
+        parser.print_help()
+        sys.exit(1)
+
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.INFO)
+    if args.prev and args.post:
+        prev_spec = read_argspec_from_file(args.prev)
+        post_spec = read_argspec_from_file(args.post)
+        diff_api_names = []
+        for as_post_name, as_post in post_spec.items():
+            as_prev = prev_spec.get(as_post_name)
+            if as_prev is None:  # the api is deleted
+                continue
+            if isinstance(as_prev, str) or isinstance(as_post, str):
+                as_prev_str = (
+                    as_prev if isinstance(as_prev, str) else repr(as_prev)
+                )
+                as_post_str = (
+                    as_post if isinstance(as_post, str) else repr(as_post)
+                )
+                if not check_compatible_str(as_prev_str, as_post_str):
+                    diff_api_names.append(as_post_name)
+            else:
+                if not check_compatible(as_prev, as_post):
+                    diff_api_names.append(as_post_name)
+        if diff_api_names:
+            print('\n'.join(diff_api_names))

tools/check_api_yaml_same.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import difflib
+import os
+import re
+import sys
+
+import yaml
+
+root_path = sys.argv[4]
+
+
+def read_yaml_ops():
+    ops_list = []
+    yaml_path = root_path + "/paddle/phi/ops/yaml/ops.yaml"
+    legacy_yaml_path = (
+        root_path + "/paddle/phi/ops/yaml/inconsistent/dygraph_ops.yaml"
+    )
+
+    with open(yaml_path, 'r') as f:
+        ops_list = yaml.load(f, Loader=yaml.FullLoader)
+    with open(legacy_yaml_path, 'r') as f:
+        ops_list.extend(yaml.load(f, Loader=yaml.FullLoader))
+
+    return ops_list
+
+
+def read_api(api_file):
+    with open(api_file, 'r') as f:
+        pr_apis = f.read()
+        pr_apis = pr_apis.splitlines()
+    result = []
+    for api in pr_apis:
+        # Delete all non-function api
+        if api.find('args') == -1:
+            continue
+        result.append(api)
+    return result
+
+
+def get_api_args(api_item):
+    result = re.search(r"args=\[(?P<args>[^\]]*)\]", api_item)
+    result = [
+        param.strip().replace('\'', '')
+        for param in result.group('args').split(',')
+    ]
+    if result[-1] == 'name':
+        result = result[:-1]
+    return result
+
+
+def get_api_name(api_item):
+    if api_item[0] == '+' or api_item[0] == '-' or api_item[0] == ' ':
+        return api_item.split(" ")[1].split(".")[-1]
+    else:
+        return api_item.split(" ")[0].split(".")[-1]
+
+
+def get_yaml_op_args(op_args):
+    args_list = op_args[1:-1].split(',')
+    args_list = [args.split('=')[0].strip() for args in args_list]
+    return [param.split(' ')[-1].strip() for param in args_list]
+
+
+def get_api_diff(dev_api_file, pr_api_file):
+    develop_apis = read_api(dev_api_file)
+    pr_apis = read_api(pr_api_file)
+
+    differ = difflib.Differ()
+    diff_obj = differ.compare(develop_apis, pr_apis)
+    result = []
+    for each_diff in diff_obj:
+        result.append(each_diff)
+    return result
+
+
+def get_yaml_diff(branch):
+    ops_yaml_path = root_path + "/paddle/phi/ops/yaml/ops.yaml"
+    legacy_yaml_path = (
+        root_path + "/paddle/phi/ops/yaml/inconsistent/dygraph_ops.yaml"
+    )
+    git_cmd = (
+        "git diff -U0 upstream/"
+        + branch
+        + " "
+        + ops_yaml_path
+        + " "
+        + legacy_yaml_path
+    )
+    yaml_diff = os.popen(git_cmd).readlines()
+    result = []
+    for line in yaml_diff:
+        result.append(line.strip('\r\n'))
+    return result
+
+
+api_diffs = get_api_diff(sys.argv[1], sys.argv[2])
+yaml_diffs = get_yaml_diff(sys.argv[3])
+yaml_ops = read_yaml_ops()  # The current PR yaml's ops
+approve_api_msg = []
+approve_yaml_msg = []
+
+api_add = []
+api_delete = []
+
+for each_diff in api_diffs:
+    if each_diff[0] == '+':
+        api_add.append(each_diff)
+    if each_diff[0] == '-':
+        api_delete.append(each_diff)
+
+# remove api that doesn't modify name and args
+add_exclude = []
+delete_exclude = []
+for each_add_diff in api_add:
+    for each_delete_diff in api_delete:
+        if get_api_name(each_add_diff) == get_api_name(
+            each_delete_diff
+        ) and get_api_args(each_add_diff) == get_api_args(each_delete_diff):
+            add_exclude.append(each_add_diff)
+            delete_exclude.append(each_delete_diff)
+
+for exclude_item in add_exclude:
+    api_add.remove(exclude_item)
+for exclude_item in delete_exclude:
+    api_delete.remove(exclude_item)
+
+
+yaml_add = []
+yaml_delete = []
+
+for each_diff in yaml_diffs:
+    if each_diff[0] == '+':
+        yaml_add.append(each_diff)
+    if each_diff[0] == '-':
+        yaml_delete.append(each_diff)
+
+# API add or modified
+for each_add in api_add:
+    add = True
+    modify = False
+    need_approve = True
+    yaml_name_found = False
+    api_name = get_api_name(each_add)
+    api_args = get_api_args(each_add)
+
+    for each_delete_api in api_delete:
+        if get_api_name(each_delete_api) == api_name:
+            modify = True
+            add = False
+
+    # If we find yaml name in yaml_delete, it shows that
+    # yaml op's name is modified.
+    for each_delete_yaml in yaml_delete:
+        if each_delete_yaml.find(api_name) != -1:
+            yaml_name_found = True
+
+    for op in yaml_ops:
+        if op['op'] == api_name:
+            yaml_name_found = True
+            if api_args == get_yaml_op_args(op['args']):
+                need_approve = False
+                break
+
+    # If API is modified and doesn't have a corresponding yaml's op
+    # We needn't approve it
+    if modify and not yaml_name_found:
+        need_approve = False
+
+    # If API is added and yaml's op is not added,
+    # it shows that new api doesn't have a corresponding yaml's op.
+    # We needn't approve it
+    if add and len(yaml_add) == 0:
+        need_approve = False
+
+    # In others, the changes need to be approved.
+    # eg: 1, The args in api is inconsistent with yaml's op
+    #     2, New Api is add, but the yaml op's name may not be inconsistent
+    #        with api's name.
+    #     3, Api's name is modified, but the yaml op's name is not modified.
+    if need_approve:
+        approve_api_msg.append(each_add)
+
+# For yaml, we don't have to consider its add or delete.
+# Because if it is related with api, code above has dealt with it.
+# But we need consider below situation:
+# The op in yaml is modified and its corresponding api is not modified.
+
+if len(api_add) == 0 and len(api_delete) == 0:
+    pr_apis = read_api(sys.argv[2])  # Get all api of this PR
+    for each_diff in yaml_delete:
+        # Note: The condition is relaxed, because symbol '-' can present delete and modification.
+        # So if op is deleted in yaml, code below is also triggered.
+        # But we mainly deal with modification here.
+
+        # If op name in yaml is modified and this name is in API,
+        # this PR need to be reviewed.
+        if each_diff.startswith('-- op'):
+            for api in pr_apis:
+                if each_diff.find(get_api_name(api)) != -1:
+                    approve_yaml_msg.append(each_diff)
+                    break
+
+        # if op args in yaml is modified and this args is in API,
+        # this PR need to be reviewed.
+        if each_diff.startswith('-  args'):
+            yaml_op_args_str = each_diff.removeprefix('-  args : ')
+            yaml_op_args = get_yaml_op_args(yaml_op_args_str)
+            for api in pr_apis:
+                if get_api_args(api) == yaml_op_args:
+                    approve_yaml_msg.append(each_diff)
+                    break
+
+# collect all msg
+approve_msg = []
+if len(approve_api_msg) != 0:
+    approve_msg = ['The APIs you changed are as follows:']
+    approve_msg.extend(approve_api_msg)
+
+if len(approve_yaml_msg) != 0:
+    approve_msg = ['The Yaml File you changed are as follows:']
+    approve_msg.extend(approve_yaml_msg)
+
+print('\r\n'.join(approve_msg))

tools/check_code_block_format.py

@@ -0,0 +1,266 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Tool to check if the code-block directives in the given files follow proper format.
+
+The following format issues are checked:
+
+- The code under code-block directive should be indented properly.
+- The code-block directive should be followed by a blank line.
+
+You can run this script as follows:
+
+    python tools/check_code_block_format.py <file1> <file2> ...
+
+If you want to check all Python and C++ files under a directory, you can provide the directory path as an argument:
+
+    python tools/check_code_block_format.py <directory_path>
+
+For example:
+
+    python tools/check_code_block_format.py python paddle
+"""
+
+import argparse
+import io
+import re
+import sys
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+REGEX_CODE_BLOCK = re.compile(r"(?P<indent>\s*)..\s+code-block::\s*(\w+)?")
+REGEX_CODE_BLOCK_ATTRS = re.compile(r"(?P<indent>\s*):(\w+):\s*(.*)?")
+
+
+def with_color(text: str, color_code: str) -> str:
+    return f"\033[{color_code}m{text}\033[0m"
+
+
+def with_red(text: str) -> str:
+    return with_color(text, "31")
+
+
+def with_cyan(text: str) -> str:
+    return with_color(text, "36")
+
+
+def with_blue(text: str) -> str:
+    return with_color(text, "34")
+
+
+@dataclass
+class CodeBlock:
+    lines: list[str]
+    start_lineno: int
+
+
+@dataclass
+class CodeBlockModifySuggestion:
+    lineno: int
+    message: str
+
+
+class Diagnostic:
+    def __init__(self, start: int, message: str):
+        self.start = start
+        self.message = message
+
+
+def is_blank_line(line: str) -> bool:
+    return line.strip() == ""
+
+
+class CodeBlockIndentationDiagnostic(Diagnostic):
+    def __init__(
+        self,
+        start: int,
+        code_block: CodeBlock,
+        suggestions: list[CodeBlockModifySuggestion] = [],
+    ):
+        super().__init__(
+            start,
+            "The code under code-block directive should be indented properly. The issue found in:\n"
+            + format_code_block_with_suggestions(code_block, suggestions),
+        )
+
+
+class CodeBlockFollowsBlankLineDiagnostic(Diagnostic):
+    def __init__(
+        self,
+        start: int,
+        code_block: CodeBlock,
+        suggestions: list[CodeBlockModifySuggestion] = [],
+    ):
+        super().__init__(
+            start,
+            "The code-block directive should be followed by a blank line. The issue found in:\n"
+            + format_code_block_with_suggestions(code_block, suggestions),
+        )
+
+
+def cli() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description='Check if the code-block follows a blank line format.'
+    )
+    parser.add_argument(
+        'files', type=str, nargs='+', help='files to be checked'
+    )
+    args = parser.parse_args()
+    return args
+
+
+def expand_glob(files) -> list[Path]:
+    expanded = []
+    for file in files:
+        path = Path(file)
+        if path.is_dir():
+            expanded.extend(path.glob('**/*.py'))
+            expanded.extend(path.glob('**/*.cc'))
+            expanded.extend(path.glob('**/*.h'))
+        else:
+            expanded.append(path)
+    return expanded
+
+
+def format_code_block_with_suggestions(
+    code_block: CodeBlock, suggestions: list[CodeBlockModifySuggestion]
+) -> str:
+    formatted_lines = []
+    for line_offset, line in enumerate(code_block.lines):
+        lineno_str = with_cyan(f"{code_block.start_lineno + line_offset:>5}|")
+        formatted_lines.append(f"{lineno_str} {line}")
+        for suggestion in suggestions:
+            if suggestion.lineno == line_offset:
+                formatted_lines.append(
+                    f"     | {with_red('^')} {with_blue(suggestion.message)}\n"
+                )
+    return ''.join(formatted_lines)
+
+
+def extract_code_blocks(
+    file: io.TextIOWrapper,
+) -> Iterator[CodeBlock]:
+    lines_iter = iter(enumerate(file))
+
+    while True:
+        next_item = next(lines_iter, None)
+        if next_item is None:
+            break
+        lineno, line = next_item
+        match = REGEX_CODE_BLOCK.match(line)
+        if match:
+            indent = match.group('indent')
+            indent_size = len(indent)
+            code_block_start_lineno = lineno + 1
+            code_block_lines = [line]
+            indent_regex = re.compile(rf"\s{{{indent_size + 1}}}")
+            while True:
+                next_item = next(lines_iter, None)
+                if next_item is None:
+                    break
+                lineno, line = next_item
+                if not indent_regex.match(line) and not is_blank_line(line):
+                    break
+                code_block_lines.append(line)
+            yield CodeBlock(code_block_lines, code_block_start_lineno)
+
+
+def check_code_block_indentation(
+    code_block: CodeBlock,
+) -> list[Diagnostic]:
+    suggestion = CodeBlockModifySuggestion(
+        lineno=0, message="Next line should be indented."
+    )
+    diagnostic = CodeBlockIndentationDiagnostic(
+        code_block.start_lineno,
+        code_block,
+        [suggestion],
+    )
+    if len(code_block.lines) == 1:
+        suggestion.lineno = 1
+        return [diagnostic]
+    for line_offset, line in enumerate(code_block.lines[1:], 1):
+        if not is_blank_line(line) and not REGEX_CODE_BLOCK_ATTRS.match(line):
+            return []
+        suggestion.lineno = line_offset
+    return [diagnostic]
+
+
+def check_code_block_follows_blank_line(
+    code_block: CodeBlock,
+) -> list[Diagnostic]:
+    suggestion = CodeBlockModifySuggestion(
+        lineno=0, message="Add a blank line here."
+    )
+    diagnostic = CodeBlockFollowsBlankLineDiagnostic(
+        code_block.start_lineno,
+        code_block,
+        [suggestion],
+    )
+    if len(code_block.lines) < 2:
+        return []
+
+    def check_lines_starts_with_blank_line(
+        lines: list[str], start_lineno: int
+    ) -> bool:
+        if not lines:
+            return True
+        first_line = lines[0]
+        if REGEX_CODE_BLOCK_ATTRS.match(first_line):
+            return check_lines_starts_with_blank_line(
+                lines[1:], start_lineno + 1
+            )
+        suggestion.lineno = start_lineno
+        return is_blank_line(first_line)
+
+    if not check_lines_starts_with_blank_line(code_block.lines[1:], 1):
+        return [diagnostic]
+    return []
+
+
+def check_code_block_format(file: io.TextIOWrapper) -> list[Diagnostic]:
+    diagnostics: list[Diagnostic] = []
+    for code_block in extract_code_blocks(file):
+        diagnostics.extend(check_code_block_indentation(code_block))
+        diagnostics.extend(check_code_block_follows_blank_line(code_block))
+    return diagnostics
+
+
+def show_diagnostic(file: Path, diagnostic: Diagnostic) -> None:
+    print(
+        f"{with_blue(str(file))}:{with_cyan(str(diagnostic.start))}: "
+        f"{with_red('error:')} {diagnostic.message}"
+    )
+
+
+def main():
+    args = cli()
+    files = args.files
+    diagnostics: list[tuple[Path, list[Diagnostic]]] = []
+    for file in expand_glob(files):
+        with open(file, 'r') as f:
+            file_diagnostics = check_code_block_format(f)
+            for diagnostic in file_diagnostics:
+                diagnostics.append((file, file_diagnostics))
+    if diagnostics:
+        for file, file_diagnostics in diagnostics:
+            for diagnostic in file_diagnostics:
+                show_diagnostic(file, diagnostic)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tools/check_ctest_hung.py

@@ -0,0 +1,52 @@
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import sys
+
+
+def escape(input):
+    o = input.replace("\n", "")
+    o = o.replace("\r", "")
+    return o
+
+
+def main():
+    usage = """Usage:
+1. Download the Paddle_PR_CI_*.log from TeamCity
+2. run: python check_ctest_hung.py Paddle_PR_CI_*.log
+3. If there is hung ctest, the result likes:
+Diff:  set(['test_parallel_executor_crf'])
+    """
+    if len(sys.argv) < 2:
+        print(usage)
+        sys.exit(0)
+
+    logfile = sys.argv[1]
+    started = set()
+    passed = set()
+    with open(logfile, "r") as fn:
+        for l in fn:
+            if l.find("Test ") != -1 and l.find("Passed") != -1:
+                m = re.search(r"Test\s+#[0-9]*\:\s([a-z0-9_]+)", escape(l))
+                passed.add(m.group(1))
+            if l.find("Start ") != -1:
+                start_parts = escape(l).split(" ")
+                m = re.search(r"Start\s+[0-9]+\:\s([a-z0-9_]+)", escape(l))
+                started.add(m.group(1))
+    print("Diff: ", started - passed)
+
+
+if __name__ == "__main__":
+    main()

tools/check_gpu_kernel_approval.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+pr_whl_path=${PADDLE_ROOT}/build/pr_whl/
+dev_whl_path=${PADDLE_ROOT}/build/python/dist/
+approval_line=$(curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000)
+
+# Install dev wheel
+echo "::group::Generate dev_phi_kernels.json and pr_phi_kernels.json"
+pip install --force-reinstall "${dev_whl_path}"/*.whl
+python -c 'import paddle;import json;print(json.dumps(paddle.base.core._get_all_register_op_kernels("phi"), indent=4))' > dev_phi_kernels.json
+# Install pr wheel
+pip install --force-reinstall "${pr_whl_path}"/*.whl
+python -c 'import paddle;import json;print(json.dumps(paddle.base.core._get_all_register_op_kernels("phi"), indent=4))' > pr_phi_kernels.json
+echo "::endgroup::"
+
+echo -e "\e[33mBegin to compare GPU kernels between dev and pr...\e[0m"
+if ! python "${PADDLE_ROOT}/tools/gpu_kernel_compare.py" dev_phi_kernels.json pr_phi_kernels.json; then
+  APPROVALS=$(echo "${approval_line}"|python "${PADDLE_ROOT}"/tools/check_pr_approval.py 1 wanghuancoder)
+  if [[ "${APPROVALS}" == "FALSE" ]]; then
+    echo -e "\e[31m**************************************************************\e[0m"
+    echo -e "\e[31mPlease ensure the added GPU kernel supports big tensors, defined as those with a number of elements (numel) greater than 2^31 - 1. In the PR description, please describe how you tested this scenario and validated the data accuracy.\e[0m"
+    echo -e "\e[31mYou must have one RD (wanghuancoder) approval.\e[0m"
+    echo -e "\e[31m**************************************************************\e[0m"
+    exit 6
+  fi
+fi
+echo -e "\e[33mComparison completed.\e[0m"
+
+echo -e "\e[32mGPU kernel approval check passed.\e[0m"

tools/check_only_change_python_files.py

@@ -0,0 +1,74 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""For the PR that only modified the unit test, get cases in pull request."""
+
+import os
+import ssl
+import sys
+
+from github import Github
+
+PADDLE_ROOT = os.getenv('PADDLE_ROOT', '/paddle/')
+PADDLE_ROOT += '/'
+PADDLE_ROOT = PADDLE_ROOT.replace('//', '/')
+ssl._create_default_https_context = ssl._create_unverified_context
+
+
+class PRChecker:
+    """PR Checker."""
+
+    def __init__(self):
+        self.github = Github(os.getenv('GITHUB_API_TOKEN'), timeout=60)
+        self.repo = self.github.get_repo('PaddlePaddle/Paddle')
+        self.pr = None
+
+    def init(self):
+        """Get pull request."""
+        pr_id = os.getenv('GIT_PR_ID')
+        if not pr_id:
+            print('PREC No PR ID')
+            sys.exit(0)
+        self.pr = self.repo.get_pull(int(pr_id))
+
+    def get_pr_files(self):
+        """Get files in pull request."""
+        page = 0
+        file_dict = {}
+        while True:
+            files = self.pr.get_files().get_page(page)
+            if not files:
+                break
+            for f in files:
+                file_dict[PADDLE_ROOT + f.filename] = f.status
+            page += 1
+        print(f"pr modify files: {file_dict}")
+        return file_dict
+
+    def check_only_change_python_file(self):
+        file_dict = self.get_pr_files()
+        for filename in file_dict:
+            if not (
+                filename.startswith(PADDLE_ROOT + 'python/')
+                and filename.endswith('.py')
+            ):
+                return False
+        return True
+
+
+if __name__ == '__main__':
+    pr_checker = PRChecker()
+    pr_checker.init()
+    if pr_checker.check_only_change_python_file():
+        with open('only_change_python_file.txt', 'w') as f:
+            f.write('yes')

tools/check_op_benchmark_result.py

@@ -0,0 +1,216 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import logging
+import os
+import sys
+
+
+def check_path_exists(path):
+    """Assert whether file/directory exists."""
+    assert os.path.exists(path), f"{path} does not exist."
+
+
+def parse_case_name(log_file_name):
+    """Parse case name."""
+    case_id, case_info = log_file_name.split("-")
+    direction = case_info.split(".")[0].split("_")[-1]
+
+    return f"{case_id} ({direction})"
+
+
+def parse_log_file(log_file):
+    """Load one case result from log file."""
+    check_path_exists(log_file)
+
+    result = None
+    with open(log_file) as f:
+        for line in f.read().strip().split('\n')[::-1]:
+            try:
+                result = json.loads(line)
+                if result.get("disabled", False):
+                    return None
+                return result
+            except ValueError:
+                pass  # do nothing
+
+    if result is None:
+        logging.warning(f"Parse {log_file} fail!")
+
+    return result
+
+
+def load_benchmark_result_from_logs_dir(logs_dir):
+    """Load benchmark result from logs directory."""
+    check_path_exists(logs_dir)
+
+    log_file_path = lambda log_file: os.path.join(logs_dir, log_file)
+    result_lambda = lambda log_file: (
+        log_file,
+        parse_log_file(log_file_path(log_file)),
+    )
+
+    return dict(map(result_lambda, os.listdir(logs_dir)))
+
+
+def check_speed_result(case_name, develop_data, pr_data, pr_result):
+    """Check speed differences between develop and pr."""
+    pr_gpu_time = pr_data.get("gpu_time")
+    develop_gpu_time = develop_data.get("gpu_time")
+    if develop_gpu_time != 0.0:
+        gpu_time_diff = (pr_gpu_time - develop_gpu_time) / develop_gpu_time
+        gpu_time_diff_str = f"{gpu_time_diff * 100:.5f}"
+    else:
+        gpu_time_diff = 0
+        gpu_time_diff_str = ""
+
+    pr_total_time = pr_data.get("total")
+    develop_total_time = develop_data.get("total")
+    total_time_diff = (pr_total_time - develop_total_time) / develop_total_time
+
+    logging.info(f"------ OP: {case_name} ------")
+    logging.info(
+        f"GPU time change: {gpu_time_diff_str} (develop: {develop_gpu_time:.7f} -> PR: {pr_gpu_time:.7f})"
+    )
+    logging.info(
+        f"Total time change: {total_time_diff * 100:.5f}% (develop: {develop_total_time:.7f} -> PR: {pr_total_time:.7f})"
+    )
+    logging.info("backward: {}".format(pr_result.get("backward")))
+    logging.info("parameters:")
+    for line in pr_result.get("parameters").strip().split("\n"):
+        logging.info(f"\t{line}")
+
+    return gpu_time_diff > 0.05
+
+
+def check_accuracy_result(case_name, pr_result):
+    """Check accuracy result."""
+    logging.info(f"------ OP: {case_name} ------")
+    logging.info("Accuracy diff: {}".format(pr_result.get("diff")))
+    logging.info("backward: {}".format(pr_result.get("backward")))
+    logging.info("parameters:")
+    for line in pr_result.get("parameters").strip().split("\n"):
+        logging.info(f"\t{line}")
+
+    return not pr_result.get("consistent")
+
+
+def compare_benchmark_result(
+    case_name, develop_result, pr_result, check_results
+):
+    """Compare the differences between develop and pr."""
+    develop_speed = develop_result.get("speed")
+    pr_speed = pr_result.get("speed")
+
+    assert type(develop_speed) == type(pr_speed), (
+        "The types of comparison results need to be consistent."
+    )
+
+    if isinstance(develop_speed, dict) and isinstance(pr_speed, dict):
+        if check_speed_result(case_name, develop_speed, pr_speed, pr_result):
+            check_results["speed"].append(case_name)
+    else:
+        if check_accuracy_result(case_name, pr_result):
+            check_results["accuracy"].append(case_name)
+
+
+def update_api_info_file(fail_case_list, api_info_file):
+    """Update api info file to auto retry benchmark test."""
+    check_path_exists(api_info_file)
+
+    # set of case names for performance check failures
+    parse_case_id_f = lambda x: x.split()[0].rsplit('_', 1)
+    fail_case_dict = dict(map(parse_case_id_f, fail_case_list))
+
+    # list of api infos for performance check failures
+    api_info_list = []
+    with open(api_info_file) as f:
+        for line in f:
+            line_list = line.split(',')
+            case = line_list[0].split(':')[0]
+            if case in fail_case_dict:
+                line_list[0] = f"{case}:{fail_case_dict[case]}"
+                api_info_list.append(','.join(line_list))
+
+    # update api info file
+    with open(api_info_file, 'w') as f:
+        f.writelines(api_info_line for api_info_line in api_info_list)
+
+
+def summary_results(check_results, api_info_file):
+    """Summary results and return sys.exit code."""
+    for case_name in check_results["speed"]:
+        logging.error(f'Check speed result with case "{case_name}" failed.')
+
+    for case_name in check_results["accuracy"]:
+        logging.error(f'Check accuracy result with case "{case_name}" failed.')
+
+    if len(check_results["speed"]) and api_info_file:
+        update_api_info_file(check_results["speed"], api_info_file)
+
+    if len(check_results["speed"]) or len(check_results["accuracy"]):
+        return 8
+    else:
+        return 0
+
+
+if __name__ == "__main__":
+    """Load result from log directories and compare the differences."""
+    logging.basicConfig(
+        level=logging.INFO,
+        format="[%(filename)s:%(lineno)d] [%(levelname)s] %(message)s",
+    )
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--develop_logs_dir",
+        type=str,
+        required=True,
+        help="Specify the benchmark result directory of develop branch.",
+    )
+    parser.add_argument(
+        "--pr_logs_dir",
+        type=str,
+        required=True,
+        help="Specify the benchmark result directory of PR branch.",
+    )
+    parser.add_argument(
+        "--api_info_file",
+        type=str,
+        required=False,
+        help="Specify the api info to run benchmark test.",
+    )
+    args = parser.parse_args()
+
+    check_results = {"accuracy": [], "speed": []}
+
+    develop_result_dict = load_benchmark_result_from_logs_dir(
+        args.develop_logs_dir
+    )
+
+    check_path_exists(args.pr_logs_dir)
+    pr_log_files = os.listdir(args.pr_logs_dir)
+    for log_file in sorted(pr_log_files):
+        develop_result = develop_result_dict.get(log_file)
+        pr_result = parse_log_file(os.path.join(args.pr_logs_dir, log_file))
+        if develop_result is None or pr_result is None:
+            continue
+        case_name = parse_case_name(log_file)
+        compare_benchmark_result(
+            case_name, develop_result, pr_result, check_results
+        )
+
+    sys.exit(summary_results(check_results, args.api_info_file))

tools/check_op_desc.py

@@ -0,0 +1,476 @@
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import sys
+
+from paddle.base.core import OpUpdateType
+from paddle.utils import OpLastCheckpointChecker
+
+INPUTS = "Inputs"
+OUTPUTS = "Outputs"
+ATTRS = "Attrs"
+
+# The constant `ADD` means that an item has been added. In particular,
+# we use `ADD_WITH_DEFAULT` to mean adding attributes with default
+# attributes, and `ADD_DISPENSABLE` to mean adding optional inputs or
+# outputs.
+ADD_WITH_DEFAULT = "Add_with_default"
+ADD_DISPENSABLE = "Add_dispensable"
+ADD = "Add"
+
+DELETE = "Delete"
+CHANGE = "Change"
+
+DUPLICABLE = "duplicable"
+INTERMEDIATE = "intermediate"
+DISPENSABLE = "dispensable"
+
+TYPE = "type"
+GENERATED = "generated"
+DEFAULT_VALUE = "default_value"
+
+# add_with_extra, add_with_quant and add_with_def
+EXTRA = "extra"
+QUANT = "quant"
+DEF = "def"
+
+error = False
+
+version_update_map = {
+    INPUTS: {
+        ADD: OpUpdateType.kNewInput,
+    },
+    OUTPUTS: {
+        ADD: OpUpdateType.kNewOutput,
+    },
+    ATTRS: {
+        ADD: OpUpdateType.kNewAttr,
+        CHANGE: OpUpdateType.kModifyAttr,
+    },
+}
+
+
+def diff_vars(origin_vars, new_vars):
+    global error
+    var_error = False
+    var_changed_error_massage = {}
+    var_add_massage = []
+    var_add_dispensable_massage = []
+    var_deleted_error_massage = []
+
+    var_add_quant_message = []
+    var_add_def_message = []
+
+    common_vars_name = set(origin_vars.keys()) & set(new_vars.keys())
+    vars_name_only_in_origin = set(origin_vars.keys()) - set(new_vars.keys())
+    vars_name_only_in_new = set(new_vars.keys()) - set(origin_vars.keys())
+
+    for var_name in common_vars_name:
+        if origin_vars.get(var_name) == new_vars.get(var_name):
+            continue
+        else:
+            error, var_error = True, True
+            for arg_name in origin_vars.get(var_name):
+                new_arg_value = new_vars.get(var_name, {}).get(arg_name)
+                origin_arg_value = origin_vars.get(var_name, {}).get(arg_name)
+                if new_arg_value != origin_arg_value:
+                    if var_name not in var_changed_error_massage.keys():
+                        var_changed_error_massage[var_name] = {}
+                    var_changed_error_massage[var_name][arg_name] = (
+                        origin_arg_value,
+                        new_arg_value,
+                    )
+
+    for var_name in vars_name_only_in_origin:
+        error, var_error = True, True
+        var_deleted_error_massage.append(var_name)
+
+    for var_name in vars_name_only_in_new:
+        var_add_massage.append(var_name)
+        if not new_vars.get(var_name).get(DISPENSABLE):
+            error, var_error = True, True
+            var_add_dispensable_massage.append(var_name)
+
+        # if added var is extra, then no need to check.
+        if new_vars.get(var_name).get(EXTRA):
+            continue
+
+        # if added var is quant, slim needs to review, needs to register.
+        if new_vars.get(var_name).get(QUANT):
+            error, var_error = True, True
+            var_add_quant_message.append(var_name)
+
+        # if added var is def, inference needs to review, needs to register.
+        if not new_vars.get(var_name).get(EXTRA) and not new_vars.get(
+            var_name
+        ).get(QUANT):
+            error, var_error = True, True
+            var_add_def_message.append(var_name)
+
+    var_diff_message = {}
+    if var_add_massage:
+        var_diff_message[ADD] = var_add_massage
+    if var_add_dispensable_massage:
+        var_diff_message[ADD_DISPENSABLE] = var_add_dispensable_massage
+    if var_changed_error_massage:
+        var_diff_message[CHANGE] = var_changed_error_massage
+    if var_deleted_error_massage:
+        var_diff_message[DELETE] = var_deleted_error_massage
+    if var_add_quant_message:
+        var_diff_message[QUANT] = var_add_quant_message
+    if var_add_def_message:
+        var_diff_message[DEF] = var_add_def_message
+
+    return var_error, var_diff_message
+
+
+def diff_attr(ori_attrs, new_attrs):
+    global error
+    attr_error = False
+
+    attr_changed_error_massage = {}
+    attr_added_error_massage = []
+    attr_added_def_error_massage = []
+    attr_deleted_error_massage = []
+
+    attr_added_quant_message = []
+    attr_added_define_message = []
+
+    common_attrs = set(ori_attrs.keys()) & set(new_attrs.keys())
+    attrs_only_in_origin = set(ori_attrs.keys()) - set(new_attrs.keys())
+    attrs_only_in_new = set(new_attrs.keys()) - set(ori_attrs.keys())
+
+    for attr_name in common_attrs:
+        if ori_attrs.get(attr_name) == new_attrs.get(attr_name):
+            continue
+        else:
+            error, attr_error = True, True
+            for arg_name in ori_attrs.get(attr_name):
+                new_arg_value = new_attrs.get(attr_name, {}).get(arg_name)
+                origin_arg_value = ori_attrs.get(attr_name, {}).get(arg_name)
+                if new_arg_value != origin_arg_value:
+                    if attr_name not in attr_changed_error_massage.keys():
+                        attr_changed_error_massage[attr_name] = {}
+                    attr_changed_error_massage[attr_name][arg_name] = (
+                        origin_arg_value,
+                        new_arg_value,
+                    )
+
+    for attr_name in attrs_only_in_origin:
+        error, attr_error = True, True
+        attr_deleted_error_massage.append(attr_name)
+
+    for attr_name in attrs_only_in_new:
+        attr_added_error_massage.append(attr_name)
+        if new_attrs.get(attr_name).get(DEFAULT_VALUE) is None:
+            error, attr_error = True, True
+            attr_added_def_error_massage.append(attr_name)
+
+        # if added attr is quant, slim needs to review, needs to register
+        if new_attrs.get(attr_name).get(QUANT):
+            error, var_error = True, True
+            attr_added_quant_message.append(attr_name)
+
+        # if added attr is def, inference needs to review, needs to register
+        if not new_attrs.get(attr_name).get(EXTRA) and not new_attrs.get(
+            attr_name
+        ).get(QUANT):
+            error, var_error = True, True
+            attr_added_define_message.append(attr_name)
+
+    attr_diff_message = {}
+    if attr_added_error_massage:
+        attr_diff_message[ADD] = attr_added_error_massage
+    if attr_added_def_error_massage:
+        attr_diff_message[ADD_WITH_DEFAULT] = attr_added_def_error_massage
+    if attr_changed_error_massage:
+        attr_diff_message[CHANGE] = attr_changed_error_massage
+    if attr_deleted_error_massage:
+        attr_diff_message[DELETE] = attr_deleted_error_massage
+    if attr_added_define_message:
+        attr_diff_message[DEF] = attr_added_define_message
+    if attr_added_quant_message:
+        attr_diff_message[QUANT] = attr_added_quant_message
+
+    return attr_error, attr_diff_message
+
+
+def check_io_registry(io_type, op, diff):
+    checker = OpLastCheckpointChecker()
+    results = {}
+    for update_type in [ADD]:
+        for item in diff.get(update_type, []):
+            infos = checker.filter_updates(
+                op, version_update_map[io_type][update_type], item
+            )
+            if not infos:
+                if update_type not in results.keys():
+                    results[update_type] = []
+                # extra not need to register.
+                qaunt_ios = diff.get(QUANT, [])
+                def_ios = diff.get(DEF, [])
+                if item in qaunt_ios or item in def_ios:
+                    results[update_type].append((op, item, io_type))
+
+    return results
+
+
+def check_attr_registry(op, diff, origin_attrs):
+    checker = OpLastCheckpointChecker()
+    results = {}
+    qaunt_attrs = diff.get(QUANT, [])
+    def_attrs = diff.get(DEF, [])
+    change_attrs = diff.get(CHANGE, {})
+    for update_type in [ADD, CHANGE]:
+        for item in diff.get(update_type, {}):
+            infos = checker.filter_updates(
+                op, version_update_map[ATTRS][update_type], item
+            )
+            if not infos:
+                if update_type == ADD:
+                    if update_type not in results.keys():
+                        results[update_type] = []
+                    # extra not need to register.
+                    if item in qaunt_attrs or item in def_attrs:
+                        results[update_type].append((op, item))
+                elif update_type == CHANGE:
+                    if CHANGE not in results.keys():
+                        results[update_type] = {}
+                    for attr_name, attr_change in change_attrs.items():
+                        # extra not need to register.
+                        if not origin_attrs.get(attr_name).get(EXTRA):
+                            results[update_type][attr_name] = attr_change
+
+    for update_type in [ADD, CHANGE]:
+        if update_type in results.keys() and len(results[update_type]) == 0:
+            del results[update_type]
+    return results
+
+
+def compare_op_desc(origin_op_desc, new_op_desc):
+    origin = json.loads(origin_op_desc)
+    new = json.loads(new_op_desc)
+    desc_error_message = {}
+    version_error_message = {}
+    if origin_op_desc == new_op_desc:
+        return desc_error_message, version_error_message
+
+    for op_type in origin:
+        # no need to compare if the operator is deleted
+        if op_type not in new:
+            continue
+
+        origin_info = origin.get(op_type, {})
+        new_info = new.get(op_type, {})
+
+        origin_inputs = origin_info.get(INPUTS, {})
+        new_inputs = new_info.get(INPUTS, {})
+        ins_error, ins_diff = diff_vars(origin_inputs, new_inputs)
+        ins_version_errors = check_io_registry(INPUTS, op_type, ins_diff)
+
+        origin_outputs = origin_info.get(OUTPUTS, {})
+        new_outputs = new_info.get(OUTPUTS, {})
+        outs_error, outs_diff = diff_vars(origin_outputs, new_outputs)
+        outs_version_errors = check_io_registry(OUTPUTS, op_type, outs_diff)
+
+        origin_attrs = origin_info.get(ATTRS, {})
+        new_attrs = new_info.get(ATTRS, {})
+        attrs_error, attrs_diff = diff_attr(origin_attrs, new_attrs)
+        attrs_version_errors = check_attr_registry(
+            op_type, attrs_diff, origin_attrs
+        )
+
+        if ins_diff:
+            desc_error_message.setdefault(op_type, {})[INPUTS] = ins_diff
+        if outs_diff:
+            desc_error_message.setdefault(op_type, {})[OUTPUTS] = outs_diff
+        if attrs_diff:
+            desc_error_message.setdefault(op_type, {})[ATTRS] = attrs_diff
+
+        if ins_version_errors:
+            version_error_message.setdefault(op_type, {})[INPUTS] = (
+                ins_version_errors
+            )
+        if outs_version_errors:
+            version_error_message.setdefault(op_type, {})[OUTPUTS] = (
+                outs_version_errors
+            )
+        if attrs_version_errors:
+            version_error_message.setdefault(op_type, {})[ATTRS] = (
+                attrs_version_errors
+            )
+
+    return desc_error_message, version_error_message
+
+
+def print_desc_error_message(error_message):
+    print(
+        "\n======================= \n"
+        "Op desc error for the changes of Inputs/Outputs/Attrs of OPs:\n"
+    )
+    for op_name in error_message:
+        print(f"For OP '{op_name}':")
+
+        # 1. print inputs error message
+        Inputs_error = error_message.get(op_name, {}).get(INPUTS, {})
+        for name in Inputs_error.get(ADD_DISPENSABLE, {}):
+            print(f" * The added Input '{name}' is not dispensable.")
+
+        for name in Inputs_error.get(DELETE, {}):
+            print(f" * The Input '{name}' is deleted.")
+
+        for name in Inputs_error.get(CHANGE, {}):
+            changed_args = Inputs_error.get(CHANGE, {}).get(name, {})
+            for arg in changed_args:
+                ori_value, new_value = changed_args.get(arg)
+                print(
+                    f" * The arg '{arg}' of Input '{name}' is changed: from '{ori_value}' to '{new_value}'."
+                )
+
+        for name in Inputs_error.get(QUANT, {}):
+            print(
+                f" * The added Input '{name}' is `quant`, need slim to review."
+            )
+
+        for name in Inputs_error.get(DEF, {}):
+            print(
+                f" * The added Input '{name}' is `def`, need inference to review."
+            )
+
+        # 2. print outputs error message
+        Outputs_error = error_message.get(op_name, {}).get(OUTPUTS, {})
+        for name in Outputs_error.get(ADD_DISPENSABLE, {}):
+            print(f" * The added Output '{name}' is not dispensable.")
+
+        for name in Outputs_error.get(DELETE, {}):
+            print(f" * The Output '{name}' is deleted.")
+
+        for name in Outputs_error.get(CHANGE, {}):
+            changed_args = Outputs_error.get(CHANGE, {}).get(name, {})
+            for arg in changed_args:
+                ori_value, new_value = changed_args.get(arg)
+                print(
+                    f" * The arg '{arg}' of Output '{name}' is changed: from '{ori_value}' to '{new_value}'."
+                )
+
+        for name in Outputs_error.get(QUANT, {}):
+            print(
+                f" * The added Output '{name}' is `quant`, need slim to review."
+            )
+
+        for name in Outputs_error.get(DEF, {}):
+            print(
+                f" * The added Output '{name}' is `def`, need inference to review."
+            )
+
+        # 3. print attrs error message
+        attrs_error = error_message.get(op_name, {}).get(ATTRS, {})
+        for name in attrs_error.get(ADD_WITH_DEFAULT, {}):
+            print(f" * The added attr '{name}' doesn't set default value.")
+
+        for name in attrs_error.get(DELETE, {}):
+            print(f" * The attr '{name}' is deleted.")
+
+        for name in attrs_error.get(CHANGE, {}):
+            changed_args = attrs_error.get(CHANGE, {}).get(name, {})
+            for arg in changed_args:
+                ori_value, new_value = changed_args.get(arg)
+                print(
+                    f" * The arg '{arg}' of attr '{name}' is changed: from '{ori_value}' to '{new_value}'."
+                )
+
+        for name in attrs_error.get(QUANT, {}):
+            # TODO(Wilber):
+            print(
+                f" * The added attr '{name}' is `quant`, need slim to review."
+            )
+
+        for name in attrs_error.get(DEF, {}):
+            # TODO(Wilber):
+            print(
+                f" * The added attr '{name}' is `def`, need inference to review."
+            )
+
+
+def print_version_error_message(error_message):
+    print(
+        "\n======================= \n"
+        "Operator registration error for the changes of Inputs/Outputs/Attrs of OPs:\n"
+    )
+    for op_name in error_message:
+        print(f"For OP '{op_name}':")
+
+        # 1. print inputs error message
+        inputs_error = error_message.get(op_name, {}).get(INPUTS, {})
+        error_list = inputs_error.get(ADD, [])
+        if error_list:
+            for tup in error_list:
+                print(f" * The added input '{tup[1]}' is not yet registered.")
+
+        # 2. print outputs error message
+        outputs_error = error_message.get(op_name, {}).get(OUTPUTS, {})
+        error_list = outputs_error.get(ADD, [])
+        if error_list:
+            for tup in error_list:
+                print(f" * The added output '{tup[1]}' is not yet registered.")
+
+        # 3. print attrs error message
+        attrs_error = error_message.get(op_name, {}).get(ATTRS, {})
+        error_list = attrs_error.get(ADD, [])
+        if error_list:
+            for tup in error_list:
+                print(
+                    f" * The added attribute '{tup[1]}' is not yet registered."
+                )
+        error_dic = (
+            error_message.get(op_name, {}).get(ATTRS, {}).get(CHANGE, {})
+        )
+        for key, val in error_dic.items():
+            print(f" * The change of attribute '{key}' is not yet registered.")
+
+
+def print_repeat_process():
+    print(
+        "Tips:"
+        " If you want to repeat the process, please follow these steps:\n"
+        "\t1. Compile and install paddle from develop branch \n"
+        "\t2. Run: python tools/print_op_desc.py  > OP_DESC_DEV.spec \n"
+        "\t3. Compile and install paddle from PR branch \n"
+        "\t4. Run: python tools/print_op_desc.py  > OP_DESC_PR.spec \n"
+        "\t5. Run: python tools/check_op_desc.py OP_DESC_DEV.spec OP_DESC_PR.spec"
+    )
+
+
+if len(sys.argv) == 3:
+    '''
+    Compare op_desc files generated by branch DEV and branch PR.
+    And print error message.
+    '''
+    with open(sys.argv[1], 'r') as f:
+        origin_op_desc = f.read()
+
+    with open(sys.argv[2], 'r') as f:
+        new_op_desc = f.read()
+
+    desc_error_message, version_error_message = compare_op_desc(
+        origin_op_desc, new_op_desc
+    )
+    if error:
+        print("-" * 30)
+        print_desc_error_message(desc_error_message)
+        print_version_error_message(version_error_message)
+        print("-" * 30)
+else:
+    print("Usage: python check_op_desc.py OP_DESC_DEV.spec OP_DESC_PR.spec")

tools/check_op_kernel_same_dtypes.py

@@ -0,0 +1,149 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Print all registered kernels of a python module in alphabet order.
+
+Usage:
+    python check_op_kernel_same_dtypes.py > all_kernels.txt
+    python check_op_kernel_same_dtypes.py OP_KERNEL_DTYPE_DEV.spec OP_KERNEL_DTYPE_PR.spec > is_valid
+"""
+
+import collections
+import re
+import sys
+
+import paddle
+
+
+def get_all_kernels():
+    all_kernels_info = paddle.framework.core._get_all_register_op_kernels()
+    print(all_kernels_info)
+    # [u'{data_type[float]; data_layout[Undefined(AnyLayout)]; place[Place(gpu:0)]; library_type[PLAIN]}']
+    op_kernel_types = collections.defaultdict(list)
+    for op_type, op_infos in all_kernels_info.items():
+        # is_grad_op = op_type.endswith("_grad")
+        # if is_grad_op: continue
+
+        pattern = re.compile(r'data_type\[([^\]]+)\]')
+        for op_info in op_infos:
+            infos = pattern.findall(op_info)
+            if infos is None or len(infos) == 0:
+                continue
+
+            register_type = infos[0].split(":")[-1]
+            op_kernel_types[op_type].append(register_type.lower())
+
+    for op_type, op_kernels in sorted(
+        op_kernel_types.items(), key=lambda x: x[0]
+    ):
+        print(op_type, " ".join(sorted(op_kernels)))
+
+
+def read_file(file_path):
+    with open(file_path, 'r') as f:
+        content = f.read()
+        content = content.splitlines()
+    return content
+
+
+def print_diff(op_type, op_kernel_dtype_set, grad_op_kernel_dtype_set):
+    if len(op_kernel_dtype_set) > len(grad_op_kernel_dtype_set):
+        lack_dtypes = list(op_kernel_dtype_set - grad_op_kernel_dtype_set)
+        print(
+            "{} supports [{}] now, but its grad op kernel not supported.".format(
+                op_type, " ".join(lack_dtypes)
+            )
+        )
+    else:
+        lack_dtypes = list(grad_op_kernel_dtype_set - op_kernel_dtype_set)
+        print(
+            "{} supports [{}] now, but its forward op kernel not supported.".format(
+                op_type + "_grad", " ".join(lack_dtypes)
+            )
+        )
+
+
+def contain_current_op(op_type, op_info_dict):
+    if not op_type.endswith("_grad"):
+        return op_type + "_grad" in op_info_dict
+    else:
+        return op_type.rstrip("_grad") in op_info_dict
+
+
+def check_change_or_add_op_kernel_dtypes_valid():
+    origin = read_file(sys.argv[1])
+    new = read_file(sys.argv[2])
+
+    origin_all_kernel_dtype_dict = {}
+    for op_msg in origin:
+        op_info = op_msg.split()
+        origin_all_kernel_dtype_dict[op_info[0]] = set(op_info[1:])
+
+    new_all_kernel_dtype_dict = {}
+    for op_msg in new:
+        op_info = op_msg.split()
+        new_all_kernel_dtype_dict[op_info[0]] = set(op_info[1:])
+
+    added_or_changed_op_info = {}
+    for op_type, dtype_set in new_all_kernel_dtype_dict.items():
+        if op_type in origin_all_kernel_dtype_dict:
+            origin_dtype_set = origin_all_kernel_dtype_dict[op_type]
+            # op kernel changed
+            if origin_dtype_set != dtype_set and not contain_current_op(
+                op_type, added_or_changed_op_info
+            ):
+                added_or_changed_op_info[op_type] = dtype_set
+            else:
+                # do nothing
+                continue
+        else:
+            # op kernel added
+            if not contain_current_op(op_type, added_or_changed_op_info):
+                added_or_changed_op_info[op_type] = dtype_set
+            else:
+                # do nothing
+                continue
+
+    for op_type, dtype_set in added_or_changed_op_info.items():
+        # if changed forward op
+        if not op_type.endswith("_grad"):
+            # only support grad op
+            grad_op_type = op_type + "_grad"
+            if grad_op_type in new_all_kernel_dtype_dict:
+                grad_op_kernel_dtype_set = set(
+                    new_all_kernel_dtype_dict[grad_op_type]
+                )
+                if dtype_set != grad_op_kernel_dtype_set:
+                    print_diff(op_type, dtype_set, grad_op_kernel_dtype_set)
+        # if changed grad op
+        else:
+            forward_op_type = op_type.rstrip("_grad")
+            if forward_op_type in new_all_kernel_dtype_dict:
+                op_kernel_dtype_set = set(
+                    new_all_kernel_dtype_dict[forward_op_type]
+                )
+                if op_kernel_dtype_set != dtype_set:
+                    print_diff(forward_op_type, op_kernel_dtype_set, dtype_set)
+
+
+if len(sys.argv) == 1:
+    get_all_kernels()
+elif len(sys.argv) == 3:
+    check_change_or_add_op_kernel_dtypes_valid()
+else:
+    print(
+        "Usage:\n"
+        "\tpython check_op_kernel_same_dtypes.py > all_kernels.txt\n"
+        "\tpython check_op_kernel_same_dtypes.py OP_KERNEL_DTYPE_DEV.spec OP_KERNEL_DTYPE_PR.spec > diff"
+    )

tools/check_op_register_type.py

@@ -0,0 +1,107 @@
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Print all registered kernels of a python module in alphabet order.
+
+Usage:
+    python check_op_register_type.py > all_kernels.txt
+    python check_op_register_type.py OP_TYPE_DEV.spec OP_TYPE_PR.spec > is_valid
+"""
+
+import collections
+import difflib
+import re
+import sys
+
+from paddle import base
+
+INTS = {'int', 'int64_t'}
+FLOATS = {'float', 'double'}
+
+
+def get_all_kernels():
+    all_kernels_info = base.core._get_all_register_op_kernels()
+    # [u'data_type[double]:data_layout[ANY_LAYOUT]:place[CPUPlace]:library_type[PLAIN]'
+    op_kernel_types = collections.defaultdict(list)
+    for op_type, op_infos in all_kernels_info.items():
+        is_grad_op = op_type.endswith("_grad")
+        if is_grad_op:
+            continue
+
+        pattern = re.compile(r'data_type\[([^\]]+)\]')
+        for op_info in op_infos:
+            infos = pattern.findall(op_info)
+            if infos is None or len(infos) == 0:
+                continue
+
+            register_type = infos[0].split(":")[-1]
+            op_kernel_types[op_type].append(register_type.lower())
+
+    for op_type, op_kernels in sorted(
+        op_kernel_types.items(), key=lambda x: x[0]
+    ):
+        print(op_type, " ".join(sorted(op_kernels)))
+
+
+def read_file(file_path):
+    with open(file_path, 'r') as f:
+        content = f.read()
+        content = content.splitlines()
+    return content
+
+
+def print_diff(op_type, register_types):
+    lack_types = set()
+    if len(INTS - register_types) == 1:
+        lack_types |= INTS - register_types
+    if len(FLOATS - register_types) == 1:
+        lack_types |= FLOATS - register_types
+
+    print(
+        "{} only supports [{}] now, but lacks [{}].".format(
+            op_type, " ".join(register_types), " ".join(lack_types)
+        )
+    )
+
+
+def check_add_op_valid():
+    origin = read_file(sys.argv[1])
+    new = read_file(sys.argv[2])
+
+    differ = difflib.Differ()
+    result = differ.compare(origin, new)
+
+    for each_diff in result:
+        if each_diff[0] in ['+'] and len(each_diff) > 2:  # if change or add op
+            op_info = each_diff[1:].split()
+            if len(op_info) < 2:
+                continue
+            register_types = set(op_info[1:])
+            if (
+                len(FLOATS - register_types) == 1
+                or len(INTS - register_types) == 1
+            ):
+                print_diff(op_info[0], register_types)
+
+
+if len(sys.argv) == 1:
+    get_all_kernels()
+elif len(sys.argv) == 3:
+    check_add_op_valid()
+else:
+    print(
+        "Usage:\n"
+        "\tpython check_op_register_type.py > all_kernels.txt\n"
+        "\tpython check_op_register_type.py OP_TYPE_DEV.spec OP_TYPE_PR.spec > diff"
+    )

tools/check_pr_approval.py

@@ -0,0 +1,58 @@
+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import sys
+
+
+def check_approval(count, required_reviewers):
+    json_buff = ""
+    for line in sys.stdin:
+        json_buff = "".join([json_buff, line])
+    json_resp = json.loads(json_buff)
+    approves = 0
+    approved_user_ids = []
+    approved_user_logins = set()
+    for review in json_resp:
+        if review["state"] == "APPROVED":
+            approves += 1
+            approved_user_ids.append(review["user"]["id"])
+            approved_user_logins.add(review["user"]["login"])
+
+    # convert to int
+    required_reviewers_int = set()
+    required_reviewers_login = set()
+    for rr in required_reviewers:
+        if rr.isdigit():
+            required_reviewers_int.add(int(rr))
+        else:
+            required_reviewers_login.add(rr)
+
+    if (
+        len(set(approved_user_ids) & required_reviewers_int)
+        + len(approved_user_logins & required_reviewers_login)
+        >= count
+    ):
+        print("TRUE")
+    else:
+        print("FALSE")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1 and sys.argv[1].isdigit():
+        check_approval(int(sys.argv[1]), sys.argv[2:])
+    else:
+        print(
+            "Usage: python check_pr_approval.py [count] [required reviewer id] ..."
+        )

tools/check_sequence_op.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../" && pwd )"
+
+function check_sequence_op_unittests(){
+    check_white_list_file=$1
+    function_grep=$2
+    INVALID_SEQUENCE_OP_UNITTEST=""
+    all_sequence_ops=`grep 'OP(sequence_' ${PADDLE_ROOT}/build/paddle/fluid/pybind/pybind.h | grep -Ev '^$' | cut -d'(' -f 2 | cut -d')' -f 1`
+    for op_name in ${all_sequence_ops}; do
+        in_white_list=`python ${PADDLE_ROOT}/${check_white_list_file} ${op_name}`
+        if [ "${in_white_list}" == "True" ]; then
+            continue
+        fi
+        unittest_file="test/sequence/test_${op_name}.py"
+        if [ ! -f "${PADDLE_ROOT}/${unittest_file}" ]; then
+            INVALID_SEQUENCE_OP_UNITTEST="${INVALID_SEQUENCE_OP_UNITTEST}${unittest_file} (unittest file does not exists)\n"
+            continue
+        fi
+        batch_size_1_function_calls=`grep ${function_grep} ${PADDLE_ROOT}/${unittest_file} || true`
+        if [ "${batch_size_1_function_calls}" == "" ]; then
+            INVALID_SEQUENCE_OP_UNITTEST="${INVALID_SEQUENCE_OP_UNITTEST}${unittest_file} (missing required function call)\n"
+        fi
+    done
+    echo ${INVALID_SEQUENCE_OP_UNITTEST}
+}
+
+check_white_list_file="test/white_list/check_op_sequence_batch_1_input_white_list.py"
+function_grep="self.get_sequence_batch_size_1_input("
+INVALID_SEQUENCE_OP_UNITTEST=$(check_sequence_op_unittests ${check_white_list_file} ${function_grep})
+if [ "${INVALID_SEQUENCE_OP_UNITTEST}" != "" ]; then
+    echo "************************************"
+    echo -e "It is required to include batch size 1 LoDTensor input in sequence OP test, please use self.get_sequence_batch_size_1_input() method."
+    echo -e "For more information, please refer to [https://github.com/PaddlePaddle/Paddle/wiki/It-is-required-to-include-LoDTensor-input-with-batch_size=1-in-sequence-OP-test]."
+    echo -e "Please check the following unittest files:\n${INVALID_SEQUENCE_OP_UNITTEST}"
+    echo "************************************"
+    exit 1
+fi
+
+check_white_list_file="test/white_list/check_op_sequence_instance_0_input_white_list.py"
+function_grep="self.get_sequence_instance_size_0_input("
+INVALID_SEQUENCE_OP_UNITTEST=$(check_sequence_op_unittests ${check_white_list_file} ${function_grep})
+if [ "${INVALID_SEQUENCE_OP_UNITTEST}" != "" ]; then
+    echo "************************************"
+    echo -e "It is required to include instance size 0 LoDTensor input in sequence OP test, please use self.get_sequence_instance_size_0_input() method."
+    echo -e "For more information, please refer to [https://github.com/PaddlePaddle/Paddle/wiki/It-is-required-to-include-LoDTensor-input-with-instance_size=0-in-sequence-OP-test]. "
+    echo -e "Please check the following unittest files:\n${INVALID_SEQUENCE_OP_UNITTEST}"
+    echo "************************************"
+    exit 1
+fi

tools/codestyle/.cmakelintrc

@@ -0,0 +1 @@
+filter=-readability/wonkycase,-syntax,-convention/filename,-package/stdargs,-whitespace/indent,-whitespace/extra,-linelength,-readability/mixedcase

tools/codestyle/clang-tidy.py

@@ -0,0 +1,506 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# ===- run-clang-tidy.py - Parallel clang-tidy runner ---------*- python -*--===#
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+#
+# ===------------------------------------------------------------------------===#
+# FIXME: Integrate with clang-tidy-diff.py
+
+"""
+Parallel clang-tidy runner
+==========================
+
+Runs clang-tidy over all files in a compilation database. Requires clang-tidy
+and clang-apply-replacements in $PATH.
+
+Example invocations.
+- Run clang-tidy on all files in the current working directory with a default
+  set of checks and show warnings in the cpp files and all project headers.
+    run-clang-tidy.py $PWD
+
+- Fix all header guards.
+    run-clang-tidy.py -fix -checks=-*,llvm-header-guard
+
+- Fix all header guards included from clang-tidy and header guards
+  for clang-tidy headers.
+    run-clang-tidy.py -fix -checks=-*,llvm-header-guard extra/clang-tidy \
+                      -header-filter=extra/clang-tidy
+
+Compilation database setup:
+http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
+"""
+
+import argparse
+import glob
+import json
+import multiprocessing
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import traceback
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+
+is_py2 = sys.version[0] == '2'
+
+if is_py2:
+    import Queue as queue
+else:
+    import queue
+
+
+def find_compilation_database(path, result="./"):
+    """Adjusts the directory until a compilation database is found."""
+    result = './'
+    while not os.path.isfile(os.path.join(result, path)):
+        if os.path.realpath(result) == '/':
+            print('Warning: could not find compilation database.')
+            return None
+        result += '../'
+    return os.path.realpath(result)
+
+
+def make_absolute(f, directory):
+    """Convert a relative file path to an absolute file path."""
+    if os.path.isabs(f):
+        return f
+    return os.path.normpath(os.path.join(directory, f))
+
+
+def analysis_gitignore(path, filename=".gitignore"):
+    """Analysis gitignore file and return ignore file list"""
+    with open(path + "/" + filename, "r") as f:
+        lines = f.readlines()
+        ignore_file_list = []
+        for line in lines:
+            # Blank row
+            if line == "\n" or line == "\r\n":
+                continue
+
+            # explanatory note
+            line = line.replace("\n", "").strip()
+            if "#" in line:
+                if not line.startswith("#"):
+                    ignore_file_list.append(
+                        line[: line.index("#")].replace(" ", "")
+                    )
+                continue
+
+            # TODO(gouzil): support more gitignore rules
+            if "*" in line:
+                continue
+
+            ignore_file_list.append(line.replace(" ", ""))
+
+    return ignore_file_list
+
+
+def skip_check_file(database, build_path):
+    """Skip checking some files"""
+    skip_file_list = []
+    skip_file_list.append(".cu")
+    skip_file_list.append(os.path.join(os.getcwd(), build_path))
+    skip_file_list += analysis_gitignore(os.getcwd())
+    res_list = []
+    for entry in database:
+        write_in = True
+        for ignore_file in skip_file_list:
+            if ignore_file in entry["file"]:
+                write_in = False
+                break
+        if write_in:
+            res_list.append(entry)
+
+    return res_list
+
+
+def get_tidy_invocation(
+    f,
+    clang_tidy_binary,
+    checks,
+    tmpdir,
+    build_path,
+    header_filter,
+    extra_arg,
+    extra_arg_before,
+    quiet,
+    config,
+):
+    """Gets a command line for clang-tidy."""
+    start = [clang_tidy_binary]
+    if header_filter is not None:
+        start.append('-header-filter=' + header_filter)
+    if checks:
+        start.append('-checks=' + checks)
+    if tmpdir is not None:
+        start.append('-export-fixes')
+        # Get a temporary file. We immediately close the handle so clang-tidy can
+        # overwrite it.
+        (handle, name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir)
+        os.close(handle)
+        start.append(name)
+    for arg in extra_arg:
+        start.append(f'-extra-arg={arg}')
+    for arg in extra_arg_before:
+        start.append(f'-extra-arg-before={arg}')
+    start.append('-p=' + build_path)
+    if quiet:
+        start.append('-quiet')
+    if config:
+        start.append('-config=' + config)
+    start.append(f)
+    return start
+
+
+def merge_replacement_files(tmpdir, mergefile):
+    """Merge all replacement files in a directory into a single file"""
+    # The fixes suggested by clang-tidy >= 4.0.0 are given under
+    # the top level key 'Diagnostics' in the output yaml files
+    mergekey = "Diagnostics"
+    merged = []
+    for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')):
+        content = yaml.safe_load(open(replacefile, 'r'))
+        if not content:
+            continue  # Skip empty files.
+        merged.extend(content.get(mergekey, []))
+
+    if merged:
+        # MainSourceFile: The key is required by the definition inside
+        # include/clang/Tooling/ReplacementsYaml.h, but the value
+        # is actually never used inside clang-apply-replacements,
+        # so we set it to '' here.
+        output = {'MainSourceFile': '', mergekey: merged}
+        with open(mergefile, 'w') as out:
+            yaml.safe_dump(output, out)
+    else:
+        # Empty the file:
+        open(mergefile, 'w').close()
+
+
+def check_clang_apply_replacements_binary(args):
+    """Checks if invoking supplied clang-apply-replacements binary works."""
+    try:
+        subprocess.check_call(
+            [args.clang_apply_replacements_binary, '--version']
+        )
+    except:
+        print(
+            'Unable to run clang-apply-replacements. Is clang-apply-replacements '
+            'binary correctly specified?',
+            file=sys.stderr,
+        )
+        traceback.print_exc()
+        sys.exit(1)
+
+
+def apply_fixes(args, tmpdir):
+    """Calls clang-apply-fixes on a given directory."""
+    invocation = [args.clang_apply_replacements_binary]
+    if args.format:
+        invocation.append('-format')
+    if args.style:
+        invocation.append('-style=' + args.style)
+    invocation.append(tmpdir)
+    subprocess.call(invocation)
+
+
+def run_tidy(args, tmpdir, build_path, queue, lock, failed_files):
+    """Takes filenames out of queue and runs clang-tidy on them."""
+    while True:
+        name = queue.get()
+        invocation = get_tidy_invocation(
+            name,
+            args.clang_tidy_binary,
+            args.checks,
+            tmpdir,
+            build_path,
+            args.header_filter,
+            args.extra_arg,
+            args.extra_arg_before,
+            args.quiet,
+            args.config,
+        )
+
+        proc = subprocess.Popen(
+            invocation, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
+        output, err = proc.communicate()
+        if proc.returncode != 0:
+            failed_files.append(name)
+        with lock:
+            sys.stdout.write(
+                ' '.join(invocation) + '\n' + output.decode('utf-8')
+            )
+            if len(err) > 0:
+                sys.stdout.flush()
+                sys.stderr.write(err.decode('utf-8'))
+        queue.task_done()
+
+
+def main():
+    """Runs clang-tidy over all files in a compilation database."""
+    parser = argparse.ArgumentParser(
+        description='Runs clang-tidy over all files '
+        'in a compilation database. Requires '
+        'clang-tidy and clang-apply-replacements in '
+        '$PATH.'
+    )
+    parser.add_argument(
+        '-clang-tidy-binary',
+        metavar='PATH',
+        default='clang-tidy-15',
+        help='path to clang-tidy binary',
+    )
+    parser.add_argument(
+        '-clang-apply-replacements-binary',
+        metavar='PATH',
+        default='clang-apply-replacements-15',
+        help='path to clang-apply-replacements binary',
+    )
+    parser.add_argument(
+        '-checks',
+        default=None,
+        help='checks filter, when not specified, use clang-tidy default',
+    )
+    parser.add_argument(
+        '-config',
+        default=None,
+        help='Specifies a configuration in YAML/JSON format: '
+        '  -config="{Checks: \'*\', '
+        '                       CheckOptions: [{key: x, '
+        '                                       value: y}]}" '
+        'When the value is empty, clang-tidy will '
+        'attempt to find a file named .clang-tidy for '
+        'each source file in its parent directories.',
+    )
+    parser.add_argument(
+        '-header-filter',
+        default=None,
+        help='regular expression matching the names of the '
+        'headers to output diagnostics from. Diagnostics from '
+        'the main file of each translation unit are always '
+        'displayed.',
+    )
+    if yaml:
+        parser.add_argument(
+            '-export-fixes',
+            metavar='filename',
+            dest='export_fixes',
+            help='Create a yaml file to store suggested fixes in, '
+            'which can be applied with clang-apply-replacements.',
+        )
+    parser.add_argument(
+        '-j',
+        type=int,
+        default=0,
+        help='number of tidy instances to be run in parallel.',
+    )
+    parser.add_argument(
+        'files',
+        nargs='*',
+        default=['.*'],
+        help='files to be processed (regex on path)',
+    )
+    parser.add_argument('-fix', action='store_true', help='apply fix-its')
+    parser.add_argument(
+        '-format',
+        action='store_true',
+        help='Reformat code after applying fixes',
+    )
+    parser.add_argument(
+        '-style',
+        default='file',
+        help='The style of reformat code after applying fixes',
+    )
+    parser.add_argument(
+        '-p',
+        dest='build_path',
+        help='Path used to read a compile command database.',
+    )
+    parser.add_argument(
+        '-extra-arg',
+        dest='extra_arg',
+        action='append',
+        default=[],
+        help='Additional argument to append to the compiler command line.',
+    )
+    parser.add_argument(
+        '-extra-arg-before',
+        dest='extra_arg_before',
+        action='append',
+        default=[],
+        help='Additional argument to prepend to the compiler command line.',
+    )
+    parser.add_argument(
+        '-quiet', action='store_true', help='Run clang-tidy in quiet mode'
+    )
+    args = parser.parse_args()
+
+    db_path = 'compile_commands.json'
+
+    if args.build_path is not None:
+        build_path = args.build_path
+        if not os.path.isfile(os.path.join(build_path, db_path)):
+            print(
+                f'Warning: could not find compilation database in {build_path}, skip clang-tidy check.'
+            )
+            build_path = None
+    else:
+        # Find our database
+        build_path = find_compilation_database(db_path)
+    if build_path is None:
+        sys.exit(0)
+
+    try:
+        invocation = [args.clang_tidy_binary, '-list-checks']
+        invocation.append('-p=' + build_path)
+        if args.checks:
+            invocation.append('-checks=' + args.checks)
+        invocation.append('-')
+        if args.quiet:
+            # Even with -quiet we still want to check if we can call clang-tidy.
+            with open(os.devnull, 'w') as dev_null:
+                subprocess.check_call(invocation, stdout=dev_null)
+        else:
+            subprocess.check_call(invocation)
+    except:
+        print("Unable to run clang-tidy.", file=sys.stderr)
+        sys.exit(0)
+
+    # Load the database and extract all files.
+    database = json.load(open(os.path.join(build_path, db_path)))
+    database = skip_check_file(database, build_path)
+    files = {
+        make_absolute(entry['file'], entry['directory']) for entry in database
+    }
+
+    max_task = args.j
+    if max_task == 0:
+        max_task = multiprocessing.cpu_count()
+
+    tmpdir = None
+    if args.fix or (yaml and args.export_fixes):
+        check_clang_apply_replacements_binary(args)
+        tmpdir = tempfile.mkdtemp()
+
+    # Build up a big regex filter from all command line arguments.
+    file_name_re = re.compile('|'.join(args.files))
+
+    return_code = 0
+    try:
+        # Spin up a bunch of tidy-launching threads.
+        task_queue = queue.Queue(max_task)
+        # List of files with a non-zero return code.
+        failed_files = []
+        lock = threading.Lock()
+        for _ in range(max_task):
+            t = threading.Thread(
+                target=run_tidy,
+                args=(args, tmpdir, build_path, task_queue, lock, failed_files),
+            )
+            t.daemon = True
+            t.start()
+
+        # Fill the queue with files.
+        for name in files:
+            if file_name_re.search(name):
+                task_queue.put(name)
+
+        # Wait for all threads to be done.
+        task_queue.join()
+        if len(failed_files):
+            return_code = 1
+
+    except KeyboardInterrupt:
+        # This is a sad hack. Unfortunately subprocess goes
+        # bonkers with ctrl-c and we start forking merrily.
+        print('\nCtrl-C detected, goodbye.')
+        if tmpdir:
+            shutil.rmtree(tmpdir)
+        os.kill(0, 9)
+
+    if yaml and args.export_fixes:
+        print('Writing fixes to ' + args.export_fixes + ' ...')
+        try:
+            merge_replacement_files(tmpdir, args.export_fixes)
+        except:
+            print('Error exporting fixes.\n', file=sys.stderr)
+            traceback.print_exc()
+            return_code = 1
+
+    if args.fix:
+        print('Applying fixes ...')
+        try:
+            apply_fixes(args, tmpdir)
+        except:
+            print('Error applying fixes.\n', file=sys.stderr)
+            traceback.print_exc()
+            return_code = 1
+
+    if tmpdir:
+        shutil.rmtree(tmpdir)
+    sys.exit(return_code)
+
+
+if __name__ == '__main__':
+    if os.getenv("SKIP_CLANG_TIDY_CHECK", "").lower() in [
+        'y',
+        'yes',
+        't',
+        'true',
+        'on',
+        '1',
+    ]:
+        print(
+            "SKIP_CLANG_TIDY_CHECK is set, skip clang-tidy check.",
+            file=sys.stderr,
+        )
+        sys.exit(0)
+
+    target_version = "15.0"
+    try:
+        out = subprocess.check_output(['clang-tidy', '--version'], shell=True)
+        version = out.decode('utf-8')
+        if version.find(target_version) == -1:
+            print(
+                f"clang-tidy version == {target_version} not found, attempting auto-install...",
+                file=sys.stderr,
+            )
+            subprocess.check_output(
+                'pip install --no-cache clang-tidy=="15.0.2.1"',
+                shell=True,
+            )
+    except:
+        print(
+            "clang-tidy not found, attempting auto-install...", file=sys.stderr
+        )
+        subprocess.check_output(
+            'pip install --no-cache clang-tidy=="15.0.2.1"',
+            shell=True,
+        )
+    main()

tools/codestyle/copyright.py

@@ -0,0 +1,137 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import datetime
+import os
+import re
+import sys
+
+COPYRIGHT = '''Copyright (c) {year} PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.'''
+
+
+def _generate_copyright(comment_mark):
+    year = datetime.datetime.now().year
+    copyright = COPYRIGHT.format(year=year)
+
+    return [
+        (
+            f"{comment_mark} {line}{os.linesep}"
+            if line
+            else f"{comment_mark}{os.linesep}"
+        )
+        for line in copyright.splitlines()
+    ]
+
+
+def _get_comment_mark(path):
+    lang_type = re.compile(r"\.(py|pyi|sh)$")
+    if lang_type.search(path) is not None:
+        return "#"
+
+    lang_type = re.compile(r"\.(h|c|hpp|cc|cpp|cu|go|cuh|proto)$")
+    if lang_type.search(path) is not None:
+        return "//"
+
+    return None
+
+
+RE_ENCODE = re.compile(r"^[ \t\v]*#.*?coding[:=]", re.IGNORECASE)
+RE_COPYRIGHT = re.compile(r".*Copyright \(c\) \d{4}", re.IGNORECASE)
+RE_SHEBANG = re.compile(r"^[ \t\v]*#[ \t]?\!")
+
+
+def _check_copyright(path):
+    head = []
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            head = [next(f) for x in range(4)]
+    except StopIteration:
+        pass
+
+    for idx, line in enumerate(head):
+        if RE_COPYRIGHT.search(line) is not None:
+            return True
+
+    return False
+
+
+def generate_copyright(path, comment_mark):
+    original_contents = open(path, 'r', encoding="utf-8").readlines()
+    head = original_contents[0:4]
+
+    insert_line_no = 0
+    for i, line in enumerate(head):
+        if RE_ENCODE.search(line) or RE_SHEBANG.search(line):
+            insert_line_no = i + 1
+
+    copyright = _generate_copyright(comment_mark)
+    if insert_line_no == 0:
+        new_contents = copyright
+        if (
+            len(original_contents) > 0
+            and len(original_contents[0].strip()) != 0
+        ):
+            new_contents.append(os.linesep)
+        new_contents.extend(original_contents)
+    else:
+        new_contents = original_contents[0:insert_line_no]
+        new_contents.append(os.linesep)
+        new_contents.extend(copyright)
+        if (
+            len(original_contents) > insert_line_no
+            and len(original_contents[insert_line_no].strip()) != 0
+        ):
+            new_contents.append(os.linesep)
+        new_contents.extend(original_contents[insert_line_no:])
+    new_contents = "".join(new_contents)
+
+    with open(path, 'w', encoding='utf-8') as output_file:
+        output_file.write(new_contents)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(
+        description='Checker for copyright declaration.'
+    )
+    parser.add_argument('filenames', nargs='*', help='Filenames to check')
+    args = parser.parse_args(argv)
+
+    retv = 0
+    for path in args.filenames:
+        comment_mark = _get_comment_mark(path)
+        if comment_mark is None:
+            print("warning:Unsupported file", path, file=sys.stderr)
+            continue
+
+        if _check_copyright(path):
+            continue
+
+        generate_copyright(path, comment_mark)
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tools/codestyle/pre_commit.sh

@@ -0,0 +1,68 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set +x
+
+# use pre-commit 2.17
+if ! [[ $(pre-commit --version) == *"2.17.0"* ]]; then
+    pip install pre-commit==2.17.0 1>nul
+fi
+
+# Install clang-format before git commit to avoid repeat installation due to
+# pre-commit multi-thread running.
+readonly VERSION="13.0.0"
+version=$(clang-format -version)
+if ! [[ $(python -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $1$2}') -ge 36 ]]; then
+    echo "clang-format installation by pip need python version great equal 3.6,
+          please change the default python to higher version."
+    exit 1
+fi
+
+diff_files=$(git diff --name-only --diff-filter=ACMR ${BRANCH})
+num_diff_files=$(echo "$diff_files" | wc -l)
+echo -e "diff files between pr and ${BRANCH}:\n${diff_files}"
+
+echo "Checking code style by pre-commit ..."
+pre-commit run --files ${diff_files};check_error=$?
+
+if test ! -z "$(git diff)"; then
+    echo -e '\n************************************************************************************'
+    echo -e "These files have been formatted by code format hook. You should use pre-commit to \
+format them before git push."
+    echo -e '************************************************************************************\n'
+    git diff 2>&1
+fi
+
+echo -e '\n************************************************************************************'
+if [ ${check_error} != 0 ];then
+    echo "Your PR code style check failed."
+    echo "Please install pre-commit locally and set up git hook scripts:"
+    echo ""
+    echo "    pip install pre-commit==2.17.0"
+    echo "    pre-commit install"
+    echo ""
+    if [[ $num_diff_files -le 100 ]];then
+        echo "Then, run pre-commit to check codestyle issues in your PR:"
+        echo ""
+        echo "    pre-commit run --files" $(echo ${diff_files} | tr "\n" " ")
+        echo ""
+    fi
+    echo "For more information, please refer to our codestyle check guide:"
+    echo "https://www.paddlepaddle.org.cn/documentation/docs/zh/develop/dev_guides/git_guides/codestyle_check_guide_cn.html"
+else
+    echo "Your PR code style check passed."
+fi
+echo -e '************************************************************************************\n'
+
+exit ${check_error}

tools/codestyle/sort_txt_file.py

@@ -0,0 +1,31 @@
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+
+def sort_by_dict_order(file_path):
+    with open(file_path, 'r') as f:
+        lines = f.readlines()
+    sorted_lines = sorted(lines)
+    with open(file_path, 'w') as f:
+        f.writelines(sorted_lines)
+
+
+if __name__ == '__main__':
+    file_paths = sys.argv[1:]
+    for file_path in file_paths:
+        file_path = os.path.normpath(file_path)
+        sort_by_dict_order(file_path)

tools/commitlist.py

@@ -0,0 +1,762 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import asdict, dataclass, fields
+from pathlib import Path
+from typing import TYPE_CHECKING
+from urllib import error, request
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+DEFAULT_OWNER = "PaddlePaddle"
+DEFAULT_REPO = "Paddle"
+HTML_COMMENT_RE = re.compile(r"<!--.*?-->", re.DOTALL)
+PR_NUMBER_RE = re.compile(r"\(#(\d+)\)")
+HEADING_RE = re.compile(r"(?m)^###\s*(.+?)\s*$")
+RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
+
+PR_CATEGORIES = [
+    'User Experience',
+    'Execute Infrastructure',
+    'Operator Mechanism',
+    'CINN',
+    'Custom Device',
+    'Performance Optimization',
+    'Distributed Strategy',
+    'Parameter Server',
+    'Communication Library',
+    'Auto Parallel',
+    'Inference',
+    'Environment Adaptation',
+    'Others',
+]
+
+PR_TYPES = [
+    'New features',
+    'Bug fixes',
+    'Improvements',
+    'Performance',
+    'BC Breaking',
+    'Deprecations',
+    'Docs',
+    'Devs',
+    'Not User Facing',
+    'Security',
+    'Others',
+]
+
+
+@dataclass(frozen=True)
+class CommitRecord:
+    commit_hash: str
+    title: str
+    git_author: str
+    pr_number: int | None
+
+
+@dataclass(frozen=True)
+class PullRequestRecord:
+    number: int
+    title: str
+    author: str
+    labels: list[str]
+    reviewers: list[str]
+    category: str
+    topic: str
+    description: str
+
+
+@dataclass(frozen=True)
+class CommitRow:
+    commit_hash: str
+    category: str
+    topic: str
+    title: str
+    pr_link: str
+    author: str
+    labels: str
+    accepter_1: str
+    accepter_2: str
+    accepter_3: str
+    description: str
+
+
+COMMIT_FIELDS = tuple(field.name for field in fields(CommitRow))
+
+
+def run_git(*args: str) -> str:
+    completed = subprocess.run(
+        ["git", *args],
+        check=True,
+        text=True,
+        capture_output=True,
+    )
+    return completed.stdout.strip()
+
+
+def normalize_whitespace(text: str) -> str:
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def strip_html_comments(text: str) -> str:
+    return HTML_COMMENT_RE.sub("", text or "")
+
+
+def cleanup_markdown_text(text: str) -> str:
+    cleaned_lines = []
+    for line in strip_html_comments(text).replace("\r\n", "\n").splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        line = re.sub(r"^[-*+]\s*", "", line)
+        line = re.sub(r"^\d+\.\s*", "", line)
+        cleaned_lines.append(line)
+    return normalize_whitespace(" ".join(cleaned_lines))
+
+
+def cleanup_title(title: str) -> str:
+    return normalize_whitespace(PR_NUMBER_RE.sub("", title or ""))
+
+
+def extract_pr_number(title: str) -> int | None:
+    matches = PR_NUMBER_RE.findall(title or "")
+    return int(matches[-1]) if matches else None
+
+
+def split_known_choice(raw_text: str, choices: list[str], default: str) -> str:
+    parts = [
+        cleanup_markdown_text(part)
+        for part in re.split(r'[,|\n]+', raw_text or '')
+    ]
+    parts = [part for part in parts if part]
+    if not parts:
+        return default
+    for part in parts:
+        for choice in choices:
+            if part.casefold() == choice.casefold():
+                return choice
+    for part in parts:
+        lowered = part.casefold()
+        for choice in choices:
+            if choice.casefold() in lowered:
+                return choice
+    return default
+
+
+def label_value(labels: Iterable[str], prefix: str) -> str | None:
+    prefix_lower = prefix.casefold()
+    for label in labels:
+        if label.casefold().startswith(prefix_lower):
+            _, value = label.split(':', 1)
+            return value.strip()
+    return None
+
+
+def markdown_sections(body: str) -> dict[str, str]:
+    cleaned = (
+        strip_html_comments(body).replace("\r\n", "\n").replace("\r", "\n")
+    )
+    matches = list(HEADING_RE.finditer(cleaned))
+    sections: dict[str, str] = {}
+    for index, match in enumerate(matches):
+        start = match.end()
+        end = (
+            matches[index + 1].start()
+            if index + 1 < len(matches)
+            else len(cleaned)
+        )
+        sections[normalize_whitespace(match.group(1))] = cleaned[
+            start:end
+        ].strip()
+    return sections
+
+
+def section_value(sections: dict[str, str], name: str) -> str:
+    for key, value in sections.items():
+        if key.casefold() == name.casefold():
+            return value
+    return ""
+
+
+def resolve_category(labels: list[str], body: str) -> str:
+    value = label_value(labels, 'release notes:')
+    if value:
+        return split_known_choice(value, PR_CATEGORIES, 'Others')
+    sections = markdown_sections(body)
+    return split_known_choice(
+        section_value(sections, 'PR Category'), PR_CATEGORIES, 'Others'
+    )
+
+
+def resolve_topic(labels: list[str], body: str) -> str:
+    value = label_value(labels, 'topic:')
+    if value:
+        return split_known_choice(value, PR_TYPES, 'Others')
+    sections = markdown_sections(body)
+    return split_known_choice(
+        section_value(sections, 'PR Types'), PR_TYPES, 'Others'
+    )
+
+
+def resolve_description(body: str) -> str:
+    sections = markdown_sections(body)
+    return cleanup_markdown_text(section_value(sections, 'Description'))
+
+
+def load_token(explicit_token: str | None) -> str | None:
+    if explicit_token:
+        return explicit_token
+    for env_name in ('GITHUB_API_TOKEN', 'GITHUB_TOKEN', 'GH_TOKEN'):
+        value = os.getenv(env_name)
+        if value:
+            return value
+    token_file = Path('~/.gh_tokenrc').expanduser()
+    if token_file.exists():
+        match = re.search(r'github_oauth\s*=\s*(\S+)', token_file.read_text())
+        if match:
+            return match.group(1)
+    return None
+
+
+def collect_commits(
+    base_ref: str, head_ref: str, use_merge_base: bool
+) -> list[CommitRecord]:
+    start_ref = base_ref
+    if use_merge_base:
+        start_ref = run_git('merge-base', base_ref, head_ref)
+    raw_log = run_git(
+        'log',
+        '--reverse',
+        '--pretty=format:%H%x1f%s%x1f%an%x1e',
+        f'{start_ref}..{head_ref}',
+    )
+    if not raw_log:
+        return []
+
+    commits = []
+    for record in raw_log.split('\x1e'):
+        record = record.strip()
+        if not record:
+            continue
+        commit_hash, title, author = record.split('\x1f')
+        commits.append(
+            CommitRecord(
+                commit_hash=commit_hash,
+                title=title,
+                git_author=author,
+                pr_number=extract_pr_number(title),
+            )
+        )
+    return commits
+
+
+class PullRequestCache:
+    def __init__(self, path: Path):
+        self.path = path
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._data: dict[str, dict[str, object]] = {}
+        if self.path.exists():
+            self._data = json.loads(self.path.read_text())
+
+    def get_many(self, numbers: Iterable[int]) -> dict[int, PullRequestRecord]:
+        records = {}
+        for number in numbers:
+            cached = self._data.get(str(number))
+            if cached is not None:
+                cached = dict(cached)
+                cached.setdefault('title', f'PR #{number}')
+                records[number] = PullRequestRecord(**cached)
+        return records
+
+    def update_many(self, records: Iterable[PullRequestRecord]) -> None:
+        changed = False
+        for record in records:
+            self._data[str(record.number)] = asdict(record)
+            changed = True
+        if changed:
+            self.path.write_text(
+                json.dumps(self._data, indent=2, sort_keys=True) + '\n'
+            )
+
+
+class GitHubClient:
+    def __init__(self, token: str | None, owner: str, repo: str):
+        self.owner = owner
+        self.repo = repo
+        self.headers = {'Accept': 'application/vnd.github+json'}
+        if token:
+            self.headers['Authorization'] = f'bearer {token}'
+
+    def _request_graphql(self, query: str) -> dict[str, object]:
+        last_error: Exception | None = None
+        for attempt in range(3):
+            try:
+                payload = json.dumps({'query': query}).encode()
+                http_request = request.Request(
+                    'https://api.github.com/graphql',
+                    data=payload,
+                    headers={
+                        **self.headers,
+                        'Content-Type': 'application/json',
+                    },
+                    method='POST',
+                )
+                with request.urlopen(http_request, timeout=60) as response:
+                    payload = json.loads(response.read().decode())
+                if payload.get('errors'):
+                    raise RuntimeError(
+                        f"GitHub GraphQL query failed: {payload['errors']}"
+                    )
+                return payload
+            except error.HTTPError as http_error:
+                last_error = http_error
+                if http_error.code in RETRYABLE_STATUS_CODES and attempt < 2:
+                    time.sleep(1 + attempt)
+                    continue
+                raise
+            except (error.URLError, ValueError, RuntimeError) as request_error:
+                last_error = request_error
+                if attempt == 2:
+                    raise
+                time.sleep(1 + attempt)
+        assert last_error is not None
+        raise last_error
+
+    def _fetch_batch(self, batch: list[int]) -> dict[int, PullRequestRecord]:
+        query_parts = []
+        for number in batch:
+            query_parts.append(
+                f'''
+                pr_{number}: pullRequest(number: {number}) {{
+                  title
+                  author {{ login }}
+                  body
+                  labels(first: 50) {{
+                    nodes {{ name }}
+                  }}
+                  reviews(first: 100, states: APPROVED) {{
+                    nodes {{
+                      author {{ login }}
+                    }}
+                  }}
+                }}
+                '''
+            )
+        payload = self._request_graphql(
+            f'''
+            query {{
+              repository(owner: "{self.owner}", name: "{self.repo}") {{
+                {''.join(query_parts)}
+              }}
+            }}
+            '''
+        )
+        repository = payload['data']['repository']
+        results: dict[int, PullRequestRecord] = {}
+        for number in batch:
+            node = repository.get(f'pr_{number}')
+            if node is None:
+                continue
+            labels = [item['name'] for item in node['labels']['nodes']]
+            reviewers = sorted(
+                {
+                    review['author']['login']
+                    for review in node['reviews']['nodes']
+                    if review.get('author')
+                }
+            )
+            body = node.get('body') or ''
+            results[number] = PullRequestRecord(
+                number=number,
+                title=node.get('title') or f'PR #{number}',
+                author=(node.get('author') or {}).get('login', ''),
+                labels=labels,
+                reviewers=reviewers,
+                category=resolve_category(labels, body),
+                topic=resolve_topic(labels, body),
+                description=resolve_description(body),
+            )
+        return results
+
+    def fetch_pull_requests(
+        self, numbers: list[int], batch_size: int, workers: int
+    ) -> dict[int, PullRequestRecord]:
+        unique_numbers = sorted(set(numbers))
+        if not unique_numbers:
+            return {}
+        batches = [
+            unique_numbers[index : index + batch_size]
+            for index in range(0, len(unique_numbers), batch_size)
+        ]
+        results: dict[int, PullRequestRecord] = {}
+        if workers <= 1 or len(batches) == 1:
+            for batch in batches:
+                results.update(self._fetch_batch(batch))
+            return results
+
+        with ThreadPoolExecutor(max_workers=workers) as executor:
+            future_to_batch = {
+                executor.submit(self._fetch_batch, batch): batch
+                for batch in batches
+            }
+            for future in as_completed(future_to_batch):
+                results.update(future.result())
+        return results
+
+
+def build_commit_rows(
+    commits: list[CommitRecord],
+    pull_requests: dict[int, PullRequestRecord],
+    owner: str,
+    repo: str,
+) -> tuple[list[CommitRow], list[int]]:
+    rows: list[CommitRow] = []
+    missing_prs: set[int] = set()
+
+    for commit in commits:
+        if commit.pr_number is None:
+            rows.append(
+                CommitRow(
+                    commit_hash=commit.commit_hash,
+                    category='Others',
+                    topic='Others',
+                    title=commit.title,
+                    pr_link='',
+                    author=commit.git_author,
+                    labels='',
+                    accepter_1='',
+                    accepter_2='',
+                    accepter_3='',
+                    description='',
+                )
+            )
+            continue
+
+        pr_record = pull_requests.get(commit.pr_number)
+        if pr_record is None:
+            missing_prs.add(commit.pr_number)
+            rows.append(
+                CommitRow(
+                    commit_hash=commit.commit_hash,
+                    category='Others',
+                    topic='Others',
+                    title=commit.title,
+                    pr_link=f'https://github.com/{owner}/{repo}/pull/{commit.pr_number}',
+                    author=commit.git_author,
+                    labels='',
+                    accepter_1='',
+                    accepter_2='',
+                    accepter_3='',
+                    description='',
+                )
+            )
+            continue
+
+        accepters = [*pr_record.reviewers, '', '', ''][:3]
+        rows.append(
+            CommitRow(
+                commit_hash=commit.commit_hash,
+                category=pr_record.category,
+                topic=pr_record.topic,
+                title=pr_record.title or commit.title,
+                pr_link=f'https://github.com/{owner}/{repo}/pull/{commit.pr_number}',
+                author=pr_record.author or commit.git_author,
+                labels=','.join(pr_record.labels),
+                accepter_1=accepters[0],
+                accepter_2=accepters[1],
+                accepter_3=accepters[2],
+                description=pr_record.description,
+            )
+        )
+
+    return rows, sorted(missing_prs)
+
+
+def ordered_values(
+    items: Iterable[str], preferred_order: list[str]
+) -> list[str]:
+    values = {item for item in items if item}
+    ordered = [value for value in preferred_order if value in values]
+    extras = sorted(values - set(preferred_order), key=str.casefold)
+    return ordered + extras
+
+
+def get_hash_or_pr_url(commit: CommitRow) -> str:
+    if not commit.pr_link:
+        return commit.commit_hash
+    matches = re.findall(
+        r'https://github.com/[^/]+/[^/]+/pull/(\d+)',
+        commit.pr_link,
+    )
+    if not matches:
+        return commit.commit_hash
+    return f'[#{matches[0]}]({commit.pr_link})'
+
+
+def markdown_entry_text(commit: CommitRow) -> str:
+    return cleanup_title(commit.title)
+
+
+def category_output_name(category: str) -> str:
+    cleaned = re.sub(r'[^0-9A-Za-z._-]+', '_', category.strip())
+    cleaned = cleaned.strip('._')
+    return cleaned or 'Others'
+
+
+def get_markdown_header(category: str) -> str:
+    return (
+        f'# Release Notes worksheet {category}\n'
+        '- polish PR title to make it human read friendly.\n'
+        '- edit, delete, merge multiple PRs.\n'
+        '- summarize notes for this category.\n\n'
+    )
+
+
+def render_markdown(
+    commits: list[CommitRow], base_ref: str, head_ref: str
+) -> str:
+    lines = [
+        '# Release Notes\n\n',
+        f'- Range: `{base_ref}..{head_ref}`\n',
+        f'- Commits: {len(commits)}\n\n',
+    ]
+
+    categories = ordered_values(
+        (commit.category for commit in commits), PR_CATEGORIES
+    )
+    for category in categories:
+        lines.append(f'## {category}\n\n')
+        category_commits = [
+            commit for commit in commits if commit.category == category
+        ]
+        topics = ordered_values(
+            (commit.topic for commit in category_commits), PR_TYPES
+        )
+        for topic in topics:
+            topic_commits = [
+                commit for commit in category_commits if commit.topic == topic
+            ]
+            if not topic_commits:
+                continue
+            lines.append(f'### {topic}\n\n')
+            for commit in topic_commits:
+                lines.append(
+                    f'- {markdown_entry_text(commit)} ({get_hash_or_pr_url(commit)})\n'
+                )
+            lines.append('\n')
+    return ''.join(lines)
+
+
+def render_category_markdown(category: str, commits: list[CommitRow]) -> str:
+    lines = [get_markdown_header(category), f'## {category}\n\n']
+    topics = ordered_values((commit.topic for commit in commits), PR_TYPES)
+    for topic in topics:
+        topic_commits = [commit for commit in commits if commit.topic == topic]
+        if not topic_commits:
+            continue
+        lines.append(f'### {topic}\n\n')
+        for commit in topic_commits:
+            lines.append(
+                f'- {markdown_entry_text(commit)} ({get_hash_or_pr_url(commit)})\n'
+            )
+        lines.append('\n')
+    return ''.join(lines)
+
+
+def write_csv(path: Path, rows: list[CommitRow]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open('w', newline='') as csv_file:
+        writer = csv.writer(csv_file)
+        writer.writerow(COMMIT_FIELDS)
+        for row in rows:
+            writer.writerow([getattr(row, field) for field in COMMIT_FIELDS])
+
+
+def write_lines(path: Path, lines: Iterable[str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(''.join(f'{line}\n' for line in lines))
+
+
+def write_category_exports(
+    export_dir: Path, rows: list[CommitRow]
+) -> list[Path]:
+    export_dir.mkdir(parents=True, exist_ok=True)
+    written_files: list[Path] = []
+    categories = ordered_values((row.category for row in rows), PR_CATEGORIES)
+    for category in categories:
+        category_rows = [row for row in rows if row.category == category]
+        if not category_rows:
+            continue
+        category_name = category_output_name(category)
+        category_dir = export_dir / category_name
+        markdown_path = category_dir / f'result_{category_name}.md'
+        csv_path = category_dir / f'result_{category_name}.csv'
+        markdown_path.parent.mkdir(parents=True, exist_ok=True)
+        markdown_path.write_text(
+            render_category_markdown(category, category_rows)
+        )
+        write_csv(csv_path, category_rows)
+        written_files.extend([markdown_path, csv_path])
+    return written_files
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description='Generate release-note CSV and Markdown between two commits in one command.'
+    )
+    parser.add_argument('base_ref', help='Base git ref or commit')
+    parser.add_argument('head_ref', help='Head git ref or commit')
+    parser.add_argument(
+        '--output-dir',
+        default='results',
+        help='Directory for generated files (default: results)',
+    )
+    parser.add_argument(
+        '--owner',
+        default=DEFAULT_OWNER,
+        help=f'GitHub owner (default: {DEFAULT_OWNER})',
+    )
+    parser.add_argument(
+        '--repo',
+        default=DEFAULT_REPO,
+        help=f'GitHub repo (default: {DEFAULT_REPO})',
+    )
+    parser.add_argument(
+        '--token',
+        help='GitHub token; falls back to env vars or ~/.gh_tokenrc',
+    )
+    parser.add_argument(
+        '--cache-path',
+        default='results/pr_cache.json',
+        help='PR metadata cache path (default: results/pr_cache.json)',
+    )
+    parser.add_argument(
+        '--batch-size',
+        type=int,
+        default=25,
+        help='How many PRs to fetch per GitHub GraphQL request (default: 25)',
+    )
+    parser.add_argument(
+        '--workers',
+        type=int,
+        default=4,
+        help='How many GraphQL batches to fetch concurrently (default: 4)',
+    )
+    parser.add_argument(
+        '--direct-range',
+        action='store_true',
+        help='Use base_ref..head_ref directly instead of merge-base(base_ref, head_ref)..head_ref',
+    )
+    parser.add_argument(
+        '--local-only',
+        action='store_true',
+        help='Skip GitHub API calls and only use local git metadata',
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    output_dir = Path(args.output_dir)
+    csv_path = output_dir / 'commitlist.csv'
+    export_dir = output_dir / 'export'
+    contributors_path = output_dir / 'contributors.txt'
+
+    commits = collect_commits(
+        args.base_ref,
+        args.head_ref,
+        use_merge_base=not args.direct_range,
+    )
+    if not commits:
+        raise SystemExit(
+            f'No commits found between {args.base_ref} and {args.head_ref}.'
+        )
+
+    pull_requests: dict[int, PullRequestRecord] = {}
+    pr_numbers = sorted(
+        {commit.pr_number for commit in commits if commit.pr_number is not None}
+    )
+    if args.local_only and pr_numbers:
+        print(
+            'Warning: running in --local-only mode. PR metadata will not be fetched, '
+            'so category/topic fall back to Others and descriptions stay empty.',
+            file=sys.stderr,
+        )
+    if not args.local_only and pr_numbers:
+        token = load_token(args.token)
+        if not token:
+            print(
+                'Warning: GitHub token not found. Falling back to unauthenticated '
+                'GitHub API requests; this may hit rate limits on large ranges.',
+                file=sys.stderr,
+            )
+        cache = PullRequestCache(Path(args.cache_path))
+        cached = cache.get_many(pr_numbers)
+        missing = [
+            number
+            for number in pr_numbers
+            if number not in cached or cached[number].title == f'PR #{number}'
+        ]
+        fetched = GitHubClient(
+            args.token or token, args.owner, args.repo
+        ).fetch_pull_requests(
+            missing,
+            batch_size=max(1, args.batch_size),
+            workers=max(1, args.workers),
+        )
+        cache.update_many(fetched.values())
+        pull_requests = {**cached, **fetched}
+
+    rows, missing_prs = build_commit_rows(
+        commits, pull_requests, args.owner, args.repo
+    )
+    contributors = sorted(
+        {commit.git_author for commit in commits if commit.git_author},
+        key=str.casefold,
+    )
+
+    write_csv(csv_path, rows)
+    exported_files = write_category_exports(export_dir, rows)
+    write_lines(contributors_path, contributors)
+
+    if missing_prs and not args.local_only:
+        print(
+            'Warning: failed to fetch metadata for PRs '
+            + ', '.join(f'#{number}' for number in missing_prs)
+            + '. Fallback commit metadata was used.',
+            file=sys.stderr,
+        )
+
+    print(f'Generated {len(rows)} rows from {len(commits)} commits.')
+    print(f'CSV: {csv_path}')
+    print(f'Export dir: {export_dir}')
+    print(f'Category exports: {len(exported_files)} files')
+    print(f'Contributors: {contributors_path}')
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

tools/continuous_integration/bisect.py

@@ -0,0 +1,147 @@
+#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A script to bisect the mainline commits and find the culprit commit.
+# The default 'git bisect' checks feature branches, which is not desired
+# because commits in feature branch might not pass tests or compile.
+#
+# Example:
+#   python ../bisect.py --git_dir=$PWD/../Paddle --build_dir=$PWD \
+#       --good_commit=3647ed6 --bad_commit=279aa6 \
+#       --test_target=test_rnn_encoder_decoder
+
+import argparse
+import os
+import subprocess
+import sys
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument(
+    '--git_dir', type=str, default='', help='git repo root directory.'
+)
+parser.add_argument(
+    '--build_dir', type=str, default='', help='build directory.'
+)
+parser.add_argument(
+    '--good_commit',
+    type=str,
+    default='',
+    help='The old commit known to be good.',
+)
+parser.add_argument(
+    '--bad_commit', type=str, default='', help='The new commit known to be bad.'
+)
+parser.add_argument(
+    '--test_target', type=str, default='', help='The test target to evaluate.'
+)
+parser.add_argument(
+    '--bisect_branch',
+    type=str,
+    default='develop',
+    help='The mainline branch to bisect (feature branch ignored).',
+)
+parser.add_argument(
+    '--log_file', type=str, default='', help='The file use to log outputs.'
+)
+parser.add_argument(
+    '--test_times',
+    type=int,
+    default=10,
+    help="Number of times to run the test target.",
+)
+parser.add_argument(
+    '--build_parallel', type=int, default=32, help="make parallelism."
+)
+args = parser.parse_args()
+
+if not args.log_file:
+    args.log_file = f'/tmp/{args.good_commit}...{args.bad_commit}.log'
+
+
+def print_arguments():
+    print('-----------  Configuration Arguments -----------')
+    for arg, value in sorted(vars(args).iteritems()):
+        print(f'{arg}: {value}')
+    print('------------------------------------------------')
+
+
+print_arguments()
+
+# List the commits in mainline branch.
+os.chdir(args.git_dir)
+ret = subprocess.check_output(
+    [f'git rev-list --first-parent {args.good_commit}...{args.bad_commit}'],
+    shell=True,
+)
+sys.stdout.write(f'commits found:\n{ret}\n')
+commits = ret.strip().split('\n')
+os.chdir(args.build_dir)
+# Clean up previous logs.
+subprocess.check_output([f'echo "" > {args.log_file}'], shell=True)
+
+last_culprit = ''
+while True:
+    # Get to the mainline branch and clean up
+    os.chdir(args.git_dir)
+    subprocess.check_output(
+        [
+            f'git checkout {args.bisect_branch} && git clean -fd && git checkout .'
+        ],
+        shell=True,
+    )
+
+    if not commits:
+        sys.stdout.write('no commits to bisect\n')
+        sys.exit()
+    # checkout the picked branch.
+    pick_idx = len(commits) / 2
+    pick = commits[pick_idx]
+    os.chdir(args.git_dir)
+    subprocess.check_output([f'git checkout {pick}'], shell=True)
+
+    # Clean builds and compile.
+    # We assume mainline commits should always compile.
+    os.chdir(args.build_dir)
+    sys.stdout.write(f'eval commit {pick_idx}/{len(commits)}: {pick}\n')
+    # Link error can happen without complete clean up.
+    cmd = (
+        'rm -rf * && '
+        f'cmake -DWITH_TESTING=ON {args.git_dir} >> {args.log_file} && make -j{args.build_parallel} >> {args.log_file}'
+    )
+    sys.stdout.write(f'cmd: {cmd}\n')
+    try:
+        subprocess.check_output([cmd], shell=True)
+    except subprocess.CalledProcessError as e:
+        sys.stderr.write(f'failed to build commit: {pick}\n{e}\n')
+        sys.exit()
+    # test the selected branch.
+    passed = True
+    try:
+        cmd = f'ctest --repeat-until-fail {args.test_times} -R {args.test_target} >> {args.log_file}'
+        sys.stdout.write(f'cmd: {cmd}\n')
+        subprocess.check_output([cmd], shell=True)
+    except subprocess.CalledProcessError as e:
+        passed = False
+        last_culprit = pick
+    sys.stdout.write(f'eval {pick} passed: {passed}\n')
+    if passed:
+        if pick_idx == 0:
+            break
+        commits = commits[:pick_idx]
+    else:
+        if pick_idx + 1 >= len(commits):
+            break
+        commits = commits[pick_idx + 1 :]
+
+sys.stdout.write(f'Culprit commit: {last_culprit}\n')

tools/count_api_without_core_ops.py

@@ -0,0 +1,216 @@
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import hashlib
+import importlib
+import inspect
+import sys
+
+__all__ = [
+    'get_apis_with_and_without_core_ops',
+]
+
+# APIs that should not be printed into API.spec
+omitted_list = [
+    "paddle.base.LoDTensor.set",  # Do not know why it should be omitted
+    "paddle.base.io.ComposeNotAligned",
+    "paddle.base.io.ComposeNotAligned.__init__",
+]
+
+
+def md5(doc):
+    try:
+        hashinst = hashlib.md5()
+        hashinst.update(str(doc).encode('utf-8'))
+        md5sum = hashinst.hexdigest()
+    except UnicodeDecodeError as e:
+        md5sum = None
+        print(
+            f"Error({e}) occurred when `md5({doc})`, discard it.",
+            file=sys.stderr,
+        )
+    return md5sum
+
+
+def split_with_and_without_core_ops(member, cur_name):
+    if cur_name in omitted_list:
+        return
+
+    if member.__doc__.find(':api_attr: Static Graph') != -1:
+        return
+
+    if (
+        cur_name.find('EagerParamBase') != -1
+        or cur_name.find('Parameter') != -1
+        or cur_name.find('Variable') != -1
+        or cur_name.find('control_flow') != -1
+        or cur_name.find('contrib.mixed_precision') != -1
+    ):
+        return
+
+    if inspect.isclass(member):
+        pass
+    else:
+        try:
+            source = inspect.getsource(member)
+            if source.find('append_op') != -1:
+                if source.find('core.ops') != -1 or source.find('_C_ops') != -1:
+                    api_with_ops.append(cur_name)
+                else:
+                    api_without_ops.append(cur_name)
+        except:
+            # If getsource failed (pybind API or function inherit from father class), just skip
+            pass
+
+
+def get_md5_of_func(member, cur_name):
+    if cur_name in omitted_list:
+        return
+
+    doc_md5 = md5(member.__doc__)
+
+    if inspect.isclass(member):
+        pass
+    else:
+        try:
+            source = inspect.getsource(member)
+            func_dict[cur_name] = md5(source)
+        except:
+            # If getsource failed (pybind API or function inherit from father class), just skip
+            pass
+
+
+def visit_member(parent_name, member, func):
+    cur_name = ".".join([parent_name, member.__name__])
+    if inspect.isclass(member):
+        func(member, cur_name)
+        for name, value in inspect.getmembers(member):
+            if hasattr(value, '__name__') and (
+                not name.startswith("_") or name == "__init__"
+            ):
+                visit_member(cur_name, value, func)
+    elif inspect.ismethoddescriptor(member):
+        return
+    elif callable(member):
+        func(member, cur_name)
+    elif inspect.isgetsetdescriptor(member):
+        return
+    else:
+        raise RuntimeError(
+            f"Unsupported generate signature of member, type {type(member)}"
+        )
+
+
+def is_primitive(instance):
+    int_types = (int,)
+    primitive_types = (*int_types, float, str)
+    if isinstance(instance, primitive_types):
+        return True
+    elif isinstance(instance, (list, tuple, set)):
+        for obj in instance:
+            if not is_primitive(obj):
+                return False
+
+        return True
+    else:
+        return False
+
+
+ErrorSet = set()
+IdSet = set()
+visited_modules = set()
+
+
+def visit_all_module(mod, func):
+    mod_name = mod.__name__
+    if mod_name != 'paddle' and not mod_name.startswith('paddle.'):
+        return
+
+    if mod_name.startswith('paddle.base.core'):
+        return
+
+    if mod in visited_modules:
+        return
+    visited_modules.add(mod)
+
+    member_names = dir(mod)
+    if hasattr(mod, "__all__"):
+        member_names += mod.__all__
+    for member_name in member_names:
+        if member_name.startswith('_'):
+            continue
+        cur_name = mod_name + '.' + member_name
+        try:
+            instance = getattr(mod, member_name)
+            if inspect.ismodule(instance):
+                visit_all_module(instance, func)
+            else:
+                instance_id = id(instance)
+                if instance_id in IdSet:
+                    continue
+                IdSet.add(instance_id)
+                visit_member(mod.__name__, instance, func)
+        except:
+            if cur_name not in ErrorSet:
+                ErrorSet.add(cur_name)
+
+
+def get_apis_with_and_without_core_ops(modules):
+    global api_with_ops, api_without_ops
+    api_with_ops = []
+    api_without_ops = []
+    for m in modules:
+        visit_all_module(
+            importlib.import_module(m), split_with_and_without_core_ops
+        )
+    return api_with_ops, api_without_ops
+
+
+def get_api_source_desc(modules):
+    global func_dict
+    func_dict = collections.OrderedDict()
+    for m in modules:
+        visit_all_module(importlib.import_module(m), get_md5_of_func)
+    return func_dict
+
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        modules = sys.argv[2].split(",")
+        if sys.argv[1] == '-c':
+            api_with_ops, api_without_ops = get_apis_with_and_without_core_ops(
+                modules
+            )
+
+            print('api_with_ops:', len(api_with_ops))
+            print('\n'.join(api_with_ops))
+            print('\n==============\n')
+            print('api_without_ops:', len(api_without_ops))
+            print('\n'.join(api_without_ops))
+
+        if sys.argv[1] == '-p':
+            func_dict = get_api_source_desc(modules)
+            for name in func_dict:
+                print(name, func_dict[name])
+
+    else:
+        print(
+            """Usage:
+            1. Count and list all operator-related APIs that contains append_op but not _legacy_C_ops.xx.
+                python ./count_api_without_core_ops.py -c paddle
+            2. Print api and the md5 of source code of the api.
+                python ./count_api_without_core_ops.py -p paddle
+            """
+        )

tools/coverage/coverage_diff.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+usage: coverage_diff.py info_file diff_file > > coverage-diff.info
+"""
+
+import sys
+
+
+def get_diff_file_lines(diff_file):
+    """
+    Args:
+        diff_file (str): File to get modified lines.
+
+    Returns:
+        dict: The diff lines of files.
+    """
+    diff_file_lines = {}
+
+    current_file = None
+    current_line = -1
+
+    with open(diff_file) as diff_file:
+        for line in diff_file:
+            line = line.strip()
+
+            if line.startswith('+++ '):
+                current_file = line.removeprefix('+++ ')
+
+                diff_file_lines[current_file] = []
+
+                continue
+
+            elif line.startswith('@@ '):
+                current_line = line.split()[2]
+                current_line = current_line.lstrip('+').split(',')[0]
+                current_line = int(current_line)
+
+                continue
+
+            elif line.startswith('-'):
+                continue
+
+            elif line.startswith('+'):
+                diff_file_lines[current_file].append(current_line)
+
+            current_line += 1
+
+    return diff_file_lines
+
+
+def get_info_file_lines(info_file, diff_file):
+    """
+    Args:
+        info_file (str): File generated by lcov.
+        diff_file (str): File to get modified lines.
+
+    Returns:
+        None
+    """
+    diff_file_lines = get_diff_file_lines(diff_file)
+
+    current_lines = []
+    current_lf = 0
+    current_lh = 0
+
+    with open(info_file) as info_file:
+        for line in info_file:
+            line = line.strip()
+
+            if line.startswith('SF:'):
+                current_file = line.lstrip('SF:')
+
+                current_file = current_file.removeprefix('/paddle/')
+
+                current_lines = diff_file_lines.get(current_file, [])
+
+            elif line.startswith('DA:'):
+                da = line.lstrip('DA:').split(',')
+
+                if int(da[0]) in current_lines:
+                    current_lf += 1
+
+                    if not line.endswith(',0'):
+                        current_lh += 1
+
+                    print(line)
+
+                continue
+
+            elif line.startswith('LF:'):
+                print(f'LF:{current_lf}')
+
+                continue
+
+            elif line.startswith('LH:'):
+                print(f'LH:{current_lh}')
+
+                continue
+
+            print(line)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        sys.exit()
+
+    info_file = sys.argv[1]
+    diff_file = sys.argv[2]
+
+    get_info_file_lines(info_file, diff_file)

tools/coverage/coverage_diff_list.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+usage: coverage_diff_list.py list_file max_rate > coverage-diff-list-90.out
+"""
+
+import sys
+
+
+def filter_by(list_file, max_rate):
+    """
+    Args:
+        list_file (str): File of list.
+        max_rate (float): Max rate.
+
+    Returns:
+        tuple: File and coverage rate.
+    """
+    with open(list_file) as list_file:
+        for line in list_file:
+            line = line.strip()
+
+            split = line.split('|')
+
+            # name
+
+            name = split[0].strip()
+
+            name = name.removeprefix('/paddle/')
+
+            # rate
+
+            try:
+                rate = split[1].split()[0].strip('%')
+                rate = float(rate)
+
+                if rate >= max_rate:
+                    continue
+            except:
+                pass
+
+            print(name, rate)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        sys.exit()
+
+    list_file = sys.argv[1]
+    max_rate = float(sys.argv[2])
+
+    filter_by(list_file, max_rate)

tools/coverage/coverage_lines.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+usage: coverage_lines.py info_file expected
+"""
+
+import os
+import sys
+
+
+def get_lines(info_file):
+    """
+    Args:
+        info_file (str): File generated by lcov.
+
+    Returns:
+        float: Coverage rate.
+    """
+    hits = 0.0
+    total = 0.0
+
+    with open(info_file) as info_file:
+        for line in info_file:
+            line = line.strip()
+
+            if not line.startswith('DA:'):
+                continue
+
+            line = line[3:]
+
+            total += 1
+
+            if int(line.split(',')[1]) > 0:
+                hits += 1
+
+    if total == 0:
+        print('no data found')
+        sys.exit()
+
+    return hits / total
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        sys.exit()
+
+    info_file = sys.argv[1]
+    expected = float(sys.argv[2])
+
+    if not os.path.isfile(info_file):
+        print(f'info file {info_file} is not exists, ignored')
+        sys.exit()
+
+    actual = get_lines(info_file)
+    actual = round(actual, 3)
+
+    if actual < expected:
+        print(
+            f'expected >= {round(expected * 100, 1)} %, actual {round(actual * 100, 1)} %, failed'
+        )
+
+        sys.exit(1)
+
+    print(
+        f'expected >= {round(expected * 100, 1)} %, actual {round(actual * 100, 1)} %, passed'
+    )

tools/coverage/cuda_clean.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""usage: cuda_clean.py pull_id."""
+
+import os
+import sys
+
+from github import Github
+
+
+def get_pull(pull_id):
+    """
+    Args:
+        pull_id (int): Pull id.
+
+    Returns:
+        github.PullRequest.PullRequest: The pull request.
+    """
+    token = os.getenv('GITHUB_API_TOKEN')
+    github = Github(token, timeout=60)
+    repo = github.get_repo('PaddlePaddle/Paddle')
+    pull = repo.get_pull(pull_id)
+
+    return pull
+
+
+def get_files(pull_id):
+    """
+    Args:
+        pull_id (int): Pull id.
+
+    Returns:
+       iterable: The generator will yield every filename.
+    """
+
+    pull = get_pull(pull_id)
+
+    for file in pull.get_files():
+        yield file.filename
+
+
+def clean(pull_id):
+    """
+    Args:
+        pull_id (int): Pull id.
+
+    Returns:
+        None.
+    """
+
+    changed = []
+
+    for file in get_files(pull_id):
+        # changed.append('/paddle/build/{}.gcda'.format(file))
+        changed.append(file)
+
+    for parent, dirs, files in os.walk('/paddle/build/'):
+        for gcda in files:
+            if gcda.endswith('.gcda'):
+                file_name = gcda.replace('.gcda', '')
+                dir_name_list = parent.replace('/paddle/build/', '').split('/')
+                dir_name_list = dir_name_list[:-2]
+                dir_name = '/'.join(dir_name_list)
+                src_name = dir_name + '/' + file_name
+
+                # remove no changed gcda
+
+                if src_name not in changed:
+                    unused_file = parent + '/' + gcda
+                    # print unused_file
+                    os.remove(gcda)
+                else:
+                    print(src_name)
+
+
+if __name__ == '__main__':
+    pull_id = sys.argv[1]
+    pull_id = int(pull_id)
+
+    clean(pull_id)

tools/coverage/gcda_clean.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""usage: gcda_clean.py pull_id."""
+
+import os
+import sys
+import time
+
+from github import Github
+
+
+def get_pull(pull_id):
+    """Get pull.
+
+    Args:
+        pull_id (int): Pull id.
+
+    Returns:
+        github.PullRequest.PullRequest
+    """
+    token = os.getenv('GITHUB_API_TOKEN')
+    github = Github(token, timeout=60)
+    idx = 1
+    while idx < 4:
+        try:
+            repo = github.get_repo('PaddlePaddle/Paddle')
+        except Exception as e:
+            print(e)
+            print(f"get_repo error, retry {idx} times after {idx * 10} secs.")
+        else:
+            break
+        idx += 1
+        time.sleep(idx * 10)
+    pull = repo.get_pull(pull_id)
+
+    return pull
+
+
+def get_files(pull_id):
+    """Get files.
+
+    Args:
+        pull_id (int): Pull id.
+
+    Returns:
+       iterable: The generator will yield every filename.
+    """
+    pull = get_pull(pull_id)
+
+    for file in pull.get_files():
+        yield file.filename
+
+
+def clean(pull_id):
+    """Clean.
+
+    Args:
+        pull_id (int): Pull id.
+
+    Returns:
+        None.
+    """
+    changed = []
+
+    for file in get_files(pull_id):
+        changed.append(f'/paddle/build/{file}.gcda')
+
+    for parent, dirs, files in os.walk('/paddle/build/'):
+        for gcda in files:
+            if gcda.endswith('.gcda'):
+                trimmed = parent
+
+                # convert paddle/fluid/imperative/CMakeFiles/layer.dir/layer.cc.gcda
+                # to paddle/fluid/imperative/layer.cc.gcda
+                # modified to make it more robust
+                # convert /paddle/build/paddle/phi/backends/CMakeFiles/phi_backends.dir/gpu/cuda/cuda_info.cc.gcda
+                # to /paddle/build/paddle/phi/backends/gpu/cuda/cuda_info.cc.gcda
+                trimmed_tmp = []
+                for p in trimmed.split('/'):
+                    if p.endswith('.dir') or p.endswith('CMakeFiles'):
+                        continue
+                    trimmed_tmp.append(p)
+                trimmed = '/'.join(trimmed_tmp)
+
+                # remove no changed gcda
+
+                if os.path.join(trimmed, gcda) not in changed:
+                    gcda = os.path.join(parent, gcda)
+                    os.remove(gcda)
+
+
+if __name__ == '__main__':
+    pull_id = sys.argv[1]
+    pull_id = int(pull_id)
+
+    clean(pull_id)

tools/coverage/paddle_coverage.sh

@@ -0,0 +1,253 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )"
+
+# install lcov
+if [ ! -f "/root/.cache/lcov-1.16.tar.gz" ];then
+wget -P /home https://paddle-ci.cdn.bcebos.com/coverage/lcov-1.16.tar.gz --no-proxy --no-check-certificate || exit 101
+cp /home/lcov-1.16.tar.gz /root/.cache/lcov-1.16.tar.gz
+else
+    cp /root/.cache/lcov-1.16.tar.gz /home/lcov-1.16.tar.gz
+fi
+tar -xf /home/lcov-1.16.tar.gz -C /
+cd /lcov-1.16
+make install
+
+# run paddle coverage
+
+cd /paddle/build
+
+python ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID} || exit 101
+lcov --ignore-errors gcov --capture -d ./ -o coverage.info --rc lcov_branch_coverage=0
+
+
+# full html report
+
+function gen_full_html_report_cinn(){
+        lcov --extract coverage.info \
+        '/paddle/paddle/cinn/adt/*' \
+        '/paddle/paddle/cinn/api/*' \
+        '/paddle/paddle/cinn/ast_gen_ius/*' \
+        '/paddle/paddle/cinn/backends/*' \
+        '/paddle/paddle/cinn/common/*' \
+        '/paddle/paddle/cinn/frontend/*' \
+        '/paddle/paddle/cinn/hlir/*' \
+        '/paddle/paddle/cinn/ir/*' \
+        '/paddle/paddle/cinn/lang/*' \
+        '/paddle/paddle/cinn/operator_fusion/*' \
+        '/paddle/paddle/cinn/optim/*' \
+        '/paddle/paddle/cinn/poly/*' \
+        '/paddle/paddle/cinn/runtime/*' \
+        '/paddle/paddle/cinn/utils/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+}
+
+
+function gen_full_html_report() {
+    lcov --extract coverage.info \
+        '/paddle/paddle/fluid/framework/*' \
+        '/paddle/paddle/fluid/imperative/*' \
+        '/paddle/paddle/fluid/inference/*' \
+        '/paddle/paddle/fluid/memory/*' \
+        '/paddle/paddle/fluid/operators/*' \
+        '/paddle/paddle/fluid/eager/*' \
+        '/paddle/paddle/fluid/pir/*' \
+        '/paddle/paddle/fluid/ir_adaptor/*' \
+        '/paddle/paddle/phi/*' \
+        '/paddle/paddle/pir/*' \
+        '/paddle/paddle/utils/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+
+    lcov --remove coverage-full.info \
+        '/paddle/paddle/fluid/framework/*_test*' \
+        '/paddle/paddle/fluid/*/*test*' \
+        '/paddle/paddle/fluid/*/*/*test*' \
+        '/paddle/paddle/fluid/inference/tests/*' \
+        '/paddle/paddle/fluid/inference/api/demo_ci/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+}
+
+function gen_full_html_report_xpu() {
+    lcov --extract coverage.info \
+        '/paddle/paddle/phi/kernels/xpu/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+
+    lcov --remove coverage-full.info \
+        '/paddle/paddle/fluid/framework/*_test*' \
+        '/paddle/paddle/fluid/*/*test*' \
+        '/paddle/paddle/fluid/*/*/*test*' \
+        '/paddle/paddle/fluid/inference/tests/*' \
+        '/paddle/paddle/fluid/inference/api/demo_ci/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+}
+
+function gen_full_html_report_npu() {
+    lcov --extract coverage.info \
+        '/paddle/paddle/fluid/operators/*npu*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+
+    lcov --remove coverage-full.info \
+        '/paddle/paddle/fluid/framework/*_test*' \
+        '/paddle/paddle/fluid/*/*test*' \
+        '/paddle/paddle/fluid/*/*/*test*' \
+        '/paddle/paddle/fluid/inference/tests/*' \
+        '/paddle/paddle/fluid/inference/api/demo_ci/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+}
+
+if [ ${WITH_XPU:-OFF} == "ON" ]; then
+    gen_full_html_report_xpu || true
+else
+    gen_full_html_report || true
+fi
+
+if [ ${WITH_CINN:-OFF} == "ON" ]; then
+    gen_full_html_report_cinn || true
+else
+    gen_full_html_report || true
+fi
+
+# diff html report
+
+function gen_diff_html_report() {
+    if [ "${GIT_PR_ID}" != "" ]; then
+
+        COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
+
+        python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out
+    fi
+
+    lcov --extract coverage-full.info \
+        ${COVERAGE_DIFF_PATTERN} \
+        -o coverage-diff.info \
+        --rc lcov_branch_coverage=0
+
+    python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp
+
+    mv -f coverage-diff.tmp coverage-diff.info
+
+    genhtml -o coverage-diff -t 'Diff Coverage' --no-function-coverage --no-branch-coverage coverage-diff.info
+}
+
+gen_diff_html_report || true
+
+# python coverage
+
+export COVERAGE_FILE=/paddle/build/python-coverage.data
+
+coverage combine `$(ls python-coverage.data.*)` || NO_PYTHON_COVERAGE_DATA=1
+
+`$(coverage xml -i -o python-coverage.xml)` || [[ "${NO_PYTHON_COVERAGE_DATA}" == "1" ]]
+
+sed -i 's/mnt\/paddle/paddle/g' python-coverage.xml
+
+`$(python ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info)` || [[ "${NO_PYTHON_COVERAGE_DATA}" == "1" ]]
+
+# python full html report
+#
+function gen_python_full_html_report() {
+    lcov --extract python-coverage.info \
+        '/paddle/python/*' \
+        -o python-coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f python-coverage-full.tmp python-coverage-full.info
+
+    lcov --remove python-coverage-full.info \
+        '/*/tests/*' \
+        -o python-coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f python-coverage-full.tmp python-coverage-full.info
+}
+
+gen_python_full_html_report || true
+
+# python diff html report
+
+function gen_python_diff_html_report() {
+    if [ "${GIT_PR_ID}" != "" ]; then
+        COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
+
+        python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > python-git-diff.out
+    fi
+
+    lcov --extract python-coverage-full.info \
+        ${COVERAGE_DIFF_PATTERN} \
+        -o python-coverage-diff.info \
+        --rc lcov_branch_coverage=0
+
+    python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out > python-coverage-diff.tmp
+
+    mv -f python-coverage-diff.tmp python-coverage-diff.info
+
+    genhtml -o python-coverage-diff \
+        -t 'Python Diff Coverage' \
+        --no-function-coverage \
+        --no-branch-coverage \
+        --ignore-errors source \
+        python-coverage-diff.info
+}
+
+gen_python_diff_html_report || true
+
+# assert coverage lines
+
+echo "Assert Diff Coverage"
+
+python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py coverage-diff.info 0.9 || COVERAGE_LINES_ASSERT=1
+
+echo "Assert Python Diff Coverage"
+
+if [ ${WITH_XPU:-OFF} == "ON" ]; then
+    echo "XPU has no python coverage!"
+else
+    if [[ "${NO_PYTHON_COVERAGE_DATA}" != "1" ]];then
+        python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9 || PYTHON_COVERAGE_LINES_ASSERT=1
+    fi
+fi
+
+if [ "$COVERAGE_LINES_ASSERT" = "1" ] || [ "$PYTHON_COVERAGE_LINES_ASSERT" = "1" ]; then
+    echo "exit 9" > /tmp/paddle_coverage.result
+    python ${PADDLE_ROOT}/tools/get_pr_title.py skip_coverage_check && NOT_CHECK_COVERAGE_PR=1
+    if [[ "${NOT_CHECK_COVERAGE_PR}" = "1" ]];then
+        echo "Skip coverage check in the PR-CI-Coverage pipeline."
+        exit 0
+    fi
+    exit 9
+fi

tools/coverage/paddle_coverage_new.sh

@@ -0,0 +1,281 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )"
+
+function lcov_init(){
+    # install lcov
+    if [ ! -f "/root/.cache/lcov-1.14.tar.gz" ];then
+        wget -P /home https://paddle-ci.gz.bcebos.com/coverage/lcov-1.14.tar.gz --no-proxy --no-check-certificate || exit 101
+        cp /home/lcov-1.14.tar.gz /root/.cache/lcov-1.14.tar.gz
+    else
+        cp /root/.cache/lcov-1.14.tar.gz /home/lcov-1.14.tar.gz
+    fi
+    tar -xf /home/lcov-1.14.tar.gz -C /
+    cd /lcov-1.14
+    make install
+    cd -
+}
+
+function gen_cpp_covinfo(){
+    # run paddle coverage
+    cd /paddle/build
+    python ${PADDLE_ROOT}/tools/coverage/gcda_clean.py ${GIT_PR_ID} || exit 101
+    lcov --capture -d ./ -o coverage.info --rc lcov_branch_coverage=0
+}
+
+
+# full html report
+
+function gen_full_html_report() {
+    lcov --extract coverage.info \
+        '/paddle/paddle/fluid/framework/*' \
+        '/paddle/paddle/fluid/imperative/*' \
+        '/paddle/paddle/fluid/inference/*' \
+        '/paddle/paddle/fluid/operators/*' \
+        '/paddle/paddle/fluid/eager/*' \
+        '/paddle/paddle/phi/*' \
+        '/paddle/paddle/utils/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+
+    lcov --remove coverage-full.info \
+        '/paddle/paddle/fluid/framework/*_test*' \
+        '/paddle/paddle/fluid/*/*test*' \
+        '/paddle/paddle/fluid/*/*/*test*' \
+        '/paddle/paddle/fluid/inference/tests/*' \
+        '/paddle/paddle/fluid/inference/api/demo_ci/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+}
+
+function gen_full_html_report_xpu() {
+    lcov --extract coverage.info \
+        '/paddle/paddle/fluid/operators/*xpu*' \
+        '/paddle/paddle/phi/kernels/xpu/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+
+    lcov --remove coverage-full.info \
+        '/paddle/paddle/fluid/framework/*_test*' \
+        '/paddle/paddle/fluid/*/*test*' \
+        '/paddle/paddle/fluid/*/*/*test*' \
+        '/paddle/paddle/fluid/inference/tests/*' \
+        '/paddle/paddle/fluid/inference/api/demo_ci/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+}
+
+function gen_full_html_report_npu() {
+    lcov --extract coverage.info \
+        '/paddle/paddle/fluid/operators/*npu*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+
+    lcov --remove coverage-full.info \
+        '/paddle/paddle/fluid/framework/*_test*' \
+        '/paddle/paddle/fluid/*/*test*' \
+        '/paddle/paddle/fluid/*/*/*test*' \
+        '/paddle/paddle/fluid/inference/tests/*' \
+        '/paddle/paddle/fluid/inference/api/demo_ci/*' \
+        -o coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f coverage-full.tmp coverage-full.info
+}
+
+# if [ ${WITH_XPU:-OFF} == "ON" ]; then
+#     gen_full_html_report_xpu || true
+# else
+#     gen_full_html_report || true
+# fi
+
+# diff html report
+
+function gen_diff_html_report() {
+    if [ "${GIT_PR_ID}" != "" ]; then
+
+        COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
+
+        python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > git-diff.out
+    fi
+
+    lcov --extract coverage-full.info \
+        ${COVERAGE_DIFF_PATTERN} \
+        -o coverage-diff.info \
+        --rc lcov_branch_coverage=0
+
+    python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py coverage-diff.info git-diff.out > coverage-diff.tmp
+
+    mv -f coverage-diff.tmp coverage-diff.info
+
+    genhtml -o coverage-diff -t 'Diff Coverage' --no-function-coverage --no-branch-coverage coverage-diff.info
+}
+
+# gen_diff_html_report || true
+
+function gen_py_covinfo(){
+    # python coverage
+
+    export COVERAGE_FILE=/paddle/build/python-coverage.data
+    coverage combine $(ls python-coverage.data.*) || NO_PYTHON_COVERAGE_DATA=1
+    $(coverage xml -i -o python-coverage.xml) || [[ "${NO_PYTHON_COVERAGE_DATA}" == "1" ]]
+    sed -i 's/mnt\/paddle/paddle/g' python-coverage.xml
+    $(python ${PADDLE_ROOT}/tools/coverage/python_coverage.py > python-coverage.info) || [[ "${NO_PYTHON_COVERAGE_DATA}" == "1" ]]
+}
+
+
+# python full html report
+#
+function gen_python_full_html_report() {
+    lcov --extract python-coverage.info \
+        '/paddle/python/*' \
+        -o python-coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f python-coverage-full.tmp python-coverage-full.info
+
+    lcov --remove python-coverage-full.info \
+        '/*/tests/*' \
+        -o python-coverage-full.tmp \
+        --rc lcov_branch_coverage=0
+
+    mv -f python-coverage-full.tmp python-coverage-full.info
+}
+
+# gen_python_full_html_report || true
+
+# python diff html report
+
+function gen_python_diff_html_report() {
+    if [ "${GIT_PR_ID}" != "" ]; then
+        COVERAGE_DIFF_PATTERN="`python ${PADDLE_ROOT}/tools/coverage/pull_request.py files ${GIT_PR_ID}`"
+
+        python ${PADDLE_ROOT}/tools/coverage/pull_request.py diff ${GIT_PR_ID} > python-git-diff.out
+    fi
+
+    lcov --extract python-coverage-full.info \
+        ${COVERAGE_DIFF_PATTERN} \
+        -o python-coverage-diff.info \
+        --rc lcov_branch_coverage=0
+
+    python ${PADDLE_ROOT}/tools/coverage/coverage_diff.py python-coverage-diff.info python-git-diff.out > python-coverage-diff.tmp
+
+    mv -f python-coverage-diff.tmp python-coverage-diff.info
+
+    genhtml -o python-coverage-diff \
+        -t 'Python Diff Coverage' \
+        --no-function-coverage \
+        --no-branch-coverage \
+        --ignore-errors source \
+        python-coverage-diff.info
+}
+
+# gen_python_diff_html_report || true
+
+# assert coverage lines
+
+function covinfo_combine_full(){
+    if [ -f "other-coverage.info" ] && [ -s "other-coverage.info" ];then
+        if [ -f "infer-coverage.info" ] && [ -s "infer-coverage.info" ];then
+            lcov -a other-coverage.info -a infer-coverage.info -o coverage.info
+        else
+            mv other-coverage.info coverage.info
+        fi
+    elif [ -f "infer-coverage.info" ] && [ -s "infer-coverage.info" ];then
+        mv infer-coverage.info coverage.info
+    else
+        echo "Cannot found coverage.info"
+    fi
+
+    if [ -f "other-python-coverage.info" ] && [ -s "other-python-coverage.info" ];then
+        if [ -f "infer-python-coverage.info" ] && [ -s "other-python-coverage.info" ];then
+            lcov -a other-python-coverage.info -a infer-python-coverage.info -o python-coverage.info
+        else
+            mv other-python-coverage.info python-coverage.info
+        fi
+    elif [ -f "infer-python-coverage.info" ] && [ -s "infer-python-coverage.info" ];then
+        mv infer-python-coverage.info python-coverage.info
+    else
+        echo "Cannot found python coverage.info"
+    fi
+    gen_python_full_html_report || true
+    gen_full_html_report || true
+}
+
+function cov_rate_judge(){
+    echo "Assert CPP Diff Coverage"
+    python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py coverage-diff.info 0.9 || COVERAGE_LINES_ASSERT=1
+
+    echo "Assert Python Diff Coverage"
+
+    if [ ${WITH_XPU:-OFF} == "ON" ]; then
+        echo "XPU has no python coverage!"
+    else
+        if [[ python-coverage-diff.info ]];then
+            python ${PADDLE_ROOT}/tools/coverage/coverage_lines.py python-coverage-diff.info 0.9 || PYTHON_COVERAGE_LINES_ASSERT=1
+        fi
+    fi
+    if [ "$COVERAGE_LINES_ASSERT" = "1" ] || [ "$PYTHON_COVERAGE_LINES_ASSERT" = "1" ]; then
+        echo "exit 9" > /tmp/paddle_coverage.result
+        exit 9
+    fi
+}
+
+function print_usage() {
+    echo -e "\n${RED}Usage${NONE}:
+    ${BOLD}${SCRIPT_NAME}${NONE} [OPTION]"
+
+    echo -e "\n${RED}Options${NONE}:
+    ${BLUE}gen_cov_info${NONE}: generate coverage info
+    ${BLUE}test${NONE}: coverage info combine
+    "
+}
+
+function main () {
+    local CMD=$1
+    lcov_init
+    case $CMD in
+      gen_cov_info)
+        gen_cpp_covinfo
+        gen_py_covinfo
+        ;;
+      combine_cov_info)
+      covinfo_combine_full
+      gen_diff_html_report || true
+      gen_python_diff_html_report || true
+      cov_rate_judge
+        ;;
+      *)
+        print_usage
+        exit 1
+        ;;
+      esac
+}
+
+main $@

tools/coverage/pull_request.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+usage: pull_request.py files pull_id
+       pull_request.py diff  pull_id
+"""
+
+import argparse
+import os
+
+from github import Github
+
+token = os.getenv('GITHUB_API_TOKEN')
+
+
+def get_pull(pull_id):
+    """
+    Args:
+        pull_id (int): Pull id.
+
+    Returns:
+        github.PullRequest.PullRequest
+    """
+    github = Github(token, timeout=60)
+    repo = github.get_repo('PaddlePaddle/Paddle')
+    pull = repo.get_pull(pull_id)
+
+    return pull
+
+
+def get_files(args):
+    """
+    Args:
+        args (argparse.ArgumentParser().parse_args()): Arguments.
+
+    Returns:
+        None.
+    """
+
+    pull = get_pull(args.pull_id)
+
+    for file in pull.get_files():
+        print(f'/paddle/{file.filename}')
+
+
+def diff(args):
+    """
+    Args:
+        args (argparse.ArgumentParser().parse_args()): Arguments.
+
+    Returns:
+        None.
+    """
+
+    pull = get_pull(args.pull_id)
+
+    for file in pull.get_files():
+        print(f'+++ {file.filename}')
+        print(file.patch)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers()
+
+    files_parser = subparsers.add_parser('files')
+    files_parser.add_argument('pull_id', type=int)
+    files_parser.set_defaults(func=get_files)
+
+    diff_parser = subparsers.add_parser('diff')
+    diff_parser.add_argument('pull_id', type=int)
+    diff_parser.set_defaults(func=diff)
+
+    args = parser.parse_args()
+    args.func(args)

tools/coverage/python_coverage.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+usage: python_coverage.py > python-coverage.info
+"""
+
+from os import path
+from xml.etree import ElementTree
+
+tree = ElementTree.parse('python-coverage.xml')
+root = tree.getroot()
+
+sources = root.findall('sources/source')
+
+source = sources[-1].text
+
+for clazz in root.findall('packages/package/classes/class'):
+    clazz_filename = clazz.attrib.get('filename')
+    clazz_filename = path.join(source, clazz_filename)
+
+    if clazz_filename.startswith('/paddle/build/python/'):
+        clazz_filename = (
+            '/paddle/python/' + clazz_filename[len('/paddle/build/python/') :]
+        )
+
+    if not path.exists(clazz_filename):
+        continue
+
+    print('TN:')
+    print(f'SF:{clazz_filename}')
+
+    branch_index = 0
+
+    for line in clazz.findall('lines/line'):
+        line_hits = line.attrib.get('hits')
+        line_number = line.attrib.get('number')
+
+        line_branch = line.attrib.get('branch')
+        line_condition_coverage = line.attrib.get('condition-coverage')
+        line_missing_branches = line.attrib.get('missing-branches')
+
+        if line_branch == 'true':
+            line_condition_coverage = line_condition_coverage.split()
+            line_condition_coverage = line_condition_coverage[1].strip('()')
+            line_condition_coverage = line_condition_coverage.split('/')
+
+            taken = line_condition_coverage[0]
+            taken = int(taken)
+
+            for _ in range(taken):
+                print(f'BRDA:{line_number},{0},{branch_index},{line_hits}')
+                branch_index += 1
+
+            if line_missing_branches:
+                for missing_branch in line_missing_branches.split(','):
+                    print(f'BRDA:{line_number},{0},{branch_index},{0}')
+                    branch_index += 1
+
+        print(f'DA:{line_number},{line_hits}')
+
+    print('end_of_record')

tools/diff_api.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import difflib
+import sys
+
+with open(sys.argv[1], 'r') as f:
+    origin = f.read()
+    origin = origin.splitlines()
+
+with open(sys.argv[2], 'r') as f:
+    new = f.read()
+    new = new.splitlines()
+
+differ = difflib.Differ()
+result = differ.compare(origin, new)
+
+error = False
+diffs = []
+for each_diff in result:
+    if each_diff[0] in ['-', '?']:  # delete or change API is not allowed
+        error = True
+    elif each_diff[0] == '+':
+        error = True
+
+    if each_diff[0] != ' ':
+        diffs.append(each_diff)
+'''
+If you modify/add/delete the API files, including code and comment,
+please follow these steps in order to pass the CI:
+
+  1. cd ${paddle_path}, compile paddle;
+  2. pip install build/python/dist/(build whl package);
+  3. run "python tools/print_signatures.py paddle.base> paddle/fluid/API.spec"
+'''
+if error:
+    print('API Difference is: ')
+    for each_diff in diffs:
+        print(each_diff)

tools/diff_unittest.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+try:
+    f1 = open(sys.argv[1], 'r')
+    origin = f1.read()
+    origin = origin.splitlines()
+except:
+    sys.exit(0)
+else:
+    f1.close()
+
+try:
+    f2 = open(sys.argv[2], 'r')
+    new = f2.read()
+    new = new.splitlines()
+except:
+    sys.exit(0)
+else:
+    f2.close()
+
+error = False
+diffs = []
+for i in origin:
+    if i not in new:
+        error = True
+        diffs.append(i)
+'''
+If you delete the unit test, such as commenting it out,
+please ask for approval of one RD below for passing CI:
+
+    - kolinwei(recommended)
+'''
+if error:
+    for each_diff in diffs:
+        print(f"- {each_diff}")

tools/diff_use_default_grad_op_maker.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+os.environ['CUDA_VISIBLE_DEVICES'] = ''
+
+import sys
+
+from paddle import base
+
+
+def generate_spec(filename):
+    with open(filename, 'w') as f:
+        ops = base.core._get_use_default_grad_op_desc_maker_ops()
+        f.writelines(op + '\n' for op in ops)
+
+
+def read_spec(filename):
+    with open(filename, 'r') as f:
+        return {line.strip() for line in f}
+
+
+def get_spec_diff(dev_filename, pr_filename):
+    ops_dev = read_spec(dev_filename)
+    ops_pr = read_spec(pr_filename)
+
+    added_ops = []
+    removed_ops = []
+
+    for op in ops_pr:
+        if op not in ops_dev:
+            added_ops.append(op)
+        else:
+            removed_ops.append(op)
+
+    return added_ops
+
+
+if len(sys.argv) == 2:
+    generate_spec(sys.argv[1])
+elif len(sys.argv) == 3:
+    added_ops = get_spec_diff(sys.argv[1], sys.argv[2])
+    if added_ops:
+        print(added_ops)
+else:
+    print(
+        'Usage 1: python diff_use_default_grad_op_maker.py [filepath] to generate new spec file\n'
+        'Usage 2: python diff_use_default_grad_op_maker.py [dev_filepath] [pr_filepath] to diff spec file'
+    )
+    sys.exit(1)

tools/dockerfile/Dockerfile.centos

@@ -0,0 +1,72 @@
+# NOTE The manylinux1 policy mandates CentOS-5. We replace it with CentOS-6 in
+# order to satisfy the build of capnproto library (a nupic.core dependency),
+# which requires some headers and symbols not present on CentOS-5 (e.g.,
+# signalfd.h, pipe2, O_NONBLOCK, SOCK_NONBLOCK, etc.). See
+# https://github.com/sandstorm-io/capnproto/issues/350.
+FROM nvidia/cuda:<baseimg>
+MAINTAINER Numenta, based on the ManyLinux project
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+ENV PATH /opt/rh/devtoolset-2/root/usr/bin:$PATH
+ENV LD_LIBRARY_PATH /usr/local/ssl/lib:/opt/rh/devtoolset-2/root/usr/lib64:/opt/rh/devtoolset-2/root/usr/lib:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH}
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+
+RUN yum update -y && yum install -y bzip2 gettext-devel sqlite-devel zlib-devel openssl-devel pcre-devel vim tk-devel libtool xz graphviz wget curl-devel patch perl swig
+COPY build_scripts /build_scripts
+RUN bash build_scripts/build.sh
+#RUN bash build_scripts/install_nccl2.sh
+RUN bash build_scripts/install_trt.sh
+RUN rm -rf build_scripts
+RUN ln -s /usr/local/ssl/include/openssl /usr/include
+
+# git 2.17.1
+RUN wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz && \
+  tar -xvf git-2.17.1.tar.gz && \
+  cd git-2.17.1 && \
+  ./configure --with-openssl CFLAGS="-Dsocklen_t=uint32_t" --prefix=/usr/local && \
+  make -j8 && make install
+
+ENV SSL_CERT_FILE=/opt/_internal/certs.pem
+ENV GOROOT=/usr/local/go GOPATH=/root/gopath
+ENV PATH=/usr/local/ssl:${GOROOT}/bin:${GOPATH}/bin:${PATH}
+ENV LIBRARY_PATH=/usr/local/ssl/lib:$LIBRARY_PATH
+
+
+# for paddle
+RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.15.12.linux-amd64.tar.gz | \
+    tar -xz -C /usr/local && \
+    mkdir /root/gopath && \
+    mkdir /root/gopath/bin && \
+    mkdir /root/gopath/src
+
+
+RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt
+
+
+RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install setuptools -U && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install setuptools -U && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install setuptools -U
+
+RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install -r /root/requirements.txt && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install -r /root/requirements.txt && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install -r /root/requirements.txt && \
+    go get github.com/Masterminds/glide && \
+    rm -rf /root/requirements.txt
+
+RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install pre-commit 'ipython==5.3.0' && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install pre-commit 'ipython==5.3.0'
+
+
+# ccache 4.8.2
+RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
+    tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
+    make -j8 && make install && \
+    ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
+    cd ../../ && rm -rf ccache-4.8.2.tar.gz
+
+CMD ["bash", "/paddle/paddle/scripts/docker/build.sh"]

tools/dockerfile/Dockerfile.develop.dtk

@@ -0,0 +1,116 @@
+# Docker Image for PaddlePaddle Hygon DCU2
+
+FROM sugonhub/kylin:v10-dev
+LABEL maintainer="PaddlePaddle Authors <paddle-dev@baidu.com>"
+
+RUN yum install -y bzip2-devel openssh-server elfutils-devel diffutils libtool iproute \
+        blas-devel lapack-devel make git patch unzip bison hostname yasm libsndfile-devel \
+        automake which file net-tools zlib-devel libffi-devel vim tk-devel tkinter rpm-build \
+        sqlite-devel xz-devel wget curl-devel initscripts mesa-libGL numactl-devel pcre-devel \
+        openssl-devel libjpeg-turbo-devel libpng-devel ninja-build pciutils libzstd-devel zstd
+
+COPY tools/dockerfile/build_scripts /build_scripts
+RUN bash /build_scripts/install_gcc.sh gcc82
+RUN ln -sf /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc
+RUN ln -sf /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++
+RUN ln -sf /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc
+RUN ln -sf /usr/local/gcc-8.2/bin/g++ /usr/bin/g++
+ENV PATH=/usr/local/gcc-8.2/bin:$PATH
+
+# workdir
+WORKDIR /opt
+
+# cmake 3.27.7
+RUN wget -q https://cmake.org/files/v3.27/cmake-3.27.7-linux-x86_64.sh && \
+    chmod +x cmake-3.27.7-linux-x86_64.sh && mkdir -p /opt/cmake-3.27.7 && \
+    ./cmake-3.27.7-linux-x86_64.sh --prefix=/opt/cmake-3.27.7 --skip-license && \
+    rm -rf cmake-3.27.7-linux-x86_64.sh && rm -rf /opt/cmake
+RUN rm -rf /usr/bin/cmake /usr/bin/cmake3 && \
+    ln -s /opt/cmake-3.27.7/bin/cmake /usr/bin/cmake && \
+    ln -s /opt/cmake-3.27.7/bin/cmake /usr/bin/cmake3
+ENV PATH=/opt/cmake-3.27.7/bin:${PATH}
+
+# Python 3.10
+RUN wget -q https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz && \
+    tar xzf Python-3.10.14.tgz && cd Python-3.10.14 && \
+    CFLAGS="-Wformat" ./configure --prefix=/usr/local/ --enable-shared > /dev/null && \
+    make -j16 > /dev/null && make altinstall > /dev/null && ldconfig && \
+    cd ../ && rm -rf Python-3.10.14 && rm -rf Python-3.10.14.tgz
+ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
+
+# create venv and activate
+RUN /usr/local/bin/python3.10 -m venv /opt/py310
+# update env
+ENV PATH=/opt/py310/bin:$PATH
+
+# upgrade pip
+RUN pip3.10 install --upgrade pip setuptools wheel
+
+# install pylint and pre-commit
+RUN pip3.10 install pre-commit==2.17.0 pylint pytest astroid isort coverage qtconsole distro
+
+RUN pip3.10 install attrs pyyaml pathlib2 scipy requests psutil Cython clang-format==13.0.0 PyGithub
+
+# install Paddle requirement
+RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O requirements.txt && \
+    pip3.10 install -r requirements.txt && rm -rf requirements.txt
+
+RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/unittest_py/requirements.txt -O requirements.txt && \
+    pip3.10 install -r requirements.txt && rm -rf requirements.txt
+
+# git credential to skip password typing
+RUN git config --global credential.helper store && \
+    git config --global pull.rebase false
+
+# Fix locales to en_US.UTF-8
+RUN yum -y install glibc-locale-source glibc-langpack-en
+RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+
+# patchelf 0.14.5 - https://github.com/NixOS/patchelf/pull/216
+RUN wget -q https://github.com/NixOS/patchelf/archive/refs/tags/0.14.5.tar.gz && \
+    tar xzf 0.14.5.tar.gz && cd patchelf-0.14.5 && \
+    ./bootstrap.sh  > /dev/null && ./configure > /dev/null && \
+    make -j16 > /dev/null && make install > /dev/null && \
+    cd .. && rm -rf patchelf-0.14.5 && rm -rf 0.14.5.tar.gz
+
+# ccache 4.6.3
+RUN wget -q https://github.com/ccache/ccache/releases/download/v4.6.3/ccache-4.6.3.tar.gz && \
+    tar xf ccache-4.6.3.tar.gz && mkdir /usr/local/ccache-4.6.3 && cd ccache-4.6.3 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_STORAGE_BACKEND=OFF \
+    -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.6.3 ..  > /dev/null && \
+    make -j16 > /dev/null && make install > /dev/null && \
+    cd ../../ && rm -rf ccache-4.6.3.tar.gz && rm -rf ccache-4.6.3 && \
+    ln -s /usr/local/ccache-4.6.3/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE=50G \
+    CCACHE_LIMIT_MULTIPLE=0.8 \
+    CCACHE_SLOPPINESS=clang_index_store,time_macros,include_file_mtime
+
+# configure ssh
+RUN sed -i "s/^#PermitRootLogin/PermitRootLogin/" /etc/ssh/sshd_config && \
+    sed -i "s/^#PubkeyAuthentication/PubkeyAuthentication/" /etc/ssh/sshd_config && \
+    sed -i "s/^#RSAAuthentication/RSAAuthentication/" /etc/ssh/sshd_config && \
+    sed -i "s/#UseDNS .*/UseDNS no/" /etc/ssh/sshd_config
+RUN ssh-keygen -A
+
+# yum clean
+RUN yum clean all && \
+    rm -rf /var/cache/yum && \
+    rm -rf /var/lib/yum/yumdb && \
+    rm -rf /var/lib/yum/history
+
+# Install DTK
+RUN wget -q --no-proxy https://cancon.hpccube.com:65024/file/1/DTK-24.04.1/CentOS7.6/DTK-24.04.1-CentOS7.6-x86_64.tar.gz --no-check-certificate && \
+    tar zxf DTK-24.04.1-CentOS7.6-x86_64.tar.gz && rm -rf DTK-24.04.1-CentOS7.6-x86_64.tar.gz
+# Replace if you use other device type, e.g. Z100, Z100L, K100
+RUN wget -q https://paddle-device.bj.bcebos.com/dcu/hyhal-K100AI.tar.gz && \
+    tar zxf hyhal-K100AI.tar.gz && rm -rf hyhal-K100AI.tar.gz
+RUN echo "source /opt/dtk-24.04.1/env.sh" >> /root/.bashrc
+# Disable compile warnings
+RUN sed -i '74d' /opt/dtk-24.04.1/include/rocrand/rocrand_common.h
+
+# generate core dump
+RUN echo "kernel.core_pattern=core_%e_%p_%t" >>  /etc/sysctl.conf && \
+    echo "kernel.core_uses_pid=0" >>  /etc/sysctl.conf
+
+EXPOSE 22

tools/dockerfile/Dockerfile.develop.npu

@@ -0,0 +1,87 @@
+# Docker Image for PaddlePaddle Ascend NPU
+
+FROM registry.baidubce.com/device/paddle-cpu:ubuntu20-npu-base-x86_64-gcc84
+LABEL maintainer="PaddlePaddle Authors <paddle-dev@baidu.com>"
+
+ARG CANN_VERSION=8.0.T113
+ARG SYSTEM=x86_64
+ARG NPU_VERSION=910b
+
+# HwHiAiUser
+RUN groupadd -g 1000 HwHiAiUser && \
+    useradd -u 1000 -g 1000 -m -d /home/HwHiAiUser HwHiAiUser
+
+RUN mkdir -p /usr/local/Ascend/driver
+WORKDIR /usr/local/Ascend
+
+# install CANN requirement
+# https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/700alpha003/softwareinstall/instg/instg_0026.html
+RUN apt-get update -y && apt-get install -y zlib1g zlib1g-dev libsqlite3-dev openssl libssl-dev libffi-dev libbz2-dev \
+    libxslt1-dev unzip pciutils net-tools libblas-dev gfortran libblas3 liblapack-dev liblapack3 libopenblas-dev zstd
+
+RUN pip3.10 install --upgrade pip setuptools wheel
+
+RUN pip3.10 install 'numpy>=1.19.2' 'decorator>=4.4.0' 'sympy>=1.5.1' 'cffi>=1.12.3' 'protobuf>=3.13.0'
+
+RUN pip3.10 install attrs pyyaml pathlib2 scipy requests psutil absl-py
+
+# update envs for driver
+ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64:$LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64/common:$LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64/driver:$LD_LIBRARY_PATH
+
+# Install Ascend toolkit
+RUN wget -q --no-proxy https://paddle-ascend.bj.bcebos.com/cvmodel/ascend-materials/Ascend-cann-toolkit_${CANN_VERSION}_linux-${SYSTEM}.run --no-check-certificate && \
+    chmod +x Ascend-cann-toolkit_${CANN_VERSION}_linux-*.run && \
+    ./Ascend-cann-toolkit_${CANN_VERSION}_linux-*.run --install --quiet && \
+    echo "source /usr/local/Ascend/ascend-toolkit/set_env.sh" >>~/.bashrc && \
+    rm -rf Ascend-cann-toolkit_${CANN_VERSION}_linux-*.run
+
+# Install Ascend Kernels
+ARG ASCEND_KLS=Ascend-cann-kernels-${NPU_VERSION}_${CANN_VERSION}_linux-${SYSTEM}.run
+RUN wget -q --no-proxy https://paddle-ascend.bj.bcebos.com/cvmodel/ascend-materials/${ASCEND_KLS} --no-check-certificate && \
+    chmod +x ${ASCEND_KLS} && ./${ASCEND_KLS} --install --quiet && rm -rf ${ASCEND_KLS}
+
+# Install Ascend nnal
+RUN wget -q --no-proxy https://paddle-ascend.bj.bcebos.com/cvmodel/ascend-materials/Ascend-cann-nnal_${CANN_VERSION}_linux-${SYSTEM}.run --no-check-certificate && \
+    chmod +x Ascend-cann-nnal_${CANN_VERSION}_linux-${SYSTEM}.run && \
+    . /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    ./Ascend-cann-nnal_${CANN_VERSION}_linux-${SYSTEM}.run --install --quiet &&\
+    rm -rf Ascend-cann-nnal_${CANN_VERSION}_linux-${SYSTEM}.run && \
+    echo "source /usr/local/Ascend/nnal/atb/set_env.sh" >> /root/.bashrc
+
+# install post process ops
+RUN wget -q --no-proxy https://paddle-ascend.bj.bcebos.com/code-share-master.zip --no-check-certificate && \
+    . /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    unzip code-share-master.zip && \
+    cd code-share-master/build && bash build_ops.sh > /dev/null && \
+    chmod +x aie_ops.run && ./aie_ops.run --extract=/usr/local/Ascend/ && \
+    rm -rf /usr/local/Ascend/code-share-master
+
+# update env for ascendc
+ENV ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/vendors/aie_ascendc
+
+# DEV image should open error level log
+# 0 debug; 1 info; 2 warning; 3 error; 4 null
+ENV ASCEND_GLOBAL_LOG_LEVEL=3
+
+# environment for HCCL
+ENV HCCL_CONNECT_TIMEOUT=7200
+ENV HCCL_WHITELIST_DISABLE=1
+ENV HCCL_SECURITY_MODE=1
+ENV HCCL_BUFFSIZE=120
+
+# environment for PaddlePaddle
+ENV FLAGS_npu_storage_format=0
+ENV FLAGS_use_stride_kernel=0
+ENV FLAGS_allocator_strategy=naive_best_fit
+ENV PADDLE_XCCL_BACKEND=npu
+
+# map this folder in docker run
+RUN rm -rf /usr/local/Ascend/driver
+
+# Clean
+RUN apt-get clean -y
+RUN pip cache purge
+
+EXPOSE 22

tools/dockerfile/Dockerfile.develop.xre

@@ -0,0 +1,126 @@
+# Docker Image for PaddlePaddle Kunlun XPU
+
+FROM ubuntu:20.04
+LABEL maintainer="PaddlePaddle Authors <paddle-dev@baidu.com>"
+
+RUN apt-get update && apt-get install -y apt-utils
+RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && add-apt-repository ppa:ubuntu-toolchain-r/test
+RUN apt-get update && apt-get install -y curl wget vim git unzip unrar tar ntp xz-utils libssl-dev bzip2 gzip make automake \
+    coreutils language-pack-zh-hans libsm6 libxext6 libxrender-dev libgl1-mesa-glx libsqlite3-dev libopenblas-dev liblapack3 \
+    bison libjpeg-dev zlib1g zlib1g-dev swig locales net-tools libtool numactl libnuma-dev liblzma-dev libbz2-dev libblas-dev \
+    openssl openssh-server libffi-dev pciutils libblas3 liblapack-dev libzstd-dev default-jre libgcc-s1 gcc g++ gfortran gdb
+
+# workdir
+WORKDIR /opt
+
+# GCC 8.4
+RUN apt-get install -y gcc-8 g++-8 gfortran-8
+RUN update-alternatives --install /usr/bin/g++  g++  /usr/bin/g++-8  90 && \
+    update-alternatives --install /usr/bin/gcc  gcc  /usr/bin/gcc-8  90 && \
+    update-alternatives --install /usr/bin/gfortran  gfortran  /usr/bin/gfortran-8  90
+
+# cmake 3.27.7
+RUN wget -q https://cmake.org/files/v3.27/cmake-3.27.7-linux-x86_64.sh && \
+    chmod +x cmake-3.27.7-linux-x86_64.sh && mkdir -p /opt/cmake-3.27.7 && \
+    ./cmake-3.27.7-linux-x86_64.sh --prefix=/opt/cmake-3.27.7 --skip-license && \
+    rm -rf cmake-3.27.7-linux-x86_64.sh
+ENV PATH=/opt/cmake-3.27.7/bin:${PATH}
+
+# default python version
+ARG PY_VERSION=3.10
+RUN apt-get update && apt-get install -y python3.10-distutils python3.10 python3.10-dev
+
+# install pip
+RUN curl -s -q https://bootstrap.pypa.io/get-pip.py | /usr/bin/python3.10
+
+# set default python
+RUN rm -rf /usr/bin/python3 && ln -s /usr/bin/python${PY_VERSION} /usr/bin/python3 && \
+    rm -rf /usr/bin/python  && ln -s /usr/bin/python${PY_VERSION} /usr/bin/python
+
+# install pylint and pre-commit
+RUN pip3.10 install pre-commit==2.17.0 pylint pytest astroid isort coverage qtconsole distro
+RUN pip3.10 install attrs pyyaml pathlib2 scipy requests psutil Cython clang-format==13.0.0 setuptools==76.1.0
+
+# add more libs
+RUN apt-get update && apt-get install libprotobuf-dev protobuf-compiler libprotoc-dev lsof libgeos-dev zstd \
+    pkg-config libhdf5-103 libhdf5-dev lrzsz libsndfile1 tree ninja-build -y
+
+# install Paddle requirement
+RUN wget --no-check-certificate https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O requirements.txt && \
+    pip3.10 install -r requirements.txt -i https://pypi.org/simple && rm -rf requirements.txt
+
+RUN wget --no-check-certificate https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/unittest_py/requirements.txt -O requirements.txt && \
+    pip3.10 install -r requirements.txt -i https://pypi.org/simple && rm -rf requirements.txt
+
+# git credential to skip password typing
+RUN git config --global credential.helper store
+
+# Fix locales to en_US.UTF-8
+RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+
+# patchelf 0.14.5 - https://github.com/NixOS/patchelf/pull/216
+RUN wget -q --no-check-certificate https://github.com/NixOS/patchelf/archive/refs/tags/0.14.5.tar.gz && \
+    tar xzf 0.14.5.tar.gz && cd patchelf-0.14.5 && \
+    ./bootstrap.sh  > /dev/null && ./configure > /dev/null && \
+    make -j16 > /dev/null && make install > /dev/null && \
+    cd .. && rm -rf patchelf-0.14.5 && rm -rf 0.14.5.tar.gz
+
+# ccache 4.6.3
+RUN wget -q https://github.com/ccache/ccache/releases/download/v4.6.3/ccache-4.6.3.tar.gz && \
+    tar xf ccache-4.6.3.tar.gz && mkdir /usr/local/ccache-4.6.3 && cd ccache-4.6.3 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_STORAGE_BACKEND=OFF \
+    -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.6.3 ..  > /dev/null && \
+    make -j16 > /dev/null && make install > /dev/null && \
+    cd ../../ && rm -rf ccache-4.6.3.tar.gz && rm -rf ccache-4.6.3 && \
+    ln -s /usr/local/ccache-4.6.3/bin/ccache /usr/local/bin/ccache
+ENV CCACHE_MAXSIZE=80G \
+    CCACHE_LIMIT_MULTIPLE=0.8 \
+    CCACHE_SLOPPINESS=clang_index_store,time_macros,include_file_mtime
+
+# Install XRE 5.0.21.21
+WORKDIR /opt
+ARG XRE_VERSION=5.0.21.21
+ARG XRE_INSTALL=/usr/local/xpu-${XRE_VERSION}
+RUN wget -q https://klx-sdk-release-public.su.bcebos.com/xre/kl3-release/${XRE_VERSION}/xre-ubuntu_2004-x86_64-${XRE_VERSION}.tar.gz --no-check-certificate && \
+    tar -zxf xre-ubuntu_2004-x86_64-${XRE_VERSION}.tar.gz && \
+    mkdir -p ${XRE_INSTALL} && \
+    mv -f /opt/xre-ubuntu_2004-x86_64-${XRE_VERSION}/* ${XRE_INSTALL}/ && \
+    ln -sf ${XRE_INSTALL} /usr/local/xpu && \
+    ln -sf ${XRE_INSTALL}/bin/xpu_smi /usr/local/bin/xpu_smi && \
+    rm -rf xre-ubuntu_2004-x86_64-${XRE_VERSION}.tar.gz
+ENV PATH=${XRE_INSTALL}/bin:$PATH
+
+# Update RDMA
+RUN wget "https://su.bcebos.com/v1/klx-sdk-release-public/xccl/resource/MLNX_OFED_LINUX-24.10-2.1.8.0-ubuntu20.04-x86_64.tgz?authorization=bce-auth-v1%2FALTAKlxQapmxlH5xQFcp7rEkCr%2F2025-05-28T02%3A24%3A09Z%2F-1%2Fhost%2Fbaad25d036a6eb868dad8ab19468884e5016507fdd6879fe1259db4bbef694e6" --no-check-certificate \
+    -O MLNX_OFED_LINUX-24.10-2.1.8.0-ubuntu20.04-x86_64.tgz && \
+    tar -zxf MLNX_OFED_LINUX-24.10-2.1.8.0-ubuntu20.04-x86_64.tgz && \
+    cd MLNX_OFED_LINUX-24.10-2.1.8.0-ubuntu20.04-x86_64 && \
+    ./mlnxofedinstall --user-space-only --skip-distro-check --without-fw-update --force && \
+    rm -rf MLNX_OFED_LINUX-24.10-2.1.8.0-ubuntu20.04-x86_64.tgz && \
+    rm -rf MLNX_OFED_LINUX-24.10-2.1.8.0-ubuntu20.04-x86_64
+
+
+# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
+RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && \
+    sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && \
+    sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
+
+# Set language environment in .bashrc
+RUN echo "export LANG=en_US.UTF-8" >> /root/.bashrc && \
+    echo "export LANGUAGE=en_US.UTF-8" >> /root/.bashrc && \
+    echo "export LC_ALL=en_US.UTF-8" >> /root/.bashrc
+
+CMD source ~/.bashrc
+
+# /proc/sys/kernel/core_pattern
+RUN mkdir -p /var/core
+
+
+# Clean
+RUN apt-get clean -y
+RUN pip cache purge
+
+EXPOSE 22

tools/dockerfile/Dockerfile.develop.xre.ubuntu22

@@ -0,0 +1,158 @@
+# Docker Image for PaddlePaddle Kunlun XPU P800
+
+FROM ubuntu:22.04
+LABEL maintainer="PaddlePaddle Authors <paddle-dev@baidu.com>"
+
+RUN apt-get update && apt-get install -y apt-utils
+RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    add-apt-repository ppa:ubuntu-toolchain-r/test
+RUN apt-get update && apt-get install -y curl wget vim git unzip unrar tar ntp xz-utils libssl-dev bzip2 gzip make automake \
+    coreutils language-pack-zh-hans libsm6 libxext6 libxrender-dev libgl1-mesa-glx libsqlite3-dev libopenblas-dev liblapack3 \
+    bison libjpeg-dev zlib1g zlib1g-dev swig locales net-tools libtool numactl libnuma-dev liblzma-dev libbz2-dev libblas-dev \
+    openssl openssh-server libffi-dev pciutils libblas3 liblapack-dev libzstd-dev default-jre libgcc-s1 gdb
+
+# workdir
+WORKDIR /opt
+
+# GCC 12
+RUN rm -f /usr/bin/gcc && rm -f /usr/bin/g++
+RUN apt-get install -y gcc-12 g++-12 gfortran-12
+RUN ln -s /usr/bin/gcc-12 /usr/bin/gcc
+RUN ln -s /usr/bin/g++-12 /usr/bin/g++
+RUN ln -s /usr/bin/gfortran-12 /usr/bin/gfortran
+
+# patchelf 0.15.0 - https://github.com/NixOS/patchelf/pull/216
+RUN wget -q --no-check-certificate https://github.com/NixOS/patchelf/archive/refs/tags/0.15.0.tar.gz && \
+    tar xzf 0.15.0.tar.gz && cd patchelf-0.15.0 && \
+    ./bootstrap.sh  > /dev/null && ./configure > /dev/null && \
+    make -j16 > /dev/null && make install > /dev/null && \
+    cd .. && rm -rf patchelf-0.15.0 && rm -rf 0.15.0.tar.gz
+
+# install cmake && ccache
+RUN apt-get remove --purge cmake && apt-get install -y cmake
+# ccache version 4.9.1
+RUN apt-get install -y ccache
+
+# default python version
+RUN apt-get update && apt-get install -y python3.10-distutils python3.10 python3.10-dev && \
+    apt-get install python-is-python3
+# set default python
+RUN rm -rf /usr/bin/python && ln -s /usr/bin/python3.10 /usr/bin/python && \
+    rm -rf /usr/bin/python3 && ln -s /usr/bin/python3.10 /usr/bin/python3
+
+
+# install pip
+RUN wget -q https://bootstrap.pypa.io/get-pip.py
+RUN sed -i 's#"install", "--upgrade", "--force-reinstall"#"install", "--upgrade", "--force-reinstall", "--break-system-packages"#' get-pip.py
+
+RUN python3.10 get-pip.py && \
+    rm -rf /opt/get-pip.py
+
+RUN python3.10 -m pip install setuptools==68.2.0
+
+
+# binutils >= 2.27
+RUN apt-get install -y binutils
+
+
+# git credential to skip password typing
+RUN git config --global credential.helper store
+
+# Fix locales to en_US.UTF-8
+RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+
+#For pre-commit
+RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.10 /usr/local/bin/pip && \
+    rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.10 /usr/local/bin/pip3
+
+RUN python3.10 -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.10 -m pip --no-cache-dir install ipykernel==4.6.0 wheel
+
+# install pylint and pre-commit
+RUN python3.10 -m pip --no-cache-dir install pre-commit==2.17.0 pylint pytest astroid isort coverage qtconsole distro \
+    attrs pyyaml pathlib2 scipy requests psutil Cython clang-format==13.0.0
+
+
+
+# add more libs
+RUN apt-get update && apt-get install libprotobuf-dev protobuf-compiler libprotoc-dev lsof libgeos-dev zstd \
+    pkg-config libhdf5-103 libhdf5-dev lrzsz libsndfile1 tree ninja-build iproute2 iputils-ping -y
+
+# install Paddle requirement
+RUN wget --no-check-certificate https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O requirements.txt && \
+    python3.10 -m pip --no-cache-dir install -r requirements.txt -i https://pypi.org/simple && \
+    rm -rf requirements.txt
+
+RUN wget --no-check-certificate https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/unittest_py/requirements.txt -O requirements.txt && \
+    python3.10 -m pip --no-cache-dir install -r requirements.txt --ignore-installed -i https://pypi.org/simple && \
+    rm -rf requirements.txt
+
+# Install XRE 5.0.21.21
+WORKDIR /opt
+ARG XRE_VERSION=5.0.21.21
+ARG XRE_INSTALL=/usr/local/xpu-${XRE_VERSION}
+RUN wget -q https://klx-sdk-release-public.su.bcebos.com/xre/kl3-release/${XRE_VERSION}/xre-ubuntu_2004-x86_64-${XRE_VERSION}.tar.gz --no-check-certificate && \
+    tar -zxf xre-ubuntu_2004-x86_64-${XRE_VERSION}.tar.gz && \
+    mkdir -p ${XRE_INSTALL} && \
+    mv -f /opt/xre-ubuntu_2004-x86_64-${XRE_VERSION}/* ${XRE_INSTALL}/ && \
+    ln -sf ${XRE_INSTALL} /usr/local/xpu && \
+    ln -sf ${XRE_INSTALL}/bin/xpu_smi /usr/local/bin/xpu_smi && \
+    rm -rf xre-ubuntu_2004-x86_64-${XRE_VERSION}.tar.gz
+ENV PATH=${XRE_INSTALL}/bin:$PATH
+
+RUN wget "https://su.bcebos.com/v1/klx-sdk-release-public/xccl/resource/MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu22.04-x86_64.tgz" --no-check-certificate \
+-O MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu22.04-x86_64.tgz && \
+tar -zxf MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu22.04-x86_64.tgz && \
+cd MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu22.04-x86_64 && \
+./mlnxofedinstall --user-space-only --skip-distro-check --without-fw-update --force && \
+rm -rf /opt/MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu22.04-x86_64.tgz && \
+rm -rf /opt/MLNX_OFED_LINUX-24.01-0.3.3.1-ubuntu22.04-x86_64
+
+# mpich 4.1a1
+RUN wget https://www.mpich.org/static/downloads/4.1a1/mpich-4.1a1.tar.gz && \
+    tar -xvf mpich-4.1a1.tar.gz && \
+    cd mpich-4.1a1 && \
+    ./configure \
+    --prefix=/opt/mpich \
+    --disable-fortran \
+    --with-hwloc=embedded \
+    --with-device=ch3:nemesis \
+    CC=gcc CXX=g++ && \
+    make -j$(nproc) && \
+    make install && \
+rm -rf /opt/mpich-4.1a1.tar.gz /opt/mpich-4.1a1
+
+ENV PATH=/opt/mpich/bin:$PATH
+ENV LD_LIBRARY_PATH=/opt/mpich/lib:$LD_LIBRARY_PATH
+
+# git env
+ENV GIT_SSH_COMMAND="ssh -o HostKeyAlgorithms=+ssh-rsa -o PubkeyAcceptedAlgorithms=+ssh-rsa"
+
+# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
+RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && \
+    sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && \
+    sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
+
+# Set language environment in .bashrc
+RUN echo "export LANG=en_US.UTF-8" >> /root/.bashrc && \
+    echo "export LANGUAGE=en_US.UTF-8" >> /root/.bashrc && \
+    echo "export LC_ALL=en_US.UTF-8" >> /root/.bashrc
+
+CMD source ~/.bashrc
+
+# /proc/sys/kernel/core_pattern
+RUN mkdir -p /var/core
+
+# clang14
+RUN apt-get update &&\
+    apt install -y clang-14
+
+# Clean
+RUN apt-get clean -y
+RUN pip cache purge
+
+EXPOSE 22

tools/dockerfile/Dockerfile.doc

@@ -0,0 +1,39 @@
+ARG BASEIMAGE=registry.baidubce.com/paddleopen/fluiddoc-sphinx:20230828-py310-cuda11.8.0-cudnn8-runtime-ubuntu20.04-v3
+FROM ${BASEIMAGE}
+
+RUN apt-get update \
+ && apt-get install -yq --no-install-recommends \
+        git \
+        jq \
+        wget \
+        pigz \
+        zstd \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+ARG PYPI_MIRROR=https://pypi.tuna.tsinghua.edu.cn/simple
+ARG no_proxy=127.0.0.1,localhost,bcebos.com,tuna.tsinghua.edu.cn,mirror.baidu.com,bce.baidu.com
+#ARG PYPI_MIRROR=https://mirror.baidu.com/pypi/simple
+RUN pip3 install --no-cache-dir -i ${PYPI_MIRROR} pylint && \
+    pip3 install --no-cache-dir -i ${PYPI_MIRROR} pre-commit && \
+    pip3 install --no-cache-dir -i ${PYPI_MIRROR} yapf==0.16.0 && \
+    pip3 install --no-cache-dir -i ${PYPI_MIRROR} importlib-metadata==4.11.0
+
+
+RUN curl -o /entrypoint.sh -Lk https://paddle-dev-tools-open.bj.bcebos.com/fluiddoc-preview/entrypoint-paddle-docs-review.sh && \
+    chmod +x /entrypoint.sh && \
+    curl -o /tmp/linux-bcecmd-0.3.0.zip https://sdk.bce.baidu.com/console-sdk/linux-bcecmd-0.3.0.zip && \
+    python -m zipfile -e /tmp/linux-bcecmd-0.3.0.zip /opt && \
+    chmod +x /opt/linux-bcecmd-0.3.0/bcecmd && \
+    rm /tmp/linux-bcecmd-0.3.0.zip && \
+    curl -o /tmp/boscmdconfig.tgz https://paddle-dev-tools-open.bj.bcebos.com/fluiddoc-preview/boscmdconfig.tgz && \
+    tar xzf /tmp/boscmdconfig.tgz -C /opt/linux-bcecmd-0.3.0/ && \
+    rm /tmp/boscmdconfig.tgz
+# COPY boscmdconfig /opt/linux-bcecmd-0.3.0/boscmdconfig/
+
+ENV BCECMD=/opt/linux-bcecmd-0.3.0/bcecmd
+ENV BCECMD_CONFIG=/opt/linux-bcecmd-0.3.0/boscmdconfig
+# credentials file is empty, please build it if need.
+
+WORKDIR /
+ENTRYPOINT ["bash"]

tools/dockerfile/Dockerfile.ipu

@@ -0,0 +1,100 @@
+# A image for building paddle binaries
+
+# build docker image
+# docker build -t paddlepaddle/paddle:latest-dev-ipu -f tools/dockerfile/Dockerfile.ipu .
+
+# run a container
+# docker run --ulimit memlock=-1:-1 --net=host --cap-add=IPC_LOCK --device=/dev/infiniband/ --ipc=host --rm -it paddlepaddle/paddle:latest-dev-ipu bash
+
+FROM graphcore/poplar:3.0.0
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+
+# ENV variables
+ARG WITH_AVX
+ENV HOME /root
+ENV WITH_AVX=${WITH_AVX:-ON}
+
+# install pkgs
+RUN apt-get update && apt-get install -y apt-utils
+RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
+RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && add-apt-repository ppa:ubuntu-toolchain-r/test
+RUN apt-get update && apt-get install -y curl wget vim git unzip unrar tar xz-utils libssl-dev bzip2 gzip make libgcc-s1 sudo openssh-server \
+    coreutils ntp language-pack-zh-hans python-qt4 libsm6 libxext6 libxrender-dev libgl1-mesa-glx libsqlite3-dev libopenblas-dev \
+    bison graphviz libjpeg-dev zlib1g zlib1g-dev automake locales swig net-tools libtool module-init-tools numactl libnuma-dev \
+    openssl libffi-dev pciutils libblas-dev gfortran libblas3 liblapack-dev liblapack3 default-jre screen tmux gdb lldb gcc g++
+RUN apt-get update && apt-get install -y rdma-core librdmacm1
+
+# Downgrade gcc&&g++
+WORKDIR /usr/bin
+COPY tools/dockerfile/build_scripts /build_scripts
+RUN bash /build_scripts/install_gcc.sh gcc82 && rm -rf /build_scripts
+RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++
+RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc
+RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++
+RUN ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc
+RUN ln -s /usr/local/gcc-8.2/bin/g++ /usr/bin/g++
+ENV PATH=/usr/local/gcc-8.2/bin:$PATH
+
+# install cmake
+WORKDIR /home
+RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.18.0-Linux-x86_64.tar.gz && rm cmake-3.18.0-Linux-x86_64.tar.gz
+ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH
+
+# conda 4.9.2
+WORKDIR /opt
+ARG CONDA_FILE=Miniconda3-py38_23.10.0-1-Linux-x86_64.sh
+RUN cd /opt && wget -q https://repo.anaconda.com/miniconda/${CONDA_FILE} && chmod +x ${CONDA_FILE}
+RUN mkdir /opt/conda && ./${CONDA_FILE} -b -f -p "/opt/conda" && rm -rf ${CONDA_FILE}
+ENV PATH=/opt/conda/bin:${PATH}
+RUN conda init bash && conda install -n base jupyter jupyterlab
+RUN conda install -n base spdlog==1.8.0 -y
+
+# Install Go and glide
+RUN wget -qO- https://paddle-ci.cdn.bcebos.com/go1.8.1.linux-amd64.tar.gz | \
+    tar -xz -C /usr/local && \
+    mkdir /root/gopath && \
+    mkdir /root/gopath/bin && \
+    mkdir /root/gopath/src
+ENV GOROOT=/usr/local/go GOPATH=/root/gopath
+# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT.
+ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin
+# install glide
+RUN curl -s -q https://glide.sh/get | sh
+
+# git credential to skip password typing
+RUN git config --global credential.helper store
+
+# Fix locales to en_US.UTF-8
+RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+
+# install pytest and pre-commit
+RUN /opt/conda/bin/pip install pre-commit pytest protocol PyGithub
+
+# install Paddle requirement
+RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt
+RUN /opt/conda/bin/pip install -r /root/requirements.txt && \
+    rm -rf /root/requirements.txt
+
+RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/unittest_py/requirements.txt -O /root/requirements.txt
+RUN /opt/conda/bin/pip install -r /root/requirements.txt && rm -rf /root/requirements.txt
+
+# Older versions of patchelf limited the size of the files being processed and were fixed in this pr.
+# https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa
+# So install a newer version here.
+RUN wget -q https://paddle-ci.cdn.bcebos.com/patchelf_0.10-2_amd64.deb && \
+    dpkg -i patchelf_0.10-2_amd64.deb
+
+# ccache 3.7.9
+RUN wget https://paddle-ci.gz.bcebos.com/ccache-3.7.9.tar.gz && \
+    tar xf ccache-3.7.9.tar.gz && mkdir /usr/local/ccache-3.7.9 && cd ccache-3.7.9 && \
+    ./configure -prefix=/usr/local/ccache-3.7.9 && \
+    make -j8 && make install && \
+    ln -s /usr/local/ccache-3.7.9/bin/ccache /usr/local/bin/ccache
+
+# # clang-form 3.8.0
+# RUN wget https://paddle-ci.cdn.bcebos.com/clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && \
+#     tar xf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz && cd clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && \
+#     cp -r * /usr/local && cd .. && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04 && rm -rf clang+llvm-3.8.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+
+RUN apt-get clean -y

tools/dockerfile/Dockerfile.release.ubuntu20

@@ -0,0 +1,126 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+FROM <baseimg>
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+<setcuda>
+
+ENV HOME /root
+# Add bash enhancements
+COPY paddle/scripts/docker/root/ /root/
+
+RUN chmod 777 /tmp
+
+RUN apt-key del 7fa2af80
+RUN rm /etc/apt/sources.list.d/*
+RUN apt-key adv --fetch-keys https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
+
+RUN apt-get update --allow-unauthenticated && \
+  apt-get install -y software-properties-common && \
+  add-apt-repository ppa:deadsnakes/ppa && \
+  apt-get update && \
+  apt-get install -y curl wget vim git unzip unrar tar xz-utils libssl-dev bzip2 gzip \
+    coreutils ntp language-pack-zh-hans libsm6 libxext6 libxrender-dev libgl1-mesa-glx \
+    bison graphviz libjpeg-dev zlib1g-dev automake locales swig net-tools libtool kmod
+<install_cpu_package>
+
+# Downgrade gcc&&g++
+WORKDIR /usr/bin
+COPY tools/dockerfile/build_scripts /build_scripts
+RUN bash /build_scripts/install_trt.sh
+# Older versions of patchelf limited the size of the files being processed and were fixed in this pr.
+# # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa
+# # So install a newer version here.
+RUN bash /build_scripts/install_patchelf.sh
+RUN bash /build_scripts/install_gcc.sh gcc121
+RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++
+RUN ln -s /usr/local/gcc-12.1/bin/gcc /usr/local/bin/gcc
+RUN ln -s /usr/local/gcc-12.1/bin/g++ /usr/local/bin/g++
+RUN ln -s /usr/local/gcc-12.1/bin/gcc /usr/bin/gcc
+RUN ln -s /usr/local/gcc-12.1/bin/g++ /usr/bin/g++
+ENV PATH=/usr/local/gcc-12.1/bin:$PATH
+
+RUN bash /build_scripts/install_cudnn.sh cudnn841
+ENV CUDNN_VERSION=8.4.1
+#RUN bash /build_scripts/install_nccl2.sh
+RUN rm -rf /build_script
+
+# install cmake
+WORKDIR /home
+RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.18.0-Linux-x86_64.tar.gz && rm cmake-3.18.0-Linux-x86_64.tar.gz
+ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH
+
+RUN apt-get update && \
+  apt-get install -y python3.10 python3.10-dev python3.10-distutils && \
+  apt-get install python-is-python3 && \
+  rm /usr/bin/python && ln -s /usr/bin/python3.10 /usr/bin/python && \
+  rm /usr/bin/python3 && ln -s /usr/bin/python3.10 /usr/bin/python3
+
+WORKDIR /home
+RUN wget https://files.pythonhosted.org/packages/ef/cc/93f7213b2ab5ed383f98ce8020e632ef256b406b8569606c3f160ed8e1c9/setuptools-68.2.2.tar.gz && tar xf setuptools-68.2.2.tar.gz
+WORKDIR /home/setuptools-68.2.2
+RUN python3.10 setup.py build && python3.10 setup.py install
+
+WORKDIR /home
+RUN wget https://files.pythonhosted.org/packages/1f/7f/4da15e07ccd11c84c1ccc8f6e24288d5e76c99441bf80e315b33542db951/pip-23.3.1.tar.gz && tar -zxf pip-23.3.1.tar.gz
+WORKDIR pip-23.3.1
+RUN python3.10 setup.py install
+
+WORKDIR /home
+RUN rm setuptools-68.2.2.tar.gz pip-23.3.1.tar.gz && \
+    rm -r setuptools-68.2.2 pip-23.3.1
+
+# remove them when apt-get support 2.27 and higher version
+RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \
+    tar -xzf binutils-2.33.1.tar.gz && \
+    cd binutils-2.33.1 && \
+    ./configure && make -j && make install && cd .. && rm -rf binutils-2.33.1 binutils-2.33.1.tar.gz
+
+# Install Go and glide
+RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.17.2.linux-amd64.tar.gz | \
+    tar -xz -C /usr/local && \
+    mkdir /root/gopath && \
+    mkdir /root/gopath/bin && \
+    mkdir /root/gopath/src
+ENV GOROOT=/usr/local/go GOPATH=/root/gopath
+
+# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT.
+ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin
+
+# install glide
+RUN apt-get install -y golang-glide
+
+# git credential to skip password typing
+RUN git config --global credential.helper store
+
+# Fix locales to en_US.UTF-8
+RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+
+RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.10 /usr/local/bin/pip && \
+    rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.10 /usr/local/bin/pip3
+
+COPY ./python/requirements.txt /root/
+RUN pip3.10 --no-cache-dir install -r /root/requirements.txt
+
+# ccache 4.2.0
+RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
+    tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
+    make -j8 && make install && \
+    ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
+    cd ../../ && rm -rf ccache-4.8.2.tar.gz
+
+# clang12
+RUN apt-get update &&\
+    apt install -y clang-12
+
+EXPOSE 22

tools/dockerfile/Dockerfile.ubuntu20

@@ -0,0 +1,200 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+FROM <baseimg>
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+<setcuda>
+
+ENV HOME /root
+# Add bash enhancements
+COPY paddle/scripts/docker/root/ /root/
+
+RUN chmod 777 /tmp
+
+RUN apt-key del 7fa2af80
+RUN rm /etc/apt/sources.list.d/*
+RUN apt-key adv --fetch-keys https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
+
+RUN apt-get update --allow-unauthenticated && \
+  apt-get install -y software-properties-common && \
+  apt-get install -y curl wget vim git unzip pigz zstd unrar tar xz-utils libssl-dev bzip2 gzip \
+    coreutils ntp language-pack-zh-hans libsm6 libxext6 libxrender-dev libgl1-mesa-glx \
+    bison graphviz libjpeg-dev zlib1g-dev automake locales swig net-tools libtool kmod \
+    libbz2-dev libexpat1-dev libffi-dev libgdbm-dev liblzma-dev libncursesw5-dev libreadline-dev \
+    libsqlite3-dev pkg-config tk-dev uuid-dev
+<install_cpu_package>
+
+# Downgrade gcc&&g++
+WORKDIR /usr/bin
+COPY tools/dockerfile/build_scripts /build_scripts
+RUN bash /build_scripts/install_trt.sh
+# Older versions of patchelf limited the size of the files being processed and were fixed in this pr.
+# # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa
+# # So install a newer version here.
+RUN bash /build_scripts/install_patchelf.sh
+#RUN apt-get install -y gcc-8 g++-8
+RUN bash /build_scripts/install_gcc.sh gcc82
+RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++
+RUN ln -sf /usr/local/gcc-8.2/bin/gcc /usr/local/bin/gcc
+RUN ln -sf /usr/local/gcc-8.2/bin/g++ /usr/local/bin/g++
+RUN ln -sf /usr/local/gcc-8.2/bin/gcc /usr/bin/gcc
+RUN ln -sf /usr/local/gcc-8.2/bin/g++ /usr/bin/g++
+RUN ln -sf /usr/local/gcc-8.2/bin/gcc /usr/bin/cc
+ENV PATH=/usr/local/gcc-8.2/bin:$PATH
+
+RUN bash /build_scripts/install_cudnn.sh cudnn841
+ENV CUDNN_VERSION=8.4.1
+RUN rm -rf /build_script
+
+# install cmake
+WORKDIR /home
+RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && tar -zxvf cmake-3.18.0-Linux-x86_64.tar.gz && rm cmake-3.18.0-Linux-x86_64.tar.gz
+ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH
+
+RUN wget -q https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-2.5.1.tar.gz && \
+    tar -xzf mpdecimal-2.5.1.tar.gz && \
+    cd mpdecimal-2.5.1 && \
+    ./configure --prefix=/usr/local > /dev/null && \
+    make -j"$(nproc)" > /dev/null && \
+    make install > /dev/null && \
+    cd .. && \
+    rm -rf mpdecimal-2.5.1 mpdecimal-2.5.1.tar.gz && \
+    ldconfig
+
+# Build free-threaded 3.13 before normal 3.13 so python3.13 stays non-free-threaded.
+RUN set -eux; \
+  build_python() { \
+    py_ver="$1"; \
+    shift; \
+    wget -q "https://www.python.org/ftp/python/${py_ver}/Python-${py_ver}.tgz"; \
+    tar -xzf "Python-${py_ver}.tgz"; \
+    cd "Python-${py_ver}"; \
+    configure_opts="--prefix=/usr/local --enable-shared --enable-optimizations --with-lto --with-system-expat --with-system-libmpdec --with-ensurepip=install"; \
+    if ./configure --help | grep -q -- "--with-system-ffi"; then \
+      configure_opts="${configure_opts} --with-system-ffi"; \
+    fi; \
+    CPPFLAGS="-I/usr/local/include" LDFLAGS="-L/usr/local/lib" CFLAGS="-Wformat" ./configure ${configure_opts} "$@" > /dev/null; \
+    make -j"$(nproc)" > /dev/null; \
+    make altinstall > /dev/null; \
+    cd ..; \
+    rm -rf "Python-${py_ver}" "Python-${py_ver}.tgz"; \
+    ldconfig; \
+  }; \
+  build_python 3.10.0; \
+  build_python 3.11.0; \
+  build_python 3.12.0; \
+  build_python 3.13.0 --disable-gil --with-mimalloc; \
+  build_python 3.13.0
+ENV LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH}
+# Keep Ubuntu's /usr/bin/python3 for system scripts such as lsb_release.
+RUN ln -sf /usr/local/bin/python3.10 /usr/bin/python
+
+RUN for py in 3.10 3.11 3.12 3.13 3.13t; do \
+      python${py} -m pip install --no-cache-dir --upgrade pip; \
+    done && \
+  python3.10 -m pip install setuptools==69.5.1 && \
+  python3.11 -m pip install setuptools==69.5.1 && \
+  python3.12 -m pip install --break-system-packages setuptools==69.5.1 && \
+  for py in 3.10 3.11 3.12 3.13 3.13t; do \
+    printf '#!/bin/sh\nexec /usr/local/bin/python%s -m pip "$@"\n' "$py" > "/usr/local/bin/pip${py}" && \
+    chmod +x "/usr/local/bin/pip${py}"; \
+  done && \
+  ln -sf /usr/local/bin/pip3.10 /usr/local/bin/pip && \
+  ln -sf /usr/local/bin/pip3.10 /usr/local/bin/pip3 && \
+  python3.13 -m pip install setuptools==69.5.1 && \
+  python3.13t -m pip install setuptools==69.5.1
+
+
+RUN wget -q https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.gz && \
+    tar -xzf binutils-2.33.1.tar.gz && \
+    cd binutils-2.33.1 && \
+    ./configure && make -j && make install && cd .. && rm -rf binutils-2.33.1 binutils-2.33.1.tar.gz
+
+# Install Go and glide
+RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.17.2.linux-amd64.tar.gz | \
+    tar -xz -C /usr/local && \
+    mkdir /root/gopath && \
+    mkdir /root/gopath/bin && \
+    mkdir /root/gopath/src
+ENV GOROOT=/usr/local/go GOPATH=/root/gopath
+# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT.
+ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin
+# install glide
+RUN apt-get install -y golang-glide
+
+# git credential to skip password typing
+RUN git config --global credential.helper store
+
+# Fix locales to en_US.UTF-8
+RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+
+RUN pip3.10 --no-cache-dir install ipython==5.3.0 && \
+    pip3.10 --no-cache-dir install ipykernel==4.6.0 wheel && \
+    pip3.11 --no-cache-dir install ipython==5.3.0 && \
+    pip3.11 --no-cache-dir install ipykernel==4.6.0 wheel && \
+    pip3.12 --no-cache-dir install ipython==5.3.0 && \
+    pip3.12 --no-cache-dir install ipykernel==4.6.0 wheel && \
+    python3.13 -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.13 -m pip --no-cache-dir install ipykernel==4.6.0 wheel && \
+    python3.13t -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.13t -m pip --no-cache-dir install ipykernel==4.6.0 wheel
+
+# For PaddleTest CE
+RUN pip3.10 --no-cache-dir install pytest && \
+    pip3.11 --no-cache-dir install pytest && \
+    pip3.12 --no-cache-dir install pytest && \
+    python3.13 -m pip --no-cache-dir install pytest && \
+    python3.13t -m pip --no-cache-dir install pytest
+
+RUN pip3.10 --no-cache-dir install pre-commit==2.17.0 && \
+    pip3.10 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    pip3.11 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    pip3.12 --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.13 -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.13t -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
+
+COPY ./python/requirements.txt  /root/
+COPY ./python/unittest_py/requirements.txt /home/
+COPY ./paddle/scripts/compile_requirements.txt /home/
+
+# Free-threaded Python 3.13 only installs compile helpers because broader
+# requirements currently pull packages without cp313t wheels.
+RUN pip3.10 --no-cache-dir install -r /root/requirements.txt && \
+    pip3.10 --no-cache-dir install -r /home/requirements.txt && \
+    pip3.10 --no-cache-dir install -r /home/compile_requirements.txt && \
+    pip3.11 --no-cache-dir install -r /root/requirements.txt && \
+    pip3.11 --no-cache-dir install -r /home/requirements.txt && \
+    pip3.11 --no-cache-dir install -r /home/compile_requirements.txt && \
+    pip3.12 --no-cache-dir install -r /root/requirements.txt && \
+    pip3.12 --no-cache-dir install -r /home/requirements.txt && \
+    pip3.12 --no-cache-dir install -r /home/compile_requirements.txt && \
+    python3.13 -m pip --no-cache-dir install -r /root/requirements.txt && \
+    python3.13 -m pip --no-cache-dir install -r /home/requirements.txt && \
+    python3.13 -m pip --no-cache-dir install -r /home/compile_requirements.txt && \
+    python3.13t -m pip --no-cache-dir install -r /home/compile_requirements.txt
+
+# clang12
+RUN apt-get update &&\
+    apt install -y clang-12
+
+ARG CC=/usr/bin/clang-12
+ARG CXX=/usr/bin/clang++-12
+
+# ccache 4.2.0
+RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
+    tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
+    make -j8 && make install && \
+    ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
+    cd ../../ && rm -rf ccache-4.8.2.tar.gz
+
+EXPOSE 22

tools/dockerfile/Dockerfile.ubuntu22

@@ -0,0 +1,154 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+FROM <baseimg>
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+<setcuda>
+
+ENV HOME /root
+# Add bash enhancements
+COPY paddle/scripts/docker/root/ /root/
+
+RUN chmod 777 /tmp
+
+RUN mv /etc/apt/sources.list.d/cuda-ubuntu2204-x86_64.list /etc/apt/sources.list.d/cuda-ubuntu2204-x86_64.list.bak
+RUN apt-get update && apt-get install -y curl
+RUN curl https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu2204/x86_64/3bf863cc.pub | gpg --dearmor | tee /usr/share/keyrings/cuda-archive-keyring.gpg
+RUN mv /etc/apt/sources.list.d/cuda-ubuntu2204-x86_64.list.bak /etc/apt/sources.list.d/cuda-ubuntu2204-x86_64.list
+RUN sed -i 's#deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 /#deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 /#' /etc/apt/sources.list.d/cuda-ubuntu2204-x86_64.list
+
+RUN apt-get update --allow-unauthenticated && \
+  apt-get install -y software-properties-common && \
+  add-apt-repository ppa:deadsnakes/ppa && \
+  apt-get update && \
+  apt-get install -y build-essential curl wget vim git unzip pigz zstd unrar tar xz-utils libssl-dev bzip2 gzip \
+    coreutils ntp language-pack-zh-hans libsm6 libxext6 libxrender-dev libgl1 libglx-mesa0 \
+    bison graphviz libjpeg-dev zlib1g-dev automake locales swig net-tools libtool kmod
+<install_cpu_package>
+
+WORKDIR /usr/bin
+COPY tools/dockerfile/build_scripts /build_scripts
+RUN bash /build_scripts/install_trt.sh
+# Older versions of patchelf limited the size of the files being processed and were fixed in this pr.
+# # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa
+# # So install a newer version here.
+RUN bash /build_scripts/install_patchelf.sh
+RUN apt-get install -y gcc-12 g++-12
+RUN cp gcc gcc.bak && cp g++ g++.bak && rm gcc && rm g++
+RUN ln -s /usr/bin/gcc-12 /usr/local/bin/gcc
+RUN ln -s /usr/bin/g++-12 /usr/local/bin/g++
+RUN ln -s /usr/bin/gcc-12 /usr/bin/gcc
+RUN ln -s /usr/bin/g++-12 /usr/bin/g++
+ENV PATH=/usr/local/gcc-12/bin:$PATH
+
+<install_cudnn>
+RUN rm -rf /build_script
+
+# install cmake && ccache
+RUN apt-get remove --purge cmake && apt-get install -y cmake
+# ccache version 4.9.1
+RUN apt-get install -y ccache
+
+RUN apt-get update && \
+  apt-get install -y python3.10 python3.10-dev python3.10-distutils \
+  python3.11 python3.11-dev python3.11-distutils \
+  python3.12 python3.12-dev \
+  python3.13 python3.13-dev python3.13-nogil && \
+  apt-get install python-is-python3
+RUN rm /usr/bin/python && ln -s /usr/bin/python3.10 /usr/bin/python && \
+    rm /usr/bin/python3 && ln -s /usr/bin/python3.10 /usr/bin/python3
+
+WORKDIR /home
+RUN wget -q https://bootstrap.pypa.io/get-pip.py
+RUN sed -i 's#"install", "--upgrade", "--force-reinstall"#"install", "--upgrade", "--force-reinstall", "--break-system-packages"#' get-pip.py
+
+RUN python3.10 get-pip.py && \
+  python3.11 get-pip.py && \
+  python3.12 get-pip.py
+
+RUN python3.13t get-pip.py && \
+  mv /usr/local/bin/pip3.13 /usr/local/bin/pip3.13t && \
+  python3.13 get-pip.py
+
+RUN python3.10 -m pip install setuptools==68.2.0 && \
+  python3.11 -m pip install setuptools==68.2.0 && \
+  python3.12 -m pip install --break-system-packages setuptools==68.2.0 && \
+  python3.13 -m pip install setuptools==69.5.0 && \
+  python3.13t -m pip install setuptools==69.5.0
+
+# binutils >= 2.27
+RUN apt-get install -y binutils
+
+# Install Go and glide
+RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.17.2.linux-amd64.tar.gz | \
+    tar -xz -C /usr/local && \
+    mkdir /root/gopath && \
+    mkdir /root/gopath/bin && \
+    mkdir /root/gopath/src
+ENV GOROOT=/usr/local/go GOPATH=/root/gopath
+# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT.
+ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin
+# install glide
+# RUN apt-get install -y golang-glide
+
+# git credential to skip password typing
+RUN git config --global credential.helper store
+
+# Fix locales to en_US.UTF-8
+RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+
+#For pre-commit
+RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.10 /usr/local/bin/pip && \
+    rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.10 /usr/local/bin/pip3
+
+RUN python3.10 -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.10 -m pip --no-cache-dir install ipykernel==4.6.0 wheel && \
+    python3.11 -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.11 -m pip --no-cache-dir install ipykernel==4.6.0 wheel && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages ipython==5.3.0 && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages ipykernel==4.6.0 wheel && \
+    python3.13 -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.13 -m pip --no-cache-dir install ipykernel==4.6.0 wheel && \
+    python3.13t -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.13t -m pip --no-cache-dir install ipykernel==4.6.0 wheel
+
+# For PaddleTest CE
+RUN python3.10 -m pip --no-cache-dir install pytest && \
+    python3.11 -m pip --no-cache-dir install pytest && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages pytest && \
+    python3.13 -m pip --no-cache-dir install pytest && \
+    python3.13t -m pip --no-cache-dir install pytest
+
+RUN python3.10 -m pip --no-cache-dir install pre-commit==2.17.0 && \
+    python3.10 -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.11 -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.13 -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.13t -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
+
+COPY ./python/requirements.txt /root/
+COPY ./python/unittest_py/requirements.txt /home/
+
+RUN python3.10 -m pip --no-cache-dir install -r /root/requirements.txt && \
+    python3.10 -m pip --no-cache-dir install --break-system-packages --ignore-installed -r /home/requirements.txt && \
+    python3.11 -m pip --no-cache-dir install -r /root/requirements.txt && \
+    python3.11 -m pip --no-cache-dir install --break-system-packages --ignore-installed -r /home/requirements.txt && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages -r /root/requirements.txt && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages --ignore-installed -r /home/requirements.txt && \
+    python3.13 -m pip --no-cache-dir install -r /root/requirements.txt && \
+    python3.13 -m pip --no-cache-dir install --break-system-packages --ignore-installed -r /home/requirements.txt && \
+    python3.13t -m pip --no-cache-dir install -r /root/requirements.txt
+
+# clang14
+RUN apt-get update &&\
+    apt install -y clang-14
+
+EXPOSE 22

tools/dockerfile/Dockerfile.ubuntu24

@@ -0,0 +1,166 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+FROM <baseimg>
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+<setcuda>
+
+ENV HOME /root
+# Add bash enhancements
+COPY paddle/scripts/docker/root/ /root/
+
+RUN chmod 777 /tmp
+
+RUN mv /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/cuda.list.bak
+RUN apt-get update && apt-get install -y curl
+RUN curl https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub | gpg --dearmor | tee /usr/share/keyrings/cuda-archive-keyring.gpg
+RUN mv /etc/apt/sources.list.d/cuda.list.bak /etc/apt/sources.list.d/cuda.list
+RUN sed -i 's#deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64 /#deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64 /#' /etc/apt/sources.list.d/cuda.list
+
+RUN apt-get update --allow-unauthenticated && \
+  apt-get install -y software-properties-common && \
+  add-apt-repository ppa:deadsnakes/ppa && \
+  apt-get update && \
+  apt-get install -y build-essential curl wget vim git unzip pigz zstd unrar tar xz-utils libssl-dev bzip2 gzip \
+    coreutils ntp language-pack-zh-hans libsm6 libxext6 libxrender-dev libgl1 libglx-mesa0 \
+    bison graphviz libjpeg-dev zlib1g-dev automake locales swig net-tools libtool kmod
+<install_cpu_package>
+
+WORKDIR /usr/bin
+COPY tools/dockerfile/build_scripts /build_scripts
+RUN bash /build_scripts/install_trt.sh
+# Older versions of patchelf limited the size of the files being processed and were fixed in this pr.
+# # https://github.com/NixOS/patchelf/commit/ba2695a8110abbc8cc6baf0eea819922ee5007fa
+# # So install a newer version here.
+RUN bash /build_scripts/install_patchelf.sh
+
+#RUN bash /build_scripts/install_nccl2.sh
+RUN rm -rf /build_script
+
+# install cmake && ccache
+RUN apt-get remove --purge cmake && apt-get install -y cmake=3.28.3-1build7
+# ccache version 4.9.1
+RUN apt-get install -y ccache
+
+RUN apt-get update && \
+  apt-get install -y python3.10 python3.10-dev python3.10-distutils \
+  python3.11 python3.11-dev python3.11-distutils \
+  python3.12 python3.12-dev \
+  python3.13 python3.13-dev python3.13-nogil \
+  python3.14 python3.14-dev python3.14-nogil && \
+  apt-get install python-is-python3
+RUN rm /usr/bin/python && ln -s /usr/bin/python3.10 /usr/bin/python && \
+    rm /usr/bin/python3 && ln -s /usr/bin/python3.10 /usr/bin/python3
+
+WORKDIR /home
+RUN wget -q https://bootstrap.pypa.io/get-pip.py
+RUN sed -i 's#"install", "--upgrade", "--force-reinstall"#"install", "--upgrade", "--force-reinstall", "--break-system-packages"#' get-pip.py
+
+RUN python3.10 get-pip.py && \
+  python3.11 get-pip.py && \
+  python3.12 get-pip.py
+
+RUN python3.13t get-pip.py && \
+  mv /usr/local/bin/pip3.13 /usr/local/bin/pip3.13t && \
+  python3.13 get-pip.py
+
+RUN python3.14t get-pip.py && \
+  mv /usr/local/bin/pip3.14 /usr/local/bin/pip3.14t && \
+  python3.14 get-pip.py
+
+RUN python -m pip config set global.break-system-packages true
+
+RUN python3.10 -m pip install setuptools==68.2.0 && \
+    python3.11 -m pip install setuptools==68.2.0 && \
+    python3.12 -m pip install --break-system-packages setuptools==68.2.0 && \
+    python3.13 -m pip install setuptools==69.5.0 && \
+    python3.13t -m pip install setuptools==69.5.0 && \
+    python3.14 -m pip install setuptools==69.5.0 && \
+    python3.14t -m pip install setuptools==69.5.0
+
+# binutils >= 2.27
+RUN apt-get install -y binutils
+
+# Install Go and glide
+RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.17.2.linux-amd64.tar.gz | \
+    tar -xz -C /usr/local && \
+    mkdir /root/gopath && \
+    mkdir /root/gopath/bin && \
+    mkdir /root/gopath/src
+ENV GOROOT=/usr/local/go GOPATH=/root/gopath
+# should not be in the same line with GOROOT definition, otherwise docker build could not find GOROOT.
+ENV PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin
+# install glide
+# RUN apt-get install -y golang-glide
+
+# git credential to skip password typing
+RUN git config --global credential.helper store
+
+# Fix locales to en_US.UTF-8
+RUN localedef -i en_US -f UTF-8 en_US.UTF-8
+
+#For pre-commit
+RUN rm -f /usr/local/bin/pip && ln -s /usr/local/bin/pip3.10 /usr/local/bin/pip && \
+    rm -f /usr/local/bin/pip3 && ln -s /usr/local/bin/pip3.10 /usr/local/bin/pip3
+
+RUN python3.10 -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.10 -m pip --no-cache-dir install ipykernel==4.6.0 wheel && \
+    python3.11 -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.11 -m pip --no-cache-dir install ipykernel==4.6.0 wheel && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages ipython==5.3.0 && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages ipykernel==4.6.0 wheel && \
+    python3.13 -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.13 -m pip --no-cache-dir install ipykernel==4.6.0 wheel && \
+    python3.13t -m pip --no-cache-dir install ipython==5.3.0 && \
+    python3.13t -m pip --no-cache-dir install ipykernel==4.6.0 wheel && \
+    python3.14 -m pip --no-cache-dir install ipython==5.3.0 ipykernel==4.6.0 wheel && \
+    python3.14t -m pip --no-cache-dir install ipython==5.3.0 ipykernel==4.6.0 wheel
+
+# For PaddleTest CE
+RUN python3.10 -m pip --no-cache-dir install pytest && \
+    python3.11 -m pip --no-cache-dir install pytest && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages pytest && \
+    python3.13 -m pip --no-cache-dir install pytest && \
+    python3.13t -m pip --no-cache-dir install pytest && \
+    python3.14 -m pip --no-cache-dir install pytest && \
+    python3.14t -m pip --no-cache-dir install pytest
+
+RUN python3.10 -m pip --no-cache-dir install pre-commit==2.17.0 && \
+    python3.10 -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.11 -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.13 -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.13t -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.14 -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0 && \
+    python3.14t -m pip --no-cache-dir install cpplint==1.6.0 clang-format==13.0.0
+
+COPY ./python/requirements.txt /root/
+COPY ./python/unittest_py/requirements.txt /home/
+
+RUN python3.10 -m pip --no-cache-dir install -r /root/requirements.txt && \
+    python3.10 -m pip --no-cache-dir install --break-system-packages --ignore-installed -r /home/requirements.txt && \
+    python3.11 -m pip --no-cache-dir install -r /root/requirements.txt && \
+    python3.11 -m pip --no-cache-dir install --break-system-packages --ignore-installed -r /home/requirements.txt && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages -r /root/requirements.txt && \
+    python3.12 -m pip --no-cache-dir install --break-system-packages --ignore-installed -r /home/requirements.txt && \
+    python3.13 -m pip --no-cache-dir install -r /root/requirements.txt && \
+    python3.13 -m pip --no-cache-dir install --break-system-packages --ignore-installed -r /home/requirements.txt && \
+    python3.13t -m pip --no-cache-dir install -r /root/requirements.txt && \
+    python3.14 -m pip --no-cache-dir install -r /root/requirements.txt && \
+    python3.14 -m pip --no-cache-dir install --break-system-packages --ignore-installed -r /home/requirements.txt && \
+    python3.14t -m pip --no-cache-dir install -r /root/requirements.txt
+
+
+# clang14
+RUN apt-get update &&\
+    apt install -y clang-14
+
+EXPOSE 22

tools/dockerfile/Dockerfile.ubuntu24.arm.py312

@@ -0,0 +1,93 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+ARG CUDA_VERSION=13.0
+ARG BASE_TARGET=cuda${CUDA_VERSION}
+
+FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-ubuntu24.04 as base
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+ARG WITH_ARM
+ARG PYTHON_VERSION=3.12
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV WITH_ARM=${WITH_ARM:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION}/compat:/usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib:$LD_LIBRARY_PATH
+
+ENV HOME /root
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+
+ENV http_proxy=${HTTP_PROXY} \
+    https_proxy=${HTTPS_PROXY} \
+    no_proxy=${NO_PROXY}
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository -y ppa:deadsnakes/ppa && \
+    apt-get update --allow-unauthenticated && \
+    apt-get install -y --no-install-recommends \
+      git \
+      vim \
+      curl \
+      wget \
+      make \
+      libgl1 \
+      libglib2.0-0 \
+      libssl-dev \
+      autoconf \
+      automake \
+      libtool \
+      libmlx5-1 \
+      libibverbs-dev \
+      ninja-build \
+      zlib1g-dev \
+      python${PYTHON_VERSION} \
+      python${PYTHON_VERSION}-dev \
+      python${PYTHON_VERSION}-venv \
+      python3-pip && \
+    ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python && \
+    ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /home
+
+RUN wget -q https://cmake.org/files/v3.31/cmake-3.31.0-linux-aarch64.tar.gz && \
+    tar -zxf cmake-3.31.0-linux-aarch64.tar.gz && \
+    rm cmake-3.31.0-linux-aarch64.tar.gz && \
+    rm -rf /home/cmake-3.31.0-linux-aarch64/doc /home/cmake-3.31.0-linux-aarch64/man
+ENV PATH=/home/cmake-3.31.0-linux-aarch64/bin:$PATH
+
+ARG TMP_DIR=patchelf_tmp
+RUN rm -rf "$TMP_DIR" && git clone -b 0.15.0 https://github.com/NixOS/patchelf "$TMP_DIR" && \
+    cd "$TMP_DIR" && ./bootstrap.sh && \
+    ./configure && make && make install && \
+    cd .. && rm -rf "$TMP_DIR"
+
+RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
+    tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
+    make -j8 && make install && \
+    ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/bin/ccache && \
+    cd ../../ && rm -rf ccache-4.8.2.tar.gz && rm -rf ccache-4.8.2
+
+COPY paddle/scripts/compile_requirements.txt /root
+COPY python/requirements.txt /root
+
+RUN pip config set global.break-system-packages true
+
+RUN pip install --break-system-packages -r /root/requirements.txt && \
+    pip install --break-system-packages -r /root/compile_requirements.txt && \
+    pip install --break-system-packages wheel pyyaml && \
+    rm -rf /root/compile_requirements.txt /root/requirements.txt
+
+#install gdrcopy
+RUN cd /usr/local && wget -q https://paddle-ci.gz.bcebos.com/gdrcopy.tar && tar xf gdrcopy.tar && rm -rf gdrcopy.tar
+ENV GDRCOPY_HOME=/usr/local/gdrcopy

tools/dockerfile/Dockerfile.ubuntu24.arm.py313

@@ -0,0 +1,96 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+ARG CUDA_VERSION=13.0
+ARG BASE_TARGET=cuda${CUDA_VERSION}
+
+FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-ubuntu24.04 as base
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+ARG WITH_ARM
+ARG PYTHON_VERSION=3.13
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV WITH_ARM=${WITH_ARM:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION}/compat:/usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib:$LD_LIBRARY_PATH
+
+ENV HOME /root
+
+ARG HTTP_PROXY
+ARG HTTPS_PROXY
+ARG NO_PROXY
+
+ENV http_proxy=${HTTP_PROXY} \
+    https_proxy=${HTTPS_PROXY} \
+    no_proxy=${NO_PROXY}
+
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository -y ppa:deadsnakes/ppa && \
+    apt-get update --allow-unauthenticated && \
+    apt-get install -y --no-install-recommends \
+      git \
+      vim \
+      curl \
+      wget \
+      make \
+      libgl1 \
+      libglib2.0-0 \
+      libssl-dev \
+      autoconf \
+      automake \
+      libtool \
+      libmlx5-1 \
+      libibverbs-dev \
+      ninja-build \
+      zlib1g-dev \
+      python${PYTHON_VERSION} \
+      python${PYTHON_VERSION}-dev \
+      python${PYTHON_VERSION}-venv && \
+    ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python && \
+    ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python3 && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /home
+
+RUN wget -q https://cmake.org/files/v3.31/cmake-3.31.0-linux-aarch64.tar.gz && \
+    tar -zxf cmake-3.31.0-linux-aarch64.tar.gz && \
+    rm cmake-3.31.0-linux-aarch64.tar.gz && \
+    rm -rf /home/cmake-3.31.0-linux-aarch64/doc /home/cmake-3.31.0-linux-aarch64/man
+ENV PATH=/home/cmake-3.31.0-linux-aarch64/bin:$PATH
+
+ARG TMP_DIR=patchelf_tmp
+RUN rm -rf "$TMP_DIR" && git clone -b 0.15.0 https://github.com/NixOS/patchelf "$TMP_DIR" && \
+    cd "$TMP_DIR" && ./bootstrap.sh && \
+    ./configure && make && make install && \
+    cd .. && rm -rf "$TMP_DIR"
+
+RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
+    tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
+    make -j8 && make install && \
+    ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/bin/ccache && \
+    cd ../../ && rm -rf ccache-4.8.2.tar.gz && rm -rf ccache-4.8.2
+
+COPY paddle/scripts/compile_requirements.txt /root
+COPY python/requirements.txt /root
+
+#install pip3
+RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} && \
+    ln -sf /usr/bin/pip${PYTHON_VERSION} /usr/bin/pip3 && \
+    ln -sf /usr/bin/pip${PYTHON_VERSION} /usr/bin/pip
+
+RUN pip install --break-system-packages -r /root/requirements.txt && \
+    pip install --break-system-packages -r /root/compile_requirements.txt && \
+    pip install wheel && \
+    rm -rf /root/compile_requirements.txt /root/requirements.txt
+RUN pip config set global.break-system-packages true
+
+#install gdrcopy
+RUN cd /usr/local && wget -q https://paddle-ci.gz.bcebos.com/gdrcopy.tar && tar xf gdrcopy.tar && rm -rf gdrcopy.tar
+ENV GDRCOPY_HOME=/usr/local/gdrcopy

tools/dockerfile/build-image.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# Usage:
+# export CANN_VERSION=8.0.RC1
+# bash build-image.sh ${CANN_VERSION}
+
+CANN_VERSION=${1:-8.0.0} # default 8.0.0
+SYSTEM=${2:-x86_64} # default x86_64
+NPU_VERSION=${3:-910b} # default 910b
+
+DOCKER_VERSION=${CANN_VERSION//[^0-9A-Z]/} # 80T13
+
+# Download packages from https://www.hiascend.com/software/cann/community first
+if [ ! -f Ascend-cann-toolkit_${CANN_VERSION}_linux-$(uname -m).run ]; then
+  echo "Please download CANN installation packages first!"
+  exit 1
+fi
+
+sed "s#<baseimg>#registry.baidubce.com/device/paddle-cpu:ubuntu20-npu-base-$(uname -m)-gcc84#g" Dockerfile.npu.ubuntu20.gcc84 > Dockerfile.npu.ubuntu20.gcc84.test
+docker build --network=host -f Dockerfile.npu.ubuntu20.gcc84.test \
+  --build-arg CANN_VERSION=${CANN_VERSION} \
+  --build-arg SYSTEM=${SYSTEM} \
+  --build-arg NPU_VERSION=${NPU_VERSION} \
+  --build-arg http_proxy=${proxy} \
+  --build-arg https_proxy=${proxy} \
+  --build-arg ftp_proxy=${proxy} \
+  --build-arg no_proxy=bcebos.com \
+  -t ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-npu:cann${DOCKER_VERSION}-ubuntu20-npu-${NPU_VERSION}-base-$(uname -m)-gcc84 .
+docker push ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-npu:cann${DOCKER_VERSION}-ubuntu20-npu-${NPU_VERSION}-base-$(uname -m)-gcc84
+rm -rf Dockerfile.npu.ubuntu20.gcc84.test

tools/dockerfile/centos7_manylinux.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -xe
+
+REPO="${REPO:-paddledocker}"
+
+function make_cuda112cudnn821trt8034() {
+  sed 's/<baseimg>/11.2.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN yum remove -y libcudnn8-devel.x86_64 libcudnn8.x86_64 \nRUN bash build_scripts/install_cudnn.sh cudnn821 \nENV CUDNN_VERSION=8.2.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+  sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8034#g" Dockerfile.tmp
+  sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
+}
+
+function make_cuda116cudnn840trt8406() {
+  sed 's/<baseimg>/11.6.2-cudnn8-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+  sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8406#g" Dockerfile.tmp
+  sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
+}
+
+function make_cuda117cudnn841trt8424() {
+  sed 's/<baseimg>/11.7.1-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/install_cudnn.sh cudnn841 \nENV CUDNN_VERSION=8.4.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+  sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8424#g" Dockerfile.tmp
+  sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
+}
+
+function make_cuda118cudnn860trt8531() {
+  sed 's/<baseimg>/11.8.0-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc82 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-8.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-8.2/bin:\$PATH \nRUN bash build_scripts/install_cudnn.sh cudnn860 \nENV CUDNN_VERSION=8.6.0 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+  sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8531#g" Dockerfile.tmp
+  sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
+}
+
+function make_cuda120cudnn891trt8616() {
+  sed 's/<baseimg>/12.0.1-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc122 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-12.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-12.2/bin:\$PATH \nRUN bash build_scripts/install_cudnn.sh cudnn891 \nENV CUDNN_VERSION=8.9.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+  sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8616#g" Dockerfile.tmp
+  sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
+}
+
+function make_cuda123cudnn900trt8616() {
+  sed 's/<baseimg>/12.3.1-devel-centos7/g' Dockerfile.centos >Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc122 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-12.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-12.2/bin:\$PATH \nRUN bash build_scripts/install_cudnn.sh cudnn900 \nENV CUDNN_VERSION=9.0.0 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+  sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8616#g" Dockerfile.tmp
+  sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
+}
+
+function make_cuda124cudnn911trt8616() {
+  sed 's/<baseimg>/12.4.1-cudnn-devel-rockylinux8/g' Dockerfile.rockylinux8 >Dockerfile.tmp
+  sed -i "s#<install_gcc>#RUN dnf install -y gcc-toolset-12-gcc* \&\& source /opt/rh/gcc-toolset-12/enable \&\& echo 'source /opt/rh/gcc-toolset-12/enable' >>~/.bashrc \nENV PATH=/opt/rh/gcc-toolset-12/root/usr/bin:/usr/share/Modules/bin:\$PATH \nENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:\$LD_LIBRARY_PATH #g" Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_cudnn.sh cudnn911 \nENV CUDNN_VERSION=9.1.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+  sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8616#g" Dockerfile.tmp
+  sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
+}
+
+function make_cuda125cudnn911trt8616() {
+  sed 's/<baseimg>/12.5.1-cudnn-devel-rockylinux8/g' Dockerfile.rockylinux8 >Dockerfile.tmp
+  sed -i "s#RUN bash build_scripts/build.sh#RUN bash build_scripts/install_gcc.sh gcc122 \nRUN mv /usr/bin/cc /usr/bin/cc.bak \&\& ln -s /usr/local/gcc-12.2/bin/gcc /usr/bin/cc \nENV PATH=/usr/local/gcc-12.2/bin:\$PATH \nRUN bash build_scripts/install_cudnn.sh cudnn911 \nENV CUDNN_VERSION=9.1.1 \nRUN bash build_scripts/build.sh#g" Dockerfile.tmp
+  sed -i "s#build_scripts/install_trt.sh#build_scripts/install_trt.sh trt8616#g" Dockerfile.tmp
+  sed -i '/CMD/iRUN ldconfig' Dockerfile.tmp
+}
+
+
+function main() {
+  local CMD=$1
+  case $CMD in
+    cuda112cudnn821trt8034)
+      make_cuda112cudnn821trt8034
+     ;;
+    cuda116cudnn840trt8406)
+      make_cuda116cudnn840trt8406
+     ;;
+    cuda117cudnn841trt8424)
+      make_cuda117cudnn841trt8424
+     ;;
+    cuda118cudnn860trt8531)
+      make_cuda118cudnn860trt8531
+     ;;
+    cuda120cudnn891trt8616)
+      make_cuda120cudnn891trt8616
+     ;;
+    cuda123cudnn900trt8616)
+     make_cuda123cudnn900trt8616
+     ;;
+    cuda124cudnn911trt8616)
+     make_cuda124cudnn911trt8616
+     ;;
+    cuda125cudnn911trt8616)
+     make_cuda125cudnn911trt8616
+     ;;
+    *)
+      echo "Make dockerfile error, Without this parameter."
+      exit 1
+      ;;
+  esac
+}
+
+main "$@"

tools/dockerfile/ci_dockerfile.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+function make_cpu_dockerfile(){
+  dockerfile_name="Dockerfile.cuda9_cudnn7_gcc48_py35_centos6"
+  sed "s#<baseimg>#ubuntu:20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
+  sed -i "s#<setcuda>##g" ${dockerfile_name}
+  sed -i "s#WITH_GPU:-ON#WITH_GPU:-OFF#g" ${dockerfile_name}
+  sed -i "s#RUN apt-key del 7fa2af80##g" ${dockerfile_name}
+  sed -i 's#RUN rm /etc/apt/sources.list.d/\*##g' ${dockerfile_name}
+  sed -i "s#RUN apt-key adv --fetch-keys https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub##g" ${dockerfile_name}
+  dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
+  sed -i 's#RUN bash /build_scripts/install_trt.sh##g' ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
+     tar -xzf     hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libsndfile1 zstd pigz ninja-build" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN pip install wheel PyGithub distro" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libcurl4-openssl-dev gettext pigz \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
+    tar -xvf git-2.17.1.tar.gz \&\& \
+    cd git-2.17.1 \&\& \
+    ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
+    make -j8 \&\& make install " ${dockerfile_name}
+  sed -i 's#<install_cpu_package>#RUN apt-get install -y gcc g++ make#g' ${dockerfile_name}
+  sed -i 's#RUN bash /build_scripts/install_gcc.sh gcc82#RUN add-apt-repository ppa:ubuntu-toolchain-r/test \&\& apt-get update \&\& apt-get install -y gcc-13 g++-13#g' ${dockerfile_name}
+  sed -i 's#/usr/local/gcc-8.2/bin/gcc#/usr/bin/gcc-13#g' ${dockerfile_name}
+  sed -i 's#/usr/local/gcc-8.2/bin/g++#/usr/bin/g++-13#g' ${dockerfile_name}
+  sed -i 's#ENV PATH=/usr/local/gcc-8.2/bin:$PATH##g' ${dockerfile_name}
+}
+
+
+function make_sot_dockerfile(){
+  dockerfile_name="Dockerfile.gcc133_ubuntu24_cpu_sot"
+  sed "s#<baseimg>#ubuntu:24.04#g" ./Dockerfile.ubuntu24 >${dockerfile_name}
+  sed -i "s#<setcuda>##g" ${dockerfile_name}
+  sed -i "s#WITH_GPU:-ON#WITH_GPU:-OFF#g" ${dockerfile_name}
+  sed -i "/\/etc\/apt\/sources.list.d\/cuda/ d" ${dockerfile_name}
+  sed -i '/RUN curl https:\/\/developer.download.nvidia.cn/ d' ${dockerfile_name}
+  sed -i "s#RUN apt-key adv --fetch-keys https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub##g" ${dockerfile_name}
+  dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
+  sed -i 's#RUN bash /build_scripts/install_trt.sh##g' ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN pip install wheel PyGithub distro" ${dockerfile_name}
+  sed -i 's#<install_cpu_package>#RUN apt-get install -y gcc g++ make libcurl4-openssl-dev gettext ninja-build#g' ${dockerfile_name}
+}
+
+
+function make_ce_framework_dockerfile(){
+  dockerfile_name="Dockerfile.cuda11.2_cudnn8_gcc82_trt8"
+  sed "s#<baseimg>#nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
+  dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
+  sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-11.8/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+  sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+  sed -i "7i RUN chmod 777 /tmp" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
+     tar -xzf  hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN apt-get update && apt install -y zstd pigz libcurl4-openssl-dev gettext ninja-build" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN pip3.10 install wheel distro jinja2 bce-python-sdk==0.8.74" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN pip3.10 install nvidia-cuda-cupti-cu11==11.8.87 nvidia-cuda-runtime-cu11==11.8.89 nvidia-cudnn-cu11==8.7.0.84 nvidia-cublas-cu11==11.11.3.6 nvidia-cufft-cu11==10.9.0.58 nvidia-curand-cu11==10.3.0.86 nvidia-cusolver-cu11==11.4.1.48 nvidia-cusparse-cu11==11.7.5.86 nvidia-nccl-cu11==2.19.3" ${dockerfile_name}
+  sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8531#g' ${dockerfile_name}
+}
+
+
+function make_ubuntu20_cu12_dockerfile(){
+  dockerfile_name="Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage"
+  sed "s#<baseimg>#nvidia/cuda:12.0.1-cudnn8-devel-ubuntu22.04#g" ./Dockerfile.ubuntu22 >${dockerfile_name}
+  sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.0/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+  sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+  sed -i "7i ENV TZ=Asia/Beijing" ${dockerfile_name}
+  sed -i "8i RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone" ${dockerfile_name}
+  sed -i "27i RUN apt-get update && apt-get install -y liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev libsndfile1" ${dockerfile_name}
+  dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
+  sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
+     tar -xzf  hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build  \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
+    tar -xvf git-2.17.1.tar.gz \&\& \
+    cd git-2.17.1 \&\& \
+    ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
+    make -j8 \&\& make install " ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3.12 install PyGithub wheel distro jinja2" ${dockerfile_name}
+  sed -i 's# && rm /etc/apt/sources.list.d/nvidia-ml.list##g' ${dockerfile_name}
+  sed -i 's#RUN bash /build_scripts/install_trt.sh##g' ${dockerfile_name}
+  sed -i 's#<install_cudnn>#RUN bash /build_scripts/install_cudnn.sh cudnn896 #g' ${dockerfile_name}
+}
+
+
+function make_ubuntu20_cu123_dockerfile(){
+  dockerfile_name="Dockerfile.cuda123_cudnn9_gcc122_ubuntu20"
+  sed "s#<baseimg>#nvidia/cuda:12.3.1-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
+  sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.3/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+  sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+  sed -i "7i ENV TZ=Asia/Beijing" ${dockerfile_name}
+  sed -i "8i RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone" ${dockerfile_name}
+  sed -i "27i RUN apt-get update && apt-get install -y liblzma-dev openmpi-bin openmpi-doc libopenmpi-dev libsndfile1" ${dockerfile_name}
+  dockerfile_line=$(wc -l ${dockerfile_name}|awk '{print $1}')
+  sed -i "${dockerfile_line}i RUN wget --no-check-certificate -q https://paddle-edl.bj.bcebos.com/hadoop-2.7.7.tar.gz \&\& \
+     tar -xzf  hadoop-2.7.7.tar.gz && mv hadoop-2.7.7 /usr/local/" ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN apt remove git -y \&\& apt update \&\& apt install -y libcurl4-openssl-dev gettext pigz zstd ninja-build \&\& wget -q https://paddle-ci.gz.bcebos.com/git-2.17.1.tar.gz \&\& \
+    tar -xvf git-2.17.1.tar.gz \&\& \
+    cd git-2.17.1 \&\& \
+    ./configure --with-openssl --with-curl --prefix=/usr/local \&\& \
+    make -j8 \&\& make install " ${dockerfile_name}
+  sed -i "${dockerfile_line}i RUN pip install wheel \&\& pip3 install PyGithub wheel distro" ${dockerfile_name}
+  sed -i 's# && rm /etc/apt/sources.list.d/nvidia-ml.list##g' ${dockerfile_name}
+  sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
+  sed -i 's#RUN bash /build_scripts/install_cudnn.sh cudnn841#RUN bash /build_scripts/install_cudnn.sh cudnn900 #g' ${dockerfile_name}
+  sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=9.0.0#g' ${dockerfile_name}
+  sed -i 's#gcc82#gcc121#g' ${dockerfile_name}
+  sed -i 's#/usr/local/gcc-8.2/bin/gcc#/usr/local/gcc-12.1/bin/gcc#g' ${dockerfile_name}
+  sed -i 's#/usr/local/gcc-8.2/bin/gcc#/usr/local/gcc-12.1/bin/gcc#g' ${dockerfile_name}
+  sed -i 's#/usr/local/gcc-8.2/bin/g++#/usr/local/gcc-12.1/bin/g++#g' ${dockerfile_name}
+  sed -i 's#/usr/local/gcc-8.2/bin/g++#/usr/local/gcc-12.1/bin/g++#g' ${dockerfile_name}
+  sed -i 's#PATH=/usr/local/gcc-8.2/bin:$PATH#PATH=/usr/local/gcc-12.1/bin:$PATH#g' ${dockerfile_name}
+}
+
+function main() {
+  make_cpu_dockerfile
+  make_sot_dockerfile
+  make_ce_framework_dockerfile
+  make_ubuntu20_cu12_dockerfile
+  make_ubuntu20_cu123_dockerfile
+}
+
+main "$@"

tools/dockerfile/manylinux/Dockerfile

@@ -0,0 +1,138 @@
+ARG CUDA_VERSION=11.8
+ARG BASE_TARGET=cuda${CUDA_VERSION}
+FROM amd64/almalinux:8 as base
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+
+ARG DEVTOOLSET_VERSION=11
+
+ENV LC_ALL en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US.UTF-8
+
+RUN yum -y update
+RUN yum -y install epel-release
+RUN yum install -y \
+    sudo wget curl perl util-linux xz bzip2 git patch which zlib-devel openssl-devel \
+    yum-utils autoconf automake make gcc-toolset-${DEVTOOLSET_VERSION}-toolchain \
+    unzip bison diffutils file vim libffi-devel ncurses-devel sqlite-devel \
+    readline-devel tk-devel gdbm-devel glibc-devel libstdc++-devel glib2-devel \
+    libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel \
+    libSM-devel freetype-devel libpng-devel kernel-devel kmod \
+    rdma-core-devel libibverbs-devel
+
+# Just add everything as a safe.directory for git since these will be used in multiple places with git
+RUN git config --global --add safe.directory '*'
+RUN ln -s /opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib/gcc/x86_64-redhat-linux/${DEVTOOLSET_VERSION} /usr/lib/gcc/x86_64-redhat-linux/${DEVTOOLSET_VERSION}
+ENV PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/bin:$PATH
+
+# cmake-3.31.0
+WORKDIR /home
+RUN wget -q https://cmake.org/files/v3.31/cmake-3.31.0-linux-x86_64.tar.gz && tar -zxvf cmake-3.31.0-linux-x86_64.tar.gz && rm cmake-3.31.0-linux-x86_64.tar.gz
+ENV PATH=/home/cmake-3.31.0-linux-x86_64/bin:$PATH
+
+# go-1.15.12
+WORKDIR /home
+RUN wget --no-check-certificate -qO- https://paddle-ci.gz.bcebos.com/go1.15.12.linux-amd64.tar.gz | \
+    tar -xz -C /usr/local && \
+    mkdir /root/gopath && \
+    mkdir /root/gopath/bin && \
+    mkdir /root/gopath/src
+ENV GOROOT=/usr/local/go GOPATH=/root/gopath
+ENV PATH=/usr/local/ssl:${GOROOT}/bin:${GOPATH}/bin:${PATH}
+
+FROM base as python
+# Install python
+ADD ./common/install_python.sh install_python.sh
+RUN bash ./install_python.sh && rm install_python.sh
+
+FROM base as openssl
+# Install openssl
+ADD ./common/install_openssl.sh install_openssl.sh
+RUN bash ./install_openssl.sh && rm install_openssl.sh
+
+FROM base as patchelf
+# Install patchelf
+ADD ./common/install_patchelf.sh install_patchelf.sh
+RUN bash ./install_patchelf.sh && rm install_patchelf.sh
+
+FROM base as ccache
+# Install ccache
+ADD ./common/install_ccache.sh install_ccache.sh
+RUN bash ./install_ccache.sh && rm install_ccache.sh
+
+
+# Install CUDA
+FROM base as cuda
+ARG CUDA_VERSION
+RUN rm -rf /usr/local/cuda-*
+ENV CUDA_HOME=/usr/local/cuda-${CUDA_VERSION}
+ENV CUDA_VERSION=${CUDA_VERSION}
+ENV PATH=/usr/local/cuda-${CUDA_VERSION}/bin:$PATH
+ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION}/compat:$LD_LIBRARY_PATH
+ADD ./common/install_cuda.sh install_cuda.sh
+ADD ./common/install_gdrcopy.sh install_gdrcopy.sh
+
+FROM cuda as cuda11.8
+RUN bash ./install_cuda.sh 11.8
+ENV DESIRED_CUDA=11.8
+
+FROM cuda as cuda12.3
+RUN bash ./install_cuda.sh 12.3
+ENV DESIRED_CUDA=12.3
+
+FROM cuda as cuda12.4
+RUN bash ./install_cuda.sh 12.4
+ENV DESIRED_CUDA=12.4
+
+FROM cuda as cuda12.6
+RUN bash ./install_cuda.sh 12.6
+ENV DESIRED_CUDA=12.6
+
+FROM cuda as cuda12.9
+RUN bash ./install_cuda.sh 12.9 && bash ./install_gdrcopy.sh
+ENV DESIRED_CUDA=12.9
+ENV GDRCOPY_HOME=/usr/local/gdrcopy
+
+FROM cuda as cuda13.0
+RUN bash ./install_cuda.sh 13.0 && bash ./install_gdrcopy.sh
+ENV DESIRED_CUDA=13.0
+ENV GDRCOPY_HOME=/usr/local/gdrcopy
+
+FROM cuda as cuda13.2
+RUN bash ./install_cuda.sh 13.2 && bash ./install_gdrcopy.sh
+ENV DESIRED_CUDA=13.2
+ENV GDRCOPY_HOME=/usr/local/gdrcopy
+
+
+# Install paddle
+FROM python as paddle
+RUN wget https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O /root/requirements.txt
+
+RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install setuptools pyyaml wheel -U && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install setuptools pyyaml wheel -U && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install setuptools pyyaml wheel -U && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.13.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.13.0/bin/pip3 install setuptools pyyaml wheel -U && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.14.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.14.0/bin/pip3 install setuptools pyyaml wheel -U
+
+RUN LD_LIBRARY_PATH=/opt/_internal/cpython-3.10.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.10.0/bin/pip3 install -r /root/requirements.txt && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.11.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.11.0/bin/pip3 install -r /root/requirements.txt && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.12.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.12.0/bin/pip3 install -r /root/requirements.txt && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.13.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.13.0/bin/pip3 install -r /root/requirements.txt && \
+    LD_LIBRARY_PATH=/opt/_internal/cpython-3.14.0/lib/:${LD_LIBRARY_PATH} /opt/_internal/cpython-3.14.0/bin/pip3 install -r /root/requirements.txt
+RUN go get github.com/Masterminds/glide
+RUN rm -rf /root/requirements.txt
+
+ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib64:/opt/rh/gcc-toolset-${DEVTOOLSET_VERSION}/root/usr/lib:$LD_LIBRARY_PATH
+RUN rm -rf /usr/local/cuda
+RUN chmod o+rw /usr/local
+
+
+# Final step
+FROM ${BASE_TARGET} as final
+COPY --from=openssl          /usr/local/ssl          /usr/local/ssl
+COPY --from=patchelf         /usr/local/bin/patchelf /usr/local/bin/patchelf
+COPY --from=ccache           /usr/local/bin/ccache   /usr/local/bin/ccache
+COPY --from=paddle           /opt/_internal          /opt/_internal

tools/dockerfile/manylinux/Dockerfile-118

@@ -0,0 +1,68 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+ARG CUDA_VERSION=11.8
+ARG CUDA_PADDLE=cu118
+ARG BASE_TARGET=cuda${CUDA_VERSION}
+
+FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu22.04 as base
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+ARG PYTHON_VERSION=3.10
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION}/compat:/usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib:$LD_LIBRARY_PATH
+
+ENV HOME /root
+
+RUN apt-get update --allow-unauthenticated && \
+    apt-get install -y --no-install-recommends \
+      git \
+      vim \
+      curl \
+      wget \
+      make \
+      libssl-dev \
+      autoconf \
+      automake \
+      libtool \
+      python${PYTHON_VERSION} \
+      python${PYTHON_VERSION}-dev \
+      python3-pip && \
+    ln -sf /usr/bin/python3 /usr/bin/python && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /home
+RUN wget -q https://cmake.org/files/v3.31/cmake-3.31.0-linux-x86_64.tar.gz && \
+    tar -zxf cmake-3.31.0-linux-x86_64.tar.gz && \
+    rm cmake-3.31.0-linux-x86_64.tar.gz && \
+    rm -rf /home/cmake-3.31.0-linux-x86_64/doc /home/cmake-3.31.0-linux-x86_64/man
+
+ENV PATH=/home/cmake-3.31.0-linux-x86_64/bin:$PATH
+
+
+ARG TMP_DIR=patchelf_tmp
+RUN rm -rf "$TMP_DIR" && git clone -b 0.15.0 https://github.com/NixOS/patchelf "$TMP_DIR" && \
+    cd "$TMP_DIR" && ./bootstrap.sh && \
+    ./configure && make && make install && \
+    cd .. && rm -rf "$TMP_DIR"
+
+RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
+    tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
+    make -j8 && make install && \
+    ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
+    cd ../../ && rm -rf ccache-4.8.2.tar.gz && rm -rf ccache-4.8.2
+
+COPY paddle/scripts/compile_requirements.txt /root
+COPY python/requirements.txt /root
+RUN pip install -r /root/requirements.txt && \
+    pip install -r /root/compile_requirements.txt && \
+    rm -rf /root/compile_requirements.txt /root/requirements.txt

tools/dockerfile/manylinux/Dockerfile-126

@@ -0,0 +1,70 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+ARG CUDA_VERSION=12.6
+ARG CUDA_PADDLE=126
+ARG BASE_TARGET=cuda${CUDA_VERSION}
+
+FROM nvidia/cuda:12.6.3-cudnn-devel-ubuntu22.04 as base
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+ARG PYTHON_VERSION=3.10
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION}/compat:/usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib:$LD_LIBRARY_PATH
+
+ENV HOME /root
+
+RUN apt-get update --allow-unauthenticated && \
+    apt-get install -y --no-install-recommends \
+      git \
+      vim \
+      curl \
+      wget \
+      make \
+      libssl-dev \
+      autoconf \
+      automake \
+      libtool \
+      libmlx5-1 \
+      libibverbs-dev \
+      python${PYTHON_VERSION} \
+      python${PYTHON_VERSION}-dev \
+      python3-pip && \
+    ln -sf /usr/bin/python3 /usr/bin/python && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /home
+RUN wget -q https://cmake.org/files/v3.31/cmake-3.31.0-linux-x86_64.tar.gz && \
+    tar -zxf cmake-3.31.0-linux-x86_64.tar.gz && \
+    rm cmake-3.31.0-linux-x86_64.tar.gz && \
+    rm -rf /home/cmake-3.31.0-linux-x86_64/doc /home/cmake-3.31.0-linux-x86_64/man
+
+ENV PATH=/home/cmake-3.31.0-linux-x86_64/bin:$PATH
+
+
+ARG TMP_DIR=patchelf_tmp
+RUN rm -rf "$TMP_DIR" && git clone -b 0.15.0 https://github.com/NixOS/patchelf "$TMP_DIR" && \
+    cd "$TMP_DIR" && ./bootstrap.sh && \
+    ./configure && make && make install && \
+    cd .. && rm -rf "$TMP_DIR"
+
+RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
+    tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
+    make -j8 && make install && \
+    ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
+    cd ../../ && rm -rf ccache-4.8.2.tar.gz && rm -rf ccache-4.8.2
+
+COPY paddle/scripts/compile_requirements.txt /root
+COPY python/requirements.txt /root
+RUN pip install -r /root/requirements.txt && \
+    pip install -r /root/compile_requirements.txt && \
+    rm -rf /root/compile_requirements.txt /root/requirements.txt

tools/dockerfile/manylinux/Dockerfile-129

@@ -0,0 +1,71 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+ARG CUDA_VERSION=12.9
+ARG BASE_TARGET=cuda${CUDA_VERSION}
+
+FROM nvidia/cuda:12.9.0-cudnn-devel-ubuntu22.04 as base
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+ARG PYTHON_VERSION=3.10
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION}/compat:/usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib:$LD_LIBRARY_PATH
+
+ENV HOME /root
+
+RUN apt-get update --allow-unauthenticated && \
+    apt-get install -y --no-install-recommends \
+      git \
+      vim \
+      curl \
+      wget \
+      make \
+      libgl1 \
+      libglib2.0-0 \
+      libssl-dev \
+      autoconf \
+      automake \
+      libtool \
+      libmlx5-1 \
+      libibverbs-dev \
+      python${PYTHON_VERSION} \
+      python${PYTHON_VERSION}-dev \
+      python3-pip && \
+    ln -sf /usr/bin/python3 /usr/bin/python && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /home
+RUN wget -q https://cmake.org/files/v3.31/cmake-3.31.0-linux-x86_64.tar.gz && \
+    tar -zxf cmake-3.31.0-linux-x86_64.tar.gz && \
+    rm cmake-3.31.0-linux-x86_64.tar.gz && \
+    rm -rf /home/cmake-3.31.0-linux-x86_64/doc /home/cmake-3.31.0-linux-x86_64/man
+
+ENV PATH=/home/cmake-3.31.0-linux-x86_64/bin:$PATH
+
+
+ARG TMP_DIR=patchelf_tmp
+RUN rm -rf "$TMP_DIR" && git clone -b 0.15.0 https://github.com/NixOS/patchelf "$TMP_DIR" && \
+    cd "$TMP_DIR" && ./bootstrap.sh && \
+    ./configure && make && make install && \
+    cd .. && rm -rf "$TMP_DIR"
+
+RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
+    tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
+    make -j8 && make install && \
+    ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
+    cd ../../ && rm -rf ccache-4.8.2.tar.gz && rm -rf ccache-4.8.2
+
+COPY paddle/scripts/compile_requirements.txt /root
+COPY python/requirements.txt /root
+RUN pip install -r /root/requirements.txt && \
+    pip install -r /root/compile_requirements.txt && \
+    rm -rf /root/compile_requirements.txt /root/requirements.txt

tools/dockerfile/manylinux/Dockerfile-130

@@ -0,0 +1,71 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+ARG CUDA_VERSION=13.0
+ARG BASE_TARGET=cuda${CUDA_VERSION}
+
+FROM nvcr.io/nvidia/cuda:13.0.1-cudnn-devel-ubuntu22.04 as base
+MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
+
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+ARG PYTHON_VERSION=3.10
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION}/compat:/usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib:$LD_LIBRARY_PATH
+
+ENV HOME /root
+
+RUN apt-get update --allow-unauthenticated && \
+    apt-get install -y --no-install-recommends \
+      git \
+      vim \
+      curl \
+      wget \
+      make \
+      libgl1 \
+      libglib2.0-0 \
+      libssl-dev \
+      autoconf \
+      automake \
+      libtool \
+      libmlx5-1 \
+      libibverbs-dev \
+      python${PYTHON_VERSION} \
+      python${PYTHON_VERSION}-dev \
+      python3-pip && \
+    ln -sf /usr/bin/python3 /usr/bin/python && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /home
+RUN wget -q https://cmake.org/files/v3.31/cmake-3.31.0-linux-x86_64.tar.gz && \
+    tar -zxf cmake-3.31.0-linux-x86_64.tar.gz && \
+    rm cmake-3.31.0-linux-x86_64.tar.gz && \
+    rm -rf /home/cmake-3.31.0-linux-x86_64/doc /home/cmake-3.31.0-linux-x86_64/man
+
+ENV PATH=/home/cmake-3.31.0-linux-x86_64/bin:$PATH
+
+
+ARG TMP_DIR=patchelf_tmp
+RUN rm -rf "$TMP_DIR" && git clone -b 0.15.0 https://github.com/NixOS/patchelf "$TMP_DIR" && \
+    cd "$TMP_DIR" && ./bootstrap.sh && \
+    ./configure && make && make install && \
+    cd .. && rm -rf "$TMP_DIR"
+
+RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
+    tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
+    mkdir build && cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
+    make -j8 && make install && \
+    ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
+    cd ../../ && rm -rf ccache-4.8.2.tar.gz && rm -rf ccache-4.8.2
+
+COPY paddle/scripts/compile_requirements.txt /root
+COPY python/requirements.txt /root
+RUN pip install -r /root/requirements.txt && \
+    pip install -r /root/compile_requirements.txt && \
+    rm -rf /root/compile_requirements.txt /root/requirements.txt

tools/dockerfile/manylinux/Dockerfile-132

@@ -0,0 +1,115 @@
+# A image for building paddle binaries
+# Use cuda devel base image for both cpu and gpu environment
+# When you modify it, please be aware of cudnn-runtime version
+ARG CUDA_VERSION=13.2
+ARG BASE_TARGET=cuda${CUDA_VERSION}
+
+FROM nvcr.io/nvidia/cuda:13.2.0-cudnn-devel-ubuntu24.04 as base
+LABEL maintainer="paddle-dev@baidu.com"
+
+
+# ENV variables
+ARG WITH_GPU
+ARG WITH_AVX
+ARG PYTHON_VERSION=3.12
+ARG PYTHON_SOURCE_VERSION=3.12.13
+ARG TMP_DIR=patchelf_tmp
+
+ENV WITH_GPU=${WITH_GPU:-ON}
+ENV WITH_AVX=${WITH_AVX:-ON}
+ENV DEBIAN_FRONTEND=noninteractive
+ENV GDRCOPY_HOME=/usr/local/gdrcopy
+
+ENV HOME /root
+
+RUN apt-get update --allow-unauthenticated && \
+    apt-get install -y --no-install-recommends \
+      git \
+      vim \
+      curl \
+      wget \
+      make \
+      zstd \
+      rsync \
+      ca-certificates \
+      build-essential \
+      libgl1 \
+      libglib2.0-0 \
+      libssl-dev \
+      autoconf \
+      automake \
+      libtool \
+      libmlx5-1 \
+      libibverbs-dev \
+      pkg-config \
+      zlib1g-dev \
+      libbz2-dev \
+      libreadline-dev \
+      libsqlite3-dev \
+      libncursesw5-dev \
+      libffi-dev \
+      libgdbm-dev \
+      libgdbm-compat-dev \
+      liblzma-dev \
+      libexpat1-dev \
+      tk-dev \
+      uuid-dev \
+      xz-utils \
+      ccache \
+      libnccl2=2.29.7-1+cuda13.2 \
+      libnccl-dev=2.29.7-1+cuda13.2 && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN cd /tmp && \
+    wget -q https://www.python.org/ftp/python/${PYTHON_SOURCE_VERSION}/Python-${PYTHON_SOURCE_VERSION}.tar.xz && \
+    tar -xf Python-${PYTHON_SOURCE_VERSION}.tar.xz && \
+    cd Python-${PYTHON_SOURCE_VERSION} && \
+    ./configure \
+      --prefix=/usr/local \
+      --enable-optimizations \
+      --with-ensurepip=install && \
+    make -j"$(nproc)" && \
+    make altinstall && \
+    ln -sf /usr/local/bin/python${PYTHON_VERSION} /usr/local/bin/python3 && \
+    ln -sf /usr/local/bin/python3 /usr/local/bin/python && \
+    ln -sf /usr/local/bin/pip${PYTHON_VERSION} /usr/local/bin/pip3 && \
+    ln -sf /usr/local/bin/pip3 /usr/local/bin/pip && \
+    python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel packaging && \
+    cd / && \
+    rm -rf /tmp/Python-${PYTHON_SOURCE_VERSION} /tmp/Python-${PYTHON_SOURCE_VERSION}.tar.xz
+
+WORKDIR /home
+RUN set -eux; \
+        if [ "${TARGETARCH:-}" = "amd64" ] || [ "$(uname -m)" = "x86_64" ]; then \
+            CMAKE_ARCH="x86_64"; \
+        elif [ "${TARGETARCH:-}" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then \
+            CMAKE_ARCH="aarch64"; \
+        else \
+            echo "Unsupported architecture: TARGETARCH=${TARGETARCH:-unknown}, uname -m=$(uname -m)"; \
+            exit 1; \
+        fi; \
+        CMAKE_PKG="cmake-3.31.0-linux-${CMAKE_ARCH}.tar.gz"; \
+        wget -q "https://cmake.org/files/v3.31/${CMAKE_PKG}"; \
+        tar -zxf "${CMAKE_PKG}"; \
+        rm "${CMAKE_PKG}"; \
+        mv "/home/cmake-3.31.0-linux-${CMAKE_ARCH}" /home/cmake-3.31.0; \
+        rm -rf /home/cmake-3.31.0/doc /home/cmake-3.31.0/man
+
+ENV PATH=/home/cmake-3.31.0/bin:$PATH
+
+RUN rm -rf "$TMP_DIR" && git clone --depth 1 --branch 0.15.0 https://github.com/NixOS/patchelf "$TMP_DIR" && \
+    cd "$TMP_DIR" && ./bootstrap.sh && \
+    ./configure && make && make install && \
+    cd .. && rm -rf "$TMP_DIR"
+
+COPY paddle/scripts/compile_requirements.txt /root
+COPY python/requirements.txt /root
+RUN pip config set global.break-system-packages true && \
+    pip install -r /root/requirements.txt && \
+    pip install -r /root/compile_requirements.txt && \
+    rm -rf /root/compile_requirements.txt /root/requirements.txt
+
+RUN cd /usr/local && \
+    wget -q https://paddle-ci.gz.bcebos.com/gdrcopy.tar && \
+    tar xf gdrcopy.tar && \
+    rm -f gdrcopy.tar

tools/dockerfile/manylinux/common/install_ccache.sh

@@ -0,0 +1,21 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz
+tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2
+mkdir build && cd build
+cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 ..
+make -j8 && make install
+ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache
+cd ../../ && rm -rf ccache-4.8.2.tar.gz

tools/dockerfile/manylinux/common/install_cuda.sh

@@ -0,0 +1,530 @@
+#!/bin/bash
+
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+NCCL_VERSION=v2.21.5-1
+CUDNN_VERSION=9.5.1.17
+
+function install_cusparselt_040 {
+    # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html
+    mkdir tmp_cusparselt && pushd tmp_cusparselt
+    wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.4.0.7-archive.tar.xz
+    tar xf libcusparse_lt-linux-x86_64-0.4.0.7-archive.tar.xz
+    cp -a libcusparse_lt-linux-x86_64-0.4.0.7-archive/include/* /usr/local/cuda/include/
+    cp -a libcusparse_lt-linux-x86_64-0.4.0.7-archive/lib/* /usr/local/cuda/lib64/
+    popd
+    rm -rf tmp_cusparselt
+}
+
+function install_cusparselt_052 {
+    # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html
+    mkdir tmp_cusparselt && pushd tmp_cusparselt
+    wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.5.2.1-archive.tar.xz
+    tar xf libcusparse_lt-linux-x86_64-0.5.2.1-archive.tar.xz
+    cp -a libcusparse_lt-linux-x86_64-0.5.2.1-archive/include/* /usr/local/cuda/include/
+    cp -a libcusparse_lt-linux-x86_64-0.5.2.1-archive/lib/* /usr/local/cuda/lib64/
+    popd
+    rm -rf tmp_cusparselt
+}
+
+function install_cusparselt_062 {
+    # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html
+    mkdir tmp_cusparselt && pushd tmp_cusparselt
+    wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.6.2.3-archive.tar.xz
+    tar xf libcusparse_lt-linux-x86_64-0.6.2.3-archive.tar.xz
+    cp -a libcusparse_lt-linux-x86_64-0.6.2.3-archive/include/* /usr/local/cuda/include/
+    cp -a libcusparse_lt-linux-x86_64-0.6.2.3-archive/lib/* /usr/local/cuda/lib64/
+    popd
+    rm -rf tmp_cusparselt
+}
+
+function install_cusparselt_063 {
+    # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html
+    mkdir tmp_cusparselt && pushd tmp_cusparselt
+    wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.6.3.2-archive.tar.xz
+    tar xf libcusparse_lt-linux-x86_64-0.6.3.2-archive.tar.xz
+    cp -a libcusparse_lt-linux-x86_64-0.6.3.2-archive/include/* /usr/local/cuda/include/
+    cp -a libcusparse_lt-linux-x86_64-0.6.3.2-archive/lib/* /usr/local/cuda/lib64/
+    popd
+    rm -rf tmp_cusparselt
+}
+
+function install_cusparselt_090_cuda13 {
+    # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html
+    mkdir tmp_cusparselt && pushd tmp_cusparselt
+    wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.9.0.3_cuda13-archive.tar.xz
+    tar xf libcusparse_lt-linux-x86_64-0.9.0.3_cuda13-archive.tar.xz
+    cp -a libcusparse_lt-linux-x86_64-0.9.0.3_cuda13-archive/include/* /usr/local/cuda/include/
+    cp -a libcusparse_lt-linux-x86_64-0.9.0.3_cuda13-archive/lib/* /usr/local/cuda/lib64/
+    popd
+    rm -rf tmp_cusparselt
+}
+
+function install_cusparselt_081 {
+    # cuSparseLt license: https://docs.nvidia.com/cuda/cusparselt/license.html
+    mkdir tmp_cusparselt && pushd tmp_cusparselt
+    wget -q https://developer.download.nvidia.com/compute/cusparselt/redist/libcusparse_lt/linux-x86_64/libcusparse_lt-linux-x86_64-0.8.1.1_cuda13-archive.tar.xz
+    tar xf libcusparse_lt-linux-x86_64-0.8.1.1_cuda13-archive.tar.xz
+    cp -a libcusparse_lt-linux-x86_64-0.8.1.1_cuda13-archive/include/* /usr/local/cuda/include/
+    cp -a libcusparse_lt-linux-x86_64-0.8.1.1_cuda13-archive/lib/* /usr/local/cuda/lib64/
+    popd
+    rm -rf tmp_cusparselt
+}
+
+function install_nccl_2162 {
+    wget -q https://nccl2-deb.cdn.bcebos.com/nccl_2.16.2-1+cuda11.8_x86_64.txz --no-check-certificate --no-proxy
+    tar xf nccl_2.16.2-1+cuda11.8_x86_64.txz
+    cp -a nccl_2.16.2-1+cuda11.8_x86_64/include/* /usr/include/
+    cp -a nccl_2.16.2-1+cuda11.8_x86_64/lib/* /usr/lib64
+    rm -rf nccl_2.16.2-1+cuda11.8_x86_64 nccl_2.16.2-1+cuda11.8_x86_64.txz
+}
+
+function install_nccl_2203 {
+    wget -q https://nccl2-deb.cdn.bcebos.com/nccl_2.20.3-1+cuda12.3_x86_64.txz --no-check-certificate --no-proxy
+    tar xf nccl_2.20.3-1+cuda12.3_x86_64.txz
+    cp -a nccl_2.20.3-1+cuda12.3_x86_64/include/* /usr/include/
+    cp -a nccl_2.20.3-1+cuda12.3_x86_64/lib/* /usr/lib64
+    rm -rf nccl_2.20.3-1+cuda12.3_x86_64 nccl_2.20.3-1+cuda12.3_x86_64.txz
+}
+
+function install_nccl_2215 {
+    wget -q https://nccl2-deb.cdn.bcebos.com/nccl_2.21.5-1+cuda12.4_x86_64.txz --no-check-certificate --no-proxy
+    tar xf nccl_2.21.5-1+cuda12.4_x86_64.txz
+    cp -a nccl_2.21.5-1+cuda12.4_x86_64/include/* /usr/include/
+    cp -a nccl_2.21.5-1+cuda12.4_x86_64/lib/* /usr/lib64
+    rm -rf nccl_2.21.5-1+cuda12.4_x86_64 nccl_2.21.5-1+cuda12.4_x86_64.txz
+}
+
+function install_nccl_2234 {
+    wget -q https://nccl2-deb.cdn.bcebos.com/nccl_2.23.4-1+cuda12.6_x86_64.txz --no-check-certificate --no-proxy
+    tar xf nccl_2.23.4-1+cuda12.6_x86_64.txz
+    cp -a nccl_2.23.4-1+cuda12.6_x86_64/include/* /usr/include/
+    cp -a nccl_2.23.4-1+cuda12.6_x86_64/lib/* /usr/lib64
+    rm -rf nccl_2.23.4-1+cuda12.6_x86_64 nccl_2.23.4-1+cuda12.6_x86_64.txz
+}
+
+function install_nccl_2297_cuda132 {
+    yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/cuda-rhel8.repo
+    # `install_132` uses toolkit-only installation, so install the official
+    # forward-compat driver libraries to provide /usr/local/cuda-13.2/compat/libcuda.so.1.
+    yum install -y \
+        cuda-compat-13-2 \
+        libnccl-2.29.7-1+cuda13.2 \
+        libnccl-devel-2.29.7-1+cuda13.2 \
+        libnccl-static-2.29.7-1+cuda13.2
+}
+
+function install_nccl_2283 {
+    wget -q https://nccl2-deb.cdn.bcebos.com/nccl_2.28.3-1+cuda13.0_x86_64.txz --no-check-certificate --no-proxy
+    tar xf nccl_2.28.3-1+cuda13.0_x86_64.txz
+    cp -a nccl_2.28.3-1+cuda13.0_x86_64/include/* /usr/include/
+    cp -a nccl_2.28.3-1+cuda13.0_x86_64/lib/* /usr/lib64
+    rm -rf nccl_2.28.3-1+cuda13.0_x86_64 nccl_2.28.3-1+cuda13.0_x86_64.txz
+}
+
+function install_trt_8616 {
+    wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-11.8.tar.gz --no-check-certificate --no-proxy
+    tar -zxf TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-11.8.tar.gz -C /usr/local
+    cp -rf /usr/local/TensorRT-8.6.1.6/include/* /usr/include/ && cp -rf /usr/local/TensorRT-8.6.1.6/lib/* /usr/lib/
+    rm -f TensorRT-8.6.1.6.Linux.x86_64-gnu.cuda-11.8.tar.gz
+}
+
+function install_trt_105018 {
+    wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT-10.5.0.18.Linux.x86_64-gnu.cuda-12.6.tar.gz --no-check-certificate --no-proxy
+    tar -zxf TensorRT-10.5.0.18.Linux.x86_64-gnu.cuda-12.6.tar.gz -C /usr/local
+    cp -rf /usr/local/TensorRT-10.5.0.18/include/* /usr/include/ && cp -rf /usr/local/TensorRT-10.5.0.18/lib/* /usr/lib/
+    rm -f TensorRT-10.5.0.18.Linux.x86_64-gnu.cuda-12.6.tar.gz
+}
+
+function install_trt_1016111 {
+    wget -q https://developer.download.nvidia.com/compute/machine-learning/tensorrt/10.16.1/tars/TensorRT-10.16.1.11.Linux.x86_64-gnu.cuda-13.2.tar.gz --no-check-certificate --no-proxy
+    tar -zxf TensorRT-10.16.1.11.Linux.x86_64-gnu.cuda-13.2.tar.gz -C /usr/local
+    cp -rf /usr/local/TensorRT-10.16.1.11/include/* /usr/include/ && cp -rf /usr/local/TensorRT-10.16.1.11/lib/* /usr/lib/
+    rm -f TensorRT-10.16.1.11.Linux.x86_64-gnu.cuda-13.2.tar.gz
+}
+
+function install_trt_101339 {
+    wget -q https://paddle-ci.gz.bcebos.com/TRT/TensorRT-10.13.3.9.Linux.x86_64-gnu.cuda-13.0.tar.gz --no-check-certificate --no-proxy
+    tar -zxf TensorRT-10.13.3.9.Linux.x86_64-gnu.cuda-13.0.tar.gz -C /usr/local
+    cp -rf /usr/local/TensorRT-10.13.3.9/include/* /usr/include/ && cp -rf /usr/local/TensorRT-10.13.3.9/lib/* /usr/lib/
+    rm -f TensorRT-10.13.3.9.Linux.x86_64-gnu.cuda-13.0.tar.gz
+}
+
+function install_118 {
+    CUDNN_VERSION=8.9.7.29
+    NCCL_VERSION=2.16.5
+    TensorRT_VERSION=8.6.1.6
+    echo "Installing CUDA 11.8 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and TensorRT ${TensorRT_VERSION} and cuSparseLt-0.4.0"
+    rm -rf /usr/local/cuda-11.8 /usr/local/cuda
+    # install CUDA 11.8.0 in the same container
+    wget -q https://developer.download.nvidia.com/compute/cuda/11.8.0/local_installers/cuda_11.8.0_520.61.05_linux.run
+    chmod +x cuda_11.8.0_520.61.05_linux.run
+    ./cuda_11.8.0_520.61.05_linux.run --toolkit --driver --silent --no-drm --kernel-source-path=/usr/src/kernels/4.18.0-553.34.1.el8_10.x86_64
+    rm -f cuda_11.8.0_520.61.05_linux.run
+    rm -f /usr/local/cuda && ln -s /usr/local/cuda-11.8 /usr/local/cuda
+    rm -rf /usr/bin/nvidia-smi
+
+    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
+    mkdir tmp_cudnn && cd tmp_cudnn
+    wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive.tar.xz
+    tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive.tar.xz
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive/include/* /usr/local/cuda/include/
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda11-archive/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_cudnn
+
+    install_nccl_2162
+    install_trt_8616
+    install_cusparselt_040
+
+    ldconfig
+}
+
+function install_123 {
+    CUDNN_VERSION=9.1.1.17
+    NCCL_VERSION=2.20.3
+    TensorRT_VERSION=10.5
+    echo "Installing CUDA 12.3 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and TensorRT ${TensorRT_VERSION} and cuSparseLt-0.5.2"
+    rm -rf /usr/local/cuda-12.3 /usr/local/cuda
+    # install CUDA 12.3.0 in the same container
+    wget -q https://developer.download.nvidia.com/compute/cuda/12.3.2/local_installers/cuda_12.3.2_545.23.08_linux.run
+    chmod +x cuda_12.3.2_545.23.08_linux.run
+    ./cuda_12.3.2_545.23.08_linux.run --toolkit --driver --silent --kernel-source-path=/usr/src/kernels/4.18.0-553.34.1.el8_10.x86_64
+    rm -f cuda_12.3.2_545.23.08_linux.run
+    rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.3 /usr/local/cuda
+    rm -rf /usr/bin/nvidia-smi
+
+    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
+    mkdir tmp_cudnn && cd tmp_cudnn
+    wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
+    tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/include/* /usr/local/cuda/include/
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_cudnn
+
+    install_nccl_2203
+    install_trt_105018
+    install_cusparselt_052
+
+    ldconfig
+}
+
+function install_124 {
+    CUDNN_VERSION=9.1.1.17
+    NCCL=2.21.5
+    TensorRT_VERSION=10.5
+    echo "Installing CUDA 12.4.1 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and TensorRT ${TensorRT_VERSION} and cuSparseLt-0.6.2"
+    rm -rf /usr/local/cuda-12.4 /usr/local/cuda
+    # install CUDA 12.4.1 in the same container
+    wget -q https://developer.download.nvidia.com/compute/cuda/12.4.1/local_installers/cuda_12.4.1_550.54.15_linux.run
+    chmod +x cuda_12.4.1_550.54.15_linux.run
+    ./cuda_12.4.1_550.54.15_linux.run --toolkit --driver --silent --kernel-source-path=/usr/src/kernels/4.18.0-553.34.1.el8_10.x86_64
+    rm -f cuda_12.4.1_550.54.15_linux.run
+    rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.4 /usr/local/cuda
+    rm -rf /usr/bin/nvidia-smi
+
+    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
+    mkdir tmp_cudnn && cd tmp_cudnn
+    wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
+    tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/include/* /usr/local/cuda/include/
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_cudnn
+
+    install_nccl_2215
+    install_trt_105018
+    install_cusparselt_062
+
+    ldconfig
+}
+
+function install_126 {
+    CUDNN_VERSION=9.5.1.17
+    NCCL_VERSION=2.23.4
+    TensorRT_VERSION=10.5
+    echo "Installing CUDA 12.6.3 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and TensorRT ${TensorRT_VERSION} and cuSparseLt-0.6.3"
+    rm -rf /usr/local/cuda-12.6 /usr/local/cuda
+    # install CUDA 12.6.3 in the same container
+    wget -q https://developer.download.nvidia.com/compute/cuda/12.6.3/local_installers/cuda_12.6.3_560.35.05_linux.run
+    chmod +x cuda_12.6.3_560.35.05_linux.run
+    ./cuda_12.6.3_560.35.05_linux.run --toolkit --driver --silent --kernel-source-path=/usr/src/kernels/4.18.0-553.34.1.el8_10.x86_64
+    rm -f cuda_12.6.3_560.35.05_linux.run
+    rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.6 /usr/local/cuda
+    rm -rf /usr/bin/nvidia-smi
+
+    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
+    mkdir tmp_cudnn && cd tmp_cudnn
+    wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
+    tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/include/* /usr/local/cuda/include/
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_cudnn
+
+    install_nccl_2234
+    install_trt_105018
+    install_cusparselt_063
+
+    ldconfig
+}
+
+
+function install_129 {
+    CUDNN_VERSION=9.9.0.52
+    NCCL_VERSION=2.23.4
+    TensorRT_VERSION=10.5
+    echo "Installing CUDA 12.9.0 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and TensorRT ${TensorRT_VERSION} and cuSparseLt-0.6.3"
+    rm -rf /usr/local/cuda-12.9 /usr/local/cuda
+    # install CUDA 12.9.0 in the same container
+    wget -q https://developer.download.nvidia.com/compute/cuda/12.9.0/local_installers/cuda_12.9.0_575.51.03_linux.run
+    chmod +x cuda_12.9.0_575.51.03_linux.run
+    ./cuda_12.9.0_575.51.03_linux.run --toolkit --driver --silent --kernel-source-path=/usr/src/kernels/4.18.0-553.34.1.el8_10.x86_64
+    rm -f cuda_12.9.0_575.51.03_linux.run
+    rm -f /usr/local/cuda && ln -s /usr/local/cuda-12.9 /usr/local/cuda
+    rm -rf /usr/bin/nvidia-smi
+
+    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
+    mkdir tmp_cudnn && cd tmp_cudnn
+    wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
+    tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive.tar.xz
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/include/* /usr/local/cuda/include/
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda12-archive/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_cudnn
+
+    install_nccl_2234
+    install_trt_105018
+    install_cusparselt_063
+
+    ldconfig
+}
+
+function install_130 {
+    CUDNN_VERSION=9.13.0.50
+    NCCL_VERSION=2.28.3
+    TensorRT_VERSION=10.13
+    echo "Installing CUDA 13.0.1 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and TensorRT ${TensorRT_VERSION} and cuSparseLt-0.8.1"
+    rm -rf /usr/local/cuda-13.0 /usr/local/cuda
+    # install CUDA 13.0.1 in the same container
+    wget -q https://developer.download.nvidia.com/compute/cuda/13.0.1/local_installers/cuda_13.0.1_580.82.07_linux.run
+    chmod +x cuda_13.0.1_580.82.07_linux.run
+    ./cuda_13.0.1_580.82.07_linux.run --toolkit --driver --silent --kernel-source-path=/usr/src/kernels/4.18.0-553.76.1.el8_10.x86_64
+    rm -f cuda_13.0.1_580.82.07_linux.run
+    rm -f /usr/local/cuda && ln -s /usr/local/cuda-13.0 /usr/local/cuda
+    rm -rf /usr/bin/nvidia-smi
+
+    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
+    mkdir tmp_cudnn && cd tmp_cudnn
+    wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda13-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda13-archive.tar.xz
+    tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda13-archive.tar.xz
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda13-archive/include/* /usr/local/cuda/include/
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda13-archive/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_cudnn
+
+    install_nccl_2283
+    install_trt_101339
+    install_cusparselt_081
+
+    ldconfig
+}
+
+function install_132 {
+    CUDNN_VERSION=9.20.0.48
+    NCCL_VERSION=2.29.7
+    TensorRT_VERSION=10.16.1.11
+    echo "Installing CUDA 13.2.0 and cuDNN ${CUDNN_VERSION} and NCCL ${NCCL_VERSION} and TensorRT ${TensorRT_VERSION} and cuSparseLt-0.9.0"
+    rm -rf /usr/local/cuda-13.2 /usr/local/cuda
+    # install CUDA 13.2.0 in the same container
+    wget -q https://developer.download.nvidia.com/compute/cuda/13.2.0/local_installers/cuda_13.2.0_595.45.04_linux.run
+    chmod +x cuda_13.2.0_595.45.04_linux.run
+    ./cuda_13.2.0_595.45.04_linux.run --toolkit --driver --silent --kernel-source-path=/usr/src/kernels/4.18.0-553.34.1.el8_10.x86_64
+    rm -f cuda_13.2.0_595.45.04_linux.run
+    rm -f /usr/local/cuda && ln -s /usr/local/cuda-13.2 /usr/local/cuda
+    rm -rf /usr/bin/nvidia-smi
+
+    # cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
+    mkdir tmp_cudnn && cd tmp_cudnn
+    wget -q https://developer.download.nvidia.com/compute/cudnn/redist/cudnn/linux-x86_64/cudnn-linux-x86_64-${CUDNN_VERSION}_cuda13-archive.tar.xz -O cudnn-linux-x86_64-${CUDNN_VERSION}_cuda13-archive.tar.xz
+    tar xf cudnn-linux-x86_64-${CUDNN_VERSION}_cuda13-archive.tar.xz
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda13-archive/include/* /usr/local/cuda/include/
+    cp -a cudnn-linux-x86_64-${CUDNN_VERSION}_cuda13-archive/lib/* /usr/local/cuda/lib64/
+    cd ..
+    rm -rf tmp_cudnn
+
+    install_nccl_2297_cuda132
+    install_trt_1016111
+    install_cusparselt_090_cuda13
+
+    ldconfig
+}
+
+function prune_118 {
+    echo "Pruning CUDA 11.8 and cuDNN"
+    #####################################################################################
+    # CUDA 11.8 prune static libs
+    #####################################################################################
+    export NVPRUNE="/usr/local/cuda-11.8/bin/nvprune"
+    export CUDA_LIB_DIR="/usr/local/cuda-11.8/lib64"
+
+    export GENCODE="-gencode arch=compute_35,code=sm_35 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
+    export GENCODE_CUDNN="-gencode arch=compute_35,code=sm_35 -gencode arch=compute_37,code=sm_37 -gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
+
+    if [[ -n "$OVERRIDE_GENCODE" ]]; then
+        export GENCODE=$OVERRIDE_GENCODE
+    fi
+
+    # all CUDA libs except CuDNN and CuBLAS (cudnn and cublas need arch 3.7 included)
+    ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis"  \
+      | xargs -I {} bash -c \
+                "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}"
+
+    # prune CuDNN and CuBLAS
+    $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a
+    $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a
+
+    #####################################################################################
+    # CUDA 11.8 prune visual tools
+    #####################################################################################
+    export CUDA_BASE="/usr/local/cuda-11.8/"
+    rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2022.3.0 $CUDA_BASE/nsight-systems-2022.4.2/
+}
+
+function prune_123 {
+  echo "Pruning CUDA 12.3"
+  #####################################################################################
+  # CUDA 12.3 prune static libs
+  #####################################################################################
+    export NVPRUNE="/usr/local/cuda-12.3/bin/nvprune"
+    export CUDA_LIB_DIR="/usr/local/cuda-12.3/lib64"
+
+    export GENCODE="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
+    export GENCODE_CUDNN="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
+
+    if [[ -n "$OVERRIDE_GENCODE" ]]; then
+        export GENCODE=$OVERRIDE_GENCODE
+    fi
+
+    # all CUDA libs except CuDNN and CuBLAS
+    ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis"  \
+      | xargs -I {} bash -c \
+                "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}"
+
+    # prune CuDNN and CuBLAS
+    $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a
+    $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a
+
+    #####################################################################################
+    # CUDA 12.3 prune visual tools
+    #####################################################################################
+    export CUDA_BASE="/usr/local/cuda-12.3/"
+    rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2023.1.0 $CUDA_BASE/nsight-systems-2023.1.2/
+}
+
+function prune_124 {
+  echo "Pruning CUDA 12.4"
+  #####################################################################################
+  # CUDA 12.4 prune static libs
+  #####################################################################################
+  export NVPRUNE="/usr/local/cuda-12.4/bin/nvprune"
+  export CUDA_LIB_DIR="/usr/local/cuda-12.4/lib64"
+
+  export GENCODE="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
+  export GENCODE_CUDNN="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
+
+  if [[ -n "$OVERRIDE_GENCODE" ]]; then
+      export GENCODE=$OVERRIDE_GENCODE
+  fi
+  if [[ -n "$OVERRIDE_GENCODE_CUDNN" ]]; then
+      export GENCODE_CUDNN=$OVERRIDE_GENCODE_CUDNN
+  fi
+
+  # all CUDA libs except CuDNN and CuBLAS
+  ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis"  \
+      | xargs -I {} bash -c \
+                "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}"
+
+  # prune CuDNN and CuBLAS
+  $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a
+  $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a
+
+  #####################################################################################
+  # CUDA 12.4 prune visual tools
+  #####################################################################################
+  export CUDA_BASE="/usr/local/cuda-12.4/"
+  rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2024.1.0 $CUDA_BASE/nsight-systems-2023.4.4/
+}
+
+function prune_126 {
+  echo "Pruning CUDA 12.6"
+  #####################################################################################
+  # CUDA 12.6 prune static libs
+  #####################################################################################
+  export NVPRUNE="/usr/local/cuda-12.6/bin/nvprune"
+  export CUDA_LIB_DIR="/usr/local/cuda-12.6/lib64"
+
+  export GENCODE="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
+  export GENCODE_CUDNN="-gencode arch=compute_50,code=sm_50 -gencode arch=compute_60,code=sm_60 -gencode arch=compute_61,code=sm_61 -gencode arch=compute_70,code=sm_70 -gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_86,code=sm_86 -gencode arch=compute_90,code=sm_90"
+
+  if [[ -n "$OVERRIDE_GENCODE" ]]; then
+      export GENCODE=$OVERRIDE_GENCODE
+  fi
+  if [[ -n "$OVERRIDE_GENCODE_CUDNN" ]]; then
+      export GENCODE_CUDNN=$OVERRIDE_GENCODE_CUDNN
+  fi
+
+  # all CUDA libs except CuDNN and CuBLAS
+  ls $CUDA_LIB_DIR/ | grep "\.a" | grep -v "culibos" | grep -v "cudart" | grep -v "cudnn" | grep -v "cublas" | grep -v "metis"  \
+      | xargs -I {} bash -c \
+                "echo {} && $NVPRUNE $GENCODE $CUDA_LIB_DIR/{} -o $CUDA_LIB_DIR/{}"
+
+  # prune CuDNN and CuBLAS
+  $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublas_static.a -o $CUDA_LIB_DIR/libcublas_static.a
+  $NVPRUNE $GENCODE_CUDNN $CUDA_LIB_DIR/libcublasLt_static.a -o $CUDA_LIB_DIR/libcublasLt_static.a
+
+  #####################################################################################
+  # CUDA 12.6 prune visual tools
+  #####################################################################################
+  export CUDA_BASE="/usr/local/cuda-12.6/"
+  rm -rf $CUDA_BASE/libnvvp $CUDA_BASE/nsightee_plugins $CUDA_BASE/nsight-compute-2024.3.2 $CUDA_BASE/nsight-systems-2024.5.1/
+}
+
+# idiomatic parameter and option handling in sh
+while test $# -gt 0
+do
+    case "$1" in
+    11.8) install_118; prune_118
+        ;;
+    12.3) install_123; prune_123
+        ;;
+    12.4) install_124; prune_124
+        ;;
+    12.6) install_126; prune_126
+        ;;
+    12.9) install_129
+        ;;
+    13.0) install_130
+        ;;
+    13.2) install_132
+        ;;
+    *) echo "bad argument $1"; exit 1
+        ;;
+    esac
+    shift
+done

tools/dockerfile/manylinux/common/install_gdrcopy.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# install gdrcopy
+cd /usr/local
+wget -q https://paddle-ci.gz.bcebos.com/gdrcopy.tar
+tar xf gdrcopy.tar
+rm -f gdrcopy.tar
+ldconfig

tools/dockerfile/manylinux/common/install_openssl.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function check_var {
+    if [ -z "$1" ]; then
+        echo "required variable not defined"
+        exit 1
+    fi
+}
+
+function check_sha256sum {
+    local fname=$1
+    check_var ${fname}
+    local sha256=$2
+    check_var ${sha256}
+
+    echo "${sha256}  ${fname}" > ${fname}.sha256
+    sha256sum -c ${fname}.sha256
+    rm ${fname}.sha256
+}
+
+function do_openssl_build {
+    ./config -fPIC --prefix=/usr/local/ssl > /dev/null
+    make > /dev/null
+    make install > /dev/null
+    ln -sf /usr/lib64/libcrypto.so.1.1 /usr/local/ssl/lib/libcrypto.so.1.1
+}
+
+
+function build_openssl {
+    local openssl_fname=$1
+    check_var ${openssl_fname}
+    local openssl_sha256=$2
+    check_var ${openssl_sha256}
+    check_var ${OPENSSL_DOWNLOAD_URL}
+    curl -sLO ${OPENSSL_DOWNLOAD_URL}/${openssl_fname}.tar.gz
+    check_sha256sum ${openssl_fname}.tar.gz ${openssl_sha256}
+    tar -xzf ${openssl_fname}.tar.gz
+    (cd ${openssl_fname} && do_openssl_build)
+    rm -rf ${openssl_fname} ${openssl_fname}.tar.gz
+}
+
+OPENSSL_ROOT=openssl-1.1.1
+OPENSSL_HASH=2836875a0f89c03d0fdf483941512613a50cfb421d6fd94b9f41d7279d586a3d
+OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source
+build_openssl $OPENSSL_ROOT $OPENSSL_HASH

tools/dockerfile/manylinux/common/install_patchelf.sh

@@ -0,0 +1,37 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+TMP_DIR=patchelf_tmp
+
+gcc_version=$(gcc --version |awk 'NR==1{print $3}')
+if [ "$gcc_version" == "5.4.0" ];then
+    patchelf_version=0.10
+else
+    patchelf_version=0.15.0
+fi
+
+rm -rf "$TMP_DIR"
+
+git clone -b $patchelf_version https://github.com/NixOS/patchelf "$TMP_DIR"
+
+cd "$TMP_DIR"
+./bootstrap.sh
+./configure
+make
+make install
+
+cd ..
+rm -rf "$TMP_DIR"

tools/dockerfile/manylinux/common/install_python.sh

@@ -0,0 +1,121 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function check_var {
+    if [ -z "$1" ]; then
+        echo "required variable not defined"
+        exit 1
+    fi
+}
+
+function lex_pyver {
+    # Echoes Python version string padded with zeros
+    # Thus:
+    # 3.2.1 -> 003002001
+    # 3     -> 003000000
+    echo $1 | awk -F "." '{printf "%03d%03d%03d", $1, $2, $3}'
+}
+
+
+function do_cpython_build {
+    local py_ver=$1
+    check_var $py_ver
+    local ucs_setting=$2
+    check_var $ucs_setting
+    tar -xzf Python-$py_ver.tgz
+    pushd Python-$py_ver
+    if [ "$ucs_setting" = "none" ]; then
+        unicode_flags=""
+        dir_suffix=""
+    else
+        local unicode_flags="--enable-unicode=$ucs_setting"
+        local dir_suffix="-$ucs_setting"
+    fi
+    local prefix="/opt/_internal/cpython-${py_ver}${dir_suffix}"
+    mkdir -p ${prefix}/lib
+
+    #if [ $1 -eq '3.13.0t' ];then
+    #    GIL='--disable-gil'
+    #fi
+
+    LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} CFLAGS="-Wformat" LDFLAGS="-Wl,-rpath,${prefix}/lib" ./configure --prefix=${prefix} --enable-shared $unicode_flags > /dev/null
+    LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} make -j8 > /dev/null
+    LD_LIBRARY_PATH=/usr/local/lib:${LD_LIBRARY_PATH} make install > /dev/null
+
+    popd
+    echo "ZZZ looking for libpython"
+    find / -name 'libpython*.so*'
+    rm -rf Python-$py_ver
+    # Some python's install as bin/python3. Make them available as bin/python.
+    if [ -e ${prefix}/bin/python3.10 ]; then
+        ln -s python3.10 ${prefix}/bin/python
+    fi
+    if [ -e ${prefix}/bin/python3.11 ]; then
+        ln -s python3.11 ${prefix}/bin/python
+    fi
+    if [ -e ${prefix}/bin/python3.12 ]; then
+        ln -s python3.12 ${prefix}/bin/python
+    fi
+    if [ -e ${prefix}/bin/python3.13 ]; then
+        ln -s python3.13 ${prefix}/bin/python
+    fi
+    if [ -e ${prefix}/bin/python3.13t ]; then
+        ln -s python3.13t ${prefix}/bin/python
+    fi
+    if [ -e ${prefix}/bin/python3.14 ]; then
+        ln -s python3.14 ${prefix}/bin/python
+    fi
+    if [ -e ${prefix}/bin/python3.14t ]; then
+        ln -s python3.14t ${prefix}/bin/python
+    fi
+
+    # NOTE Make libpython shared library visible to python calls below
+    LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python -m pip config set global.trusted-host mirrors.aliyun.com
+    LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python -m pip config set global.index-url http://mirrors.aliyun.com/pypi/simple/
+    LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/python get-pip.py
+    LD_LIBRARY_PATH="/usr/local/ssl/lib:${prefix}/lib" ${prefix}/bin/pip install wheel==0.40.0
+    cd /
+    ls ${MY_DIR}
+    abi_version=$(LD_LIBRARY_PATH="${prefix}/lib" ${prefix}/bin/python -V|awk '{print $2}'|awk -F '.' '{print $1$2}')
+    local abi_tag=$(echo cp$abi_version-cp$abi_version)
+    ln -s ${prefix} /opt/python/${abi_tag}
+}
+
+
+function build_cpython {
+    local py_ver=$1
+    check_var $py_ver
+    check_var $PYTHON_DOWNLOAD_URL
+    wget -q $PYTHON_DOWNLOAD_URL/$py_ver/Python-$py_ver.tgz
+    do_cpython_build $py_ver none
+    rm -f Python-$py_ver.tgz
+}
+
+
+function build_cpythons {
+    for py_ver in $@; do
+        check_var $GET_PIP_URL
+        curl -sLO $GET_PIP_URL
+        build_cpython $py_ver
+    done
+    rm -f get-pip.py
+    rm -f ez_setup.py
+}
+
+PYTHON_DOWNLOAD_URL=https://www.python.org/ftp/python
+GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
+CPYTHON_VERSIONS="3.14.0 3.13.0 3.12.0 3.11.0 3.10.0"
+
+mkdir -p /opt/python
+build_cpythons $CPYTHON_VERSIONS

tools/dockerfile/ubuntu20_dev.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function base_image(){
+  if [[ ${ref_CUDA_MAJOR} == "11.8" ]];then
+    dockerfile_name="Dockerfile-118"
+    sed "s#<baseimg>#nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.ubuntu20 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-11.8/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
+    sed -i 's#cudnn841#cudnn897#g' ${dockerfile_name}
+    sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=8.6.0#g' ${dockerfile_name}
+  else
+    echo "Dockerfile ERROR!!!"
+    exit 1
+  fi
+
+}
+
+export ref_CUDA_MAJOR=11.8
+base_image

tools/dockerfile/ubuntu20_release.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function base_image(){
+  if [[ ${ref_CUDA_MAJOR} == "11.2" ]];then
+    dockerfile_name="Dockerfile-112"
+    sed "s#<baseimg>#nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.release.ubuntu20 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-11.2/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name}
+    sed -i "s#gcc121#gcc82#g" ${dockerfile_name}
+    sed -i "s#gcc-12.1#gcc-8.2#g" ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8034#g' ${dockerfile_name}
+    sed -i 's#cudnn841#cudnn821#g' ${dockerfile_name}
+    sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=8.2.1#g' ${dockerfile_name}
+  elif [[ ${ref_CUDA_MAJOR} == "11.8" ]];then
+    dockerfile_name="Dockerfile-118"
+    sed "s#<baseimg>#nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.release.ubuntu20 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-11.8/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name}
+    sed -i "s#gcc121#gcc82#g" ${dockerfile_name}
+    sed -i "s#gcc-12.1#gcc-8.2#g" ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8531#g' ${dockerfile_name}
+    sed -i 's#cudnn841#cudnn860#g' ${dockerfile_name}
+    sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=8.6.0#g' ${dockerfile_name}
+  elif [[ ${ref_CUDA_MAJOR} == "12.0" ]];then
+    dockerfile_name="Dockerfile-120"
+    sed "s#<baseimg>#nvidia/cuda:12.0.0-cudnn8-devel-ubuntu20.04#g" ./Dockerfile.release.ubuntu20 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.0/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-12.0/bin:\$PATH #g" ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
+    sed -i 's#cudnn841#cudnn891#g' ${dockerfile_name}
+    sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=8.9.1#g' ${dockerfile_name}
+  elif [[ ${ref_CUDA_MAJOR} == "12.3" ]];then
+    dockerfile_name="Dockerfile-123"
+    sed "s#<baseimg>#nvidia/cuda:12.3.1-devel-ubuntu20.04#g" ./Dockerfile.release.ubuntu20 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.3/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-12.0/bin:\$PATH #g" ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
+    sed -i 's#cudnn841#cudnn900#g' ${dockerfile_name}
+    sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=9.0.0#g' ${dockerfile_name}
+  elif [[ ${ref_CUDA_MAJOR} == "12.6" ]];then
+    dockerfile_name="Dockerfile-126"
+    sed "s#<baseimg>#nvidia/cuda:12.6.0-devel-ubuntu20.04#g" ./Dockerfile.release.ubuntu20 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.s63/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-12.0/bin:\$PATH #g" ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt8616#g' ${dockerfile_name}
+    sed -i 's#cudnn841#cudnn911#g' ${dockerfile_name}
+    sed -i 's#CUDNN_VERSION=8.4.1#CUDNN_VERSION=9.1.1#g' ${dockerfile_name}
+  elif [[ ${ref_CUDA_MAJOR} == "0" ]];then
+    dockerfile_name="Dockerfile-cpu"
+    sed "s#<baseimg>#ubuntu:20.04#g" ./Dockerfile.release.ubuntu20 >${dockerfile_name}
+    sed -i "s#<setcuda>##g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>#RUN apt-get install -y gcc g++ make#g' ${dockerfile_name}
+    sed -i "s#<install_gcc>#WORKDIR /usr/bin ENV PATH=/usr/local/gcc-8.2/bin:\$PATH #g" ${dockerfile_name}
+    sed -i "s#gcc121#gcc82#g" ${dockerfile_name}
+    sed -i "s#gcc-12.1#gcc-8.2#g" ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh##g' ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_cudnn.sh cudnn841##g' ${dockerfile_name}
+    sed -i 's#ENV CUDNN_VERSION=8.4.1##g' ${dockerfile_name}
+    sed -i 's#RUN apt-key del 7fa2af80##g' ${dockerfile_name}
+    sed -i 's#RUN rm /etc/apt/sources.list.d/\*##g' ${dockerfile_name}
+    sed -i 's#RUN apt-key adv --fetch-keys https://developer.download.nvidia.cn/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub##g' ${dockerfile_name}
+    sed -i 's#ENV WITH_GPU=${WITH_GPU:-ON}#ENV WITH_GPU=${WITH_GPU:-OFF}#g' ${dockerfile_name}
+  else
+    echo "Dockerfile ERROR!!!"
+    exit 1
+  fi
+
+}
+
+
+export ref_CUDA_MAJOR=0
+base_image
+export ref_CUDA_MAJOR=11.2
+base_image
+export ref_CUDA_MAJOR=11.8
+base_image
+export ref_CUDA_MAJOR=12.0
+base_image
+export ref_CUDA_MAJOR=12.3
+base_image
+export ref_CUDA_MAJOR=12.6
+base_image

tools/dockerfile/ubuntu22_dev.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function base_image(){
+  if [[ ${ref_CUDA_MAJOR} == "12.3" ]];then
+    dockerfile_name="Dockerfile-123"
+    sed "s#<baseimg>#nvidia/cuda:12.3.2-cudnn9-devel-ubuntu22.04#g" ./Dockerfile.ubuntu22 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.3/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i 's#<install_cudnn>#RUN bash /build_scripts/install_cudnn.sh cudnn911#g' ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt105018#g' ${dockerfile_name}
+  elif [[ ${ref_CUDA_MAJOR} == "12.4" ]];then
+    dockerfile_name="Dockerfile-124"
+    sed "s#<baseimg>#nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04#g" ./Dockerfile.ubuntu22 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.4/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i 's#<install_cudnn>##g' ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt105018#g' ${dockerfile_name}
+  else
+    echo "Dockerfile ERROR!!!"
+    exit 1
+  fi
+
+}
+
+export ref_CUDA_MAJOR=12.3
+base_image
+export ref_CUDA_MAJOR=12.4
+base_image

tools/dockerfile/ubuntu24_dev.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function base_image(){
+   if [[ ${ref_CUDA_MAJOR} == "cpu" ]];then
+    dockerfile_name="Dockerfile-cpu"
+    sed "s#<baseimg>#ubuntu:24.04#g" ./Dockerfile.ubuntu24 >${dockerfile_name}
+    sed -i "s#<setcuda>##g" ${dockerfile_name}
+    sed -i "s#WITH_GPU:-ON#WITH_GPU:-OFF#g" ${dockerfile_name}
+    sed -i 's#RUN mv /etc/apt/sources.list.d/cuda.list /etc/apt/sources.list.d/cuda.list.bak##g' ${dockerfile_name}
+    sed -i 's#RUN curl .*3bf863cc.pub | gpg --dearmor | tee /usr/share/keyrings/cuda-archive-keyring.gpg##g' ${dockerfile_name}
+    sed -i 's#RUN mv /etc/apt/sources.list.d/cuda.list.bak /etc/apt/sources.list.d/cuda.list##g' ${dockerfile_name}
+    sed -i 's#RUN sed -i .*signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg.*cuda.list##g' ${dockerfile_name}
+    sed -i 's#<install_cpu_package>#RUN apt-get install -y gcc g++ make#g' ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh##g' ${dockerfile_name}
+    sed -i 's#ENV CUDNN_VERSION=9.5.0##g' ${dockerfile_name}
+  elif [[ ${ref_CUDA_MAJOR} == "12.6" ]];then
+    dockerfile_name="Dockerfile-126"
+    sed "s#<baseimg>#nvidia/cuda:12.6.2-cudnn-devel-ubuntu24.04#g" ./Dockerfile.ubuntu24 >${dockerfile_name}
+    sed -i "s#<setcuda>#ENV LD_LIBRARY_PATH=/usr/local/cuda-12.6/targets/x86_64-linux/lib:\$LD_LIBRARY_PATH #g" ${dockerfile_name}
+    sed -i 's#<install_cpu_package>##g' ${dockerfile_name}
+    sed -i 's#RUN bash /build_scripts/install_trt.sh#RUN bash /build_scripts/install_trt.sh trt105018#g' ${dockerfile_name}
+  else
+    echo "Dockerfile ERROR!!!"
+    exit 1
+  fi
+
+}
+
+export ref_CUDA_MAJOR=cpu
+base_image
+export ref_CUDA_MAJOR=12.6
+base_image

tools/document_preview.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+is_shell_attribute_set() { # attribute, like "x"
+  case "$-" in
+    *"$1"*) return 0 ;;
+    *)    return 1 ;;
+  esac
+}
+function get_docs_pr_num_from_paddle_pr_info(){
+    # get_repo_pr_info's output
+    pr_info_file=$1
+    if [ ! -r ${pr_info_file} ] ; then
+        return 1
+    fi
+
+    declare -A arr_kv
+    while read line
+    do
+        echo "$line" | grep '^\w\+\s*=\s*.*' > /dev/null
+        if [ $? = 0 ] ; then
+            kv=($(echo $line | sed 's/=/\n/g'))
+            k=($(echo "${kv[0]}" | sed 's/\s//g'))
+            v=($(echo "${kv[1]}" | sed 's/^\s*//g' | sed 's/\s*$//g'))
+            # arr_kv[${kv[1]}]=${kv[2]}
+            arr_kv[${k}]=${v}
+        fi
+    done < <(jq -r '.body' ${pr_info_file})
+
+    echo ${arr_kv[PADDLEDOCS_PR]}
+    return 0
+}
+
+# Attention:
+# 1. /FluidDoc will be used as the workspace of PaddlePaddle/docs.
+# 2. And /docs is used as the output of doc-build process.
+# 3. If conflicted with yours, please modify the definition of FLUIDDOCDIR and
+#    OUTPUTDIR in the subsequent codes.
+# 4. The doc-build process is controlled under EnvVar BUILD_DOC and UPLOAD_DOC.
+#    All the Chinese and English docs will be generated, and then uploaded.
+
+PREVIEW_URL_PROMPT="ipipe_log_param_preview_url: None"
+BUILD_DOC=${BUILD_DOC:=false}
+UPLOAD_DOC=${UPLOAD_DOC:=false}
+
+CURPWD=${PWD}
+
+if [ -f /usr/local/python3.8.0/bin/sphinx-build ] ; then
+    if [ -f /usr/local/bin/sphinx-build ] ; then
+        rm /usr/local/bin/sphinx-build
+    fi
+    ln -s /usr/local/python3.8.0/bin/sphinx-build /usr/local/bin/sphinx-build
+fi
+
+if [ "${BUILD_DOC}" = "true" ] &&  [ -x /usr/local/bin/sphinx-build ] ; then
+    export FLUIDDOCDIR=${FLUIDDOCDIR:=/FluidDoc}
+    export OUTPUTDIR=${OUTPUTDIR:=/docs}
+    export VERSIONSTR=$(echo ${BRANCH} | sed 's@release/@@g')
+
+    if [ -d ${FLUIDDOCDIR} ] ; then
+        echo "${FLUIDDOCDIR} exists, git clone will be skipped, but git clean will be done."
+        cd ${FLUIDDOCDIR}
+        git reset --hard
+        git clean -dfx
+        cd ${CURPWD}
+    else
+        git clone -b ${BRANCH} --depth=1 https://github.com/PaddlePaddle/docs.git ${FLUIDDOCDIR}
+        if [ ! "$?" = "0" ] ; then
+            git clone --depth=1 https://github.com/PaddlePaddle/docs.git ${FLUIDDOCDIR}
+        fi
+    fi
+    if [ -d ${OUTPUTDIR} ] ; then
+        echo "$0: rm -rf ${OUTPUTDIR}"
+        rm -rf ${OUTPUTDIR}
+        mkdir -p ${OUTPUTDIR}
+    fi
+
+    # install requirements
+    export no_proxy=mirror.baidu.com,${no_proxy}
+    apt-get install -y --no-install-recommends doxygen jq
+    echo 'beautifulsoup4
+Markdown
+sphinx-sitemap
+sphinx-markdown-tables
+breathe
+exhale
+sphinx_design
+nbsphinx
+' >/tmp/doc-build.requirements && \
+    pip install --no-cache-dir -i https://mirror.baidu.com/pypi/simple -r /tmp/doc-build.requirements && \
+    rm /tmp/doc-build.requirements
+
+
+    source ${FLUIDDOCDIR}/ci_scripts/utils.sh
+    paddle_pr_info=$(get_repo_pr_info "PaddlePaddle/Paddle" ${GIT_PR_ID})
+    docs_pr_id=$(get_docs_pr_num_from_paddle_pr_info ${paddle_pr_info})
+    if [ -n "${docs_pr_id}" ] ; then
+        cd ${FLUIDDOCDIR}
+        git fetch --depth=1 origin pull/${docs_pr_id}/head
+        git checkout -b "pr${docs_pr_id}" FETCH_HEAD
+        git log --pretty=oneline -10
+    fi
+    echo "docs_pr_id=${docs_pr_id}"
+
+
+    # build doc
+    /bin/bash -x ${FLUIDDOCDIR}/ci_scripts/gendoc.sh
+    if [ $? -ne 0 ];then
+        echo 'gendoc error'
+        exit 1
+    fi
+
+    if [ "${UPLOAD_DOC}" = "true" ] ; then
+        curl -o /tmp/linux-bcecmd-0.3.0.zip https://sdk.bce.baidu.com/console-sdk/linux-bcecmd-0.3.0.zip && \
+        python -m zipfile -e /tmp/linux-bcecmd-0.3.0.zip /opt && \
+        chmod +x /opt/linux-bcecmd-0.3.0/bcecmd && \
+        rm /tmp/linux-bcecmd-0.3.0.zip && \
+        curl -o /tmp/boscmdconfig.tgz https://paddle-dev-tools-open.bj.bcebos.com/fluiddoc-preview/boscmdconfig.tgz && \
+        tar xzf /tmp/boscmdconfig.tgz -C /opt/linux-bcecmd-0.3.0/ && \
+        rm /tmp/boscmdconfig.tgz
+
+        # credentials file is empty, please build it if need.
+        BCECMD=/opt/linux-bcecmd-0.3.0/bcecmd
+        BCECMD_CONFIG=/opt/linux-bcecmd-0.3.0/boscmdconfig
+
+        is_shell_attribute_set x
+        xdebug_set=$?
+        if [ $xdebug_set ] ; then
+            set +x
+        fi
+        if [ -n "${BOS_CREDENTIAL_AK}" ] && [ -n "${BOS_CREDENTIAL_SK}" ] ; then
+            echo "Ak = ${BOS_CREDENTIAL_AK}" >> ${BCECMD_CONFIG}/credentials
+            echo "Sk = ${BOS_CREDENTIAL_SK}" >> ${BCECMD_CONFIG}/credentials
+        fi
+        if [ $xdebug_set ] ; then
+            set -x
+        fi
+
+        PREVIEW_JOB_NAME="preview-paddle-pr-${GIT_PR_ID}"
+        BOSBUCKET=${BOSBUCKET:=paddle-site-web-dev}
+        ${BCECMD} --conf-path ${BCECMD_CONFIG} bos sync "${OUTPUTDIR}/en/${VERSIONSTR}" "bos:/${BOSBUCKET}/documentation/en/${PREVIEW_JOB_NAME}" \
+            --delete --yes --exclude "${OUTPUTDIR}/en/${VERSIONSTR}/_sources/"
+        ${BCECMD} --conf-path ${BCECMD_CONFIG} bos sync "${OUTPUTDIR}/en/${VERSIONSTR}" "bos:/${BOSBUCKET}/documentation/en/${PREVIEW_JOB_NAME}" \
+            --delete --yes --exclude "${OUTPUTDIR}/en/${VERSIONSTR}/_sources/"
+        ${BCECMD} --conf-path ${BCECMD_CONFIG} bos sync "${OUTPUTDIR}/zh/${VERSIONSTR}" "bos:/${BOSBUCKET}/documentation/zh/${PREVIEW_JOB_NAME}" \
+            --delete --yes --exclude "${OUTPUTDIR}/zh/${VERSIONSTR}/_sources/"
+        ${BCECMD} --conf-path ${BCECMD_CONFIG} bos sync "${OUTPUTDIR}/zh/${VERSIONSTR}" "bos:/${BOSBUCKET}/documentation/zh/${PREVIEW_JOB_NAME}" \
+            --delete --yes --exclude "${OUTPUTDIR}/zh/${VERSIONSTR}/_sources/"
+        PREVIEW_URL_PROMPT="ipipe_log_param_preview_url: http://${PREVIEW_JOB_NAME}.${PREVIEW_SITE:-paddle.run}/documentation/docs/zh/api/index_cn.html"
+    fi
+fi
+
+cd ${CURPWD}
+# print the preview url
+echo "${PREVIEW_URL_PROMPT}"

tools/enforce/count_all_enforce.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to count all PADDLE checks in the paddle/fluid directory,
+#   including the total PADDLE check number, the valid check number and the
+#   invalid check number under paddle/fluid and its main sub-directories.
+
+# Usage: bash count_all_enforce.sh (run in tools directory)
+
+# Result Example:
+
+#     paddle/fluid/framework - total: 1065, valid: 267, invalid: 798
+#     paddle/fluid/imperative - total: 135, valid: 118, invalid: 17
+#     paddle/fluid/inference - total: 449, valid: 158, invalid: 291
+#     paddle/fluid/memory - total: 60, valid: 10, invalid: 50
+#     paddle/fluid/operators - total: 4225, valid: 1061, invalid: 3164
+#     paddle/fluid/platform - total: 240, valid: 39, invalid: 201
+#     paddle/fluid/pybind - total: 98, valid: 53, invalid: 45
+#     paddle/fluid/string - total: 0, valid: 0, invalid: 0
+#     paddle/fluid/testdata - total: 0, valid: 0, invalid: 0
+#     paddle/fluid/train - total: 6, valid: 0, invalid: 6
+#     ----------------------------
+#     PADDLE ENFORCE & THROW COUNT
+#     ----------------------------
+#     All PADDLE_ENFORCE{_**} & PADDLE_THROW Count: 6278
+#     Valid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: 1706
+#     Invalid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: 4572
+
+ROOT_DIR=../../paddle/fluid
+ALL_PADDLE_CHECK_CNT=0
+VALID_PADDLE_CHECK_CNT=0
+
+function enforce_count(){
+    paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
+    total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
+    valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
+    eval $2=$total_check_cnt
+    eval $3=$valid_check_cnt
+}
+
+function count_dir_recursively(){
+    for file in `ls $1`
+    do
+        if [ -d $1"/"$file ];then
+            level=$(($2+1))
+            if [ $level -le 1 ]; then
+                enforce_count $1"/"$file total_check_cnt valid_check_cnt
+                dir_name=$1
+                echo "${dir_name#../}/"$file" | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))"
+                ALL_PADDLE_CHECK_CNT=$(($ALL_PADDLE_CHECK_CNT+$total_check_cnt))
+                VALID_PADDLE_CHECK_CNT=$(($VALID_PADDLE_CHECK_CNT+$valid_check_cnt))
+                count_dir_recursively $1"/"$file $level
+            fi
+        fi
+    done
+}
+
+main() {
+    count_dir_recursively $ROOT_DIR 0
+
+    echo "----------------------------"
+    echo "PADDLE ENFORCE & THROW COUNT"
+    echo "----------------------------"
+    echo "All PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${ALL_PADDLE_CHECK_CNT}"
+    echo "Valid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${VALID_PADDLE_CHECK_CNT}"
+    echo "Invalid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: $(($ALL_PADDLE_CHECK_CNT-$VALID_PADDLE_CHECK_CNT))"
+}
+
+if [ "${1}" != "--source-only" ]; then
+    main "${@}"
+fi

tools/enforce/count_enforce_by_dir.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to count detail PADDLE checks in the paddle/fluid directory,
+#   contains the number of PADDLE checks under each folder, the statistical data
+#   does not include subdirectories, only covers all files under the current directory.
+#
+#   The three columns of data are: total number, valid number, invalid number.
+#   The output format is easy to display as a markdown table.
+
+# Usage: bash count_enforce_by_dir.sh (run in tools directory)
+
+# Result Example:
+
+#     paddle/fluid/operators/collective | 28 | 1 | 27
+#     paddle/fluid/operators/controlflow | 60 | 59 | 1
+#     paddle/fluid/operators/detection | 276 | 146 | 130
+#     paddle/fluid/operators/elementwise | 29 | 20 | 9
+#     paddle/fluid/operators/fused | 227 | 182 | 45
+#     paddle/fluid/operators/nccl | 27 | 0 | 27
+#     paddle/fluid/operators/optimizers | 214 | 50 | 164
+#     paddle/fluid/operators/reader | 40 | 14 | 26
+#     paddle/fluid/operators/reduce_ops | 8 | 8 | 0
+#     paddle/fluid/operators/sequence_ops | 167 | 47 | 120
+#     paddle/fluid/operators/tensorrt | 7 | 4 | 3
+#     paddle/fluid/operators | 2144 | 999 | 1145
+
+. ./count_all_enforce.sh --source-only
+
+ROOT_DIR=../../paddle/fluid
+
+function count_dir_independently(){
+    local sub_dir_total_check_cnt=0
+    local sub_dir_valid_check_cnt=0
+    for file in `ls $1`
+    do
+        if [ -d $1"/"$file ];then
+            enforce_count $1"/"$file dir_total_check_cnt dir_valid_check_cnt
+            sub_dir_total_check_cnt=$(($sub_dir_total_check_cnt+$dir_total_check_cnt))
+            sub_dir_valid_check_cnt=$(($sub_dir_valid_check_cnt+$dir_valid_check_cnt))
+
+            count_dir_independently $1"/"$file $dir_total_check_cnt $dir_valid_check_cnt
+        fi
+    done
+    total_check_cnt=$(($2-$sub_dir_total_check_cnt))
+    valid_check_cnt=$(($3-$sub_dir_valid_check_cnt))
+    dir_name=$1
+    echo "${dir_name#../} | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))"
+}
+
+main() {
+    count_dir_independently $ROOT_DIR 0 0
+}
+
+if [ "${1}" != "--source-only" ]; then
+    main "${@}"
+fi

tools/enforce/count_enforce_by_file.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to count PADDLE checks by files in the paddle/fluid/operators directory,
+#   contains the number of PADDLE checks under each file.
+#
+#   The three columns of data are: total number, valid number, invalid number.
+#   The output format is easy to display as a markdown table.
+
+# Usage: bash count_enforce_by_file.sh  [target directory or file] (run in tools directory)
+#   - The default check path is paddle/fluid/operators
+
+# Result Example:
+
+#   **paddle/fluid/operators/math** | **200** | **7** | **193**
+#   - beam_search.cc | 1 | 0 | 1
+#   - beam_search.cu | 1 | 0 | 1
+#   - blas.cc | 1 | 0 | 1
+#   - blas_impl.cu.h | 8 | 1 | 7
+#   - blas_impl.h | 15 | 0 | 15
+#   - concat_test.cc | 16 | 0 | 16
+#   - context_project.h | 1 | 0 | 1
+#   - cpu_vec.h | 1 | 0 | 1
+#   - cross_entropy.cu | 1 | 0 | 1
+#   - cross_entropy.h | 1 | 0 | 1
+#   - im2col.cc | 12 | 0 | 12
+#   - im2col.cu | 12 | 0 | 12
+#   - math_function.cc | 2 | 0 | 2
+#   - math_function.cu | 4 | 0 | 4
+#   - math_function_impl.h | 10 | 0 | 10
+
+. ./count_all_enforce.sh --source-only
+
+ROOT_DIR=../paddle/fluid/operators
+
+if [ "$1" != "" ]; then
+    ROOT_DIR=$1
+fi
+
+FILE_WHITE_LIST="\
+    box_clip_op.cc \
+    box_clip_op.h \
+    elementwise_op_function.cu.h \
+    fused_elemwise_activation_op.cc \
+    auc_op.cu \
+    unsqueeze_op.h \
+    unsqueeze_op.cc \
+    enforce.h \
+    errors_test.cc \
+    cross_entropy.cu \
+    cross_entropy.h \
+    unpooling.cu"
+
+function count_file_recursively(){
+    dir_name=$1
+    echo "**${dir_name#../}** | **$2** | **$3** | **$(($2-$3))**"
+    local i=0
+    local dir_array
+    for file in `ls $1`
+    do
+        if [ -f $1"/"$file ];then
+            in_white_list=$(echo $FILE_WHITE_LIST | grep "${file}")
+            if [[ "$in_white_list" == "" ]];then
+                enforce_count $1"/"$file file_total_check_cnt file_valid_check_cnt
+                file_invalid_check_cnt=$(($total_check_cnt-$valid_check_cnt))
+                if [ $file_invalid_check_cnt -gt 0 ];then
+                    echo "- $file | ${file_total_check_cnt} | ${file_valid_check_cnt} | ${file_invalid_check_cnt}"
+                fi
+            fi
+        fi
+        if [ -d $1"/"$file ];then
+            dir_array[$i]=$1"/"$file
+            ((i++))
+        fi
+    done
+    for sub_dir_name in ${dir_array[@]}
+    do
+        enforce_count $sub_dir_name dir_total_check_cnt dir_valid_check_cnt
+        count_file_recursively $sub_dir_name $dir_total_check_cnt $dir_valid_check_cnt
+    done
+}
+
+main() {
+    count_file_recursively $ROOT_DIR 0 0
+}
+
+if [ "${1}" != "--source-only" ]; then
+    main "${@}"
+fi

tools/enforce/grep_invalid_enforce.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to grep invalid PADDLE checks by directory or file in the paddle/fluid/,
+#   the result show all invalid PADDLE checks in specified directory or file.
+
+# Usage:
+#   - bash grep_invalid_enforce.sh [target directory or file] (run in tools directory)
+#       - The default check path is paddle/fluid/operators
+
+# Result Examples:
+# 1. grep invalid PADDLE checks in directory
+
+#     - Command: /work/paddle/tools {develop} bash grep_invalid_enforce.sh ../paddle/fluid/imperative
+#     - Results:
+#         - paddle/fluid/imperative/gradient_accumulator.cc
+#         PADDLE_ENFORCE_EQ(dst_tensor->numel() == numel, true,
+#                             "dst_numel %d vs. src_numel %d", dst_tensor->numel(),
+#                             numel);
+
+#         - paddle/fluid/imperative/nccl_context.cc
+#         PADDLE_ENFORCE_EQ(addr.size(), 2UL,
+#                             "The endpoint should contain host and port: %s", ep);
+#         PADDLE_THROW("create server fd failed");
+#         PADDLE_THROW("set socket opt failed");
+#         PADDLE_THROW("binding failed on ep: %s", ep);
+#         PADDLE_THROW("listen on server fd failed");
+#         PADDLE_THROW("accept the new socket fd failed");
+#         PADDLE_THROW("reading the ncclUniqueId from socket failed");
+#         PADDLE_ENFORCE_EQ(addr.size(), 2UL,
+#                             "The endpoint should contain host and port: %s", ep);
+#         PADDLE_THROW("create socket failed");
+#         PADDLE_THROW("invalid address: %s", ep);
+
+#         - paddle/fluid/imperative/jit/program_desc_tracer.cc
+#         PADDLE_ENFORCE_NOT_NULL(new_var);
+#         PADDLE_ENFORCE_EQ(inner_var.IsInitialized(), true);
+#         PADDLE_THROW("Not support variable type %s",
+#                         framework::ToTypeName(inner_var.Type()));
+
+# 2. grep invalid PADDLE checks in file
+
+#     - Command: /work/paddle/tools {develop} bash grep_invalid_enforce.sh ../paddle/fluid/pybind/reader_py.cc
+#     - Results:
+#         - paddle/fluid/pybind/reader_py.cc
+#         PADDLE_THROW(
+#                     "Place cannot be CUDAPlace when use_double_buffer is False");
+#         PADDLE_ENFORCE_NOT_NULL(exceptions_[i]);
+#         PADDLE_ENFORCE_EQ(status, Status::kException);
+#         PADDLE_ENFORCE_EQ(status, Status::kSuccess);
+
+. ./count_enforce_by_file.sh --source-only
+
+ROOT_DIR=../paddle/fluid/operators
+
+if [ "$1" != "" ]; then
+    ROOT_DIR=$1
+fi
+
+function enforce_grep(){
+    paddle_check=`grep -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
+    valid_check=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' || true`
+    invalid_check=`echo "$paddle_check" | grep -vxF "$valid_check" || true`
+    if [ "${invalid_check}" != "" ];then
+        file_path=$1
+        echo -e "\n- ${file_path#../}"
+        echo "${invalid_check}"
+    fi
+}
+
+function grep_file_recursively(){
+    local i=0
+    local dir_array
+    for file in `ls $1`
+    do
+        if [ -f $1"/"$file ];then
+            in_white_list=$(echo $FILE_WHITE_LIST | grep "${file}")
+            if [[ "$in_white_list" == "" ]];then
+                enforce_grep $1"/"$file
+            fi
+        fi
+        if [ -d $1"/"$file ];then
+            dir_array[$i]=$1"/"$file
+            ((i++))
+        fi
+    done
+    for sub_dir_name in ${dir_array[@]}
+    do
+        grep_file_recursively $sub_dir_name
+    done
+}
+
+function grep_file(){
+    file_path=$1
+    file_name=`echo ${file_path##*/} `
+    if [ -f $file_path ];then
+        in_white_list=$(echo $FILE_WHITE_LIST | grep "${file_name}")
+        if [[ "$in_white_list" == "" ]];then
+            enforce_grep $file_path
+        fi
+    fi
+}
+
+main() {
+    if [ -f $ROOT_DIR ];then
+        grep_file $ROOT_DIR
+    else
+        grep_file_recursively $ROOT_DIR
+    fi
+}
+
+if [ "${1}" != "--source-only" ]; then
+    main "${@}"
+fi

tools/externalError/README.md

@@ -0,0 +1,25 @@
+#### **Introduction for crawling new error message:**
+
+
+
+1. add new spider code in spider.py for crawling error message from website.
+
+2. run `bash start.sh` in current  directory to generate new externalErrorMsg_${date}.tar.gz file, for example `externalErrorMsg_20210928.tar.gz`.
+
+3. upload above tar file into bos https://paddlepaddledeps.bj.bcebos.com **paddlepaddledeps** bucket, and copy download link `${download_url}`. ***\*Be careful not to delete original tar file\****.
+
+4. compute md5 value of above tar file `${md5}`, and modify cmake/third_party.cmake file
+
+   ```
+   set(URL  "${download_url}" CACHE STRING "" FORCE)
+   file_download_and_uncompress(${URL} "externalError" MD5 ${md5})
+   ```
+
+   for example:
+
+   ```
+   set(URL  "https://paddlepaddledeps.bj.bcebos.com/externalErrorMsg_20210928.tar.gz" CACHE STRING "" FORCE)
+   file_download_and_uncompress(${URL} "externalError" MD5 a712a49384e77ca216ad866712f7cafa)
+   ```
+
+5. commit your changes, and create pull request.

tools/externalError/spider.py

@@ -0,0 +1,413 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import getopt
+import re
+import ssl
+import sys
+import urllib.request
+from html.parser import HTMLParser
+
+import external_error_pb2
+
+
+def parsing(externalErrorDesc):
+    # *********************************************************************************************#
+    # *********************************** CUDA Error Message **************************************#
+    print("start crawling errorMessage for nvidia CUDA API--->")
+    url = 'https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html#group__CUDART__TYPES_1g3f51e3575c2178246db0a94a430e0038'
+
+    allMessageDesc = externalErrorDesc.errors.add()
+    allMessageDesc.type = external_error_pb2.CUDA
+
+    ssl._create_default_https_context = ssl._create_unverified_context
+    html = urllib.request.urlopen(url).read().decode('utf-8')
+    res_div = r'<div class="section">.*?<p>CUDA error types </p>.*?</div>.*?<div class="enum-members">(.*?)</div>'
+    m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
+
+    res_dt = r'<dt>(.*?)</dt>.*?<dd>(.*?)</dd>'
+    m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
+    for error in m_dt:
+        res_type = r'<span class="enum-member-name-def">(.*?) = <span class="ph ph apiData">(.*?)</span></span>'
+        m_type = re.findall(res_type, error[0], re.DOTALL | re.MULTILINE)[0]
+        m_message = error[1]
+        m_message = m_message.replace('\n', '')
+        res_a = r'(<a class=.*?</a>)'
+        res_shape = r'<a class=.*?>(.*?)</a>'
+        list_a = re.findall(res_a, m_message, re.DOTALL | re.MULTILINE)
+        list_shape = re.findall(res_shape, m_message, re.DOTALL | re.MULTILINE)
+        assert len(list_a) == len(list_shape)
+        for idx in range(len(list_a)):
+            m_message = m_message.replace(list_a[idx], list_shape[idx])
+
+        m_message = m_message.replace(
+            '<h6 class="deprecated_header">Deprecated</h6>', ''
+        )
+
+        res_span = r'(<span class=.*?</span>)'
+        res_span_detail = r'<span class=.*?>(.*?)</span>'
+        list_span = re.findall(res_span, m_message, re.DOTALL | re.MULTILINE)
+        list_span_detail = re.findall(
+            res_span_detail, m_message, re.DOTALL | re.MULTILINE
+        )
+        assert len(list_span) == len(list_span_detail)
+        for idx in range(len(list_span)):
+            m_message = m_message.replace(list_span[idx], list_span_detail[idx])
+
+        res_p = r'(<p>.*?</p>)'
+        res_p_detail = r'<p>(.*?)</p>'
+        list_p = re.findall(res_p, m_message, re.DOTALL | re.MULTILINE)
+        list_p_detail = re.findall(
+            res_p_detail, m_message, re.DOTALL | re.MULTILINE
+        )
+        assert len(list_p) == len(list_p_detail)
+        for idx in range(len(list_p)):
+            m_message = m_message.replace(list_p[idx], list_p_detail[idx])
+
+        m_message = m_message.replace('  ', '')
+        _Messages = allMessageDesc.messages.add()
+        try:
+            _Messages.code = int(m_type[1])
+        except ValueError:
+            if re.match('0x', m_type[1]):
+                _Messages.code = int(m_type[1], 16)
+            else:
+                raise ValueError
+        _Messages.message = f"'{m_type[0]}'. {m_message}"
+    print("End crawling errorMessage for nvidia CUDA API!\n")
+
+    # ***********************************************************************************************#
+    # *********************************** CURAND Error Message **************************************#
+    print("start crawling errorMessage for nvidia CURAND API--->")
+    url = 'https://docs.nvidia.com/cuda/curand/group__HOST.html#group__HOST_1gb94a31d5c165858c96b6c18b70644437'
+
+    allMessageDesc = externalErrorDesc.errors.add()
+    allMessageDesc.type = external_error_pb2.CURAND
+
+    html = urllib.request.urlopen(url).read().decode('utf-8')
+
+    res_div = r'<div class="section">.*?<p>CURAND function call status types </p>.*?</div>.*?<div class="enum-members">(.*?)</div>'
+    m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
+
+    res_dt = r'<dt>(.*?)</dt>.*?<dd>(.*?)</dd>'
+    m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
+    for error in m_dt:
+        res_type = r'<span class="enum-member-name-def">(.*?) = <span class="ph ph apiData">(.*?)</span></span>'
+        m_type = re.findall(res_type, error[0], re.DOTALL | re.MULTILINE)[0]
+        m_message = error[1]
+
+        _Messages = allMessageDesc.messages.add()
+        try:
+            _Messages.code = int(m_type[1])
+        except ValueError:
+            if re.match('0x', m_type[1]):
+                _Messages.code = int(m_type[1], 16)
+            else:
+                raise ValueError
+        _Messages.message = f"'{m_type[0]}'. {m_message}"
+    print("End crawling errorMessage for nvidia CURAND API!\n")
+
+    # **************************************************************************************************#
+    # *********************************** CUDNN Error Message ******************************************#
+    cudnnStatus_t = {
+        "CUDNN_STATUS_SUCCESS": 0,
+        "CUDNN_STATUS_NOT_INITIALIZED": 1,
+        "CUDNN_STATUS_ALLOC_FAILED": 2,
+        "CUDNN_STATUS_BAD_PARAM": 3,
+        "CUDNN_STATUS_INTERNAL_ERROR": 4,
+        "CUDNN_STATUS_INVALID_VALUE": 5,
+        "CUDNN_STATUS_ARCH_MISMATCH": 6,
+        "CUDNN_STATUS_MAPPING_ERROR": 7,
+        "CUDNN_STATUS_EXECUTION_FAILED": 8,
+        "CUDNN_STATUS_NOT_SUPPORTED": 9,
+        "CUDNN_STATUS_LICENSE_ERROR": 10,
+        "CUDNN_STATUS_RUNTIME_PREREQUISITE_MISSING": 11,
+        "CUDNN_STATUS_RUNTIME_IN_PROGRESS": 12,
+        "CUDNN_STATUS_RUNTIME_FP_OVERFLOW": 13,
+    }
+
+    print("start crawling errorMessage for nvidia CUDNN API--->")
+    url = 'https://docs.nvidia.com/deeplearning/cudnn/api/index.html#cudnnStatus_t'
+
+    allMessageDesc = externalErrorDesc.errors.add()
+    allMessageDesc.type = external_error_pb2.CUDNN
+
+    html = urllib.request.urlopen(url).read().decode('utf-8')
+    f = open('1.txt', 'w')
+    f.write(html)
+
+    res_div = r'<div class="section" id="cudnnStatus_t__section_lmp_dgr_2jb"><a name="cudnnStatus_t__section_lmp_dgr_2jb" shape="rect">(.*?)</div>'
+    m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
+
+    res_dt = r'<dt class="dt dlterm"><samp class="ph codeph">(.*?)</samp></dt>.*?<dd class="dd">(.*?)</dd>'
+    m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
+    for error in m_dt:
+        m_message = error[1]
+
+        res_class = r'<p class="p">.*?</p>'
+        res_class_detail = r'<p class="p">(.*?)</p>'
+        list_class = re.findall(res_class, m_message, re.DOTALL | re.MULTILINE)
+        list_class_detail = re.findall(
+            res_class_detail, m_message, re.DOTALL | re.MULTILINE
+        )
+        assert len(list_class) == len(list_class_detail)
+        for idx in range(len(list_class)):
+            m_message = m_message.replace(
+                list_class[idx], list_class_detail[idx]
+            )
+
+        res_a = r'(<a class="xref".*?</a>)'
+        res_shape = r'<a class="xref".*?>(.*?)</a>'
+        list_a = re.findall(res_a, m_message, re.DOTALL | re.MULTILINE)
+        list_shape = re.findall(res_shape, m_message, re.DOTALL | re.MULTILINE)
+        assert len(list_a) == len(list_shape)
+        for idx in range(len(list_a)):
+            m_message = m_message.replace(list_a[idx], list_shape[idx])
+
+        res_span = r'(<span class="ph">.*?</span>)'
+        res_span_detail = r'<span class="ph">(.*?)</span>'
+        list_span = re.findall(res_span, m_message, re.DOTALL | re.MULTILINE)
+        list_span_detail = re.findall(
+            res_span_detail, m_message, re.DOTALL | re.MULTILINE
+        )
+        assert len(list_span) == len(list_span_detail)
+        for idx in range(len(list_span)):
+            m_message = m_message.replace(list_span[idx], list_span_detail[idx])
+
+        res_samp = r'(<samp class="ph codeph">.*?</samp>)'
+        res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
+        list_samp = re.findall(res_samp, m_message, re.DOTALL | re.MULTILINE)
+        list_samp_detail = re.findall(
+            res_samp_detail, m_message, re.DOTALL | re.MULTILINE
+        )
+        assert len(list_samp) == len(list_samp_detail)
+        for idx in range(len(list_samp)):
+            m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
+
+        m_message = re.sub(r'\n +', ' ', m_message)
+
+        _Messages = allMessageDesc.messages.add()
+        _Messages.code = int(cudnnStatus_t[error[0]])
+        _Messages.message = f"'{error[0]}'. {m_message}"
+    print("End crawling errorMessage for nvidia CUDNN API!\n")
+
+    # *************************************************************************************************#
+    # *********************************** CUBLAS Error Message ****************************************#
+    cublasStatus_t = {
+        "CUBLAS_STATUS_SUCCESS": 0,
+        "CUBLAS_STATUS_NOT_INITIALIZED": 1,
+        "CUBLAS_STATUS_ALLOC_FAILED": 3,
+        "CUBLAS_STATUS_INVALID_VALUE": 7,
+        "CUBLAS_STATUS_ARCH_MISMATCH": 8,
+        "CUBLAS_STATUS_MAPPING_ERROR": 11,
+        "CUBLAS_STATUS_EXECUTION_FAILED": 13,
+        "CUBLAS_STATUS_INTERNAL_ERROR": 14,
+        "CUBLAS_STATUS_NOT_SUPPORTED": 15,
+        "CUBLAS_STATUS_LICENSE_ERROR": 16,
+    }
+
+    print("start crawling errorMessage for nvidia CUBLAS API--->")
+    url = 'https://docs.nvidia.com/cuda/cublas/index.html#cublasstatus_t'
+
+    allMessageDesc = externalErrorDesc.errors.add()
+    allMessageDesc.type = external_error_pb2.CUBLAS
+
+    html = urllib.request.urlopen(url).read().decode('utf-8')
+
+    res_div = r'<p class="p">The type is used for function status returns. All cuBLAS library.*?<div class="tablenoborder">(.*?)</div>'
+    m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
+
+    res_dt = r'<p class="p"><samp class="ph codeph">(.*?)</samp></p>.*?colspan="1">(.*?)</td>'
+    m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
+
+    for error in m_dt:
+        m_message = error[1]
+        m_message = re.sub(r'\n +', ' ', m_message)
+
+        res_p = r'<p class="p">.*?</p>'
+        res_p_detail = r'<p class="p">(.*?)</p>'
+        list_p = re.findall(res_p, m_message, re.DOTALL | re.MULTILINE)
+        list_p_detail = re.findall(
+            res_p_detail, m_message, re.DOTALL | re.MULTILINE
+        )
+        assert len(list_p) == len(list_p_detail)
+        for idx in range(len(list_p)):
+            m_message = m_message.replace(list_p[idx], list_p_detail[idx])
+
+        res_samp = r'<samp class="ph codeph">.*?</samp>'
+        res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
+        list_samp = re.findall(res_samp, m_message, re.DOTALL | re.MULTILINE)
+        list_samp_detail = re.findall(
+            res_samp_detail, m_message, re.DOTALL | re.MULTILINE
+        )
+        assert len(list_samp) == len(list_samp_detail)
+        for idx in range(len(list_samp)):
+            m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
+
+        _Messages = allMessageDesc.messages.add()
+        _Messages.code = int(cublasStatus_t[error[0]])
+        _Messages.message = f"'{error[0]}'. {m_message}"
+    print("End crawling errorMessage for nvidia CUBLAS API!\n")
+
+    # *************************************************************************************************#
+    # *********************************** CUSOLVER Error Message **************************************#
+    cusolverStatus_t = {
+        "CUSOLVER_STATUS_SUCCESS": 0,
+        "CUSOLVER_STATUS_NOT_INITIALIZED": 1,
+        "CUSOLVER_STATUS_ALLOC_FAILED": 2,
+        "CUSOLVER_STATUS_INVALID_VALUE": 3,
+        "CUSOLVER_STATUS_ARCH_MISMATCH": 4,
+        "CUSOLVER_STATUS_MAPPING_ERROR": 5,
+        "CUSOLVER_STATUS_EXECUTION_FAILED": 6,
+        "CUSOLVER_STATUS_INTERNAL_ERROR": 7,
+        "CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED": 8,
+        "CUSOLVER_STATUS_NOT_SUPPORTED": 9,
+        "CUSOLVER_STATUS_ZERO_PIVOT": 10,
+        "CUSOLVER_STATUS_INVALID_LICENSE": 11,
+        "CUSOLVER_STATUS_IRS_PARAMS_NOT_INITIALIZED": 12,
+        "CUSOLVER_STATUS_IRS_PARAMS_INVALID": 13,
+        "CUSOLVER_STATUS_IRS_INTERNAL_ERROR": 14,
+        "CUSOLVER_STATUS_IRS_NOT_SUPPORTED": 15,
+        "CUSOLVER_STATUS_IRS_OUT_OF_RANGE": 16,
+        "CUSOLVER_STATUS_IRS_NRHS_NOT_SUPPORTED_FOR_REFINE_GMRES": 17,
+        "CUSOLVER_STATUS_IRS_INFOS_NOT_INITIALIZED": 18,
+    }
+    print("start crawling errorMessage for nvidia CUSOLVER API--->")
+    url = 'https://docs.nvidia.com/cuda/cusolver/index.html#cuSolverSPstatus'
+
+    allMessageDesc = externalErrorDesc.errors.add()
+    allMessageDesc.type = external_error_pb2.CUSOLVER
+
+    html = urllib.request.urlopen(url).read().decode('utf-8')
+
+    res_div = r'This is a status type returned by the library functions and.*?<div class="tablenoborder">(.*?)</div>'
+    m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
+
+    res_dt = (
+        r'<samp class="ph codeph">(.*?)</samp></td>.*?colspan="1">(.*?)</td>'
+    )
+    m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
+
+    for error in m_dt:
+        m_message = error[1]
+        m_message = re.sub(r'\n +', '', m_message)
+        m_message = re.sub(r'<p class="p"></p>', '', m_message)
+
+        res_p = r'<p class="p">.*?</p>'
+        res_p_detail = r'<p class="p">(.*?)</p>'
+        list_p = re.findall(res_p, m_message, re.DOTALL | re.MULTILINE)
+        list_p_detail = re.findall(
+            res_p_detail, m_message, re.DOTALL | re.MULTILINE
+        )
+        assert len(list_p) == len(list_p_detail)
+        for idx in range(len(list_p)):
+            m_message = m_message.replace(list_p[idx], list_p_detail[idx])
+
+        res_samp = r'<samp class="ph codeph">.*?</samp>'
+        res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
+        list_samp = re.findall(res_samp, m_message, re.DOTALL | re.MULTILINE)
+        list_samp_detail = re.findall(
+            res_samp_detail, m_message, re.DOTALL | re.MULTILINE
+        )
+        assert len(list_samp) == len(list_samp_detail)
+        for idx in range(len(list_samp)):
+            m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
+
+        res_strong = r'<strong class="ph b">.*?</strong>'
+        res_strong_detail = r'<strong class="ph b">(.*?)</strong>'
+        list_strong = re.findall(
+            res_strong, m_message, re.DOTALL | re.MULTILINE
+        )
+        list_strong_detail = re.findall(
+            res_strong_detail, m_message, re.DOTALL | re.MULTILINE
+        )
+        assert len(list_strong) == len(list_strong_detail)
+        for idx in range(len(list_strong)):
+            m_message = m_message.replace(
+                list_strong[idx], list_strong_detail[idx]
+            )
+
+        _Messages = allMessageDesc.messages.add()
+        _Messages.code = int(cusolverStatus_t[error[0]])
+        _Messages.message = f"'{error[0]}'. {m_message}"
+    print("End crawling errorMessage for nvidia CUSOLVER API!\n")
+
+    # **********************************************************************************************#
+    # *************************************** NCCL error *******************************************#
+    print("start crawling errorMessage for nvidia NCCL API--->")
+    url = 'https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/types.html#ncclresult-t'
+    allMessageDesc = externalErrorDesc.errors.add()
+    allMessageDesc.type = external_error_pb2.NCCL
+    html = urllib.request.urlopen(url).read().decode('utf-8')
+    res_div = r'<code class="descname">ncclResult_t</code>(.*?)</div>'
+    m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
+
+    res_dt = r'<code class="descname">(.*?)</code>.*?<span class="pre">(.*?)</span></code>\)(.*?)</p>\n</dd></dl>'
+    m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
+    for error in m_dt:
+        m_message = re.sub(r'\n', '', error[2])
+        _Messages = allMessageDesc.messages.add()
+        _Messages.code = int(error[1])
+        _Messages.message = f"'{error[0]}'. {m_message}"
+    print("End crawling errorMessage for nvidia NCCL API!\n")
+
+    # *************************************************************************************************#
+    # *********************************** CUFFT Error Message **************************************#
+    print("start crawling errorMessage for nvidia CUFFT API--->")
+    url = 'https://docs.nvidia.com/cuda/cufft/index.html#cufftresult'
+
+    allMessageDesc = externalErrorDesc.errors.add()
+    allMessageDesc.type = external_error_pb2.CUFFT
+
+    html = urllib.request.urlopen(url).read().decode('utf-8')
+
+    class CUFFTHTMLParser(HTMLParser):
+        '''CUFFTHTML Parser'''
+
+        def handle_data(self, data):
+            if 'typedef enum cufftResult_t' in data:
+                for line in data.strip().splitlines()[1:-1]:
+                    status, code, desc = re.split('=|//', line.strip())
+                    _Messages = allMessageDesc.messages.add()
+                    _Messages.code = int(code.strip(' ,'))
+                    _Messages.message = f"'{status.strip()}'. {desc.strip()}"
+
+    CUFFTHTMLParser().feed(html)
+
+
+def main(argv):
+    try:
+        opts, _ = getopt.getopt(argv, "h", ["help"])
+    except getopt.GetoptError:
+        print('python spider.py')
+        sys.exit(2)
+    for opt, _ in opts:
+        if opt in ("-h", "--help"):
+            print('python spider.py')
+            sys.exit(2)
+    externalErrorDesc = external_error_pb2.ExternalErrorDesc()
+    parsing(externalErrorDesc)
+
+    serializedString = externalErrorDesc.SerializeToString()
+    with open("externalErrorMsg.pb", "wb") as f:
+        # save for externalErrorMsg.pb from Python-protobuf interface
+        # load from C++-protobuf interface and get error message
+        f.write(serializedString)
+    print(
+        "Generating data file [externalErrorMsg.pb] for external third_party API error has been done!"
+    )
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

tools/externalError/start.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+SYSTEM=`uname -s`
+rm -f protoc-3.11.3-linux-x86_64.*
+if [ "$SYSTEM" == "Linux" ]; then
+    wget --no-check-certificate https://github.com/protocolbuffers/protobuf/releases/download/v3.11.3/protoc-3.11.3-linux-x86_64.zip
+    unzip -d protobuf -o protoc-3.11.3-linux-x86_64.zip
+    rm protoc-3.11.3-linux-x86_64.*
+elif [ "$SYSTEM" == "Darwin" ]; then
+    wget --no-check-certificate https://github.com/protocolbuffers/protobuf/releases/download/v3.11.3/protoc-3.11.3-osx-x86_64.zip
+    unzip -d protobuf -o protoc-3.11.3-osx-x86_64.zip
+    rm protoc-3.11.3-osx-x86_64.*
+else
+    echo "please run on Mac/Linux"
+    exit 1
+fi
+protobuf/bin/protoc -I../../paddle/phi/core/ --python_out . ../../paddle/phi/core/external_error.proto
+
+python3.10 spider.py
+tar czvf externalErrorMsg_$(date +'%Y%m%d').tar.gz externalErrorMsg.pb

tools/final_ut_parallel_rule.py

@@ -0,0 +1,156 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import sys
+
+
+def classify_cases_by_mem(rootPath):
+    """classify cases by mem"""
+    case_filename = f'{rootPath}/build/classify_case_by_cardNum.txt'
+    case_exec_100 = [
+        'test_conv_eltwiseadd_bn_fuse_pass',
+        'test_trt_convert_pool2d',
+        'test_fc_fuse_pass',
+        'test_trt_convert_depthwise_conv2d',
+        'test_quant2_int8_resnet50_onednn',
+        'test_conv_elementwise_add_act_fuse_pass',
+        'test_trt_convert_conv2d',
+        'test_paddle_save_load',
+        'test_logical_op',
+        'test_pool2d_op',
+        'test_conv3d_transpose_op',
+        'test_lstmp_op',
+        'test_cross_entropy2_op',
+        'test_sgd_op',
+        'test_imperative_ptq',
+        'test_model',
+        'test_custom_relu_op_setup',
+        'test_dropout_op',
+        'test_concat_op',
+    ]  # 木桶原理 70s-100s之间的case
+
+    case_exec_200 = [
+        'test_post_training_quantization_mnist',
+        'test_trt_dynamic_shape_ernie_fp16_ser_deser',
+        'test_trt_dynamic_shape_ernie',
+        'test_layer_norm_op',
+        'trt_quant_int8_yolov3_r50_test',
+        'test_gru_op',
+        'test_post_training_quantization_while',
+        'test_onednn_log_softmax_op',
+        'test_onednn_matmulv2_op',
+        'test_onednn_shape_op',
+        'interceptor_pipeline_short_path_test',
+        'interceptor_pipeline_long_path_test',
+        'test_cpuonly_spawn',
+    ]  # 木桶原理 110s-200s之间的case 以及容易timeout
+
+    case_always_timeout = [
+        'test_quant2_int8_resnet50_channelwise_onednn',
+        'test_parallel_dygraph_unused_variables_gloo',
+        'test_seq2seq',
+        'test_pool3d_op',
+        'test_trilinear_interp_v2_op',
+        'test_dropout_op',
+        'test_parallel_dygraph_sync_batch_norm',
+        'test_conv3d_op',
+        'test_quant2_int8_resnet50_range_onednn',
+    ]  # always timeout
+
+    f = open(case_filename)
+    lines = f.readlines()
+    all_tests_by_card = {}
+    for line in lines:
+        if line.startswith('single_card_tests:'):
+            all_tests_by_card['single_card_tests'] = []
+            line = line.split('single_card_tests: ^job$|')[1].split('|')
+            for case in line:
+                case = case.replace('^', '').replace('$', '').strip()
+                all_tests_by_card['single_card_tests'].append(case)
+        elif line.startswith('multiple_card_tests:'):
+            all_tests_by_card['multiple_card_tests'] = []
+            line = line.split('multiple_card_tests: ^job$|')[1].split('|')
+            for case in line:
+                case = case.replace('^', '').replace('$', '').strip()
+                all_tests_by_card['multiple_card_tests'].append(case)
+        elif line.startswith('exclusive_card_tests:'):
+            all_tests_by_card['exclusive_card_tests'] = []
+            line = line.split('exclusive_card_tests: ^job$')[1].split('|')
+            for case in line:
+                case = case.replace('^', '').replace('$', '').strip()
+                all_tests_by_card['exclusive_card_tests'].append(case)
+
+    if not os.path.exists("/pre_test"):
+        os.mkdir("/pre_test")
+
+    with open("/pre_test/classify_case_by_cardNum.json", "w") as f:
+        json.dump(all_tests_by_card, f)
+
+    with open("/pre_test/ut_mem_map.json", 'r') as load_f:
+        new_latest_mem = json.load(load_f)
+    no_parallel_case = '^job$'
+    for cardType in all_tests_by_card:
+        case_mem_0 = '^job$'
+        case_mem_1 = {}
+        for case in all_tests_by_card[cardType]:
+            if case in case_exec_100 or case in case_exec_200:
+                continue
+            if case in case_always_timeout:
+                no_parallel_case = no_parallel_case + '|^' + case + '$'
+                continue
+
+            if case not in new_latest_mem:
+                continue
+
+            # mem = 0
+            if new_latest_mem[case]["mem_nvidia"] == 0:
+                case_mem_0 = case_mem_0 + '|^' + case + '$'
+            # mem != 0
+            else:
+                case_mem_1[case] = new_latest_mem[case]["mem_nvidia"]
+
+        with open(f'/pre_test/{cardType}_mem0', 'w') as f:
+            f.write(case_mem_0)
+            f.close()
+
+        case_mem_1_sort = sorted(case_mem_1.items(), key=lambda x: x[1])
+        case_mem_1_line = '^job$'
+        mem_1_sum = 0
+        with open(f'/pre_test/{cardType}', 'w') as f_not_0:
+            for index in case_mem_1_sort:
+                if mem_1_sum < 14 * 1024 * 2:
+                    mem_1_sum += index[1]
+                    case_mem_1_line = case_mem_1_line + '|^' + index[0] + '$'
+                else:
+                    f_not_0.write(case_mem_1_line + '\n')
+                    case_mem_1_line = '^job$|^' + index[0] + '$'
+                    mem_1_sum = index[1]
+            f_not_0.write(case_mem_1_line + '\n')
+
+            if cardType == 'single_card_tests':
+                for cases in [case_exec_100, case_exec_200]:
+                    case_mem_1_line = '^job$'
+                    for case in cases:
+                        case_mem_1_line = case_mem_1_line + '|^' + case + '$'
+                    f_not_0.write(case_mem_1_line + '\n')
+            f_not_0.close()
+
+    os.system(f'cp {rootPath}/build/nightly_case /pre_test/')
+
+
+if __name__ == '__main__':
+    rootPath = sys.argv[1]
+    classify_cases_by_mem(rootPath)

tools/gen_alias_mapping.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Brief:
+#     This code is used for generating the mapping list of Paddle API alias.
+#     Only the APIs set with the `DEFINE_ALIAS` flag is enable.
+#
+# Arguments:
+#     None
+#
+# Usage:
+#     Go into the `Paddle` folder and just run `./tools/gen_alias_mapping.sh`
+#
+# Returns:
+#     succ: 0
+#
+#     Will also print the mapping list to stdout. The format of each line is as below:
+#         <real API implement>\t<API recommend>,<API other alias name1>,<API other alias name2>,...
+
+
+PADDLE_ROOT="$(dirname $(readlink -f ${BASH_SOURCE[0]}))/.."
+
+find ${PADDLE_ROOT}/python/ -name '*.py' \
+    | xargs  grep -v '^#' \
+    | grep 'DEFINE_ALIAS' \
+    | perl -ne '
+        if (/\/python\/(.*):from (\.*)(\w.*) import (.*?)\s+#DEFINE_ALIAS\s+$/) {
+            my @arr = split(", ", $4);
+            foreach $i (@arr) {
+                printf "%s|%s|%s|%d\n", $3, $i, substr($1, 0, -3), length($2);
+            }
+        }' \
+    | awk -F '[|/]' '
+        {
+            key = "";
+            val = "";
+            if ($2 ~ /.* as .*/) {
+                split($2, arr, " as ");
+                old = arr[1];
+                new = arr[2];
+            } else {
+                old = $2;
+                new = $2;
+            }
+            for (i = 3; i <= (NF - 1 - $NF); ++i) {
+                val = val""$i".";
+            }
+            val =  val""$1"."old
+            for (i = 3; i <= (NF - 1); ++i) {
+                if ($i != "__init__") {
+                    key = key""$i".";
+                }
+            }
+            key = key""new;
+            n2o[key] = val;
+        }
+        END {
+            for (new in n2o) {
+                old = n2o[new] in n2o ? n2o[n2o[new]] : n2o[new];
+                print old, length(new), new;
+            }
+        }' \
+    | sort -k 1,1 -k 2n,2 \
+    | awk '
+        {
+            o2n[$1] = o2n[$1] ? o2n[$1]","$3 : $3;
+        }
+        END {
+            for (i in o2n) {
+                print i"\t"o2n[i];
+            }
+        }'

tools/gen_pybind11_stub.py

@@ -0,0 +1,921 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import argparse
+import functools
+import importlib
+import inspect
+import keyword
+import logging
+import os
+import py_compile
+import re
+import shutil
+import tempfile
+import traceback
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, TypedDict
+
+import yaml
+from pybind11_stubgen import (
+    CLIArgs,
+    Printer,
+    Writer,
+    run,
+    stub_parser_from_args,
+    to_output_and_subdir,
+)
+from pybind11_stubgen.structs import (
+    Annotation,
+    InvalidExpression,
+    Value,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+# some invalid attr can NOT be parsed.
+# to avoid syntax error, we can only do plain replacement.
+# e.g. {'a': 'b'}, do replace 'a' -> 'b' .
+BAD_ATTR = {}
+
+# add some import modules
+# e.g. {'a': 'b'}, if not found ' a.' in stub file,
+# we insert 'b' after 'from __future__ import annotations'.
+# 'a' should be converted to ' a.'
+EXTRA_IMPORTS = {
+    'paddle': 'import paddle',
+    'typing': 'import typing',
+    'typing_extensions': 'import typing_extensions',
+    'pybind11_stubgen': 'import pybind11_stubgen',
+    'npt': 'import numpy.typing as npt',
+    'types': 'import types',
+}
+
+# some invalid attr from pybind11.
+# ref:
+# - https://pybind11.readthedocs.io/en/latest/advanced/misc.html#avoiding-cpp-types-in-docstrings
+# - https://pybind11.readthedocs.io/en/latest/advanced/functions.html#default-arguments-revisited
+# we can add some mappings for conversion, e.g. {'paddle::Tensor': 'paddle.Tensor'}
+PYBIND11_ATTR_MAPPING = {}
+
+# some bad full expression pybind11-stubgen can not catch as invalid exp
+PYBIND11_INVALID_FULL_MAPPING = {
+    'PyCodeObject': 'types.CodeType',
+    'PyInterpreterFrameProxy': 'typing.Any',
+    '_object': 'typing.Any',
+    'float16': 'numpy.float16',
+    'Variant': 'typing.Any',
+    'capsule': 'typing_extensions.CapsuleType',
+    'cudaDeviceProp': 'typing.Any',
+    'Tensor': 'paddle.Tensor',
+    'TensorLike': 'paddle._typing.TensorLike',
+    'DTypeLike': 'paddle._typing.DTypeLike',
+    'ShapeLike': 'paddle._typing.ShapeLike',
+    'Numeric': 'paddle._typing.Numeric',
+    'TypeGuard': 'typing_extensions.TypeGuard',
+    '_Interpolation': 'paddle.tensor.stat._Interpolation',
+    'ParamAttrLike': 'paddle._typing.ParamAttrLike',
+    '_POrder': 'paddle.tensor.linalg._POrder',
+    'TensorOrTensors': 'paddle._typing.TensorOrTensors',
+}
+
+# some bad partial expression pybind11-stubgen can not catch as invalid exp
+_PYBIND11_INVALID_PART_MAPPING = {
+    'NestedSequence': 'paddle._typing.NestedSequence',
+    'Dep': 'Node.Dep',
+    'Tensor': 'paddle.Tensor',
+    'TypeGuard': 'typing.TypeGuard',
+}
+PYBIND11_INVALID_PART_MAPPING: dict[re.Pattern, str] = {
+    re.compile(rf'(?<![\.a-zA-Z\:]){k}(?![\.a-zA-Z\:])'): v
+    for k, v in _PYBIND11_INVALID_PART_MAPPING.items()
+}
+
+# data type mapping,
+# ref: paddle/phi/api/generator/api_base.py
+INPUT_TYPES_MAP = {
+    'Tensor': 'paddle.Tensor',
+    'Tensor[]': 'list[paddle.Tensor]',
+}
+ATTR_TYPES_MAP = {
+    'IntArray': 'list[int]',
+    'Scalar': 'float',
+    'Scalar(int)': 'int',
+    'Scalar(int64_t)': 'int',
+    'Scalar(float)': 'float',
+    'Scalar(double)': 'float',
+    'Scalar[]': 'list[float]',
+    'int': 'int',
+    'int32_t': 'int',
+    'int64_t': 'int',
+    'long': 'float',
+    'size_t': 'int',
+    'float': 'float',
+    'float[]': 'list[float]',
+    'double': 'float',
+    'bool': 'bool',
+    'bool[]': 'list[bool]',
+    'str': 'str',
+    'str[]': 'list[str]',
+    'Place': 'paddle._typing.PlaceLike',
+    'DataLayout': 'paddle._typing.DataLayoutND',
+    'DataType': 'paddle._typing.DTypeLike',
+    'int64_t[]': 'list[int]',
+    'int[]': 'list[int]',
+}
+OPTIONAL_TYPES_TRANS = {
+    'Tensor': 'paddle.Tensor',
+    'Tensor[]': 'list[paddle.Tensor]',
+    'int': 'int',
+    'int32_t': 'int',
+    'int64_t': 'int',
+    'float': 'float',
+    'double': 'float',
+    'bool': 'bool',
+    'Place': 'paddle._typing.PlaceLike',
+    'DataLayout': 'paddle._typing.DataLayoutND',
+    'DataType': 'paddle._typing.DTypeLike',
+}
+OUTPUT_TYPE_MAP = {
+    'Tensor': 'paddle.Tensor',
+    'Tensor[]': 'list[paddle.Tensor]',
+}
+
+# for parse ops yaml
+PATTERN_FUNCTION = re.compile(r'^def (?P<name>.*)\(\*args, \*\*kwargs\):')
+FUNCTION_VALUE_TRANS = {
+    'true': 'True',
+    'false': 'False',
+}
+# TODO: Duplicate of python/paddle/tensor/tensor.prototype.pyi
+# Consider a better way to manage these common mappings.
+OPS_YAML_IMPORTS = """
+# Import common typings for generated methods
+# isort: off
+from typing import *  # noqa: F403
+from typing_extensions import *  # type: ignore # noqa: F403
+from paddle._typing import *  # noqa: F403
+
+# isort: on
+from builtins import (  # noqa: F401
+    bool as _bool,
+    bytes as _bytes,
+    complex as _complex,
+    float as _float,
+    int as _int,
+    str as _str,
+)
+from collections.abc import Iterator
+from typing import Any, Literal, overload
+
+import numpy.typing as npt
+
+import paddle
+from paddle import (
+    ParamAttr,  # noqa: F401
+    _typing,
+)
+from paddle.base.dygraph.tensor_patch_methods import (
+    TensorHookRemoveHelper,  # noqa: F401
+)
+from paddle.tensor.linalg import _POrder  # noqa: F401
+from paddle.tensor.stat import _Interpolation  # noqa: F401
+
+# Special types already defined in tensor.prototype.pyi
+from paddle import Tensor
+"""
+
+
+def _get_pybind11_stubgen_annotation_text(annotation: Annotation) -> str:
+    if isinstance(annotation, InvalidExpression):
+        return annotation.text
+    return str(annotation)
+
+
+def _patch_pybind11_invalid_name():
+    # patch name with suffix '_' if `name` is a keyword like `in` to `in_`
+    def wrap_name(func):
+        @functools.wraps(func)
+        def wrapper(self, arg: Any):
+            if hasattr(arg, 'name') and keyword.iskeyword(arg.name):
+                arg.name += '_'
+            return func(self, arg)
+
+        return wrapper
+
+    Printer.print_argument = wrap_name(Printer.print_argument)
+    Printer.print_function = wrap_name(Printer.print_function)
+
+
+def _patch_pybind11_invalid_annotation():
+    # patch invalid annotation as `Value`, e.g. 'capsule' to 'typing_extensions.CapsuleType'
+    def wrap_name(func):
+        @functools.wraps(func)
+        def wrapper(self, arg: Annotation):
+            _arg_str = _get_pybind11_stubgen_annotation_text(arg)
+            if _arg_str in PYBIND11_INVALID_FULL_MAPPING:
+                arg = Value(PYBIND11_INVALID_FULL_MAPPING[_arg_str])
+            else:
+                _arg = _arg_str
+                for (
+                    sig_pattern,
+                    mapping,
+                ) in PYBIND11_INVALID_PART_MAPPING.items():
+                    _arg = sig_pattern.sub(mapping, _arg)
+
+                if _arg != _arg_str:
+                    arg = Value(_arg)
+
+            return func(self, arg)
+
+        return wrapper
+
+    Printer.print_annotation = wrap_name(Printer.print_annotation)
+
+
+def _patch_pybind11_invalid_exp():
+    # patch invalid exp with `"xxx"` as a `typing.Any`
+    def print_invalid_exp(self, invalid_expr: InvalidExpression) -> str:
+        _text = invalid_expr.text
+        _text = PYBIND11_ATTR_MAPPING.get(_text, f'"{_text}"')
+        if (
+            self.invalid_expr_as_ellipses
+            and _text not in PYBIND11_INVALID_FULL_MAPPING.values()
+        ):
+            return "typing.Any"
+        return _text
+
+    Printer.print_invalid_exp = print_invalid_exp
+
+
+def patch_pybind11_stubgen_printer():
+    _patch_pybind11_invalid_name()
+    _patch_pybind11_invalid_annotation()
+    _patch_pybind11_invalid_exp()
+
+
+def gen_stub(
+    output_dir: str,
+    module_name: str,
+    ignore_all_errors: bool = False,
+    print_invalid_expressions_as_is: bool = False,
+) -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(name)s - [%(levelname)7s] %(message)s",
+    )
+
+    args = CLIArgs(
+        output_dir=output_dir,
+        root_suffix=None,
+        ignore_invalid_expressions=None,
+        ignore_invalid_identifiers=None,
+        ignore_unresolved_names=None,
+        ignore_all_errors=ignore_all_errors,
+        enum_class_locations=[],
+        numpy_array_wrap_with_annotated=False,
+        numpy_array_use_type_var=False,
+        numpy_array_remove_parameters=False,
+        print_invalid_expressions_as_is=print_invalid_expressions_as_is,
+        print_safe_value_reprs=None,
+        exit_code=False,
+        dry_run=False,
+        stub_extension='pyi',
+        module_name=module_name,
+    )
+
+    parser = stub_parser_from_args(args)
+    printer = Printer(
+        invalid_expr_as_ellipses=not args.print_invalid_expressions_as_is
+    )
+
+    out_dir, sub_dir = to_output_and_subdir(
+        output_dir=args.output_dir,
+        module_name=args.module_name,
+        root_suffix=args.root_suffix,
+    )
+
+    run(
+        parser,
+        printer,
+        args.module_name,
+        out_dir,
+        sub_dir=sub_dir,
+        dry_run=args.dry_run,
+        writer=Writer(stub_ext=args.stub_extension),
+    )
+
+
+def replace_bad_attr(filename: str):
+    def wrap_text(text):
+        """wrap text to avoid bad math"""
+        return ' ' + text
+
+    stub_file = None
+    with open(filename, encoding='utf-8') as f:
+        stub_file = f.read()
+
+    for _bad_attr, _replacement in BAD_ATTR.items():
+        bad_attr = wrap_text(_bad_attr)
+        replacement = wrap_text(_replacement)
+        stub_file = stub_file.replace(bad_attr, replacement)
+
+    with open(filename, 'w', encoding='utf-8') as f:
+        f.write(stub_file)
+
+
+def insert_import_modules(filename: str):
+    def wrap_text(text):
+        """wrap text to avoid bad math"""
+        return ' ' + text + '.'
+
+    future_import = 'from __future__ import annotations\n'
+
+    stub_file = None
+    with open(filename, encoding='utf-8') as f:
+        stub_file = f.read()
+
+    for _module, _import_txt in EXTRA_IMPORTS.items():
+        module = wrap_text(_module)
+        if module in stub_file and _import_txt not in stub_file:
+            stub_file = stub_file.replace(
+                future_import, future_import + _import_txt + '\n'
+            )
+
+    with open(filename, 'w', encoding='utf-8') as f:
+        f.write(stub_file)
+
+
+def check_remove_syntax_error(filename: str, limit: int = 10000):
+    """
+    Args:
+        filename: xxx.pyi
+        limit: the max times try to check syntax error
+    """
+    pattern_check = re.compile(
+        rf"File.*{re.escape(filename)}.*line (?P<lineno>\d+)"
+    )
+
+    while limit > 0:
+        limit -= 1
+
+        # check syntax error
+        err = ""
+
+        try:
+            py_compile.compile(filename, doraise=True)
+            break
+        except py_compile.PyCompileError as e:
+            err = traceback.format_exc()
+
+        print(f">>> Syntax error: find syntax error in file: {filename}")
+
+        # find the line with syntax error
+        match_obj = pattern_check.search(err)
+        if match_obj is not None:
+            line_no = int(match_obj.group("lineno"))
+
+            # read file
+            source_lines = []
+            with open(filename, "r", encoding="utf-8") as f:
+                source_lines = f.readlines()
+
+            del source_lines[line_no - 1]
+
+            # write new lines
+            with open(filename, "w", encoding="utf-8") as f:
+                f.writelines(source_lines)
+
+            print(
+                f">>> Syntax error: remove the error line {line_no}, and continue to check ..."
+            )
+        else:
+            print(">>> Syntax error: no match obj, just continue ...")
+            break
+
+
+def post_process(output_dir: str):
+    """
+    1. replace some bad attr
+    2. check and remove syntax error lines
+    3. insert some import modules. This should be the last process.
+    """
+    for root, _, files in os.walk(output_dir):
+        if not files:
+            continue
+
+        for f in files:
+            # only patch stub files: *.pyi
+            if not f.endswith('.pyi'):
+                continue
+
+            filename = str(Path(root) / f)
+            # insert modules
+            insert_import_modules(filename)
+
+            replace_bad_attr(filename)
+            check_remove_syntax_error(filename)
+
+            # insert modules if necessary
+            insert_import_modules(filename)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        type=str,
+        default="python/paddle/",
+    )
+    parser.add_argument(
+        "-m",
+        "--module-name",
+        type=str,
+        default="",
+    )
+
+    parser.add_argument(
+        "--ignore-all-errors",
+        default=False,
+        action="store_true",
+        help="Ignore all errors during module parsing",
+    )
+
+    parser.add_argument(
+        "--print-invalid-expressions-as-is",
+        default=False,
+        action="store_true",
+        help="Suppress the replacement with 'typing.Any' of invalid expressions"
+        "found in annotations",
+    )
+
+    parser.add_argument(
+        "--ops-yaml",
+        nargs='*',
+        help="Parse ops yaml, the input should be `<yaml path>;<ops module>` or `<yaml path>;<ops module>;<op_prefix>`"
+        "like `/foo/bar/ops.yaml;paddle.x.y.ops` or /foo/bar/ops.yaml;paddle.x.y.ops;sparse",
+    )
+
+    parser.add_argument(
+        "--python-api-info-yaml-path",
+        type=str,
+        default=None,
+        help="the yaml file path for python api info",
+    )
+
+    args = parser.parse_args()
+
+    return args
+
+
+def generate_stub_file(
+    output_dir: str,
+    module_name: str,
+    ignore_all_errors: bool = False,
+    print_invalid_expressions_as_is: bool = False,
+    ops_yaml: list[str] | None = None,
+    python_api_info_yaml_path: str | None = None,
+):
+    # patch `pybind11-stubgen`
+    patch_pybind11_stubgen_printer()
+
+    # generate stub files
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        # gen stub
+        gen_stub(
+            output_dir=tmpdirname,  # e.g.: 'Paddle/python/',
+            module_name=module_name,  # e.g.: 'paddle.base.libpaddle',
+            ignore_all_errors=ignore_all_errors,
+            print_invalid_expressions_as_is=print_invalid_expressions_as_is,
+        )
+
+        # parse ops yaml into file
+        if ops_yaml is not None:
+            ops_yaml_helper = OpsYamlBaseAPI()
+            python_api_info: dict[str, list[str]] = {}
+            if python_api_info_yaml_path is not None:
+                python_api_info = ops_yaml_helper.parse_python_api_info(
+                    python_api_info_yaml_path
+                )
+            for (
+                yaml_path,
+                dst_module,
+                op_prefix,
+            ) in ops_yaml_helper._parse_yaml_inputs(ops_yaml):
+                dst_module_path = ops_yaml_helper._parse_dst_module_to_path(
+                    tmpdirname,
+                    module_name,
+                    dst_module,
+                )
+
+                ops_yaml_helper.parse_yaml_ops(
+                    yaml_path,
+                    dst_module_path,
+                    python_api_info,
+                    op_prefix,
+                )
+                ops_yaml_helper.insert_yaml_imports(dst_module_path)
+
+        # move stub files into output_dir
+        paths = module_name.split('.')
+        source_path = Path(tmpdirname).joinpath(*paths)
+
+        if source_path.is_dir():
+            _path_dst = Path(output_dir).joinpath(paths[-1])
+            if _path_dst.exists():
+                shutil.rmtree(str(_path_dst))
+        else:
+            paths[-1] += '.pyi'
+            _path_dst = Path(output_dir).joinpath(paths[-1])
+            if _path_dst.exists():
+                os.remove(str(_path_dst))
+
+        shutil.move(str(source_path), output_dir)
+
+    # post process
+    post_process(output_dir)
+
+
+def load_python_api_function_by_name(name: str) -> Any:
+    components = name.split('.')
+    mod = importlib.import_module(components[0])
+    fn = mod
+    for comp in components[1:]:
+        fn = getattr(fn, comp)
+    return fn
+
+
+class _OpsYamlInputs(TypedDict):
+    names: list[str]
+    input_info: dict[str, str]
+
+
+class _OpsYamlAttr(TypedDict):
+    names: list[str]
+    attr_info: dict[str, tuple[str, str]]
+
+
+class OpsYamlBaseAPI:
+    """ref: paddle/phi/api/generator/api_base.py"""
+
+    import_inserted: set[str] = set()
+
+    # ref: paddle/phi/api/generator/api_base.py
+    def parse_input_and_attr(
+        self, api_name: str, args_config: str, optional_vars: list[str] = []
+    ) -> tuple[_OpsYamlInputs, _OpsYamlAttr]:
+        inputs = {'names': [], 'input_info': {}}
+        attrs = {'names': [], 'attr_info': {}}
+        args_str = args_config.strip()
+        assert args_str.startswith('(') and args_str.endswith(')'), (
+            f"Args declaration should start with '(' and end with ')', please check the args of {api_name} in yaml."
+        )
+        args_str = args_str[1:-1]
+        pattern = re.compile(r',(?![^{]*\})')  # support int[] a={1,3}
+        args_list = re.split(pattern, args_str.strip())
+        args_list = [x.strip() for x in args_list]
+
+        for item in args_list:
+            item = item.strip()
+            type_and_name = item.split(' ')
+            # match the input tensor
+            has_input = False
+            for in_type_symbol, in_type in INPUT_TYPES_MAP.items():
+                if type_and_name[0] == in_type_symbol:
+                    input_name = type_and_name[1].strip()
+                    assert len(input_name) > 0, (
+                        f"The input tensor name should not be empty. Please check the args of {api_name} in yaml."
+                    )
+                    assert len(attrs['names']) == 0, (
+                        f"The input Tensor should appear before attributes. please check the position of {api_name}:input({input_name}) in yaml"
+                    )
+
+                    if input_name in optional_vars:
+                        in_type = OPTIONAL_TYPES_TRANS[in_type_symbol]
+
+                    inputs['names'].append(input_name)
+                    inputs['input_info'][input_name] = in_type
+                    has_input = True
+                    break
+            if has_input:
+                continue
+
+            # match the attribute
+            for attr_type_symbol, attr_type in ATTR_TYPES_MAP.items():
+                if type_and_name[0] == attr_type_symbol:
+                    attr_name = item[len(attr_type_symbol) :].strip()
+                    assert len(attr_name) > 0, (
+                        f"The attribute name should not be empty. Please check the args of {api_name} in yaml."
+                    )
+                    default_value = None
+                    if '=' in attr_name:
+                        attr_infos = attr_name.split('=')
+                        attr_name = attr_infos[0].strip()
+                        default_value = attr_infos[1].strip()
+
+                    if attr_name in optional_vars:
+                        attr_type = OPTIONAL_TYPES_TRANS[attr_type_symbol]
+
+                    attrs['names'].append(attr_name)
+                    attrs['attr_info'][attr_name] = (attr_type, default_value)
+                    break
+
+        return inputs, attrs
+
+    # ref: paddle/phi/api/generator/api_base.py
+    def parse_output(
+        self, api_name: str, output_config: str
+    ) -> tuple[list[str], Any, Any]:
+        def parse_output_item(output_item):
+            result = re.search(
+                r"(?P<out_type>[a-zA-Z0-9_[\]]+)\s*(?P<name>\([a-zA-Z0-9_@]+\))?\s*(?P<expr>\{[^\}]+\})?",
+                output_item,
+            )
+            assert result is not None, (
+                f"{api_name} : the output config parse error."
+            )
+            out_type = result.group('out_type')
+            assert out_type in OUTPUT_TYPE_MAP, (
+                f"{api_name} : Output type error: the output type only support Tensor and Tensor[], \
+                    but now is {out_type}."
+            )
+
+            out_name = (
+                'out'
+                if result.group('name') is None
+                else result.group('name')[1:-1]
+            )
+            out_size_expr = (
+                None
+                if result.group('expr') is None
+                else result.group('expr')[1:-1]
+            )
+            return OUTPUT_TYPE_MAP[out_type], out_name, out_size_expr
+
+        temp_list = output_config.split(',')
+
+        if len(temp_list) == 1:
+            out_type, out_name, size_expr = parse_output_item(temp_list[0])
+            return [out_type], [out_name], [size_expr]
+        else:
+            out_type_list = []
+            out_name_list = []
+            out_size_expr_list = []
+            for output_item in temp_list:
+                out_type, out_name, size_expr = parse_output_item(output_item)
+                out_type_list.append(out_type)
+                out_name_list.append(out_name)
+                out_size_expr_list.append(size_expr)
+
+            return out_type_list, out_name_list, out_size_expr_list
+
+    def _make_sig_name(self, name: str) -> str:
+        # 'lambda' -> 'lambda_'
+        if keyword.iskeyword(name):
+            name += '_'
+
+        return name
+
+    def _make_attr(self, info: tuple[str, str | None]) -> str:
+        info_name, info_value = info
+        if info_value is None:
+            return info_name
+
+        if info_name.startswith('list') and '{' in info_value:
+            info_value = info_value.replace('{', '[').replace('}', ']')
+
+        elif info_value in FUNCTION_VALUE_TRANS:
+            info_value = FUNCTION_VALUE_TRANS[info_value]
+
+        elif info_name == 'float' and info_value.lower().endswith('f'):
+            info_value = info_value[:-1]
+
+        elif info_name == 'int' and info_value.lower().endswith('l'):
+            info_value = info_value[:-1]
+
+        else:
+            try:
+                eval(info_value)
+            except:
+                info_value = f'"{info_value}"'
+
+        return info_name + ' = ' + info_value
+
+    def _make_sig(self, name: str, sig: tuple[str, str]) -> str:
+        return self._make_sig_name(name) + ': ' + self._make_attr(sig)
+
+    def make_op_function(
+        self,
+        name: str,
+        inputs: _OpsYamlInputs,
+        attrs: _OpsYamlAttr,
+        output_type_list: list[str],
+    ) -> str:
+        input_info_names = inputs['names']
+        input_info = inputs['input_info']
+        attr_info_names = attrs['names']
+        attr_info = attrs['attr_info']
+
+        _sig_info = [
+            self._make_sig(_name, (input_info[_name], None))
+            for _name in input_info_names
+            if _name in input_info
+        ]
+        _sig_attr = [
+            self._make_sig(_name, attr_info[_name])
+            for _name in attr_info_names
+            if _name in attr_info
+        ]
+        sig_input = ', '.join(_sig_info + _sig_attr)
+        sig_output = (
+            output_type_list[0]
+            if len(output_type_list) == 1
+            else f'tuple[{", ".join(output_type_list)}]'
+        )
+
+        return f'def {name}({sig_input}) -> {sig_output}:\n'
+
+    def make_python_api_function(
+        self,
+        name: str,
+        python_api_names: list[str],
+    ) -> str:
+        fn = load_python_api_function_by_name(python_api_names[0])
+        sig = inspect.signature(fn)
+        return f'def {name}{sig}:\n'
+
+    def parse_python_api_info(self, yaml_path: str) -> dict[str, list[str]]:
+        # op name -> python api names
+        # e.g. {'add': ['paddle.add', 'paddle.Tensor.add']}
+        api_info: dict[str, list[str]] = {}
+        with open(yaml_path) as f:
+            api_list = yaml.load(f, Loader=yaml.FullLoader)
+            for api_item_yaml in api_list:
+                op_name = api_item_yaml['op']
+                api_names = [item.strip() for item in api_item_yaml['name']]
+                api_info[op_name] = api_names
+
+        return api_info
+
+    # ref: paddle/phi/api/generator/api_base.py
+    def parse_yaml_ops(
+        self,
+        yaml_file: str,
+        dst_module_path: str,
+        python_api_info: dict[str, list[str]],
+        op_prefix: str | None = None,
+    ) -> None:
+        ops_names = {}
+        ops_file = []
+        # read stub file generated by pybind11-stubgen and get op names
+        with open(dst_module_path) as f:
+            for line_no, line in enumerate(f.readlines()):
+                match_obj = PATTERN_FUNCTION.match(line)
+                if match_obj is not None:
+                    ops_names[match_obj.group('name')] = line_no
+                ops_file.append(line)
+
+        # read yaml
+        with open(yaml_file) as f:
+            api_list = yaml.load(f, Loader=yaml.FullLoader)
+            for api_item_yaml in api_list:
+                optional_vars = []
+                if 'optional' in api_item_yaml:
+                    optional_vars = [
+                        item.strip()
+                        for item in api_item_yaml['optional'].split(',')
+                    ]
+
+                # get op_name, and add op_prefix
+                raw_op_name = api_item_yaml['op']
+                raw_op_name = (
+                    f'{op_prefix}_{raw_op_name}'
+                    if op_prefix is not None
+                    else raw_op_name
+                )
+                op_args = api_item_yaml['args']
+                op_output = api_item_yaml['output']
+
+                # generate input and output
+                op_inputs, op_attrs = self.parse_input_and_attr(
+                    raw_op_name, op_args, optional_vars
+                )
+                output_type_list, _, _ = self.parse_output(
+                    raw_op_name, op_output
+                )
+
+                # generate full signature from op and inplace op
+                for op_name in [raw_op_name, raw_op_name + '_']:
+                    if op_name in ops_names:
+                        try:
+                            if op_name in python_api_info:
+                                op_signature = self.make_python_api_function(
+                                    op_name, python_api_info[op_name]
+                                )
+                            else:
+                                op_signature = self.make_op_function(
+                                    raw_op_name,
+                                    op_inputs,
+                                    op_attrs,
+                                    output_type_list,
+                                )
+                            # replace the line from stub file with full signature
+                            ops_file[ops_names[op_name]] = op_signature
+                        except:
+                            print(
+                                op_name, op_inputs, op_attrs, output_type_list
+                            )
+                            raise
+
+        with open(dst_module_path, 'w') as f:
+            f.writelines(ops_file)
+
+    def insert_yaml_imports(self, dst_module_path: str) -> None:
+        # insert imports into file only once
+        if dst_module_path in self.import_inserted:
+            return
+        self.import_inserted.add(dst_module_path)
+
+        ops_file = []
+        with open(dst_module_path, 'r') as f:
+            ops_file = f.readlines()
+
+        import_line_no = 0
+        for line_no, line in enumerate(ops_file):
+            if line.startswith('from __future__ import annotations'):
+                import_line_no = line_no + 1
+                break
+
+        # insert imports
+        ops_file = [
+            *ops_file[:import_line_no],
+            "\n",
+            *OPS_YAML_IMPORTS.strip().splitlines(keepends=True),
+            "\n",
+            *ops_file[import_line_no:],
+        ]
+
+        with open(dst_module_path, 'w') as f:
+            f.writelines(ops_file)
+
+    def _parse_yaml_inputs(
+        self,
+        ops_yaml: list[str],
+    ) -> Generator[tuple[str, str, str], None, None]:
+        for ops in ops_yaml:
+            _ops = ops.split(';')
+            if len(_ops) == 2:
+                yaml_path, dst_module = _ops
+                op_prefix = None
+            elif len(_ops) == 3:
+                yaml_path, dst_module, op_prefix = _ops
+            yield yaml_path, dst_module, op_prefix
+
+    def _parse_dst_module_to_path(
+        self, base_dir: str, module_name: str, dst_module: str
+    ) -> str:
+        assert dst_module.startswith(
+            module_name
+        )  # e.g.: `paddle.base.libpaddle` in `paddle.base.libpaddle.eager.ops`
+
+        paths = dst_module.split('.')
+        dst_path = Path(base_dir).joinpath(*paths)
+        if dst_path.is_dir():
+            dst_path = dst_path.joinpath('__init__.pyi')
+        else:
+            dst_path = dst_path.parent.joinpath(paths[-1] + '.pyi')
+
+        assert dst_path.exists()
+        return str(dst_path)
+
+
+def main():
+    args = parse_args()
+    generate_stub_file(
+        output_dir=args.output_dir,
+        module_name=args.module_name,
+        ignore_all_errors=args.ignore_all_errors,
+        print_invalid_expressions_as_is=args.print_invalid_expressions_as_is,
+        ops_yaml=args.ops_yaml,
+        python_api_info_yaml_path=args.python_api_info_yaml_path,
+    )
+
+
+if __name__ == '__main__':
+    main()

tools/gen_tensor_stub.py

@@ -0,0 +1,642 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import inspect
+import logging
+import re
+import sys
+import traceback
+from dataclasses import dataclass
+from functools import cached_property, lru_cache
+from typing import TYPE_CHECKING, Any, Literal, TypeAlias
+
+from typing_extensions import get_overloads
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+    from types import ModuleType
+
+logging.basicConfig(style="{", format="{message}", level=logging.INFO)
+logger = logging.getLogger("Generating stub file for paddle.Tensor")
+logger.setLevel(logging.INFO)
+
+INDENT_SIZE = 4
+INDENT = " " * INDENT_SIZE
+MEANLESS_INDEX = -1
+
+
+MemberType: TypeAlias = Literal[
+    "doc",
+    "attribute",
+    "method",
+]
+
+
+@dataclass
+class Member:
+    id: int
+    name: str
+    type: MemberType
+    aliases: list[str]
+    decorators: list[str]
+    signature: str
+    doc: str | None
+
+    def add_alias(self, alias: str):
+        self.aliases.append(alias)
+
+
+@lru_cache
+def _slot_pattern(slot_name: str) -> re.Pattern:
+    return re.compile(
+        r"(?P<indent> *)#\s*annotation:\s*\$\{" + slot_name + r"\}"
+    )
+
+
+@lru_cache
+def create_builtin_annotation_renamer():
+    # NOTE(ooooo-create): Rename built-in types to avoid naming conflicts
+    builtin_types = ["int", "bool", "str", "float", "complex", "bytes"]
+    regex_string = "|".join([rf"\b{t}\b" for t in builtin_types])
+    regex = re.compile(regex_string)
+
+    def renamer(annotations):
+        if annotations is inspect.Signature.empty:
+            return annotations
+        return regex.sub(lambda m: f"_{m.group(0)}", annotations)
+
+    return renamer
+
+
+def rename_builtin_annotation(annotation):
+    renamer = create_builtin_annotation_renamer()
+    return renamer(annotation)
+
+
+class TensorGen:
+    def __init__(self, template: str = '', prefix: str = 'tensor'):
+        self._template = template
+        self._template_codes: list[tuple[int, int, str]] = []
+        self._prefix = prefix
+
+    def find_annotation_slot(self, slot_name: str) -> tuple[str, int, int]:
+        pattern = _slot_pattern(slot_name)
+        slot = []
+        for mo in pattern.finditer(self._template):
+            _indent = mo.group('indent')
+            _start_index, _end_index = mo.span()
+            slot.append((_indent, _start_index, _end_index))
+
+        assert len(slot) == 1, self._template
+        return slot[0]
+
+    @property
+    def tensor_docstring(self):
+        return self.find_annotation_slot(f'{self._prefix}_docstring')
+
+    @property
+    def tensor_attributes(self):
+        return self.find_annotation_slot(f'{self._prefix}_attributes')
+
+    @property
+    def tensor_methods(self):
+        return self.find_annotation_slot(f'{self._prefix}_methods')
+
+    @property
+    def tensor_alias(self):
+        return self.find_annotation_slot(f'{self._prefix}_alias')
+
+    @property
+    def template_codes(self) -> list[tuple[int, int, str]]:
+        return self._template_codes
+
+    @property
+    def template_begin(self) -> int:
+        return self._template.find(f'{self._prefix}_begin')
+
+    @property
+    def template_end(self) -> int:
+        return self._template.find(f'{self._prefix}_end')
+
+    def find_apis(self, api_name: str) -> list[dict[str, tuple[str, int, int]]]:
+        if self.template_begin < 0 or self.template_end < 0:
+            return []
+
+        pattern = re.compile(
+            r"(?P<indent> *)(?P<def_api>def "
+            + api_name
+            + r")(?P<signature>\(.*?\).*?:)(?P<docstring>.*?)(?P<ellipsis>\.{3})(?P<comment>[^\n]*#[^\n]*\n)?",
+            re.DOTALL,
+        )
+        api = []
+        for mo in pattern.finditer(self._template):
+            _start_index, _end_index = mo.span()
+            if not (
+                self.template_begin < _start_index < self.template_end
+                and self.template_begin < _end_index < self.template_end
+            ):
+                continue
+
+            _indent = mo.group('indent')
+            _signature = mo.group('signature')
+            _docstring = mo.group('docstring')
+            _ellipsis = mo.group('ellipsis')
+            _comment = mo.group('comment')
+            _comment = '' if _comment is None else _comment
+
+            _start_indent, _ = mo.span('indent')
+            _start_docstring, _ = mo.span('docstring')
+            _, _end_ellipsis = mo.span('ellipsis')
+            _start_comment = _end_ellipsis
+            _end_comment = _start_comment + len(_comment)
+
+            assert _start_index == _start_indent
+            assert _end_comment == _end_index
+
+            _api = {
+                'indent': (_indent, MEANLESS_INDEX, MEANLESS_INDEX),
+                'signature': (_signature, MEANLESS_INDEX, MEANLESS_INDEX),
+                'docstring': (_docstring, _start_docstring, _end_comment),
+                'ellipsis': (_ellipsis, MEANLESS_INDEX, MEANLESS_INDEX),
+                'comment': (_comment, MEANLESS_INDEX, MEANLESS_INDEX),
+            }
+            api.append(_api)
+
+        return api
+
+    def insert_template(self, code: str, start: int, end: int) -> None:
+        if start != MEANLESS_INDEX and end != MEANLESS_INDEX:
+            self._template_codes.append((start, end, code))
+
+    def add_method(self, func: Member):
+        """
+        1. insert docstring: tensor.prototype.pyi define the method without docstring
+        2. insert method: tensor.prototype.pyi NOT define the method
+        """
+        methods = self.find_apis(func.name)
+        if methods:
+            # only use the last method
+            method = methods[-1]
+            # insert docstring if necessary
+            if not method['docstring'][0].strip():
+                doc = func.doc
+                if doc:
+                    comment = method['comment'][0]
+                    doc_start = method['docstring'][1]
+                    doc_end = method['docstring'][2]
+
+                    api_indent = method['indent'][0]
+
+                    assert len(api_indent) % INDENT_SIZE == 0
+
+                    _indent = api_indent + INDENT
+
+                    _doc = '\n'  # new line
+                    _doc += f'{_indent}r"""\n'
+                    _doc += with_indent(doc, len(_indent) // INDENT_SIZE)
+                    _doc += "\n"
+                    _doc += f'{_indent}"""\n'
+                    _doc += f'{_indent}...\n'
+                    _doc += f'{_indent}\n'
+
+                    self.insert_template(comment + _doc, doc_start, doc_end)
+        else:
+            method_code = '\n'
+            for decorator in func.decorators:
+                method_code += f"@{decorator}\n"
+
+            method_code += f"def {func.signature}:\n"
+            # do NOT insert docs from overload methods,
+            # because we always add a plain method
+            if func.doc and func.decorators != ["overload"]:
+                method_code += f'{INDENT}r"""\n'
+                method_code += with_indent(func.doc, 1)
+                method_code += "\n"
+                method_code += f'{INDENT}"""\n'
+            method_code += f"{INDENT}...\n"
+
+            _indent, _, _end_index = self.tensor_methods
+            method_code = with_indent(method_code, len(_indent) // INDENT_SIZE)
+            self.insert_template(method_code, _end_index, _end_index)
+
+    def add_alias(self, alias: str, target: str):
+        _indent, _, _end_index = self.tensor_alias
+        aliases_code = "\n"
+        aliases_code += f"{_indent}{alias} = {target}"
+        self.insert_template(aliases_code, _end_index, _end_index)
+
+    def add_attribute(self, name: str, type_: str):
+        _indent, _, _end_index = self.tensor_attributes
+        attr_code = "\n"
+        attr_code += f"{_indent}{name}: {type_}"
+        self.insert_template(attr_code, _end_index, _end_index)
+
+    def add_doc(self, doc: str):
+        _indent, _, _end_index = self.tensor_docstring
+        docstring = "\n"
+        docstring += 'r"""\n'
+        docstring += doc
+        docstring += "\n"
+        docstring += '"""\n'
+        docstring = with_indent(docstring, len(_indent) // INDENT_SIZE)
+        self.insert_template(docstring, _end_index, _end_index)
+
+    @classmethod
+    def codegen(
+        cls, template: str, template_codes: list[tuple[int, int, str]]
+    ) -> str:
+        header = (
+            '# This file is auto generated by `tools/gen_tensor_stub.py`.\n\n'
+        )
+
+        _template = []
+        start = 0
+        end = 0
+        for _start, _end, code in sorted(template_codes):
+            end = _start
+            _template.extend(
+                [
+                    template[start:end],
+                    code,
+                ]
+            )
+            start = _end
+        _template.append(template[start:])
+
+        _content = header + ''.join(_template)
+
+        return _content
+
+
+def is_inherited_member(name: str, cls: type) -> bool:
+    """Check if the member is inherited from parent class"""
+
+    # keep magic methods
+    if name.startswith("__") and name.endswith("__"):
+        return False
+
+    if name in cls.__dict__:
+        return False
+
+    for base in cls.__bases__:
+        if name in base.__dict__:
+            return True
+
+    return any(is_inherited_member(name, base) for base in cls.__bases__)
+
+
+def is_property(member: Any) -> bool:
+    """Check if the member is a property"""
+
+    return isinstance(member, (property, cached_property))
+
+
+def is_staticmethod(member: Any) -> bool:
+    """Check if the member is a staticmethod"""
+
+    return isinstance(member, staticmethod)
+
+
+def is_classmethod(member: Any) -> bool:
+    """Check if the member is a classmethod"""
+
+    return isinstance(member, classmethod)
+
+
+def process_lines(code: str, callback: Callable[[str], str]) -> str:
+    lines = code.splitlines()
+    end_with_newline = code.endswith("\n")
+    processed_lines: list[str] = []
+    for line in lines:
+        processed_lines.append(callback(line))
+    processed_code = "\n".join(processed_lines)
+    if end_with_newline:
+        processed_code += "\n"
+    return processed_code
+
+
+def with_indent(code: str, level: int) -> str:
+    def add_indent_line(line: str) -> str:
+        if not line:
+            return line
+        return INDENT + line
+
+    def remove_indent_line(line: str) -> str:
+        if not line:
+            return line
+        elif line.startswith(INDENT):
+            return line[len(INDENT) :]
+        else:
+            return line
+
+    if level == 0:
+        return code
+    elif level > 0:
+        if level == 1:
+            return process_lines(code, add_indent_line)
+        code = process_lines(code, add_indent_line)
+        return with_indent(code, level - 1)
+    else:
+        if level == -1:
+            return process_lines(code, remove_indent_line)
+        return with_indent(code, level - 1)
+
+
+def func_sig_to_method_sig(func_sig: str) -> str:
+    regex_func_sig = re.compile(
+        r"^(?P<method_name>[_a-zA-Z0-9]+)\((?P<arg0>[^,*)]+(:.+)?)?(?P<rest_args>.*)?\)",
+        re.DOTALL,
+    )
+    matched = regex_func_sig.search(func_sig)
+    if matched is None:
+        # TODO: resolve this case
+        logging.warning(f"Cannot parse function signature: {func_sig}")
+        return "_(self)"
+
+    if matched.group('rest_args').startswith('*'):
+        method_sig = regex_func_sig.sub(
+            r"\g<method_name>(self, \g<rest_args>)", func_sig
+        )
+
+    else:
+        method_sig = regex_func_sig.sub(
+            r"\g<method_name>(self\g<rest_args>)", func_sig
+        )
+
+    return method_sig
+
+
+def func_doc_to_method_doc(func_doc: str) -> str:
+    # Iterate every line, insert the indent and remove document of the first argument
+    method_doc = ""
+    is_first_arg = False
+    first_arg_offset = 0
+
+    for line in func_doc.splitlines():
+        current_line_offset = len(line) - len(line.lstrip())
+        # Remove the first argument (self in Tensor method) from docstring
+        if is_first_arg:
+            if current_line_offset <= first_arg_offset:
+                is_first_arg = False
+            if not first_arg_offset:
+                first_arg_offset = current_line_offset
+            if is_first_arg:
+                continue
+        method_doc += f"{line}\n" if line else "\n"
+        if line.lstrip().startswith("Args:"):
+            is_first_arg = True
+
+    return method_doc
+
+
+def try_import_paddle() -> ModuleType | None:
+    try:
+        return importlib.import_module('paddle')
+    except ModuleNotFoundError:
+        traceback.print_exc(file=sys.stderr)
+        sys.stderr.write(
+            '''
+ERROR: Can NOT import paddle from `tools/gen_tensor_stub.py` before installation.
+    So the stub file `python/paddle/tensor/tensor.pyi` of `paddle.Tensor` may be lost.
+    We COULD import paddle without installation with all libs (.dll or .so) copied into dir `paddle/libs`,
+    or path already been set for the system. Try the following steps to locate the problem.
+
+    1. Build with `SKIP_STUB_GEN=ON make -j$(nproc)`.
+    2. Install the wheel from `build/python/dist`.
+    3. Try to `import paddle` and check the problems.
+            '''
+        )
+    return None
+
+
+def get_tensor_members(module: str = 'paddle.Tensor') -> dict[int, Member]:
+    paddle = try_import_paddle()
+    if not paddle:
+        raise (
+            ModuleNotFoundError(
+                'Can NOT import paddle from tools/gen_tensor_stub.py.'
+            )
+        )
+
+    tensor_class = eval(module)
+
+    members: dict[int, Member] = {}
+    for name, member in inspect.getmembers(tensor_class):
+        member_id = id(member)
+        member_doc = inspect.getdoc(member)
+        member_doc_cleaned = (
+            func_doc_to_method_doc(inspect.cleandoc(member_doc))
+            if member_doc is not None
+            else None
+        )
+        try:
+            sig = inspect.signature(member)
+            sig = sig.replace(
+                parameters=[
+                    p.replace(
+                        annotation=rename_builtin_annotation(p.annotation)
+                    )
+                    for p in sig.parameters.values()
+                ],
+                return_annotation=rename_builtin_annotation(
+                    sig.return_annotation
+                ),
+            )
+            # TODO: classmethod
+            member_signature = f"{name}{sig}"
+
+        except (TypeError, ValueError):
+            member_signature = f"{name}()"
+
+        if is_inherited_member(name, tensor_class):
+            continue
+
+        # Filter out private members except magic methods
+        if name.startswith("_") and not (
+            name.startswith("__") and name.endswith("__")
+        ):
+            continue
+
+        if member_id in members:
+            members[member_id].add_alias(name)
+            continue
+
+        if name == '__doc__':
+            members[member_id] = Member(
+                member_id,
+                name,
+                "doc",
+                [],
+                [],
+                "__doc__",
+                member,
+            )
+        elif is_property(member) or inspect.isdatadescriptor(member):
+            members[member_id] = Member(
+                member_id,
+                name,
+                "method",
+                [],
+                ["property"],
+                f"{name}(self)",
+                member_doc_cleaned,
+            )
+        elif is_classmethod(member):
+            members[member_id] = Member(
+                member_id,
+                name,
+                "method",
+                [],
+                ["classmethod"],
+                member_signature,
+                member_doc_cleaned,
+            )
+        elif is_staticmethod(member):
+            members[member_id] = Member(
+                member_id,
+                name,
+                "method",
+                [],
+                ["staticmethod"],
+                member_signature,
+                member_doc_cleaned,
+            )
+        elif inspect.isfunction(member) or inspect.ismethod(member):
+            # `all_signatures`： list[[member id, decorators, signature]]
+            # with atleast an original method
+            all_signatures = [[member_id, [], member_signature]]
+
+            # try to get overloads
+            _overloads = get_overloads(member)
+            for f in _overloads:
+                _sig = inspect.signature(f)
+                _sig = _sig.replace(
+                    parameters=[
+                        p.replace(
+                            annotation=rename_builtin_annotation(p.annotation)
+                        )
+                        for p in _sig.parameters.values()
+                    ],
+                    return_annotation=rename_builtin_annotation(
+                        _sig.return_annotation
+                    ),
+                )
+                all_signatures.append(
+                    [
+                        id(f),
+                        ["overload"],
+                        f"{name}{_sig}".replace("Ellipsis", "..."),
+                    ]
+                )
+
+            for _member_id, _decorators, _sig in all_signatures:
+                members[_member_id] = Member(
+                    _member_id,
+                    name,
+                    "method",
+                    [],
+                    _decorators,
+                    func_sig_to_method_sig(_sig),
+                    member_doc_cleaned,
+                )
+        elif inspect.ismethoddescriptor(member):
+            members[member_id] = Member(
+                member_id,
+                name,
+                "method",
+                [],
+                [],
+                func_sig_to_method_sig(member_signature),
+                member_doc_cleaned,
+            )
+        else:
+            logging.debug(f"Skip unknown type of member: {name}, {member}")
+
+    return members
+
+
+def get_tensor_template(path: str) -> str:
+    with open(path) as f:
+        return ''.join(f.readlines())
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-i",
+        "--input-file",
+        type=str,
+        default="python/paddle/tensor/tensor.prototype.pyi",
+    )
+    parser.add_argument(
+        "-o",
+        "--output-file",
+        type=str,
+        default="python/paddle/tensor/tensor.pyi",
+    )
+
+    args = parser.parse_args()
+
+    return args
+
+
+def generate_stub_file(input_file=None, output_file=None):
+    # Get tensor template
+    tensor_template = get_tensor_template(input_file)
+
+    all_members: set[int] = set()
+    template_codes: list[tuple[int, int, str]] = []
+    # Get members of Tensor
+    for module, prefix in [
+        ['paddle.Tensor', 'tensor'],
+        ['paddle.base.framework.EagerParamBase', 'eager_param_base'],
+    ]:
+        tensor_members = get_tensor_members(module)
+        logging.debug(f'total members in {module}: {len(tensor_members)}')
+
+        # Generate the Tensor stub
+        tensor_gen = TensorGen(tensor_template, prefix)
+        for member_id, member in tensor_members.items():
+            if member_id in all_members:
+                continue
+
+            if member.type == "method":
+                tensor_gen.add_method(member)
+                for alias in member.aliases:
+                    tensor_gen.add_alias(alias, member.name)
+            elif member.type == "attribute":
+                tensor_gen.add_attribute(member.name, "Any")
+            elif member.type == "doc":
+                tensor_gen.add_doc(member.doc)
+
+        all_members |= tensor_members.keys()
+        template_codes += tensor_gen.template_codes
+
+    # Write to target file
+    with open(output_file, "w", encoding="utf-8") as f:
+        f.write(TensorGen.codegen(tensor_template, template_codes))
+
+
+def main():
+    args = parse_args()
+    generate_stub_file(args.input_file, args.output_file)
+
+
+if __name__ == "__main__":
+    main()

tools/gen_ut_cmakelists.hook

@@ -0,0 +1,6 @@
+
+set -e
+lists=`python tools/gen_ut_cmakelists.py -f $* |grep 'modified/new:'|cut -f 2 -d :`
+num=`echo $lists |wc -w`
+[[ $num -ge 1 ]] && git add $lists && exit 1
+exit 0

tools/gen_ut_cmakelists.py

@@ -0,0 +1,660 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import re
+import sys
+
+# port range (21200, 23000) is reserved for dist-ops
+
+
+# function to process pythonpath env
+# append "${PADDLE_BINARY_DIR}/python" to PYTHONPATH
+def _process_PYTHONPATH(pythonpath_option):
+    pythonpath_option += ":${PADDLE_BINARY_DIR}/python"
+    return pythonpath_option
+
+
+def _process_envs(envs):
+    """
+    Desc:
+        Input a str and output a str with the same function to specify some environment variables.
+    Here we can give a special process for some variable if needed.
+    Example 1:
+        Input: "http_proxy=;PYTHONPATH=.."
+        Output: "http_proxy=;PYTHONPATH=..:${PADDLE_BINARY_DIR}/python"
+    Example 2:
+        Input: "http_proxy=;https_proxy=123.123.123.123:1230"
+        Output: "http_proxy=;https_proxy=123.123.123.123:1230"
+    """
+    envs = envs.strip()
+
+    envs_parts = envs.split(";")
+    processed_envs = []
+
+    for p in envs_parts:
+        assert (
+            " " not in p
+            and re.compile("^[a-zA-Z_][0-9a-zA-Z_]*=").search(p) is not None
+        ), f"""The environment option format is wrong. The env variable name can only contains'a-z', 'A-Z', '0-9' and '_',
+and the var can not contain space in either env names or values.
+However the var's format is '{p}'."""
+
+        # if p starts with "PYTHONPATH=", then process python path
+        if re.compile("^PYTHONPATH=").search(p):
+            p = _process_PYTHONPATH(p)
+
+        processed_envs.append(p)
+
+    return ";".join(processed_envs)
+
+
+def _process_conditions(conditions):
+    """
+    Desc:
+        Input condition expression in cmake grammar and return a string wrapped by 'AND ()'.
+        If the conditions string is empty, return an empty string.
+    Example 1:
+        Input: "LINUX"
+        Output: "AND (LINUX)"
+    Example 2:
+        Input: ""
+        Output: ""
+    """
+    if len(conditions.strip()) == 0:
+        conditions = []
+    else:
+        conditions = conditions.strip().split(";")
+    return [c.strip() for c in conditions]
+
+
+def _process_archs(arch):
+    """
+    desc:
+        Input archs options and warp it with 'WITH_', 'OR' and '()' in cmakelist grammar.
+        The case is ignored.
+        If the input is empty, return "LOCAL_ALL_ARCH".
+    Example 1:
+        Input: 'gpu'
+        Output: '(WITH_GPU)'
+    Example 2:
+        Input: 'gpu;ROCM'
+        Output: '(WITH_GPU OR WITH_ROCM)'
+    """
+    archs = ""
+    arch = arch.upper().strip()
+    if len(arch) > 0:
+        for a in arch.split(";"):
+            if '' == a:
+                continue
+            assert a in ["GPU", "ROCM", "XPU"], (
+                f"""Supported arch options are "GPU", "ROCM", and "XPU", but the options is {a}"""
+            )
+            archs += "WITH_" + a.upper() + " OR "
+        arch = "(" + archs[:-4] + ")"
+    else:
+        arch = "LOCAL_ALL_ARCH"
+    return arch
+
+
+def _process_os(os_):
+    """
+    Desc:
+        Input os options and output wrapped options with 'OR' and '()'
+        If the input is empty, return "LOCAL_ALL_PLAT"
+    Example 1:
+        Input: "WIN32"
+        Output: "(WIN32)"
+    Example 2:
+        Input: "WIN32;linux"
+        Output: "(WIN32 OR LINUX)"
+    """
+    os_ = os_.strip()
+    if len(os_) > 0:
+        os_ = os_.upper()
+        for p in os_.split(';'):
+            assert p in ["WIN32", "APPLE", "LINUX"], (
+                f"""Supported os options are 'WIN32', 'APPLE' and 'LINUX', but the options is {p}"""
+            )
+        os_ = os_.replace(";", " OR ")
+        os_ = "(" + os_ + ")"
+    else:
+        os_ = "LOCAL_ALL_PLAT"
+    return os_
+
+
+# check whether run_serial is 0, 1 or empty
+def _process_run_serial(run_serial):
+    rs = run_serial.strip()
+    assert rs in [
+        "1",
+        "0",
+        "",
+    ], (
+        f"""the value of run_serial must be one of 0, 1 or empty. But this value is {rs}"""
+    )
+    if rs == "":
+        return ""
+    return rs
+
+
+def _file_with_extension(prefix, suffixes):
+    """
+    Desc:
+        check whether test file exists.
+    """
+    for ext in suffixes:
+        if os.path.isfile(prefix + ext):
+            return True
+    return False
+
+
+def _process_name(name, curdir):
+    """
+    Desc:
+        check whether name is with a legal format and check whether the test file exists.
+    """
+    name = name.strip()
+    assert re.compile("^test_[0-9a-zA-Z_]+").search(name), (
+        """If line is not the header of table, the test name must begin with "test_" """
+        """and the following substring must include at least one char of "0-9", "a-z", "A-Z" or "_"."""
+    )
+    filepath_prefix = os.path.join(curdir, name)
+    suffix = [".py", ".sh"]
+    assert _file_with_extension(filepath_prefix, suffix), (
+        f""" Please ensure the test file with the prefix '{filepath_prefix}' and one of the suffix {suffix} exists, because you specified a unittest named '{name}'"""
+    )
+
+    return name
+
+
+def _norm_dirs(dirs):
+    # reform all dirs' path as normal absolute path
+    # abspath() can automatically normalize the path format
+    norm_dirs = []
+    for d in dirs:
+        d = os.path.abspath(d)
+        if d not in norm_dirs:
+            norm_dirs.append(d)
+    return norm_dirs
+
+
+def _process_run_type(run_type):
+    rt = run_type.strip()
+    # completely match one of the strings: 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'HYBRID', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY'
+    assert re.compile(
+        "^(NIGHTLY|EXCLUSIVE|CINN|DIST|HYBRID|GPUPS|INFER|EXCLUSIVE:NIGHTLY|DIST:NIGHTLY)$"
+    ).search(rt), (
+        f""" run_type must be one of 'NIGHTLY', 'EXCLUSIVE', 'CINN', 'DIST', 'HYBRID', 'GPUPS', 'INFER', 'EXCLUSIVE:NIGHTLY' and 'DIST:NIGHTLY'"""
+        f"""but the run_type is {rt}"""
+    )
+    return rt
+
+
+class DistUTPortManager:
+    def __init__(self, ignore_dirs=[]):
+        self.dist_ut_port = 21200
+        self.assigned_ports = {}
+        self.last_test_name = ""
+        self.last_test_cmake_file = ""
+        self.no_cmake_dirs = []
+        self.processed_dirs = set()
+        self.ignore_dirs = _norm_dirs(ignore_dirs)
+
+    def reset_current_port(self, port=None):
+        self.dist_ut_port = 21200 if port is None else port
+
+    def get_current_port(self):
+        return self.dist_ut_port
+
+    def get_set_port(self, test_name, port):
+        '''
+        Get and set a port for unit test named test_name. If the test has been already holding a port, return the port it holds.
+        Else assign the input port as a new port to the test.
+        '''
+        if test_name not in self.assigned_ports:
+            self.assigned_ports[test_name] = port
+        self.dist_ut_port = max(
+            self.dist_ut_port, self.assigned_ports[test_name]
+        )
+        return self.assigned_ports[test_name]
+
+    def process_dist_port_num(self, port_num):
+        assert (
+            re.compile("^[0-9]+$").search(port_num)
+            and int(port_num) > 0
+            or port_num.strip() == ""
+        ), (
+            f"""port_num must be format as a positive integer or empty, but this port_num is '{port_num}'"""
+        )
+        port_num = port_num.strip()
+        if len(port_num) == 0:
+            return 0
+        port = self.dist_ut_port
+        assert port < 23000, "dist port is exhausted"
+        self.dist_ut_port += int(port_num)
+        return port
+
+    def _init_dist_ut_ports_from_cmakefile(self, cmake_file_name):
+        '''
+        Desc:
+            Find all signed ut ports in cmake_file and update the ASSIGNED_PORTS
+            and keep the DIST_UT_PORT max of all assigned ports
+        '''
+        with open(cmake_file_name) as cmake_file:
+            # match lines including 'PADDLE_DIST_UT_PORT=' followed by a number
+            port_reg = re.compile("PADDLE_DIST_UT_PORT=[0-9]+")
+            lines = cmake_file.readlines()
+            for idx, line in enumerate(lines):
+                matched = port_reg.search(line)
+                if matched is None:
+                    continue
+                p = matched.span()
+                port = int(line[p[0] : p[1]].split("=")[-1])
+
+                # find the test name which the port belongs to
+                for k in range(idx, 0, -1):
+                    if lines[k].strip() == "START_BASH":
+                        break
+                name = lines[k - 1].strip()
+
+                # match right tests name format, the name must start with 'test_' followed by at least one char of
+                # '0-9'. 'a-z'. 'A-Z' or '_'
+                assert re.compile(
+                    "^test_[0-9a-zA-Z_]+"
+                ).search(
+                    name
+                ), f'''we found a test for initial the latest dist_port but the test name '{name}' seems to be wrong
+                    at line {k - 1}, in file {cmake_file_name}
+                    '''
+                self.get_set_port(name, port)
+
+                # get the test_name which latest assigned port belongs to
+                if self.assigned_ports[name] == self.dist_ut_port:
+                    self.last_test_name = name
+                    self.last_test_cmake_file = cmake_file_name
+
+    def parse_assigned_dist_ut_ports(self, current_work_dir, depth=0):
+        '''
+        Desc:
+            get all assigned dist ports to keep port of unmodified test fixed.
+        '''
+        if current_work_dir in self.processed_dirs:
+            return
+
+        # if root(depth==0)
+        if depth == 0:
+            self.processed_dirs.clear()
+
+        self.processed_dirs.add(current_work_dir)
+        contents = os.listdir(current_work_dir)
+        cmake_file = os.path.join(current_work_dir, "CMakeLists.txt")
+        csv = cmake_file.replace("CMakeLists.txt", 'testslist.csv')
+
+        if os.path.isfile(csv) or os.path.isfile(cmake_file):
+            if current_work_dir not in self.ignore_dirs:
+                if os.path.isfile(cmake_file) and os.path.isfile(csv):
+                    self._init_dist_ut_ports_from_cmakefile(cmake_file)
+                elif not os.path.isfile(cmake_file):
+                    # put the directory which has csv but no cmake into NO_CMAKE_DIR_WARNING
+                    self.no_cmake_dirs.append(current_work_dir)
+
+            # recursively process the subdirectories
+            for c in contents:
+                c_path = os.path.join(current_work_dir, c)
+                if os.path.isdir(c_path):
+                    self.parse_assigned_dist_ut_ports(c_path, depth + 1)
+
+        if depth == 0:
+            # After all directories are scanned and processed
+            # 1. Get the num_port of last added test and set DIST_UT_PORT+=num_port
+            #    to guarantee the DIST_UT_PORT is not assigned
+            # 2. Summary all the directories which include csv but no cmake and show an error
+            #    if such a directory exists
+
+            # step 1
+            if (
+                len(self.last_test_name) > 0
+                and len(self.last_test_cmake_file) > 0
+            ):
+                with open(
+                    self.last_test_cmake_file.replace(
+                        "CMakeLists.txt", "testslist.csv"
+                    )
+                ) as csv_file:
+                    found = False
+                    for line in csv_file:
+                        (
+                            name,
+                            _,
+                            _,
+                            _,
+                            _,
+                            launcher,
+                            num_port,
+                            _,
+                            _,
+                            _,
+                        ) = line.strip().split(",")
+                        if name == self.last_test_name:
+                            found = True
+                            break
+                assert found, (
+                    f"no such test named '{self.last_test_name}' in file '{self.last_test_cmake_file}'"
+                )
+                if launcher[-2:] == ".sh":
+                    self.process_dist_port_num(num_port)
+
+            # step 2
+            err_msg = """==================[No Old CMakeLists.txt Error]==================================
+        Following directories has no CmakeLists.txt files:
+    """
+            for c in self.no_cmake_dirs:
+                err_msg += "   " + c + "\n"
+            err_msg += """
+        This may cause the dist ports different with the old version.
+        If the directories are newly created or there is no CMakeLists.txt before, or ignore this error, you
+        must specify the directories using the args option --ignore-cmake-dirs/-i.
+        If you want to keep the dist ports of old tests unchanged, please ensure the old
+        version CMakeLists.txt file existing before using the gen_ut_cmakelists tool to
+        generate new CmakeLists.txt files.
+    ====================================================================================
+    """
+            assert len(self.no_cmake_dirs) == 0, err_msg
+
+
+class CMakeGenerator:
+    def __init__(self, current_dirs, only_check, ignore_dirs):
+        self.processed_dirs = set()
+        self.port_manager = DistUTPortManager(ignore_dirs)
+        self.current_dirs = _norm_dirs(current_dirs)
+        self.modified_or_created_files = []
+        self._only_check = only_check
+
+    def prepare_dist_ut_port(self):
+        for c in self._find_root_dirs():
+            self.port_manager.parse_assigned_dist_ut_ports(c, depth=0)
+
+    def parse_csvs(self):
+        '''
+        parse csv files, return the lists of created or modified files
+        '''
+        self.modified_or_created_files = []
+        for c in self.current_dirs:
+            self._gen_cmakelists(c)
+        return self.modified_or_created_files
+
+    def _find_root_dirs(self):
+        root_dirs = []
+        # for each current directory, find its highest ancient directory (at least itself)
+        # which includes CMakeLists.txt or testslist.csv.txt in the file system tree
+        for c in self.current_dirs:
+            while True:
+                ppath = os.path.dirname(c)
+                if ppath == c:
+                    break
+                cmake = os.path.join(ppath, "CMakeLists.txt")
+                csv = os.path.join(ppath, "testslist.csv.txt")
+                if not (os.path.isfile(cmake) or os.path.isfile(csv)):
+                    break
+                c = ppath
+            if c not in root_dirs:
+                root_dirs.append(c)
+        return root_dirs
+
+    def _parse_line(self, line, curdir):
+        """
+        Desc:
+            Input a line in csv file and output a string in cmake grammar, adding the specified test and setting its properties.
+        Example:
+            Input: "test_allreduce,linux,gpu;rocm,120,DIST,test_runner.py,20071,1,PYTHONPATH=..;http_proxy=;https_proxy=,"
+            Output:
+                "if((WITH_GPU OR WITH_ROCM) AND (LINUX) )
+                    py_test_modules(
+                    test_allreduce
+                    MODULES
+                    test_allreduce
+                    ENVS
+                    "PADDLE_DIST_UT_PORT=20071;PYTHONPATH=..:${PADDLE_BINARY_DIR}/python;http_proxy=;https_proxy=")
+                    set_tests_properties(test_allreduce PROPERTIES  TIMEOUT "120" RUN_SERIAL 1)
+                endif()"
+        """
+
+        (
+            name,
+            os_,
+            archs,
+            timeout,
+            run_type,
+            launcher,
+            num_port,
+            run_serial,
+            envs,
+            conditions,
+        ) = line.strip().split(",")
+
+        # name == "name" means the line being parsed is the header of the table
+        # we should skip this line and return empty here.
+        if name == "name":
+            return ""
+        name = _process_name(name, curdir)
+
+        envs = _process_envs(envs)
+        conditions = _process_conditions(conditions)
+        archs = _process_archs(archs)
+        os_ = _process_os(os_)
+        run_serial = _process_run_serial(run_serial)
+
+        cmd = ""
+
+        for c in conditions:
+            cmd += f"if ({c})\n"
+
+        time_out_str = (
+            f' TIMEOUT "{timeout}"' if len(timeout.strip()) > 0 else ''
+        )
+
+        if launcher[-3:] == ".sh":
+            run_type = _process_run_type(run_type)
+            dist_ut_port = self.port_manager.process_dist_port_num(num_port)
+            dist_ut_port = self.port_manager.get_set_port(name, dist_ut_port)
+            cmd += f'''if({archs} AND {os_})
+        bash_test_modules(
+        {name}
+        START_BASH
+        {launcher}
+        {time_out_str}
+        LABELS
+        "RUN_TYPE={run_type}"
+        ENVS
+        "PADDLE_DIST_UT_PORT={dist_ut_port};{envs}")%s
+    endif()
+    '''
+            run_type_str = ""
+        else:
+            try:
+                run_type = _process_run_type(run_type)
+            except Exception as e:
+                assert run_type.strip() == "", (
+                    f"{e}\nIf use test_runner.py, the run_type can be ''"
+                )
+            cmd += f'''if({archs} AND {os_})
+        py_test_modules(
+        {name}
+        MODULES
+        {name}
+        ENVS
+        "{envs}")%s
+    endif()
+    '''
+            run_type_str = (
+                "" if len(run_type) == 0 else f' LABELS "RUN_TYPE={run_type}"'
+            )
+        run_serial_str = (
+            f' RUN_SERIAL {run_serial}' if len(run_serial) > 0 else ''
+        )
+        if (
+            len(time_out_str) > 0
+            or len(run_serial_str) > 0
+            or len(run_type_str) > 0
+        ):
+            set_properties = f'''
+        set_tests_properties({name} PROPERTIES{time_out_str}{run_serial_str}{run_type_str})'''
+        else:
+            set_properties = ""
+        cmd = cmd % set_properties
+        for _ in conditions:
+            cmd += "endif()\n"
+        return cmd
+
+    def _gen_cmakelists(self, current_work_dir, depth=0):
+        if depth == 0:
+            self.processed_dirs.clear()
+        if current_work_dir == "":
+            current_work_dir = "."
+
+        contents = os.listdir(current_work_dir)
+        contents.sort()
+        sub_dirs = []
+        for c in contents:
+            c_path = os.path.join(current_work_dir, c)
+            if c_path in self.processed_dirs:
+                return
+            if not os.path.isdir(c_path):
+                continue
+            self.processed_dirs.add(c_path)
+            if os.path.isfile(
+                os.path.join(current_work_dir, c, "testslist.csv")
+            ) or os.path.isfile(
+                os.path.join(current_work_dir, c, "CMakeLists.txt")
+            ):
+                self._gen_cmakelists(
+                    os.path.join(current_work_dir, c), depth + 1
+                )
+                sub_dirs.append(c)
+
+        if not os.path.isfile(os.path.join(current_work_dir, "testslist.csv")):
+            return
+        cmds = """# This file is generated by ${PADDLE_ROOT}/tools/gen_ut_cmakelists.py.
+    # Please don't modify this file manually.
+    # If you need to change unittests in this file, please modify testslist.csv in the current directory
+    # and then run the command `python3 ${PADDLE_ROOT}/tools/gen_ut_cmakelists.py -f ${CURRENT_DIRECTORY}/testslist.csv`
+    set(LOCAL_ALL_ARCH ON)
+    set(LOCAL_ALL_PLAT ON)\n"""
+        with open(f"{current_work_dir}/testslist.csv") as csv_file:
+            for i, line in enumerate(csv_file.readlines()):
+                try:
+                    cmds += self._parse_line(line, current_work_dir)
+                except Exception as e:
+                    print("===============PARSE LINE ERRORS OCCUR==========")
+                    print(e)
+                    print(f"[ERROR FILE]: {current_work_dir}/testslist.csv")
+                    print(f"[ERROR LINE {i + 1}]: {line.strip()}")
+                    sys.exit(1)
+
+        for sub in sub_dirs:
+            cmds += f"add_subdirectory({sub})\n"
+
+        # check whether the generated file are the same with the existing file, ignoring the blank chars
+        # if they are same, skip the waiting process
+        if os.path.isfile(f"{current_work_dir}/CMakeLists.txt"):
+            with open(
+                f"{current_work_dir}/CMakeLists.txt", "r"
+            ) as old_cmake_file:
+                char_seq = old_cmake_file.read().split()
+        else:
+            char_seq = []
+        char_seq = "".join(char_seq)
+
+        if char_seq != "".join(cmds.split()):
+            assert (
+                f"{current_work_dir}/CMakeLists.txt"
+                not in self.modified_or_created_files
+            ), (
+                f"the file {current_work_dir}/CMakeLists.txt are modified twice, which may cause some error"
+            )
+            self.modified_or_created_files.append(
+                f"{current_work_dir}/CMakeLists.txt"
+            )
+            if not self._only_check:
+                with open(
+                    f"{current_work_dir}/CMakeLists.txt", "w"
+                ) as cmake_file:
+                    print(cmds, end="", file=cmake_file)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--files",
+        "-f",
+        type=str,
+        required=False,
+        default=[],
+        nargs="+",
+        help="Input a list of files named testslist.csv and output files named CMakeLists.txt in the same directories as the csv files respectively",
+    )
+    parser.add_argument(
+        "--dirpaths",
+        "-d",
+        type=str,
+        required=False,
+        default=[],
+        nargs="+",
+        help="Input a list of dir paths including files named testslist.csv and output CMakeLists.txt in these directories respectively",
+    )
+    parser.add_argument(
+        "--ignore-cmake-dirs",
+        '-i',
+        type=str,
+        required=False,
+        default=[],
+        nargs='*',
+        help="To keep dist ports the same with old version cmake, old CMakeLists.txt files are needed to parse dist_ports. If a directories are newly created and there is no CMakeLists.txt file, the directory path must be specified by this option. The dirs are not recursive.",
+    )
+    parser.add_argument(
+        "--only-check-changed",
+        '-o',
+        type=lambda x: x.lower() not in ["false", "0", "off"],
+        required=False,
+        default=False,
+        help="Only check whether the CMake files should be rewritten, do not write it even if it should be write",
+    )
+    args = parser.parse_args()
+
+    assert not (len(args.files) == 0 and len(args.dirpaths) == 0), (
+        "You must provide at least one file or dirpath"
+    )
+    current_work_dirs = []
+    if len(args.files) >= 1:
+        for p in args.files:
+            assert os.path.basename(p) == "testslist.csv", (
+                "you must input file named testslist.csv"
+            )
+        current_work_dirs = current_work_dirs + [
+            os.path.dirname(file) for file in args.files
+        ]
+    if len(args.dirpaths) >= 1:
+        current_work_dirs = current_work_dirs + list(args.dirpaths)
+
+    cmake_generator = CMakeGenerator(
+        current_work_dirs, args.only_check_changed, args.ignore_cmake_dirs
+    )
+    cmake_generator.prepare_dist_ut_port()
+    created = cmake_generator.parse_csvs()
+
+    # summary the modified files
+    for f in created:
+        print("modified/new:", f)

tools/get_build_time.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CMAKE_BINARY_DIR=$1
+shift
+start=$(date +%s.%N)
+duration=$("/usr/bin/time" -f "%C, %E elapsed, %U user, %S sys" "$@" 2>&1)
+end=$(date +%s.%N)
+
+echo ${duration}, 'start', $start, 'end', $end, 'process', $$ >> $CMAKE_BINARY_DIR/build-time

tools/get_cpu_info.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "`uname -s`" != "Linux" ]; then
+  echo "Current scenario only support in Linux yet!"
+  exit 0
+fi
+
+echo "********** Hardware Information **********"
+sockets=`grep 'physical id' /proc/cpuinfo | sort -u | wc -l`
+cores_per_socket=`grep 'core id' /proc/cpuinfo | sort -u | wc -l`
+ht=`lscpu |grep "per core" |awk -F':' '{print $2}'|xargs`
+physical_cores=$((sockets * cores_per_socket))
+virtual_cores=`grep 'processor' /proc/cpuinfo | sort -u | wc -l`
+numa_nodes=`lscpu |grep "NUMA node(s)"|awk -F':' '{print $2}'|xargs`
+echo "CPU Name               : `cat /proc/cpuinfo |grep -i "model name" |uniq |awk -F ':' '{print $2}'|xargs`"
+echo "CPU Family             : `lscpu |grep \"CPU family\" |awk -F':' '{print $2}'|xargs`"
+echo "Socket Number          : $sockets"
+echo "Cores Per Socket       : $cores_per_socket"
+echo "Total Physical Cores   : $physical_cores"
+echo "Total Virtual Cores    : $virtual_cores"
+if [ $ht -eq 1 ]; then
+  echo "Hyper Threading        : OFF"
+  if [ $physical_cores -ne $virtual_cores ]; then
+    echo "Error: HT logical error"
+  fi
+else
+  echo "Hyper Threading        : ON"
+  if [ $physical_cores -ge $virtual_cores ]; then
+    echo "Error: HT logical error"
+  fi
+fi
+echo "NUMA Nodes             : $numa_nodes"
+if [ $numa_nodes -lt $sockets ]; then
+  echo "Warning: NUMA node is not enough for the best performance,\
+ at least $sockets"
+fi
+
+echo "********** Software Information **********"
+echo "OS Version             : `uname -o`"
+echo "Kernel Release Version : `uname -r`"
+echo "Kernel Patch Version   : `uname -v`"
+echo "GCC Version            :`gcc --version | head -n 1|awk -F '\\\(GCC\\\)' '{print $2}'`"
+if command -v cmake >/dev/null 2>&1; then
+  cmake_ver=`cmake --version | head -n 1 | awk -F 'version' '{print $2}'`
+else
+  cmake_ver=" Not installed"
+fi
+echo "CMake Version          :$cmake_ver"
+echo "******************************************"

tools/get_op_list.py

@@ -0,0 +1,105 @@
+# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+import re
+
+import numpy as np
+
+import paddle
+from paddle.inference import _get_phi_kernel_name
+
+paddle.enable_static()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--model_dir',
+        type=str,
+        default="",
+        help='Directory of the inference models that named with pdmodel.',
+    )
+    parser.add_argument(
+        '--op_list',
+        type=str,
+        default="",
+        help='List of ops like "conv2d;pool2d;relu".',
+    )
+    return parser.parse_args()
+
+
+def get_model_ops(model_file, ops_set):
+    model_bytes = paddle.static.load_from_file(model_file)
+    pg = paddle.static.deserialize_program(model_bytes)
+
+    for i in range(0, pg.desc.num_blocks()):
+        block = pg.desc.block(i)
+        size = block.op_size()
+
+        for j in range(0, size):
+            ops_set.add(block.op(j).type())
+
+
+def get_model_phi_kernels(ops_set):
+    phi_set = set()
+    phi_raw_list = [
+        "add",
+        "subtract",
+        "multiply",
+        "multiply_sr",
+        "divide",
+        "maximum",
+        "minimum",
+        "remainder",
+        "floor_divide",
+        "elementwise_pow",
+    ]
+    phi_odd_dist = {"batch_norm": "batch_norm_infer"}
+    for op in ops_set:
+        print(op)
+        phi_kernel = _get_phi_kernel_name(op)
+        print(phi_kernel)
+        phi_set.add(phi_kernel)
+        if phi_kernel in phi_raw_list:
+            phi_set.add(phi_kernel + "_raw")
+        if phi_kernel in phi_odd_dist.keys():
+            phi_set.add(phi_odd_dist[phi_kernel])
+
+    return phi_set
+
+
+if __name__ == '__main__':
+    args = parse_args()
+    ops_set = set()
+    if args.op_list != "":
+        op_list = args.op_list.split(";")
+        for op in op_list:
+            ops_set.add(op)
+
+    if args.model_dir != "":
+        for root, dirs, files in os.walk(args.model_dir, topdown=True):
+            for name in files:
+                if re.match(r'.*pdmodel', name):
+                    get_model_ops(os.path.join(root, name), ops_set)
+    phi_set = get_model_phi_kernels(ops_set)
+    ops = ";".join(ops_set)
+    kernels = ";".join(phi_set)
+    print("op_list: ", ops)
+    print("kernel_list: ", kernels)
+    ops = np.array([ops])
+    kernels = np.array([kernels])
+    np.savetxt("op_list.txt", ops, fmt='%s')
+    np.savetxt("kernel_list.txt", kernels, fmt='%s')