chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,25 @@
|
||||
#### **Introduction for crawling new error message:**
|
||||
|
||||
|
||||
|
||||
1. add new spider code in spider.py for crawling error message from website.
|
||||
|
||||
2. run `bash start.sh` in current directory to generate new externalErrorMsg_${date}.tar.gz file, for example `externalErrorMsg_20210928.tar.gz`.
|
||||
|
||||
3. upload above tar file into bos https://paddlepaddledeps.bj.bcebos.com **paddlepaddledeps** bucket, and copy download link `${download_url}`. ***\*Be careful not to delete original tar file\****.
|
||||
|
||||
4. compute md5 value of above tar file `${md5}`, and modify cmake/third_party.cmake file
|
||||
|
||||
```
|
||||
set(URL "${download_url}" CACHE STRING "" FORCE)
|
||||
file_download_and_uncompress(${URL} "externalError" MD5 ${md5})
|
||||
```
|
||||
|
||||
for example:
|
||||
|
||||
```
|
||||
set(URL "https://paddlepaddledeps.bj.bcebos.com/externalErrorMsg_20210928.tar.gz" CACHE STRING "" FORCE)
|
||||
file_download_and_uncompress(${URL} "externalError" MD5 a712a49384e77ca216ad866712f7cafa)
|
||||
```
|
||||
|
||||
5. commit your changes, and create pull request.
|
||||
@@ -0,0 +1,413 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import getopt
|
||||
import re
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
|
||||
import external_error_pb2
|
||||
|
||||
|
||||
def parsing(externalErrorDesc):
|
||||
# *********************************************************************************************#
|
||||
# *********************************** CUDA Error Message **************************************#
|
||||
print("start crawling errorMessage for nvidia CUDA API--->")
|
||||
url = 'https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html#group__CUDART__TYPES_1g3f51e3575c2178246db0a94a430e0038'
|
||||
|
||||
allMessageDesc = externalErrorDesc.errors.add()
|
||||
allMessageDesc.type = external_error_pb2.CUDA
|
||||
|
||||
ssl._create_default_https_context = ssl._create_unverified_context
|
||||
html = urllib.request.urlopen(url).read().decode('utf-8')
|
||||
res_div = r'<div class="section">.*?<p>CUDA error types </p>.*?</div>.*?<div class="enum-members">(.*?)</div>'
|
||||
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
|
||||
|
||||
res_dt = r'<dt>(.*?)</dt>.*?<dd>(.*?)</dd>'
|
||||
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
|
||||
for error in m_dt:
|
||||
res_type = r'<span class="enum-member-name-def">(.*?) = <span class="ph ph apiData">(.*?)</span></span>'
|
||||
m_type = re.findall(res_type, error[0], re.DOTALL | re.MULTILINE)[0]
|
||||
m_message = error[1]
|
||||
m_message = m_message.replace('\n', '')
|
||||
res_a = r'(<a class=.*?</a>)'
|
||||
res_shape = r'<a class=.*?>(.*?)</a>'
|
||||
list_a = re.findall(res_a, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_shape = re.findall(res_shape, m_message, re.DOTALL | re.MULTILINE)
|
||||
assert len(list_a) == len(list_shape)
|
||||
for idx in range(len(list_a)):
|
||||
m_message = m_message.replace(list_a[idx], list_shape[idx])
|
||||
|
||||
m_message = m_message.replace(
|
||||
'<h6 class="deprecated_header">Deprecated</h6>', ''
|
||||
)
|
||||
|
||||
res_span = r'(<span class=.*?</span>)'
|
||||
res_span_detail = r'<span class=.*?>(.*?)</span>'
|
||||
list_span = re.findall(res_span, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_span_detail = re.findall(
|
||||
res_span_detail, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
assert len(list_span) == len(list_span_detail)
|
||||
for idx in range(len(list_span)):
|
||||
m_message = m_message.replace(list_span[idx], list_span_detail[idx])
|
||||
|
||||
res_p = r'(<p>.*?</p>)'
|
||||
res_p_detail = r'<p>(.*?)</p>'
|
||||
list_p = re.findall(res_p, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_p_detail = re.findall(
|
||||
res_p_detail, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
assert len(list_p) == len(list_p_detail)
|
||||
for idx in range(len(list_p)):
|
||||
m_message = m_message.replace(list_p[idx], list_p_detail[idx])
|
||||
|
||||
m_message = m_message.replace(' ', '')
|
||||
_Messages = allMessageDesc.messages.add()
|
||||
try:
|
||||
_Messages.code = int(m_type[1])
|
||||
except ValueError:
|
||||
if re.match('0x', m_type[1]):
|
||||
_Messages.code = int(m_type[1], 16)
|
||||
else:
|
||||
raise ValueError
|
||||
_Messages.message = f"'{m_type[0]}'. {m_message}"
|
||||
print("End crawling errorMessage for nvidia CUDA API!\n")
|
||||
|
||||
# ***********************************************************************************************#
|
||||
# *********************************** CURAND Error Message **************************************#
|
||||
print("start crawling errorMessage for nvidia CURAND API--->")
|
||||
url = 'https://docs.nvidia.com/cuda/curand/group__HOST.html#group__HOST_1gb94a31d5c165858c96b6c18b70644437'
|
||||
|
||||
allMessageDesc = externalErrorDesc.errors.add()
|
||||
allMessageDesc.type = external_error_pb2.CURAND
|
||||
|
||||
html = urllib.request.urlopen(url).read().decode('utf-8')
|
||||
|
||||
res_div = r'<div class="section">.*?<p>CURAND function call status types </p>.*?</div>.*?<div class="enum-members">(.*?)</div>'
|
||||
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
|
||||
|
||||
res_dt = r'<dt>(.*?)</dt>.*?<dd>(.*?)</dd>'
|
||||
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
|
||||
for error in m_dt:
|
||||
res_type = r'<span class="enum-member-name-def">(.*?) = <span class="ph ph apiData">(.*?)</span></span>'
|
||||
m_type = re.findall(res_type, error[0], re.DOTALL | re.MULTILINE)[0]
|
||||
m_message = error[1]
|
||||
|
||||
_Messages = allMessageDesc.messages.add()
|
||||
try:
|
||||
_Messages.code = int(m_type[1])
|
||||
except ValueError:
|
||||
if re.match('0x', m_type[1]):
|
||||
_Messages.code = int(m_type[1], 16)
|
||||
else:
|
||||
raise ValueError
|
||||
_Messages.message = f"'{m_type[0]}'. {m_message}"
|
||||
print("End crawling errorMessage for nvidia CURAND API!\n")
|
||||
|
||||
# **************************************************************************************************#
|
||||
# *********************************** CUDNN Error Message ******************************************#
|
||||
cudnnStatus_t = {
|
||||
"CUDNN_STATUS_SUCCESS": 0,
|
||||
"CUDNN_STATUS_NOT_INITIALIZED": 1,
|
||||
"CUDNN_STATUS_ALLOC_FAILED": 2,
|
||||
"CUDNN_STATUS_BAD_PARAM": 3,
|
||||
"CUDNN_STATUS_INTERNAL_ERROR": 4,
|
||||
"CUDNN_STATUS_INVALID_VALUE": 5,
|
||||
"CUDNN_STATUS_ARCH_MISMATCH": 6,
|
||||
"CUDNN_STATUS_MAPPING_ERROR": 7,
|
||||
"CUDNN_STATUS_EXECUTION_FAILED": 8,
|
||||
"CUDNN_STATUS_NOT_SUPPORTED": 9,
|
||||
"CUDNN_STATUS_LICENSE_ERROR": 10,
|
||||
"CUDNN_STATUS_RUNTIME_PREREQUISITE_MISSING": 11,
|
||||
"CUDNN_STATUS_RUNTIME_IN_PROGRESS": 12,
|
||||
"CUDNN_STATUS_RUNTIME_FP_OVERFLOW": 13,
|
||||
}
|
||||
|
||||
print("start crawling errorMessage for nvidia CUDNN API--->")
|
||||
url = 'https://docs.nvidia.com/deeplearning/cudnn/api/index.html#cudnnStatus_t'
|
||||
|
||||
allMessageDesc = externalErrorDesc.errors.add()
|
||||
allMessageDesc.type = external_error_pb2.CUDNN
|
||||
|
||||
html = urllib.request.urlopen(url).read().decode('utf-8')
|
||||
f = open('1.txt', 'w')
|
||||
f.write(html)
|
||||
|
||||
res_div = r'<div class="section" id="cudnnStatus_t__section_lmp_dgr_2jb"><a name="cudnnStatus_t__section_lmp_dgr_2jb" shape="rect">(.*?)</div>'
|
||||
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
|
||||
|
||||
res_dt = r'<dt class="dt dlterm"><samp class="ph codeph">(.*?)</samp></dt>.*?<dd class="dd">(.*?)</dd>'
|
||||
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
|
||||
for error in m_dt:
|
||||
m_message = error[1]
|
||||
|
||||
res_class = r'<p class="p">.*?</p>'
|
||||
res_class_detail = r'<p class="p">(.*?)</p>'
|
||||
list_class = re.findall(res_class, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_class_detail = re.findall(
|
||||
res_class_detail, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
assert len(list_class) == len(list_class_detail)
|
||||
for idx in range(len(list_class)):
|
||||
m_message = m_message.replace(
|
||||
list_class[idx], list_class_detail[idx]
|
||||
)
|
||||
|
||||
res_a = r'(<a class="xref".*?</a>)'
|
||||
res_shape = r'<a class="xref".*?>(.*?)</a>'
|
||||
list_a = re.findall(res_a, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_shape = re.findall(res_shape, m_message, re.DOTALL | re.MULTILINE)
|
||||
assert len(list_a) == len(list_shape)
|
||||
for idx in range(len(list_a)):
|
||||
m_message = m_message.replace(list_a[idx], list_shape[idx])
|
||||
|
||||
res_span = r'(<span class="ph">.*?</span>)'
|
||||
res_span_detail = r'<span class="ph">(.*?)</span>'
|
||||
list_span = re.findall(res_span, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_span_detail = re.findall(
|
||||
res_span_detail, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
assert len(list_span) == len(list_span_detail)
|
||||
for idx in range(len(list_span)):
|
||||
m_message = m_message.replace(list_span[idx], list_span_detail[idx])
|
||||
|
||||
res_samp = r'(<samp class="ph codeph">.*?</samp>)'
|
||||
res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
|
||||
list_samp = re.findall(res_samp, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_samp_detail = re.findall(
|
||||
res_samp_detail, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
assert len(list_samp) == len(list_samp_detail)
|
||||
for idx in range(len(list_samp)):
|
||||
m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
|
||||
|
||||
m_message = re.sub(r'\n +', ' ', m_message)
|
||||
|
||||
_Messages = allMessageDesc.messages.add()
|
||||
_Messages.code = int(cudnnStatus_t[error[0]])
|
||||
_Messages.message = f"'{error[0]}'. {m_message}"
|
||||
print("End crawling errorMessage for nvidia CUDNN API!\n")
|
||||
|
||||
# *************************************************************************************************#
|
||||
# *********************************** CUBLAS Error Message ****************************************#
|
||||
cublasStatus_t = {
|
||||
"CUBLAS_STATUS_SUCCESS": 0,
|
||||
"CUBLAS_STATUS_NOT_INITIALIZED": 1,
|
||||
"CUBLAS_STATUS_ALLOC_FAILED": 3,
|
||||
"CUBLAS_STATUS_INVALID_VALUE": 7,
|
||||
"CUBLAS_STATUS_ARCH_MISMATCH": 8,
|
||||
"CUBLAS_STATUS_MAPPING_ERROR": 11,
|
||||
"CUBLAS_STATUS_EXECUTION_FAILED": 13,
|
||||
"CUBLAS_STATUS_INTERNAL_ERROR": 14,
|
||||
"CUBLAS_STATUS_NOT_SUPPORTED": 15,
|
||||
"CUBLAS_STATUS_LICENSE_ERROR": 16,
|
||||
}
|
||||
|
||||
print("start crawling errorMessage for nvidia CUBLAS API--->")
|
||||
url = 'https://docs.nvidia.com/cuda/cublas/index.html#cublasstatus_t'
|
||||
|
||||
allMessageDesc = externalErrorDesc.errors.add()
|
||||
allMessageDesc.type = external_error_pb2.CUBLAS
|
||||
|
||||
html = urllib.request.urlopen(url).read().decode('utf-8')
|
||||
|
||||
res_div = r'<p class="p">The type is used for function status returns. All cuBLAS library.*?<div class="tablenoborder">(.*?)</div>'
|
||||
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
|
||||
|
||||
res_dt = r'<p class="p"><samp class="ph codeph">(.*?)</samp></p>.*?colspan="1">(.*?)</td>'
|
||||
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
|
||||
|
||||
for error in m_dt:
|
||||
m_message = error[1]
|
||||
m_message = re.sub(r'\n +', ' ', m_message)
|
||||
|
||||
res_p = r'<p class="p">.*?</p>'
|
||||
res_p_detail = r'<p class="p">(.*?)</p>'
|
||||
list_p = re.findall(res_p, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_p_detail = re.findall(
|
||||
res_p_detail, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
assert len(list_p) == len(list_p_detail)
|
||||
for idx in range(len(list_p)):
|
||||
m_message = m_message.replace(list_p[idx], list_p_detail[idx])
|
||||
|
||||
res_samp = r'<samp class="ph codeph">.*?</samp>'
|
||||
res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
|
||||
list_samp = re.findall(res_samp, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_samp_detail = re.findall(
|
||||
res_samp_detail, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
assert len(list_samp) == len(list_samp_detail)
|
||||
for idx in range(len(list_samp)):
|
||||
m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
|
||||
|
||||
_Messages = allMessageDesc.messages.add()
|
||||
_Messages.code = int(cublasStatus_t[error[0]])
|
||||
_Messages.message = f"'{error[0]}'. {m_message}"
|
||||
print("End crawling errorMessage for nvidia CUBLAS API!\n")
|
||||
|
||||
# *************************************************************************************************#
|
||||
# *********************************** CUSOLVER Error Message **************************************#
|
||||
cusolverStatus_t = {
|
||||
"CUSOLVER_STATUS_SUCCESS": 0,
|
||||
"CUSOLVER_STATUS_NOT_INITIALIZED": 1,
|
||||
"CUSOLVER_STATUS_ALLOC_FAILED": 2,
|
||||
"CUSOLVER_STATUS_INVALID_VALUE": 3,
|
||||
"CUSOLVER_STATUS_ARCH_MISMATCH": 4,
|
||||
"CUSOLVER_STATUS_MAPPING_ERROR": 5,
|
||||
"CUSOLVER_STATUS_EXECUTION_FAILED": 6,
|
||||
"CUSOLVER_STATUS_INTERNAL_ERROR": 7,
|
||||
"CUSOLVER_STATUS_MATRIX_TYPE_NOT_SUPPORTED": 8,
|
||||
"CUSOLVER_STATUS_NOT_SUPPORTED": 9,
|
||||
"CUSOLVER_STATUS_ZERO_PIVOT": 10,
|
||||
"CUSOLVER_STATUS_INVALID_LICENSE": 11,
|
||||
"CUSOLVER_STATUS_IRS_PARAMS_NOT_INITIALIZED": 12,
|
||||
"CUSOLVER_STATUS_IRS_PARAMS_INVALID": 13,
|
||||
"CUSOLVER_STATUS_IRS_INTERNAL_ERROR": 14,
|
||||
"CUSOLVER_STATUS_IRS_NOT_SUPPORTED": 15,
|
||||
"CUSOLVER_STATUS_IRS_OUT_OF_RANGE": 16,
|
||||
"CUSOLVER_STATUS_IRS_NRHS_NOT_SUPPORTED_FOR_REFINE_GMRES": 17,
|
||||
"CUSOLVER_STATUS_IRS_INFOS_NOT_INITIALIZED": 18,
|
||||
}
|
||||
print("start crawling errorMessage for nvidia CUSOLVER API--->")
|
||||
url = 'https://docs.nvidia.com/cuda/cusolver/index.html#cuSolverSPstatus'
|
||||
|
||||
allMessageDesc = externalErrorDesc.errors.add()
|
||||
allMessageDesc.type = external_error_pb2.CUSOLVER
|
||||
|
||||
html = urllib.request.urlopen(url).read().decode('utf-8')
|
||||
|
||||
res_div = r'This is a status type returned by the library functions and.*?<div class="tablenoborder">(.*?)</div>'
|
||||
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
|
||||
|
||||
res_dt = (
|
||||
r'<samp class="ph codeph">(.*?)</samp></td>.*?colspan="1">(.*?)</td>'
|
||||
)
|
||||
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
|
||||
|
||||
for error in m_dt:
|
||||
m_message = error[1]
|
||||
m_message = re.sub(r'\n +', '', m_message)
|
||||
m_message = re.sub(r'<p class="p"></p>', '', m_message)
|
||||
|
||||
res_p = r'<p class="p">.*?</p>'
|
||||
res_p_detail = r'<p class="p">(.*?)</p>'
|
||||
list_p = re.findall(res_p, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_p_detail = re.findall(
|
||||
res_p_detail, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
assert len(list_p) == len(list_p_detail)
|
||||
for idx in range(len(list_p)):
|
||||
m_message = m_message.replace(list_p[idx], list_p_detail[idx])
|
||||
|
||||
res_samp = r'<samp class="ph codeph">.*?</samp>'
|
||||
res_samp_detail = r'<samp class="ph codeph">(.*?)</samp>'
|
||||
list_samp = re.findall(res_samp, m_message, re.DOTALL | re.MULTILINE)
|
||||
list_samp_detail = re.findall(
|
||||
res_samp_detail, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
assert len(list_samp) == len(list_samp_detail)
|
||||
for idx in range(len(list_samp)):
|
||||
m_message = m_message.replace(list_samp[idx], list_samp_detail[idx])
|
||||
|
||||
res_strong = r'<strong class="ph b">.*?</strong>'
|
||||
res_strong_detail = r'<strong class="ph b">(.*?)</strong>'
|
||||
list_strong = re.findall(
|
||||
res_strong, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
list_strong_detail = re.findall(
|
||||
res_strong_detail, m_message, re.DOTALL | re.MULTILINE
|
||||
)
|
||||
assert len(list_strong) == len(list_strong_detail)
|
||||
for idx in range(len(list_strong)):
|
||||
m_message = m_message.replace(
|
||||
list_strong[idx], list_strong_detail[idx]
|
||||
)
|
||||
|
||||
_Messages = allMessageDesc.messages.add()
|
||||
_Messages.code = int(cusolverStatus_t[error[0]])
|
||||
_Messages.message = f"'{error[0]}'. {m_message}"
|
||||
print("End crawling errorMessage for nvidia CUSOLVER API!\n")
|
||||
|
||||
# **********************************************************************************************#
|
||||
# *************************************** NCCL error *******************************************#
|
||||
print("start crawling errorMessage for nvidia NCCL API--->")
|
||||
url = 'https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/api/types.html#ncclresult-t'
|
||||
allMessageDesc = externalErrorDesc.errors.add()
|
||||
allMessageDesc.type = external_error_pb2.NCCL
|
||||
html = urllib.request.urlopen(url).read().decode('utf-8')
|
||||
res_div = r'<code class="descname">ncclResult_t</code>(.*?)</div>'
|
||||
m_div = re.findall(res_div, html, re.DOTALL | re.MULTILINE)[0]
|
||||
|
||||
res_dt = r'<code class="descname">(.*?)</code>.*?<span class="pre">(.*?)</span></code>\)(.*?)</p>\n</dd></dl>'
|
||||
m_dt = re.findall(res_dt, m_div, re.DOTALL | re.MULTILINE)
|
||||
for error in m_dt:
|
||||
m_message = re.sub(r'\n', '', error[2])
|
||||
_Messages = allMessageDesc.messages.add()
|
||||
_Messages.code = int(error[1])
|
||||
_Messages.message = f"'{error[0]}'. {m_message}"
|
||||
print("End crawling errorMessage for nvidia NCCL API!\n")
|
||||
|
||||
# *************************************************************************************************#
|
||||
# *********************************** CUFFT Error Message **************************************#
|
||||
print("start crawling errorMessage for nvidia CUFFT API--->")
|
||||
url = 'https://docs.nvidia.com/cuda/cufft/index.html#cufftresult'
|
||||
|
||||
allMessageDesc = externalErrorDesc.errors.add()
|
||||
allMessageDesc.type = external_error_pb2.CUFFT
|
||||
|
||||
html = urllib.request.urlopen(url).read().decode('utf-8')
|
||||
|
||||
class CUFFTHTMLParser(HTMLParser):
|
||||
'''CUFFTHTML Parser'''
|
||||
|
||||
def handle_data(self, data):
|
||||
if 'typedef enum cufftResult_t' in data:
|
||||
for line in data.strip().splitlines()[1:-1]:
|
||||
status, code, desc = re.split('=|//', line.strip())
|
||||
_Messages = allMessageDesc.messages.add()
|
||||
_Messages.code = int(code.strip(' ,'))
|
||||
_Messages.message = f"'{status.strip()}'. {desc.strip()}"
|
||||
|
||||
CUFFTHTMLParser().feed(html)
|
||||
|
||||
|
||||
def main(argv):
|
||||
try:
|
||||
opts, _ = getopt.getopt(argv, "h", ["help"])
|
||||
except getopt.GetoptError:
|
||||
print('python spider.py')
|
||||
sys.exit(2)
|
||||
for opt, _ in opts:
|
||||
if opt in ("-h", "--help"):
|
||||
print('python spider.py')
|
||||
sys.exit(2)
|
||||
externalErrorDesc = external_error_pb2.ExternalErrorDesc()
|
||||
parsing(externalErrorDesc)
|
||||
|
||||
serializedString = externalErrorDesc.SerializeToString()
|
||||
with open("externalErrorMsg.pb", "wb") as f:
|
||||
# save for externalErrorMsg.pb from Python-protobuf interface
|
||||
# load from C++-protobuf interface and get error message
|
||||
f.write(serializedString)
|
||||
print(
|
||||
"Generating data file [externalErrorMsg.pb] for external third_party API error has been done!"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -ex
|
||||
SYSTEM=`uname -s`
|
||||
rm -f protoc-3.11.3-linux-x86_64.*
|
||||
if [ "$SYSTEM" == "Linux" ]; then
|
||||
wget --no-check-certificate https://github.com/protocolbuffers/protobuf/releases/download/v3.11.3/protoc-3.11.3-linux-x86_64.zip
|
||||
unzip -d protobuf -o protoc-3.11.3-linux-x86_64.zip
|
||||
rm protoc-3.11.3-linux-x86_64.*
|
||||
elif [ "$SYSTEM" == "Darwin" ]; then
|
||||
wget --no-check-certificate https://github.com/protocolbuffers/protobuf/releases/download/v3.11.3/protoc-3.11.3-osx-x86_64.zip
|
||||
unzip -d protobuf -o protoc-3.11.3-osx-x86_64.zip
|
||||
rm protoc-3.11.3-osx-x86_64.*
|
||||
else
|
||||
echo "please run on Mac/Linux"
|
||||
exit 1
|
||||
fi
|
||||
protobuf/bin/protoc -I../../paddle/phi/core/ --python_out . ../../paddle/phi/core/external_error.proto
|
||||
|
||||
python3.10 spider.py
|
||||
tar czvf externalErrorMsg_$(date +'%Y%m%d').tar.gz externalErrorMsg.pb
|
||||
Reference in New Issue
Block a user