wehub-resource-sync
701 lines
25 KiB
Python
701 lines
25 KiB
Python
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import os
|
|
|
|
import numpy as np
|
|
|
|
|
|
# Judge whether the value is within the range indicated by fp16
|
|
def is_infinite(value, dtype=np.float16):
|
|
# return value > np.finfo(np.float16).max or value < np.finfo(np.float16).min
|
|
array = np.array([value]).astype(dtype)
|
|
return np.isinf(array) or np.isnan(array)
|
|
|
|
|
|
# Judge whether the value of fp32 is equal to that of fp16
|
|
def is_allclose(actual, expected, atol=1e-2, rtol=1e-2):
|
|
return np.allclose(
|
|
np.array([actual]), np.array([expected]), atol=atol, rtol=rtol
|
|
)
|
|
|
|
|
|
class TensorInfo:
|
|
def __init__(self):
|
|
self.device = None
|
|
self.op_type = None
|
|
self.tensor_name = None
|
|
self.dtype = None
|
|
self.numel = None
|
|
self.max_value = None
|
|
self.min_value = None
|
|
self.mean_value = None
|
|
self.has_inf = None
|
|
self.has_nan = None
|
|
self.num_zero = None
|
|
|
|
def __str__(self):
|
|
return f"[TensorInfo] device={self.device}, op_type={self.op_type}, tensor_name={self.tensor_name}, dtype={self.dtype}, numel={self.numel}, num_inf={self.has_inf}, num_nan={self.has_nan}, num_zero={self.num_zero}, max_value={self.max_value:.6f}, min_value={self.min_value:.6f}, mean_value={self.mean_value:.6f}"
|
|
|
|
def key(
|
|
self,
|
|
):
|
|
return self.op_type + "/" + self.tensor_name
|
|
|
|
def init_from_string(self, line):
|
|
try:
|
|
line_frags = line.strip().split(" ")
|
|
for frag in line_frags:
|
|
word_str = (
|
|
frag.replace("[", "").replace("]", "").replace(",", "")
|
|
)
|
|
words = word_str.split("=")
|
|
if words[0] == "op":
|
|
self.op_type = words[1]
|
|
elif words[0] == "device":
|
|
self.device = words[1]
|
|
elif words[0] == "tensor":
|
|
self.tensor_name = words[1]
|
|
elif words[0] == "dtype":
|
|
self.dtype = words[1]
|
|
elif words[0] == "numel":
|
|
self.numel = np.int64(words[1])
|
|
elif words[0] == "max":
|
|
self.max_value = np.float32(words[1])
|
|
elif words[0] == "min":
|
|
self.min_value = np.float32(words[1])
|
|
elif words[0] == "mean":
|
|
self.mean_value = np.float32(words[1])
|
|
elif words[0] == "num_inf":
|
|
self.has_inf = int(words[1])
|
|
elif words[0] == "num_nan":
|
|
self.has_nan = int(words[1])
|
|
elif words[0] == "num_zero":
|
|
self.num_zero = np.int64(words[1])
|
|
except Exception as e:
|
|
print(f"!! Error parsing {line}")
|
|
return self
|
|
|
|
|
|
class MixedPrecisionTensorInfo:
|
|
def __init__(
|
|
self, fp32_tensor_info, fp16_tensor_info, fp32_idx=0, grad_scale=1.0
|
|
):
|
|
self.is_normal = True
|
|
self.fp32_idx = fp32_idx
|
|
|
|
self.fp32_tensor_name = None
|
|
self.fp32_dtype = None
|
|
self.fp32_max_value = None
|
|
self.fp32_min_value = None
|
|
self.fp32_mean_value = None
|
|
self.fp32_num_zero = None
|
|
self.scaled_fp32_max_value = None
|
|
self.scaled_fp32_min_value = None
|
|
|
|
self.fp16_tensor_name = None
|
|
self.fp16_dtype = None
|
|
self.fp16_max_value = None
|
|
self.fp16_min_value = None
|
|
self.fp16_mean_value = None
|
|
self.fp16_num_zero = None
|
|
self.fp16_has_inf = None
|
|
self.fp16_has_nan = None
|
|
|
|
self.fp32_div_fp16_max_value = None
|
|
self.fp32_div_fp16_min_value = None
|
|
self.fp32_div_fp16_mean_value = None
|
|
|
|
if fp32_tensor_info is not None:
|
|
self.op_type = fp32_tensor_info.op_type
|
|
self.numel = fp32_tensor_info.numel
|
|
self.fp32_num_zero = fp32_tensor_info.num_zero
|
|
self.fp32_tensor_name = fp32_tensor_info.tensor_name
|
|
self.fp32_dtype = fp32_tensor_info.dtype
|
|
self.fp32_max_value = fp32_tensor_info.max_value
|
|
self.fp32_min_value = fp32_tensor_info.min_value
|
|
self.fp32_mean_value = fp32_tensor_info.mean_value
|
|
if "GRAD" in self.fp32_tensor_name:
|
|
self.scaled_fp32_max_value = (
|
|
grad_scale * fp32_tensor_info.max_value
|
|
)
|
|
self.scaled_fp32_min_value = (
|
|
grad_scale * fp32_tensor_info.min_value
|
|
)
|
|
|
|
if fp16_tensor_info is not None:
|
|
self.op_type = fp16_tensor_info.op_type
|
|
self.numel = fp16_tensor_info.numel
|
|
self.fp16_num_zero = fp16_tensor_info.num_zero
|
|
self.fp16_tensor_name = fp16_tensor_info.tensor_name
|
|
self.fp16_dtype = fp16_tensor_info.dtype
|
|
self.fp16_max_value = fp16_tensor_info.max_value
|
|
self.fp16_min_value = fp16_tensor_info.min_value
|
|
self.fp16_mean_value = fp16_tensor_info.mean_value
|
|
self.fp16_has_inf = fp16_tensor_info.has_inf
|
|
self.fp16_has_nan = fp16_tensor_info.has_nan
|
|
|
|
if fp32_tensor_info is not None and fp16_tensor_info is not None:
|
|
# Check whether the op name and data are equal
|
|
assert fp32_tensor_info.op_type == fp16_tensor_info.op_type
|
|
assert fp32_tensor_info.numel == fp16_tensor_info.numel, (
|
|
f"Error:\n\tFP32 Tensor Info:{fp32_tensor_info}\n\tFP16 Tensor Info:{fp16_tensor_info}"
|
|
)
|
|
# Fp16 divided by fp32
|
|
self.fp32_div_fp16_max_value = self._div(
|
|
self.fp16_max_value, self.fp32_max_value
|
|
)
|
|
self.fp32_div_fp16_min_value = self._div(
|
|
self.fp16_min_value, self.fp32_min_value
|
|
)
|
|
self.fp32_div_fp16_mean_value = self._div(
|
|
self.fp16_mean_value, self.fp32_mean_value
|
|
)
|
|
|
|
self._check_normal()
|
|
|
|
def __str__(self):
|
|
def _float_str(value):
|
|
return f"{value:.6f}" if value is not None else value
|
|
|
|
debug_str = f"[MixedPrecisionTensorInfo] op_type={self.op_type}, numel={self.numel}"
|
|
debug_str += f"\n FP32: tensor_name={self.fp32_tensor_name}, dtype={self.fp32_dtype}, max_value={_float_str(self.fp32_max_value)}, min_value={_float_str(self.fp32_min_value)}, mean_value={_float_str(self.fp32_mean_value)}"
|
|
debug_str += f"\n FP16: tensor_name={self.fp16_tensor_name}, dtype={self.fp16_dtype}, max_value={_float_str(self.fp16_max_value)}, min_value={_float_str(self.fp16_min_value)}, mean_value={_float_str(self.fp16_mean_value)}, has_inf={self.fp16_has_inf}, has_nan={self.fp16_has_nan}"
|
|
return debug_str
|
|
|
|
def _div(self, a, b):
|
|
if a is not None and b is not None:
|
|
return a / b if b != 0 else 1
|
|
return None
|
|
|
|
def get_tensor_name(self):
|
|
if self.fp32_tensor_name is None:
|
|
return self.fp16_tensor_name # + "#" + str(self.idx)
|
|
elif self.fp16_tensor_name is None:
|
|
return self.fp32_tensor_name + "#" + str(self.fp32_idx)
|
|
else:
|
|
return (
|
|
self.fp16_tensor_name.replace(".cast_fp16", "/.cast_fp16/")
|
|
+ "#"
|
|
+ str(self.fp32_idx)
|
|
)
|
|
|
|
def _check_normal(self):
|
|
# When the OP meets the following conditions, it is abnormal data, and use --skip_normal_tensors to retain the data in Excel:
|
|
# 1. The number of OP outputs exceeds the indication range of int32
|
|
# 2. The output data exceeds the representation range of fp16
|
|
# 3. Nan or inf appears in fp16 output data
|
|
# 4. The maximum value of fp32 is not equal to the maximum value of fp16
|
|
# 5. The minimum value of fp32 is not equal to the minimum value of fp16
|
|
if self.numel is not None and self.numel > np.iinfo(np.int32).max:
|
|
self.is_normal = False
|
|
return
|
|
|
|
check_list = [
|
|
self.fp32_max_value,
|
|
self.fp32_min_value,
|
|
self.scaled_fp32_max_value,
|
|
self.scaled_fp32_min_value,
|
|
self.fp16_max_value,
|
|
self.fp16_min_value,
|
|
]
|
|
|
|
for value in check_list:
|
|
if value is not None and is_infinite(value):
|
|
self.is_normal = False
|
|
return
|
|
|
|
if self.fp16_has_inf is not None and self.fp16_has_inf:
|
|
self.is_normal = False
|
|
return
|
|
if self.fp16_has_nan is not None and self.fp16_has_nan:
|
|
self.is_normal = False
|
|
return
|
|
|
|
if (
|
|
self.scaled_fp32_max_value is not None
|
|
and self.fp16_max_value is not None
|
|
and not is_allclose(self.fp16_max_value, self.scaled_fp32_max_value)
|
|
):
|
|
self.is_normal = False
|
|
return
|
|
if (
|
|
self.scaled_fp32_min_value is not None
|
|
and self.fp16_min_value is not None
|
|
and not is_allclose(self.fp16_min_value, self.scaled_fp32_min_value)
|
|
):
|
|
self.is_normal = False
|
|
return
|
|
|
|
|
|
class ExcelWriter:
|
|
def __init__(self, log_fp32_dir, log_fp16_dir, output_path):
|
|
self.log_fp32_dir = log_fp32_dir
|
|
self.log_fp16_dir = log_fp16_dir
|
|
|
|
try:
|
|
import xlsxwriter as xlw
|
|
except ImportError:
|
|
print(
|
|
"import xlsxwriter failed. please run 'pip install xlsxwriter==3.0.9' to install it"
|
|
)
|
|
|
|
self.workbook = xlw.Workbook(output_path)
|
|
self.title_format = self.workbook.add_format(
|
|
{
|
|
'bold': True,
|
|
'border': 1,
|
|
'font_color': 'black',
|
|
'bg_color': '#6495ED',
|
|
'align': 'center',
|
|
}
|
|
)
|
|
self.tensor_name_format = self.workbook.add_format(
|
|
{'bold': True, 'bg_color': '#F5F5F5'}
|
|
)
|
|
self.red_bg_cell_format = self.workbook.add_format(
|
|
{'bold': True, 'bg_color': 'red'}
|
|
)
|
|
self.yellow_bg_cell_format = self.workbook.add_format(
|
|
{'bold': True, 'bg_color': 'yellow'}
|
|
)
|
|
self.orange_bg_cell_format = self.workbook.add_format(
|
|
{'bold': True, 'bg_color': 'orange'}
|
|
)
|
|
|
|
def close(self):
|
|
self.workbook.close()
|
|
self.workbook = None
|
|
|
|
def _write_dtype(self, worksheet, value, row, col):
|
|
if value is None:
|
|
worksheet.write(row, col, "--")
|
|
else:
|
|
if value == "fp16":
|
|
worksheet.write(row, col, value, self.yellow_bg_cell_format)
|
|
else:
|
|
worksheet.write(row, col, value)
|
|
|
|
def _write_tensor_name(self, worksheet, mp_tensor_info, row, col):
|
|
tensor_name = mp_tensor_info.get_tensor_name()
|
|
if (
|
|
mp_tensor_info.fp32_tensor_name is not None
|
|
and mp_tensor_info.fp16_tensor_name
|
|
):
|
|
worksheet.write(row, col, tensor_name, self.tensor_name_format)
|
|
else:
|
|
worksheet.write(row, col, tensor_name)
|
|
|
|
def _write_maxmin_value(
|
|
self, worksheet, value, row, col, check_finite=True
|
|
):
|
|
if value is None:
|
|
worksheet.write(row, col, "--")
|
|
else:
|
|
if abs(value) < 1e-5:
|
|
value_str = f"{value:.6E}"
|
|
else:
|
|
value_str = f"{value:.6f}"
|
|
if check_finite and is_infinite(value, np.float16):
|
|
worksheet.write(row, col, value_str, self.red_bg_cell_format)
|
|
else:
|
|
worksheet.write(row, col, value_str)
|
|
|
|
def _write_tensor_num_zero(
|
|
self, worksheet, value, row, col, check_finite=True
|
|
):
|
|
if value is None:
|
|
worksheet.write(row, col, "--")
|
|
else:
|
|
value_str = f"{value:>10d}"
|
|
worksheet.write(row, col, value_str)
|
|
|
|
def _write_infinite_status(self, worksheet, value, row, col):
|
|
if value is None:
|
|
worksheet.write(row, col, "--")
|
|
else:
|
|
if value == 1:
|
|
worksheet.write(row, col, value, self.red_bg_cell_format)
|
|
else:
|
|
worksheet.write(row, col, value)
|
|
|
|
def _write_fp32divfp16_value(self, worksheet, value, row, col, loss_scale):
|
|
def _in_range(value, scale=1):
|
|
return value > scale * 0.95 and value < scale * 1.05
|
|
|
|
if value is None:
|
|
worksheet.write(row, col, "--")
|
|
else:
|
|
value_str = f"{value:.6f}"
|
|
if _in_range(value, scale=1) or _in_range(value, loss_scale):
|
|
worksheet.write(row, col, value_str)
|
|
else:
|
|
worksheet.write(row, col, value_str, self.orange_bg_cell_format)
|
|
|
|
def _write_titles(self, worksheet, loss_scale, row):
|
|
column_width_dict = {
|
|
"op_type": 24,
|
|
"tensor_name": 60,
|
|
"numel": 10,
|
|
"num_zero": 10,
|
|
"infinite": 8,
|
|
"dtype": 8,
|
|
"max_value": 16,
|
|
"min_value": 16,
|
|
"mean_value": 16,
|
|
"num_inf": 8,
|
|
"num_nan": 8,
|
|
}
|
|
title_names = ["op_type", "tensor_name", "numel", "infinite"]
|
|
if self.log_fp16_dir is None:
|
|
# only fp32 values
|
|
worksheet.merge_range("E1:H1", "fp32", self.title_format)
|
|
worksheet.merge_range(
|
|
"I1:J1", f"fp32 (scale={loss_scale})", self.title_format
|
|
)
|
|
title_names.extend(
|
|
[
|
|
"dtype",
|
|
"max_value",
|
|
"min_value",
|
|
"mean_value",
|
|
"max_value",
|
|
"min_value",
|
|
]
|
|
)
|
|
elif self.log_fp32_dir is None:
|
|
# only fp16 values
|
|
worksheet.merge_range(
|
|
"E1:J1", f"fp16 (scale={loss_scale})", self.title_format
|
|
)
|
|
title_names.extend(
|
|
[
|
|
"dtype",
|
|
"max_value",
|
|
"min_value",
|
|
"mean_value",
|
|
"num_zero",
|
|
"num_inf",
|
|
"num_nan",
|
|
]
|
|
)
|
|
else:
|
|
# fp32 and fp16 values
|
|
worksheet.merge_range("E1:H1", "fp32", self.title_format)
|
|
worksheet.merge_range(
|
|
"I1:N1", f"fp16 (scale={loss_scale})", self.title_format
|
|
)
|
|
worksheet.merge_range("O1:Q1", "fp16 / fp32", self.title_format)
|
|
title_names.extend(
|
|
[
|
|
"dtype",
|
|
"max_value",
|
|
"min_value",
|
|
"mean_value",
|
|
"num_zero",
|
|
"dtype",
|
|
"max_value",
|
|
"min_value",
|
|
"mean_value",
|
|
"num_zero",
|
|
"num_inf",
|
|
"num_nan",
|
|
"max_value",
|
|
"min_value",
|
|
"mean_value",
|
|
]
|
|
)
|
|
|
|
for col in range(len(title_names)):
|
|
col_char = chr(ord("A") + col)
|
|
worksheet.set_column(
|
|
col_char + ":" + col_char, column_width_dict[title_names[col]]
|
|
)
|
|
for col in range(len(title_names)):
|
|
worksheet.write(row, col, title_names[col], self.title_format)
|
|
|
|
def add_worksheet(
|
|
self, mp_tensor_info_list, sheetname, loss_scale, skip_normal_tensors
|
|
):
|
|
assert self.workbook is not None
|
|
|
|
worksheet = self.workbook.add_worksheet(sheetname)
|
|
row = 1
|
|
|
|
self._write_titles(worksheet, loss_scale, row)
|
|
row += 1
|
|
|
|
infinite_op_types = []
|
|
for tensor_info in mp_tensor_info_list:
|
|
if (
|
|
not tensor_info.is_normal
|
|
and tensor_info.op_type not in infinite_op_types
|
|
):
|
|
infinite_op_types.append(tensor_info.op_type)
|
|
|
|
if skip_normal_tensors and tensor_info.is_normal:
|
|
continue
|
|
|
|
worksheet.write(row, 0, tensor_info.op_type)
|
|
self._write_tensor_name(worksheet, tensor_info, row, 1)
|
|
|
|
if tensor_info.numel > np.iinfo(np.int32).max:
|
|
worksheet.write(
|
|
row, 2, tensor_info.numel, self.bad_value_format
|
|
)
|
|
else:
|
|
worksheet.write(row, 2, tensor_info.numel)
|
|
|
|
if tensor_info.is_normal:
|
|
worksheet.write(row, 3, "0")
|
|
else:
|
|
worksheet.write(row, 3, "1", self.red_bg_cell_format)
|
|
|
|
col = 4
|
|
|
|
if self.log_fp32_dir is not None:
|
|
self._write_dtype(worksheet, tensor_info.fp32_dtype, row, col)
|
|
self._write_maxmin_value(
|
|
worksheet, tensor_info.fp32_max_value, row, col + 1
|
|
)
|
|
self._write_maxmin_value(
|
|
worksheet, tensor_info.fp32_min_value, row, col + 2
|
|
)
|
|
self._write_maxmin_value(
|
|
worksheet, tensor_info.fp32_mean_value, row, col + 3
|
|
)
|
|
self._write_tensor_num_zero(
|
|
worksheet, tensor_info.fp32_num_zero, row, col + 4
|
|
)
|
|
col += 5
|
|
|
|
if self.log_fp16_dir is None:
|
|
self._write_maxmin_value(
|
|
worksheet, tensor_info.scaled_fp32_max_value, row, col
|
|
)
|
|
self._write_maxmin_value(
|
|
worksheet,
|
|
tensor_info.scaled_fp32_min_value,
|
|
row,
|
|
col + 1,
|
|
)
|
|
col += 2
|
|
|
|
if self.log_fp16_dir is not None:
|
|
self._write_dtype(worksheet, tensor_info.fp16_dtype, row, col)
|
|
self._write_maxmin_value(
|
|
worksheet, tensor_info.fp16_max_value, row, col + 1
|
|
)
|
|
self._write_maxmin_value(
|
|
worksheet, tensor_info.fp16_min_value, row, col + 2
|
|
)
|
|
self._write_maxmin_value(
|
|
worksheet, tensor_info.fp16_mean_value, row, col + 3
|
|
)
|
|
self._write_tensor_num_zero(
|
|
worksheet, tensor_info.fp32_num_zero, row, col + 4
|
|
)
|
|
col += 5
|
|
|
|
self._write_infinite_status(
|
|
worksheet, tensor_info.fp16_has_inf, row, col
|
|
)
|
|
self._write_infinite_status(
|
|
worksheet, tensor_info.fp16_has_nan, row, col + 1
|
|
)
|
|
col += 2
|
|
|
|
if self.log_fp32_dir is not None and self.log_fp16_dir is not None:
|
|
self._write_fp32divfp16_value(
|
|
worksheet,
|
|
tensor_info.fp32_div_fp16_max_value,
|
|
row,
|
|
col,
|
|
loss_scale,
|
|
)
|
|
self._write_fp32divfp16_value(
|
|
worksheet,
|
|
tensor_info.fp32_div_fp16_min_value,
|
|
row,
|
|
col + 1,
|
|
loss_scale,
|
|
)
|
|
self._write_fp32divfp16_value(
|
|
worksheet,
|
|
tensor_info.fp32_div_fp16_mean_value,
|
|
row,
|
|
col + 2,
|
|
loss_scale,
|
|
)
|
|
col += 3
|
|
|
|
row += 1
|
|
|
|
print(f"-- OP Types produce infinite outputs: {infinite_op_types}")
|
|
|
|
|
|
def parse_lines(lines, specified_op_list=None):
|
|
tensor_info_list = []
|
|
|
|
for i in range(len(lines)):
|
|
if i % 10 == 0:
|
|
print(
|
|
f"-- Processing {i:-8d} / {len(lines):-8d} line",
|
|
end="\r",
|
|
)
|
|
line = lines[i]
|
|
if "[PRECISION]" in line:
|
|
tensor_info = TensorInfo()
|
|
tensor_info.init_from_string(line)
|
|
if (
|
|
tensor_info.tensor_name is not None
|
|
and tensor_info.tensor_name != ""
|
|
):
|
|
has_tensor_name = True
|
|
if (
|
|
specified_op_list is None
|
|
or tensor_info.op_type in specified_op_list
|
|
):
|
|
tensor_info_list.append(tensor_info)
|
|
# print(tensor_info)
|
|
return tensor_info_list
|
|
|
|
|
|
def parse_log(log_dir, filename, specified_op_list=None):
|
|
if log_dir is None or filename is None:
|
|
return None
|
|
|
|
complete_filename = log_dir + "/" + filename
|
|
tensor_info_list = None
|
|
has_tensor_name = False
|
|
|
|
try:
|
|
with open(complete_filename, 'r') as f:
|
|
lines = f.readlines()
|
|
tensor_info_list = parse_lines(lines, specified_op_list)
|
|
except FileNotFoundError:
|
|
print("the file ", complete_filename, "is not found")
|
|
return None, has_tensor_name
|
|
return tensor_info_list, has_tensor_name
|
|
|
|
|
|
def merge_tensor_info_list(
|
|
fp32_tensor_info_list, fp16_tensor_info_list, grad_scale
|
|
):
|
|
mp_tensor_info_list = []
|
|
if fp16_tensor_info_list is not None:
|
|
fp32_tensor_info_dict = {}
|
|
fp32_write_count = {}
|
|
if fp32_tensor_info_list is not None:
|
|
for tensor_info in fp32_tensor_info_list:
|
|
tensor_info_key = tensor_info.key()
|
|
count = fp32_write_count.get(tensor_info_key, 0)
|
|
fp32_write_count[tensor_info_key] = count + 1
|
|
fp32_tensor_info_dict[tensor_info_key + "#" + str(count)] = (
|
|
tensor_info
|
|
)
|
|
|
|
fp32_read_count = {}
|
|
for i in range(len(fp16_tensor_info_list)):
|
|
if i % 10 == 0:
|
|
print(
|
|
f"-- Processing {i:-8d} / {len(fp16_tensor_info_list):-8d} FP16 Tensor Info",
|
|
end="\r",
|
|
)
|
|
fp16_tensor_info = fp16_tensor_info_list[i]
|
|
fp32_tensor_info_key = (
|
|
fp16_tensor_info.key()
|
|
.replace(".cast_fp16", "")
|
|
.replace(".cast_fp32", "")
|
|
)
|
|
count = fp32_read_count.get(fp32_tensor_info_key, 0)
|
|
fp32_tensor_info = fp32_tensor_info_dict.get(
|
|
fp32_tensor_info_key + "#" + str(count), None
|
|
)
|
|
if fp32_tensor_info is not None:
|
|
fp32_read_count[fp32_tensor_info_key] = count + 1
|
|
mp_tensor_info = MixedPrecisionTensorInfo(
|
|
fp32_tensor_info, fp16_tensor_info, count, grad_scale
|
|
)
|
|
mp_tensor_info_list.append(mp_tensor_info)
|
|
# print(mp_tensor_info)
|
|
elif fp32_tensor_info_list is not None:
|
|
fp32_count = {}
|
|
for i in range(len(fp32_tensor_info_list)):
|
|
if i % 10 == 0:
|
|
print(
|
|
f"-- Processing {i:-8d} / {len(fp32_tensor_info_list):-8d} FP32 Tensor Info",
|
|
end="\r",
|
|
)
|
|
tensor_info = fp32_tensor_info_list[i]
|
|
tensor_info_key = tensor_info.key()
|
|
count = fp32_count.get(tensor_info_key, 0)
|
|
fp32_count[tensor_info_key] = count + 1
|
|
mp_tensor_info = MixedPrecisionTensorInfo(
|
|
tensor_info, None, count, grad_scale
|
|
)
|
|
mp_tensor_info_list.append(mp_tensor_info)
|
|
|
|
return mp_tensor_info_list
|
|
|
|
|
|
def compare_accuracy(
|
|
dump_path,
|
|
another_dump_path,
|
|
output_filename,
|
|
loss_scale=1,
|
|
dump_all_tensors=False,
|
|
):
|
|
excel_writer = ExcelWriter(dump_path, another_dump_path, output_filename)
|
|
grad_scale = loss_scale
|
|
workerlog_filenames = []
|
|
filenames = os.listdir(dump_path)
|
|
for name in filenames:
|
|
if "worker_" in name:
|
|
workerlog_filenames.append(name)
|
|
print(
|
|
f"-- There are {len(workerlog_filenames)} workerlogs under {dump_path}: {workerlog_filenames}"
|
|
)
|
|
|
|
for filename in sorted(workerlog_filenames):
|
|
print(f"-- [Step 1/4] Parsing FP32 logs under {dump_path}/{filename}")
|
|
fp32_tensor_info_list, fp32_has_tensor_name = parse_log(
|
|
dump_path, filename, None
|
|
)
|
|
print(
|
|
f"-- [Step 2/4] Parsing FP16 logs under {another_dump_path}/{filename}"
|
|
)
|
|
fp16_tensor_info_list, fp16_has_tensor_name = parse_log(
|
|
another_dump_path, filename, None
|
|
)
|
|
|
|
print(f"-- [Step 3/4] Merge FP32 and FP16 tensor info for {filename}")
|
|
mp_tensor_info_list = merge_tensor_info_list(
|
|
fp32_tensor_info_list, fp16_tensor_info_list, grad_scale
|
|
)
|
|
print(
|
|
f"-- [Step 4/4] Add worksheet for mixed precision tensor info of {filename}"
|
|
)
|
|
excel_writer.add_worksheet(
|
|
mp_tensor_info_list,
|
|
filename,
|
|
loss_scale,
|
|
False,
|
|
)
|
|
|
|
print(f"-- Write to {output_filename}")
|
|
|
|
print()
|
|
excel_writer.close()
|