chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
Executable
+410
@@ -0,0 +1,410 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import glob
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from multiprocessing import Lock
|
||||
|
||||
""" Some terms to clarify the code
|
||||
in most case, one or more parameters may be set as input args for a class or a function
|
||||
in form of single variable or k-v dict
|
||||
|
||||
1. trainerId
|
||||
2. gpuId
|
||||
3. rankId
|
||||
4. gpuPerTrainer
|
||||
5. groupSize
|
||||
6. groupId
|
||||
7. groupNum
|
||||
8. displaySize
|
||||
9. dataPath
|
||||
10. resultPath
|
||||
11. fileOrganizeForm -- "byRank" OR "byTrainer" or "other"
|
||||
|
||||
"""
|
||||
|
||||
PIPELINEINFO_TRACE_NUM = 1
|
||||
|
||||
dcgmMetricParameterMap = {
|
||||
"02_gpuUtility": [("GPUTL", "GPUTL"), ("GRACT", "GRACT")],
|
||||
"03_smUtility": [("SMACT", "SMACT"), ("SMOCC", "SMOCC")],
|
||||
"04_memUtility": [("FB_USED_RATIO", "FB_USED_RATIO"), ("DRAMA", "DRAMA")],
|
||||
"05_txUtility": [
|
||||
("NVLTX", "NVLTX"),
|
||||
("NVLRX", "NVLRX"),
|
||||
("PCITX", "PCITX"),
|
||||
("PCIRX", "PCIRX"),
|
||||
],
|
||||
"06_calUtility": [
|
||||
("FP32A", "FP32A"),
|
||||
("FP16A", "FP16A"),
|
||||
("TENSO", "TENSO"),
|
||||
],
|
||||
}
|
||||
DCGMINFO_TRACE_NUM = len(dcgmMetricParameterMap.keys())
|
||||
NETINFO_TRACE_NUM = 2
|
||||
|
||||
DCGM_PATH = "dcgm"
|
||||
NET_PATH = "net"
|
||||
TIME_PATH = "time"
|
||||
PROFILE_PATH = "profile"
|
||||
|
||||
FILEORGANIZEFORM_BYRANK = "byRank"
|
||||
FILEORGANIZEFORM_BYTRAINER = "byTrainer"
|
||||
FILEORGANIZEFORM_BYOTHER = "other"
|
||||
FILEORGANIZEFORM = [
|
||||
FILEORGANIZEFORM_BYRANK,
|
||||
FILEORGANIZEFORM_BYTRAINER,
|
||||
FILEORGANIZEFORM_BYOTHER,
|
||||
]
|
||||
|
||||
|
||||
class FileReader:
|
||||
def __init__(self, logger, args):
|
||||
self._logger = logger
|
||||
self._args = args
|
||||
|
||||
self._fileList = []
|
||||
self._fileNum = 0
|
||||
|
||||
self._dataPath = ""
|
||||
self._groupSize = 0
|
||||
self._displaySize = 0
|
||||
self._organizeForm = FILEORGANIZEFORM_BYOTHER
|
||||
self._gpuPerTrainer = 0
|
||||
|
||||
self._checkArgs()
|
||||
self._getFileList()
|
||||
|
||||
self._lock = Lock()
|
||||
|
||||
def printArgs(self):
|
||||
self._logger.info("dataPath:")
|
||||
self._logger.info(self._dataPath)
|
||||
self._logger.info("groupSize:")
|
||||
self._logger.info(self._groupSize)
|
||||
self._logger.info("displaySize:")
|
||||
self._logger.info(self._displaySize)
|
||||
self._logger.info("organizeForm:")
|
||||
self._logger.info(self._organizeForm)
|
||||
self._logger.info("gpuPerTrainer:")
|
||||
self._logger.info(self._gpuPerTrainer)
|
||||
self._logger.info("minTimeStamp:")
|
||||
self._logger.info(self._minTimeStamp)
|
||||
|
||||
def _checkArgsKey(self, key, type):
|
||||
if key not in self._args:
|
||||
raise KeyError(f"args should has key [{key}]!")
|
||||
|
||||
if not isinstance(self._args[key], type):
|
||||
raise TypeError(
|
||||
f"Invalid type of key [{key}] in args dict, it should be a {type}!"
|
||||
)
|
||||
|
||||
exec(f'self._{key} = self._args["{key}"]')
|
||||
|
||||
def _align_ts(self, ts):
|
||||
return ts - self._minTimeStamp
|
||||
|
||||
def _checkArgs(self):
|
||||
if not isinstance(self._args, dict):
|
||||
raise TypeError("Invalid type of args, it should be a dict!")
|
||||
|
||||
self._checkArgsKey("organizeForm", str)
|
||||
if (
|
||||
self._organizeForm not in FILEORGANIZEFORM
|
||||
or self._organizeForm == FILEORGANIZEFORM_BYOTHER
|
||||
):
|
||||
raise NotImplementedError(
|
||||
f"we have not known how to process this form of file [{self._organizeForm}]!"
|
||||
)
|
||||
|
||||
self._checkArgsKey("gpuPerTrainer", int)
|
||||
|
||||
self._checkArgsKey("dataPath", str)
|
||||
if not os.path.exists(self._dataPath):
|
||||
raise OSError(f"input data path [{self._dataPath}] not existed!")
|
||||
|
||||
self._checkArgsKey("groupSize", int)
|
||||
self._checkArgsKey("displaySize", int)
|
||||
self._checkArgsKey("minTimeStamp", int)
|
||||
|
||||
def getFileListByGroup(self, groupId):
|
||||
lIndex = 0
|
||||
rIndex = 0
|
||||
|
||||
if self._organizeForm == FILEORGANIZEFORM_BYTRAINER:
|
||||
lIndex = groupId * self._groupSize
|
||||
rIndex = (groupId + 1) * self._groupSize
|
||||
elif self._organizeForm == FILEORGANIZEFORM_BYRANK:
|
||||
lIndex = groupId * self._groupSize * self._gpuPerTrainer
|
||||
rIndex = (groupId + 1) * self._groupSize * self._gpuPerTrainer
|
||||
|
||||
try:
|
||||
return self._fileList[lIndex:rIndex]
|
||||
except IndexError:
|
||||
raise IndexError("invalid index of file list")
|
||||
|
||||
def getFileList(self):
|
||||
return self._getFileList
|
||||
|
||||
def _cmp(self, x, y):
|
||||
return self._getId(x, self._organizeForm) - self._getId(
|
||||
y, self._organizeForm
|
||||
)
|
||||
|
||||
def _getFileList(self):
|
||||
self._fileList = glob.glob(os.path.join(self._dataPath, "*.*"))
|
||||
|
||||
# check unique
|
||||
idList = []
|
||||
newFileList = []
|
||||
for file in self._fileList:
|
||||
id = self._getId(file, self._organizeForm)
|
||||
if id not in idList:
|
||||
idList.append(id)
|
||||
newFileList.append(file)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
f"[{file}] is repeated by id, we do not know how to process it!"
|
||||
)
|
||||
|
||||
if not self._fileList:
|
||||
if (
|
||||
self._getId(self._fileList[-1]) - self._getId(self._fileList[0])
|
||||
) != len(self._fileList) - 1:
|
||||
raise Exception("The file id should be continuous!")
|
||||
|
||||
# sort
|
||||
def _sortBySuffix(elem):
|
||||
return int(elem.split(".")[-1])
|
||||
|
||||
self._fileList.sort(key=_sortBySuffix)
|
||||
|
||||
if not self._fileList:
|
||||
self._logger.warning(
|
||||
f"we can not find any file in dir [{self._dataPath}]!"
|
||||
)
|
||||
else:
|
||||
self._logger.info(
|
||||
"file list in dir [{}] is : {} !".format(
|
||||
self._dataPath, ', '.join(self._fileList)
|
||||
)
|
||||
)
|
||||
|
||||
return self._fileList
|
||||
|
||||
def _getId(self, fileName, organizeForm, sed="."):
|
||||
if self._organizeForm != organizeForm:
|
||||
raise TypeError(
|
||||
f"Can not get rank id when organize form is not {organizeForm}!"
|
||||
)
|
||||
|
||||
if not os.path.isfile(fileName):
|
||||
raise OSError(f"[{fileName}] is not a valid file!")
|
||||
|
||||
try:
|
||||
prefix_str = fileName.split(sed)[-1]
|
||||
try:
|
||||
return int(prefix_str)
|
||||
except ValueError as e:
|
||||
print(e)
|
||||
raise TypeError(f"invalid fileName [{fileName}]")
|
||||
|
||||
except IndexError as e:
|
||||
print(e)
|
||||
raise TypeError(
|
||||
f"invalid fileName [{fileName}], the prefix should be a number!"
|
||||
)
|
||||
|
||||
def getRankId(self, fileName, sed="."):
|
||||
return self._getId(fileName, FILEORGANIZEFORM_BYRANK, sed)
|
||||
|
||||
def getRankNum(self):
|
||||
if self._organizeForm == FILEORGANIZEFORM_BYRANK:
|
||||
return len(self._fileList)
|
||||
|
||||
elif self._organizeForm == FILEORGANIZEFORM_BYTRAINER:
|
||||
return len(self._fileList) * self._gpuPerTrainer
|
||||
|
||||
def getTrainerNum(self):
|
||||
if self._organizeForm == FILEORGANIZEFORM_BYRANK:
|
||||
return len(self._fileList) / self._gpuPerTrainer
|
||||
|
||||
elif self._organizeForm == FILEORGANIZEFORM_BYTRAINER:
|
||||
return len(self._fileList)
|
||||
|
||||
def getTrainerId(self, fileName, sed="."):
|
||||
return self._getId(fileName, FILEORGANIZEFORM_BYTRAINER, sed)
|
||||
|
||||
def _splitTaskListForMultiProcess(self, ls, n):
|
||||
if not isinstance(ls, list) or not isinstance(n, int):
|
||||
return []
|
||||
ls_len = len(ls)
|
||||
if n <= 0 or 0 == ls_len:
|
||||
return []
|
||||
if n >= ls_len:
|
||||
return [[i] for i in ls]
|
||||
else:
|
||||
j = int((ls_len + n - 1) / n)
|
||||
k = ls_len % n
|
||||
ls_return = []
|
||||
end = 0
|
||||
for i in range(0, (n) * j, j):
|
||||
if i < len(ls) and (i + j) < len(ls):
|
||||
ls_return.append(ls[i : i + j])
|
||||
end = i + j
|
||||
ls_return.append(ls[end:])
|
||||
return ls_return
|
||||
|
||||
def getOpInfoFileName(self, groupId, gpuId, tmpPath="./tmp"):
|
||||
return self.getFileName("opinfo", groupId, gpuId, tmpPath)
|
||||
|
||||
def getPipeLineInfoFileName(self, groupId, gpuId, tmpPath="./tmp"):
|
||||
return self.getFileName("pipelineinfo", groupId, gpuId, tmpPath)
|
||||
|
||||
def getDCGMInfoFileName(self, groupId, gpuId, tmpPath="./tmp"):
|
||||
return self.getFileName("dcgm", groupId, gpuId, tmpPath)
|
||||
|
||||
def getFileName(self, name, groupId, gpuId, tmpPath="./tmp"):
|
||||
return os.path.join(tmpPath, f"{name}_{groupId}_{gpuId}.json")
|
||||
|
||||
def getOpInfoDict(self, groupId, gpuId, tmpPath="./tmp"):
|
||||
return self.getDict("opinfo", groupId, gpuId, tmpPath)
|
||||
|
||||
def getDcgmInfoDict(self, groupId, gpuId, tmpPath="./tmp"):
|
||||
return self.getDict("dcgm", groupId, gpuId, tmpPath)
|
||||
|
||||
def getDict(self, name, groupId, gpuId, tmpPath="./tmp"):
|
||||
fileName = self.getFileName(name, groupId, gpuId, tmpPath)
|
||||
if not os.path.isfile(fileName):
|
||||
raise OSError(f"[{fileName}] does not existed!")
|
||||
|
||||
data = {}
|
||||
with open(fileName, "r") as rf:
|
||||
try:
|
||||
data = json.load(rf)
|
||||
except Exception:
|
||||
self._logger.error(f"read [{fileName}] error. not a json file!")
|
||||
raise TypeError(f"read [{fileName}] error. not a json file!")
|
||||
return data
|
||||
|
||||
def dumpOpInfoDict(
|
||||
self, data, groupId, gpuId, pretty=False, tmpPath="./tmp"
|
||||
):
|
||||
return self.dumpDict(
|
||||
data, "opinfo", groupId, gpuId, pretty=False, tmpPath="./tmp"
|
||||
)
|
||||
|
||||
def dumpDCGMDict(self, data, groupId, gpuId, pretty=False, tmpPath="./tmp"):
|
||||
return self.dumpDict(
|
||||
data, "dcgm", groupId, gpuId, pretty=False, tmpPath="./tmp"
|
||||
)
|
||||
|
||||
def dumpDict(
|
||||
self, data, name, groupId, gpuId, pretty=False, tmpPath="./tmp"
|
||||
):
|
||||
self._lock.acquire()
|
||||
if not os.path.exists(tmpPath):
|
||||
os.makedirs(tmpPath)
|
||||
self._lock.release()
|
||||
if pretty:
|
||||
jsObj = json.dumps(data, indent=4, separators=(',', ': '))
|
||||
else:
|
||||
jsObj = json.dumps(data, separators=(',', ':'))
|
||||
|
||||
fileName = self.getFileName(name, groupId, gpuId, tmpPath)
|
||||
if os.path.isfile(fileName):
|
||||
os.remove(fileName)
|
||||
|
||||
fileObject = open(fileName, 'w')
|
||||
fileObject.write(jsObj)
|
||||
fileObject.close()
|
||||
self._logger.info(f"dump [{fileName}] successfully!")
|
||||
|
||||
|
||||
def getLogger():
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
rq = time.strftime('%Y%m%d%H%M.%s', time.localtime(time.time()))
|
||||
log_path = os.path.dirname(os.getcwd()) + '/Logs/'
|
||||
if not os.path.exists(log_path):
|
||||
os.makedirs(log_path)
|
||||
|
||||
log_name = log_path + rq + '.log'
|
||||
logfile = log_name
|
||||
fh = logging.FileHandler(logfile, mode='w')
|
||||
fh.setLevel(logging.DEBUG)
|
||||
|
||||
formatter = logging.Formatter(
|
||||
"%(asctime)s - %(filename)s[line:%(lineno)d] - %(process)d - %(levelname)s: %(message)s"
|
||||
)
|
||||
fh.setFormatter(formatter)
|
||||
|
||||
logger.addHandler(fh)
|
||||
return logger
|
||||
|
||||
|
||||
def test_FileReader(args):
|
||||
try:
|
||||
testReader = FileReader(None, args)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
else:
|
||||
testReader.printArgs()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = 0
|
||||
test_FileReader(args)
|
||||
|
||||
args = {
|
||||
"dataPath": ".",
|
||||
"groupSize": 1,
|
||||
"displaySize": 1,
|
||||
"gpuPerTrainer": 8,
|
||||
"organizeForm": FILEORGANIZEFORM_BYOTHER,
|
||||
}
|
||||
test_FileReader(args)
|
||||
|
||||
args = {
|
||||
"dataPath": ".",
|
||||
"groupSize": 1,
|
||||
"displaySize": 1,
|
||||
"gpuPerTrainer": 8,
|
||||
"organizeForm": FILEORGANIZEFORM_BYTRAINER,
|
||||
}
|
||||
test_FileReader(args)
|
||||
|
||||
args = {
|
||||
"dataPath": "./res",
|
||||
"groupSize": 1,
|
||||
"displaySize": 1,
|
||||
"gpuPerTrainer": 8,
|
||||
"organizeForm": FILEORGANIZEFORM_BYTRAINER,
|
||||
}
|
||||
test_FileReader(args)
|
||||
|
||||
args = {
|
||||
"dataPath": ".",
|
||||
"groupSize": "",
|
||||
"displaySize": 1,
|
||||
"gpuPerTrainer": 8,
|
||||
"organizeForm": FILEORGANIZEFORM_BYTRAINER,
|
||||
}
|
||||
test_FileReader(args)
|
||||
Reference in New Issue
Block a user