chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+516
@@ -0,0 +1,516 @@
|
||||
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import multiprocessing
|
||||
from multiprocessing import Process
|
||||
|
||||
from CspChromeTraceFormatter import ChromeTraceFormatter
|
||||
from CspFileReader import (
|
||||
DCGMINFO_TRACE_NUM,
|
||||
FILEORGANIZEFORM_BYRANK,
|
||||
NETINFO_TRACE_NUM,
|
||||
PIPELINEINFO_TRACE_NUM,
|
||||
FileReader,
|
||||
getLogger,
|
||||
)
|
||||
|
||||
from paddle.base.proto.profiler import profiler_pb2
|
||||
|
||||
|
||||
class profileFileReader(FileReader):
|
||||
def _parseSingleFile(self, profile):
|
||||
with open(profile, 'rb') as f:
|
||||
profile_s = f.read()
|
||||
profile_pb = profiler_pb2.Profile()
|
||||
profile_pb.ParseFromString(profile_s)
|
||||
|
||||
return profile_pb
|
||||
|
||||
def _parseTask(self, taskList, q=None):
|
||||
profile_dict = {}
|
||||
|
||||
for fileName in taskList:
|
||||
rankId = self.getRankId(fileName)
|
||||
profile_dict[f"trainerRank.{rankId:03}"] = self._parseSingleFile(
|
||||
fileName
|
||||
)
|
||||
self._logger.info(f"I finish processing {fileName}!")
|
||||
|
||||
if q is not None:
|
||||
q.put(profile_dict)
|
||||
|
||||
return profile_dict
|
||||
|
||||
def _is_forwardBackwardInfo(self, items):
|
||||
if items["name"] == "marker/compute/MarkerCUDA":
|
||||
if "args" in items:
|
||||
if isinstance(items["args"], dict):
|
||||
args = items["args"]
|
||||
if "detail_info" in args:
|
||||
if (
|
||||
args["detail_info"] == "marker_forward_B"
|
||||
or args["detail_info"] == "marker_forward_E"
|
||||
or args["detail_info"] == "marker_backward_B"
|
||||
or args["detail_info"] == "marker_backward_E"
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _allocate_forwardBackwardInfo(self, restList, pid, tid):
|
||||
def _cmp_ele(items):
|
||||
return items["ts"]
|
||||
|
||||
restList.sort(key=_cmp_ele)
|
||||
newList = []
|
||||
|
||||
lastEle = {}
|
||||
for items in restList:
|
||||
if items["args"]["detail_info"].endswith("E"):
|
||||
if not lastEle:
|
||||
continue
|
||||
else:
|
||||
lastEle["dur"] = items["ts"] - lastEle["ts"]
|
||||
name = lastEle["args"]["detail_info"]
|
||||
name = name[: name.rfind('_')]
|
||||
name = name.split('_')[1]
|
||||
lastEle["name"] = name
|
||||
lastEle["args"]["detail_info"] = name
|
||||
lastEle["args"]["name"] = name
|
||||
if name == "backward":
|
||||
lastEle["cname"] = "good"
|
||||
else:
|
||||
lastEle["cname"] = "bad"
|
||||
|
||||
lastEle["tid"] = tid
|
||||
lastEle["pid"] = pid
|
||||
|
||||
newList.append(lastEle)
|
||||
else:
|
||||
lastEle = items
|
||||
|
||||
return newList
|
||||
|
||||
def _getPipeLineInfo(self, profileList, q=None):
|
||||
res = {}
|
||||
for profile in profileList:
|
||||
rankId = self.getRankId(profile)
|
||||
|
||||
profile_pb = self._parseSingleFile(profile)
|
||||
traceEventList = []
|
||||
pid = 0
|
||||
tid = rankId
|
||||
|
||||
for event in profile_pb.events:
|
||||
args = {'name': event.name}
|
||||
if event.memcopy.bytes > 0:
|
||||
args['mem_bytes'] = event.memcopy.bytes
|
||||
if hasattr(event, "detail_info") and event.detail_info:
|
||||
args['detail_info'] = event.detail_info
|
||||
|
||||
traceEvent = {}
|
||||
traceEvent['ph'] = 'X'
|
||||
traceEvent['cat'] = 'Op'
|
||||
traceEvent['name'] = event.name
|
||||
traceEvent['pid'] = pid
|
||||
traceEvent['tid'] = tid
|
||||
traceEvent['ts'] = self._align_ts(event.start_ns)
|
||||
traceEvent['dur'] = (event.end_ns - event.start_ns) / 1.0
|
||||
traceEvent['args'] = args
|
||||
|
||||
if self._is_forwardBackwardInfo(traceEvent):
|
||||
traceEventList.append(traceEvent)
|
||||
|
||||
pipeLineList = self._allocate_forwardBackwardInfo(
|
||||
traceEventList, pid, tid
|
||||
)
|
||||
|
||||
res[str(rankId)] = pipeLineList
|
||||
|
||||
if q is not None:
|
||||
q.put(res)
|
||||
|
||||
return res
|
||||
|
||||
def getPipeLineInfo(self, groupId, processNum=8):
|
||||
fileFist = self.getFileListByGroup(groupId)
|
||||
|
||||
self._logger.info(
|
||||
f"using [{processNum}] process to do this work, total task num is {len(fileFist)}!"
|
||||
)
|
||||
processPool = []
|
||||
pidList = []
|
||||
|
||||
manager = multiprocessing.Manager()
|
||||
q = manager.Queue()
|
||||
|
||||
taskList = self._splitTaskListForMultiProcess(fileFist, processNum)
|
||||
for task in taskList:
|
||||
subproc = Process(
|
||||
target=self._getPipeLineInfo,
|
||||
args=(
|
||||
task,
|
||||
q,
|
||||
),
|
||||
)
|
||||
processPool.append(subproc)
|
||||
subproc.start()
|
||||
pidList.append(subproc.pid)
|
||||
self._logger.info(
|
||||
f"[pipeline info]: process [{subproc.pid}] has been started, total task num is {len(task)} ..."
|
||||
)
|
||||
|
||||
for t in processPool:
|
||||
t.join()
|
||||
pidList.remove(t.pid)
|
||||
self._logger.info(
|
||||
f"[pipeline info]: process [{t.pid}] has exited! remained {len(pidList)} process!"
|
||||
)
|
||||
|
||||
pipeLineInfo = {}
|
||||
|
||||
metaInfo = {}
|
||||
metaInfo['name'] = 'process_name'
|
||||
metaInfo['ph'] = 'M'
|
||||
metaInfo['pid'] = 0
|
||||
metaInfo['args'] = {'name': f"{PIPELINEINFO_TRACE_NUM:02}_pipeLineInfo"}
|
||||
|
||||
for t in processPool:
|
||||
for k, v in q.get().items():
|
||||
rankId = int(k)
|
||||
gpuId = rankId % self._gpuPerTrainer
|
||||
if str(gpuId) not in pipeLineInfo.keys():
|
||||
pipeLineInfo[str(gpuId)] = [metaInfo]
|
||||
pipeLineInfo[str(gpuId)].extend(v)
|
||||
|
||||
return pipeLineInfo
|
||||
|
||||
def _allocate_pids(self, profile_dict, gpuId, initPid):
|
||||
chrome_trace = ChromeTraceFormatter()
|
||||
devices = {}
|
||||
mem_devices = {}
|
||||
|
||||
initLineNum = initPid + 1
|
||||
lineDelta = len(profile_dict.keys())
|
||||
i = 0
|
||||
for k, profile_pb in profile_dict.items():
|
||||
lineNum = initLineNum
|
||||
for event in profile_pb.events:
|
||||
if event.type == profiler_pb2.Event.CPU:
|
||||
if (k, event.device_id, "CPU") not in devices:
|
||||
pid = initPid
|
||||
initPid = initPid + 1
|
||||
devices[(k, event.device_id, "CPU")] = pid
|
||||
# -1 device id represents CUDA API(RunTime) call.(e.g. cudaLaunch, cudaMemcpy)
|
||||
if event.device_id == -1:
|
||||
chrome_trace.emit_pid(
|
||||
f"{lineNum:02}_{k}:cuda_api", pid
|
||||
)
|
||||
lineNum = lineNum + 1
|
||||
else:
|
||||
chrome_trace.emit_pid(
|
||||
f"{lineNum:02}_{k}:cpu:block:{event.device_id}",
|
||||
pid,
|
||||
)
|
||||
lineNum = lineNum + 1
|
||||
elif event.type == profiler_pb2.Event.GPUKernel:
|
||||
if (k, event.device_id, "GPUKernel") not in devices:
|
||||
if gpuId == event.device_id:
|
||||
pid = initPid
|
||||
initPid = initPid + 1
|
||||
|
||||
devices[(k, event.device_id, "GPUKernel")] = pid
|
||||
chrome_trace.emit_pid(
|
||||
f"{lineNum:02}_{k}:gpu:{event.device_id}",
|
||||
pid,
|
||||
)
|
||||
lineNum = lineNum + 1
|
||||
|
||||
if not hasattr(profile_pb, "mem_events"):
|
||||
continue
|
||||
for mevent in profile_pb.mem_events:
|
||||
if mevent.place == profiler_pb2.MemEvent.CUDAPlace:
|
||||
if (k, mevent.device_id, "GPU") not in mem_devices:
|
||||
if gpuId == mevent.device_id:
|
||||
pid = initPid
|
||||
initPid = initPid + 1
|
||||
|
||||
mem_devices[(k, mevent.device_id, "GPU")] = pid
|
||||
chrome_trace.emit_pid(
|
||||
f"{lineNum:02}_memory usage on {k}:gpu:{mevent.device_id}",
|
||||
pid,
|
||||
)
|
||||
lineNum = lineNum + 1
|
||||
elif mevent.place == profiler_pb2.MemEvent.CPUPlace:
|
||||
if (k, mevent.device_id, "CPU") not in mem_devices:
|
||||
pid = initPid
|
||||
initPid = initPid + 1
|
||||
|
||||
mem_devices[(k, mevent.device_id, "CPU")] = pid
|
||||
chrome_trace.emit_pid(
|
||||
f"{lineNum:02}_memory usage on {k}:cpu:{mevent.device_id}",
|
||||
pid,
|
||||
)
|
||||
lineNum = lineNum + 1
|
||||
elif mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace:
|
||||
if (
|
||||
k,
|
||||
mevent.device_id,
|
||||
"CUDAPinnedPlace",
|
||||
) not in mem_devices:
|
||||
if gpuId == mevent.device_id:
|
||||
pid = initPid
|
||||
initPid = initPid + 1
|
||||
|
||||
mem_devices[
|
||||
(k, mevent.device_id, "CUDAPinnedPlace")
|
||||
] = pid
|
||||
chrome_trace.emit_pid(
|
||||
f"{lineNum:02}_memory usage on {k}:cudapinnedplace:{mevent.device_id}",
|
||||
pid,
|
||||
)
|
||||
lineNum = lineNum + 1
|
||||
if (k, 0, "CPU") not in mem_devices:
|
||||
pid = initPid
|
||||
initPid = initPid + 1
|
||||
|
||||
mem_devices[(k, 0, "CPU")] = pid
|
||||
chrome_trace.emit_pid(
|
||||
f"{lineNum:02}_memory usage on {k}:cpu:{0}", pid
|
||||
)
|
||||
lineNum = lineNum + 1
|
||||
if (k, 0, "GPU") not in mem_devices:
|
||||
# if gpuId == mevent.device_id:
|
||||
pid = initPid
|
||||
initPid = initPid + 1
|
||||
|
||||
mem_devices[(k, 0, "GPU")] = pid
|
||||
chrome_trace.emit_pid(
|
||||
f"{lineNum:02}_memory usage on {k}:gpu:{0}", pid
|
||||
)
|
||||
lineNum = lineNum + 1
|
||||
if (k, 0, "CUDAPinnedPlace") not in mem_devices:
|
||||
pid = initPid
|
||||
initPid = initPid + 1
|
||||
|
||||
mem_devices[(k, 0, "CUDAPinnedPlace")] = pid
|
||||
chrome_trace.emit_pid(
|
||||
f"{lineNum:02}_memory usage on {k}:cudapinnedplace:{0}",
|
||||
pid,
|
||||
)
|
||||
lineNum = lineNum + 1
|
||||
i = i + 1
|
||||
return chrome_trace, devices, mem_devices
|
||||
|
||||
def _allocate_events(self, profile_dict, devices, gpuId):
|
||||
chrome_trace = ChromeTraceFormatter()
|
||||
for k, profile_pb in profile_dict.items():
|
||||
rankId = int(k.split(".")[-1])
|
||||
|
||||
for event in profile_pb.events:
|
||||
if event.type == profiler_pb2.Event.CPU:
|
||||
type = "CPU"
|
||||
elif event.type == profiler_pb2.Event.GPUKernel:
|
||||
type = "GPUKernel"
|
||||
|
||||
if (
|
||||
event.type == profiler_pb2.Event.GPUKernel
|
||||
and event.device_id != gpuId
|
||||
and rankId % self._gpuPerTrainer != gpuId
|
||||
):
|
||||
continue
|
||||
|
||||
pid = devices[(k, event.device_id, type)]
|
||||
args = {'name': event.name}
|
||||
if event.memcopy.bytes > 0:
|
||||
args['mem_bytes'] = event.memcopy.bytes
|
||||
if hasattr(event, "detail_info") and event.detail_info:
|
||||
args['detail_info'] = event.detail_info
|
||||
# TODO(panyx0718): Chrome tracing only handles ms. However, some
|
||||
# ops takes micro-seconds. Hence, we keep the ns here.
|
||||
chrome_trace.emit_region(
|
||||
self._align_ts(event.start_ns),
|
||||
(event.end_ns - event.start_ns) / 1.0,
|
||||
pid,
|
||||
event.sub_device_id,
|
||||
'Op',
|
||||
event.name,
|
||||
args,
|
||||
)
|
||||
return chrome_trace
|
||||
|
||||
def _allocate_memory_event(self, profile_dict, mem_devices, gpuId):
|
||||
chrome_trace = ChromeTraceFormatter()
|
||||
if not hasattr(profiler_pb2, "MemEvent"):
|
||||
return
|
||||
place_to_str = {
|
||||
profiler_pb2.MemEvent.CPUPlace: "CPU",
|
||||
profiler_pb2.MemEvent.CUDAPlace: "GPU",
|
||||
profiler_pb2.MemEvent.CUDAPinnedPlace: "CUDAPinnedPlace",
|
||||
}
|
||||
for k, profile_pb in profile_dict.items():
|
||||
rankId = int(k.split(".")[-1])
|
||||
|
||||
trainerId = rankId / self._gpuPerTrainer
|
||||
|
||||
if trainerId >= self._displaySize:
|
||||
continue
|
||||
|
||||
mem_list = []
|
||||
end_profiler = 0
|
||||
for mevent in profile_pb.mem_events:
|
||||
crt_info = {}
|
||||
crt_info['time'] = mevent.start_ns
|
||||
crt_info['size'] = mevent.bytes
|
||||
if mevent.place in place_to_str:
|
||||
place = place_to_str[mevent.place]
|
||||
else:
|
||||
place = "UnDefine"
|
||||
|
||||
if (
|
||||
mevent.place == profiler_pb2.MemEvent.CUDAPlace
|
||||
or mevent.place == profiler_pb2.MemEvent.CUDAPinnedPlace
|
||||
) and mevent.device_id != gpuId:
|
||||
continue
|
||||
|
||||
crt_info['place'] = place
|
||||
pid = mem_devices[(k, mevent.device_id, place)]
|
||||
crt_info['pid'] = pid
|
||||
crt_info['thread_id'] = mevent.thread_id
|
||||
crt_info['device_id'] = mevent.device_id
|
||||
mem_list.append(crt_info)
|
||||
crt_info = {}
|
||||
crt_info['place'] = place
|
||||
crt_info['pid'] = pid
|
||||
crt_info['thread_id'] = mevent.thread_id
|
||||
crt_info['device_id'] = mevent.device_id
|
||||
crt_info['time'] = mevent.end_ns
|
||||
crt_info['size'] = -mevent.bytes
|
||||
mem_list.append(crt_info)
|
||||
end_profiler = max(end_profiler, crt_info['time'])
|
||||
mem_list.sort(key=lambda tmp: tmp.get('time', 0))
|
||||
i = 0
|
||||
total_size = 0
|
||||
while i < len(mem_list):
|
||||
total_size += mem_list[i]['size']
|
||||
while (
|
||||
i < len(mem_list) - 1
|
||||
and mem_list[i]['time'] == mem_list[i + 1]['time']
|
||||
):
|
||||
total_size += mem_list[i + 1]['size']
|
||||
i += 1
|
||||
|
||||
chrome_trace.emit_counter(
|
||||
"Memory",
|
||||
"Memory",
|
||||
mem_list[i]['pid'],
|
||||
self._align_ts(mem_list[i]['time']),
|
||||
0,
|
||||
total_size,
|
||||
)
|
||||
i += 1
|
||||
return chrome_trace
|
||||
|
||||
def _getOPTraceInfoByGpuId(self, groupId, gpuId):
|
||||
fileFist = self.getFileListByGroup(groupId)
|
||||
newFileList = []
|
||||
for file in fileFist:
|
||||
rankId = self.getRankId(file)
|
||||
localRank = rankId % self._gpuPerTrainer
|
||||
if (
|
||||
localRank == gpuId
|
||||
and (rankId / self._gpuPerTrainer) % self._groupSize
|
||||
< self._displaySize
|
||||
):
|
||||
newFileList.append(file)
|
||||
|
||||
profile_dict = self._parseTask(newFileList)
|
||||
initPid = (
|
||||
PIPELINEINFO_TRACE_NUM + DCGMINFO_TRACE_NUM + NETINFO_TRACE_NUM
|
||||
)
|
||||
metaTrace, devicesPid, mem_devicesPid = self._allocate_pids(
|
||||
profile_dict, gpuId, initPid
|
||||
)
|
||||
eventsTrace = self._allocate_events(profile_dict, devicesPid, gpuId)
|
||||
memEventsTrace = self._allocate_memory_event(
|
||||
profile_dict, mem_devicesPid, gpuId
|
||||
)
|
||||
|
||||
trace = {}
|
||||
trace['traceEvents'] = (
|
||||
metaTrace._metadata + eventsTrace._events + memEventsTrace._events
|
||||
)
|
||||
self.dumpOpInfoDict(trace, groupId, gpuId, True)
|
||||
|
||||
return trace
|
||||
|
||||
def getOPTraceInfo(self, groupId):
|
||||
manager = multiprocessing.Manager()
|
||||
q = manager.Queue()
|
||||
processPool = []
|
||||
pidList = []
|
||||
|
||||
for gpuId in range(self._gpuPerTrainer):
|
||||
subproc = Process(
|
||||
target=self._getOPTraceInfoByGpuId,
|
||||
args=(
|
||||
groupId,
|
||||
gpuId,
|
||||
),
|
||||
)
|
||||
processPool.append(subproc)
|
||||
subproc.start()
|
||||
pidList.append(subproc.pid)
|
||||
self._logger.info(
|
||||
f"[op info]: process [{subproc.pid}] has been started, total task num is {1} ..."
|
||||
)
|
||||
|
||||
for t in processPool:
|
||||
t.join()
|
||||
pidList.remove(t.pid)
|
||||
self._logger.info(
|
||||
f"[op info]: process [{t.pid}] has exited! remained {len(pidList)} process!"
|
||||
)
|
||||
|
||||
opInfo = {}
|
||||
|
||||
return opInfo
|
||||
|
||||
def parseFileByGroup(self, groupId, processNum=8):
|
||||
fileFist = self.getFileListByGroup(groupId)
|
||||
return self._parseTask(fileFist)
|
||||
|
||||
|
||||
def test_profileFileReader():
|
||||
args = {
|
||||
"dataPath": "data/newdata/profile",
|
||||
"groupSize": 4,
|
||||
"displaySize": 8,
|
||||
"gpuPerTrainer": 8,
|
||||
"minTimeStamp": 0,
|
||||
"organizeForm": FILEORGANIZEFORM_BYRANK,
|
||||
}
|
||||
|
||||
testReader = profileFileReader(getLogger(), args)
|
||||
testReader.printArgs()
|
||||
data = testReader.getOPTraceInfo(0)
|
||||
|
||||
jsObj = json.dumps(data)
|
||||
fileObject = open('jsonFile.json', 'w')
|
||||
fileObject.write(jsObj)
|
||||
fileObject.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_profileFileReader()
|
||||
Reference in New Issue
Block a user