wehub-resource-sync
148 lines
5.0 KiB
Python
148 lines
5.0 KiB
Python
#
|
|
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
import os
|
|
import sys
|
|
import logging
|
|
import argparse
|
|
|
|
import numpy as np
|
|
import tensorrt as trt
|
|
from cuda.bindings import driver as cuda, runtime as cudart
|
|
|
|
from image_batcher import ImageBatcher
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logging.getLogger("EngineBuilder").setLevel(logging.INFO)
|
|
log = logging.getLogger("EngineBuilder")
|
|
|
|
sys.path.insert(1, os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))
|
|
import common
|
|
|
|
class EngineBuilder:
|
|
"""
|
|
Parses an ONNX graph and builds a TensorRT engine from it.
|
|
"""
|
|
|
|
def __init__(self, verbose=False, workspace=8):
|
|
"""
|
|
:param verbose: If enabled, a higher verbosity level will be set on the TensorRT logger.
|
|
:param workspace: Max memory workspace to allow, in Gb.
|
|
"""
|
|
self.trt_logger = trt.Logger(trt.Logger.INFO)
|
|
if verbose:
|
|
self.trt_logger.min_severity = trt.Logger.Severity.VERBOSE
|
|
|
|
trt.init_libnvinfer_plugins(self.trt_logger, namespace="")
|
|
|
|
self.builder = trt.Builder(self.trt_logger)
|
|
self.config = self.builder.create_builder_config()
|
|
self.config.set_memory_pool_limit(
|
|
trt.MemoryPoolType.WORKSPACE, workspace * (2**30)
|
|
)
|
|
|
|
self.batch_size = None
|
|
self.network = None
|
|
self.parser = None
|
|
|
|
def create_network(self, onnx_path):
|
|
"""
|
|
Parse the ONNX graph and create the corresponding TensorRT network definition.
|
|
:param onnx_path: The path to the ONNX graph to load.
|
|
"""
|
|
self.network = self.builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.STRONGLY_TYPED))
|
|
self.parser = trt.OnnxParser(self.network, self.trt_logger)
|
|
|
|
onnx_path = os.path.realpath(onnx_path)
|
|
with open(onnx_path, "rb") as f:
|
|
if not self.parser.parse(f.read()):
|
|
log.error("Failed to load ONNX file: {}".format(onnx_path))
|
|
for error in range(self.parser.num_errors):
|
|
log.error(self.parser.get_error(error))
|
|
sys.exit(1)
|
|
|
|
inputs = [self.network.get_input(i) for i in range(self.network.num_inputs)]
|
|
outputs = [self.network.get_output(i) for i in range(self.network.num_outputs)]
|
|
|
|
log.info("Network Description")
|
|
for input in inputs:
|
|
self.batch_size = input.shape[0]
|
|
log.info(
|
|
"Input '{}' with shape {} and dtype {}".format(
|
|
input.name, input.shape, input.dtype
|
|
)
|
|
)
|
|
for output in outputs:
|
|
log.info(
|
|
"Output '{}' with shape {} and dtype {}".format(
|
|
output.name, output.shape, output.dtype
|
|
)
|
|
)
|
|
assert self.batch_size > 0
|
|
|
|
def create_engine(
|
|
self,
|
|
engine_path,
|
|
):
|
|
"""
|
|
Build the TensorRT engine and serialize it to disk.
|
|
:param engine_path: The path where to serialize the engine to.
|
|
"""
|
|
engine_path = os.path.realpath(engine_path)
|
|
engine_dir = os.path.dirname(engine_path)
|
|
os.makedirs(engine_dir, exist_ok=True)
|
|
log.info("Building fp32 Engine in {}".format(engine_path))
|
|
|
|
engine_bytes = self.builder.build_serialized_network(self.network, self.config)
|
|
if engine_bytes is None:
|
|
log.error("Failed to create engine")
|
|
sys.exit(1)
|
|
|
|
with open(engine_path, "wb") as f:
|
|
log.info("Serializing engine to file: {:}".format(engine_path))
|
|
f.write(engine_bytes)
|
|
|
|
|
|
def main(args):
|
|
builder = EngineBuilder(args.verbose, args.workspace)
|
|
builder.create_network(args.onnx)
|
|
builder.create_engine(
|
|
args.engine,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("-o", "--onnx", help="The input ONNX model file to load")
|
|
parser.add_argument("-e", "--engine", help="The output path for the TRT engine")
|
|
parser.add_argument(
|
|
"-v", "--verbose", action="store_true", help="Enable more verbose log output"
|
|
)
|
|
parser.add_argument(
|
|
"-w",
|
|
"--workspace",
|
|
default=1,
|
|
type=int,
|
|
help="The max memory workspace size to allow in Gb, " "default: 1",
|
|
)
|
|
args = parser.parse_args()
|
|
if not all([args.onnx, args.engine]):
|
|
parser.print_help()
|
|
log.error("These arguments are required: --onnx and --engine")
|
|
sys.exit(1)
|
|
main(args)
|