chore: import upstream snapshot with attribution
Docker Image CI / build-ubuntu2004 (push) Waiting to run
Docker Image CI / build-ubuntu2004 (push) Waiting to run
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,48 @@
|
||||
# Converting To TensorRT And Running Inference
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
Polygraphy includes a high-level Python API that can convert models
|
||||
and run inference with various backends. For an overview of the Polygraphy
|
||||
Python API, see [here](../../../polygraphy/).
|
||||
|
||||
In this example, we'll look at how you can leverage the API to easily convert an ONNX
|
||||
model to TensorRT and run inference with FP16 precision enabled. We'll then save the
|
||||
engine to a file and see how you can load it again and run inference.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. **[Optional]** Inspect the model before running the example:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.onnx
|
||||
```
|
||||
|
||||
3. Run the script that builds and runs the engine:
|
||||
|
||||
```bash
|
||||
python3 build_and_run.py
|
||||
```
|
||||
|
||||
4. **[Optional]** Inspect the TensorRT engine built by the example:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.engine
|
||||
```
|
||||
|
||||
5. Run the script that loads the previously built engine, then runs it:
|
||||
|
||||
```bash
|
||||
python3 load_and_run.py
|
||||
```
|
||||
|
||||
## Further Reading
|
||||
|
||||
For more details on the Polygraphy Python API, see the
|
||||
[Polygraphy API reference](https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/polygraphy/index.html).
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script builds and runs a TensorRT engine with FP16 precision enabled
|
||||
starting from an ONNX identity model.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.trt import (
|
||||
CreateConfig,
|
||||
EngineFromNetwork,
|
||||
NetworkFromOnnxPath,
|
||||
SaveEngine,
|
||||
TrtRunner,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
# We can compose multiple lazy loaders together to get the desired conversion.
|
||||
# In this case, we want ONNX -> TensorRT Network -> TensorRT engine (w/ fp16).
|
||||
#
|
||||
# NOTE: `build_engine` is a *callable* that returns an engine, not the engine itself.
|
||||
# To get the engine directly, you can use the immediately evaluated functional API.
|
||||
# See examples/api/06_immediate_eval_api for details.
|
||||
build_engine = EngineFromNetwork(
|
||||
NetworkFromOnnxPath("identity.onnx"), config=CreateConfig(fp16=True)
|
||||
) # Note that config is an optional argument.
|
||||
|
||||
# To reuse the engine elsewhere, we can serialize and save it to a file.
|
||||
# The `SaveEngine` lazy loader will return the TensorRT engine when called,
|
||||
# which allows us to chain it together with other loaders.
|
||||
build_engine = SaveEngine(build_engine, path="identity.engine")
|
||||
|
||||
# Once our loader is ready, inference is simply a matter of constructing a runner,
|
||||
# activating it with a context manager (i.e. `with TrtRunner(...)`) and calling `infer()`.
|
||||
#
|
||||
# NOTE: You can use the activate() function instead of a context manager, but you will need to make sure to
|
||||
# deactivate() to avoid a memory leak. For that reason, a context manager is the safer option.
|
||||
with TrtRunner(build_engine) as runner:
|
||||
inp_data = np.ones(shape=(1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict={"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["y"], inp_data) # It's an identity model!
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script loads the TensorRT engine built by `build_and_run.py` and runs inference.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.common import BytesFromPath
|
||||
from polygraphy.backend.trt import EngineFromBytes, TrtRunner
|
||||
|
||||
|
||||
def main():
|
||||
# Just as we did when building, we can compose multiple loaders together
|
||||
# to achieve the behavior we want. Specifically, we want to load a serialized
|
||||
# engine from a file, then deserialize it into a TensorRT engine.
|
||||
load_engine = EngineFromBytes(BytesFromPath("identity.engine"))
|
||||
|
||||
# Inference remains virtually exactly the same as before:
|
||||
with TrtRunner(load_engine) as runner:
|
||||
inp_data = np.ones(shape=(1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict={"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["y"], inp_data) # It's an identity model!
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,45 @@
|
||||
# Comparing Frameworks
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
One of the core features of Polygraphy is comparison of model outputs across multiple
|
||||
different backends. This makes it possible to check the accuracy of one backend with
|
||||
respect to another.
|
||||
|
||||
In this example, we'll look at how you can use the Polygraphy API to run inference
|
||||
with synthetic input data using ONNX-Runtime and TensorRT, and then compare the results
|
||||
using two different comparison methods:
|
||||
|
||||
1. A simple comparison using absolute tolerance
|
||||
2. A more comprehensive comparison using distance metrics (L2 distance, cosine similarity, and PSNR)
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. Run the example
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
3. **[Optional]** Inspect the inference outputs from the example:
|
||||
|
||||
```bash
|
||||
polygraphy inspect data inference_results.json
|
||||
```
|
||||
|
||||
## Comparison Methods
|
||||
|
||||
The example demonstrates two approaches for comparing outputs:
|
||||
|
||||
- **Simple Comparison**: Uses absolute tolerance to determine if outputs match within a specified threshold.
|
||||
- **Distance Metrics**: Performs a more comprehensive comparison using multiple metrics including:
|
||||
- L2 distance (Euclidean distance)
|
||||
- Cosine similarity (measures the angle between vectors)
|
||||
- PSNR (Peak Signal-to-Noise Ratio, useful for comparing image-like data)
|
||||
|
||||
These comparison methods help validate that frameworks produce equivalent results within acceptable margins.
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script runs an identity model with ONNX-Runtime and TensorRT,
|
||||
then compares outputs.
|
||||
"""
|
||||
from polygraphy.backend.onnxrt import OnnxrtRunner, SessionFromOnnx
|
||||
from polygraphy.backend.trt import EngineFromNetwork, NetworkFromOnnxPath, TrtRunner
|
||||
from polygraphy.comparator import Comparator, CompareFunc
|
||||
|
||||
|
||||
def main():
|
||||
# The OnnxrtRunner requires an ONNX-RT session.
|
||||
# We can use the SessionFromOnnx lazy loader to construct one easily:
|
||||
build_onnxrt_session = SessionFromOnnx("identity.onnx")
|
||||
|
||||
# The TrtRunner requires a TensorRT engine.
|
||||
# To create one from the ONNX model, we can chain a couple lazy loaders together:
|
||||
build_engine = EngineFromNetwork(NetworkFromOnnxPath("identity.onnx"))
|
||||
|
||||
runners = [
|
||||
TrtRunner(build_engine),
|
||||
OnnxrtRunner(build_onnxrt_session),
|
||||
]
|
||||
|
||||
# `Comparator.run()` will run each runner separately using synthetic input data and
|
||||
# return a `RunResults` instance. See `polygraphy/comparator/struct.py` for details.
|
||||
#
|
||||
# TIP: To use custom input data, you can set the `data_loader` parameter in `Comparator.run()``
|
||||
# to a generator or iterable that yields `Dict[str, np.ndarray]`.
|
||||
run_results = Comparator.run(runners)
|
||||
|
||||
# `Comparator.compare_accuracy()` checks that outputs match between runners.
|
||||
#
|
||||
# TIP: The `compare_func` parameter can be used to control how outputs are compared (see API reference for details).
|
||||
# The default comparison function is created by `CompareFunc.simple()`, but we can construct it
|
||||
# explicitly if we want to change the default parameters, such as tolerance.
|
||||
assert bool(
|
||||
Comparator.compare_accuracy(
|
||||
run_results, compare_func=CompareFunc.simple(atol=1e-8)
|
||||
)
|
||||
)
|
||||
|
||||
# Use distance metrics comparison for more comprehensive evaluation
|
||||
assert bool(
|
||||
Comparator.compare_accuracy(
|
||||
run_results,
|
||||
compare_func=CompareFunc.distance_metrics(
|
||||
l2_tolerance=1e-5, # Maximum allowed L2 norm (Euclidean distance)
|
||||
cosine_similarity_threshold=0.99, # Minimum cosine similarity (angular similarity)
|
||||
)
|
||||
)
|
||||
)
|
||||
print("All outputs matched using distance metrics (L2 norm, Cosine Similarity)")
|
||||
|
||||
# Use quality metrics for signal quality evaluation
|
||||
assert bool(
|
||||
Comparator.compare_accuracy(
|
||||
run_results,
|
||||
compare_func=CompareFunc.quality_metrics(
|
||||
psnr_tolerance=50.0, # Minimum Peak Signal-to-Noise Ratio in dB
|
||||
snr_tolerance=25.0 # Minimum Signal-to-Noise Ratio in dB
|
||||
)
|
||||
)
|
||||
)
|
||||
print("All outputs matched using quality metrics (PSNR, SNR)")
|
||||
|
||||
# We can use `RunResults.save()` method to save the inference results to a JSON file.
|
||||
# This can be useful if you want to generate and compare results separately.
|
||||
run_results.save("inference_results.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
onnx
|
||||
onnxruntime
|
||||
@@ -0,0 +1,34 @@
|
||||
# Validating Accuracy On A Real Dataset
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `Comparator` provided by Polygraphy can be useful for comparing a small number of
|
||||
results across multiple runners, but is not well suited for validating a single runner
|
||||
with a real dataset that includes labels or golden values - especially if the dataset is large.
|
||||
|
||||
In such cases, it is recommended to use a runner directly instead.
|
||||
|
||||
*NOTE: It is possible to provide custom input data to `Comparator.run()` using the `data_loader`*
|
||||
*parameter. This may be a viable option when using a smaller dataset.*
|
||||
|
||||
In this example, we use a `TrtRunner` directly to validate an identity model on
|
||||
a trivial dataset. Unlike using the `Comparator`, using a runner gives you complete
|
||||
freedom as to how you load your input data, as well as how you validate the results.
|
||||
|
||||
Since all runners provide the same interface, you can freely drop-in other runners
|
||||
without touching the rest of your validation code. For example, in this case, validating
|
||||
the model using ONNX-Runtime would require changing just 2 lines; this is left as an
|
||||
exercise for the reader.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. Run the example
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script uses the Polygraphy Runner API to validate the outputs
|
||||
of an identity model using a trivial dataset.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.trt import EngineFromNetwork, NetworkFromOnnxPath, TrtRunner
|
||||
|
||||
# Pretend that this is a very large dataset.
|
||||
REAL_DATASET = [
|
||||
np.ones((1, 1, 2, 2), dtype=np.float32),
|
||||
np.zeros((1, 1, 2, 2), dtype=np.float32),
|
||||
np.ones((1, 1, 2, 2), dtype=np.float32),
|
||||
np.zeros((1, 1, 2, 2), dtype=np.float32),
|
||||
] # Definitely real data
|
||||
|
||||
# For an identity network, the golden output values are the same as the input values.
|
||||
# Though such a network appears useless at first glance, it can be very useful in some cases (like here!).
|
||||
EXPECTED_OUTPUTS = REAL_DATASET
|
||||
|
||||
|
||||
def main():
|
||||
build_engine = EngineFromNetwork(NetworkFromOnnxPath("identity.onnx"))
|
||||
|
||||
with TrtRunner(build_engine) as runner:
|
||||
for data, golden in zip(REAL_DATASET, EXPECTED_OUTPUTS):
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict={"x": data})
|
||||
|
||||
assert np.array_equal(outputs["y"], golden)
|
||||
|
||||
print("Validation succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,41 @@
|
||||
# Interoperating With TensorRT
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
A key feature of Polygraphy is complete interoperability with TensorRT, as well as
|
||||
with other backends. Since Polygraphy does not hide the underlying backend APIs,
|
||||
it is possible to freely switch between using the Polygraphy API and a backend API,
|
||||
such as TensorRT.
|
||||
|
||||
In this example, we'll look at how you can retain access to the advanced functionality
|
||||
provided by a backend without giving up the conveniences provided by Polygraphy - the
|
||||
best of both worlds.
|
||||
|
||||
Polygraphy provides an `extend` decorator which can be used to easily extend existing
|
||||
Polygraphy loaders. This can be useful in many scenarios, but for this example,
|
||||
we will focus on cases where you may want to:
|
||||
- Modify the TensorRT network prior to building the engine
|
||||
- Use a TensorRT builder flag not currently supported by Polygraphy
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
|
||||
2. **[Optional]** Inspect the TensorRT network generated by `load_network()`.
|
||||
This will invoke `load_network()` from within the script and display the
|
||||
generated TensorRT network, which should be named `"MyIdentity"`:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model example.py --trt-network-func load_network --show layers attrs weights
|
||||
```
|
||||
|
||||
3. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script demonstrates how to use Polygraphy in conjunction with APIs
|
||||
provided by a backend. Specifically, in this case, we use TensorRT APIs
|
||||
to print the network name and enable FP16 mode.
|
||||
"""
|
||||
import numpy as np
|
||||
import tensorrt as trt
|
||||
from polygraphy import func
|
||||
from polygraphy.backend.trt import (
|
||||
CreateConfig,
|
||||
EngineFromNetwork,
|
||||
NetworkFromOnnxPath,
|
||||
TrtRunner,
|
||||
)
|
||||
|
||||
|
||||
# TIP: The immediately evaluated functional API makes it very easy to interoperate
|
||||
# with backends like TensorRT. For details, see example 06 (`examples/api/06_immediate_eval_api`).
|
||||
|
||||
# We can use the `extend` decorator to easily extend lazy loaders provided by Polygraphy
|
||||
# The parameters our decorated function takes should match the return values of the loader we are extending.
|
||||
|
||||
|
||||
# For `NetworkFromOnnxPath`, we can see from the API documentation that it returns a TensorRT
|
||||
# builder, network and parser. That is what our function will receive.
|
||||
@func.extend(NetworkFromOnnxPath("identity.onnx"))
|
||||
def load_network(builder, network, parser):
|
||||
# Here we can modify the network. For this example, we'll just set the network name.
|
||||
network.name = "MyIdentity"
|
||||
print(f"Network name: {network.name}")
|
||||
|
||||
# Notice that we don't need to return anything - `extend()` takes care of that for us!
|
||||
|
||||
|
||||
# In case a builder configuration option is missing from Polygraphy, we can easily set it using TensorRT APIs.
|
||||
# Our function will receive a TensorRT IBuilderConfig since that's what `CreateConfig` returns.
|
||||
@func.extend(CreateConfig())
|
||||
def load_config(config):
|
||||
# Polygraphy supports the fp16 flag, but in case it didn't, we could do this:
|
||||
config.set_flag(trt.BuilderFlag.FP16)
|
||||
|
||||
|
||||
def main():
|
||||
# Since we have no further need of TensorRT APIs, we can come back to regular Polygraphy.
|
||||
#
|
||||
# NOTE: Since we're using lazy loaders, we provide the functions as arguments - we do *not* call them ourselves.
|
||||
build_engine = EngineFromNetwork(load_network, config=load_config)
|
||||
|
||||
with TrtRunner(build_engine) as runner:
|
||||
inp_data = np.ones(shape=(1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer({"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["y"], inp_data) # It's an identity model!
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,46 @@
|
||||
# Int8 Calibration In TensorRT
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
Int8 calibration in TensorRT involves providing a representative set of input data
|
||||
to TensorRT as part of the engine building process. The
|
||||
[calibration API](https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/python-api/infer/Int8/Calibrator.html)
|
||||
included in TensorRT requires the user to handle copying input data to the GPU and
|
||||
manage the calibration cache generated by TensorRT.
|
||||
|
||||
While the TensorRT API provides a higher degree of control, we can greatly simplify the
|
||||
process for many common use-cases. For that purpose, Polygraphy provides a calibrator, which
|
||||
can be used either with Polygraphy or directly with TensorRT. In the latter
|
||||
case, the Polygraphy calibrator behaves exactly like a normal TensorRT int8 calibrator.
|
||||
|
||||
In this example, we'll look at how you can use Polygraphy's calibrator to calibrate a network
|
||||
with (fake) calibration data, and how you can manage the calibration cache with just a single
|
||||
parameter.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
3. The first time you run the example, it will create a calibration cache
|
||||
called `identity-calib.cache`. If you run the example again, you should see that
|
||||
it now uses the cache instead of running calibration again:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
|
||||
## Further Reading
|
||||
|
||||
For more information on how int8 calibration works in TensorRT, see the
|
||||
[developer guide](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#optimizing_int8_c)
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script demonstrates how to use the Calibrator API provided by Polygraphy
|
||||
to calibrate a TensorRT engine to run in INT8 precision.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.trt import (
|
||||
Calibrator,
|
||||
CreateConfig,
|
||||
EngineFromNetwork,
|
||||
NetworkFromOnnxPath,
|
||||
TrtRunner,
|
||||
)
|
||||
from polygraphy.logger import G_LOGGER
|
||||
|
||||
|
||||
# The data loader argument to `Calibrator` can be any iterable or generator that yields `feed_dict`s.
|
||||
# A `feed_dict` is just a mapping of input names to corresponding inputs.
|
||||
def calib_data():
|
||||
for _ in range(4):
|
||||
# TIP: If your calibration data is already on the GPU, you can instead provide GPU pointers
|
||||
# (as `int`s), Polygraphy `DeviceView`s, or PyTorch tensors instead of NumPy arrays.
|
||||
#
|
||||
# For details on `DeviceView`, see `polygraphy/cuda/cuda.py`.
|
||||
yield {"x": np.ones(shape=(1, 1, 2, 2), dtype=np.float32)} # Totally real data
|
||||
|
||||
|
||||
def main():
|
||||
# We can provide a path or file-like object if we want to cache calibration data.
|
||||
# This lets us avoid running calibration the next time we build the engine.
|
||||
#
|
||||
# TIP: You can use this calibrator with TensorRT APIs directly (e.g. config.int8_calibrator).
|
||||
# You don't have to use it with Polygraphy loaders if you don't want to.
|
||||
calibrator = Calibrator(data_loader=calib_data(), cache="identity-calib.cache")
|
||||
|
||||
# We must enable int8 mode in addition to providing the calibrator.
|
||||
build_engine = EngineFromNetwork(
|
||||
NetworkFromOnnxPath("identity.onnx"),
|
||||
config=CreateConfig(int8=True, calibrator=calibrator),
|
||||
)
|
||||
|
||||
# When we activate our runner, it will calibrate and build the engine. If we want to
|
||||
# see the logging output from TensorRT, we can temporarily increase logging verbosity:
|
||||
with G_LOGGER.verbosity(G_LOGGER.VERBOSE), TrtRunner(build_engine) as runner:
|
||||
# Finally, we can test out our int8 TensorRT engine with some dummy input data:
|
||||
inp_data = np.ones(shape=(1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer({"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["y"], inp_data) # It's an identity model!
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,36 @@
|
||||
# Using The TensorRT Network API
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
In addition to loading existing models, TensorRT allows you to define networks by hand
|
||||
using the network API.
|
||||
|
||||
In this example, we'll look at how you can use Polygraphy's `extend` decorator, covered in
|
||||
[example 03](../03_interoperating_with_tensorrt), in conjunction with the `CreateNetwork`
|
||||
loader to seamlessly integrate a network defined using TensorRT APIs with Polygraphy.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. **[Optional]** Inspect the TensorRT network generated by `create_network()`.
|
||||
This will invoke `create_network()` from within the script and display the generated TensorRT network:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model example.py --trt-network-func create_network --show layers attrs weights
|
||||
```
|
||||
|
||||
3. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
## Further Reading
|
||||
|
||||
For more information on the TensorRT Network API, see the
|
||||
[TensorRT API documentation](https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/python-api/infer/Graph/Network.html)
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script demonstrates how to use the extend() API covered in example 03
|
||||
to construct a TensorRT network using the TensorRT Network API.
|
||||
"""
|
||||
import numpy as np
|
||||
import tensorrt as trt
|
||||
from polygraphy import func
|
||||
from polygraphy.backend.trt import CreateNetwork, EngineFromNetwork, TrtRunner
|
||||
|
||||
|
||||
INPUT_NAME = "input"
|
||||
INPUT_SHAPE = (64, 64)
|
||||
OUTPUT_NAME = "output"
|
||||
|
||||
|
||||
# Just like in example 03, we can use `extend` to add our own functionality to existing lazy loaders.
|
||||
# `CreateNetwork` will create an empty network, which we can then populate ourselves.
|
||||
@func.extend(CreateNetwork())
|
||||
def create_network(builder, network):
|
||||
# This network will add 1 to the input tensor.
|
||||
inp = network.add_input(name=INPUT_NAME, shape=INPUT_SHAPE, dtype=trt.float32)
|
||||
ones = network.add_constant(
|
||||
shape=INPUT_SHAPE, weights=np.ones(shape=INPUT_SHAPE, dtype=np.float32)
|
||||
).get_output(0)
|
||||
add = network.add_elementwise(
|
||||
inp, ones, op=trt.ElementWiseOperation.SUM
|
||||
).get_output(0)
|
||||
add.name = OUTPUT_NAME
|
||||
network.mark_output(add)
|
||||
|
||||
# Notice that we don't need to return anything - `extend()` takes care of that for us!
|
||||
|
||||
|
||||
def main():
|
||||
# After we've constructed the network, we can go back to using regular Polygraphy APIs.
|
||||
#
|
||||
# NOTE: Since we're using lazy loaders, we provide the `create_network` function as
|
||||
# an argument - we do *not* call it ourselves.
|
||||
build_engine = EngineFromNetwork(create_network)
|
||||
|
||||
with TrtRunner(build_engine) as runner:
|
||||
feed_dict = {
|
||||
INPUT_NAME: np.random.random_sample(INPUT_SHAPE).astype(np.float32)
|
||||
}
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict)
|
||||
|
||||
assert np.array_equal(outputs[OUTPUT_NAME], (feed_dict[INPUT_NAME] + 1))
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,92 @@
|
||||
# Immediately Evaluated Functional API
|
||||
|
||||
## Introduction
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
Most of the time, the lazy loaders included with Polygraphy have several advantages:
|
||||
|
||||
- They allow us to defer the work until we actually need to do it, which can potentially save
|
||||
time.
|
||||
- Since constructed loaders are extremely light-weight, runners using lazily evaluated loaders can be
|
||||
easily copied into other processes or threads, where they can then be launched.
|
||||
If runners instead referenced entire models/inference sessions, it would be non-trivial to copy them in this way.
|
||||
- They allow us to define a sequence of operations in advance by chaining loaders together, which
|
||||
provides an easy way to build reusable functions.
|
||||
For example, we could create a loader that imports a model from ONNX and generates a serialized TensorRT Engine:
|
||||
|
||||
```python
|
||||
build_engine = EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx"))
|
||||
```
|
||||
|
||||
- They allow for special semantics where if a callable is provided to a loader, it takes ownership
|
||||
of the return value, whereas otherwise it does not. These special semantics are useful for
|
||||
sharing objects between multiple loaders.
|
||||
|
||||
However, this can sometimes lead to code that is less readable, or even downright confusing.
|
||||
For example, consider the following:
|
||||
```python
|
||||
# Each line in this example looks almost the same, but has significantly
|
||||
# different behavior. Some of these lines even cause memory leaks!
|
||||
EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx")) # This is a loader instance, not an engine!
|
||||
EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx"))() # This is an engine.
|
||||
EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx")()) # And it's a loader instance again...
|
||||
EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx")())() # Back to an engine!
|
||||
EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx"))()() # This throws - can you see why?
|
||||
```
|
||||
|
||||
For that reason, Polygraphy provides immediately-evaluated functional
|
||||
equivalents of each loader. Each functional variant uses the same name as the loader, but
|
||||
`snake_case` instead of `PascalCase`. Using the functional variants, loader code like:
|
||||
|
||||
```python
|
||||
parse_network = NetworkFromOnnxPath("/path/to/model.onnx")
|
||||
create_config = CreateConfig(fp16=True, tf32=True)
|
||||
build_engine = EngineFromNetwork(parse_network, create_config)
|
||||
engine = build_engine()
|
||||
```
|
||||
|
||||
becomes:
|
||||
|
||||
```python
|
||||
builder, network, parser = network_from_onnx_path("/path/to/model.onnx")
|
||||
config = create_config(builder, network, fp16=True, tf32=True)
|
||||
engine = engine_from_network((builder, network, parser), config)
|
||||
```
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
|
||||
|
||||
In this example, we'll look at how you can leverage the functional API to convert an ONNX
|
||||
model to a TensorRT network, modify the network, build a TensorRT engine with FP16 precision
|
||||
enabled, and run inference.
|
||||
We'll also save the engine to a file to see how you can load it again and run inference.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. **[Optional]** Inspect the model before running the example:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.onnx
|
||||
```
|
||||
|
||||
3. Run the script that builds and runs the engine:
|
||||
|
||||
```bash
|
||||
python3 build_and_run.py
|
||||
```
|
||||
|
||||
4. **[Optional]** Inspect the TensorRT engine built by the example:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.engine
|
||||
```
|
||||
|
||||
5. Run the script that loads the previously built engine, then runs it:
|
||||
|
||||
```bash
|
||||
python3 load_and_run.py
|
||||
```
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script uses Polygraphy's immediately evaluated functional APIs
|
||||
to load an ONNX model, convert it into a TensorRT network, add an identity
|
||||
layer to the end of it, build an engine with FP16 mode enabled,
|
||||
save the engine, and finally run inference.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.trt import (
|
||||
TrtRunner,
|
||||
create_config,
|
||||
engine_from_network,
|
||||
network_from_onnx_path,
|
||||
save_engine,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
# In Polygraphy, loaders and runners take ownership of objects if they are provided
|
||||
# via the return values of callables. For example, we don't need to worry about object
|
||||
# lifetimes when we use lazy loaders.
|
||||
#
|
||||
# Since we are immediately evaluating, we take ownership of objects, and are responsible for freeing them.
|
||||
builder, network, parser = network_from_onnx_path("identity.onnx")
|
||||
|
||||
# Extend the network with an identity layer (purely for the sake of example).
|
||||
# Note that unlike with lazy loaders, we don't need to do anything special to modify the network.
|
||||
# If we were using lazy loaders, we would need to use `func.extend()` as described
|
||||
# in example 03 and example 05.
|
||||
prev_output = network.get_output(0)
|
||||
network.unmark_output(prev_output)
|
||||
output = network.add_identity(prev_output).get_output(0)
|
||||
output.name = "output"
|
||||
network.mark_output(output)
|
||||
|
||||
# Create a TensorRT IBuilderConfig so that we can build the engine with FP16 enabled.
|
||||
config = create_config(builder, network, fp16=True)
|
||||
|
||||
engine = engine_from_network((builder, network), config)
|
||||
|
||||
# To reuse the engine elsewhere, we can serialize it and save it to a file.
|
||||
save_engine(engine, path="identity.engine")
|
||||
|
||||
with TrtRunner(engine) as runner:
|
||||
inp_data = np.ones((1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict={"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["output"], inp_data) # It's an identity model!
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script uses Polygraphy's immediately evaluated functional APIs
|
||||
to load the TensorRT engine built by `build_and_run.py` and run inference.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.common import bytes_from_path
|
||||
from polygraphy.backend.trt import TrtRunner, engine_from_bytes
|
||||
|
||||
|
||||
def main():
|
||||
engine = engine_from_bytes(bytes_from_path("identity.engine"))
|
||||
|
||||
with TrtRunner(engine) as runner:
|
||||
inp_data = np.ones((1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict={"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["output"], inp_data) # It's an identity model!
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,121 @@
|
||||
# Using Dynamic Shapes With TensorRT
|
||||
|
||||
## Introduction
|
||||
|
||||
*NOTE: This example is intended for use with TensorRT 8.0 or newer.*
|
||||
*Older versions may require slight modifications to the example code.*
|
||||
|
||||
In order to use dynamic input shapes with TensorRT, we have to specify a range
|
||||
(or multiple ranges) of possible shapes when we build the engine.
|
||||
TensorRT optimization profiles provide the means of doing so.
|
||||
|
||||
Using the TensorRT API, the process involves two steps:
|
||||
|
||||
1. During engine building, specify one or more optimization profiles.
|
||||
An optimization profile includes 3 shapes for each input:
|
||||
- `min`: The minimum shape for which the profile should work.
|
||||
- `opt`: The shape which TensorRT should optimize for.
|
||||
Generally, you'd want this to correspond to the most commonly used shape.
|
||||
- `max`: The maximum shape for which the profile should work.
|
||||
|
||||
2. During inference, set the input shape(s) in the execution context, then
|
||||
use the `IOutputAllocator` API to provide a callback to allocate enough
|
||||
device memory for the outputs.
|
||||
|
||||
Polygraphy can simplify both steps and help you avoid common pitfalls:
|
||||
|
||||
1. It provides a `Profile` abstraction, which is an `OrderedDict` that
|
||||
can be converted to a TensorRT `IOptimizationProfile` and includes some utility functions:
|
||||
- `fill_defaults`: Fills the profile with default shapes based on the network.
|
||||
- `to_trt`: Creates a TensorRT `IOptimizationProfile` using the shapes in this `Profile`.
|
||||
|
||||
What's more, `Profile` will automatically handle complexities like the
|
||||
distinction between shape-tensor vs. non-shape-tensor inputs - you do not
|
||||
need to worry about this distinction yourself.
|
||||
|
||||
2. The `TrtRunner` will automatically handle dynamic shapes in the model.
|
||||
As in `Profile`, distinctions between shape-tensor and non-shape-tensor inputs
|
||||
are handled automatically.
|
||||
|
||||
Additionally, the runner will only update the context binding shapes when required,
|
||||
as changing the shapes has a small overhead. The output device buffers will only
|
||||
be resized if their current size is smaller that the context outputs, thus avoiding
|
||||
unnecessary reallocation.
|
||||
|
||||
|
||||
### Setting The Stage
|
||||
|
||||
For the sake of this example, we'll imagine a hypothetical scenario:
|
||||
|
||||
We're running an inference workload using an image classification model.
|
||||
|
||||
Normally, we use this model in an online scenario - i.e. we want the lowest possible
|
||||
latency, so we'll process one image at a time.
|
||||
For this case, assume `batch_size` is `[1]`.
|
||||
|
||||
However, if we have too many users, then we need to employ dynamic batching so that
|
||||
our throughput doesn't suffer. Our range of batch sizes is still small to
|
||||
keep the latency acceptable. Our most frequently used batch size is 4.
|
||||
For this case, assume `batch_size` is in the range `[1, 32]`.
|
||||
|
||||
In even rarer cases, we need to process large amounts of data offline. In this case,
|
||||
we use a very large batch size to improve our throughput.
|
||||
For this case, assume `batch_size` is `[128]`.
|
||||
|
||||
### Performance Considerations
|
||||
|
||||
In implementing our inference pipeline, we need to consider a few tradeoffs:
|
||||
|
||||
- A profile with a large range will not perform as well as for the entire range as
|
||||
multiple profiles each with smaller ranges.
|
||||
- Switching shapes within a profile has a small but non-zero cost.
|
||||
- Switching profiles within a context has a larger cost than switching shapes within a profile.
|
||||
- We can avoid the cost of switching profiles by creating a separate execution context
|
||||
for each profile and selecting the appropriate context at runtime.
|
||||
However, keep in mind that each context will require some additional memory.
|
||||
|
||||
|
||||
### A Possible Solution
|
||||
|
||||
Assuming the image size is `(3, 28, 28)`, we'll create three separate
|
||||
optimization profiles, and a separate context for each:
|
||||
|
||||
1. For the low latency case:
|
||||
`min=(1, 3, 28, 28), opt=(1, 3, 28, 28), max=(1, 3, 28, 28)`
|
||||
|
||||
2. For the dynamic batching case:
|
||||
`min=(1, 3, 28, 28), opt=(4, 3, 28, 28), max=(32, 3, 28, 28)`
|
||||
|
||||
Note that we use a batch size of `4` for `opt` since that's the most common case.
|
||||
|
||||
3. For the offline case:
|
||||
`min=(128, 3, 28, 28), opt=(128, 3, 28, 28), max=(128, 3, 28, 28)`
|
||||
|
||||
For each context, we'll create a corresponding `TrtRunner`. If we make sure that
|
||||
we own the engine and the context (by not providing them via lazy loaders), then
|
||||
the cost of activating a runner should be small - it just needs to allocate
|
||||
input and output buffers. Hence, we'll be able to activate runners on-demand quickly.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
3. **[Optional]** Inspect the generated engine:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model dynamic_identity.engine
|
||||
```
|
||||
|
||||
## Further Reading
|
||||
|
||||
For more information on using dynamic shapes with TensorRT, see the
|
||||
[developer guide](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#work_dynamic_shapes)
|
||||
@@ -0,0 +1,12 @@
|
||||
:[
|
||||
|
||||
XY"Identityonnx_dynamic_identityZ%
|
||||
X
|
||||
|
||||
|
||||
batch_size
|
||||
|
||||
|
||||
b
|
||||
Y
|
||||
B
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script builds an engine with 3 separate optimization profiles, each
|
||||
built for a specific use-case. It then creates 3 separate execution contexts
|
||||
and corresponding `TrtRunner`s for inference.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.trt import (
|
||||
CreateConfig,
|
||||
Profile,
|
||||
TrtRunner,
|
||||
engine_from_network,
|
||||
network_from_onnx_path,
|
||||
save_engine,
|
||||
)
|
||||
from polygraphy.logger import G_LOGGER
|
||||
|
||||
|
||||
def main():
|
||||
# A Profile maps each input tensor to a range of shapes.
|
||||
# The `add()` method can be used to add shapes for a single input.
|
||||
#
|
||||
# TIP: To save lines, calls to `add` can be chained:
|
||||
# profile.add("input0", ...).add("input1", ...)
|
||||
#
|
||||
# Of course, you may alternatively write this as:
|
||||
# profile.add("input0", ...)
|
||||
# profile.add("input1", ...)
|
||||
#
|
||||
profiles = [
|
||||
# The low-latency case. For best performance, min == opt == max.
|
||||
Profile().add("X", min=(1, 3, 28, 28), opt=(1, 3, 28, 28), max=(1, 3, 28, 28)),
|
||||
# The dynamic batching case. We use `4` for the opt batch size since that's our most common case.
|
||||
Profile().add("X", min=(1, 3, 28, 28), opt=(4, 3, 28, 28), max=(32, 3, 28, 28)),
|
||||
# The offline case. For best performance, min == opt == max.
|
||||
Profile().add(
|
||||
"X", min=(128, 3, 28, 28), opt=(128, 3, 28, 28), max=(128, 3, 28, 28)
|
||||
),
|
||||
]
|
||||
|
||||
# See examples/api/06_immediate_eval_api for details on immediately evaluated functional loaders like `engine_from_network`.
|
||||
# Note that we can freely mix lazy and immediately-evaluated loaders.
|
||||
engine = engine_from_network(
|
||||
network_from_onnx_path("dynamic_identity.onnx"),
|
||||
config=CreateConfig(profiles=profiles),
|
||||
)
|
||||
|
||||
# We'll save the engine so that we can inspect it with `inspect model`.
|
||||
# This should make it easy to see how the engine bindings are laid out.
|
||||
save_engine(engine, "dynamic_identity.engine")
|
||||
|
||||
# We'll create, but not activate, three separate runners, each with a separate context.
|
||||
#
|
||||
# TIP: By providing a context directly, as opposed to via a lazy loader,
|
||||
# we can ensure that the runner will *not* take ownership of it.
|
||||
#
|
||||
low_latency = TrtRunner(engine.create_execution_context())
|
||||
|
||||
# NOTE: The following two lines may cause TensorRT to display errors since profile 0
|
||||
# is already in use by the first execution context. We'll suppress them using G_LOGGER.verbosity().
|
||||
#
|
||||
with G_LOGGER.verbosity(G_LOGGER.CRITICAL):
|
||||
# We can use the `optimization_profile` parameter of the runner to ensure that the correct optimization profile is used.
|
||||
# This eliminates the need to call `set_profile()` later.
|
||||
dynamic_batching = TrtRunner(
|
||||
engine.create_execution_context(), optimization_profile=1
|
||||
) # Use the second profile, which is intended for dynamic batching.
|
||||
|
||||
# For the sake of example, we *won't* use `optimization_profile` here.
|
||||
# Instead, we'll use `set_profile()` after activating the runner.
|
||||
offline = TrtRunner(engine.create_execution_context())
|
||||
|
||||
# Finally, we can activate the runners as we need them.
|
||||
#
|
||||
# NOTE: Since the context and engine are already created, the runner will only need to
|
||||
# allocate input and output buffers during activation.
|
||||
|
||||
input_img = np.ones((1, 3, 28, 28), dtype=np.float32) # An input "image"
|
||||
|
||||
with low_latency:
|
||||
outputs = low_latency.infer({"X": input_img})
|
||||
assert np.array_equal(outputs["Y"], input_img) # It's an identity model!
|
||||
|
||||
print("Low latency runner succeeded!")
|
||||
|
||||
# While we're serving requests online, we might decide that we need dynamic batching
|
||||
# for a moment.
|
||||
#
|
||||
# NOTE: We're assuming that activating runners will be cheap here, so we can bring up
|
||||
# the dynamic batching runner just-in-time.
|
||||
#
|
||||
# TIP: If activating the runner is not cheap (e.g. input/output buffers are large),
|
||||
# it might be better to keep the runner active the whole time.
|
||||
#
|
||||
with dynamic_batching:
|
||||
# We'll create fake batches by repeating our fake input image.
|
||||
small_input_batch = np.repeat(input_img, 4, axis=0) # Shape: (4, 3, 28, 28)
|
||||
outputs = dynamic_batching.infer({"X": small_input_batch})
|
||||
assert np.array_equal(outputs["Y"], small_input_batch)
|
||||
|
||||
# If we need dynamic batching again later, we can activate the runner once more.
|
||||
#
|
||||
# NOTE: This time, we do *not* need to set the profile.
|
||||
#
|
||||
with dynamic_batching:
|
||||
# NOTE: We can use any shape that's in the range of the profile without
|
||||
# additional setup - Polygraphy handles the details behind the scenes!
|
||||
#
|
||||
large_input_batch = np.repeat(input_img, 16, axis=0) # Shape: (16, 3, 28, 28)
|
||||
outputs = dynamic_batching.infer({"X": large_input_batch})
|
||||
assert np.array_equal(outputs["Y"], large_input_batch)
|
||||
|
||||
print("Dynamic batching runner succeeded!")
|
||||
|
||||
with offline:
|
||||
# NOTE: When we first activate this runner, we need to set the profile index (it's 0 by default).
|
||||
# Since we provided our own execution context when we created the runner, we need to do this *only once*.
|
||||
# Our settings persist since the context will remain alive even after the runner is deactivated.
|
||||
# If we had instead allowed the runner to own the context, we'd need to repeat this step each time we activated the runner.
|
||||
#
|
||||
# Alternatively, we could have used the `optimization_profile` parameter (see above).
|
||||
#
|
||||
offline.set_profile(
|
||||
2
|
||||
) # Use the third profile, which is intended for the offline case.
|
||||
|
||||
large_offline_batch = np.repeat(
|
||||
input_img, 128, axis=0
|
||||
) # Shape: (128, 3, 28, 28)
|
||||
outputs = offline.infer({"X": large_offline_batch})
|
||||
assert np.array_equal(outputs["Y"], large_offline_batch)
|
||||
|
||||
print("Offline runner succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
# Working With Run Results And Saved Inputs Manually
|
||||
|
||||
## Introduction
|
||||
|
||||
Inference inputs and outputs from `Comparator.run` can be serialized and saved to JSON
|
||||
files so they can be reused. Inputs are stored as `List[Dict[str, np.ndarray]]` while outputs
|
||||
are stored in a `RunResults` object, which can keep track of the outputs of multiple runners
|
||||
from multiple inference iterations.
|
||||
|
||||
Command-line tools providing `--save-inputs` and `--save-outputs` options generally use these formats.
|
||||
|
||||
Usually, you'll only use saved inputs or `RunResults` with other Polygraphy APIs or
|
||||
tools (as in [this example](../../cli//run/06_comparing_with_custom_output_data/)
|
||||
or [this one](../../cli/inspect/05_inspecting_inference_outputs/)), but sometimes,
|
||||
you may want to work with the underlying NumPy arrays manually.
|
||||
|
||||
Polygraphy includes convenience APIs that make it easy to load and manipulate these objects.
|
||||
|
||||
This example illustrates how you can load saved inputs and/or `RunResults` from a file
|
||||
using the Python API and then access the NumPy arrays stored within.
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Generate some inference inputs and outputs:
|
||||
|
||||
```bash
|
||||
polygraphy run identity.onnx --trt --onnxrt \
|
||||
--save-inputs inputs.json --save-outputs outputs.json
|
||||
```
|
||||
|
||||
2. **[Optional]** Use `inspect data` to view the inputs on the command-line:
|
||||
|
||||
```bash
|
||||
polygraphy inspect data inputs.json --show-values
|
||||
```
|
||||
|
||||
3. **[Optional]** Use `inspect data` to view the outputs on the command-line:
|
||||
|
||||
```bash
|
||||
polygraphy inspect data outputs.json --show-values
|
||||
```
|
||||
|
||||
4. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""
|
||||
This script demonstrates how to use the `load_json` and `RunResults` APIs to load
|
||||
and manipulate inference inputs and outputs respectively.
|
||||
"""
|
||||
|
||||
from polygraphy.comparator import RunResults
|
||||
from polygraphy.json import load_json
|
||||
|
||||
|
||||
def main():
|
||||
# Use the `load_json` API to load inputs from file.
|
||||
#
|
||||
# NOTE: The `save_json` and `load_json` standalone helpers should be used only with non-Polygraphy objects.
|
||||
# Polygraphy objects that support serialization include `save` and `load` methods.
|
||||
inputs = load_json("inputs.json")
|
||||
|
||||
# Inputs are stored as a `List[Dict[str, np.ndarray]]`, i.e. a list of feed_dicts,
|
||||
# where each feed_dict maps input names to NumPy arrays.
|
||||
#
|
||||
# TIP: In the typical case, we'll only have one iteration, so we'll only look at the first item.
|
||||
# If you need to access inputs from multiple iterations, you can do something like this instead:
|
||||
#
|
||||
# for feed_dict in inputs:
|
||||
# for name, array in feed_dict.items():
|
||||
# ... # Do something with the inputs here
|
||||
#
|
||||
[feed_dict] = inputs
|
||||
for name, array in feed_dict.items():
|
||||
print(f"Input: '{name}' | Values:\n{array}")
|
||||
|
||||
# Use the `RunResults.load` API to load results from file.
|
||||
#
|
||||
# TIP: You can provide either a file path or a file-like object here.
|
||||
results = RunResults.load("outputs.json")
|
||||
|
||||
# The `RunResults` object is structured like a `Dict[str, List[IterationResult]]``,
|
||||
# mapping runner names to inference outputs from one or more iterations.
|
||||
# An `IterationResult` behaves just like a `Dict[str, np.ndarray]` mapping output names
|
||||
# to NumPy arrays.
|
||||
#
|
||||
# TIP: In the typical case, we'll only have one iteration, so we can unpack it
|
||||
# directly in the loop. If you need to access outputs from multiple iterations,
|
||||
# you can do something like this instead:
|
||||
#
|
||||
# for runner_name, iters in results.items():
|
||||
# for outputs in iters:
|
||||
# ... # Do something with the outputs here
|
||||
#
|
||||
for runner_name, [outputs] in results.items():
|
||||
print(f"\nProcessing outputs for runner: {runner_name}")
|
||||
# Now you can read or modify the outputs for each runner.
|
||||
# For the sake of this example, we'll just print them:
|
||||
for name, array in outputs.items():
|
||||
print(f"Output: '{name}' | Values:\n{array}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# Working With PyTorch Tensors
|
||||
|
||||
## Introduction
|
||||
|
||||
Some runners like `OnnxrtRunner` and `TrtRunner` can accept and return PyTorch tensors
|
||||
in addition to NumPy arrays. When PyTorch tensors are provided in the inputs, the runner
|
||||
will return the outputs as PyTorch tensors as well. This can be especially useful in
|
||||
cases where PyTorch supports a data type that is not supported by NumPy, such as BFloat16.
|
||||
|
||||
Polygraphy's included TensorRT `Calibrator` can also accept PyTorch tensors directly.
|
||||
|
||||
This example uses PyTorch tensors on the GPU where possible (i.e. if a GPU-enabled version
|
||||
of PyTorch is installed). When the tensors already reside on GPU memory, no additional copies
|
||||
are required in the runner/calibrator.
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
|
||||
2. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
|
||||
## See Also
|
||||
|
||||
* [Inference With TensorRT](../00_inference_with_tensorrt/)
|
||||
* [INT8 Calibration In TensorRT](../04_int8_calibration_in_tensorrt/)
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""
|
||||
This script demonstrates how to use PyTorch tensors with the TensorRT runner and calibrator.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from polygraphy.backend.trt import (
|
||||
Calibrator,
|
||||
CreateConfig,
|
||||
TrtRunner,
|
||||
engine_from_network,
|
||||
network_from_onnx_path,
|
||||
)
|
||||
|
||||
# If your PyTorch installation has GPU support, then we'll allocate the tensors
|
||||
# directly in GPU memory. This will mean that the calibrator and runner can skip the
|
||||
# host-to-device copy we would otherwise incur with NumPy arrays.
|
||||
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
|
||||
def calib_data():
|
||||
for _ in range(4):
|
||||
yield {"x": torch.ones((1, 1, 2, 2), dtype=torch.float32, device=DEVICE)}
|
||||
|
||||
|
||||
def main():
|
||||
calibrator = Calibrator(data_loader=calib_data())
|
||||
|
||||
engine = engine_from_network(
|
||||
network_from_onnx_path("identity.onnx"),
|
||||
config=CreateConfig(int8=True, calibrator=calibrator),
|
||||
)
|
||||
|
||||
with TrtRunner(engine) as runner:
|
||||
inp_data = torch.ones((1, 1, 2, 2), dtype=torch.float32, device=DEVICE)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
#
|
||||
# When you provide PyTorch tensors in the feed_dict, the runner will try to use
|
||||
# PyTorch tensors for the outputs. Specifically:
|
||||
# - If the `copy_outputs_to_host` argument to `infer()` is set to `True` (the default),
|
||||
# it will return PyTorch tensors in CPU memory.
|
||||
# - If `copy_outputs_to_host` is `False`, it will return:
|
||||
# - PyTorch tensors in GPU memory if you have a GPU-enabled PyTorch installation.
|
||||
# - Polygraphy `DeviceView`s otherwise.
|
||||
#
|
||||
outputs = runner.infer({"x": inp_data})
|
||||
|
||||
# `copy_outputs_to_host` defaults to True, so the outputs should be PyTorch
|
||||
# tensors in CPU memory.
|
||||
assert isinstance(outputs["y"], torch.Tensor)
|
||||
assert outputs["y"].device.type == "cpu"
|
||||
|
||||
assert torch.equal(outputs["y"], inp_data.to("cpu")) # It's an identity model!
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
tensorrt>=8.5
|
||||
torch>=1.13.0
|
||||
@@ -0,0 +1,64 @@
|
||||
# Polygraphy Python API Examples
|
||||
|
||||
This directory includes examples that use the Polygraphy Python API.
|
||||
For examples of the command-line tools, see the [cli](../cli/) directory instead.
|
||||
|
||||
You may find it useful to read the [Python API Overview](../../polygraphy/) prior
|
||||
to looking at the API examples.
|
||||
|
||||
## Generating Your Own Examples
|
||||
|
||||
In the event that the examples here do not cover a particular use-case, you can typically
|
||||
use `polygraphy run` to fill the gap; `polygraphy run` is capable of dynamically generating
|
||||
Python scripts that use the Polygraphy API that do exactly what the tool would otherwise do.
|
||||
|
||||
Thus, if `polygraphy run` includes functionality you need, but you cannot find a
|
||||
corresponding API example, try running:
|
||||
```bash
|
||||
polygraphy run --gen - <options...>
|
||||
```
|
||||
The argument to `--gen` should be the name of the file in which to write the generated script.
|
||||
The special value `-` corresponds to `stdout`.
|
||||
|
||||
For example, running:
|
||||
```bash
|
||||
polygraphy run --gen - model.onnx --trt --onnxrt
|
||||
```
|
||||
will display something like this on `stdout`:
|
||||
```py
|
||||
#!/usr/bin/env python3
|
||||
# Template auto-generated by polygraphy [v0.31.0] on 01/01/20 at 10:10:10
|
||||
# Generation Command: polygraphy run --gen - model.onnx --trt --onnxrt
|
||||
# This script compares model.onnx between TensorRT and ONNX-Runtime
|
||||
|
||||
from polygraphy.logger import G_LOGGER
|
||||
|
||||
from polygraphy.backend.onnxrt import OnnxrtRunner, SessionFromOnnx
|
||||
from polygraphy.backend.trt import EngineFromNetwork, NetworkFromOnnxPath, TrtRunner
|
||||
from polygraphy.comparator import Comparator
|
||||
import sys
|
||||
|
||||
# Loaders
|
||||
parse_network_from_onnx = NetworkFromOnnxPath('model.onnx')
|
||||
build_engine = EngineFromNetwork(parse_network_from_onnx)
|
||||
build_onnxrt_session = SessionFromOnnx('model.onnx')
|
||||
|
||||
# Runners
|
||||
runners = [
|
||||
TrtRunner(build_engine),
|
||||
OnnxrtRunner(build_onnxrt_session),
|
||||
]
|
||||
|
||||
# Runner Execution
|
||||
results = Comparator.run(runners)
|
||||
|
||||
success = True
|
||||
# Accuracy Comparison
|
||||
success &= bool(Comparator.compare_accuracy(results))
|
||||
|
||||
# Report Results
|
||||
cmd_run = ' '.join(sys.argv)
|
||||
if not success:
|
||||
G_LOGGER.critical(f"FAILED | Command: {cmd_run}"))
|
||||
G_LOGGER.finish(f"PASSED | Command: {cmd_run}"))
|
||||
```
|
||||
Reference in New Issue
Block a user