chore: import upstream snapshot with attribution
Docker Image CI / build-ubuntu2004 (push) Has been cancelled
Docker Image CI / build-ubuntu2004 (push) Has been cancelled
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,5 @@
|
||||
# Examples
|
||||
|
||||
This directory includes various examples covering the Polygraphy [CLI](./cli), [Python API](./api), and [development practices](./dev).
|
||||
|
||||
The paths used in each example assume that the example is being run from within that example's directory.
|
||||
@@ -0,0 +1,48 @@
|
||||
# Converting To TensorRT And Running Inference
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
Polygraphy includes a high-level Python API that can convert models
|
||||
and run inference with various backends. For an overview of the Polygraphy
|
||||
Python API, see [here](../../../polygraphy/).
|
||||
|
||||
In this example, we'll look at how you can leverage the API to easily convert an ONNX
|
||||
model to TensorRT and run inference with FP16 precision enabled. We'll then save the
|
||||
engine to a file and see how you can load it again and run inference.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. **[Optional]** Inspect the model before running the example:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.onnx
|
||||
```
|
||||
|
||||
3. Run the script that builds and runs the engine:
|
||||
|
||||
```bash
|
||||
python3 build_and_run.py
|
||||
```
|
||||
|
||||
4. **[Optional]** Inspect the TensorRT engine built by the example:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.engine
|
||||
```
|
||||
|
||||
5. Run the script that loads the previously built engine, then runs it:
|
||||
|
||||
```bash
|
||||
python3 load_and_run.py
|
||||
```
|
||||
|
||||
## Further Reading
|
||||
|
||||
For more details on the Polygraphy Python API, see the
|
||||
[Polygraphy API reference](https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/polygraphy/index.html).
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script builds and runs a TensorRT engine with FP16 precision enabled
|
||||
starting from an ONNX identity model.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.trt import (
|
||||
CreateConfig,
|
||||
EngineFromNetwork,
|
||||
NetworkFromOnnxPath,
|
||||
SaveEngine,
|
||||
TrtRunner,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
# We can compose multiple lazy loaders together to get the desired conversion.
|
||||
# In this case, we want ONNX -> TensorRT Network -> TensorRT engine (w/ fp16).
|
||||
#
|
||||
# NOTE: `build_engine` is a *callable* that returns an engine, not the engine itself.
|
||||
# To get the engine directly, you can use the immediately evaluated functional API.
|
||||
# See examples/api/06_immediate_eval_api for details.
|
||||
build_engine = EngineFromNetwork(
|
||||
NetworkFromOnnxPath("identity.onnx"), config=CreateConfig(fp16=True)
|
||||
) # Note that config is an optional argument.
|
||||
|
||||
# To reuse the engine elsewhere, we can serialize and save it to a file.
|
||||
# The `SaveEngine` lazy loader will return the TensorRT engine when called,
|
||||
# which allows us to chain it together with other loaders.
|
||||
build_engine = SaveEngine(build_engine, path="identity.engine")
|
||||
|
||||
# Once our loader is ready, inference is simply a matter of constructing a runner,
|
||||
# activating it with a context manager (i.e. `with TrtRunner(...)`) and calling `infer()`.
|
||||
#
|
||||
# NOTE: You can use the activate() function instead of a context manager, but you will need to make sure to
|
||||
# deactivate() to avoid a memory leak. For that reason, a context manager is the safer option.
|
||||
with TrtRunner(build_engine) as runner:
|
||||
inp_data = np.ones(shape=(1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict={"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["y"], inp_data) # It's an identity model!
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script loads the TensorRT engine built by `build_and_run.py` and runs inference.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.common import BytesFromPath
|
||||
from polygraphy.backend.trt import EngineFromBytes, TrtRunner
|
||||
|
||||
|
||||
def main():
|
||||
# Just as we did when building, we can compose multiple loaders together
|
||||
# to achieve the behavior we want. Specifically, we want to load a serialized
|
||||
# engine from a file, then deserialize it into a TensorRT engine.
|
||||
load_engine = EngineFromBytes(BytesFromPath("identity.engine"))
|
||||
|
||||
# Inference remains virtually exactly the same as before:
|
||||
with TrtRunner(load_engine) as runner:
|
||||
inp_data = np.ones(shape=(1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict={"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["y"], inp_data) # It's an identity model!
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,45 @@
|
||||
# Comparing Frameworks
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
One of the core features of Polygraphy is comparison of model outputs across multiple
|
||||
different backends. This makes it possible to check the accuracy of one backend with
|
||||
respect to another.
|
||||
|
||||
In this example, we'll look at how you can use the Polygraphy API to run inference
|
||||
with synthetic input data using ONNX-Runtime and TensorRT, and then compare the results
|
||||
using two different comparison methods:
|
||||
|
||||
1. A simple comparison using absolute tolerance
|
||||
2. A more comprehensive comparison using distance metrics (L2 distance, cosine similarity, and PSNR)
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. Run the example
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
3. **[Optional]** Inspect the inference outputs from the example:
|
||||
|
||||
```bash
|
||||
polygraphy inspect data inference_results.json
|
||||
```
|
||||
|
||||
## Comparison Methods
|
||||
|
||||
The example demonstrates two approaches for comparing outputs:
|
||||
|
||||
- **Simple Comparison**: Uses absolute tolerance to determine if outputs match within a specified threshold.
|
||||
- **Distance Metrics**: Performs a more comprehensive comparison using multiple metrics including:
|
||||
- L2 distance (Euclidean distance)
|
||||
- Cosine similarity (measures the angle between vectors)
|
||||
- PSNR (Peak Signal-to-Noise Ratio, useful for comparing image-like data)
|
||||
|
||||
These comparison methods help validate that frameworks produce equivalent results within acceptable margins.
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script runs an identity model with ONNX-Runtime and TensorRT,
|
||||
then compares outputs.
|
||||
"""
|
||||
from polygraphy.backend.onnxrt import OnnxrtRunner, SessionFromOnnx
|
||||
from polygraphy.backend.trt import EngineFromNetwork, NetworkFromOnnxPath, TrtRunner
|
||||
from polygraphy.comparator import Comparator, CompareFunc
|
||||
|
||||
|
||||
def main():
|
||||
# The OnnxrtRunner requires an ONNX-RT session.
|
||||
# We can use the SessionFromOnnx lazy loader to construct one easily:
|
||||
build_onnxrt_session = SessionFromOnnx("identity.onnx")
|
||||
|
||||
# The TrtRunner requires a TensorRT engine.
|
||||
# To create one from the ONNX model, we can chain a couple lazy loaders together:
|
||||
build_engine = EngineFromNetwork(NetworkFromOnnxPath("identity.onnx"))
|
||||
|
||||
runners = [
|
||||
TrtRunner(build_engine),
|
||||
OnnxrtRunner(build_onnxrt_session),
|
||||
]
|
||||
|
||||
# `Comparator.run()` will run each runner separately using synthetic input data and
|
||||
# return a `RunResults` instance. See `polygraphy/comparator/struct.py` for details.
|
||||
#
|
||||
# TIP: To use custom input data, you can set the `data_loader` parameter in `Comparator.run()``
|
||||
# to a generator or iterable that yields `Dict[str, np.ndarray]`.
|
||||
run_results = Comparator.run(runners)
|
||||
|
||||
# `Comparator.compare_accuracy()` checks that outputs match between runners.
|
||||
#
|
||||
# TIP: The `compare_func` parameter can be used to control how outputs are compared (see API reference for details).
|
||||
# The default comparison function is created by `CompareFunc.simple()`, but we can construct it
|
||||
# explicitly if we want to change the default parameters, such as tolerance.
|
||||
assert bool(
|
||||
Comparator.compare_accuracy(
|
||||
run_results, compare_func=CompareFunc.simple(atol=1e-8)
|
||||
)
|
||||
)
|
||||
|
||||
# Use distance metrics comparison for more comprehensive evaluation
|
||||
assert bool(
|
||||
Comparator.compare_accuracy(
|
||||
run_results,
|
||||
compare_func=CompareFunc.distance_metrics(
|
||||
l2_tolerance=1e-5, # Maximum allowed L2 norm (Euclidean distance)
|
||||
cosine_similarity_threshold=0.99, # Minimum cosine similarity (angular similarity)
|
||||
)
|
||||
)
|
||||
)
|
||||
print("All outputs matched using distance metrics (L2 norm, Cosine Similarity)")
|
||||
|
||||
# Use quality metrics for signal quality evaluation
|
||||
assert bool(
|
||||
Comparator.compare_accuracy(
|
||||
run_results,
|
||||
compare_func=CompareFunc.quality_metrics(
|
||||
psnr_tolerance=50.0, # Minimum Peak Signal-to-Noise Ratio in dB
|
||||
snr_tolerance=25.0 # Minimum Signal-to-Noise Ratio in dB
|
||||
)
|
||||
)
|
||||
)
|
||||
print("All outputs matched using quality metrics (PSNR, SNR)")
|
||||
|
||||
# We can use `RunResults.save()` method to save the inference results to a JSON file.
|
||||
# This can be useful if you want to generate and compare results separately.
|
||||
run_results.save("inference_results.json")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
onnx
|
||||
onnxruntime
|
||||
@@ -0,0 +1,34 @@
|
||||
# Validating Accuracy On A Real Dataset
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `Comparator` provided by Polygraphy can be useful for comparing a small number of
|
||||
results across multiple runners, but is not well suited for validating a single runner
|
||||
with a real dataset that includes labels or golden values - especially if the dataset is large.
|
||||
|
||||
In such cases, it is recommended to use a runner directly instead.
|
||||
|
||||
*NOTE: It is possible to provide custom input data to `Comparator.run()` using the `data_loader`*
|
||||
*parameter. This may be a viable option when using a smaller dataset.*
|
||||
|
||||
In this example, we use a `TrtRunner` directly to validate an identity model on
|
||||
a trivial dataset. Unlike using the `Comparator`, using a runner gives you complete
|
||||
freedom as to how you load your input data, as well as how you validate the results.
|
||||
|
||||
Since all runners provide the same interface, you can freely drop-in other runners
|
||||
without touching the rest of your validation code. For example, in this case, validating
|
||||
the model using ONNX-Runtime would require changing just 2 lines; this is left as an
|
||||
exercise for the reader.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. Run the example
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script uses the Polygraphy Runner API to validate the outputs
|
||||
of an identity model using a trivial dataset.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.trt import EngineFromNetwork, NetworkFromOnnxPath, TrtRunner
|
||||
|
||||
# Pretend that this is a very large dataset.
|
||||
REAL_DATASET = [
|
||||
np.ones((1, 1, 2, 2), dtype=np.float32),
|
||||
np.zeros((1, 1, 2, 2), dtype=np.float32),
|
||||
np.ones((1, 1, 2, 2), dtype=np.float32),
|
||||
np.zeros((1, 1, 2, 2), dtype=np.float32),
|
||||
] # Definitely real data
|
||||
|
||||
# For an identity network, the golden output values are the same as the input values.
|
||||
# Though such a network appears useless at first glance, it can be very useful in some cases (like here!).
|
||||
EXPECTED_OUTPUTS = REAL_DATASET
|
||||
|
||||
|
||||
def main():
|
||||
build_engine = EngineFromNetwork(NetworkFromOnnxPath("identity.onnx"))
|
||||
|
||||
with TrtRunner(build_engine) as runner:
|
||||
for data, golden in zip(REAL_DATASET, EXPECTED_OUTPUTS):
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict={"x": data})
|
||||
|
||||
assert np.array_equal(outputs["y"], golden)
|
||||
|
||||
print("Validation succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,41 @@
|
||||
# Interoperating With TensorRT
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
A key feature of Polygraphy is complete interoperability with TensorRT, as well as
|
||||
with other backends. Since Polygraphy does not hide the underlying backend APIs,
|
||||
it is possible to freely switch between using the Polygraphy API and a backend API,
|
||||
such as TensorRT.
|
||||
|
||||
In this example, we'll look at how you can retain access to the advanced functionality
|
||||
provided by a backend without giving up the conveniences provided by Polygraphy - the
|
||||
best of both worlds.
|
||||
|
||||
Polygraphy provides an `extend` decorator which can be used to easily extend existing
|
||||
Polygraphy loaders. This can be useful in many scenarios, but for this example,
|
||||
we will focus on cases where you may want to:
|
||||
- Modify the TensorRT network prior to building the engine
|
||||
- Use a TensorRT builder flag not currently supported by Polygraphy
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
|
||||
2. **[Optional]** Inspect the TensorRT network generated by `load_network()`.
|
||||
This will invoke `load_network()` from within the script and display the
|
||||
generated TensorRT network, which should be named `"MyIdentity"`:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model example.py --trt-network-func load_network --show layers attrs weights
|
||||
```
|
||||
|
||||
3. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script demonstrates how to use Polygraphy in conjunction with APIs
|
||||
provided by a backend. Specifically, in this case, we use TensorRT APIs
|
||||
to print the network name and enable FP16 mode.
|
||||
"""
|
||||
import numpy as np
|
||||
import tensorrt as trt
|
||||
from polygraphy import func
|
||||
from polygraphy.backend.trt import (
|
||||
CreateConfig,
|
||||
EngineFromNetwork,
|
||||
NetworkFromOnnxPath,
|
||||
TrtRunner,
|
||||
)
|
||||
|
||||
|
||||
# TIP: The immediately evaluated functional API makes it very easy to interoperate
|
||||
# with backends like TensorRT. For details, see example 06 (`examples/api/06_immediate_eval_api`).
|
||||
|
||||
# We can use the `extend` decorator to easily extend lazy loaders provided by Polygraphy
|
||||
# The parameters our decorated function takes should match the return values of the loader we are extending.
|
||||
|
||||
|
||||
# For `NetworkFromOnnxPath`, we can see from the API documentation that it returns a TensorRT
|
||||
# builder, network and parser. That is what our function will receive.
|
||||
@func.extend(NetworkFromOnnxPath("identity.onnx"))
|
||||
def load_network(builder, network, parser):
|
||||
# Here we can modify the network. For this example, we'll just set the network name.
|
||||
network.name = "MyIdentity"
|
||||
print(f"Network name: {network.name}")
|
||||
|
||||
# Notice that we don't need to return anything - `extend()` takes care of that for us!
|
||||
|
||||
|
||||
# In case a builder configuration option is missing from Polygraphy, we can easily set it using TensorRT APIs.
|
||||
# Our function will receive a TensorRT IBuilderConfig since that's what `CreateConfig` returns.
|
||||
@func.extend(CreateConfig())
|
||||
def load_config(config):
|
||||
# Polygraphy supports the fp16 flag, but in case it didn't, we could do this:
|
||||
config.set_flag(trt.BuilderFlag.FP16)
|
||||
|
||||
|
||||
def main():
|
||||
# Since we have no further need of TensorRT APIs, we can come back to regular Polygraphy.
|
||||
#
|
||||
# NOTE: Since we're using lazy loaders, we provide the functions as arguments - we do *not* call them ourselves.
|
||||
build_engine = EngineFromNetwork(load_network, config=load_config)
|
||||
|
||||
with TrtRunner(build_engine) as runner:
|
||||
inp_data = np.ones(shape=(1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer({"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["y"], inp_data) # It's an identity model!
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,46 @@
|
||||
# Int8 Calibration In TensorRT
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
Int8 calibration in TensorRT involves providing a representative set of input data
|
||||
to TensorRT as part of the engine building process. The
|
||||
[calibration API](https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/python-api/infer/Int8/Calibrator.html)
|
||||
included in TensorRT requires the user to handle copying input data to the GPU and
|
||||
manage the calibration cache generated by TensorRT.
|
||||
|
||||
While the TensorRT API provides a higher degree of control, we can greatly simplify the
|
||||
process for many common use-cases. For that purpose, Polygraphy provides a calibrator, which
|
||||
can be used either with Polygraphy or directly with TensorRT. In the latter
|
||||
case, the Polygraphy calibrator behaves exactly like a normal TensorRT int8 calibrator.
|
||||
|
||||
In this example, we'll look at how you can use Polygraphy's calibrator to calibrate a network
|
||||
with (fake) calibration data, and how you can manage the calibration cache with just a single
|
||||
parameter.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
3. The first time you run the example, it will create a calibration cache
|
||||
called `identity-calib.cache`. If you run the example again, you should see that
|
||||
it now uses the cache instead of running calibration again:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
|
||||
## Further Reading
|
||||
|
||||
For more information on how int8 calibration works in TensorRT, see the
|
||||
[developer guide](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#optimizing_int8_c)
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script demonstrates how to use the Calibrator API provided by Polygraphy
|
||||
to calibrate a TensorRT engine to run in INT8 precision.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.trt import (
|
||||
Calibrator,
|
||||
CreateConfig,
|
||||
EngineFromNetwork,
|
||||
NetworkFromOnnxPath,
|
||||
TrtRunner,
|
||||
)
|
||||
from polygraphy.logger import G_LOGGER
|
||||
|
||||
|
||||
# The data loader argument to `Calibrator` can be any iterable or generator that yields `feed_dict`s.
|
||||
# A `feed_dict` is just a mapping of input names to corresponding inputs.
|
||||
def calib_data():
|
||||
for _ in range(4):
|
||||
# TIP: If your calibration data is already on the GPU, you can instead provide GPU pointers
|
||||
# (as `int`s), Polygraphy `DeviceView`s, or PyTorch tensors instead of NumPy arrays.
|
||||
#
|
||||
# For details on `DeviceView`, see `polygraphy/cuda/cuda.py`.
|
||||
yield {"x": np.ones(shape=(1, 1, 2, 2), dtype=np.float32)} # Totally real data
|
||||
|
||||
|
||||
def main():
|
||||
# We can provide a path or file-like object if we want to cache calibration data.
|
||||
# This lets us avoid running calibration the next time we build the engine.
|
||||
#
|
||||
# TIP: You can use this calibrator with TensorRT APIs directly (e.g. config.int8_calibrator).
|
||||
# You don't have to use it with Polygraphy loaders if you don't want to.
|
||||
calibrator = Calibrator(data_loader=calib_data(), cache="identity-calib.cache")
|
||||
|
||||
# We must enable int8 mode in addition to providing the calibrator.
|
||||
build_engine = EngineFromNetwork(
|
||||
NetworkFromOnnxPath("identity.onnx"),
|
||||
config=CreateConfig(int8=True, calibrator=calibrator),
|
||||
)
|
||||
|
||||
# When we activate our runner, it will calibrate and build the engine. If we want to
|
||||
# see the logging output from TensorRT, we can temporarily increase logging verbosity:
|
||||
with G_LOGGER.verbosity(G_LOGGER.VERBOSE), TrtRunner(build_engine) as runner:
|
||||
# Finally, we can test out our int8 TensorRT engine with some dummy input data:
|
||||
inp_data = np.ones(shape=(1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer({"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["y"], inp_data) # It's an identity model!
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,36 @@
|
||||
# Using The TensorRT Network API
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
In addition to loading existing models, TensorRT allows you to define networks by hand
|
||||
using the network API.
|
||||
|
||||
In this example, we'll look at how you can use Polygraphy's `extend` decorator, covered in
|
||||
[example 03](../03_interoperating_with_tensorrt), in conjunction with the `CreateNetwork`
|
||||
loader to seamlessly integrate a network defined using TensorRT APIs with Polygraphy.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. **[Optional]** Inspect the TensorRT network generated by `create_network()`.
|
||||
This will invoke `create_network()` from within the script and display the generated TensorRT network:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model example.py --trt-network-func create_network --show layers attrs weights
|
||||
```
|
||||
|
||||
3. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
## Further Reading
|
||||
|
||||
For more information on the TensorRT Network API, see the
|
||||
[TensorRT API documentation](https://docs.nvidia.com/deeplearning/tensorrt/latest/_static/python-api/infer/Graph/Network.html)
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script demonstrates how to use the extend() API covered in example 03
|
||||
to construct a TensorRT network using the TensorRT Network API.
|
||||
"""
|
||||
import numpy as np
|
||||
import tensorrt as trt
|
||||
from polygraphy import func
|
||||
from polygraphy.backend.trt import CreateNetwork, EngineFromNetwork, TrtRunner
|
||||
|
||||
|
||||
INPUT_NAME = "input"
|
||||
INPUT_SHAPE = (64, 64)
|
||||
OUTPUT_NAME = "output"
|
||||
|
||||
|
||||
# Just like in example 03, we can use `extend` to add our own functionality to existing lazy loaders.
|
||||
# `CreateNetwork` will create an empty network, which we can then populate ourselves.
|
||||
@func.extend(CreateNetwork())
|
||||
def create_network(builder, network):
|
||||
# This network will add 1 to the input tensor.
|
||||
inp = network.add_input(name=INPUT_NAME, shape=INPUT_SHAPE, dtype=trt.float32)
|
||||
ones = network.add_constant(
|
||||
shape=INPUT_SHAPE, weights=np.ones(shape=INPUT_SHAPE, dtype=np.float32)
|
||||
).get_output(0)
|
||||
add = network.add_elementwise(
|
||||
inp, ones, op=trt.ElementWiseOperation.SUM
|
||||
).get_output(0)
|
||||
add.name = OUTPUT_NAME
|
||||
network.mark_output(add)
|
||||
|
||||
# Notice that we don't need to return anything - `extend()` takes care of that for us!
|
||||
|
||||
|
||||
def main():
|
||||
# After we've constructed the network, we can go back to using regular Polygraphy APIs.
|
||||
#
|
||||
# NOTE: Since we're using lazy loaders, we provide the `create_network` function as
|
||||
# an argument - we do *not* call it ourselves.
|
||||
build_engine = EngineFromNetwork(create_network)
|
||||
|
||||
with TrtRunner(build_engine) as runner:
|
||||
feed_dict = {
|
||||
INPUT_NAME: np.random.random_sample(INPUT_SHAPE).astype(np.float32)
|
||||
}
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict)
|
||||
|
||||
assert np.array_equal(outputs[OUTPUT_NAME], (feed_dict[INPUT_NAME] + 1))
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,92 @@
|
||||
# Immediately Evaluated Functional API
|
||||
|
||||
## Introduction
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
Most of the time, the lazy loaders included with Polygraphy have several advantages:
|
||||
|
||||
- They allow us to defer the work until we actually need to do it, which can potentially save
|
||||
time.
|
||||
- Since constructed loaders are extremely light-weight, runners using lazily evaluated loaders can be
|
||||
easily copied into other processes or threads, where they can then be launched.
|
||||
If runners instead referenced entire models/inference sessions, it would be non-trivial to copy them in this way.
|
||||
- They allow us to define a sequence of operations in advance by chaining loaders together, which
|
||||
provides an easy way to build reusable functions.
|
||||
For example, we could create a loader that imports a model from ONNX and generates a serialized TensorRT Engine:
|
||||
|
||||
```python
|
||||
build_engine = EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx"))
|
||||
```
|
||||
|
||||
- They allow for special semantics where if a callable is provided to a loader, it takes ownership
|
||||
of the return value, whereas otherwise it does not. These special semantics are useful for
|
||||
sharing objects between multiple loaders.
|
||||
|
||||
However, this can sometimes lead to code that is less readable, or even downright confusing.
|
||||
For example, consider the following:
|
||||
```python
|
||||
# Each line in this example looks almost the same, but has significantly
|
||||
# different behavior. Some of these lines even cause memory leaks!
|
||||
EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx")) # This is a loader instance, not an engine!
|
||||
EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx"))() # This is an engine.
|
||||
EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx")()) # And it's a loader instance again...
|
||||
EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx")())() # Back to an engine!
|
||||
EngineBytesFromNetwork(NetworkFromOnnxPath("/path/to/model.onnx"))()() # This throws - can you see why?
|
||||
```
|
||||
|
||||
For that reason, Polygraphy provides immediately-evaluated functional
|
||||
equivalents of each loader. Each functional variant uses the same name as the loader, but
|
||||
`snake_case` instead of `PascalCase`. Using the functional variants, loader code like:
|
||||
|
||||
```python
|
||||
parse_network = NetworkFromOnnxPath("/path/to/model.onnx")
|
||||
create_config = CreateConfig(fp16=True, tf32=True)
|
||||
build_engine = EngineFromNetwork(parse_network, create_config)
|
||||
engine = build_engine()
|
||||
```
|
||||
|
||||
becomes:
|
||||
|
||||
```python
|
||||
builder, network, parser = network_from_onnx_path("/path/to/model.onnx")
|
||||
config = create_config(builder, network, fp16=True, tf32=True)
|
||||
engine = engine_from_network((builder, network, parser), config)
|
||||
```
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
|
||||
|
||||
In this example, we'll look at how you can leverage the functional API to convert an ONNX
|
||||
model to a TensorRT network, modify the network, build a TensorRT engine with FP16 precision
|
||||
enabled, and run inference.
|
||||
We'll also save the engine to a file to see how you can load it again and run inference.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. **[Optional]** Inspect the model before running the example:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.onnx
|
||||
```
|
||||
|
||||
3. Run the script that builds and runs the engine:
|
||||
|
||||
```bash
|
||||
python3 build_and_run.py
|
||||
```
|
||||
|
||||
4. **[Optional]** Inspect the TensorRT engine built by the example:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.engine
|
||||
```
|
||||
|
||||
5. Run the script that loads the previously built engine, then runs it:
|
||||
|
||||
```bash
|
||||
python3 load_and_run.py
|
||||
```
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script uses Polygraphy's immediately evaluated functional APIs
|
||||
to load an ONNX model, convert it into a TensorRT network, add an identity
|
||||
layer to the end of it, build an engine with FP16 mode enabled,
|
||||
save the engine, and finally run inference.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.trt import (
|
||||
TrtRunner,
|
||||
create_config,
|
||||
engine_from_network,
|
||||
network_from_onnx_path,
|
||||
save_engine,
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
# In Polygraphy, loaders and runners take ownership of objects if they are provided
|
||||
# via the return values of callables. For example, we don't need to worry about object
|
||||
# lifetimes when we use lazy loaders.
|
||||
#
|
||||
# Since we are immediately evaluating, we take ownership of objects, and are responsible for freeing them.
|
||||
builder, network, parser = network_from_onnx_path("identity.onnx")
|
||||
|
||||
# Extend the network with an identity layer (purely for the sake of example).
|
||||
# Note that unlike with lazy loaders, we don't need to do anything special to modify the network.
|
||||
# If we were using lazy loaders, we would need to use `func.extend()` as described
|
||||
# in example 03 and example 05.
|
||||
prev_output = network.get_output(0)
|
||||
network.unmark_output(prev_output)
|
||||
output = network.add_identity(prev_output).get_output(0)
|
||||
output.name = "output"
|
||||
network.mark_output(output)
|
||||
|
||||
# Create a TensorRT IBuilderConfig so that we can build the engine with FP16 enabled.
|
||||
config = create_config(builder, network, fp16=True)
|
||||
|
||||
engine = engine_from_network((builder, network), config)
|
||||
|
||||
# To reuse the engine elsewhere, we can serialize it and save it to a file.
|
||||
save_engine(engine, path="identity.engine")
|
||||
|
||||
with TrtRunner(engine) as runner:
|
||||
inp_data = np.ones((1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict={"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["output"], inp_data) # It's an identity model!
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script uses Polygraphy's immediately evaluated functional APIs
|
||||
to load the TensorRT engine built by `build_and_run.py` and run inference.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.common import bytes_from_path
|
||||
from polygraphy.backend.trt import TrtRunner, engine_from_bytes
|
||||
|
||||
|
||||
def main():
|
||||
engine = engine_from_bytes(bytes_from_path("identity.engine"))
|
||||
|
||||
with TrtRunner(engine) as runner:
|
||||
inp_data = np.ones((1, 1, 2, 2), dtype=np.float32)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
outputs = runner.infer(feed_dict={"x": inp_data})
|
||||
|
||||
assert np.array_equal(outputs["output"], inp_data) # It's an identity model!
|
||||
|
||||
print("Inference succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
numpy
|
||||
@@ -0,0 +1,121 @@
|
||||
# Using Dynamic Shapes With TensorRT
|
||||
|
||||
## Introduction
|
||||
|
||||
*NOTE: This example is intended for use with TensorRT 8.0 or newer.*
|
||||
*Older versions may require slight modifications to the example code.*
|
||||
|
||||
In order to use dynamic input shapes with TensorRT, we have to specify a range
|
||||
(or multiple ranges) of possible shapes when we build the engine.
|
||||
TensorRT optimization profiles provide the means of doing so.
|
||||
|
||||
Using the TensorRT API, the process involves two steps:
|
||||
|
||||
1. During engine building, specify one or more optimization profiles.
|
||||
An optimization profile includes 3 shapes for each input:
|
||||
- `min`: The minimum shape for which the profile should work.
|
||||
- `opt`: The shape which TensorRT should optimize for.
|
||||
Generally, you'd want this to correspond to the most commonly used shape.
|
||||
- `max`: The maximum shape for which the profile should work.
|
||||
|
||||
2. During inference, set the input shape(s) in the execution context, then
|
||||
use the `IOutputAllocator` API to provide a callback to allocate enough
|
||||
device memory for the outputs.
|
||||
|
||||
Polygraphy can simplify both steps and help you avoid common pitfalls:
|
||||
|
||||
1. It provides a `Profile` abstraction, which is an `OrderedDict` that
|
||||
can be converted to a TensorRT `IOptimizationProfile` and includes some utility functions:
|
||||
- `fill_defaults`: Fills the profile with default shapes based on the network.
|
||||
- `to_trt`: Creates a TensorRT `IOptimizationProfile` using the shapes in this `Profile`.
|
||||
|
||||
What's more, `Profile` will automatically handle complexities like the
|
||||
distinction between shape-tensor vs. non-shape-tensor inputs - you do not
|
||||
need to worry about this distinction yourself.
|
||||
|
||||
2. The `TrtRunner` will automatically handle dynamic shapes in the model.
|
||||
As in `Profile`, distinctions between shape-tensor and non-shape-tensor inputs
|
||||
are handled automatically.
|
||||
|
||||
Additionally, the runner will only update the context binding shapes when required,
|
||||
as changing the shapes has a small overhead. The output device buffers will only
|
||||
be resized if their current size is smaller that the context outputs, thus avoiding
|
||||
unnecessary reallocation.
|
||||
|
||||
|
||||
### Setting The Stage
|
||||
|
||||
For the sake of this example, we'll imagine a hypothetical scenario:
|
||||
|
||||
We're running an inference workload using an image classification model.
|
||||
|
||||
Normally, we use this model in an online scenario - i.e. we want the lowest possible
|
||||
latency, so we'll process one image at a time.
|
||||
For this case, assume `batch_size` is `[1]`.
|
||||
|
||||
However, if we have too many users, then we need to employ dynamic batching so that
|
||||
our throughput doesn't suffer. Our range of batch sizes is still small to
|
||||
keep the latency acceptable. Our most frequently used batch size is 4.
|
||||
For this case, assume `batch_size` is in the range `[1, 32]`.
|
||||
|
||||
In even rarer cases, we need to process large amounts of data offline. In this case,
|
||||
we use a very large batch size to improve our throughput.
|
||||
For this case, assume `batch_size` is `[128]`.
|
||||
|
||||
### Performance Considerations
|
||||
|
||||
In implementing our inference pipeline, we need to consider a few tradeoffs:
|
||||
|
||||
- A profile with a large range will not perform as well as for the entire range as
|
||||
multiple profiles each with smaller ranges.
|
||||
- Switching shapes within a profile has a small but non-zero cost.
|
||||
- Switching profiles within a context has a larger cost than switching shapes within a profile.
|
||||
- We can avoid the cost of switching profiles by creating a separate execution context
|
||||
for each profile and selecting the appropriate context at runtime.
|
||||
However, keep in mind that each context will require some additional memory.
|
||||
|
||||
|
||||
### A Possible Solution
|
||||
|
||||
Assuming the image size is `(3, 28, 28)`, we'll create three separate
|
||||
optimization profiles, and a separate context for each:
|
||||
|
||||
1. For the low latency case:
|
||||
`min=(1, 3, 28, 28), opt=(1, 3, 28, 28), max=(1, 3, 28, 28)`
|
||||
|
||||
2. For the dynamic batching case:
|
||||
`min=(1, 3, 28, 28), opt=(4, 3, 28, 28), max=(32, 3, 28, 28)`
|
||||
|
||||
Note that we use a batch size of `4` for `opt` since that's the most common case.
|
||||
|
||||
3. For the offline case:
|
||||
`min=(128, 3, 28, 28), opt=(128, 3, 28, 28), max=(128, 3, 28, 28)`
|
||||
|
||||
For each context, we'll create a corresponding `TrtRunner`. If we make sure that
|
||||
we own the engine and the context (by not providing them via lazy loaders), then
|
||||
the cost of activating a runner should be small - it just needs to allocate
|
||||
input and output buffers. Hence, we'll be able to activate runners on-demand quickly.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
2. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
3. **[Optional]** Inspect the generated engine:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model dynamic_identity.engine
|
||||
```
|
||||
|
||||
## Further Reading
|
||||
|
||||
For more information on using dynamic shapes with TensorRT, see the
|
||||
[developer guide](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#work_dynamic_shapes)
|
||||
@@ -0,0 +1,12 @@
|
||||
:[
|
||||
|
||||
XY"Identityonnx_dynamic_identityZ%
|
||||
X
|
||||
|
||||
|
||||
batch_size
|
||||
|
||||
|
||||
b
|
||||
Y
|
||||
B
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
This script builds an engine with 3 separate optimization profiles, each
|
||||
built for a specific use-case. It then creates 3 separate execution contexts
|
||||
and corresponding `TrtRunner`s for inference.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.backend.trt import (
|
||||
CreateConfig,
|
||||
Profile,
|
||||
TrtRunner,
|
||||
engine_from_network,
|
||||
network_from_onnx_path,
|
||||
save_engine,
|
||||
)
|
||||
from polygraphy.logger import G_LOGGER
|
||||
|
||||
|
||||
def main():
|
||||
# A Profile maps each input tensor to a range of shapes.
|
||||
# The `add()` method can be used to add shapes for a single input.
|
||||
#
|
||||
# TIP: To save lines, calls to `add` can be chained:
|
||||
# profile.add("input0", ...).add("input1", ...)
|
||||
#
|
||||
# Of course, you may alternatively write this as:
|
||||
# profile.add("input0", ...)
|
||||
# profile.add("input1", ...)
|
||||
#
|
||||
profiles = [
|
||||
# The low-latency case. For best performance, min == opt == max.
|
||||
Profile().add("X", min=(1, 3, 28, 28), opt=(1, 3, 28, 28), max=(1, 3, 28, 28)),
|
||||
# The dynamic batching case. We use `4` for the opt batch size since that's our most common case.
|
||||
Profile().add("X", min=(1, 3, 28, 28), opt=(4, 3, 28, 28), max=(32, 3, 28, 28)),
|
||||
# The offline case. For best performance, min == opt == max.
|
||||
Profile().add(
|
||||
"X", min=(128, 3, 28, 28), opt=(128, 3, 28, 28), max=(128, 3, 28, 28)
|
||||
),
|
||||
]
|
||||
|
||||
# See examples/api/06_immediate_eval_api for details on immediately evaluated functional loaders like `engine_from_network`.
|
||||
# Note that we can freely mix lazy and immediately-evaluated loaders.
|
||||
engine = engine_from_network(
|
||||
network_from_onnx_path("dynamic_identity.onnx"),
|
||||
config=CreateConfig(profiles=profiles),
|
||||
)
|
||||
|
||||
# We'll save the engine so that we can inspect it with `inspect model`.
|
||||
# This should make it easy to see how the engine bindings are laid out.
|
||||
save_engine(engine, "dynamic_identity.engine")
|
||||
|
||||
# We'll create, but not activate, three separate runners, each with a separate context.
|
||||
#
|
||||
# TIP: By providing a context directly, as opposed to via a lazy loader,
|
||||
# we can ensure that the runner will *not* take ownership of it.
|
||||
#
|
||||
low_latency = TrtRunner(engine.create_execution_context())
|
||||
|
||||
# NOTE: The following two lines may cause TensorRT to display errors since profile 0
|
||||
# is already in use by the first execution context. We'll suppress them using G_LOGGER.verbosity().
|
||||
#
|
||||
with G_LOGGER.verbosity(G_LOGGER.CRITICAL):
|
||||
# We can use the `optimization_profile` parameter of the runner to ensure that the correct optimization profile is used.
|
||||
# This eliminates the need to call `set_profile()` later.
|
||||
dynamic_batching = TrtRunner(
|
||||
engine.create_execution_context(), optimization_profile=1
|
||||
) # Use the second profile, which is intended for dynamic batching.
|
||||
|
||||
# For the sake of example, we *won't* use `optimization_profile` here.
|
||||
# Instead, we'll use `set_profile()` after activating the runner.
|
||||
offline = TrtRunner(engine.create_execution_context())
|
||||
|
||||
# Finally, we can activate the runners as we need them.
|
||||
#
|
||||
# NOTE: Since the context and engine are already created, the runner will only need to
|
||||
# allocate input and output buffers during activation.
|
||||
|
||||
input_img = np.ones((1, 3, 28, 28), dtype=np.float32) # An input "image"
|
||||
|
||||
with low_latency:
|
||||
outputs = low_latency.infer({"X": input_img})
|
||||
assert np.array_equal(outputs["Y"], input_img) # It's an identity model!
|
||||
|
||||
print("Low latency runner succeeded!")
|
||||
|
||||
# While we're serving requests online, we might decide that we need dynamic batching
|
||||
# for a moment.
|
||||
#
|
||||
# NOTE: We're assuming that activating runners will be cheap here, so we can bring up
|
||||
# the dynamic batching runner just-in-time.
|
||||
#
|
||||
# TIP: If activating the runner is not cheap (e.g. input/output buffers are large),
|
||||
# it might be better to keep the runner active the whole time.
|
||||
#
|
||||
with dynamic_batching:
|
||||
# We'll create fake batches by repeating our fake input image.
|
||||
small_input_batch = np.repeat(input_img, 4, axis=0) # Shape: (4, 3, 28, 28)
|
||||
outputs = dynamic_batching.infer({"X": small_input_batch})
|
||||
assert np.array_equal(outputs["Y"], small_input_batch)
|
||||
|
||||
# If we need dynamic batching again later, we can activate the runner once more.
|
||||
#
|
||||
# NOTE: This time, we do *not* need to set the profile.
|
||||
#
|
||||
with dynamic_batching:
|
||||
# NOTE: We can use any shape that's in the range of the profile without
|
||||
# additional setup - Polygraphy handles the details behind the scenes!
|
||||
#
|
||||
large_input_batch = np.repeat(input_img, 16, axis=0) # Shape: (16, 3, 28, 28)
|
||||
outputs = dynamic_batching.infer({"X": large_input_batch})
|
||||
assert np.array_equal(outputs["Y"], large_input_batch)
|
||||
|
||||
print("Dynamic batching runner succeeded!")
|
||||
|
||||
with offline:
|
||||
# NOTE: When we first activate this runner, we need to set the profile index (it's 0 by default).
|
||||
# Since we provided our own execution context when we created the runner, we need to do this *only once*.
|
||||
# Our settings persist since the context will remain alive even after the runner is deactivated.
|
||||
# If we had instead allowed the runner to own the context, we'd need to repeat this step each time we activated the runner.
|
||||
#
|
||||
# Alternatively, we could have used the `optimization_profile` parameter (see above).
|
||||
#
|
||||
offline.set_profile(
|
||||
2
|
||||
) # Use the third profile, which is intended for the offline case.
|
||||
|
||||
large_offline_batch = np.repeat(
|
||||
input_img, 128, axis=0
|
||||
) # Shape: (128, 3, 28, 28)
|
||||
outputs = offline.infer({"X": large_offline_batch})
|
||||
assert np.array_equal(outputs["Y"], large_offline_batch)
|
||||
|
||||
print("Offline runner succeeded!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
# Working With Run Results And Saved Inputs Manually
|
||||
|
||||
## Introduction
|
||||
|
||||
Inference inputs and outputs from `Comparator.run` can be serialized and saved to JSON
|
||||
files so they can be reused. Inputs are stored as `List[Dict[str, np.ndarray]]` while outputs
|
||||
are stored in a `RunResults` object, which can keep track of the outputs of multiple runners
|
||||
from multiple inference iterations.
|
||||
|
||||
Command-line tools providing `--save-inputs` and `--save-outputs` options generally use these formats.
|
||||
|
||||
Usually, you'll only use saved inputs or `RunResults` with other Polygraphy APIs or
|
||||
tools (as in [this example](../../cli//run/06_comparing_with_custom_output_data/)
|
||||
or [this one](../../cli/inspect/05_inspecting_inference_outputs/)), but sometimes,
|
||||
you may want to work with the underlying NumPy arrays manually.
|
||||
|
||||
Polygraphy includes convenience APIs that make it easy to load and manipulate these objects.
|
||||
|
||||
This example illustrates how you can load saved inputs and/or `RunResults` from a file
|
||||
using the Python API and then access the NumPy arrays stored within.
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Generate some inference inputs and outputs:
|
||||
|
||||
```bash
|
||||
polygraphy run identity.onnx --trt --onnxrt \
|
||||
--save-inputs inputs.json --save-outputs outputs.json
|
||||
```
|
||||
|
||||
2. **[Optional]** Use `inspect data` to view the inputs on the command-line:
|
||||
|
||||
```bash
|
||||
polygraphy inspect data inputs.json --show-values
|
||||
```
|
||||
|
||||
3. **[Optional]** Use `inspect data` to view the outputs on the command-line:
|
||||
|
||||
```bash
|
||||
polygraphy inspect data outputs.json --show-values
|
||||
```
|
||||
|
||||
4. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""
|
||||
This script demonstrates how to use the `load_json` and `RunResults` APIs to load
|
||||
and manipulate inference inputs and outputs respectively.
|
||||
"""
|
||||
|
||||
from polygraphy.comparator import RunResults
|
||||
from polygraphy.json import load_json
|
||||
|
||||
|
||||
def main():
|
||||
# Use the `load_json` API to load inputs from file.
|
||||
#
|
||||
# NOTE: The `save_json` and `load_json` standalone helpers should be used only with non-Polygraphy objects.
|
||||
# Polygraphy objects that support serialization include `save` and `load` methods.
|
||||
inputs = load_json("inputs.json")
|
||||
|
||||
# Inputs are stored as a `List[Dict[str, np.ndarray]]`, i.e. a list of feed_dicts,
|
||||
# where each feed_dict maps input names to NumPy arrays.
|
||||
#
|
||||
# TIP: In the typical case, we'll only have one iteration, so we'll only look at the first item.
|
||||
# If you need to access inputs from multiple iterations, you can do something like this instead:
|
||||
#
|
||||
# for feed_dict in inputs:
|
||||
# for name, array in feed_dict.items():
|
||||
# ... # Do something with the inputs here
|
||||
#
|
||||
[feed_dict] = inputs
|
||||
for name, array in feed_dict.items():
|
||||
print(f"Input: '{name}' | Values:\n{array}")
|
||||
|
||||
# Use the `RunResults.load` API to load results from file.
|
||||
#
|
||||
# TIP: You can provide either a file path or a file-like object here.
|
||||
results = RunResults.load("outputs.json")
|
||||
|
||||
# The `RunResults` object is structured like a `Dict[str, List[IterationResult]]``,
|
||||
# mapping runner names to inference outputs from one or more iterations.
|
||||
# An `IterationResult` behaves just like a `Dict[str, np.ndarray]` mapping output names
|
||||
# to NumPy arrays.
|
||||
#
|
||||
# TIP: In the typical case, we'll only have one iteration, so we can unpack it
|
||||
# directly in the loop. If you need to access outputs from multiple iterations,
|
||||
# you can do something like this instead:
|
||||
#
|
||||
# for runner_name, iters in results.items():
|
||||
# for outputs in iters:
|
||||
# ... # Do something with the outputs here
|
||||
#
|
||||
for runner_name, [outputs] in results.items():
|
||||
print(f"\nProcessing outputs for runner: {runner_name}")
|
||||
# Now you can read or modify the outputs for each runner.
|
||||
# For the sake of this example, we'll just print them:
|
||||
for name, array in outputs.items():
|
||||
print(f"Output: '{name}' | Values:\n{array}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# Working With PyTorch Tensors
|
||||
|
||||
## Introduction
|
||||
|
||||
Some runners like `OnnxrtRunner` and `TrtRunner` can accept and return PyTorch tensors
|
||||
in addition to NumPy arrays. When PyTorch tensors are provided in the inputs, the runner
|
||||
will return the outputs as PyTorch tensors as well. This can be especially useful in
|
||||
cases where PyTorch supports a data type that is not supported by NumPy, such as BFloat16.
|
||||
|
||||
Polygraphy's included TensorRT `Calibrator` can also accept PyTorch tensors directly.
|
||||
|
||||
This example uses PyTorch tensors on the GPU where possible (i.e. if a GPU-enabled version
|
||||
of PyTorch is installed). When the tensors already reside on GPU memory, no additional copies
|
||||
are required in the runner/calibrator.
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Install prerequisites
|
||||
* Ensure that TensorRT is installed
|
||||
* Install other dependencies with `python3 -m pip install -r requirements.txt`
|
||||
|
||||
|
||||
2. Run the example:
|
||||
|
||||
```bash
|
||||
python3 example.py
|
||||
```
|
||||
|
||||
|
||||
## See Also
|
||||
|
||||
* [Inference With TensorRT](../00_inference_with_tensorrt/)
|
||||
* [INT8 Calibration In TensorRT](../04_int8_calibration_in_tensorrt/)
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""
|
||||
This script demonstrates how to use PyTorch tensors with the TensorRT runner and calibrator.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from polygraphy.backend.trt import (
|
||||
Calibrator,
|
||||
CreateConfig,
|
||||
TrtRunner,
|
||||
engine_from_network,
|
||||
network_from_onnx_path,
|
||||
)
|
||||
|
||||
# If your PyTorch installation has GPU support, then we'll allocate the tensors
|
||||
# directly in GPU memory. This will mean that the calibrator and runner can skip the
|
||||
# host-to-device copy we would otherwise incur with NumPy arrays.
|
||||
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
|
||||
def calib_data():
|
||||
for _ in range(4):
|
||||
yield {"x": torch.ones((1, 1, 2, 2), dtype=torch.float32, device=DEVICE)}
|
||||
|
||||
|
||||
def main():
|
||||
calibrator = Calibrator(data_loader=calib_data())
|
||||
|
||||
engine = engine_from_network(
|
||||
network_from_onnx_path("identity.onnx"),
|
||||
config=CreateConfig(int8=True, calibrator=calibrator),
|
||||
)
|
||||
|
||||
with TrtRunner(engine) as runner:
|
||||
inp_data = torch.ones((1, 1, 2, 2), dtype=torch.float32, device=DEVICE)
|
||||
|
||||
# NOTE: The runner owns the output buffers and is free to reuse them between `infer()` calls.
|
||||
# Thus, if you want to store results from multiple inferences, you should use `copy.deepcopy()`.
|
||||
#
|
||||
# When you provide PyTorch tensors in the feed_dict, the runner will try to use
|
||||
# PyTorch tensors for the outputs. Specifically:
|
||||
# - If the `copy_outputs_to_host` argument to `infer()` is set to `True` (the default),
|
||||
# it will return PyTorch tensors in CPU memory.
|
||||
# - If `copy_outputs_to_host` is `False`, it will return:
|
||||
# - PyTorch tensors in GPU memory if you have a GPU-enabled PyTorch installation.
|
||||
# - Polygraphy `DeviceView`s otherwise.
|
||||
#
|
||||
outputs = runner.infer({"x": inp_data})
|
||||
|
||||
# `copy_outputs_to_host` defaults to True, so the outputs should be PyTorch
|
||||
# tensors in CPU memory.
|
||||
assert isinstance(outputs["y"], torch.Tensor)
|
||||
assert outputs["y"].device.type == "cpu"
|
||||
|
||||
assert torch.equal(outputs["y"], inp_data.to("cpu")) # It's an identity model!
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
tensorrt>=8.5
|
||||
torch>=1.13.0
|
||||
@@ -0,0 +1,64 @@
|
||||
# Polygraphy Python API Examples
|
||||
|
||||
This directory includes examples that use the Polygraphy Python API.
|
||||
For examples of the command-line tools, see the [cli](../cli/) directory instead.
|
||||
|
||||
You may find it useful to read the [Python API Overview](../../polygraphy/) prior
|
||||
to looking at the API examples.
|
||||
|
||||
## Generating Your Own Examples
|
||||
|
||||
In the event that the examples here do not cover a particular use-case, you can typically
|
||||
use `polygraphy run` to fill the gap; `polygraphy run` is capable of dynamically generating
|
||||
Python scripts that use the Polygraphy API that do exactly what the tool would otherwise do.
|
||||
|
||||
Thus, if `polygraphy run` includes functionality you need, but you cannot find a
|
||||
corresponding API example, try running:
|
||||
```bash
|
||||
polygraphy run --gen - <options...>
|
||||
```
|
||||
The argument to `--gen` should be the name of the file in which to write the generated script.
|
||||
The special value `-` corresponds to `stdout`.
|
||||
|
||||
For example, running:
|
||||
```bash
|
||||
polygraphy run --gen - model.onnx --trt --onnxrt
|
||||
```
|
||||
will display something like this on `stdout`:
|
||||
```py
|
||||
#!/usr/bin/env python3
|
||||
# Template auto-generated by polygraphy [v0.31.0] on 01/01/20 at 10:10:10
|
||||
# Generation Command: polygraphy run --gen - model.onnx --trt --onnxrt
|
||||
# This script compares model.onnx between TensorRT and ONNX-Runtime
|
||||
|
||||
from polygraphy.logger import G_LOGGER
|
||||
|
||||
from polygraphy.backend.onnxrt import OnnxrtRunner, SessionFromOnnx
|
||||
from polygraphy.backend.trt import EngineFromNetwork, NetworkFromOnnxPath, TrtRunner
|
||||
from polygraphy.comparator import Comparator
|
||||
import sys
|
||||
|
||||
# Loaders
|
||||
parse_network_from_onnx = NetworkFromOnnxPath('model.onnx')
|
||||
build_engine = EngineFromNetwork(parse_network_from_onnx)
|
||||
build_onnxrt_session = SessionFromOnnx('model.onnx')
|
||||
|
||||
# Runners
|
||||
runners = [
|
||||
TrtRunner(build_engine),
|
||||
OnnxrtRunner(build_onnxrt_session),
|
||||
]
|
||||
|
||||
# Runner Execution
|
||||
results = Comparator.run(runners)
|
||||
|
||||
success = True
|
||||
# Accuracy Comparison
|
||||
success &= bool(Comparator.compare_accuracy(results))
|
||||
|
||||
# Report Results
|
||||
cmd_run = ' '.join(sys.argv)
|
||||
if not success:
|
||||
G_LOGGER.critical(f"FAILED | Command: {cmd_run}"))
|
||||
G_LOGGER.finish(f"PASSED | Command: {cmd_run}"))
|
||||
```
|
||||
@@ -0,0 +1,6 @@
|
||||
# Polygraphy CLI Examples
|
||||
|
||||
This directory includes examples that use the Polygraphy CLI.
|
||||
For examples of the Python API, see the [api](../api/) directory instead.
|
||||
|
||||
You may find the [CLI User Guide](../../polygraphy/tools/) useful to navigate the CLI examples.
|
||||
@@ -0,0 +1,125 @@
|
||||
# checking An ONNX Model
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `check lint` subtool validates ONNX Models and generates a JSON report detailing any bad/unused nodes or model errors.
|
||||
|
||||
## Running The Example
|
||||
|
||||
### Lint the ONNX model:
|
||||
|
||||
<!-- Polygraphy Test: XFAIL Start -->
|
||||
```bash
|
||||
polygraphy check lint bad_graph.onnx -o report.json
|
||||
```
|
||||
<!-- Polygraphy Test: XFAIL End -->
|
||||
The output should look something like this:
|
||||
```bash
|
||||
[I] RUNNING | Command: polygraphy check lint bad_graph.onnx -o report.json
|
||||
[I] Loading model: bad_graph.onnx
|
||||
[E] LINT | Field 'name' of 'graph' is required to be non-empty.
|
||||
[I] Will generate inference input data according to provided TensorMetadata: {E [dtype=float32, shape=(1, 4)],
|
||||
F [dtype=float32, shape=(4, 1)],
|
||||
G [dtype=int64, shape=(4, 4)],
|
||||
D [dtype=float32, shape=(4, 1)],
|
||||
C [dtype=float32, shape=(3, 4)],
|
||||
A [dtype=float32, shape=(1, 3)],
|
||||
B [dtype=float32, shape=(4, 4)]}
|
||||
[E] LINT | Name: MatMul_3, Op: MatMul | Incompatible dimensions for matrix multiplication
|
||||
[E] LINT | Name: Add_0, Op: Add | Incompatible dimensions
|
||||
[E] LINT | Name: MatMul_0, Op: MatMul | Incompatible dimensions for matrix multiplication
|
||||
[W] LINT | Input: 'A' does not affect outputs, can be removed.
|
||||
[W] LINT | Input: 'B' does not affect outputs, can be removed.
|
||||
[W] LINT | Name: MatMul_0, Op: MatMul | Does not affect outputs, can be removed.
|
||||
[I] Saving linting report to report.json
|
||||
[E] FAILED | Runtime: 1.006s | Command: polygraphy check lint bad_graph.onnx -o report.json
|
||||
```
|
||||
|
||||
- This will create a `report.json` that contains information about what's wrong with the model.
|
||||
- The above example uses a faulty ONNX Model `bad_graph.onnx` that has multiple errors/warnings captured by the linter.
|
||||
The errors are:
|
||||
1. Model has an empty name.
|
||||
2. Nodes `Add_0`, `MatMul_0` and `MatMul_3` have incompatible input shapes.
|
||||
The warnings are:
|
||||
1. Inputs `A` and `B` are unused output.
|
||||
2. Node `MatMul_0` is unused by output.
|
||||
|
||||
### Example Report:
|
||||
|
||||
The generated report looks as follows:
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
```json
|
||||
{
|
||||
"summary": {
|
||||
"passing": [
|
||||
"MatMul_1",
|
||||
"cast_to_int64",
|
||||
"NonZero"
|
||||
],
|
||||
"failing": [
|
||||
"MatMul_0",
|
||||
"MatMul_3",
|
||||
"Add_0"
|
||||
]
|
||||
},
|
||||
"lint_entries": [
|
||||
{
|
||||
"level": "exception",
|
||||
"source": "onnx_checker",
|
||||
"message": "Field 'name' of 'graph' is required to be non-empty."
|
||||
},
|
||||
{
|
||||
"level": "exception",
|
||||
"source": "onnxruntime",
|
||||
"message": " Incompatible dimensions for matrix multiplication",
|
||||
"nodes": [
|
||||
"MatMul_3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"level": "exception",
|
||||
"source": "onnxruntime",
|
||||
"message": " Incompatible dimensions",
|
||||
"nodes": [
|
||||
"Add_0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"level": "exception",
|
||||
"source": "onnxruntime",
|
||||
"message": " Incompatible dimensions for matrix multiplication",
|
||||
"nodes": [
|
||||
"MatMul_0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"level": "warning",
|
||||
"source": "onnx_graphsurgeon",
|
||||
"message": "Input: 'A' does not affect outputs, can be removed."
|
||||
},
|
||||
{
|
||||
"level": "warning",
|
||||
"source": "onnx_graphsurgeon",
|
||||
"message": "Input: 'B' does not affect outputs, can be removed."
|
||||
},
|
||||
{
|
||||
"level": "warning",
|
||||
"source": "onnx_graphsurgeon",
|
||||
"message": "Does not affect outputs, can be removed.",
|
||||
"nodes": [
|
||||
"MatMul_0"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
|
||||
### Notes
|
||||
Since it runs ONNX Runtime under the hood, it is possible to specify execution providers using `--providers`. Defaults to CPU.
|
||||
|
||||
It is also possible to override the input shapes using `--input-shapes`, or provide custom input data. For more details, refer [how-to/use_custom_input_data](../../../../how-to/use_custom_input_data.md).
|
||||
|
||||
For more information on usage, use `polygraphy check lint --help`.
|
||||
Binary file not shown.
@@ -0,0 +1,52 @@
|
||||
# Int8 Calibration In TensorRT
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
In [API example 04](../../../api/04_int8_calibration_in_tensorrt/), we saw how we can leverage
|
||||
Polygraphy's included calibrator to easily run int8 calibration with TensorRT.
|
||||
|
||||
But what if we wanted to do the same thing on the command-line?
|
||||
|
||||
To do this, we need a way to supply custom input data to our command-line tools.
|
||||
Polygraphy provides multiple ways to do so, which are detailed [here](../../../../how-to/use_custom_input_data.md).
|
||||
|
||||
In this example, we'll use a data loader script by defining a `load_data` function in a Python
|
||||
script called `data_loader.py` and then use `polygraphy convert` to build the TensorRT engine.
|
||||
|
||||
*TIP: We can use a similar approach with `polygraphy run` to build and run the engine.*
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Convert the model, using the custom data loader script to supply calibration data,
|
||||
saving a calibration cache for future use:
|
||||
|
||||
```bash
|
||||
polygraphy convert identity.onnx --int8 \
|
||||
--data-loader-script ./data_loader.py \
|
||||
--calibration-cache identity_calib.cache \
|
||||
-o identity.engine
|
||||
```
|
||||
|
||||
2. **[Optional]** Rebuild the engine using the cache to skip calibration:
|
||||
|
||||
```bash
|
||||
polygraphy convert identity.onnx --int8 \
|
||||
--calibration-cache identity_calib.cache \
|
||||
-o identity.engine
|
||||
```
|
||||
|
||||
Since the calibration cache is already populated, calibration will be skipped.
|
||||
Hence, we do *not* need to supply input data.
|
||||
|
||||
|
||||
3. **[Optional]** Use the data loader directly from the API example.
|
||||
|
||||
The method outlined here is so flexible that we can even use the data loader we defined in the API example!
|
||||
We just need to specify the function name since the example does not call it `load_data`:
|
||||
|
||||
```bash
|
||||
polygraphy convert identity.onnx --int8 \
|
||||
--data-loader-script ../../../api/04_int8_calibration_in_tensorrt/example.py:calib_data \
|
||||
-o identity.engine
|
||||
```
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
Defines a `load_data` function that returns a generator yielding
|
||||
feed_dicts so that this script can be used as the argument for
|
||||
the --data-loader-script command-line parameter.
|
||||
"""
|
||||
import numpy as np
|
||||
|
||||
INPUT_SHAPE = (1, 1, 2, 2)
|
||||
|
||||
|
||||
def load_data():
|
||||
for _ in range(5):
|
||||
yield {
|
||||
"x": np.ones(shape=INPUT_SHAPE, dtype=np.float32)
|
||||
} # Still totally real data
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
# Deterministic Engine Building In TensorRT
|
||||
|
||||
**NOTE: This example requires TensorRT 8.7 or newer.**
|
||||
|
||||
## Introduction
|
||||
|
||||
During engine building, TensorRT runs and times several kernels in order to select
|
||||
the most optimal ones. Since kernel timings may vary slightly from run to run, this
|
||||
process is inherently non-deterministic.
|
||||
|
||||
In many cases, deterministic engine builds may be desirable. One way of achieving this
|
||||
is to use a timing cache to ensure the same kernels are picked each time.
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Build an engine and save a timing cache:
|
||||
|
||||
```bash
|
||||
polygraphy convert identity.onnx \
|
||||
--save-timing-cache timing.cache \
|
||||
-o 0.engine
|
||||
```
|
||||
|
||||
2. Use the timing cache for another engine build:
|
||||
|
||||
```bash
|
||||
polygraphy convert identity.onnx \
|
||||
--load-timing-cache timing.cache --error-on-timing-cache-miss \
|
||||
-o 1.engine
|
||||
```
|
||||
|
||||
We specify `--error-on-timing-cache-miss` so that we can be sure that the new engine
|
||||
used the entries from the timing cache for each layer.
|
||||
|
||||
3. Verify that the engines are exactly the same:
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
```bash
|
||||
diff <(polygraphy inspect model 0.engine --show layers attrs) <(polygraphy inspect model 1.engine --show layers attrs)
|
||||
```
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
# Working With Models With Dynamic Shapes In TensorRT
|
||||
|
||||
## Introduction
|
||||
|
||||
In order to use dynamic input shapes with TensorRT, we have to specify a range
|
||||
(or multiple ranges) of possible shapes when we build the engine.
|
||||
For details on how this works, refer to
|
||||
[API example 07](../../../api/07_tensorrt_and_dynamic_shapes/).
|
||||
|
||||
When using the CLI, we can specify the per-input minimum, optimum, and maximum
|
||||
shapes one or more times. If shapes are specified more than
|
||||
once per input, multiple optimization profiles are created.
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Build an engine with 3 separate profiles:
|
||||
|
||||
```bash
|
||||
polygraphy convert dynamic_identity.onnx -o dynamic_identity.engine \
|
||||
--trt-min-shapes X:[1,3,28,28] --trt-opt-shapes X:[1,3,28,28] --trt-max-shapes X:[1,3,28,28] \
|
||||
--trt-min-shapes X:[1,3,28,28] --trt-opt-shapes X:[4,3,28,28] --trt-max-shapes X:[32,3,28,28] \
|
||||
--trt-min-shapes X:[128,3,28,28] --trt-opt-shapes X:[128,3,28,28] --trt-max-shapes X:[128,3,28,28]
|
||||
```
|
||||
|
||||
For models with multiple inputs, simply provide multiple arguments to each `--trt-*-shapes` parameter.
|
||||
For example: `--trt-min-shapes input0:[10,10] input1:[10,10] input2:[10,10] ...`
|
||||
|
||||
*TIP: If we want to use only a single profile where min == opt == max, we can leverage the runtime input*
|
||||
*shapes option: `--input-shapes` as a conveneint shorthand instead of setting min/opt/max separately.*
|
||||
|
||||
|
||||
2. **[Optional]** Inspect the resulting engine:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model dynamic_identity.engine
|
||||
```
|
||||
|
||||
|
||||
## Further Reading
|
||||
|
||||
For more information on using dynamic shapes with TensorRT, see the
|
||||
[developer guide](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#work_dynamic_shapes)
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
:[
|
||||
|
||||
XY"Identityonnx_dynamic_identityZ%
|
||||
X
|
||||
|
||||
|
||||
batch_size
|
||||
|
||||
|
||||
b
|
||||
Y
|
||||
B
|
||||
@@ -0,0 +1,49 @@
|
||||
# Converting ONNX Models To FP16
|
||||
|
||||
## Introduction
|
||||
|
||||
When debugging accuracy issues with using TensorRT reduced precision
|
||||
optimizations (`--fp16` and `--tf32` flags) on an ONNX model trained in FP32,
|
||||
it can be helpful to convert the model to FP16 and run it under ONNX-Runtime
|
||||
to check if there are might be problems inherent to running the model
|
||||
with reduced precision.
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Convert the model to FP16:
|
||||
|
||||
```bash
|
||||
polygraphy convert --fp-to-fp16 -o identity_fp16.onnx identity.onnx
|
||||
```
|
||||
|
||||
2. **[Optional]** Inspect the resulting model:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity_fp16.onnx
|
||||
```
|
||||
|
||||
3. **[Optional]** Run the FP32 and FP16 models under ONNX-Runtime and then compare the results:
|
||||
|
||||
```bash
|
||||
polygraphy run --onnxrt identity.onnx \
|
||||
--save-inputs inputs.json --save-outputs outputs_fp32.json
|
||||
```
|
||||
|
||||
```bash
|
||||
polygraphy run --onnxrt identity_fp16.onnx \
|
||||
--load-inputs inputs.json --load-outputs outputs_fp32.json \
|
||||
--atol 0.001 --rtol 0.001
|
||||
```
|
||||
|
||||
4. **[Optional]** Check if any intermediate outputs of the FP16 model
|
||||
contain NaN or infinity (see [Checking for Intermediate NaN or Infinities](../../../../examples/cli/run/07_checking_nan_inf)):
|
||||
|
||||
```bash
|
||||
polygraphy run --onnxrt identity_fp16.onnx --onnx-outputs mark all --validate
|
||||
```
|
||||
|
||||
## See Also
|
||||
|
||||
* [Comparing Across Runs](../../../../examples/cli/run/02_comparing_across_runs)
|
||||
* [Checking for Intermediate NaN or Infinities](../../../../examples/cli/run/07_checking_nan_inf)
|
||||
* [Debugging TensorRT Accuracy Issues](../../../../how-to/debug_accuracy.md)
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
# Debugging Flaky TensorRT Tactics
|
||||
|
||||
**IMPORTANT: This example no longer works reliably for newer versions of TensorRT, since they make some**
|
||||
**tactic choices that are not exposed via the IAlgorithmSelector interface (Deprecated in TensorRT 10.8.**
|
||||
**Please use editable mode in ITimingCache instead). Thus, the approach outlined below**
|
||||
**cannot guarantee deterministic engine builds. With TensorRT 8.7 and newer, you can use the**
|
||||
**tactic timing cache (`--save-timing-cache` and `--load-timing-cache` in Polygraphy) to ensure**
|
||||
**determinism, but these files are opaque and thus cannot be interpreted by `inspect diff-tactics`**
|
||||
|
||||
## Introduction
|
||||
|
||||
Sometimes, a tactic in TensorRT may produce incorrect results, or have
|
||||
otherwise buggy behavior. Since the TensorRT builder relies on timing
|
||||
tactics, engine builds are non-deterministic, which can make tactic bugs
|
||||
manifest as flaky/intermittent failures.
|
||||
|
||||
One approach to tackling the problem is to run the builder several times,
|
||||
saving tactic replay files from each run. Once we have a set of known-good and
|
||||
known-bad tactics, we can compare them to determine which tactic
|
||||
is likely to be the source of error.
|
||||
|
||||
The `debug build` subtool allows you to automate this process.
|
||||
|
||||
For more details on how the `debug` tools work, see the help output:
|
||||
`polygraphy debug -h` and `polygraphy debug build -h`.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Generate golden outputs from ONNX-Runtime:
|
||||
|
||||
```bash
|
||||
polygraphy run identity.onnx --onnxrt \
|
||||
--save-outputs golden.json
|
||||
```
|
||||
|
||||
2. Use `debug build` to repeatedly build TensorRT engines and compare results against the golden outputs,
|
||||
saving a tactic replay file each time:
|
||||
|
||||
```bash
|
||||
polygraphy debug build identity.onnx --fp16 --save-tactics replay.json \
|
||||
--artifacts-dir replays --artifacts replay.json --until=10 \
|
||||
--check polygraphy run polygraphy_debug.engine --trt --load-outputs golden.json
|
||||
```
|
||||
|
||||
Let's break this down:
|
||||
|
||||
- Like other `debug` subtools, `debug build` generates an intermediate artifact each iteration
|
||||
(`./polygraphy_debug.engine` by default). This artifact in this case is a TensorRT engine.
|
||||
|
||||
*TIP: `debug build` supports all the TensorRT builder configuration options supported*
|
||||
*by other tools, like `convert` or `run`.*
|
||||
|
||||
- In order for `debug build` to determine whether each engine fails or passes,
|
||||
we provide a `--check` command. Since we're looking at a (fake) accuracy issue,
|
||||
we can use `polygraphy run` to compare the outputs of the engine to our golden values.
|
||||
|
||||
*TIP: Like other `debug` subtools, an interactive mode is also supported, which you can*
|
||||
*use simply by omitting the `--check` argument.*
|
||||
|
||||
- Unlike other `debug` subtools, `debug build` has no automatic terminating condition, so we need
|
||||
to provide the `--until` option so that the tool knows when to stop. This can either be a number
|
||||
of iterations, or `"good"` or `"bad"`. In the latter case, the tool will stop after finding the
|
||||
first passing or failing iteration respectively.
|
||||
|
||||
- Since we eventually want to compare the good and bad tactic replays, we specify `--save-tactics`
|
||||
to save tactic replay files from each iteration, then use `--artifacts` to tell `debug build`
|
||||
to manage them, which involves sorting them into `good` and `bad` subdirectories under the
|
||||
main artifacts directory, specified with `--artifacts-dir`.
|
||||
|
||||
|
||||
3. Use `inspect diff-tactics` to determine which tactics could be bad:
|
||||
|
||||
```bash
|
||||
polygraphy inspect diff-tactics --dir replays
|
||||
```
|
||||
|
||||
*NOTE: This last step should report that it could not determine potentially bad tactics since*
|
||||
*our `bad` directory should be empty at this point (please file a TensorRT issue otherwise!):*
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
```
|
||||
[I] Loaded 2 good tactic replays.
|
||||
[I] Loaded 0 bad tactic replays.
|
||||
[I] Could not determine potentially bad tactics. Try generating more tactic replay files?
|
||||
```
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
|
||||
|
||||
## Further Reading
|
||||
|
||||
For more information on the `debug` tool, as well as tips and tricks applicable
|
||||
to all `debug` subtools, see the
|
||||
[how-to guide for `debug` subtools](../../../../how-to/use_debug_subtools_effectively.md).
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
# Reducing Failing ONNX Models
|
||||
|
||||
## Introduction
|
||||
|
||||
When a model fails for any reason (for example, an accuracy issue in TensorRT) it is often
|
||||
useful to reduce it to the smallest possible subgraph that triggers the failure. That makes
|
||||
it easier to pinpoint the cause of the failure.
|
||||
|
||||
One approach to doing so is to generate successively smaller subgraphs of the original ONNX model.
|
||||
At each iteration, we can check whether the subgraph works or is still failing; once we have a working
|
||||
subgraph, we know that the subgraph generated by the previous iteration is the smallest failing
|
||||
subgraph.
|
||||
|
||||
The `debug reduce` subtool allows us to automate this process.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
For the sake of this example, we'll assume our model (`./model.onnx`) has accuracy issues
|
||||
in TensorRT. Since the model actually does work in TensorRT (please report a bug if not!),
|
||||
we'll outline the commands that you would normally run followed by commands you can run to
|
||||
simulate a failure to get a feel for how the tool looks in practice.
|
||||
|
||||
Our simulated failures will trigger whenever there's a `Mul` node in the model:
|
||||
|
||||
|
||||
|
||||
Hence, the final reduced model should contain just the `Mul` node (since the other nodes don't cause a failure).
|
||||
|
||||
1. For models that use dynamic input shapes or contain shape operations, freeze the input
|
||||
shapes and fold shape operations with:
|
||||
|
||||
```bash
|
||||
polygraphy surgeon sanitize model.onnx -o folded.onnx --fold-constants \
|
||||
--override-input-shapes x0:[1,3,224,224] x1:[1,3,224,224]
|
||||
```
|
||||
|
||||
2. Let's assume ONNX-Runtime gives us correct outputs. We'll start by generating golden
|
||||
values for every tensor in the network. We'll also save the inputs we use:
|
||||
|
||||
```bash
|
||||
polygraphy run folded.onnx --onnxrt \
|
||||
--save-inputs inputs.json \
|
||||
--onnx-outputs mark all --save-outputs layerwise_golden.json
|
||||
```
|
||||
|
||||
Then we'll combine the inputs and layerwise outputs into a single layerwise inputs file
|
||||
using the `data to-input` subtool (we'll see why this is necessary in the next step):
|
||||
|
||||
```bash
|
||||
polygraphy data to-input inputs.json layerwise_golden.json -o layerwise_inputs.json
|
||||
```
|
||||
|
||||
|
||||
3. Next, we'll use `debug reduce` in `bisect` mode:
|
||||
|
||||
```bash
|
||||
polygraphy debug reduce folded.onnx -o initial_reduced.onnx --mode=bisect --load-inputs layerwise_inputs.json \
|
||||
--check polygraphy run polygraphy_debug.onnx --trt \
|
||||
--load-inputs layerwise_inputs.json --load-outputs layerwise_golden.json
|
||||
```
|
||||
|
||||
Let's break this down:
|
||||
|
||||
- Like the other `debug` subtools, `debug reduce` generates an intermediate artifact each iteration
|
||||
(`./polygraphy_debug.onnx` by default). The artifact in this case is some subgraph of the original ONNX model.
|
||||
|
||||
- In order for `debug reduce` to determine whether each subgraph fails or passes,
|
||||
we provide a `--check` command. Since we're looking into an accuracy issue,
|
||||
we can use `polygraphy run` to compare against our golden outputs from before.
|
||||
|
||||
*TIP: Like other `debug` subtools, an interactive mode is also supported, which you can*
|
||||
*use simply by omitting the `--check` argument.*
|
||||
|
||||
- In the `--check` command, we provide the layerwise inputs via `--load-inputs`, since otherwise, `polygraphy run`
|
||||
would generate new inputs for the subgraph tensors, which may not match the values those tensors
|
||||
had when we generated our golden data. An alternative approach is to run the reference implementation
|
||||
(ONNX-Runtime here) during each iteration of `debug reduce` rather than ahead of time.
|
||||
|
||||
- Since we're using non-default input data, we also provide the layerwise inputs via `--load-inputs` directly to the
|
||||
`debug reduce` command (in addition to providing it to the `--check` command).
|
||||
This is important in models with multiple parallel branches (*referring to paths in the model rather than control flow*) like:
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
```
|
||||
inp0 inp1
|
||||
| |
|
||||
Abs Abs
|
||||
\ /
|
||||
Sum
|
||||
|
|
||||
out
|
||||
```
|
||||
In such cases, `debug reduce` needs to be able to replace one branch with a constant.
|
||||
To do so, it needs to know the input data you are using so that it can replace it with the correct values.
|
||||
Though we're using a file here, input data can be provided via any other Polygraphy data loader argument covered in
|
||||
[the CLI user guide](../../../../how-to/use_custom_input_data.md).
|
||||
|
||||
In case you're not sure whether you need to provide a data loader,
|
||||
`debug reduce` will emit a warning like this when it tries to replace a branch:
|
||||
```
|
||||
[W] This model includes multiple branches/paths. In order to continue reducing, one branch needs to be folded away.
|
||||
Please ensure that you have provided a data loader argument to `debug reduce` if your `--check` command is using a non-default data loader.
|
||||
Not doing so may result in false negatives!
|
||||
```
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
|
||||
- We specify the `-o` option so that the reduced model will be written to `initial_reduced.onnx`.
|
||||
|
||||
**To Simulate A Failure:** We can use `polygraphy inspect model` in conjunction with `--fail-regex` to trigger
|
||||
a failure whenever the model contains a `Mul` node:
|
||||
|
||||
```bash
|
||||
polygraphy debug reduce folded.onnx -o initial_reduced.onnx --mode=bisect \
|
||||
--fail-regex "Op: Mul" \
|
||||
--check polygraphy inspect model polygraphy_debug.onnx --show layers
|
||||
```
|
||||
|
||||
4. **[Optional]** As a sanity check, we can inspect our reduced model to ensure that it does contain the `Mul` node:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model initial_reduced.onnx --show layers
|
||||
```
|
||||
|
||||
5. Since we used `bisect` mode in the previous step, the model may not be as minimal as it could be.
|
||||
To further refine it, we'll run `debug reduce` again in `linear` mode:
|
||||
|
||||
```bash
|
||||
polygraphy debug reduce initial_reduced.onnx -o final_reduced.onnx --mode=linear --load-inputs layerwise_inputs.json \
|
||||
--check polygraphy run polygraphy_debug.onnx --trt \
|
||||
--load-inputs layerwise_inputs.json --load-outputs layerwise_golden.json
|
||||
```
|
||||
|
||||
**To Simulate A Failure:** We'll use the same technique as before:
|
||||
|
||||
```bash
|
||||
polygraphy debug reduce initial_reduced.onnx -o final_reduced.onnx --mode=linear \
|
||||
--fail-regex "Op: Mul" \
|
||||
--check polygraphy inspect model polygraphy_debug.onnx --show layers
|
||||
```
|
||||
|
||||
6. **[Optional]** At this stage, `final_reduced.onnx` should contain just the failing node - the `Mul`.
|
||||
We can verify this with `inspect model`:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model final_reduced.onnx --show layers
|
||||
```
|
||||
|
||||
|
||||
## Further Reading
|
||||
|
||||
- For more details on how the `debug` tools work, see the help output:
|
||||
`polygraphy debug -h` and `polygraphy debug reduce -h`.
|
||||
|
||||
- Also see the [`debug reduce` how-to guide](../../../../how-to/use_debug_reduce_effectively.md)
|
||||
for more information, tips, and tricks.
|
||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 48 KiB |
@@ -0,0 +1,37 @@
|
||||
# Inspecting A TensorRT Network
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `inspect model` subtool can automatically convert supported formats
|
||||
into TensorRT networks, and then display them.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Display the TensorRT network after parsing an ONNX model:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.onnx \
|
||||
--show layers --display-as=trt
|
||||
```
|
||||
|
||||
This will display something like:
|
||||
|
||||
```
|
||||
[I] ==== TensorRT Network ====
|
||||
Name: Unnamed Network 0 | Explicit Batch Network
|
||||
|
||||
---- 1 Network Input(s) ----
|
||||
{x [dtype=float32, shape=(1, 1, 2, 2)]}
|
||||
|
||||
---- 1 Network Output(s) ----
|
||||
{y [dtype=float32, shape=(1, 1, 2, 2)]}
|
||||
|
||||
---- 1 Layer(s) ----
|
||||
Layer 0 | node_of_y [Op: LayerType.IDENTITY]
|
||||
{x [dtype=float32, shape=(1, 1, 2, 2)]}
|
||||
-> {y [dtype=float32, shape=(1, 1, 2, 2)]}
|
||||
```
|
||||
|
||||
It is also possible to show detailed layer information, including layer attributes, using `--show layers attrs weights`.
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
# Inspecting A TensorRT Engine
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `inspect model` subtool can load and display information
|
||||
about TensorRT engines, i.e. plan files:
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Generate an engine with dynamic shapes and 2 profiles:
|
||||
|
||||
```bash
|
||||
polygraphy run dynamic_identity.onnx --trt \
|
||||
--trt-min-shapes X:[1,2,1,1] --trt-opt-shapes X:[1,2,3,3] --trt-max-shapes X:[1,2,5,5] \
|
||||
--trt-min-shapes X:[1,2,2,2] --trt-opt-shapes X:[1,2,4,4] --trt-max-shapes X:[1,2,6,6] \
|
||||
--save-engine dynamic_identity.engine
|
||||
```
|
||||
|
||||
You can also dump unfused intermediate tensors by adding `--mark-unfused-tensors-as-debug-tensors` and
|
||||
`--save-outputs output.json` options. Later, this tensor information can be combined with the inspector output.
|
||||
|
||||
2. Inspect the engine:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model dynamic_identity.engine \
|
||||
--show layers
|
||||
```
|
||||
|
||||
NOTE: `--show layers` only works if the engine was built with a `profiling_verbosity` other than `NONE`.
|
||||
Higher verbosities make more per-layer information available.
|
||||
|
||||
This will display something like:
|
||||
|
||||
```
|
||||
[I] ==== TensorRT Engine ====
|
||||
Name: Unnamed Network 0 | Explicit Batch Engine
|
||||
|
||||
---- 1 Engine Input(s) ----
|
||||
{X [dtype=float32, shape=(1, 2, -1, -1)]}
|
||||
|
||||
---- 1 Engine Output(s) ----
|
||||
{Y [dtype=float32, shape=(1, 2, -1, -1)]}
|
||||
|
||||
---- Memory ----
|
||||
Device Memory: 0 bytes
|
||||
|
||||
---- 2 Profile(s) (2 Tensor(s) Each) ----
|
||||
- Profile: 0
|
||||
Tensor: X (Input), Index: 0 | Shapes: min=(1, 2, 1, 1), opt=(1, 2, 3, 3), max=(1, 2, 5, 5)
|
||||
Tensor: Y (Output), Index: 1 | Shape: (1, 2, -1, -1)
|
||||
|
||||
- Profile: 1
|
||||
Tensor: X (Input), Index: 0 | Shapes: min=(1, 2, 2, 2), opt=(1, 2, 4, 4), max=(1, 2, 6, 6)
|
||||
Tensor: Y (Output), Index: 1 | Shape: (1, 2, -1, -1)
|
||||
|
||||
---- 1 Layer(s) Per Profile ----
|
||||
- Profile: 0
|
||||
Layer 0 | node_of_Y [Op: Reformat]
|
||||
{X [shape=(1, 2, -1, -1)]}
|
||||
-> {Y [shape=(1, 2, -1, -1)]}
|
||||
|
||||
- Profile: 1
|
||||
Layer 0 | node_of_Y [profile 1] [Op: MyelinReformat]
|
||||
{X [profile 1] [shape=(1, 2, -1, -1)]}
|
||||
-> {Y [profile 1] [shape=(1, 2, -1, -1)]}
|
||||
```
|
||||
|
||||
It is also possible to show more detailed layer information using `--show layers attrs`.
|
||||
|
||||
You can also combine tensor value statistics using `--combine-tensor-info output.json` where the JSON file is got from
|
||||
`--mark-unfused-tensors-as-debug-tensors` and `--save-outputs output.json`.
|
||||
|
||||
The statistics will be added to the input and output tensors of each layer:
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
```
|
||||
{X [dtype=float32, shape=(1, 2, -1, -1), Format: Float, min=0.42, max=0.72, avg=0.57]}
|
||||
-> {Y [dtype=float32, shape=(1, 2, -1, -1), Format: Float, min=0.42, max=0.72, avg=0.57]}
|
||||
```
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
backend_test:y
|
||||
|
||||
XY"Identityonnx_dynamic_identityZ&
|
||||
X!
|
||||
|
||||
|
||||
|
||||
height
|
||||
widthb&
|
||||
Y!
|
||||
|
||||
|
||||
|
||||
height
|
||||
widthB
|
||||
@@ -0,0 +1,38 @@
|
||||
# Inspecting An ONNX Model
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `inspect model` subtool can display ONNX models.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Inspect the ONNX model:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.onnx --show layers
|
||||
```
|
||||
|
||||
This will display something like:
|
||||
|
||||
```
|
||||
[I] ==== ONNX Model ====
|
||||
Name: test_identity | ONNX Opset: 8
|
||||
|
||||
---- 1 Graph Input(s) ----
|
||||
{x [dtype=float32, shape=(1, 1, 2, 2)]}
|
||||
|
||||
---- 1 Graph Output(s) ----
|
||||
{y [dtype=float32, shape=(1, 1, 2, 2)]}
|
||||
|
||||
---- 0 Initializer(s) ----
|
||||
{}
|
||||
|
||||
---- 1 Node(s) ----
|
||||
Node 0 | [Op: Identity]
|
||||
{x [dtype=float32, shape=(1, 1, 2, 2)]}
|
||||
-> {y [dtype=float32, shape=(1, 1, 2, 2)]}
|
||||
```
|
||||
|
||||
It is also possible to show detailed layer information, including layer attributes, using `--show layers attrs weights`.
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# Inspecting A TensorFlow Graph
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `inspect model` subtool can display TensorFlow graphs.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Inspect a TensorFlow frozen model:
|
||||
|
||||
```bash
|
||||
polygraphy inspect model identity.pb --model-type=frozen
|
||||
```
|
||||
|
||||
This will display something like:
|
||||
|
||||
```
|
||||
[I] ==== TensorFlow Graph ====
|
||||
---- 1 Graph Inputs ----
|
||||
{Input:0 [dtype=float32, shape=(1, 15, 25, 30)]}
|
||||
|
||||
---- 1 Graph Outputs ----
|
||||
{Identity_2:0 [dtype=float32, shape=(1, 15, 25, 30)]}
|
||||
|
||||
---- 4 Nodes ----
|
||||
```
|
||||
@@ -0,0 +1,17 @@
|
||||
|
||||
>
|
||||
Input Placeholder*
|
||||
dtype0*
|
||||
shape:
|
||||
$
|
||||
IdentityIdentityInput*
|
||||
T0
|
||||
)
|
||||
|
||||
Identity_1IdentityIdentity*
|
||||
T0
|
||||
+
|
||||
|
||||
Identity_2Identity
|
||||
Identity_1*
|
||||
T0"
|
||||
@@ -0,0 +1,34 @@
|
||||
# Inspecting Inference Outputs
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `inspect data` subtool can display information about the
|
||||
`RunResults` object generated by `Comparator.run()`, which represents inference outputs.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Generate some inference outputs using ONNX-Runtime:
|
||||
|
||||
```bash
|
||||
polygraphy run identity.onnx --onnxrt --save-outputs outputs.json
|
||||
```
|
||||
|
||||
2. Inspect the results:
|
||||
|
||||
```bash
|
||||
polygraphy inspect data outputs.json --show-values
|
||||
```
|
||||
|
||||
This will display something like:
|
||||
|
||||
```
|
||||
[I] ==== Run Results (1 runners) ====
|
||||
|
||||
---- onnxrt-runner-N0-07/15/21-10:46:07 (1 iterations) ----
|
||||
|
||||
y [dtype=float32, shape=(1, 1, 2, 2)] | Stats: mean=0.35995, std-dev=0.25784, var=0.066482, median=0.35968, min=0.00011437 at (0, 0, 1, 0), max=0.72032 at (0, 0, 0, 1), avg-magnitude=0.35995, p90=0.62933, p95=0.67483, p99=0.71123
|
||||
[[[[4.17021990e-01 7.20324516e-01]
|
||||
[1.14374816e-04 3.02332580e-01]]]]
|
||||
```
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# Inspecting Input Data
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `inspect data` subtool can display information about input data generated
|
||||
by a data loader.
|
||||
|
||||
|
||||
## Running The Example
|
||||
1. Generate some input data by running inference:
|
||||
|
||||
```bash
|
||||
polygraphy run identity.onnx --onnxrt --save-inputs inputs.json
|
||||
```
|
||||
|
||||
2. Inspect the input data:
|
||||
|
||||
```bash
|
||||
polygraphy inspect data inputs.json --show-values
|
||||
```
|
||||
|
||||
This will display something like:
|
||||
|
||||
```
|
||||
[I] ==== Data (1 iterations) ====
|
||||
|
||||
x [dtype=float32, shape=(1, 1, 2, 2)] | Stats: mean=0.35995, std-dev=0.25784, var=0.066482, median=0.35968, min=0.00011437 at (0, 0, 1, 0), max=0.72032 at (0, 0, 0, 1), avg-magnitude=0.35995, p90=0.62933, p95=0.67483, p99=0.71123
|
||||
[[[[4.17021990e-01 7.20324516e-01]
|
||||
[1.14374816e-04 3.02332580e-01]]]]
|
||||
```
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
# Inspecting Tactic Replay Files
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `inspect tactics` subtool can display information about TensorRT tactic replay
|
||||
files generated by Polygraphy.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Generate a tactic replay file:
|
||||
|
||||
```bash
|
||||
polygraphy run model.onnx --trt --save-tactics replay.json
|
||||
```
|
||||
|
||||
2. Inspect the tactic replay:
|
||||
|
||||
```bash
|
||||
polygraphy inspect tactics replay.json
|
||||
```
|
||||
|
||||
This will display something like:
|
||||
|
||||
```
|
||||
[I] Layer: ONNXTRT_Broadcast
|
||||
Algorithm: (Implementation: 2147483661, Tactic: 0) | Inputs: (('DataType.FLOAT'),) | Outputs: (('DataType.FLOAT'),)
|
||||
Layer: node_of_z
|
||||
Algorithm: (Implementation: 2147483651, Tactic: 1) | Inputs: (('DataType.FLOAT'), ('DataType.FLOAT')) | Outputs: (('DataType.FLOAT'),)
|
||||
```
|
||||
Binary file not shown.
@@ -0,0 +1,34 @@
|
||||
# Inspecting TensorRT ONNX Support
|
||||
|
||||
## Introduction
|
||||
|
||||
The `inspect capability` subtool provides detailed information on TensorRT's ONNX operator support for a given ONNX graph.
|
||||
It is also able to partition and save supported and unsupported subgraphs from the original model in order to report all the dynamically checked errors with a given model.
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Generate the capability report
|
||||
|
||||
```bash
|
||||
polygraphy inspect capability --with-partitioning model.onnx
|
||||
```
|
||||
|
||||
2. This should display a summary table like:
|
||||
|
||||
```
|
||||
[I] ===== Summary =====
|
||||
Operator | Count | Reason | Nodes
|
||||
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Fake | 1 | In node 0 with name: and operator: Fake (checkFallbackPluginImporter): INVALID_NODE: creator && "Plugin not found, are the plugin name, version, and namespace correct?" | [[2, 3]]
|
||||
```
|
||||
|
||||
## Understanding The Output
|
||||
|
||||
In this example, `model.onnx` contains a `Fake` node that is not supported by TensorRT.
|
||||
The summary table shows the unsupported operator, the reason it's unsupported, how many times it appears in the graph,
|
||||
and the index range of these nodes in the graph in case there are multiple unsupported nodes in a row.
|
||||
Note that this range uses an inclusive start index and an exclusive end index.
|
||||
|
||||
It is important to note that the graph partitioning logic (`--with-partitioning`) currently does not support surfacing issues with nodes inside local functions (`FunctionProto`s). See the description of the default flow (without `--with-partitioning` option, described in the example `09_inspecting_tensorrt_static_onnx_support`) for static error reporting that properly handles nodes inside local functions.
|
||||
|
||||
For more information and options, see `polygraphy inspect capability --help`.
|
||||
Binary file not shown.
+31
@@ -0,0 +1,31 @@
|
||||
# Inspecting TensorRT ONNX Support
|
||||
|
||||
## Introduction
|
||||
|
||||
The `inspect capability` subtool provides detailed information on TensorRT's ONNX operator support for a given ONNX graph.
|
||||
It is also able to partition and save supported and unsupported subgraphs from the original model in order to report all the dynamically checked errors with a given model (see the example `08_inspecting_tensorrt_onnx_support`).
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Generate the capability report
|
||||
|
||||
```bash
|
||||
polygraphy inspect capability nested_local_function.onnx
|
||||
```
|
||||
|
||||
2. This should display a summary table like:
|
||||
|
||||
```
|
||||
[I] ===== Summary =====
|
||||
Stack trace | Operator | Node | Reason
|
||||
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
onnx_graphsurgeon_node_1 (OuterFunction) -> onnx_graphsurgeon_node_1 (NestedLocalFake2) | Fake_2 | nested_node_fake_2 | In node 0 with name: nested_node_fake_2 and operator: Fake_2 (checkFallbackPluginImporter): INVALID_NODE: creator && "Plugin not found, are the plugin name, version, and namespace correct?"
|
||||
onnx_graphsurgeon_node_1 (OuterFunction) | Fake_1 | nested_node_fake_1 | In node 0 with name: nested_node_fake_1 and operator: Fake_1 (checkFallbackPluginImporter): INVALID_NODE: creator && "Plugin not found, are the plugin name, version, and namespace correct?"
|
||||
```
|
||||
|
||||
## Understanding The Output
|
||||
|
||||
In this example, `nested_local_function.onnx` contains `Fake_1` and `Fake_2` nodes that are not supported by TensorRT. `Fake_1` node is located inside a local function `OuterFunction` and `Fake_2` node is located inside a nested local function, `NestedLocalFake2`.
|
||||
The summary table shows the current stack trace consisting of local functions, the operator in which the error occurred and the reason it's unsupported.
|
||||
|
||||
For more information and options, see `polygraphy inspect capability --help`.
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,36 @@
|
||||
# Using Shard To Convert a SD Model to MD
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
The `shard` tool can be used to convert single-device (SD) models containing attention layers into multi-device (MD) models intended to be run on multiple GPUs using a hints file.
|
||||
|
||||
In this example, we'll show how to shard a simple model containing an attention layer
|
||||
|
||||
|
||||
|
||||
## Hint Configuration
|
||||
|
||||
For this example we'll be using [this hints file](./hint.json).
|
||||
|
||||
See the [Shard README](../../../../polygraphy/tools/multi_device/README.md#sharding-hints-file-format) for an explanation of the hints file format.
|
||||
|
||||
|
||||
## Running the Example
|
||||
|
||||
```bash
|
||||
polygraphy multi-device shard \
|
||||
../attention.onnx \
|
||||
-s hint.json \
|
||||
-o attention_md.onnx
|
||||
```
|
||||
|
||||
Looking at the result, we can now see the model is ready to be run on multiple GPUs through TensorRT
|
||||
|
||||
|
||||
|
||||
|
||||
### A Note On Gathering Q
|
||||
|
||||
If we changed `gather_q` in the hints to `true` the model effectively becomes SD, and a final all-gather will not be inserted. All attention layers must have Q consistently sharded, as it affects whether or not to place an all-gather at the output of the model
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"parallelism": "CP",
|
||||
"group_size": 0,
|
||||
"root": 0,
|
||||
"groups": [],
|
||||
"attention_layers": [
|
||||
{
|
||||
"q": "q",
|
||||
"gather_kv": true,
|
||||
"gather_q": false,
|
||||
"polygraphy_class": "AttentionLayerHint"
|
||||
}
|
||||
],
|
||||
"inputs": [
|
||||
{
|
||||
"name": "input",
|
||||
"seq_len_idx": 0,
|
||||
"rank": 3,
|
||||
"polygraphy_class": "ShardTensor"
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "output",
|
||||
"seq_len_idx": 0,
|
||||
"rank": 3,
|
||||
"polygraphy_class": "ShardTensor"
|
||||
}
|
||||
],
|
||||
"k_seq_len_idx": 0,
|
||||
"v_seq_len_idx": 0,
|
||||
"kv_rank": null,
|
||||
"reduce_scatter_reduce_op": "max",
|
||||
"polygraphy_class": "ShardHints"
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 42 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 58 KiB |
Binary file not shown.
@@ -0,0 +1,126 @@
|
||||
# Matching and replacing a subgraph with a plugin in an onnx model
|
||||
|
||||
## Introduction
|
||||
|
||||
The `plugin` tool offers subtools to find and replace subgraphs in an onnx model.
|
||||
|
||||
Subgraph substition is a three-step process:
|
||||
|
||||
1. Find matching subgraphs based on the plugin's graph pattern (pattern.py) and list the potential substitutions in a user-editable intermediate file (config.yaml)
|
||||
2. Review and edit (if necessary) the list of potential substitutions (config.yaml)
|
||||
3. Replace subgraphs with plugins based on the list of potential substitutions (config.yaml)
|
||||
|
||||
`original.onnx` -------> `match` -------> `config.yaml` -------> `replace` -------> `replaced.onnx`
|
||||
`plugins` ----------------^ `usr input`---^ `plugins`--------^
|
||||
|
||||
## Details
|
||||
|
||||
### Match
|
||||
|
||||
Finding matchings subgraphs in a model is done based on a graph pattern description (`pattern.py`) provided by the plugins.
|
||||
The graph pattern description (`pattern.py`) contains information about the topology and additional constraints for the graph nodes, and a way to calculate the plugin's attributes based on the matching subgraph.
|
||||
Only plugins which provide a graph pattern description (pattern.py) are considered for matching.
|
||||
|
||||
The result of the matching is stored in an intermediate file called `config.yaml`.
|
||||
The user should review and edit this file, as it serves as a TODO list for the replacement step. For example, if there are 2 matching subgraphs, but only one should be substituted, the result can be removed from the file.
|
||||
|
||||
As a preview/dry-run step, the `plugin list` subtool can show the list of potential substitutions without generating an intermediate file.
|
||||
|
||||
### Replace
|
||||
|
||||
Replacement of subgraphs with plugins uses the `config.yaml` file generated in the matching stage. Any matching subgraph listed in this file is going to be removed and replaced with a single node representing the plugin. The original file is kept, and a new file is saved where the replacements are done. This file by default is called `replaced.onnx`.
|
||||
|
||||
### Compare
|
||||
|
||||
The original and the replaced model can be compared to check if they behave the same way before and after plugin substitution:
|
||||
`polygraphy run original.onnx --trt --save-outputs model_output.json`
|
||||
`polygraphy run replaced.onnx --trt --load-outputs model_output.json`
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Find and save matches of toyPlugin in the example network:
|
||||
|
||||
```bash
|
||||
polygraphy plugin match toy_subgraph.onnx \
|
||||
--plugin-dir ./plugins -o config.yaml
|
||||
```
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
|
||||
This will display something like:
|
||||
|
||||
```
|
||||
checking toyPlugin in model
|
||||
[I] Start a subgraph matching...
|
||||
[I] Checking node: n1 against pattern node: Anode.
|
||||
[I] No match because: Op did not match. Node op was: O but pattern op was: A.
|
||||
[I] Start a subgraph matching...
|
||||
[I] Found a matched subgraph!
|
||||
[I] Start a subgraph matching...
|
||||
```
|
||||
|
||||
The resulting config.yaml will look like:
|
||||
|
||||
```
|
||||
name: toyPlugin
|
||||
instances:
|
||||
- inputs:
|
||||
- i1
|
||||
- i1
|
||||
outputs:
|
||||
- o1
|
||||
- o2
|
||||
attributes:
|
||||
x: 1
|
||||
```
|
||||
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
|
||||
2. **[Optional]** List matches of toyPlugin in the example network, without saving config.yaml:
|
||||
|
||||
```bash
|
||||
polygraphy plugin list toy_subgraph.onnx \
|
||||
--plugin-dir ./plugins
|
||||
```
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
|
||||
This will display something like:
|
||||
|
||||
```
|
||||
checking toyPlugin in model
|
||||
[I] Start a subgraph matching...
|
||||
[I] Checking node: n1 against pattern node: Anode.
|
||||
[I] No match because: Op did not match. Node op was: O but pattern op was: A.
|
||||
[I] Start a subgraph matching...
|
||||
...
|
||||
[I] Found a matched subgraph!
|
||||
[I] Start a subgraph matching...
|
||||
[I] Checking node: n6 against pattern node: Anode.
|
||||
[I] No match because: Op did not match. Node op was: E but pattern op was: A.
|
||||
the following plugins would be used:
|
||||
{'toyPlugin': 1}
|
||||
```
|
||||
|
||||
There will be no resulting config.yaml, as this command is only for printing the number of matches per plugin
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
|
||||
The `plugin replace` subtool replaces subgraphs in an onnx model with plugins
|
||||
|
||||
3. Replace parts of the example network with toyPlugin:
|
||||
|
||||
```bash
|
||||
polygraphy plugin replace toy_subgraph.onnx \
|
||||
--plugin-dir ./plugins --config config.yaml -o replaced.onnx
|
||||
```
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
|
||||
This will display something like:
|
||||
|
||||
```
|
||||
[I] Loading model: toy_subgraph.onnx
|
||||
```
|
||||
|
||||
The result file is replaced.onnx, where a subgraph in the example network is replaced by toyPlugin
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
from polygraphy import mod
|
||||
gs = mod.lazy_import("onnx_graphsurgeon>=0.5.0")
|
||||
from typing import List,Dict
|
||||
|
||||
def get_plugin_pattern():
|
||||
"""
|
||||
Toy plugin pattern:
|
||||
A B
|
||||
\ /
|
||||
C, attrs['x'] < 2.0
|
||||
/ \
|
||||
D E
|
||||
"""
|
||||
pattern = gs.GraphPattern()
|
||||
in_0 = pattern.variable()
|
||||
in_1 = pattern.variable()
|
||||
a_out = pattern.add("Anode", "A", inputs=[in_0])
|
||||
b_out = pattern.add("Bnode", "B", inputs=[in_1])
|
||||
check_function = lambda node : node.attrs["x"] < 2.0
|
||||
c_out = pattern.add("Cnode", "C", inputs=[a_out, b_out], check_func=check_function)
|
||||
d_out = pattern.add("Dnode", "D", inputs=[c_out])
|
||||
e_out = pattern.add("Enode", "E", inputs=[c_out])
|
||||
pattern.set_output_tensors([d_out, e_out])
|
||||
|
||||
return pattern
|
||||
|
||||
def get_matching_subgraphs(graph) -> List[Dict[str,str]]:
|
||||
gp = get_plugin_pattern()
|
||||
matches = gp.match_all(graph)
|
||||
ans = []
|
||||
for m in matches:
|
||||
# save the input and output tensor names of the matching subgraph(s)
|
||||
input_tensors = list(set([ip_tensor.name for ip_tensor in m.inputs]))
|
||||
output_tensors = list(set([op_tensor.name for op_tensor in m.outputs]))
|
||||
|
||||
attrs = {"ToyX": int(m.get("Cnode").attrs["x"]) * 2}
|
||||
ioa = {
|
||||
'inputs':input_tensors,
|
||||
'outputs':output_tensors,
|
||||
'attributes':attrs
|
||||
}
|
||||
ans.append(ioa)
|
||||
return ans
|
||||
|
||||
def get_plugin_metadata() -> Dict[str,str]:
|
||||
return {'name':'toyPlugin',
|
||||
'op':'CustomToyPlugin',
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,153 @@
|
||||
# Comparing Frameworks
|
||||
|
||||
## Introduction
|
||||
|
||||
You can use the `run` subtool to compare a model between different frameworks.
|
||||
In the simplest case, you can supply a model, and one or more framework flags.
|
||||
By default, it will generate synthetic input data, run inference using the
|
||||
specified frameworks, then compare outputs of the specified frameworks.
|
||||
|
||||
## Running The Example
|
||||
|
||||
In this example, we'll outline various common use-cases for the `run` subtool:
|
||||
|
||||
- [Comparing TensorRT And ONNX-Runtime Outputs](#comparing-tensorrt-and-onnx-runtime-outputs)
|
||||
- [Comparing TensorRT Precisions](#comparing-tensorrt-precisions)
|
||||
- [Changing Tolerances](#changing-tolerances)
|
||||
- [Changing Comparison Metrics](#changing-comparison-metrics)
|
||||
- [Comparing Per-Layer Outputs Between ONNX-Runtime And TensorRT](#comparing-per-layer-outputs-between-onnx-runtime-and-tensorrt)
|
||||
|
||||
### Comparing TensorRT And ONNX-Runtime Outputs
|
||||
|
||||
To run the model in Polygraphy with both frameworks and perform an output
|
||||
comparison:
|
||||
|
||||
```bash
|
||||
polygraphy run dynamic_identity.onnx --trt --onnxrt
|
||||
```
|
||||
|
||||
The `dynamic_identity.onnx` model has dynamic input shapes. By default,
|
||||
Polygraphy will override any dynamic input dimensions in the model to
|
||||
`constants.DEFAULT_SHAPE_VALUE` (defined as `1`) and warn you:
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
```
|
||||
[W] Input tensor: X (dtype=DataType.FLOAT, shape=(1, 2, -1, -1)) | No shapes provided; Will use shape: [1, 2, 1, 1] for min/opt/max in profile.
|
||||
[W] This will cause the tensor to have a static shape. If this is incorrect, please set the range of shapes for this input tensor.
|
||||
```
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
|
||||
In order to suppress this message and explicitly provide input shapes to
|
||||
Polygraphy, use the `--input-shapes` option:
|
||||
|
||||
```
|
||||
polygraphy run dynamic_identity.onnx --trt --onnxrt \
|
||||
--input-shapes X:[1,2,4,4]
|
||||
```
|
||||
|
||||
### Comparing TensorRT Precisions
|
||||
|
||||
To build a TensorRT engine with reduced precision layers for comparison against
|
||||
ONNXRT, use one of the supported precision flags (e.g. `--tf32`, `--fp16`,`--int8`, etc.).
|
||||
For example:
|
||||
|
||||
```bash
|
||||
polygraphy run dynamic_identity.onnx --trt --fp16 --onnxrt \
|
||||
--input-shapes X:[1,2,4,4]
|
||||
```
|
||||
|
||||
> :warning: Getting acceptable accuracy with INT8 precision typically requires an additional calibration step:
|
||||
see the [developer guide](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#working-with-int8)
|
||||
and instructions on [how to do calibration](../../../../examples/cli/convert/01_int8_calibration_in_tensorrt)
|
||||
with Polygraphy on the command line.
|
||||
|
||||
### Changing Tolerances
|
||||
|
||||
The default tolerances used by `run` are usually appropriate for FP32 precision
|
||||
but may not be appropriate for reduced precisions. In order to relax tolerances,
|
||||
you can use the `--atol` and `--rtol` options to set absolute and relative
|
||||
tolerance respectively.
|
||||
|
||||
### Changing Comparison Metrics
|
||||
|
||||
You can use the `--check-error-stat` option to change the metric used for
|
||||
comparison. By default, Polygraphy uses an "elementwise" metric
|
||||
(`--check-error-stat elemwise`).
|
||||
|
||||
Other possible metrics for `--check-error-stat` are `mean`, `median`, and `max`, which
|
||||
compares the mean, median, and maximum absolute/relative error across the tensor, respectively.
|
||||
|
||||
To better understand this, suppose we are
|
||||
comparing two outputs `out0` and `out1`. Polygraphy takes
|
||||
the elementwise absolute and relative difference of these tensors:
|
||||
|
||||
<!-- Polygraphy Test: Ignore Start -->
|
||||
```
|
||||
absdiff = out0 - out1
|
||||
reldiff = absdiff / abs(out1)
|
||||
```
|
||||
<!-- Polygraphy Test: Ignore End -->
|
||||
|
||||
Then, for each index `i` in the output, Polygraphy checks whether
|
||||
`absdiff[i] > atol and reldiff[i] > rtol`. If any index satisfies this,
|
||||
then the comparison will fail. This is less stringent than comparing the maximum
|
||||
absolute and relative error across the entire tensor (`--check-error-stat max`) since if
|
||||
*different* indices `i` and `j` satisfy `absdiff[i] > atol` and `reldiff[j] > rtol`,
|
||||
then the `max` comparison will fail but the `elemwise` comparison may
|
||||
pass.
|
||||
|
||||
Putting it all together, the below example runs a `median` comparison between
|
||||
TensorRT using FP16 and ONNX-Runtime, using absolute and relative tolerances of `0.001`:
|
||||
|
||||
```bash
|
||||
polygraphy run dynamic_identity.onnx --trt --fp16 --onnxrt \
|
||||
--input-shapes X:[1,2,4,4] \
|
||||
--atol 0.001 --rtol 0.001 --check-error-stat median
|
||||
```
|
||||
|
||||
> You can also specify per-output values for `--atol`/`--rtol`/`--check-error-stat`.
|
||||
See the help output of the `run` subtool for more information.
|
||||
|
||||
### Comparing Per-Layer Outputs Between ONNX-Runtime And TensorRT
|
||||
|
||||
When network outputs do not match, it can be useful to compare per-layer outputs
|
||||
to see where the error is introduced. To do so, you can use the `--trt-outputs`
|
||||
and `--onnx-outputs` options respectively. These options accept one or more
|
||||
output names as their arguments. The special value `mark all` indicates that all
|
||||
tensors in the model should be compared:
|
||||
|
||||
```bash
|
||||
polygraphy run dynamic_identity.onnx --trt --onnxrt \
|
||||
--trt-outputs mark all \
|
||||
--onnx-outputs mark all
|
||||
```
|
||||
|
||||
To find the first mismatched output more easily, you can use the `--fail-fast`
|
||||
option which will cause the tool to exit after the first mismatch between
|
||||
outputs.
|
||||
|
||||
Note that use of `--trt-outputs mark all` can sometimes perturb the generated
|
||||
engine due to differences in timing, layer fusion choices, and format
|
||||
constraints, which can hide the failure. In that case, you may have to use a
|
||||
more sophisticated approach to bisect the failing model and generate a reduced
|
||||
test case that reproduces the error. See [Reducing Failing ONNX
|
||||
Models](../../../../examples/cli/debug/02_reducing_failing_onnx_models) for a tutorial on
|
||||
how to do this with Polygraphy.
|
||||
|
||||
## Further Reading
|
||||
|
||||
* In some cases you may need to do comparisons across multiple Polygraphy runs
|
||||
(for example, when comparing the output of a pre-built TensorRT engine or
|
||||
[Polygraphy network script](../../../../examples/cli/run/04_defining_a_tensorrt_network_or_config_manually)
|
||||
against ONNX-Runtime). See [Comparing Across Runs](../../../../examples/cli/run/02_comparing_across_runs) for a tutorial on how to
|
||||
accomplish this.
|
||||
|
||||
* For more details on working with dynamic shapes in TensorRT:
|
||||
* See [Dynamic Shapes in TensorRT](../../../../examples/cli/convert/03_dynamic_shapes_in_tensorrt/) for how to specify
|
||||
optimization profiles for use with the engine using the Polygraphy CLI
|
||||
* See [TensorRT and Dynamic Shapes](../../../../examples/api/07_tensorrt_and_dynamic_shapes/) for details on
|
||||
how to do this with the Polygraphy API
|
||||
|
||||
* For details on how to supply real input data, see [Comparing with Custom Input Data](../05_comparing_with_custom_input_data/).
|
||||
|
||||
* See [Debugging TensorRT Accuracy Issues](../../../../how-to/debug_accuracy.md) for a broader tutorial on how to debug accuracy failures using Polygraphy.
|
||||
@@ -0,0 +1,19 @@
|
||||
:·
|
||||
|
||||
X intermediate"Identity
|
||||
|
||||
intermediateY"Identityonnx_dynamic_identityZ&
|
||||
X!
|
||||
|
||||
|
||||
|
||||
height
|
||||
widthb
|
||||
Y
|
||||
j1
|
||||
intermediate!
|
||||
|
||||
|
||||
|
||||
height
|
||||
widthB
|
||||
@@ -0,0 +1,72 @@
|
||||
# Comparing Across Runs
|
||||
|
||||
## Prerequisites
|
||||
For a general overview of how to use `polygraphy run` to compare the outputs of
|
||||
different frameworks, see the example on [Comparing Frameworks](../../../../examples/cli/run/01_comparing_frameworks).
|
||||
|
||||
## Introduction
|
||||
|
||||
There are situations where you may need to compare results across different invocations
|
||||
of the `polygraphy run` command. Some examples of this include:
|
||||
|
||||
* Comparing results across different platforms
|
||||
* Comparing results across different versions of TensorRT
|
||||
* Comparing different model types with compatible input(s)/output(s)
|
||||
|
||||
In this example, we'll demonstrate how to accomplish this with Polygraphy.
|
||||
|
||||
## Running The Example
|
||||
|
||||
### Comparing Across Runs
|
||||
|
||||
1. Save the input and output values from the first run:
|
||||
|
||||
```bash
|
||||
polygraphy run identity.onnx --onnxrt \
|
||||
--save-inputs inputs.json --save-outputs run_0_outputs.json
|
||||
```
|
||||
|
||||
2. Run the model again, this time loading the saved inputs and outputs from
|
||||
the first run. The saved inputs will be used as inputs for the current run, and
|
||||
the saved outputs will be used to compare against the first run.
|
||||
|
||||
```bash
|
||||
polygraphy run identity.onnx --onnxrt \
|
||||
--load-inputs inputs.json --load-outputs run_0_outputs.json
|
||||
```
|
||||
|
||||
The `--atol/--rtol/--check-error-stat` options all work the same as in the
|
||||
[Comparing Frameworks](../../../../examples/cli/run/01_comparing_frameworks) example:
|
||||
|
||||
```bash
|
||||
polygraphy run identity.onnx --onnxrt \
|
||||
--load-inputs inputs.json --load-outputs run_0_outputs.json \
|
||||
--atol 0.001 --rtol 0.001 --check-error-stat median
|
||||
```
|
||||
|
||||
### Comparing Different Models
|
||||
|
||||
We can also use this technique to compare different models, like TensorRT engines
|
||||
and ONNX modles (if they have matching outputs).
|
||||
|
||||
1. Convert the ONNX model to a TensorRT engine and save it to disk:
|
||||
|
||||
```bash
|
||||
polygraphy convert identity.onnx -o identity.engine
|
||||
```
|
||||
|
||||
2. Run the saved engine in Polygraphy, using the saved inputs from the ONNX-Runtime run as
|
||||
inputs to the engine, and compare the engine's outputs to the saved ONNX-Runtime outputs:
|
||||
|
||||
```bash
|
||||
polygraphy run --trt identity.engine --model-type=engine \
|
||||
--load-inputs inputs.json --load-outputs run_0_outputs.json
|
||||
```
|
||||
|
||||
|
||||
## Further Reading
|
||||
|
||||
For details on how to access and work with the saved outputs
|
||||
using the Python API, refer to [API example 08](../../../api/08_working_with_run_results_and_saved_inputs_manually/).
|
||||
|
||||
For information on comparing against custom outputs, refer to [`run` example 06](../06_comparing_with_custom_output_data/).
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
|
||||
# Generating A Script For Advanced Comparisons
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
For more advanced requirements, you may want to use the [API](../../../../polygraphy).
|
||||
Instead of writing a script from scratch, you can use `run`'s `--gen-script` option
|
||||
to create a Python script that you can use as a starting point.
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Generate a comparison script:
|
||||
|
||||
```bash
|
||||
polygraphy run identity.onnx --trt --onnxrt \
|
||||
--gen-script=compare_trt_onnxrt.py
|
||||
```
|
||||
|
||||
The generated script will do exactly what the `run` command would otherwise do.
|
||||
|
||||
2. Run the comparison script, optionally after modifying it:
|
||||
|
||||
```bash
|
||||
python3 compare_trt_onnxrt.py
|
||||
```
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
# Defining A TensorRT Network Or Config Manually
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
In some cases, it can be useful to define a TensorRT network from scratch using the Python API,
|
||||
or modify a network created by other means (e.g. a parser). Normally, this would restrict you
|
||||
from using CLI tools, at least until you build an engine, since the network cannot be serialized
|
||||
to disk and loaded on the command-line.
|
||||
|
||||
Polygraphy CLI tools provide a work-around for this - if your Python script defines a function
|
||||
named `load_network`, which takes no parameters and returns a TensorRT builder, network,
|
||||
and optionally parser, then you can provide your Python script in place of a model argument.
|
||||
|
||||
Similarly, we can create a custom TensorRT builder configuration using a script that defines
|
||||
a function called `load_config` which accepts a builder and network and returns a builder configuration.
|
||||
|
||||
In this example, the included `define_network.py` script parses an ONNX model and appends an identity
|
||||
layer to it. Since it returns the builder, network, and parser in a function called `load_network`,
|
||||
we can build and run a TensorRT engine from it using just a single command. The `create_config.py`
|
||||
script creates a new TensorRT builder configuration and enables FP16 mode.
|
||||
|
||||
|
||||
### TIP: Generating Script Templates Automatically
|
||||
|
||||
Instead of writing the network script from scratch, you can use
|
||||
`polygraphy template trt-network` to give you a starting point:
|
||||
|
||||
```bash
|
||||
polygraphy template trt-network -o my_define_network.py
|
||||
```
|
||||
|
||||
If you want to start from a model and modify the resulting TensorRT network instead
|
||||
of creating one from scratch, simply provide the model as an argument to `template trt-network`:
|
||||
|
||||
```bash
|
||||
polygraphy template trt-network identity.onnx -o my_define_network.py
|
||||
```
|
||||
|
||||
Similarly, you can generate a template script for the config using `polygraphy template trt-config`:
|
||||
|
||||
```bash
|
||||
polygraphy template trt-config -o my_create_config.py
|
||||
```
|
||||
|
||||
You can also specify builder configuration options to pre-populate the script.
|
||||
For example, to enable FP16 mode:
|
||||
|
||||
```bash
|
||||
polygraphy template trt-config --fp16 -o my_create_config.py
|
||||
```
|
||||
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Run the network defined in `define_network.py`:
|
||||
|
||||
```bash
|
||||
polygraphy run --trt define_network.py --model-type=trt-network-script
|
||||
```
|
||||
|
||||
2. Run the network from step (1) using the builder configuration defined in `create_config.py`:
|
||||
|
||||
```bash
|
||||
polygraphy run --trt define_network.py --model-type=trt-network-script --trt-config-script=create_config.py
|
||||
```
|
||||
|
||||
Note that we could have defined both `load_network` and `load_config` in the same script.
|
||||
In fact, we could have retrieved these functions from arbitrary scripts, or even modules.
|
||||
|
||||
*TIP: We can use the same approach with `polygraphy convert` to build, but not run, the engine.*
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""
|
||||
Creates a TensorRT builder configuration and enables FP16 tactics.
|
||||
"""
|
||||
import tensorrt as trt
|
||||
from polygraphy import func
|
||||
from polygraphy.backend.trt import CreateConfig
|
||||
|
||||
|
||||
# If we define a function called `load_config`, polygraphy can use it to
|
||||
# create the builder configuration.
|
||||
#
|
||||
# TIP: If our function isn't called `load_config`, we can explicitly specify
|
||||
# the name with the script argument, separated by a colon. For example: `create_config.py:my_func`.
|
||||
@func.extend(CreateConfig())
|
||||
def load_config(config):
|
||||
# NOTE: func.extend() causes the signature of this function to be `(builder, network) -> config`
|
||||
# For details on how this works, see examples/api/03_interoperating_with_tensorrt
|
||||
|
||||
config.set_flag(trt.BuilderFlag.FP16)
|
||||
|
||||
# Notice that we don't need to return anything - `extend()` takes care of that for us!
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
"""
|
||||
Parses an ONNX model, and then extends it with an Identity layer.
|
||||
"""
|
||||
from polygraphy import func
|
||||
from polygraphy.backend.trt import NetworkFromOnnxPath
|
||||
|
||||
parse_onnx = NetworkFromOnnxPath("identity.onnx")
|
||||
|
||||
|
||||
# If we define a function called `load_network`, polygraphy can
|
||||
# use it directly in place of using a model file.
|
||||
#
|
||||
# TIP: If our function isn't called `load_network`, we can explicitly specify
|
||||
# the name with the model argument, separated by a colon. For example, `define_network.py:my_func`.
|
||||
@func.extend(parse_onnx)
|
||||
def load_network(builder, network, parser):
|
||||
# NOTE: func.extend() causes the signature of this function to be `() -> (builder, network, parser)`
|
||||
# For details on how this works, see examples/api/03_interoperating_with_tensorrt
|
||||
|
||||
# Append an identity layer to the network
|
||||
prev_output = network.get_output(0)
|
||||
network.unmark_output(prev_output)
|
||||
|
||||
output = network.add_identity(prev_output).get_output(0)
|
||||
network.mark_output(output)
|
||||
|
||||
# Notice that we don't need to return anything - `extend()` takes care of that for us!
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
# Comparing With Custom Input Data
|
||||
|
||||
## Introduction
|
||||
|
||||
In some cases, we may want to run comparisons using custom input data.
|
||||
Polygraphy provides multiple ways to do so, which are detailed [here](../../../../how-to/use_custom_input_data.md).
|
||||
|
||||
In this example, we'll demonstrate 2 different approaches:
|
||||
|
||||
1. Using a data loader script by defining a `load_data()` function in a Python script (`data_loader.py`).
|
||||
Polygraphy will use `load_data()` to generate inputs at runtime.
|
||||
|
||||
2. Using a JSON file containing pre-generated inputs.
|
||||
For convenience, we'll use our script from above (`data_loader.py`) to save the inputs
|
||||
generated by `load_data()` to a file called `custom_inputs.json`.
|
||||
|
||||
*TIP: Generally, a data loader script is preferrable when working with large amounts of input data*
|
||||
*as it avoids the need to write to the disk.*
|
||||
*On the other hand, JSON files may be more portable and can help ensure reproducibility.*
|
||||
|
||||
Finally, we'll supply our custom input data to `polygraphy run` and compare outputs between
|
||||
ONNX-Runtime and TensorRT.
|
||||
|
||||
Since our model has dynamic shapes, we'll need to set up a TensorRT Optimization Profile.
|
||||
For details on how we can do this via the command-line,
|
||||
see [`convert` example 03](../../convert/03_dynamic_shapes_in_tensorrt).
|
||||
For simplicitly, we'll create a profile where `min` == `opt` == `max`.
|
||||
|
||||
*NOTE: It is important that our optimization profile works with the shapes provided by our*
|
||||
*custom data loader. In our very simple case, the data loader always generates inputs of*
|
||||
*shape (1, 2, 28, 28), so we just need to ensure this falls within [`min`, `max`].*
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Run the script to save input data to the disk.
|
||||
*NOTE: This is only necessary for option 2.*
|
||||
```bash
|
||||
python3 data_loader.py
|
||||
```
|
||||
|
||||
2. Run the model with TensorRT and ONNX-Runtime using custom input data:
|
||||
- Option 1: Using the data loader script:
|
||||
|
||||
```bash
|
||||
polygraphy run dynamic_identity.onnx --trt --onnxrt \
|
||||
--trt-min-shapes X:[1,2,28,28] --trt-opt-shapes X:[1,2,28,28] --trt-max-shapes X:[1,2,28,28] \
|
||||
--data-loader-script data_loader.py
|
||||
```
|
||||
|
||||
- Option 2: Using the JSON file containing the saved inputs:
|
||||
|
||||
```bash
|
||||
polygraphy run dynamic_identity.onnx --trt --onnxrt \
|
||||
--trt-min-shapes X:[1,2,28,28] --trt-opt-shapes X:[1,2,28,28] --trt-max-shapes X:[1,2,28,28] \
|
||||
--load-inputs custom_inputs.json
|
||||
```
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
Demonstrates two methods of loading custom input data in Polygraphy:
|
||||
|
||||
Option 1: Defines a `load_data` function that returns a generator yielding
|
||||
feed_dicts so that this script can be used as the argument for
|
||||
the --data-loader-script command-line parameter.
|
||||
|
||||
Option 2: Writes input data to a JSON file that can be used as the argument for
|
||||
the --load-inputs command-line parameter.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.json import save_json
|
||||
|
||||
INPUT_SHAPE = (1, 2, 28, 28)
|
||||
|
||||
|
||||
# Option 1: Define a function that will yield feed_dicts (i.e. Dict[str, np.ndarray])
|
||||
def load_data():
|
||||
for _ in range(5):
|
||||
yield {
|
||||
"x": np.ones(shape=INPUT_SHAPE, dtype=np.float32)
|
||||
} # Still totally real data
|
||||
|
||||
|
||||
# Option 2: Create a JSON file containing the input data using the `save_json()` helper.
|
||||
# The input to `save_json()` should have type: List[Dict[str, np.ndarray]].
|
||||
# For convenience, we'll reuse our `load_data()` implementation to generate the list.
|
||||
input_data = list(load_data())
|
||||
save_json(input_data, "custom_inputs.json", description="custom input data")
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
backend_test:y
|
||||
|
||||
XY"Identityonnx_dynamic_identityZ&
|
||||
X!
|
||||
|
||||
|
||||
|
||||
height
|
||||
widthb&
|
||||
Y!
|
||||
|
||||
|
||||
|
||||
height
|
||||
widthB
|
||||
@@ -0,0 +1,43 @@
|
||||
# Comparing With Custom Output Data
|
||||
|
||||
## Introduction
|
||||
|
||||
In some cases, it may be useful to compare against output values generated outside Polygraphy.
|
||||
The simplest way to do so is to create a `RunResults` object and save it to a file.
|
||||
|
||||
This example illustrates how you can generate custom input and output data outside of Polygraphy
|
||||
and seamlessly load it into Polygraphy for comparison.
|
||||
|
||||
## Running The Example
|
||||
|
||||
1. Generate the input and output data:
|
||||
|
||||
```bash
|
||||
python3 generate_data.py
|
||||
```
|
||||
|
||||
2. **[Optional]** Inspect the data.
|
||||
For inputs:
|
||||
|
||||
```bash
|
||||
polygraphy inspect data custom_inputs.json
|
||||
```
|
||||
|
||||
For outputs:
|
||||
|
||||
```bash
|
||||
polygraphy inspect data custom_outputs.json
|
||||
```
|
||||
|
||||
3. Run inference with the generated input data and then compare outputs against the custom outputs:
|
||||
|
||||
```bash
|
||||
polygraphy run identity.onnx --trt \
|
||||
--load-inputs custom_inputs.json \
|
||||
--load-outputs custom_outputs.json
|
||||
```
|
||||
|
||||
## Further Reading
|
||||
|
||||
For details on how to access and work with the outputs stored in `RunResults` objects
|
||||
using the Python API, refer to [API example 08](../../../api/08_working_with_run_results_and_saved_inputs_manually/).
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
"""
|
||||
Generates input and output data for an identity model and saves it to disk.
|
||||
"""
|
||||
import numpy as np
|
||||
from polygraphy.comparator import RunResults
|
||||
from polygraphy.json import save_json
|
||||
|
||||
INPUT_SHAPE = (1, 1, 2, 2)
|
||||
|
||||
|
||||
# We'll generate arbitrary input data and then "compute" the expected output data before saving both to disk.
|
||||
# In order for Polygraphy to load the input and output data, they must be in the following format:
|
||||
# - Input Data: List[Dict[str, np.ndarray]] (A list of feed_dicts)
|
||||
# - Output Data: RunResults
|
||||
|
||||
|
||||
# Generate arbitrary input data compatible with the model.
|
||||
#
|
||||
# TIP: We could have alternatively used a generator as in `run` example 05 (05_comparing_with_custom_input_data).
|
||||
# In that case, we would simply provide this script to `--data-loader-script` instead of saving the inputs here
|
||||
# and then using `--load-inputs`.
|
||||
input_data = {"x": np.ones(shape=INPUT_SHAPE, dtype=np.float32)}
|
||||
|
||||
# NOTE: Input data must be in a list (to support multiple sets of inputs), so we create one before saving it.
|
||||
# The `description` argument is optional:
|
||||
save_json([input_data], "custom_inputs.json", description="custom input data")
|
||||
|
||||
|
||||
# "Compute" the outputs based on the input data. Since this is an identity model, we can just copy the inputs.
|
||||
output_data = {"y": input_data["x"]}
|
||||
|
||||
# To save output data, we can create a RunResults object:
|
||||
custom_outputs = RunResults()
|
||||
|
||||
# The `add()` helper function allows us to easily add entries.
|
||||
#
|
||||
# NOTE: As with input data, output data must be in a list, so we create one before saving it.
|
||||
#
|
||||
# TIP: Alternatively, we can manually add entries using an approach like:
|
||||
# runner_name = "custom_runner"
|
||||
# custom_outputs[runner_name] = [IterationResult(output_data, runner_name=runner_name), ...]
|
||||
#
|
||||
# TIP: To store outputs from multiple different implementations, you can specify different `runner_name`s to `add()`.
|
||||
# If `runner_name` is omitted, a default is used.
|
||||
custom_outputs.add([output_data], runner_name="custom_runner")
|
||||
custom_outputs.save("custom_outputs.json")
|
||||
@@ -0,0 +1,15 @@
|
||||
backend-test:[
|
||||
|
||||
xy"Identity
|
||||
test_identityZ
|
||||
x
|
||||
|
||||
|
||||
|
||||
|
||||
b
|
||||
y
|
||||
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user