chore: import upstream snapshot with attribution
Docker Image CI / build-ubuntu2004 (push) Has been cancelled
Docker Image CI / build-ubuntu2004 (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
#
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
add_executable(trtexec_safe trtSafeExec.cpp delayStreamKernel.cu)
|
||||
|
||||
if(TRT_SAFETY_INFERENCE_ONLY)
|
||||
target_link_libraries(trtexec_safe PRIVATE trt_global_definitions tensorrt_headers)
|
||||
target_include_directories(trtexec_safe PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/../common
|
||||
)
|
||||
# Gate out the CUDA kernel in delayStreamKernel.cu that uses __cudaLaunchKernel
|
||||
# (not available in SafeCUDA); SafetyInferenceOnly.cmake sets the CMake variable
|
||||
# but does not propagate it as a preprocessor define.
|
||||
target_compile_definitions(trtexec_safe PRIVATE TRT_SAFETY_INFERENCE_ONLY)
|
||||
else()
|
||||
target_link_libraries(trtexec_safe PRIVATE trt_samples_common TRTSAFE::nvinfer_safe_debug)
|
||||
endif()
|
||||
|
||||
if(TRT_OUT_DIR)
|
||||
set_target_properties(trtexec_safe
|
||||
PROPERTIES
|
||||
RUNTIME_OUTPUT_DIRECTORY "${TRT_OUT_DIR}"
|
||||
)
|
||||
endif()
|
||||
|
||||
add_dependencies(tensorrt_samples trtexec_safe)
|
||||
|
||||
installLibraries(
|
||||
TARGETS trtexec_safe
|
||||
OPTIONAL
|
||||
COMPONENT external
|
||||
EXPORT TensorRT
|
||||
)
|
||||
@@ -0,0 +1,86 @@
|
||||
# TensorRT Safety Runtime Command-Line Wrapper: trtexec_safe
|
||||
|
||||
**Table Of Contents**
|
||||
- [Description](#description)
|
||||
- [Building `trtexec_safe`](#building-trtexec_safe)
|
||||
- [Using `trtexec_safe`](#using-trtexec_safe)
|
||||
- [Tool command line arguments](#tool-command-line-arguments)
|
||||
- [Additional resources](#additional-resources)
|
||||
- [License](#license)
|
||||
- [Changelog](#changelog)
|
||||
- [Known issues](#known-issues)
|
||||
|
||||
## Description
|
||||
|
||||
`trtexec_safe` is a performance testing tool for TensorRT safety runtime. `trtexec_safe` can load a serialized engine file and test the performance of running inference with TensorRT safety runtime. `trtexec_safe` provides many options, like iterations, average runs, streams, for performance timing.
|
||||
|
||||
**Note:** `trtexec_safe` only provides the functionality of doing inference with a pre-built engine file. The engine should be built with `EngineCapability::kSAFETY`. `trtexec_safe` is not safety-certified, therefore, it could produce AUTOSAR violations in the source code.
|
||||
|
||||
## Building `trtexec_safe`
|
||||
|
||||
Compile the sample by following the build instructions in the [TensorRT README](https://github.com/NVIDIA/TensorRT/). This will build the `trtexec_safe` binary.
|
||||
|
||||
The options that control compiling against safety runtime or standard runtime are summarized in the following table.
|
||||
|
||||
| Compile Option | Default |Description|
|
||||
| ------------------------------- | ------- |---------- |
|
||||
|TRT_SAFETY_INFERENCE_ONLY | OFF |When enabled, build the infer part only, skip compiling the builder part.|
|
||||
|
||||
## Using `trtexec_safe`
|
||||
|
||||
1. `trtexec_safe` requires a prebuilt safe TensorRT engine as input. Use `trtexec` to build a safe TensorRT engine.
|
||||
```
|
||||
trtexec --safe --saveEngine=/path/to/safe/engine --model=/path/to/model
|
||||
```
|
||||
|
||||
where `--safe` indicates `trtexec` to build a safety certified TensorRT engine and `--saveEngine` specifies the path to save the output engine. For other details of `trtexec`, see `<TensorRT root directory>/samples/trtexec/README.md`.
|
||||
|
||||
|
||||
2. Use `trtexec_safe` to load the engine and run performance testing.
|
||||
|
||||
```
|
||||
trtexec_safe --loadEngine=/path/to/safe/engine
|
||||
```
|
||||
`trtexec_safe` will do inference for several iterations and give a final median execution time on both the CPU and GPU. For other options, refer to the [Tool command line arguments](#tool-command-line-arguments) section.
|
||||
|
||||
## Tool command line arguments
|
||||
|
||||
To see the full list of available options and their descriptions, use the `-h` or `--help` command line option.
|
||||
|
||||
```
|
||||
trtexec_safe [-h or --help]
|
||||
```
|
||||
|
||||
## Additional resources
|
||||
|
||||
The following resources provide more details about `trtexec_safe`:
|
||||
|
||||
**Documentation**
|
||||
- [TensorRT trtexec](https://docs.nvidia.com/deeplearning/tensorrt/developer-guide/index.html#trtexec)
|
||||
- [TensorRT Sample Support Guide](https://docs.nvidia.com/deeplearning/tensorrt/sample-support-guide/index.html)
|
||||
- [NVIDIA’s TensorRT Documentation Library](https://docs.nvidia.com/deeplearning/tensorrt/index.html)
|
||||
|
||||
# License
|
||||
|
||||
For terms and conditions for use, reproduction, and distribution, see the [TensorRT Software License Agreement](https://docs.nvidia.com/deeplearning/tensorrt/sla/index.html)
|
||||
documentation.
|
||||
|
||||
# Changelog
|
||||
|
||||
June 2020
|
||||
This is the first release of this `README.md` file.
|
||||
|
||||
September 2020
|
||||
This sample was updated to fit TensorRt API changes:
|
||||
IRuntime::validateSerializedEngine is removed.
|
||||
|
||||
March 2022
|
||||
This sample was updated for DriveOS 6.0 and later releases.
|
||||
|
||||
Dec. 2025
|
||||
This sample was updated to use the CMake-based build system.
|
||||
|
||||
# Known issues
|
||||
|
||||
There are no known issues in this sample.
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "delayStreamKernel.h"
|
||||
|
||||
#include <tuple>
|
||||
|
||||
#ifndef TRT_SAFETY_INFERENCE_ONLY
|
||||
namespace
|
||||
{
|
||||
__global__ void delayKernel(long long nanoSeconds)
|
||||
{
|
||||
// It is supported with compute capability 7.0 or higher.
|
||||
__nanosleep(nanoSeconds);
|
||||
}
|
||||
} // namespace
|
||||
#endif // TRT_SAFETY_INFERENCE_ONLY
|
||||
|
||||
namespace nvinfer1
|
||||
{
|
||||
cudaError_t delayStream(cudaStream_t stream, float timeInMsec) noexcept
|
||||
{
|
||||
#ifndef TRT_SAFETY_INFERENCE_ONLY
|
||||
auto nanoSeconds = static_cast<long long>(1000000 * timeInMsec);
|
||||
delayKernel<<<1, 1, 0, stream>>>(nanoSeconds);
|
||||
return cudaGetLastError();
|
||||
#else
|
||||
// QNX SafeCUDA does not support PTX JIT or __cudaLaunchKernel; the delay is
|
||||
// optional timing-measurement padding and has no effect on inference correctness.
|
||||
std::ignore = stream;
|
||||
std::ignore = timeInMsec;
|
||||
return cudaSuccess;
|
||||
#endif // TRT_SAFETY_INFERENCE_ONLY
|
||||
}
|
||||
} // namespace nvinfer1
|
||||
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DELAY_STREAM_KERNEL_H
|
||||
#define DELAY_STREAM_KERNEL_H
|
||||
|
||||
#include <cstdint>
|
||||
#include <cuda_runtime_api.h>
|
||||
|
||||
namespace nvinfer1
|
||||
{
|
||||
cudaError_t delayStream(cudaStream_t stream, float timeInMsec) noexcept;
|
||||
} // namespace nvinfer1
|
||||
#endif // DELAY_STREAM_KERNEL_H
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user