wehub/nvidia--tensorrt
1
0
Fork 0
Code Issues Pull Requests Actions 5 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
Docker Image CI / build-ubuntu2004 (push) Waiting to run
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:36:55 +08:00
commit c8a779b1bb
1887 changed files with 3245738 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/.clang-tidy
+11
View File
@@ -0,0 +1,11 @@
InheritParentConfig: true
CheckOptions:
- { key: readability-identifier-naming.FunctionIgnoredRegexp, value: '^[a-z][a-z0-9_]*$' }
- { key: readability-identifier-naming.LocalVariableIgnoredRegexp, value: '^[a-z][a-z0-9_]*$' }
- { key: readability-identifier-naming.GlobalVariableIgnoredRegexp, value: '^[a-z][a-z0-9_]*$' }
- { key: readability-identifier-naming.StaticVariableIgnoredRegexp, value: '^[a-z][a-z0-9_]*$' }
- { key: readability-identifier-naming.ParameterIgnoredRegexp, value: '^[a-z][a-z0-9_]*$' }
- { key: readability-identifier-naming.GlobalConstantIgnoredRegexp, value: '^[a-z][a-z0-9_]*$' }
- { key: readability-identifier-naming.StaticConstantIgnoredRegexp, value: '^(?:[a-z][a-z0-9_]*|[A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+)$' }
- { key: readability-identifier-naming.ConstexprVariableIgnoredRegexp, value: '^[a-z][a-z0-9_]*$' }
- { key: readability-identifier-naming.ClassConstantIgnoredRegexp, value: '^[a-z][a-z0-9_]*$' }
python/CMakeLists.txt
+462
View File
@@ -0,0 +1,462 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
cmake_minimum_required(VERSION 3.31 FATAL_ERROR)
project(TRTPyBinds LANGUAGES CXX)
option(TRT_BUILD_ENABLE_NEW_PYTHON_FLOW "Use new build logic based on the main CMake build." OFF)
if (${TRT_BUILD_ENABLE_NEW_PYTHON_FLOW})
if(MSVC)
set(DEFAULT_PY_EXT_PATH "${TOOLS_BASE}/win32")
else()
set(DEFAULT_PY_EXT_PATH "/externals")
endif()
set(TRT_BUILD_PYTHON_EXTERNALS_PATH ${DEFAULT_PY_EXT_PATH} CACHE PATH "Path to the parent folder of the many versioned python headers/libs.")
set(BUILD_PYTHON_PY_VERSIONS 3.8 3.9 3.10 3.11 3.12 3.13 3.14)
set(TRT_BUILD_PYTHON_PY_VERSIONS ${BUILD_PYTHON_PY_VERSIONS} CACHE STRING "The list of python versions to build TensorRT bindings for.")
message(STATUS "TRT_BUILD_PYTHON_PY_VERSIONS: ${TRT_BUILD_PYTHON_PY_VERSIONS}")
if(NOT ${TRT_BUILD_WINML})
set(TRT_PYTHON_MODULE_NAMES
"tensorrt"
"tensorrt_lean"
"tensorrt_dispatch")
else()
set(TRT_PYTHON_MODULE_NAMES "tensorrt_rtx")
endif()
# The "main" tensorrt bindings depend on the parser, so if we aren't building it, we need to skip it.
if(NOT ${TRT_BUILD_ONNX_PARSER})
message(STATUS "Not building the tensorrt python bindings as the ONNX Parser was disabled.")
list(REMOVE_ITEM TRT_PYTHON_MODULE_NAMES "tensorrt")
endif()
set(CMAKE_CXX_STANDARD 20 CACHE STRING "")
set(CMAKE_CXX_STANDARD_REQUIRED ON CACHE BOOL "")
set(CMAKE_CXX_EXTENSIONS OFF CACHE BOOL "")
find_package(
Python3
COMPONENTS Interpreter
REQUIRED
)
# Disable automatic python detection since we need to build bindings for many python versions in one go.
set(PYBIND11_NOPYTHON ON CACHE INTERNAL "")
include(FetchPyBind11)
# Pybind11 would normally enable this by default, but does not do so under NOPYTHON mode, so we do it manually.
if(${TRT_BUILD_PLATFORM} STREQUAL ${TRT_PLATFORM_X86})
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON)
endif()
add_custom_target(tensorrt_python_bindings)
# Creates the binding library for the specified module and python version.
#
# \param moduleName The module name to create the bindings for. One of "tensorrt", "tensorrt_dispatch", or "tensorrt_lean".
# \param pyVersion The python version to create bindings for, i.e. "3.12".
function(createBindingLibrary moduleName pyVersion)
set(libName tensorrt_bindings_${moduleName}_${pyVersion})
add_library(${libName} MODULE)
# Set options unique to the "full" bindings.
# The subdirs will use the value of TRT_PYTHON_IS_FULL_BINDINGS to set sources appropriately.
if(${moduleName} STREQUAL "tensorrt" OR ${moduleName} STREQUAL "tensorrt_rtx")
target_compile_definitions(${libName} PRIVATE
tensorrt_EXPORTS=1
)
set(TRT_PYTHON_IS_FULL_BINDINGS ON)
else()
set(TRT_PYTHON_IS_FULL_BINDINGS OFF)
endif()
function(add_${libName}_source)
target_sources(${libName} PRIVATE ${ARGN})
endfunction()
# Create an indirect refernce to the add_${libName}_source function which can be called by the subdirectories.
# This allows each subdir to add files to the individual targets with unique binary dirs on each call.
set(ADD_SOURCES_FUNCTION add_${libName}_source)
set(SUBDIR_BINARY_DIR_PREFIX subbuild/${libName})
add_subdirectory(src ${SUBDIR_BINARY_DIR_PREFIX}/src)
target_link_libraries(${libName} PRIVATE
pybind11::module
)
if(MSVC)
target_link_libraries(${libName} PRIVATE
pybind11::windows_extras
)
else()
# This allows us to use TRT libs shipped with standalone wheels.
set_target_properties(${libName} PROPERTIES SKIP_BUILD_RPATH ON)
target_link_options(${libName} PRIVATE "LINKER:--rpath=$ORIGIN,--disable-new-dtags")
endif()
# Find the main python headers in the relevant python<ver> subfolder in the externals.
find_path(
PYTHON_INCLUDES Python.h
HINTS ${TRT_BUILD_PYTHON_EXTERNALS_PATH}/python${pyVersion}
PATH_SUFFIXES include
NO_CACHE
REQUIRED
NO_CMAKE_FIND_ROOT_PATH
)
# Most of the headers are in that path we just found, except "pyconfig.h", which is platform-specific
# and in a platform-specific directory with an inconsistent naming scheme.
# So... go hunt for that. It's "mostly" located at /externals/python<ver>/include/<triple>/python<ver>/
# Except on windows, where instead of <triple> it's just "win".
if(${TRT_BUILD_PLATFORM} STREQUAL ${TRT_PLATFORM_X86})
set(PYCONFIG_H_PATH "x86_64-linux-gnu/python${pyVersion}")
elseif(${TRT_BUILD_PLATFORM} STREQUAL ${TRT_PLATFORM_AARCH64})
set(PYCONFIG_H_PATH "aarch64-linux-gnu/python${pyVersion}")
else()
message(FATAL_ERROR "The current platform \"${TRT_BUILD_PLATFORM}\" cannot be used to build the TRT Python Bindings.")
endif()
find_path(
PYCONFIG_INCLUDE pyconfig.h
HINTS ${PYTHON_INCLUDES}/${PYCONFIG_H_PATH}
NO_CACHE
REQUIRED
NO_CMAKE_FIND_ROOT_PATH
)
# Add the python headers as SYSTEM headers to silence warnings.
target_include_directories(${libName} SYSTEM PRIVATE
${PYTHON_INCLUDES}
${PYCONFIG_INCLUDE}
)
target_include_directories(${libName} PRIVATE
"include"
"docstrings"
)
# Setup links against the TRT Libraries.
if(${moduleName} STREQUAL "tensorrt")
set(TRT_LIBS tensorrt nvonnxparser)
if(${TRT_BUILD_PLUGINS})
list(APPEND TRT_LIBS tensorrt_plugins)
endif()
elseif(${moduleName} STREQUAL "tensorrt_rtx")
set(TRT_LIBS tensorrt nvonnxparser)
elseif(${moduleName} STREQUAL "tensorrt_lean")
set(TRT_LIBS tensorrt_lean_runtime)
elseif(${moduleName} STREQUAL "tensorrt_dispatch")
set(TRT_LIBS tensorrt_dispatch_runtime)
else()
message(FATAL_ERROR "Unknown TensorRT module " ${moduleName})
endif()
target_link_libraries(${libName} PRIVATE
${TRT_LIBS}
$<COMPILE_ONLY:TRT::cudart> # We need the cuda headers to compile, but we don't link against them.
$<COMPILE_ONLY:trt_global_definitions>
)
# Tell the files what module they are currently building.
target_compile_definitions(${libName} PRIVATE
TENSORRT_MODULE=${moduleName}
)
# Remove the `lib` prefix from the binding .so's and correct the output name.
set_target_properties(${libName}
PROPERTIES PREFIX ""
CXX_VISIBILITY_PRESET hidden
VISIBILITY_INLINES_HIDDEN ON
LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${moduleName}_bindings-py${pyVersion}
RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${moduleName}_bindings-py${pyVersion}
ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${moduleName}_bindings-py${pyVersion}
OUTPUT_NAME ${moduleName}
)
add_dependencies(tensorrt_python_bindings ${libName})
if(MSVC)
# Prevent pybind11 from sharing resources with other, potentially ABI incompatible modules
# https://github.com/pybind/pybind11/issues/2898
add_definitions(-DPYBIND11_COMPILER_TYPE="_${PROJECT_NAME}_abi")
# The python lib is python<maj><minor>.lib, but pyVersion is <maj>.<minor>, so we need to remove the dot.
string(REPLACE "." "" pyVerStr ${pyVersion})
if(NOT TARGET python${pyVerStr})
# Windows needs an explicit link against the python library.
find_library(
PYTHON${pyVerStr}_LIBRARY_PATH python${pyVerStr}.lib
HINTS ${TRT_BUILD_PYTHON_EXTERNALS_PATH}/python${pyVersion}
PATH_SUFFIXES lib
REQUIRED
NO_CMAKE_FIND_ROOT_PATH
)
add_library(python${pyVerStr} STATIC IMPORTED)
set_target_properties(python${pyVerStr} PROPERTIES IMPORTED_LOCATION "${PYTHON${pyVerStr}_LIBRARY_PATH}")
endif()
target_link_libraries(${libName} PRIVATE python${pyVerStr})
endif()
endfunction()
# Enumerate all the combinations and create the per-python per-module targets.
foreach(moduleName IN LISTS TRT_PYTHON_MODULE_NAMES)
foreach(pyVersion IN LISTS TRT_BUILD_PYTHON_PY_VERSIONS)
createBindingLibrary(${moduleName} ${pyVersion})
endforeach()
endforeach()
# Enter the packaging subdir to actually build the wheels.
add_subdirectory(packaging)
else() # TRT_BUILD_ENABLE_NEW_PYTHON_FLOW - old flow is below this line
set(TRT_BUILD_WINML OFF)
if (${TENSORRT_MODULE} STREQUAL "tensorrt_rtx")
# The "old" flow doesn't already have TRT_BUILD_WINML set, because it's
# coming from the legacy make build. So if we get the tensorrt_rtx module,
# we assume we're in the TensorRT-RTX build.
set(TRT_BUILD_WINML ON)
endif()
# Ensure TRT_WINML is either 0 or 1
if (NOT TRT_WINML)
set(TRT_WINML 0)
if (TRT_BUILD_WINML)
set(TRT_WINML 1)
endif()
endif()
# Sets variable to a value if variable is unset.
macro(set_ifndef var val)
if(NOT DEFINED ${var})
set(${var} ${val})
endif()
endmacro()
function(message)
if(VERBOSE)
_message(${ARGN})
endif()
endfunction()
# -------- CMAKE OPTIONS --------
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/${TENSORRT_MODULE}/)
set(CPP_STANDARD
17
CACHE STRING "CPP Standard Version")
set(CMAKE_CXX_STANDARD ${CPP_STANDARD})
# -------- PATHS --------
find_package(CUDAToolkit REQUIRED)
message(STATUS "EXT_PATH: ${EXT_PATH}")
message(STATUS "TENSORRT_BUILD: ${TENSORRT_BUILD}")
message(STATUS "CMAKE_BINARY_DIR: ${CMAKE_BINARY_DIR}")
message(STATUS "CUDAToolkit_LIBRARY_ROOT: ${CUDAToolkit_LIBRARY_ROOT}")
message(STATUS "CUDAToolkit_INCLUDE_DIRS: ${CUDAToolkit_INCLUDE_DIRS}")
message(STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
set_ifndef(TENSORRT_ROOT ../)
message(STATUS "TENSORRT_ROOT: ${TENSORRT_ROOT}")
set_ifndef(WIN_EXTERNALS ${EXT_PATH})
message(STATUS "WIN_EXTERNALS: ${WIN_EXTERNALS}")
# Convert to an absolute path.
set_ifndef(ONNX_INC_DIR ${TENSORRT_ROOT}/parsers/onnx)
find_path(
PYBIND11_DIR pybind11/pybind11.h
HINTS ${EXT_PATH} ${WIN_EXTERNALS}
PATH_SUFFIXES pybind11/include
REQUIRED
)
message(STATUS "ONNX_INC_DIR: ${ONNX_INC_DIR}")
message(STATUS "PYBIND11_DIR: ${PYBIND11_DIR}")
# Source Files
if(${TENSORRT_MODULE} STREQUAL "tensorrt" OR ${TENSORRT_MODULE} STREQUAL "tensorrt_rtx")
# tensorrt full dependencies
file(GLOB_RECURSE SOURCE_FILES src/*.cpp)
set(_NEED_EXPORTS_DEF ON)
else()
# tensorrt_lean and tensorrt_dispatch dependencies
set(SOURCE_FILES src/pyTensorRT.cpp src/utils.cpp src/infer/pyCore.cpp src/infer/pyPlugin.cpp
src/infer/pyFoundationalTypes.cpp)
endif()
set(PYTHON python${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION})
set(PYTHON_LIB_NAME python${PYTHON_MAJOR_VERSION}${PYTHON_MINOR_VERSION})
message(STATUS "PYTHON: ${PYTHON}")
message(STATUS "TENSORRT_MODULE: ${TENSORRT_MODULE}")
set(PY_MODULE_NAME ${TENSORRT_MODULE})
# Find headers
set(LIBPATH_SUFFIX lib)
if(MSVC)
find_path(
PY_INCLUDE Python.h
HINTS ${WIN_EXTERNALS}/${PYTHON} ${EXT_PATH}/${PYTHON}
PATH_SUFFIXES include
REQUIRED
)
find_path(
PY_LIB_DIR ${PYTHON_LIB_NAME}.lib
HINTS ${WIN_EXTERNALS}/${PYTHON} ${EXT_PATH}/${PYTHON}
PATH_SUFFIXES ${LIBPATH_SUFFIX}
REQUIRED
)
message(STATUS "PY_LIB_DIR: ${PY_LIB_DIR}")
else()
find_path(
PY_INCLUDE Python.h
HINTS ${EXT_PATH}/${PYTHON} /usr/include/${PYTHON}
PATH_SUFFIXES include
REQUIRED
)
endif()
message(STATUS "PY_INCLUDE: ${PY_INCLUDE}")
if(MSVC)
set(PY_TARGET_DIR win)
else()
set(PY_TARGET_DIR ${TARGET}-linux-gnu)
if(${TARGET} STREQUAL ppc64le)
set(PY_TARGET_DIR powerpc64le-linux-gnu)
endif()
endif()
# The per-platform pyconfig.h is located at /externals/python<ver>/<triple>/python<ver>/.
# Not sure why it's setup that way.
find_path(
PY_CONFIG_INCLUDE pyconfig.h
HINTS ${PY_INCLUDE}
PATH_SUFFIXES ${PY_TARGET_DIR}/${PYTHON} ${PY_TARGET_DIR}/${PYTHON}m
REQUIRED
)
message(STATUS "PY_CONFIG_INCLUDE: ${PY_CONFIG_INCLUDE}")
# -------- GLOBAL COMPILE OPTIONS --------
include_directories(${TENSORRT_ROOT}/include ${PROJECT_SOURCE_DIR}/include ${CUDAToolkit_INCLUDE_DIRS}
${PROJECT_SOURCE_DIR}/docstrings ${ONNX_INC_DIR} ${PYBIND11_DIR})
link_directories(${TENSORRT_BUILD} ${TENSORRT_LIBPATH})
if(MSVC)
# Prevent pybind11 from sharing resources with other, potentially ABI incompatible modules
# https://github.com/pybind/pybind11/issues/2898
add_definitions(-DPYBIND11_COMPILER_TYPE="_${PROJECT_NAME}_abi")
endif()
if(MSVC)
include_directories(SYSTEM BEFORE ${ADDITIONAL_PLATFORM_INCLUDE_DIRS})
link_libraries(${ADDITIONAL_PLATFORM_LIB_FLAGS})
link_directories(${PY_LIB_DIR})
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /bigobj")
if(${NV_GEN_PDB})
# PDB is only useful in release mode.
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Zi /bigobj")
set(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /DEBUG /OPT:REF /OPT:ICF")
endif()
else()
set(CMAKE_CXX_FLAGS
"${CMAKE_CXX_FLAGS} ${GLIBCXX_USE_CXX11_ABI_FLAG} -fvisibility=hidden -std=c++${CPP_STANDARD} -Wno-deprecated-declarations"
)
endif()
# ---------- MODULE DEPENDENCIES ----------
add_compile_definitions(TENSORRT_MODULE=${TENSORRT_MODULE})
if("${TRT_VCAST}" STREQUAL "1" OR "${TRT_VCAST_SAFE}" STREQUAL "1")
set(vfc_suffix "_static")
else()
set(vfc_suffix "")
endif()
if(MSVC)
set(nvinfer_lib_name "${TRT_NVINFER_NAME}_${TENSORRT_MAJOR_VERSION}${TRT_LIB_SUFFIX}")
set(nvinfer_plugin_lib_name "nvinfer_plugin_${TENSORRT_MAJOR_VERSION}")
set(nvonnxparser_lib_name "${TRT_ONNXPARSER_NAME}_${TENSORRT_MAJOR_VERSION}${TRT_LIB_SUFFIX}")
set(nvinfer_lean_lib_name "nvinfer_lean_${TENSORRT_MAJOR_VERSION}${vfc_suffix}")
set(nvinfer_dispatch_lib_name "nvinfer_dispatch_${TENSORRT_MAJOR_VERSION}${vfc_suffix}")
else()
if(${TRT_BUILD_WINML})
set(nvinfer_lib_name "-l:lib${TRT_NVINFER_NAME}.so")
else()
set(nvinfer_lib_name "${TRT_NVINFER_NAME}")
endif()
set(nvinfer_plugin_lib_name "nvinfer_plugin")
set(nvonnxparser_lib_name "${TRT_ONNXPARSER_NAME}")
set(nvinfer_lean_lib_name "nvinfer_lean${vfc_suffix}")
set(nvinfer_dispatch_lib_name "nvinfer_dispatch${vfc_suffix}")
endif()
message(STATUS "NVInferLib = ${nvinfer_lib_name}")
message(STATUS "NVOnnxParserLib = ${nvonnx_parser_lib_name}")
if(${TENSORRT_MODULE} STREQUAL "tensorrt" OR ${TENSORRT_MODULE} STREQUAL "tensorrt_rtx")
set(TRT_LIBS ${nvinfer_lib_name} ${nvonnxparser_lib_name})
list(APPEND TRT_LIBS ${nvinfer_plugin_lib_name})
elseif(${TENSORRT_MODULE} STREQUAL "tensorrt_lean")
set(TRT_LIBS ${nvinfer_lean_lib_name})
elseif(${TENSORRT_MODULE} STREQUAL "tensorrt_dispatch")
set(TRT_LIBS ${nvinfer_dispatch_lib_name})
else()
message(FATAL_ERROR "Unknown TensorRT module " ${TENSORRT_MODULE})
endif()
# -------- BUILDING --------
set(LIB_NAME ${PY_MODULE_NAME})
# Set up target
add_library(${LIB_NAME} SHARED ${SOURCE_FILES})
if(_NEED_EXPORTS_DEF)
target_compile_definitions(${LIB_NAME} PRIVATE
tensorrt_EXPORTS=1
)
endif()
message(STATUS "Python library name ${LIB_NAME}")
target_include_directories(${LIB_NAME} BEFORE PUBLIC ${PY_CONFIG_INCLUDE} ${PY_INCLUDE})
if(MSVC)
# For some reason, we must explicitly link against the Python library on Windows.
message(STATUS "Dependencies ${TRT_LIBS} ${PYTHON_LIB_NAME}")
target_link_libraries(${LIB_NAME} PRIVATE ${TRT_LIBS} ${PYTHON_LIB_NAME})
else()
message(STATUS "Dependencies ${TRT_LIBS} ")
target_link_libraries(${LIB_NAME} PRIVATE ${TRT_LIBS})
endif()
# Note that we have to remove the `lib` prefix from the binding .so's
set_target_properties(${LIB_NAME} PROPERTIES PREFIX "")
endif()
python/docstrings/infer/pyCoreDoc.h
+2309
View File
File diff suppressed because it is too large Load Diff
python/docstrings/infer/pyFoundationalTypesDoc.h
+230
View File
@@ -0,0 +1,230 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file contains all Dims docstrings, since these are typically too long to keep in the binding code.
#pragma once
namespace tensorrt
{
namespace DataTypeDoc
{
constexpr const char* descr = R"trtdoc(
Represents data types.
:ivar itemsize: :class:`int` The size in bytes of this :class:`DataType` .
)trtdoc";
constexpr char const* float32 = R"trtdoc(32-bit floating point format.)trtdoc";
constexpr char const* float16 = R"trtdoc(IEEE 16-bit floating-point format.)trtdoc";
constexpr char const* bfloat16 = R"trtdoc(Brain float -- has an 8 bit exponent and 8 bit significand)trtdoc";
constexpr char const* int8 = R"trtdoc(Signed 8-bit integer representing a quantized floating-point value.)trtdoc";
constexpr char const* int32 = R"trtdoc(Signed 32-bit integer format.)trtdoc";
constexpr char const* int64 = R"trtdoc(Signed 64-bit integer format.)trtdoc";
constexpr char const* boolean = R"trtdoc(8-bit boolean. 0 = false, 1 = true, other values undefined.)trtdoc";
constexpr char const* uint8 = R"trtdoc(
Unsigned 8-bit integer format.
Cannot be used to represent quantized floating-point values.
Use the IdentityLayer to convert ``uint8`` network-level inputs to {``float32``, ``float16``} prior
to use with other TensorRT layers, or to convert intermediate output
before ``uint8`` network-level outputs from {``float32``, ``float16``} to ``uint8``.
``uint8`` conversions are only supported for {``float32``, ``float16``}.
``uint8`` to {``float32``, ``float16``} conversion will convert the integer values
to equivalent floating point values.
{``float32``, ``float16``} to ``uint8`` conversion will convert the floating point values
to integer values by truncating towards zero. This conversion has undefined behavior for
floating point values outside the range [0.0f, 256.0) after truncation.
``uint8`` conversions are not supported for {``int8``, ``int32``, ``bool``}.
)trtdoc";
constexpr char const* fp8 = R"trtdoc(
Signed 8-bit floating point with 1 sign bit, 4 exponent bits, 3 mantissa
bits, and exponent-bias 7.
)trtdoc";
constexpr char const* int4 = R"trtdoc(Signed 4-bit integer representing a quantized floating-point value.)trtdoc";
constexpr char const* fp4
= R"trtdoc(Signed 4-bit floating point with 1 sign bit, 2 exponent bits and 1 mantissa bits.)trtdoc";
constexpr char const* e8m0 = R"trtdoc(Unsigned 8-bit exponent-only floating point.)trtdoc";
} // namespace DataTypeDoc
namespace WeightsRoleDoc
{
constexpr const char* descr
= R"trtdoc(How a layer uses particular Weights. The power weights of an IScaleLayer are omitted. Refitting those is not supported.)trtdoc";
constexpr const char* KERNEL = R"trtdoc(Kernel for :class:`IConvolutionLayer` or :class:`IDeconvolutionLayer` .)trtdoc";
constexpr const char* BIAS = R"trtdoc(Bias for :class:`IConvolutionLayer` or :class:`IDeconvolutionLayer` .)trtdoc";
constexpr const char* SHIFT = R"trtdoc(Shift part of :class:`IScaleLayer` .)trtdoc";
constexpr const char* SCALE = R"trtdoc(Scale part of :class:`IScaleLayer` .)trtdoc";
constexpr const char* CONSTANT = R"trtdoc(Weights for :class:`IConstantLayer` .)trtdoc";
constexpr const char* ANY = R"trtdoc(Any other weights role.)trtdoc";
} // namespace WeightsRoleDoc
namespace WeightsDoc
{
constexpr const char* descr = R"trtdoc(
An array of weights used as a layer parameter.
The weights are held by reference until the engine has been built - deep copies are not made automatically.
:ivar dtype: :class:`DataType` The type of the weights.
:ivar size: :class:`int` The number of weights in the array.
:ivar nbytes: :class:`int` Total bytes consumed by the elements of the weights buffer.
)trtdoc";
// FIXME: Weird bug occurring here. Cannot provide :arg:
constexpr const char* init_type = R"trtdoc(
Initializes an empty (0-length) Weights object with the specified type.
:type: A type to initialize the weights with. Default: :class:`tensorrt.float32`
)trtdoc";
constexpr const char* init_ptr = R"trtdoc(
Initializes a Weights object with the specified data.
:type: A type to initialize the weights with.
:ptr: A pointer to the data.
:count: The number of weights.
)trtdoc";
// FIXME: Weird bug occurring here. Cannot provide :arg:
constexpr const char* init_numpy = R"trtdoc(
:a: A numpy array whose values to use. No deep copies are made.
)trtdoc";
constexpr const char* numpy = R"trtdoc(
Create a numpy array using the underlying buffer of this weights object.
The resulting array is just a view over the existing data, i.e. no deep copy is made.
If the weights cannot be converted to NumPy (e.g. due to unsupported data type), the original weights are returned.
:returns: The NumPy array or the original weights.
)trtdoc";
} // namespace WeightsDoc
namespace DimsDoc
{
constexpr const char* descr = R"trtdoc(
Structure to define the dimensions of a tensor. :class:`Dims` and all derived classes behave like Python :class:`tuple` s. Furthermore, the TensorRT API can implicitly convert Python iterables to :class:`Dims` objects, so :class:`tuple` or :class:`list` can be used in place of this class.
)trtdoc";
constexpr const char* volume = R"trtdoc(
Computes the total volume of the dimensions
:returns: Total volume. `0` for empty dimensions.
)trtdoc";
constexpr const char* get_type = R"trtdoc(
Queries the type of a dimension.
:returns: The type of the specified dimension.
)trtdoc";
constexpr const char* MAX_DIMS = R"trtdoc(
The maximum number of dimensions supported by :class:`Dims`.
)trtdoc";
} // namespace DimsDoc
namespace Dims2Doc
{
constexpr const char* descr = R"trtdoc(
Structure to define 2D shape.
)trtdoc";
} // namespace Dims2Doc
namespace DimsHWDoc
{
constexpr const char* descr = R"trtdoc(
Structure to define 2D shape with height and width.
:ivar h: :class:`int` The first dimension (height).
:ivar w: :class:`int` The second dimension (width).
)trtdoc";
} // namespace DimsHWDoc
namespace Dims3Doc
{
constexpr const char* descr = R"trtdoc(
Structure to define 3D shape.
)trtdoc";
} // namespace Dims3Doc
namespace DimsCHWDoc
{
constexpr const char* descr = R"trtdoc(
Structure to define 3D tensor with a channel dimension, height, and width.
:ivar c: :class:`int` The first dimension (channel).
:ivar h: :class:`int` The second dimension (height).
:ivar w: :class:`int` The third dimension (width).
)trtdoc";
} // namespace DimsCHWDoc
namespace Dims4Doc
{
constexpr const char* descr = R"trtdoc(
Structure to define 4D tensor.
)trtdoc";
} // namespace Dims4Doc
namespace IVersionedInterfaceDoc
{
constexpr const char* descr = R"trtdoc(
Base class for all versioned interfaces.
)trtdoc";
} // namespace IVersionedInterfaceDoc
namespace APILanguageDoc
{
constexpr const char* descr = R"trtdoc(
The language used in the implementation of a TensorRT interface.
)trtdoc";
} // namespace APILanguageDoc
namespace InterfaceInfoDoc
{
constexpr const char* descr = R"trtdoc(
Version information for a TensorRT interface.
)trtdoc";
} // namespace InterfaceInfoDoc
namespace DimsNCHWDoc
{
constexpr const char* descr = R"trtdoc(
Structure to define 4D tensor with a batch dimension, a channel dimension, height and width.
:ivar n: :class:`int` The first dimension (batch).
:ivar c: :class:`int` The second dimension (channel).
:ivar h: :class:`int` The third dimension (height).
:ivar w: :class:`int` The fourth dimension (width).
)trtdoc";
} // namespace DimsNCHWDoc
namespace IHostMemoryDoc
{
constexpr const char* descr = R"trtdoc(
Handles library allocated memory that is accessible to the user.
The memory allocated via the host memory object is owned by the library and will be de-allocated when object is destroyed.
This class exposes a buffer interface using Python's buffer protocol.
:ivar dtype: :class:`DataType` The data type of this buffer.
:ivar nbytes: :class:`int` Total bytes consumed by the elements of the buffer.
)trtdoc";
} // namespace IHostMemoryDoc
} // namespace tensorrt
python/docstrings/infer/pyGraphDoc.h
+3147
View File
File diff suppressed because it is too large Load Diff
python/docstrings/infer/pyInt8Doc.h
+18
View File
@@ -0,0 +1,18 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
python/docstrings/infer/pyPluginDoc.h
+1066
View File
File diff suppressed because it is too large Load Diff
python/docstrings/parsers/pyOnnxDoc.h
+388
View File
@@ -0,0 +1,388 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Docstrings for the pyOnnx parser bindings.
#pragma once
namespace tensorrt
{
namespace OnnxParserDoc
{
constexpr char const* descr = R"trtdoc(
This class is used for parsing ONNX models into a TensorRT network definition
:ivar num_errors: :class:`int` The number of errors that occurred during prior calls to :func:`parse`
)trtdoc";
constexpr char const* init = R"trtdoc(
:arg network: The network definition to which the parser will write.
:arg logger: The logger to use.
)trtdoc";
constexpr char const* parse = R"trtdoc(
Parse a serialized ONNX model into the TensorRT network.
:arg model: The serialized ONNX model.
:arg path: The path to the model file. Only required if the model has externally stored weights.
:returns: true if the model was parsed successfully
)trtdoc";
constexpr char const* parse_from_file = R"trtdoc(
Parse an ONNX model from file into a TensorRT network.
:arg model: The path to an ONNX model.
:returns: true if the model was parsed successfully
)trtdoc";
constexpr char const* supports_model_v2 = R"trtdoc(
Check whether TensorRT supports a particular ONNX model.
Query each subgraph with num_subgraphs, is_subgraph_supported, get_subgraph_nodes.
:arg model: The serialized ONNX model.
:arg path: The path to the model file. Only required if the model has externally stored weights.
:returns: true if the model is supported
)trtdoc";
constexpr char const* num_subgraphs = R"trtdoc(
Get the number of subgraphs. Calling before \p supportsModelV2 is an undefined behavior. Will return 0 by default.
:returns: Number of subgraphs
)trtdoc";
constexpr char const* is_subgraph_supported = R"trtdoc(
Returns whether the subgraph is supported. Calling before \p supportsModelV2 is an undefined behavior.
Will return false by default.
:arg index: Index of the subgraph to be checked.
:returns: true if subgraph is supported
)trtdoc";
constexpr char const* get_subgraph_nodes = R"trtdoc(
Get the nodes of the specified subgraph. Calling before \p supportsModelV2 is an undefined behavior.
Will return an empty list by default.
:arg index: Index of the subgraph.
:returns: List[int]
A list of node indices in the subgraph.
)trtdoc";
constexpr char const* supports_operator = R"trtdoc(
Returns whether the specified operator may be supported by the parser.
Note that a result of true does not guarantee that the operator will be supported in all cases (i.e., this function may return false-positives).
:arg op_name: The name of the ONNX operator to check for support
)trtdoc";
constexpr char const* get_error = R"trtdoc(
Get an error that occurred during prior calls to :func:`parse`
:arg index: Index of the error
)trtdoc";
constexpr char const* clear_errors = R"trtdoc(
Clear errors from prior calls to :func:`parse`
)trtdoc";
constexpr char const* clear_flag = R"trtdoc(
Clears the parser flag from the enabled flags.
:arg flag: The flag to clear.
)trtdoc";
constexpr char const* get_flag = R"trtdoc(
Check if a build mode flag is set.
:arg flag: The flag to check.
:returns: A `bool` indicating whether the flag is set.
)trtdoc";
constexpr char const* set_flag = R"trtdoc(
Add the input parser flag to the already enabled flags.
:arg flag: The flag to set.
)trtdoc";
constexpr char const* get_layer_output_tensor = R"trtdoc(
Get the i-th output ITensor object for the ONNX layer "name".
In the case of multiple nodes sharing the same name this function will return
the output tensors of the first instance of the node in the ONNX graph.
:arg name: The name of the ONNX layer.
:arg i: The index of the output.
:returns: The output tensor or None if the layer was not found or an invalid index was provided.
)trtdoc";
constexpr char const* get_used_vc_plugin_libraries = R"trtdoc(
Query the plugin libraries needed to implement operations used by the parser in a version-compatible engine.
This provides a list of plugin libraries on the filesystem needed to implement operations
in the parsed network. If you are building a version-compatible engine using this network,
provide this list to IBuilderConfig.set_plugins_to_serialize() to serialize these plugins along
with the version-compatible engine, or, if you want to ship these plugin libraries externally
to the engine, ensure that IPluginRegistry.load_library() is used to load these libraries in the
appropriate runtime before deserializing the corresponding engine.
:returns: List[str] List of plugin libraries found by the parser.
:raises: :class:`RuntimeError` if an internal error occurred when trying to fetch the list of plugin libraries.
)trtdoc";
constexpr char const* load_model_proto = R"trtdoc(
Load a serialized ONNX model into the parser. Unlike the parse() or parse_from_file()
functions, this function does not immediately convert the model into a TensorRT INetworkDefinition. Using this function
allows users to provide their own initializers for the ONNX model through the load_initializer() function.
Only one model can be loaded at a time. Subsequent calls to load_model_proto() will result in an error.
To begin the conversion of the model into a TensorRT INetworkDefinition, use parse_model_proto().
:arg model: The serialized ONNX model.
:arg path: The path to the model file. Only required if the model has externally stored weights.
:returns: true if the model was loaded successfully
)trtdoc";
constexpr char const* load_initializer = R"trtdoc(
Prompt the ONNX parser to load an initializer with user-provided binary data.
The lifetime of the data must exceed the lifetime of the parser.
All user-provided initializers must be provided prior to calling parse_model_proto().
This function can be called multiple times to specify the names of multiple initializers.
Calling this function with an initializer previously specified will overwrite the previous instance.
This function will return false if initializer validation fails. Possible validation errors are:
* This function was called prior to load_model_proto().
* The requested initializer was not found in the model.
* The size of the data provided is different from the corresponding initializer in the model.
:arg name: name of the initializer.
:arg data: binary data of the initializer.
:arg size: the size of the binary data.
:returns: true if the initializer was successfully loaded
)trtdoc";
constexpr char const* parse_model_proto = R"trtdoc(
Begin the parsing and conversion process of the loaded ONNX model into a TensorRT INetworkDefinition.
:returns: true if the model was parsed successfully.
)trtdoc";
constexpr char const* set_builder_config = R"trtdoc(
Set the BuilderConfig for the parser.
:arg builder_config: The BuilderConfig to set.
:returns: true if the BuilderConfig was set successfully, false otherwise.
)trtdoc";
} // namespace OnnxParserDoc
namespace OnnxParserRefitterDoc
{
constexpr char const* descr = R"trtdoc(
This is an interface designed to refit weights from an ONNX model.
)trtdoc";
constexpr char const* init = R"trtdoc(
:arg refitter: The Refitter object used to refit the model.
:arg logger: The logger to use.
)trtdoc";
constexpr char const* refit_from_bytes = R"trtdoc(
Load a serialized ONNX model from memory and perform weight refit.
:arg model: The serialized ONNX model.
:arg path: The path to the model file. Only required if the model has externally stored weights.
:returns: true if all the weights in the engine were refit successfully.
)trtdoc";
constexpr char const* refit_from_file = R"trtdoc(
Load and parse a ONNX model from disk and perform weight refit.
:arg model: The path to an ONNX model.
:returns: true if the model was loaded successfully, and if all the weights in the engine were refit successfully.
)trtdoc";
constexpr char const* get_error = R"trtdoc(
Get an error that occurred during prior calls to :func:`refitFromBytes` or :func:`refitFromFile`.
:arg index: Index of the error
)trtdoc";
constexpr char const* clear_errors = R"trtdoc(
Clear errors from prior calls to :func:`refitFromBytes` or :func:`refitFromFile`.
)trtdoc";
constexpr char const* load_model_proto = R"trtdoc(
Load a serialized ONNX model into the refitter. Unlike the refit() or refit_from_file()
functions, this function does not immediately begin the refit process. Using this function
allows users to provide their own initializers for the ONNX model through the load_initializer() function.
Only one model can be loaded at a time. Subsequent calls to load_model_proto() will result in an error.
To begin the refit process, use refit_model_proto().
:arg model: The serialized ONNX model.
:arg path: The path to the model file. Only required if the model has externally stored weights.
:returns: true if the model was loaded successfully.
)trtdoc";
constexpr char const* load_initializer = R"trtdoc(
Prompt the ONNX refitter to load an initializer with user-provided binary data.
The lifetime of the data must exceed the lifetime of the refitter.
All user-provided initializers must be provided prior to calling refit_model_proto().
This function can be called multiple times to specify the names of multiple initializers.
Calling this function with an initializer previously specified will overwrite the previous instance.
This function will return false if initializer validation fails. Possible validation errors are:
* This function was called prior to load_model_proto().
* The requested initializer was not found in the model.
* The size of the data provided is different from the corresponding initializer in the model.
:arg name: name of the initializer.
:arg data: binary data of the initializer.
:arg size: the size of the binary data.
:returns: true if the initializer was successfully loaded.
)trtdoc";
constexpr char const* refit_model_proto = R"trtdoc(
Begin the refit process from the loaded ONNX model.
:returns: true if the model was refit successfully.
)trtdoc";
} // namespace OnnxParserRefitterDoc
namespace ErrorCodeDoc
{
constexpr char const* descr = R"trtdoc(
The type of parser error
)trtdoc";
} // namespace ErrorCodeDoc
namespace OnnxParserFlagDoc
{
constexpr char const* descr = R"trtdoc(
Flags that control how an ONNX model gets parsed.
)trtdoc";
constexpr char const* NATIVE_INSTANCENORM = R"trtdoc(
Parse the ONNX model into the INetworkDefinition with the intention of using TensorRT's native layer implementation over the plugin implementation for InstanceNormalization nodes.
This flag is required when building version-compatible or hardware-compatible engines.
This flag is ON by default.
)trtdoc";
constexpr char const* ENABLE_UINT8_AND_ASYMMETRIC_QUANTIZATION_DLA = R"trtdoc(
Enable UINT8 as a quantization data type and asymmetric quantization with non-zero zero-point values in Quantize and Dequantize nodes.
The resulting engine must be built targeting DLA version >= 3.16.
This flag is OFF by default.
)trtdoc";
constexpr char const* REPORT_CAPABILITY_DLA = R"trtdoc(
Parse the ONNX model with per-node validation for DLA. If this flag is set, is_subgraph_supported() will
also return capability in the context of DLA support.
When this flag is set, a valid BuilderConfig must be provided to the parser via set_builder_config().
This flag is OFF by default.
)trtdoc";
constexpr char const* ENABLE_PLUGIN_OVERRIDE = R"trtdoc(
Allow a loaded plugin with the same name as an ONNX operator type to override the default ONNX implementation,
even if the plugin namespace attribute is not set.
This flag is useful for custom plugins that are intended to replace standard ONNX operators, for example to provide
alternative implementations or improved performance.
This flag is OFF by default.
)trtdoc";
constexpr char const* ADJUST_FOR_DLA = R"trtdoc(
Parse the ONNX model with adjustments to make layers more amenable to running on DLA.
This flag is OFF by default.
)trtdoc";
} // namespace OnnxParserFlagDoc
namespace ParserErrorDoc
{
constexpr char const* descr = R"trtdoc(
An object containing information about an error
)trtdoc";
constexpr char const* code = R"trtdoc(
:returns: The error code
)trtdoc";
constexpr char const* desc = R"trtdoc(
:returns: Description of the error
)trtdoc";
constexpr char const* file = R"trtdoc(
:returns: Source file in which the error occurred
)trtdoc";
constexpr char const* line = R"trtdoc(
:returns: Source line at which the error occurred
)trtdoc";
constexpr char const* func = R"trtdoc(
:returns: Source function in which the error occurred
)trtdoc";
constexpr char const* node = R"trtdoc(
:returns: Index of the Onnx model node in which the error occurred
)trtdoc";
constexpr char const* node_name = R"trtdoc(
:returns: Name of the node in the model in which the error occurred
)trtdoc";
constexpr char const* node_operator = R"trtdoc(
:returns: Name of the node operation in the model in which the error occurred
)trtdoc";
constexpr char const* local_function_stack = R"trtdoc(
:returns: Current stack trace of local functions in which the error occurred
)trtdoc";
constexpr char const* local_function_stack_size = R"trtdoc(
:returns: Size of the current stack trace of local functions in which the error occurred
)trtdoc";
} // namespace ParserErrorDoc
constexpr char const* get_nv_onnx_parser_version = R"trtdoc(
:returns: The Onnx Parser version
)trtdoc";
} // namespace tensorrt
python/docstrings/pyTensorRTDoc.h
+24
View File
@@ -0,0 +1,24 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Top level docstring, for the whole Python package.
#pragma once
namespace tensorrt
{
} // namespace tensorrt
python/include/ForwardDeclarations.h
+89
View File
@@ -0,0 +1,89 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TRT_PYTHON_FORWARD_DECLARATIONS_H
#define TRT_PYTHON_FORWARD_DECLARATIONS_H
// clang-format off
// Hack for missing declarations on Windows.
// These headers must be included before pybind11.h as some dependencies are missing.
#ifdef _MSC_VER
#include <cstdint>
using ssize_t = int64_t;
#endif // _MSC_VER
// clang-format on
#include <pybind11/pybind11.h>
// True if we are building full TensorRT bindings (as opposed to lean/dispatch bindings).
#if defined(tensorrt_EXPORTS)
#define EXPORT_ALL_BINDINGS 1
#else
#define EXPORT_ALL_BINDINGS 0
#endif
#include "NvInferRuntimeCommon.h"
// We need to avoid making copies of PluginField because it does not own any of it's members.
// When there are multiple PluginFields pointing to the same data in Python, bad things happen.
// Making this opaque allows us to create lists of PluginFields without creating unwanted copies.
PYBIND11_MAKE_OPAQUE(std::vector<nvinfer1::PluginField>)
namespace tensorrt
{
// Set some global namespace aliases.
namespace py = pybind11;
// This is for literal operators (like _a for default args)
using namespace pybind11::literals;
// Hack for situations where the C++ object does not own a member string/const char*.
// Cannot reference python strings, so we make a copy and keep it alive on the C++ side.
struct FallbackString
{
FallbackString() = default;
FallbackString(std::string const& other)
: mData{other}
{
}
FallbackString(py::str other)
: mData{std::string(other)}
{
}
const char* c_str() const
{
return mData.c_str();
}
const char* c_str()
{
return mData.c_str();
}
std::string mData{};
};
// Infer
void bindFoundationalTypes(py::module& m);
void bindPlugin(py::module& m);
#if EXPORT_ALL_BINDINGS
void bindGraph(py::module& m);
#endif
void bindCore(py::module& m);
// Parsers
#if EXPORT_ALL_BINDINGS
void bindOnnx(py::module& m);
#endif
} // namespace tensorrt
#endif // TRT_PYTHON_FORWARD_DECLARATIONS_H
python/include/utils.h
+224
View File
@@ -0,0 +1,224 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TRT_PYTHON_UTILS_H
#define TRT_PYTHON_UTILS_H
// These headers must be included before pybind11.h as some dependencies are otherwise missing on Windows.
// clang-format off
#include "ForwardDeclarations.h"
// clang-format on
#include <pybind11/numpy.h>
#include <pybind11/pybind11.h>
#include "NvInfer.h"
#include <functional>
#include <iostream>
#include <memory>
#include <string>
#if defined(__GLIBC__)
#include <sys/resource.h>
#include <sys/sysinfo.h>
#include <unistd.h>
#endif
#define CUDA_LIB_NAME "cuda"
//! Macro evaluates the expression \p EXPR, and if not equal to `CUDA_SUCCESS`, logs to `std::cerr` and evaluates return
//! RET_ON_FAIL.
#define CUDA_CALL_WITH_RET(EXPR, RET_ON_FAIL) \
if (CUresult retCode = (EXPR); retCode != CUDA_SUCCESS) \
{ \
std::cerr << "[ERROR] Failed to " << #EXPR << " with error " << retCode << std::endl; \
return RET_ON_FAIL; \
}
namespace tensorrt
{
namespace utils
{
namespace py = pybind11;
// Wrapper function for dlopen/dlclose/dlsym, support both Windows and Linux
//! \brief Attempts to open the library
//!
//! \param libName. The returned library handle may be nullptr on failure and must be closed with `dllClose`.
[[nodiscard]] void* nvdllOpen(char const* libName);
//! Closes a library opened with `nvdllOpen`.
void dllClose(void* handle);
//! \brief get symbol from the library
//!
//! \param name in a dll
//! \param handle, loaded by `nvdllOpen`.
//!
//! \return the pointer to the symbol named
[[nodiscard]] void* dllGetSym(void* handle, char const* name);
// Returns the size in bytes of the specified data type.
size_t size(nvinfer1::DataType type);
// Converts a TRT datatype to its corresponding numpy dtype.
// Returns nullptr if the type could not be converted to NumPy.
std::unique_ptr<py::dtype> nptype(nvinfer1::DataType type);
// Returns the TRT type corresponding to the specified numpy type.
nvinfer1::DataType type(py::dtype const& type);
// Return a numpy array (that doesn't own the data, but rather refers to it)
static const auto weights_to_numpy = [](nvinfer1::Weights const& self) -> py::object {
// The py::cast(self) allows us to return the buffer by reference rather than by copy.
// See https://stackoverflow.com/questions/49181258/pybind11-create-numpy-view-of-data
auto const npType = nptype(self.type);
if (npType)
{
return py::array{*npType, self.count, self.values, py::cast(self)};
}
return py::cast(self);
};
inline int64_t volume(nvinfer1::Dims const& dims)
{
return std::accumulate(dims.d, dims.d + dims.nbDims, int64_t{1}, std::multiplies<int64_t>{});
}
// Method for calling the python function and returning the value (returned from python) used in cpp trampoline
// classes. Prints an error if no such method is overriden in python.
// T* must NOT be a trampoline class!
template <typename T>
py::function getOverride(const T* self, std::string const& overloadName, bool showWarning = true)
{
py::function overload = py::get_override(self, overloadName.c_str());
if (!overload && showWarning)
{
std::cerr << "Method: " << overloadName
<< " was not overriden. Please provide an implementation for this method." << std::endl;
}
return overload;
}
// Deprecation helpers
void issueDeprecationWarning(const char* useInstead);
// TODO: Figure out how to de-duplicate these two
template <typename RetVal, typename... Args>
struct DeprecatedFunc
{
using Func = RetVal (*)(Args...);
RetVal operator()(Args... args) const
{
issueDeprecationWarning(useInstead);
return (*func)(std::forward<Args>(args)...);
}
const Func func;
const char* useInstead;
};
template <typename RetVal, typename... Args>
constexpr auto deprecate(RetVal (*func)(Args...), const char* useInstead) -> DeprecatedFunc<RetVal, Args...>
{
return DeprecatedFunc<RetVal, Args...>{func, useInstead};
}
template <bool isConst, typename RetVal, typename Cls, typename... Args>
struct DeprecatedMemberFunc
{
using Func = std::conditional_t<isConst, RetVal (Cls::*)(Args...) const, RetVal (Cls::*)(Args...)>;
RetVal operator()(Cls& self, Args... args) const
{
issueDeprecationWarning(useInstead);
return (std::forward<Cls>(self).*func)(std::forward<Args>(args)...);
}
const Func func;
const char* useInstead;
};
template <typename RetVal, typename Cls, typename... Args>
constexpr auto deprecateMember(RetVal (Cls::*func)(Args...) const, const char* useInstead)
-> DeprecatedMemberFunc</*isConst=*/true, RetVal, Cls, Args...>
{
return DeprecatedMemberFunc</*isConst=*/true, RetVal, Cls, Args...>{func, useInstead};
}
template <typename RetVal, typename Cls, typename... Args>
constexpr auto deprecateMember(RetVal (Cls::*func)(Args...), const char* useInstead)
-> DeprecatedMemberFunc</*isConst=*/false, RetVal, Cls, Args...>
{
return DeprecatedMemberFunc</*isConst=*/false, RetVal, Cls, Args...>{func, useInstead};
}
template <typename T>
constexpr auto deprecateInTrtRtxOnly(T&& func, const char* /*unused*/) -> T&&
{
return std::forward<T>(func);
}
template <typename T>
constexpr auto deprecateMemberInTrtRtxOnly(T&& func, const char* /*unused*/) -> T&&
{
return std::forward<T>(func);
}
template <typename T>
void doNothingDel(const T& self)
{
issueDeprecationWarning("del obj");
}
// https://nvbugs/3479811 Create a wrapper for C++ to python throw
[[noreturn]] void throwPyError(PyObject* type, std::string const& message = "python error");
} // namespace utils
#define PY_ASSERT_RUNTIME_ERROR(assertion, msg) \
do \
{ \
if (!(assertion)) \
{ \
utils::throwPyError(PyExc_RuntimeError, msg); \
} \
} while (false)
#define PY_ASSERT_INDEX_ERROR(assertion) \
do \
{ \
if (!(assertion)) \
{ \
utils::throwPyError(PyExc_IndexError, "Out of bounds"); \
} \
} while (false)
#define PY_ASSERT_VALUE_ERROR(assertion, msg) \
do \
{ \
if (!(assertion)) \
{ \
utils::throwPyError(PyExc_ValueError, msg); \
} \
} while (false)
} // namespace tensorrt
#endif // TRT_PYTHON_UTILS_H
python/packaging/CMakeLists.txt
+168
View File
@@ -0,0 +1,168 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
include(FlagToInt)
# Processes one or more wheel templates file, replacing any markers with concrete information and
# copying the result into the per-python per-module build dir.
#
# \param moduleName The module name to create the bindings for. One of "tensorrt", "tensorrt_dispatch", or "tensorrt_lean".
# \param pyVersion The python version to create bindings for, i.e. "3.12".
# \param ARGN A list of file paths, relative to './packaging/bindings_wheel/tensorrt', of the file(s) to copy.
# \returns generatedWheelFiles A list containing paths to all generated files, which can be used to create a custom target.
# \returns generatedFileOutDir A string containing the output directory for the generated files. Contains generator expressions.
function(processWheelTemplates wheelType moduleName pyVersion)
set(outputDir ${CMAKE_CURRENT_BINARY_DIR}/${moduleName}_${wheelType}-py${pyVersion}/$<CONFIG>)
foreach(filePath IN LISTS ARGN)
# Target files that start with tensorrt/ need to be expanded into ${moduleName}/ instead of tensorrt/
# So we need to rewrite the relative path, the source directory, *and* the output directory to match.
# Note that we do not update the "outputDir" variable, since we want to report it back based on the original file path.
if(filePath MATCHES "^tensorrt/")
string(REGEX REPLACE "^tensorrt/" "" adjustedFilePath ${filePath})
set(__srcDir ${CMAKE_CURRENT_LIST_DIR}/tensorrt)
set(__srcFilePath ${adjustedFilePath})
# Bit of a hack branch for standalone binding wheels
if(wheelType STREQUAL "binding_standalone")
set(__outDir ${outputDir}/${moduleName}_bindings)
set(__outputFile ${outputDir}/${moduleName}_bindings/${adjustedFilePath})
else()
set(__outDir ${outputDir}/${moduleName})
set(__outputFile ${outputDir}/${moduleName}/${adjustedFilePath})
endif()
elseif(filePath MATCHES "^tensorrt_libs/") # Standalone lib wheels use <module>_libs/ instead of <module>/
string(REGEX REPLACE "^tensorrt_libs/" "" adjustedFilePath ${filePath})
set(__srcDir ${CMAKE_CURRENT_LIST_DIR}/tensorrt_libs)
set(__srcFilePath ${adjustedFilePath})
set(__outDir ${outputDir}/${moduleName}_libs)
set(__outputFile ${outputDir}/${moduleName}_libs/${adjustedFilePath})
else()
set(__srcDir ${CMAKE_CURRENT_LIST_DIR})
set(__srcFilePath ${filePath})
set(__outDir ${outputDir})
set(__outputFile ${outputDir}/${filePath})
endif()
flagToInt(TRT_BUILD_WINML)
add_custom_command(
OUTPUT ${__outputFile}
COMMAND
${Python3_EXECUTABLE} ${TensorRT_SOURCE_DIR}/python/scripts/process_wheel_template.py
--src-dir ${__srcDir}
--dst-dir ${__outDir}
--filepath ${__srcFilePath}
--trt-module ${moduleName}
--trt-py-version ${TensorRT_PACKAGE_VERSION}
--cuda-version ${TRT_CUDA_VERSION}
--trt-version ${TensorRT_VERSION}
--plugin-disabled ${TRT_BUILD_WINML_INT}
--trt-nvinfer-name ${TENSORRT_BASE_NAME}
--trt-onnxparser-name ${TRT_ONNXPARSER_NAME} # The TRT_ONNXPARSER_NAME var is populated in the root CMakeLists.txt
DEPENDS
${TensorRT_SOURCE_DIR}/python/scripts/process_wheel_template.py
${CMAKE_CURRENT_LIST_DIR}/${filePath}
COMMENT "Expanding wheel template ${filePath} for ${moduleName}"
VERBATIM
)
list(APPEND generatedFiles ${__outputFile})
endforeach()
set(generatedWheelFiles ${generatedFiles} PARENT_SCOPE)
set(generatedFileOutDir ${outputDir} PARENT_SCOPE)
endfunction()
# Copy the custom build backend to the generated directory.
# The backend is referenced via backend-path in pyproject.toml.
# Note: This macro is intended to be called immediately after processWheelTemplates and relies on information generated by it.
macro(copyTRTBuildBackend)
set(copiedBuildBackend ${generatedFileOutDir}/tensorrt_build_backend/__init__.py)
add_custom_command(
OUTPUT ${copiedBuildBackend}
COMMAND ${CMAKE_COMMAND} -E copy_directory
${TensorRT_SOURCE_DIR}/python/packaging/tensorrt_build_backend
${generatedFileOutDir}/tensorrt_build_backend
DEPENDS ${TensorRT_SOURCE_DIR}/python/packaging/tensorrt_build_backend/__init__.py
COMMENT "Copying tensorrt_build_backend to ${generatedFileOutDir}"
VERBATIM
)
endmacro()
# Create meta targets used to group the post-processed wheel files and all the python wheels.
add_custom_target(tensorrt_python_wheels)
# Specify a single output directory to hold all the generated wheels.
# This is similar to CMAKE_ARCHIVE_OUTPUT_DIRECTORY, but for wheels.
set(TRT_WHEEL_OUTPUT_DIR "${CMAKE_BINARY_DIR}/$<CONFIG>/dist" CACHE PATH "")
# Install packaging requirements into the build dir so they survive container rebuilds.
set(TRT_PACKAGING_DEPS_DIR ${CMAKE_CURRENT_BINARY_DIR}/packaging_deps)
add_custom_command(
OUTPUT ${TRT_PACKAGING_DEPS_DIR}/.installed
COMMAND ${Python3_EXECUTABLE} -m pip install --upgrade
--target ${TRT_PACKAGING_DEPS_DIR}
-r ${TensorRT_SOURCE_DIR}/python/packaging/requirements.txt
${maybe_pypi_index}
COMMAND ${CMAKE_COMMAND} -E touch ${TRT_PACKAGING_DEPS_DIR}/.installed
DEPENDS ${TensorRT_SOURCE_DIR}/python/packaging/requirements.txt
VERBATIM
)
add_custom_target(trt_packaging_requirements_installed DEPENDS ${TRT_PACKAGING_DEPS_DIR}/.installed)
# Environment variables for Python packaging commands.
set(TRT_PACKAGING_ENV "PYTHONPATH=${TRT_PACKAGING_DEPS_DIR}")
if(NOT CMAKE_VERBOSE_MAKEFILE)
# Suppress deprecation that we can't fix while python/packaging/requirements.txt
# specifies setuptools~=75.3.2 for python_version<3.12.
list(APPEND TRT_PACKAGING_ENV "PYTHONWARNINGS=ignore::setuptools.warnings.SetuptoolsDeprecationWarning")
endif()
# Computes the current wheel platform name and stores it the variable named by outVar.
#
# \param isStandalone Boolean indicating whether to compute the platform for a standalone wheel.
# \param outVar Name of the variable to store the resulting platform name in.
# \returns wheelPlatform The computed wheel platform name. Standalone wheels use manylinux tags.
function(get_wheel_platform isStandalone outVar)
if(WIN32)
set(wheelPlatform win_${TRT_LOWERCASE_CMAKE_SYSTEM_PROCESSOR})
elseif(isStandalone)
if(NOT GLIBC_VERSION)
# Determine glibc version for standalone wheels
execute_process(
COMMAND ldd --version
OUTPUT_VARIABLE LDD_OUTPUT
OUTPUT_STRIP_TRAILING_WHITESPACE
)
if(LDD_OUTPUT MATCHES "([0-9]+\\.[0-9]+)")
set(GLIBC_VERSION ${CMAKE_MATCH_1} CACHE STRING "Detected glibc version for standalone wheels")
else()
message(FATAL_ERROR "Failed to determine glibc version from ldd output: ${LDD_OUTPUT}")
endif()
endif()
string(REPLACE "." "_" GLIBC_VERSION_FLAT ${GLIBC_VERSION})
set(wheelPlatform manylinux_${GLIBC_VERSION_FLAT}_${TRT_LOWERCASE_CMAKE_SYSTEM_PROCESSOR})
else()
set(wheelPlatform linux_${TRT_LOWERCASE_CMAKE_SYSTEM_PROCESSOR})
endif()
set(${outVar} ${wheelPlatform} PARENT_SCOPE)
endfunction()
add_subdirectory(bindings_wheel)
add_subdirectory(libs_wheel)
add_subdirectory(frontend_sdist)
add_subdirectory(metapackage)
python/packaging/bindings_wheel/CMakeLists.txt
+173
View File
@@ -0,0 +1,173 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
add_custom_target(tensorrt_bindings_wheels ALL)
add_custom_target(trt_bindings_wheel_files)
define_property(TARGET
PROPERTY TRT_WHEEL_STAGING_DIR
BRIEF_DOCS "The directory containing all the files that were packaged into this wheel."
)
# \brief Creates a target named tensorrt_bindings_wheel_${moduleName}_${pyVersion} which will build the Bindings Wheel for that combination.
#
# \details The wheel is created by expanding all template files (from this directory) into the per-module per-python build directory.
# Then, the binding library (tensorrt.so) is copied into the same directory as the generated files.
# Finally, the wheel is built by running setup.py with the appropriate arguments in the binary directory.
#
# \param moduleName The module name to create the bindings for. One of "tensorrt", "tensorrt_dispatch", or "tensorrt_lean".
# \param pyVersion The python version to create bindings for, i.e. "3.12".
# \param isStandalone If we are building the wheels for distribution with the standalone libs wheels.
# If true, the binding wheels' top-level folder will be <module_name>_bindings instead of <module_name>
function(buildBindingsWheel moduleName pyVersion isStandalone)
if (isStandalone)
set(standaloneMarker "_standalone")
set(bindingSuffix "_bindings")
else()
set(standaloneMarker "")
set(bindingSuffix "")
endif()
set(filesTarget trt_wheel_files_binding_${moduleName}_${pyVersion}${standaloneMarker})
string(REPLACE "." "" pyVerStr ${pyVersion})
get_wheel_platform(${isStandalone} wheelPlatform)
set(wheelTemplateFiles
tensorrt/__init__.py
LICENSE.txt
pyproject.toml
)
if(${TRT_BUILD_PLUGINS})
list(APPEND wheelTemplateFiles
tensorrt/plugin/__init__.py
tensorrt/plugin/_autotune.py
tensorrt/plugin/_export.py
tensorrt/plugin/_lib.py
tensorrt/plugin/_plugin_class.py
tensorrt/plugin/_tensor.py
tensorrt/plugin/_top_level.py
tensorrt/plugin/_utils.py
tensorrt/plugin/_validate.py
)
endif()
# Expands all template files for the bindings for the target module and python version.
# File paths starting with "tensorrt/" are expanded into "${moduleName}/".
processWheelTemplates(binding${standaloneMarker} ${moduleName} ${pyVersion} ${wheelTemplateFiles})
copyTRTBuildBackend()
# We need to copy the binding library into the directory that the wheel is created from.
set(copiedBindingLibrary ${generatedFileOutDir}/${moduleName}${bindingSuffix}/${moduleName}${CMAKE_SHARED_LIBRARY_SUFFIX})
# I bet you thought you were free from more windows exceptions, I certainly did.
# But here we are again, with yet another naming convention™
if(MSVC)
set(copiedBindingLibrary ${generatedFileOutDir}/${moduleName}${bindingSuffix}/${moduleName}.cp${pyVerStr}-${wheelPlatform}.pyd)
endif()
# Creates a new custom target, and makes trt_bindings_wheel_files depend on the new target.
add_custom_target(${filesTarget} DEPENDS ${generatedWheelFiles} ${copiedBindingLibrary} ${copiedBuildBackend})
add_dependencies(trt_bindings_wheel_files ${filesTarget})
# Copies the binding library (tensorrt.so) into the same directory as the generated files.
add_custom_command(
OUTPUT ${copiedBindingLibrary}
COMMAND ${CMAKE_COMMAND} -E copy
$<TARGET_FILE:tensorrt_bindings_${moduleName}_${pyVersion}>
${copiedBindingLibrary}
DEPENDS tensorrt_bindings_${moduleName}_${pyVersion}
COMMENT "Copying bindings library for ${moduleName} to ${copiedBindingLibrary}"
VERBATIM
)
# Define the output directory for the wheel
set(wheelOutDir ${TRT_WHEEL_OUTPUT_DIR}/bindings${standaloneMarker})
# Wheel type determines how the build backend computes the package name
if(isStandalone)
set(wheelType "binding_standalone")
set(wheelOutputFile ${wheelOutDir}/${moduleName}_cu${CUDAToolkit_VERSION_MAJOR}_bindings-${TensorRT_PACKAGE_VERSION}-cp${pyVerStr}-none-${wheelPlatform}.whl)
else()
set(wheelType "binding")
set(wheelOutputFile ${wheelOutDir}/${moduleName}-${TensorRT_PACKAGE_VERSION}-cp${pyVerStr}-none-${wheelPlatform}.whl)
endif()
# Add a custom command to build the wheel using PEP 517 compliant build frontend.
# The custom tensorrt_build_backend handles package naming and wheel tags via config settings.
add_custom_command(
OUTPUT ${wheelOutputFile}
COMMAND ${CMAKE_COMMAND} -E env ${TRT_PACKAGING_ENV}
${Python3_EXECUTABLE} -m build --wheel --no-isolation
--config-setting=--quiet
--config-setting=wheel-type=${wheelType}
--config-setting=python-tag=cp${pyVerStr}
--config-setting=plat-name=${wheelPlatform}
--outdir ${wheelOutDir}
WORKING_DIRECTORY ${generatedFileOutDir}
DEPENDS ${copiedBindingLibrary} ${copiedBuildBackend} ${generatedWheelFiles} trt_packaging_requirements_installed
VERBATIM
)
set(wheelTarget tensorrt_bindings_wheel_${moduleName}_${pyVersion}${standaloneMarker})
# Add a custom target for the wheel
add_custom_target(
${wheelTarget}
ALL
DEPENDS ${wheelOutputFile}
)
add_dependencies(${wheelTarget} trt_bindings_wheel_files)
add_dependencies(tensorrt_bindings_wheels ${wheelTarget})
set_target_properties(${wheelTarget}
PROPERTIES TRT_WHEEL_STAGING_DIR "${generatedFileOutDir}"
)
if(isStandalone)
set(bindingsInstallComponent internal)
else()
set(bindingsInstallComponent external)
endif()
install(FILES
${wheelOutputFile}
DESTINATION python${standaloneMarker}
COMPONENT ${bindingsInstallComponent}
OPTIONAL
)
endfunction()
foreach(moduleName IN LISTS TRT_PYTHON_MODULE_NAMES)
foreach(pyVersion IN LISTS TRT_BUILD_PYTHON_PY_VERSIONS)
buildBindingsWheel(${moduleName} ${pyVersion} FALSE)
endforeach()
endforeach()
add_dependencies(tensorrt_python_wheels tensorrt_bindings_wheels)
# For the standalone wheels, we need to build an additional copy of the wheels that uses <moduleName>_bindings as the package directory.
# This prevents clashes with the metapackage, which uses the "tensorrt" module name for the standalone path.
# TODO TRT-11.0: Can we drop this and change the name of the metapackage?
if(${TRT_BUILD_PYTHON_STANDALONE_WHEELS})
foreach(moduleName IN LISTS TRT_PYTHON_MODULE_NAMES)
foreach(pyVersion IN LISTS TRT_BUILD_PYTHON_PY_VERSIONS)
buildBindingsWheel(${moduleName} ${pyVersion} TRUE)
endforeach()
endforeach()
endif()
python/packaging/bindings_wheel/LICENSE.txt
+180
View File
@@ -0,0 +1,180 @@
Abstract
This document is the Software License Agreement (SLA) for NVIDIA TensorRT. This document contains specific license terms and conditions for NVIDIA TensorRT. By accepting this agreement, you agree to comply with all the terms and conditions applicable to the specific product(s) included herein.
If you are receiving TensorRT under the NVIDIA Prerelease License Agreement (also known as NPLA) or under the NVIDIA Software License Agreement (previously known as the NVIDIA Tegra Software License Agreement), your use of TensorRT is governed by such applicable terms and conditions. All other uses of TensorRT are governed by the terms and conditions of the below license agreement.
NVIDIA SOFTWARE LICENSE AGREEMENT
Important: READ BEFORE DOWNLOADING, INSTALLING, COPYING OR USING THE LICENSED SOFTWARE
This Software License Agreement ("SLA”), made and entered into as of the time and date of click through action (“Effective Date”),is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs the use of the NVIDIA computer software and the documentation made available for use with such NVIDIA software. By downloading, installing, copying, or otherwise using the NVIDIA software and/or documentation, you agree to be bound by the terms of this SLA. If you do not agree to the terms of this SLA, do not download, install, copy or use the NVIDIA software or documentation. IF YOU ARE ENTERING INTO THIS SLAON BEHALF OF A COMPANY OR OTHER LEGAL ENTITY, YOU REPRESENT THAT YOU HAVE THE LEGAL AUTHORITY TO BIND THE ENTITY TO THIS SLA, IN WHICH CASE “YOU” WILL MEAN THE ENTITY YOU REPRESENT. IF YOU DON’T HAVE SUCH AUTHORITY, OR IF YOU DON’T ACCEPT ALL THE TERMS AND CONDITIONS OF THIS SLA, THEN NVIDIA DOES NOT AGREETO LICENSE THE LICENSED SOFTWARETO YOU, AND YOU MAY NOT DOWNLOAD, INSTALL, COPY OR USE IT.
Preface
This document is the Software License Agreement (SLA) for NVIDIA TensorRT. This document contains specific license terms and conditions for NVIDIA TensorRT. By accepting this agreement, you agree to comply with all the terms and conditions applicable to the specific product(s) included herein.
If you are receiving TensorRT under the NVIDIA Prerelease License Agreement (also known as NPLA) or under the NVIDIA Software License Agreement (previously known as the NVIDIA Tegra Software License Agreement), your use of TensorRT is governed by such applicable terms and conditions. All other uses of TensorRT are governed by the terms and conditions of the below license agreement.
NVIDIA SOFTWARE LICENSE AGREEMENT
Important: READ BEFORE DOWNLOADING, INSTALLING, COPYING OR USING THE LICENSED SOFTWARE
This Software License Agreement ("SLA”), made and entered into as of the time and date of click through action (“Effective Date”),is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs the use of the NVIDIA computer software and the documentation made available for use with such NVIDIA software. By downloading, installing, copying, or otherwise using the NVIDIA software and/or documentation, you agree to be bound by the terms of this SLA. If you do not agree to the terms of this SLA, do not download, install, copy or use the NVIDIA software or documentation. IF YOU ARE ENTERING INTO THIS SLAON BEHALF OF A COMPANY OR OTHER LEGAL ENTITY, YOU REPRESENT THAT YOU HAVE THE LEGAL AUTHORITY TO BIND THE ENTITY TO THIS SLA, IN WHICH CASE “YOU” WILL MEAN THE ENTITY YOU REPRESENT. IF YOU DON’T HAVE SUCH AUTHORITY, OR IF YOU DON’T ACCEPT ALL THE TERMS AND CONDITIONS OF THIS SLA, THEN NVIDIA DOES NOT AGREETO LICENSE THE LICENSED SOFTWARETO YOU, AND YOU MAY NOT DOWNLOAD, INSTALL, COPY OR USE IT.
1. LICENSE.
1.1. License Grant
Subject to the terms of the AGREEMENT, NVIDIA hereby grants you a non-exclusive, non-transferable license, without the right to sublicense (except as expressly set forth in a Supplement), during the applicable license term unless earlier terminated as provided below, to have Authorized Users install and use the Software, including modifications (if expressly permitted in a Supplement), in accordance with the Documentation. You are only licensed to activate and use Licensed Software for which you a have a valid license, even if during the download or installation you are presented with other product options. No Orders are binding on NVIDIA until accepted by NVIDIA. Your Orders are subject to the AGREEMENT.
SLA Supplements: Certain Licensed Software licensed under this SLA may be subject to additional terms and conditions that will be presented to you in a Supplement for acceptance prior to the delivery of such Licensed Software under this SLA and the applicable Supplement. Licensed Software will only be delivered to you upon your acceptance of all applicable terms.
1.2. Limited Purpose Licenses
If your license is provided for one of the purposes indicated below, then notwithstanding contrary terms in License Grant or in a Supplement, such licenses are for internal use and do not include any right or license to sub-license and distribute the Licensed Software or its output in any way in any public release, however limited, and/or in any manner that provides third parties with use of or access to the Licensed Software or its functionality or output, including (but not limited to) external alpha or beta testing or development phases. Further:
Evaluation License. You may use evaluation licenses solely for your internal evaluation of the Licensed Software for broader adoption within your Enterprise or in connection with a NVIDIA product purchase decision, and such licenses have an expiration date as indicated by NVIDIA in its sole discretion (or ninety days from the date of download if no other duration is indicated).
Educational/Academic License. You may use educational/academic licenses solely for educational purposes and all users must be enrolled or employed by an academic institution. If you do not meet NVIDIA’s academic program requirements for educational institutions, you have no rights under this license.
Test/Development License. You may use test/development licenses solely for your internal development, testing and/or debugging of your software applications or for interoperability testing with the Licensed Software, and such licenses have an expiration date as indicated by NVIDIA in its sole discretion (or one year from the date of download if no other duration is indicated). NVIDIA Confidential Information under the AGREEMENT includes output from Licensed Software developer tools identified as “Pro” versions, where the output reveals functionality or performance data pertinent to NVIDIA hardware or software products.
1.3. Pre-Release Licenses
With respect to alpha, beta, preview, and other pre-release Software and Documentation (“Pre-Release Licensed Software”) delivered to you under the AGREEMENT you acknowledge and agree that such Pre-Release Licensed Software (i) may not be fully functional, may contain errors or design flaws, and may have reduced or different security, privacy, accessibility, availability, and reliability standards relative to commercially provided NVIDIA software and documentation, and (ii) use of such Pre-Release Licensed Software may result in unexpected results, loss of data, project delays or other unpredictable damage or loss. THEREFORE, PRE-RELEASE LICENSED SOFTWARE IS NOT INTENDED FOR USE, AND SHOULD NOT BE USED, IN PRODUCTION OR BUSINESS-CRITICAL SYSTEMS. NVIDIA has no obligation to make available a commercial version of any Pre-Release Licensed Software and NVIDIA has the right to abandon development of Pre-Release Licensed Software at any time without liability.
1.4. Enterprise and Contractor Usage
You may allow your Enterprise employees and Contractors to access and use the Licensed Software pursuant to the terms of the AGREEMENT solely to perform work on your behalf, provided further that with respect to Contractors: (i) you obtain a written agreement from each Contractor which contains terms and obligations with respect to access to and use of Licensed Software no less protective of NVIDIA than those set forth in the AGREEMENT, and (ii) such Contractor’s access and use expressly excludes any sublicensing or distribution rights for the Licensed Software. You are responsible for the compliance with the terms and conditions of the AGREEMENT by your Enterprise and Contractors. Any act or omission that, if committed by you, would constitute a breach of the AGREEMENT shall be deemed to constitute a breach of the AGREEMENT if committed by your Enterprise or Contractors.
1.5. Services
Except as expressly indicated in an Order, NVIDIA is under no obligation to provide support for the Licensed Software or to provide any patches, maintenance, updates or upgrades under the AGREEMENT. Unless patches, maintenance, updates or upgrades are provided with their separate governing terms and conditions, they constitute Licensed Software licensed to you under the AGREEMENT.
2. LIMITATIONS.
2.1. License Restrictions
Except as expressly authorized in the AGREEMENT, you agree that you will not (nor authorize third parties to): (i) copy and use Software that was licensed to you for use in one or more NVIDIA hardware products in other unlicensed products (provided that copies solely for backup purposes are allowed); (ii) reverse engineer, decompile, disassemble (except to the extent applicable laws specifically require that such activities be permitted) or attempt to derive the source code, underlying ideas, algorithm or structure of Software provided to you in object code form; (iii) sell, transfer, assign, distribute, rent, loan, lease, sublicense or otherwise make available the Licensed Software or its functionality to third parties (a) as an application services provider or service bureau, (b) by operating hosted/virtual system environments, (c) by hosting, time sharing or providing any other type of services, or (d) otherwise by means of the internet; (iv) modify, translate or otherwise create any derivative works of any Licensed Software; (v) remove, alter, cover or obscure any proprietary notice that appears on or with the Licensed Software or any copies thereof; (vi) use the Licensed Software, or allow its use, transfer, transmission or export in violation of any applicable export control laws, rules or regulations; (vii) distribute, permit access to, or sublicense the Licensed Software as a stand-alone product; (viii) bypass, disable, circumvent or remove any form of copy protection, encryption, security or digital rights management or authentication mechanism used by NVIDIA in connection with the Licensed Software, or use the Licensed Software together with any authorization code, serial number, or other copy protection device not supplied by NVIDIA directly or through an authorized reseller; (ix) use the Licensed Software for the purpose of developing competing products or technologies or assisting a third party in such activities; (x) use the Licensed Software with any system or application where the use or failure of such system or application can reasonably be expected to threaten or result in personal injury, death, or catastrophic loss including, without limitation, use in connection with any nuclear, avionics, navigation, military, medical, life support or other life critical application (“Critical Applications”), unless the parties have entered into a Critical Applications agreement; (xi) distribute any modification or derivative work you make to the Licensed Software under or by reference to the same name as used by NVIDIA; or (xii) use the Licensed Software in any manner that would cause the Licensed Software to become subject to an Open Source License. Nothing in the AGREEMENT shall be construed to give you a right to use, or otherwise obtain access to, any source code from which the Software or any portion thereof is compiled or interpreted. You acknowledge that NVIDIA does not design, test, manufacture or certify the Licensed Software for use in the context of a Critical Application and NVIDIA shall not be liable to you or any third party, in whole or in part, for any claims or damages arising from such use. You agree to defend, indemnify and hold harmless NVIDIA and its Affiliates, and their respective employees, contractors, agents, officers and directors, from and against any and all claims, damages, obligations, losses, liabilities, costs or debt, fines, restitutions and expenses (including but not limited to attorney’s fees and costs incident to establishing the right of indemnification) arising out of or related to you and your Enterprise, and their respective employees, contractors, agents, distributors, resellers, end users, officers and directors use of Licensed Software outside of the scope of the AGREEMENT or any other breach of the terms of the AGREEMENT.
2.2. Third Party License Obligations
You acknowledge and agree that the Licensed Software may include or incorporate third party technology (collectively “Third Party Components”), which is provided for use in or with the Software and not otherwise used separately. If the Licensed Software includes or incorporates Third Party Components, then the third-party pass-through terms and conditions (“Third Party Terms”) for the particular Third Party Component will be bundled with the Software or otherwise made available online as indicated by NVIDIA and will be incorporated by reference into the AGREEMENT. In the event of any conflict between the terms in the AGREEMENT and the Third Party Terms, the Third Party Terms shall govern. Copyright to Third Party Components are held by the copyright holders indicated in the copyright notices indicated in the Third Party Terms.
Audio/Video Encoders and Decoders: You acknowledge and agree that it is your sole responsibility to obtain any additional third party licenses required to make, have made, use, have used, sell, import, and offer for sale your products or services that include or incorporate any Third Party Components and content relating to audio and/or video encoders and decoders from, including but not limited to, Microsoft, Thomson, Fraunhofer IIS, Sisvel S.p.A., MPEG-LA, and Coding Technologies as NVIDIA does not grant to you under the AGREEMENT any necessary patent or other rights with respect to audio and/or video encoders and decoders.
2.3. Limited Rights
Your rights in the Licensed Software are limited to those expressly granted under the AGREEMENT and no other licenses are granted whether by implication, estoppel or otherwise. NVIDIA reserves all rights, title and interest in and to the Licensed Software not expressly granted under the AGREEMENT.
3. CONFIDENTIALITY
Neither party will use the other party’s Confidential Information, except as necessary for the performance of the AGREEMENT, nor will either party disclose such Confidential Information to any third party, except to personnel of NVIDIA and its Affiliates, you, your Enterprise, your Enterprise Contractors, and each party’s legal and financial advisors that have a need to know such Confidential Information for the performance of the AGREEMENT, provided that each such personnel, employee and Contractor is subject to a written agreement that includes confidentiality obligations consistent with those set forth herein. Each party will use all reasonable efforts to maintain the confidentiality of all of the other party’s Confidential Information in its possession or control, but in no event less than the efforts that it ordinarily uses with respect to its own Confidential Information of similar nature and importance. The foregoing obligations will not restrict either party from disclosing the other party’s Confidential Information or the terms and conditions of the AGREEMENT as required under applicable securities regulations or pursuant to the order or requirement of a court, administrative agency, or other governmental body, provided that the party required to make such disclosure (i) gives reasonable notice to the other party to enable it to contest such order or requirement prior to its disclosure (whether through protective orders or otherwise), (ii) uses reasonable effort to obtain confidential treatment or similar protection to the fullest extent possible to avoid such public disclosure, and (iii) discloses only the minimum amount of information necessary to comply with such requirements.
4. OWNERSHIP
You are not obligated to disclose to NVIDIA any modifications that you, your Enterprise or your Contractors make to the Licensed Software as permitted under the AGREEMENT. As between the parties, all modifications are owned by NVIDIA and licensed to you under the AGREEMENT unless otherwise expressly provided in a Supplement. The Licensed Software and all modifications owned by NVIDIA, and the respective Intellectual Property Rights therein, are and will remain the sole and exclusive property of NVIDIA or its licensors, whether the Licensed Software is separate from or combined with any other products or materials. You shall not engage in any act or omission that would impair NVIDIA’s and/or its licensors’ Intellectual Property Rights in the Licensed Software or any other materials, information, processes or subject matter proprietary to NVIDIA. NVIDIA’s licensors are intended third party beneficiaries with the right to enforce provisions of the AGREEMENT with respect to their Confidential Information and/or Intellectual Property Rights.
5. FEEDBACK
You have no obligation to provide Feedback to NVIDIA. However, NVIDIA and/or its Affiliates may use and include any Feedback that you provide to improve the Licensed Software or other NVIDIA products, technologies or materials. Accordingly, if you provide Feedback, you agree that NVIDIA and/or its Affiliates, at their option, may, and may permit their licensees, to make, have made, use, have used, reproduce, license, distribute and otherwise commercialize the Feedback in the Licensed Software or in other NVIDIA products, technologies or materials without the payment of any royalties or fees to you. All Feedback becomes the sole property of NVIDIA and may be used in any manner NVIDIA sees fit, and you hereby assign to NVIDIA all of your right, title and interest in and to any Feedback. NVIDIA has no obligation to respond to Feedback or to incorporate Feedback into the Licensed Software.
6. NO WARRANTIES
THE LICENSED SOFTWARE AND ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES ARE PROVIDED BY NVIDIA “AS IS” AND “WITH ALL FAULTS,” AND NVIDIA EXPRESSLY DISCLAIMS ALL OTHER WARRANTIES OF ANY KIND OR NATURE, WHETHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES OF OPERABILITY, CONDITION, VALUE, ACCURACY OF DATA, OR QUALITY, AS WELL AS ANY WARRANTIES OF MERCHANTABILITY, SYSTEM INTEGRATION, WORKMANSHIP, SUITABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, OR THE ABSENCE OF ANY DEFECTS THEREIN, WHETHER LATENT OR PATENT. NO WARRANTY IS MADE BY NVIDIA ON THE BASIS OF TRADE USAGE, COURSE OF DEALING OR COURSE OF TRADE. NVIDIA DOES NOT WARRANT THAT THE LICENSED SOFTWARE OR ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES PROVIDED BY NVIDIA UNDER THE AGREEMENT WILL MEET YOUR REQUIREMENTS OR THAT THE OPERATION THEREOF WILL BE UNINTERRUPTED OR ERROR-FREE, OR THAT ALL ERRORS WILL BE CORRECTED. YOU ACKNOWLEDGE THAT NVIDIA’S OBLIGATIONS UNDER THE AGREEMENT ARE FOR THE BENEFIT OF YOU ONLY. Nothing in this warranty section affects any statutory rights of consumers or other recipients to the extent that they cannot be waived or limited by contract under applicable law.
7. LIMITATION OF LIABILITY
TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA OR ITS LICENSORS SHALL NOT BE LIABLE FOR ANY SPECIAL, INCIDENTAL, PUNITIVE OR CONSEQUENTIAL DAMAGES, OR ANY LOST PROFITS, LOSS OF USE, LOSS OF DATA OR LOSS OF GOODWILL, OR THE COSTS OF PROCURING SUBSTITUTE PRODUCTS, ARISING OUT OF OR IN CONNECTION WITH THE AGREEMENT OR THE USE OR PERFORMANCE OF THE LICENSED SOFTWARE AND ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES PROVIDED BY NVIDIA UNDER THE AGREEMENT, WHETHER SUCH LIABILITY ARISES FROM ANY CLAIM BASED UPON BREACH OF CONTRACT, BREACH OF WARRANTY, TORT (INCLUDING NEGLIGENCE), PRODUCT LIABILITY OR ANY OTHER CAUSE OF ACTION OR THEORY OF LIABILITY. IN NO EVENT WILL NVIDIA’S TOTAL CUMULATIVE LIABILITY UNDER OR ARISING OUT OF THE AGREEMENT EXCEED THE NET AMOUNTS RECEIVED BY NVIDIA FOR YOUR USE OF THE PARTICULAR LICENSED SOFTWARE DURING THE TWELVE (12) MONTHS BEFORE THE LIABILITY AROSE (or up to US$10.00 if you acquired the Licensed Software for no charge). THE NATURE OF THE LIABILITY, THE NUMBER OF CLAIMS OR SUITS OR THE NUMBER OF PARTIES WITHIN YOUR ENTERPRISE THAT ACCEPTED THE TERMS OF THE AGREEMENT SHALL NOT ENLARGE OR EXTEND THIS LIMIT. THE FOREGOING LIMITATIONS SHALL APPLY REGARDLESS OF WHETHER NVIDIA OR ITS LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES AND REGARDLESS OF WHETHER ANY REMEDY FAILS ITS ESSENTIAL PURPOSE. The disclaimers, exclusions and limitations of liability set forth in the AGREEMENT form an essential basis of the bargain between the parties, and, absent any such disclaimers, exclusions or limitations of liability, the provisions of the AGREEMENT, including, without limitation, the economic terms, would be substantially different.
8. TERM AND TERMINATION.
8.1. AGREEMENT, Licenses and Services
This SLA shall become effective upon the Effective Date, each Supplement upon their acceptance, and both this SLA and Supplements shall continue in effect until your last access or use of the Licensed Software and/or services hereunder, unless earlier terminated as provided in this “Term and Termination” section. Each Licensed Software license ends at the earlier of (a) the expiration of the applicable license term, or (b) termination of such license or the AGREEMENT. Each service ends at the earlier of (x) the expiration of the applicable service term, (y) termination of such service or the AGREEMENT, or (z) expiration or termination of the associated license and no credit or refund will be provided upon the expiration or termination of the associated license for any service fees paid.
8.2. Termination and Effect of Expiration or Termination
NVIDIA may terminate the AGREEMENT in whole or in part: (i) if you breach any term of the AGREEMENT and fail to cure such breach within thirty (30) days following notice thereof from NVIDIA (or immediately if you violate NVIDIA’s Intellectual Property Rights); (ii) if you become the subject of a voluntary or involuntary petition in bankruptcy or any proceeding relating to insolvency, receivership, liquidation or composition for the benefit of creditors, if that petition or proceeding is not dismissed with prejudice within sixty (60) days after filing, or if you cease to do business; or (iii) if you commence or participate in any legal proceeding against NVIDIA, with respect to the Licensed Software that is the subject of the proceeding during the pendency of such legal proceeding. If you or your authorized NVIDIA reseller fail to pay license fees or service fees when due then NVIDIA may, in its sole discretion, suspend or terminate your license grants, services and any other rights provided under the AGREEMENT for the affected Licensed Software, in addition to any other remedies NVIDIA may have at law or equity. Upon any expiration or termination of the AGREEMENT, a license or a service provided hereunder, (a) any amounts owed to NVIDIA become immediately due and payable, (b) you must promptly discontinue use of the affected Licensed Software and/or service, and (c) you must promptly destroy or return to NVIDIA all copies of the affected Licensed Software and all portions thereof in your possession or control, and each party will promptly destroy or return to the other all of the other party’s Confidential Information within its possession or control. Upon written request, you will certify in writing that you have complied with your obligations under this section. Upon expiration or termination of the AGREEMENT all provisions survive except for the license grant provisions.
9. CONSENT TO COLLECTION AND USE OF INFORMATION.
You hereby agree and acknowledge that the Software may access, collect non-personally identifiable information about your Enterprise computer systems in order to properly optimize such systems for use with the Software. To the extent that you use the Software, you hereby consent to all of the foregoing, and represent and warrant that you have the right to grant such consent. In addition, you agree that you are solely responsible for maintaining appropriate data backups and system restore points for your Enterprise systems, and that NVIDIA will have no responsibility for any damage or loss to such systems (including loss of data or access) arising from or relating to (a) any changes to the configuration, application settings, environment variables, registry, drivers, BIOS, or other attributes of the systems (or any part of such systems) initiated through the Software; or (b) installation of any Software or third party software patches initiated through the Software. In certain systems you may change your system update preferences by unchecking "Automatically check for updates" in the "Preferences" tab of the control panel for the Software.
In connection with the receipt of the Licensed Software or services you may receive access to links to third party websites and services and the availability of those links does not imply any endorsement by NVIDIA. NVIDIA encourages you to review the privacy statements on those sites and services that you choose to visit so that you can understand how they may collect, use and share personal information of individuals. NVIDIA is not responsible or liable for: (i) the availability or accuracy of such links; or (ii) the products, services or information available on or through such links; or (iii) the privacy statements or practices of sites and services controlled by other companies or organizations.
To the extent that you or members of your Enterprise provide to NVIDIA during registration or otherwise personal information, you acknowledge that such information will be collected, used and disclosed by NVIDIA in accordance with NVIDIA's privacy policy, available at URL http://www.nvidia.com/object/privacy_policy.html.
10. GENERAL.
This SLA, any Supplements incorporated hereto, and Orders constitute the entire agreement of the parties with respect to the subject matter hereto and supersede all prior negotiations, conversations, or discussions between the parties relating to the subject matter hereto, oral or written, and all past dealings or industry custom. Any additional and/or conflicting terms and conditions on purchase order(s) or any other documents issued by you are null, void, and invalid. Any amendment or waiver under the AGREEMENT must be in writing and signed by representatives of both parties.
The AGREEMENT and the rights and obligations thereunder may not be assigned by you, in whole or in part, including by merger, consolidation, dissolution, operation of law, or any other manner, without written consent of NVIDIA, and any purported assignment in violation of this provision shall be void and of no effect. NVIDIA may assign, delegate or transfer the AGREEMENT and its rights and obligations hereunder, and if to a non-Affiliate you will be notified.
Each party acknowledges and agrees that the other is an independent contractor in the performance of the AGREEMENT, and each party is solely responsible for all of its employees, agents, contractors, and labor costs and expenses arising in connection therewith. The parties are not partners, joint ventures or otherwise affiliated, and neither has any authority to make any statements, representations or commitments of any kind to bind the other party without prior written consent.
Neither party will be responsible for any failure or delay in its performance under the AGREEMENT (except for any payment obligations) to the extent due to causes beyond its reasonable control for so long as such force majeure event continues in effect.
The AGREEMENT will be governed by and construed under the laws of the State of Delaware and the United States without regard to the conflicts of law provisions thereof and without regard to the United Nations Convention on Contracts for the International Sale of Goods. The parties consent to the personal jurisdiction of the federal and state courts located in Santa Clara County, California. You acknowledge and agree that a breach of any of your promises or agreements contained in the AGREEMENT may result in irreparable and continuing injury to NVIDIA for which monetary damages may not be an adequate remedy and therefore NVIDIA is entitled to seek injunctive relief as well as such other and further relief as may be appropriate. If any court of competent jurisdiction determines that any provision of the AGREEMENT is illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect. Unless otherwise specified, remedies are cumulative.
The Licensed Software has been developed entirely at private expense and is “commercial items” consisting of “commercial computer software” and “commercial computer software documentation” provided with RESTRICTED RIGHTS. Use, duplication or disclosure by the U.S. Government or a U.S. Government subcontractor is subject to the restrictions set forth in the AGREEMENT pursuant to DFARS 227.7202-3(a) or as set forth in subparagraphs (c)(1) and (2) of the Commercial Computer Software - Restricted Rights clause at FAR 52.227-19, as applicable. Contractor/manufacturer is NVIDIA, 2701 San Tomas Expressway, Santa Clara, CA 95050.
You acknowledge that the Licensed Software described under the AGREEMENT is subject to export control under the U.S. Export Administration Regulations (EAR) and economic sanctions regulations administered by the U.S. Department of Treasury’s Office of Foreign Assets Control (OFAC). Therefore, you may not export, reexport or transfer in-country the Licensed Software without first obtaining any license or other approval that may be required by BIS and/or OFAC. You are responsible for any violation of the U.S. or other applicable export control or economic sanctions laws, regulations and requirements related to the Licensed Software. By accepting this SLA, you confirm that you are not a resident or citizen of any country currently embargoed by the U.S. and that you are not otherwise prohibited from receiving the Licensed Software.
Any notice delivered by NVIDIA to you under the AGREEMENT will be delivered via mail, email or fax. Please direct your legal notices or other correspondence to NVIDIA Corporation, 2701 San Tomas Expressway, Santa Clara, California 95050, United States of America, Attention: Legal Department.
11. GLOSSARY OF TERMS
Certain capitalized terms, if not otherwise defined elsewhere in this SLA, shall have the meanings set forth below:
“Affiliate”
“Affiliate” means any legal entity that Owns, is Owned by, or is commonly Owned with a party. “Own” means having more than 50% ownership or the right to direct the management of the entity.
“AGREEMENT”
“AGREEMENT” means this SLA and all associated Supplements entered by the parties referencing this SLA.
“Authorized Users”
“Authorized Users” means your Enterprise individual employees and any of your Enterprise’s Contractors, subject to the terms of the “Enterprise and Contractors Usage” section.
“Confidential Information”
“Confidential Information” means the Licensed Software (unless made publicly available by NVIDIA without confidentiality obligations), and any NVIDIA business, marketing, pricing, research and development, know-how, technical, scientific, financial status, proposed new products or other information disclosed by NVIDIA to you which, at the time of disclosure, is designated in writing as confidential or proprietary (or like written designation), or orally identified as confidential or proprietary or is otherwise reasonably identifiable by parties exercising reasonable business judgment, as confidential. Confidential Information does not and will not include information that: (i) is or becomes generally known to the public through no fault of or breach of the AGREEMENT by the receiving party; (ii) is rightfully known by the receiving party at the time of disclosure without an obligation of confidentiality; (iii) is independently developed by the receiving party without use of the disclosing party’s Confidential Information; or (iv) is rightfully obtained by the receiving party from a third party without restriction on use or disclosure.
“Contractor”
“Contractor” means an individual who works primarily for your Enterprise on a contractor basis from your secure network. means an individual who works primarily for your Enterprise on a contractor basis from your secure network.
“Documentation”
“Documentation” means the NVIDIA documentation made available for use with the Software, including (without limitation) user manuals, datasheets, operations instructions, installation guides, release notes and other materials provided to you under the AGREEMENT.
“Enterprise”
“Enterprise” means you or any company or legal entity for which you accepted the terms of this SLA, and their subsidiaries of which your company or legal entity owns more than fifty percent (50%) of the issued and outstanding equity.
“Feedback”
“Feedback” means any and all suggestions, feature requests, comments or other feedback regarding the Licensed Software, including possible enhancements or modifications thereto.
“Intellectual Property Rights”
“Intellectual Property Rights” means all patent, copyright, trademark, trade secret, trade dress, trade names, utility models, mask work, moral rights, rights of attribution or integrity service marks, master recording and music publishing rights, performance rights, author’s rights, database rights, registered design rights and any applications for the protection or registration of these rights, or other intellectual or industrial property rights or proprietary rights, howsoever arising and in whatever media, whether now known or hereafter devised, whether or not registered, (including all claims and causes of action for infringement, misappropriation or violation and all rights in any registrations and renewals), worldwide and whether existing now or in the future.
“Licensed Software”
“Licensed Software” means Software, Documentation and all modifications owned by NVIDIA.
“Open Source License”
“Open Source License” includes, without limitation, a software license that requires as a condition of use, modification, and/or distribution of such software that the Software be (i) disclosed or distributed in source code form; (ii) be licensed for the purpose of making derivative works; or (iii) be redistributable at no charge.
“Order”
“Order” means a purchase order issued by you, a signed purchase agreement with you, or other ordering document issued by you to NVIDIA or a NVIDIA authorized reseller (including any on-line acceptance process) that references and incorporates the AGREEMENT and is accepted by NVIDIA.
“Software”
“Software” means the NVIDIA software programs licensed to you under the AGREEMENT including, without limitation, libraries, sample code, utility programs and programming code.
“Supplement”
“Supplement” means the additional terms and conditions beyond those stated in this SLA that apply to certain Licensed Software licensed hereunder.
12. TensorRT SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT
TensorRT SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT
The terms set forth in this TensorRT Supplement (“Supplement”) govern your use of the NVIDIA GPU inference engine (the “TensorRT Licensed Software”) under the terms of your software license agreement (“SLA”) as modified by this Supplement. This Supplement is an exhibit to the SLA and is hereby incorporated as an integral part thereto. Capitalized terms used but not defined herein shall have the meaning assigned to them in the SLA. In the event of conflict between the terms in this Supplement and the terms in the SLA, this Supplement shall control.
12.1. TensorRT DISTRIBUTION
Subject to the terms of the SLA and this Supplement, NVIDIA hereby grants you a non-exclusive, nontransferable license during the applicable license term unless earlier terminated pursuant to the SLA, to distribute the libnvinfer and libnvinfer_plugin libraries when delivered to you as part of the TensorRT Licensed Software in source code form or binary form (but not when provided to you as part of a hardware product), subject to the following: such distribution is solely in binary form to your licensees (“Customers”) only as a component of your own software products having additional material functionality beyond the TensorRT Licensed Software (each, a “Licensee Application"). Subject to the terms and conditions of the SLA and this Supplement, you may further authorize Customers to redistribute the libnvinfer and libnvinfer_plugin libraries as incorporated into a Licensee Application, solely in binary form, provided, however, that you shall require in your agreements with your Customers that their distributions be on terms at least as restrictive as those applicable for your use of such TensorRT Licensed Software within a Licensee Application. The expiration or termination of your licenses to the above described TensorRT Licensed Software under the SLA and this Supplement will not affect rights previously granted by you to recipients that were in compliance with the SLA and this Supplement.
In addition to the rights above, for parties that are developing software intended solely for use on Jetson development kits or Jetson modules and running Linux for Tegra software the following shall apply: TensorRT Licensed Software licensed hereunder may be distributed in its entirety, as provided by NVIDIA and without separation of its components, for you and/or your licensees to create software development kits for use only on the Jetson platform and running Linux for Tegra software. You shall require in your agreements with your licensees that their distributions be on terms at least as restrictive as those applicable for your distribution of TensorRT Licensed Software as described in this Section 1.
In addition to the rights above, for parties that are developing software intended solely for use on Jetson development kits or Jetson modules and running Linux for Tegra software the following shall apply: TensorRT Licensed Software licensed hereunder may be distributed in its entirety, as provided by NVIDIA and without separation of its components, for you and/or your licensees to create software development kits for use only on the Jetson platform and running Linux for Tegra software. You shall require in your agreements with your licensees that their distributions be on terms at least as restrictive as those applicable for your distribution of TensorRT Licensed Software as described in this Section 1.
12.2. LICENSE DURATION
Each TensorRT Licensed Software is licensed to you for an initial duration of one year starting from the date of delivery or download. The licenses granted will automatically renew for successive one year periods, provided that NVIDIA reserves the right to terminate licenses upon ninety days (90) days written notice to you prior to the commencement of a renewal year in addition to the termination rights set forth in the SLA.
12.3. EXPIRATION OF TERMINATION OF THIS SUPPLEMENT
Your failure to comply with the terms of this Supplement is ground for termination for breach by NVIDIA under the SLA. This Supplement will automatically expire or terminate upon the expiration or termination of your rights to TensorRT Licensed Software under the SLA or this Supplement.
Notices
Notice
This document is provided for information purposes only and shall not be regarded as a warranty of a certain functionality, condition, or quality of a product. NVIDIA Corporation (“NVIDIA”) makes no representations or warranties, expressed or implied, as to the accuracy or completeness of the information contained in this document and assumes no responsibility for any errors contained herein. NVIDIA shall have no liability for the consequences or use of such information or for any infringement of patents or other rights of third parties that may result from its use. This document is not a commitment to develop, release, or deliver any Material (defined below), code, or functionality.
NVIDIA reserves the right to make corrections, modifications, enhancements, improvements, and any other changes to this document, at any time without notice.
Customer should obtain the latest relevant information before placing orders and should verify that such information is current and complete.
NVIDIA products are sold subject to the NVIDIA standard terms and conditions of sale supplied at the time of order acknowledgement, unless otherwise agreed in an individual sales agreement signed by authorized representatives of NVIDIA and customer (“Terms of Sale”). NVIDIA hereby expressly objects to applying any customer general terms and conditions with regards to the purchase of the NVIDIA product referenced in this document. No contractual obligations are formed either directly or indirectly by this document.
NVIDIA products are not designed, authorized, or warranted to be suitable for use in medical, military, aircraft, space, or life support equipment, nor in applications where failure or malfunction of the NVIDIA product can reasonably be expected to result in personal injury, death, or property or environmental damage. NVIDIA accepts no liability for inclusion and/or use of NVIDIA products in such equipment or applications and therefore such inclusion and/or use is at customer’s own risk.
NVIDIA makes no representation or warranty that products based on this document will be suitable for any specified use. Testing of all parameters of each product is not necessarily performed by NVIDIA. It is customer’s sole responsibility to evaluate and determine the applicability of any information contained in this document, ensure the product is suitable and fit for the application planned by customer, and perform the necessary testing for the application in order to avoid a default of the application or the product. Weaknesses in customer’s product designs may affect the quality and reliability of the NVIDIA product and may result in additional or different conditions and/or requirements beyond those contained in this document. NVIDIA accepts no liability related to any default, damage, costs, or problem which may be based on or attributable to: (i) the use of the NVIDIA product in any manner that is contrary to this document or (ii) customer product designs.
No license, either expressed or implied, is granted under any NVIDIA patent right, copyright, or other NVIDIA intellectual property right under this document. Information published by NVIDIA regarding third-party products or services does not constitute a license from NVIDIA to use such products or services or a warranty or endorsement thereof. Use of such information may require a license from a third party under the patents or other intellectual property rights of the third party, or a license from NVIDIA under the patents or other intellectual property rights of NVIDIA.
Reproduction of information in this document is permissible only if approved in advance by NVIDIA in writing, reproduced without alteration and in full compliance with all applicable export laws and regulations, and accompanied by all associated conditions, limitations, and notices.
THIS DOCUMENT AND ALL NVIDIA DESIGN SPECIFICATIONS, REFERENCE BOARDS, FILES, DRAWINGS, DIAGNOSTICS, LISTS, AND OTHER DOCUMENTS (TOGETHER AND SEPARATELY, “MATERIALS”) ARE BEING PROVIDED “AS IS.” NVIDIA MAKES NO WARRANTIES, EXPRESSED, IMPLIED, STATUTORY, OR OTHERWISE WITH RESPECT TO THE MATERIALS, AND EXPRESSLY DISCLAIMS ALL IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTABILITY, AND FITNESS FOR A PARTICULAR PURPOSE. TO THE EXTENT NOT PROHIBITED BY LAW, IN NO EVENT WILL NVIDIA BE LIABLE FOR ANY DAMAGES, INCLUDING WITHOUT LIMITATION ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, PUNITIVE, OR CONSEQUENTIAL DAMAGES, HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY, ARISING OUT OF ANY USE OF THIS DOCUMENT, EVEN IF NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. Notwithstanding any damages that customer might incur for any reason whatsoever, NVIDIA’s aggregate and cumulative liability towards customer for the products described herein shall be limited in accordance with the Terms of Sale for the product.
VESA DisplayPort
DisplayPort and DisplayPort Compliance Logo, DisplayPort Compliance Logo for Dual-mode Sources, and DisplayPort Compliance Logo for Active Cables are trademarks owned by the Video Electronics Standards Association in the United States and other countries.
HDMI
HDMI, the HDMI logo, and High-Definition Multimedia Interface are trademarks or registered trademarks of HDMI Licensing LLC.
ARM
ARM, AMBA and ARM Powered are registered trademarks of ARM Limited. Cortex, MPCore and Mali are trademarks of ARM Limited. All other brands or product names are the property of their respective holders. "ARM" is used to represent ARM Holdings plc; its operating company ARM Limited; and the regional subsidiaries ARM Inc.; ARM KK; ARM Korea Limited.; ARM Taiwan Limited; ARM France SAS; ARM Consulting (Shanghai) Co. Ltd.; ARM Germany GmbH; ARM Embedded Technologies Pvt. Ltd.; ARM Norway, AS and ARM Sweden AB.
OpenCL
OpenCL is a trademark of Apple Inc. used under license to the Khronos Group Inc.
Trademarks
NVIDIA, the NVIDIA logo, and cuBLAS, CUDA, CUDA Toolkit, cuDNN, DALI, DIGITS, DGX, DGX-1, DGX-2, DGX Station, DLProf, GPU, JetPack, Jetson, Kepler, Maxwell, NCCL, Nsight Compute, Nsight Systems, NVCaffe, NVIDIA Ampere GPU architecture, NVIDIA Deep Learning SDK, NVIDIA Developer Program, NVIDIA GPU Cloud, NVLink, NVSHMEM, PerfWorks, Pascal, SDK Manager, T4, Tegra, TensorRT, TensorRT Inference Server, Tesla, TF-TRT, Triton Inference Server, Turing, and Volta are trademarks and/or registered trademarks of NVIDIA Corporation in the United States and other countries. Other company and product names may be trademarks of the respective companies with which they are associated.
Copyright
© 2021 NVIDIA Corporation. All rights reserved.
python/packaging/bindings_wheel/tensorrt/__init__.py
+224
View File
@@ -0,0 +1,224 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import ctypes
import os
import sys
import warnings
# For standalone wheels, attempt to import the wheel containing the libraries.
_libs_wheel_imported = False
try:
import ##TENSORRT_MODULE##_libs
except (ImportError, ModuleNotFoundError):
pass
else:
_libs_wheel_imported = True
_trt_lib_suffix = ""
if "##TENSORRT_NVINFER_NAME##".strip() == "tensorrt_rtx":
_trt_lib_suffix = "_##TENSORRT_MINOR##"
if not _libs_wheel_imported and sys.platform.startswith("win"):
log_found_dlls = bool(int(os.environ.get("TRT_LOG_FOUND_DLLS", 0)))
# On Windows, we need to manually open the TensorRT libraries - otherwise we are unable to
# load the bindings. If we imported the tensorrt_libs wheel, then that should have taken care of it for us.
def find_lib(name):
paths = os.environ["PATH"].split(os.path.pathsep)
# Add ../##TENSORRT_MODULE##_libs to the search path. This allows repackaging non-standalone TensorRT wheels as standalone
# using delvewheel (with the --no-mangle-all flag set) to work properly.
paths.append(os.path.join(os.path.dirname(__file__), os.pardir, "##TENSORRT_MODULE##_libs"))
for path in paths:
libpath = os.path.join(path, name)
if os.path.isfile(libpath):
if log_found_dlls:
print(f"Found {name} in path: {libpath}")
return libpath
if ##TENSORRT_PLUGIN_DISABLED## and name.startswith("nvinfer_plugin"):
return None
if name.startswith("nvinfer_builder_resource"):
return None
raise FileNotFoundError(
"Could not find: {:}. Is it on your PATH?\nNote: Paths searched were:\n{:}".format(name, paths)
)
# Order matters here because of dependencies
LIBRARIES = {
"tensorrt": [
f"##TENSORRT_NVINFER_NAME##_##TENSORRT_MAJOR##{_trt_lib_suffix}.dll",
"nvinfer_plugin_##TENSORRT_MAJOR##.dll",
f"##TENSORRT_ONNXPARSER_NAME##_##TENSORRT_MAJOR##{_trt_lib_suffix}.dll",
"nvinfer_builder_resource_##TENSORRT_MAJOR##.dll",
],
"tensorrt_rtx": [
f"##TENSORRT_NVINFER_NAME##_##TENSORRT_MAJOR##{_trt_lib_suffix}.dll",
"nvinfer_plugin_##TENSORRT_MAJOR##.dll",
f"##TENSORRT_ONNXPARSER_NAME##_##TENSORRT_MAJOR##{_trt_lib_suffix}.dll",
"nvinfer_builder_resource_##TENSORRT_MAJOR##.dll",
],
"tensorrt_dispatch": [
"nvinfer_dispatch_##TENSORRT_MAJOR##.dll",
],
"tensorrt_lean": [
"nvinfer_lean_##TENSORRT_MAJOR##.dll",
],
}["##TENSORRT_MODULE##"]
for lib in LIBRARIES:
lib_path = find_lib(lib)
if not lib_path:
continue
assert os.path.isfile(lib_path)
ctypes.CDLL(lib_path)
del _libs_wheel_imported
del _trt_lib_suffix
from .##TENSORRT_MODULE## import *
__version__ = "##TENSORRT_PYTHON_VERSION##"
# Provides Python's `with` syntax
def common_enter(this):
warnings.warn(
"Context managers for TensorRT types are deprecated. "
"Memory will be freed automatically when the reference count reaches 0.",
DeprecationWarning,
)
return this
def common_exit(this, exc_type, exc_value, traceback):
"""
Context managers are deprecated and have no effect. Objects are automatically freed when
the reference count reaches 0.
"""
pass
# Logger does not have a destructor.
ILogger.__enter__ = common_enter
ILogger.__exit__ = lambda this, exc_type, exc_value, traceback: None
ICudaEngine.__enter__ = common_enter
ICudaEngine.__exit__ = common_exit
IExecutionContext.__enter__ = common_enter
IExecutionContext.__exit__ = common_exit
Runtime.__enter__ = common_enter
Runtime.__exit__ = common_exit
IHostMemory.__enter__ = common_enter
IHostMemory.__exit__ = common_exit
if "##TENSORRT_MODULE##" == "tensorrt" or "##TENSORRT_MODULE##" == "tensorrt_rtx":
Builder.__enter__ = common_enter
Builder.__exit__ = common_exit
INetworkDefinition.__enter__ = common_enter
INetworkDefinition.__exit__ = common_exit
OnnxParser.__enter__ = common_enter
OnnxParser.__exit__ = common_exit
IBuilderConfig.__enter__ = common_enter
IBuilderConfig.__exit__ = common_exit
# Add logger severity into the default implementation to preserve backwards compatibility.
Logger.Severity = ILogger.Severity
for attr, value in ILogger.Severity.__members__.items():
setattr(Logger, attr, value)
# Computes the volume of an iterable.
def volume(iterable):
"""
Computes the volume of an iterable.
:arg iterable: Any python iterable, including a :class:`Dims` object.
:returns: The volume of the iterable. This will return 1 for empty iterables, as a scalar has an empty shape and the volume of a tensor with empty shape is 1.
"""
vol = 1
for elem in iterable:
vol *= elem
return vol
# Converts a TensorRT datatype to the equivalent numpy type.
def nptype(trt_type):
"""
Returns the numpy-equivalent of a TensorRT :class:`DataType` .
:arg trt_type: The TensorRT data type to convert.
:returns: The equivalent numpy type.
"""
import numpy as np
mapping = {
float32: np.float32,
float16: np.float16,
int8: np.int8,
int32: np.int32,
int64: np.int64,
bool: np.bool_,
uint8: np.uint8,
# Note: fp8 and bfloat16 have no equivalent numpy type
}
if trt_type in mapping:
return mapping[trt_type]
raise TypeError("Could not resolve TensorRT datatype to an equivalent numpy datatype.")
# Add a numpy-like itemsize property to the datatype.
def _itemsize(trt_type):
"""
Returns the size in bytes of this :class:`DataType`.
The returned size is a rational number, possibly a `Real` denoting a fraction of a byte.
:arg trt_type: The TensorRT data type.
:returns: The size of the type.
"""
mapping = {
float32: 4,
float16: 2,
bfloat16: 2,
int8: 1,
int32: 4,
int64: 8,
bool: 1,
uint8: 1,
fp8: 1,
int4: 0.5,
fp4: 0.5,
}
if trt_type in mapping:
return mapping[trt_type]
DataType.itemsize = property(lambda this: _itemsize(this))
python/packaging/bindings_wheel/tensorrt/plugin/__init__.py
+46
View File
@@ -0,0 +1,46 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import tensorrt as trt
logger = trt.Logger()
logger.log(trt.Logger.WARNING, "Functionality provided through tensorrt.plugin module is experimental.")
# export.public_api() will expose things here. To make sure that happens, we just need to
# import all the submodules so that the decorator is actually executed (__discover_modules() below).
__all__ = []
def __discover_modules():
import importlib
import pkgutil
mods = [importlib.import_module(__package__)]
while mods:
mod = mods.pop(0)
yield mod
if hasattr(mod, "__path__"):
mods.extend(
[
importlib.import_module(f"{mod.__name__}.{submod.name}")
for submod in pkgutil.iter_modules(mod.__path__)
]
)
_ = list(__discover_modules())
python/packaging/bindings_wheel/tensorrt/plugin/_autotune.py
+270
View File
@@ -0,0 +1,270 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import builtins
import tensorrt as trt
from typing import List, Iterable
import copy
from ._utils import _str_to_data_type
from ._export import public_api
# "onesided" means either type or format combinations. After combinations for each are separately generated, we will combine them later.
# e.g. io_variants = ["FP32|FP16", "FP32|FP16", "FP32*FP16"] for a plugin with 3 I/Os. i.e. I/O indices 0 and 1 are dependently either FP32/FP16 and index 2 is independently FP32/FP16.
# There will be 2 * 2 = 4 combinations here: ["FP32", "FP32", "FP32"], ["FP16", "FP16", "FP32"], ["FP32", "FP32", "FP16"], ["FP16", "FP16", "FP16"]
def _gen_onesided_combinations(io_variants):
# Algorithm:
# (1) Ignore independent variants and count the (max) number of dependent variants `mx_poly`
# (2) Compile initial list of #`mx_poly` combinations using the first option (option 0) for any independent variants
# (3) For each independent variant IO index, add combinations with that index replaced by option 1, 2, ...
combinations = []
mx_poly = 0 # This is the number of dependent variants
for io_variant in io_variants:
io_variant_list = io_variant.split("|")
if len(io_variant_list) > 1:
if "*" in io_variant:
raise ValueError(
f"Type/Format '{io_variant}' contains both '|' and '*'"
)
if mx_poly > 1:
if mx_poly != len(io_variant_list):
raise ValueError(
f"Type/Format combinations {io_variants} contain illegal dependent lengths"
)
mx_poly = builtins.max(mx_poly, len(io_variant_list))
for _ in range(mx_poly):
combinations.append([None] * len(io_variants))
for j, io_variant in enumerate(io_variants):
io_variant_list = io_variant.split("|")
if len(io_variant_list) == 1:
if "*" in io_variant:
io_variant_list = io_variant.split("*")
for i in range(len(combinations)):
combinations[i][j] = io_variant_list[0]
else:
for k in range(len(io_variant_list)):
combinations[k][j] = io_variant_list[k]
for j, io_variant in enumerate(io_variants):
new_combs = []
if "*" in io_variant:
io_variant_list = io_variant.split("*")
for k in range(1, len(io_variant_list)):
for c in combinations:
new_c = copy.deepcopy(c)
new_c[j] = io_variant_list[k]
new_combs.append(new_c)
combinations.extend(new_combs)
return combinations
class _TypeFormatCombination:
def __init__(self, num=0):
self.types = [None] * num
self.layouts = [None] * num
self.tactics = []
def set_types(self, types):
self.types = types
def set_layouts(self, layouts=None):
if isinstance(layouts, List):
self.layouts = layouts
else:
self.layouts = [layouts] * len(self.types)
def __hash__(self):
return hash((tuple(self.types), tuple(self.layouts)))
def __eq__(self, other):
return (
isinstance(other, _TypeFormatCombination)
and self.types == other.types
and self.layouts == other.layouts
)
def __str__(self) -> str:
return "{" + str(self.types) + ", " + str(self.layouts) + "}"
@public_api()
class AutoTuneCombination:
def __init__(
self, io_types: str = None, layouts: str = None, tactics: Iterable[int] = None
):
"""
Construct a set of supported type/format combinations of a plugin's I/O.
Any custom *tactic* s per each such type/format combination can also be advertised. A tactic is simply another way to
calculate the output of a plugin for the same type/format combination of the I/O (e.g. if there are multiple kernels available).
Args:
io_types (str, optional): A string representation of a type combination.
Valid format is "type0,type1,...,type#io" where 'type' is of the form "TYPE0[sep]TYPE1[sep]...".
TYPE is a valid string representation of a `trt.DataType`. These include "FP32" for trt.float32, "FP16" for trt.float16. The string representation of other data types is the same as their name in the trt.DataType enum.
[sep] is a valid separator, which is either '|' or '*'. Only one of these separators can appear in a given `io_types`.
(1). '|' indicates a dependent combination: the dependence of the type of one I/O to another I/O. e.g. "FP32|FP16,FP32|FP16" indicates the IO can only be both FP32 or both FP16.
(2). '*' indicates an independent combination. e.g. "FP32*FP16,FP32|FP16,FP32|FP16" indicates that the first input is independently either FP32 or FP16 regardless of the rest of the IO.
layouts (str, optional): A string representation of a format combination.
Valid format is "format0,format1,...,format#io" where 'format' is of the form "FORMAT0[sep]FORMAT1[sep]...".
FORMAT is a valid string representation of a `trt.TensorFormat`. These are string versions for the enum values of `trt.TensorFormat`. e.g. "LINEAR" for `trt.TensorFormat.LINEAR`.
[sep] is a valid separator, which is either '|' or '*'. The rules are the same as for `io_types`.
tactics (Iterable[int], optional): Custom tactics for this type/format combination. Each custom tactic must be a positive integer. Defaults to default tactic (0).
.. code-block:: python
:linenos:
:caption: For a plugin with 3 I/Os, I/O indices 0 and 1 are dependently either FP32/FP16 and index 2 is independently FP32/FP16.
@trtp.autotune("my::plugin")
def autotune(inp0: trtp.TensorDesc, inp1: trtp.TensorDesc, outputs: Tuple[trtp.TensorDesc]) -> List[trtp.AutoTuneCombination]:
# The following would result in the following type combinations:
# [FP32, FP32, FP32], [FP16, FP16, FP32], [FP32, FP32, FP16], [FP16, FP16, FP16]
return [trtp.AutoTuneCombination("FP32|FP16, FP32|FP16, FP32|FP16", "LINEAR", [1, 2])]
.. code-block:: python
:linenos:
:caption: For a plugin with 2 I/Os, the input/output supports either LINEAR or HWC format for FP32 and LINEAR format for FP16.
@trtp.autotune("my::plugin")
def autotune(inp0: trtp.TensorDesc, outputs: Tuple[trtp.TensorDesc]) -> List[trtp.AutoTuneCombination]:
# Even though (FP16, HWC) is not a valid combination (see next example), TRT should intelligently reject those
# and pass the following combinations to the impl function:
# [{FP32, FP32}, {LINEAR, LINEAR}], [{FP32, FP32}, {HWC, LINEAR}], [{FP16, FP32}, {LINEAR, LINEAR}]
return [trtp.AutoTuneCombination("FP32*FP16, FP32", "LINEAR*HWC, LINEAR", [1, 2])]
.. code-block:: python
:linenos:
:caption: For a plugin with 2 I/Os, the input/output supports either LINEAR or HWC format for FP32 and LINEAR format for FP16 (second method).
@trtp.autotune("my::plugin")
def autotune(inp0: trtp.TensorDesc, outputs: Tuple[trtp.TensorDesc]) -> List[trtp.AutoTuneCombination]:
# We can use two AutoTuneCombination objects to avoid communicating illegal combinations
return [trtp.AutoTuneCombination("FP32*FP16, FP32", "LINEAR, LINEAR", [1, 2]), trtp.AutoTuneCombination("FP32, FP32", "HWC, LINEAR", [1, 2])]
"""
if io_types is not None:
self.io_types = [s.strip() for s in io_types.split(",")]
if layouts is None:
layouts = "LINEAR"
self.layouts = [s.strip() for s in layouts.split(",")]
if len(self.layouts) > 1:
assert len(self.io_types) == len(self.layouts)
if len(self.io_types) > len(self.layouts):
assert len(self.layouts) == 1
self.layouts = [self.layouts[0]] * len(self.io_types)
else:
self.io_types = []
self.layouts = []
self.combinations = []
self._tactics = tactics
def pos(self, pos: Iterable[int], io_types: str, layouts: str = "LINEAR") -> None:
"""
Specify I/O types and formats for a specified set of I/O indices.
Args:
pos (Iterable[int]): I/O indices. Input indices are [0, 1, ..., #inputs - 1] and output indices are [#inputs, #inputs + 1, ..., #inputs + #outputs - 1].
io_types (str): Data types for these I/O indices.
layouts (str, optional): Tensor format(s) for these I/O indices. Defaults to "LINEAR".
Raises:
ValueError: If types or layouts for any of these I/O indices is already specified.
.. code-block:: python
:linenos:
:caption: For a plugin with 3 I/Os, I/O indices 0 and 1 are dependently either FP32/FP16 and index 2 is independently FP32/FP16.
@trtp.autotune("my::plugin")
def autotune(inp0: trtp.TensorDesc, inp1: trtp.TensorDesc, outputs: Tuple[trtp.TensorDesc]) -> List[trtp.AutoTuneCombination]:
c = trtp.AutoTuneCombination()
c.pos([0, 1], "FP32|FP16", "LINEAR")
c.pos(2, "FP32*FP16") # Omitting format is the same as declaring it to be LINEAR.
c.tactics([1, 2])
return [c]
"""
if max(pos) >= len(self.io_types):
self.io_types.extend([None] * (max(pos) + 1 - len(self.io_types)))
self.layouts.extend([None] * (max(pos) + 1 - len(self.layouts)))
assert len(self.io_types) == len(self.layouts)
for p in pos:
if self.io_types[p] is not None:
raise ValueError(f"Type(s) for position {p} already specified")
if self.layouts[p] is not None:
raise ValueError(f"Layout(s) for position {p} already specified")
self.io_types[p] = io_types
self.layouts[p] = layouts
def tactics(self, tactics: Iterable[int]) -> None:
"""
Specify custom tactics for this type/format combination
Args:
tactics (Iterable[int]): Custom tactics. These must be positive integers.
"""
self._tactics = tactics
def _generate_combinations(self):
self.combinations = []
type_combinations = _gen_onesided_combinations(self.io_types)
layout_combinations = _gen_onesided_combinations(self.layouts)
for t in type_combinations:
for l in layout_combinations:
c = _TypeFormatCombination(len(self.io_types))
c.types = [_str_to_data_type(tt) for tt in t]
c.layouts = [getattr(trt.TensorFormat, ff) for ff in l]
c.tactics = self._tactics
self.combinations.append(c)
def _get_combinations(self):
self._generate_combinations()
return self.combinations
def _check(self, pos, type, layout):
for i in range(len(self.combinations)):
if (
self.combinations[i].types[pos] == _str_to_data_type(type)
and self.combinations[i].layouts[pos] == layout.name
):
return True
return False
python/packaging/bindings_wheel/tensorrt/plugin/_export.py
+39
View File
@@ -0,0 +1,39 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import tensorrt as trt
from types import ModuleType
import importlib
def public_api(module: ModuleType = None, symbol: str = None):
def export_impl(obj):
nonlocal module, symbol
module = module or importlib.import_module(__package__)
symbol = symbol or obj.__name__
if not hasattr(module, "__all__"):
module.__all__ = []
module.__all__.append(symbol)
setattr(module, symbol, obj)
return obj
return export_impl
IS_AOT_ENABLED = hasattr(trt, "QuickPluginCreationRequest")
python/packaging/bindings_wheel/tensorrt/plugin/_lib.py
+695
View File
@@ -0,0 +1,695 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import tensorrt as trt
import types
import typing
from typing import Callable, Tuple, List
import numpy as np
from ._plugin_class import _TemplateJITPlugin
from ._export import IS_AOT_ENABLED
if IS_AOT_ENABLED:
from ._plugin_class import _TemplateAOTPlugin
from ._validate import (
_parse_register_inputs,
_parse_register_return,
_validate_autotune,
_validate_impl,
_validate_aot_impl,
_validate_name_and_namespace,
)
from ._utils import (
_built_in_to_plugin_field_type,
_join_with,
_numpy_to_plugin_field_type,
_is_numpy_array,
_infer_numpy_type,
)
from ._export import public_api
# Namespace to which plugins are dynamically bound
# A namespace can be thought of as a library of plugins from the same author/common objective
class _PluginNamespace(types.ModuleType):
def __init__(self, namespace):
super().__init__("tensorrt.plugin.op." + namespace)
self._namespace = namespace
def define(self, name, plugin_def):
assert not hasattr(self, name)
setattr(self, name, plugin_def)
def __getattr__(self, name):
raise AttributeError(
f"'{self.__class__.__name__}' object '{self._namespace}' has no attribute '{name}'"
)
def __repr__(self):
return f'_PluginNamespace(namespace="{self._namespace}")'
# `tensorrt.plugin.op` module to which plugin namespaces are dynamically bound
class _Op(types.ModuleType):
def __init__(self):
super().__init__("tensorrt.plugin.op")
def define_or_get(self, namespace):
if hasattr(self, namespace):
return getattr(self, namespace)
ns = _PluginNamespace(namespace)
setattr(self, namespace, ns)
return ns
def __getattr__(self, name):
raise AttributeError(
f"'{self.__class__.__name__}' object has no attribute '{name}'"
)
op = _Op()
public_api(symbol="op")(op)
QDP_CREATORS = {}
QDP_REGISTRY = {}
# Contains metadata about a registered plugin and `__call__()`` that allows for a plugin instance to be created
class PluginDef:
def __init__(self):
self.plugin_id = None # includes namespace (format is ns::name)
self.register_func = None
self.impl_func = None
self.aot_impl_func = None
self.autotune_func = None
self.autotune_attr_names = None
self.input_tensor_names = None
self.input_attrs = None # map name -> type
self.impl_attr_names = None
self.aot_impl_attr_names = None
self.num_outputs = None
self.input_arg_schema = None
self.expects_tactic = None
def __call__(
self, *args, **kwargs
) -> Tuple[List[trt.ITensor], List[trt.ITensor], trt.IPluginV3]:
namespace, name = self.plugin_id.split("::")
input_tensors = []
schema_chunks = []
for t in args:
if not isinstance(t, trt.ITensor):
raise ValueError(
f"Expected trt.ITensor but got input of type {type(t)}"
)
schema_chunks.append("ITensor")
input_tensors.append(t)
attrs = {}
for key, value in kwargs.items():
if key not in self.input_attrs:
raise ValueError(
f"Unexpected attribute {key} provided. Expected one of {self.input_attrs.keys()}."
)
attrs[key] = value
attr_annotation = self.input_attrs[key]
if isinstance(value, np.ndarray):
if typing.get_origin(attr_annotation) == np.ndarray:
np_dtype = typing.get_args(typing.get_args(attr_annotation)[1])[0]
if np.dtype(np_dtype) != np.dtype(value.dtype):
raise ValueError(
f"Unexpected dtype '{np.dtype(value.dtype)}' for attribute '{key}'. Expected '{np_dtype}'."
)
else:
if attr_annotation is not type(value):
raise ValueError(
f"Unexpected type '{type(value)}' for attribute '{key}'. Expected '{attr_annotation}'."
)
schema_chunks.append(key)
expected_schema = (
f"({_join_with(['ITensor'] * len(self.input_tensor_names))}"
+ _join_with(self.input_attrs.keys(), True)
+ ")"
)
schema = f"({', '.join(schema_chunks)})"
if schema != expected_schema:
raise ValueError(
f"Unexpected schema {schema} received. Expected {expected_schema}."
)
if self.plugin_id in QDP_CREATORS:
plg_creator = trt.get_plugin_registry().get_creator(name, "1", namespace)
else:
attrs_types = {}
for key, value in kwargs.items():
if isinstance(value, np.ndarray):
attrs_types[key] = (False, value.dtype) # (builtin?, type)
else:
attrs_types[key] = (True, type(value)) # (builtin?, type)
plg_creator = _register_plugin_creator(name, namespace, attrs_types)
fields = []
for key, value in attrs.items():
if isinstance(value, np.ndarray):
np_type = np.dtype(value.dtype)
if np_type == np.float16:
fields.append(
trt.PluginField(
key, value.tobytes(), trt.PluginFieldType.UNKNOWN
)
)
else:
fields.append(
trt.PluginField(
key, value, _numpy_to_plugin_field_type[np_type]
)
)
elif isinstance(value, str):
fields.append(
trt.PluginField(key, value.encode(), trt.PluginFieldType.CHAR)
)
elif isinstance(value, bytes):
fields.append(trt.PluginField(key, value, trt.PluginFieldType.UNKNOWN))
else:
fields.append(
trt.PluginField(
key,
np.array([value]),
_built_in_to_plugin_field_type[type(value)],
)
)
def create_plugin_instance(quick_plugin_creation_request: "trt.QuickPluginCreationRequest" = None):
if quick_plugin_creation_request is None:
plg = plg_creator.create_plugin(
name,
namespace,
trt.PluginFieldCollection(fields),
trt.TensorRTPhase.BUILD
)
else:
plg = plg_creator.create_plugin(
name,
namespace,
trt.PluginFieldCollection(fields),
trt.TensorRTPhase.BUILD,
quick_plugin_creation_request
)
return input_tensors, [], plg
return create_plugin_instance
class _TemplatePluginCreator(trt.IPluginCreatorV3Quick):
def __init__(self, name, namespace, attrs):
trt.IPluginCreatorV3Quick.__init__(self)
self.name = name
self.plugin_namespace = namespace
self.plugin_version = "1"
field_names = []
for name, (builtin, type_) in attrs.items():
if builtin:
if type_ is str:
field_names.append(
trt.PluginField(name, b"", trt.PluginFieldType.CHAR)
)
elif type_ is bytes:
field_names.append(
trt.PluginField(name, b"", trt.PluginFieldType.UNKNOWN)
)
else:
field_names.append(
trt.PluginField(
name, np.array([]), _built_in_to_plugin_field_type[type_]
)
)
else:
field_names.append(
trt.PluginField(
name, np.array([]), _numpy_to_plugin_field_type[np.dtype(type_)]
)
)
self.field_names = trt.PluginFieldCollection(field_names)
def create_plugin(self, name, namespace, fc, phase, qpcr: "trt.QuickPluginCreationRequest" = None):
desc = QDP_REGISTRY[f"{namespace}::{name}"]
name = name
namespace = namespace
attrs = {}
for f in fc:
if f.name not in desc.input_attrs:
raise AssertionError(
f"Unexpected attribute {f.name} provided to create_plugin. Expected one of {desc.input_attrs.keys()}."
)
attr_type_annot = desc.input_attrs[f.name]
if _is_numpy_array(attr_type_annot):
np_type = _infer_numpy_type(attr_type_annot)
if np_type == np.float16:
attrs[f.name] = np.frombuffer(f.data.tobytes(), dtype=np.float16)
else:
attrs[f.name] = f.data.astype(np_type)
else:
if issubclass(attr_type_annot, str):
attrs[f.name] = f.data.tobytes().decode("utf-8")
else:
attrs[f.name] = attr_type_annot(f.data)
jit_or_aot = None # True if JIT is to be created, False if AOT. Not None will be asserted before plugin creation.
if qpcr is None:
plg = _TemplateJITPlugin(name, namespace, desc.num_outputs)
plg.init(
desc.register_func,
attrs,
desc.impl_attr_names,
desc.impl_func,
desc.autotune_attr_names,
desc.autotune_func,
desc.expects_tactic,
)
return plg
# If there is a strict preference, that takes precedence
if qpcr == trt.QuickPluginCreationRequest.STRICT_AOT:
if desc.aot_impl_func is None:
raise ValueError(f"AOT implementation requested, but not defined for '{desc.plugin_id}'. Was @trt.plugin.aot_impl defined?")
jit_or_aot = False
elif qpcr == trt.QuickPluginCreationRequest.STRICT_JIT:
if desc.impl_func is None:
raise ValueError(f"JIT implementation requested, but not defined for '{desc.plugin_id}'. Was @trt.plugin.impl defined?")
jit_or_aot = True
else:
aot_defined = desc.aot_impl_func is not None
jit_defined = desc.impl_func is not None
# A preferemce must be indicated if both AOT and JIT implementations are defined
if aot_defined and jit_defined:
if qpcr == trt.QuickPluginCreationRequest.PREFER_AOT:
jit_or_aot = False
elif qpcr == trt.QuickPluginCreationRequest.PREFER_JIT:
jit_or_aot = True
else:
raise ValueError(f"Plugin '{desc.plugin_id}' has both AOT and JIT implementations. NetworkDefinitionCreationFlag.PREFER_AOT_PYTHON_PLUGINS or NetworkDefinitionCreationFlag.PREFER_JIT_PYTHON_PLUGINS should be specified.")
else:
# If only one implementation is defined, use that.
# Any preference specified is ignored. If the preference is strong, a strict flag should have been specified.
if aot_defined:
jit_or_aot = False
elif jit_defined:
jit_or_aot = True
else:
raise ValueError(f"Plugin '{desc.plugin_id}' does not have either a AOT or JIT implementation.")
assert jit_or_aot is not None
if jit_or_aot:
plg = _TemplateJITPlugin(name, namespace, desc.num_outputs)
plg.init(
desc.register_func,
attrs,
desc.impl_attr_names,
desc.impl_func,
desc.autotune_attr_names,
desc.autotune_func,
desc.expects_tactic,
)
else:
plg = _TemplateAOTPlugin(name, namespace, desc.num_outputs)
plg.init(
desc.register_func,
attrs,
desc.aot_impl_attr_names,
desc.aot_impl_func,
desc.autotune_attr_names,
desc.autotune_func
)
# the caller can determine if the created plugin is an AOT or JIT plugin by inspecting the interface info
return plg
def _register_plugin_creator(name: str, namespace: str, attrs_types):
plg_registry = trt.get_plugin_registry()
plg_creator = _TemplatePluginCreator(name, namespace, attrs_types)
plg_registry.register_creator(plg_creator, namespace)
plg_creator = plg_registry.get_creator(name, "1", namespace)
QDP_CREATORS[f"{namespace}::{name}"] = plg_creator
return plg_creator
# Decorator for `tensorrt.plugin.register`
# By default, the plugin will be immediately registered in the TRT plugin registry
# During plugin development/when building engine, lazy registration may be used to delay plugin registration until the plugin is explicitly instantiated using `trt.plugin.op.ns.plugin_name(...)`
@public_api()
def register(plugin_id: str, lazy_register: bool = False) -> Callable:
"""
Wraps a function to register and describe a TensorRT plugin's IO characteristics. In addition, a complete plugin at least needs an `trt.plugin.impl` function to be registered.
This API is only intended to be used as a decorator. The decorated function must have type hints for all inputs as well as return value.
.. code-block:: text
(inp0: TensorDesc, inp1: TensorDesc, ..., attr0: SupportedAttrType, attr1: SupportedAttrType, ...) -> Union[TensorDesc, Tuple[TensorDesc]]
* Input tensors are declared first, each described by a tensor descriptor TensorDesc.
* Plugin attributes are declared next. "SupportedAttrType" must be one of:
* Supported built-in types: int, float, str, bool, bytes (Note: Lists/tuples of these types are not supported)
* 1-D Numpy arrays of the following types: int8, int16, int32, int64, float16, float32, float64, bool. These must be annotated with 'numpy.typing.NDArray[dtype]', where 'dtype' is the expected numpy dtype.
* If the plugin has only one output, the return annotation could be TensorDesc. Tuple[TensorDesc] could be used for any number of outputs.
By default, the plugin will be immediately registered in the TRT plugin registry. Use the lazy_register argument to change this.
Args:
plugin_id: An ID for the plugin in the form "{namespace}::{name}",
e.g. "my_project::add_plugin". The namespace is used to avoid collisions
so using your product/project name is recommended.
lazy_register: During plugin development/when building engine, lazy registration may be used to delay plugin registration until the plugin is explicitly instantiated using `trt.plugin.op.ns.plugin_name(...)`
.. code-block:: python
:linenos:
:caption: Registration of an elementwise plugin (output has same characteristics as the input)
import tensorrt.plugin as trtp
@trtp.register("my::add_plugin")
def add_plugin_desc(inp0: trtp.TensorDesc, block_size: int) -> Tuple[trtp.TensorDesc]:
return inp0.like()
"""
def decorator(register_func: Callable):
plugin_ns, plugin_name = plugin_id.split("::")
_validate_name_and_namespace(plugin_ns, plugin_name)
op_namespace = op.define_or_get(plugin_ns)
if hasattr(op_namespace, plugin_name):
raise ValueError(
f"'{op.__class__.__name__}' already has a defintion for '{plugin_name}'"
)
(
tensor_names,
input_attrs,
input_arg_schema,
attrs_types,
) = _parse_register_inputs(register_func, lazy_register)
plugin_def = PluginDef()
plugin_def.plugin_id = plugin_id
plugin_def.register_func = register_func
plugin_def.input_tensor_names = tensor_names
plugin_def.input_attrs = input_attrs
plugin_def.input_arg_schema = input_arg_schema
num_outputs = _parse_register_return(register_func)
plugin_def.num_outputs = num_outputs
QDP_REGISTRY[plugin_id] = plugin_def
if not lazy_register:
_register_plugin_creator(plugin_name, plugin_ns, attrs_types)
op_namespace.define(plugin_name, plugin_def)
return register_func
return decorator
# Decorator for `tensorrt.plugin.impl`
@public_api()
def impl(plugin_id: str) -> Callable:
"""
Wraps a function to define an implementation for a plugin already registered through `trt.plugin.register`.
This API is only intended to be used as a decorator. The decorated function is not required to have type hints for input arguments or return value;
however, any type hints specified will be validated against the `trt.plugin.register` signature for consistency.
The schema for the function is as follows:
.. code-block:: text
(inp0: Tensor, inp1: Tensor, ..., attr0: SupportedAttrType, attr1: SupportedAttrType, outputs: Tuple[Tensor], stream: int, tactic: Optional[int]) -> None
* Input tensors are passed first, each described by a `Tensor`.
* Plugin attributes are declared next.
* Not all attributes included in `trt.plugin.register` must be specified here -- they could be a subset.
* Included attributes will be serialized to the TRT engine. Therefore, only attributes the plugin actually needs to perform inference (within the body of `trt.plugin.impl`) should be included.
* `tactic` is an optional argument. If the plugin is using custom tactics, it must be specified to receive the tactic value to use for the current execution of the plugin.
Args:
plugin_id: The ID for the plugin in the form "{namespace}::{name}", which must match that used during `trt.plugin.register`
.. code-block:: python
:linenos:
:caption: Implementation of an elementwise plugin with an OpenAI Triton kernel
import tensorrt.plugin as trtp
import triton
import triton.language as tl
@triton.jit
def add_kernel(x_ptr, y_ptr, n_elements, BLOCK_SIZE: tl.constexpr):
pid = tl.program_id(0)
offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
x = tl.load(x_ptr + offsets, mask=mask)
tl.store(y_ptr + offsets, x + 1, mask=mask)
@trtp.register("my::add_plugin")
def add_plugin_desc(inp0: trtp.TensorDesc, block_size: int) -> Tuple[trtp.TensorDesc]:
return inp0.like()
@trtp.impl("my::add_plugin")
def add_plugin_impl(inp0: trtp.Tensor, block_size: int, outputs: Tuple[trtp.Tensor], stream: int) -> None:
n = inp0.numel()
inp0_t = torch.as_tensor(inp0, device="cuda")
out_t = torch.as_tensor(outputs[0], device="cuda")
add_kernel[(triton.cdiv(n, block_size),)](inp0_t, out_t, n, BLOCK_SIZE = block_size)
"""
def decorator(impl_func: Callable):
if plugin_id not in QDP_REGISTRY:
raise ValueError(
f"Plugin {plugin_id} is not registered. Did you register it with tensorrt.plugin.register API?"
)
plugin_def = QDP_REGISTRY[plugin_id]
impl_attr_names, found_tactic = _validate_impl(impl_func, plugin_def)
plugin_def.impl_func = impl_func
plugin_def.impl_attr_names = impl_attr_names
plugin_def.expects_tactic = found_tactic
return impl_func
return decorator
# Decorator for `tensorrt.plugin.aot_impl`
@public_api()
def aot_impl(plugin_id: str) -> Callable:
"""
Wraps a function to define an Ahead-of-Time (AOT) implementation for a plugin already registered through `trt.plugin.register`.
This API is only intended to be used as a decorator. The decorated function is not required to have type hints for input arguments or return value;
however, any type hints specified will be validated against the `trt.plugin.register` signature for consistency.
The schema for the function is as follows:
.. code-block:: text
(inp0: TensorDesc, inp1: TensorDesc, ..., attr0: SupportedAttrType, attr1: SupportedAttrType, outputs: Tuple[TensorDesc], tactic: Optional[int]) -> Tuple[str, str, KernelLaunchParams, SymExprs]
* Input tensors are passed first, each described by a `TensorDesc`.
* Plugin attributes are declared next.
* Not all attributes included in `trt.plugin.register` must be specified here -- they could be a subset.
* NOTE: Plugin attributes are not serialized into the engine when using an AOT implementation.
* `tactic` is an optional argument. If the plugin is using custom tactics, it must be specified to receive the tactic value to use for the current execution of the plugin.
Args:
plugin_id: The ID for the plugin in the form "{namespace}::{name}", which must match that used during `trt.plugin.register`
:returns:
- kernel_name: The name of the kernel.
- compiled_kernel: Compiled form of the kernel. Presently, only PTX is supported.
- launch_params: The launch parameters for the kernel
- extra_args: Symbolic expressions for scalar inputs to the kernel, located after the tensor inputs and before the tensor outputs
.. code-block:: python
:linenos:
:caption: Implementation of an elementwise plugin with an OpenAI Triton kernel
import tensorrt.plugin as trtp
import triton
import triton.language as tl
@triton.jit
def add_kernel(x_ptr, n_elements, y_ptr, BLOCK_SIZE: tl.constexpr):
pid = tl.program_id(0)
offsets = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
x = tl.load(x_ptr + offsets, mask=mask)
tl.store(y_ptr + offsets, x + 1, mask=mask)
@trtp.register("my::add_plugin")
def add_plugin_desc(inp0: trtp.TensorDesc, block_size: int) -> Tuple[trtp.TensorDesc]:
return inp0.like()
@trtp.aot_impl("my::elemwise_add_plugin")
def add_plugin_aot_impl(
inp0: trtp.TensorDesc, block_size: int, single_tactic: bool, outputs: Tuple[trtp.TensorDesc], tactic: int
) -> Tuple[Union[str, bytes], Union[str, bytes], trtp.KernelLaunchParams, trtp.SymExprs]:
type_str = "fp32" if inp0.dtype == trt.float32 else "fp16"
src = triton.compiler.ASTSource(
fn=add_kernel,
signature={
"x_ptr": f"*{type_str}",
"n_elements": "i32",
"y_ptr": f"*{type_str}",
},
constexprs={
"BLOCK_SIZE": block_size,
},
)
compiled_kernel = triton.compile(src)
N = inp0.shape_expr.numel()
launch_params = trtp.KernelLaunchParams()
# grid dims
launch_params.grid_x = trtp.cdiv(N, block_size)
# block dims
launch_params.block_x = compiled_kernel.metadata.num_warps * 32
# shared memory
launch_params.shared_mem = compiled_kernel.metadata.shared
extra_args = trtp.SymIntExprs(1)
extra_args[0] = trtp.SymInt32(N)
return compiled_kernel.metadata.name, compiled_kernel.asm["ptx"], launch_params, extra_args
"""
def decorator(aot_impl_func: Callable):
if plugin_id not in QDP_REGISTRY:
raise ValueError(
f"Plugin {plugin_id} is not registered. Did you register it with tensorrt.plugin.register API?"
)
plugin_def = QDP_REGISTRY[plugin_id]
aot_impl_attr_names = _validate_aot_impl(aot_impl_func, plugin_def)
plugin_def.aot_impl_func = aot_impl_func
plugin_def.aot_impl_attr_names = aot_impl_attr_names
return aot_impl_func
return decorator
# Decorator for `tensorrt.plugin.autotune`
@public_api()
def autotune(plugin_id: str) -> Callable:
"""
Wraps a function to define autotune logic for a plugin already registered through `trt.plugin.register`.
Autotuning is the process by which TensorRT executes the plugin over IO type/format combinations, and any custom tactics advertised as being supported by the plugin.
The (type, format, tactic) combination with the lowest latency is used to execute the plugin once the engine is built.
.. note:: An autotune function is optional. If not specified, TensorRT will assume the plugin only supports input types specified at network creation, output types specifeid through `trt.plugin.register`, and linear formats for all I/O.
This API is only intended to be used as a decorator. The decorated function is not required to have type hints for input arguments or return value; however, any type hints specified will be validated against the `trt.plugin.register` signature for consistency.
The schema for the function is as follows:
.. code-block:: text
(inp0: TensorDesc, inp1: TensorDesc, ..., attr0: SupportedAttrType, attr1: SupportedAttrType, outputs: Tuple[TensorDesc]) -> List[AutoTuneCombination]
* Input tensors are passed first, each described by a :class:`TensorDesc`.
* Plugin attributes are declared next. Not all attributes included in `trt.plugin.register` must be specified here -- they could be a subset.
* The function should return a list of :class:`AutoTuneCombination`\s.
Args:
plugin_id: The ID for the plugin in the form "{namespace}::{name}", which must match that used during `trt.plugin.register`
.. code-block:: python
:linenos:
:caption: An elementwise add plugin which supports both FP32 and FP16 linear I/O and wants to be tuned over 2 custom tactics.
import tensorrt.plugin as trtp
@trtp.register("my::add_plugin")
def add_plugin_desc(inp0: trtp.TensorDesc, block_size: int) -> Tuple[trtp.TensorDesc]:
return inp0.like()
@trtp.autotune("my::add_plugin")
def add_plugin_autotune(inp0: trtp.TensorDesc, block_size: int, outputs: Tuple[trtp.TensorDesc]) -> List[trtp.AutoTuneCombination]:
return [trtp.AutoTuneCombination("FP32|FP16, FP32|FP16", "LINEAR", [1, 2])]
.. code-block:: python
:linenos:
:caption: Same as above example but using index-by-index construction of an `AutoTuneCombination`
import tensorrt.plugin as trtp
@trtp.register("my::add_plugin")
def add_plugin_desc(inp0: trtp.TensorDesc, block_size: int) -> Tuple[trtp.TensorDesc]:
return inp0.like()
@trtp.autotune("my::add_plugin")
def add_plugin_autotune(inp0: trtp.TensorDesc, block_size: int, outputs: Tuple[trtp.TensorDesc]) -> List[trtp.AutoTuneCombination]:
c = trtp.AutoTuneCombination()
c.pos(0, "FP32|FP16", "LINEAR")
c.pos(1, "FP32|FP16") # index 1 is the output. Omitting format is the same as declaring it to be LINEAR.
c.tactics([1, 2])
return [c]
"""
def decorator(autotune_func: Callable):
if plugin_id not in QDP_REGISTRY:
raise ValueError(
f"Plugin {plugin_id} is not registered. Did you register it with tensorrt.plugin.register API?"
)
plugin_def = QDP_REGISTRY[plugin_id]
autotune_attr_names = _validate_autotune(autotune_func, plugin_def)
plugin_def.autotune_func = autotune_func
plugin_def.autotune_attr_names = autotune_attr_names
return autotune_func
return decorator
python/packaging/bindings_wheel/tensorrt/plugin/_plugin_class.py
+445
View File
@@ -0,0 +1,445 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import tensorrt as trt
from typing import Tuple, Union
import numpy as np
from ._utils import _numpy_to_plugin_field_type, _built_in_to_plugin_field_type
from ._tensor import TensorDesc, Tensor, Shape, ShapeExpr, ShapeExprs, SymIntExpr, SymExprs, SymInt32
from ._export import IS_AOT_ENABLED
if IS_AOT_ENABLED:
from ._tensor import KernelLaunchParams
from ._autotune import _TypeFormatCombination
from ._export import public_api
class _TemplatePluginBase(
trt.IPluginV3,
trt.IPluginV3QuickCore,
trt.IPluginV3QuickBuild,
):
def __init__(self, name, namespace, num_outputs):
trt.IPluginV3.__init__(self)
trt.IPluginV3QuickCore.__init__(self)
trt.IPluginV3QuickBuild.__init__(self)
self.plugin_version = "1"
self.input_types = []
self.aliased_map = {} # output index -> input index
self.plugin_namespace = namespace
self.plugin_name = name
self.num_outputs = num_outputs
self.autotune_combs = []
self.supported_combs = {}
self.curr_comb = None
def get_num_outputs(self):
return self.num_outputs
def get_output_data_types(self, input_types, ranks):
self.input_types = input_types
input_descs = [None] * len(input_types)
input_desc_map = {}
for i in range(len(input_types)):
input_descs[i] = TensorDesc()
input_descs[i].dtype = input_types[i]
input_descs[i].shape_expr = ShapeExprs(ranks[i], _is_dummy=True)
input_descs[i]._immutable = True
input_desc_map[id(input_descs[i])] = i
output_descs = self.register_function(*input_descs, **self.attrs)
if not isinstance(output_descs, Tuple):
output_descs = tuple([output_descs])
self.output_types = []
for i in range(len(output_descs)):
self.output_types.append(output_descs[i].dtype)
if output_descs[i].get_aliased() is not None:
self.aliased_map[i] = input_desc_map[id(output_descs[i].get_aliased())]
else:
self.aliased_map[i] = -1
return self.output_types
def get_fields_to_serialize(self):
fields = []
for key, value in self.attrs.items():
if key in self.impl_attr_names:
if isinstance(value, np.ndarray):
if np.dtype(value.dtype) == np.float16:
fields.append(trt.PluginField(key, value.tobytes(), trt.PluginFieldType.UNKNOWN))
else:
fields.append(
trt.PluginField(
key,
value,
_numpy_to_plugin_field_type[np.dtype(value.dtype)],
)
)
elif isinstance(value, str):
fields.append(trt.PluginField(key, value.encode(), trt.PluginFieldType.CHAR))
elif isinstance(value, bytes):
fields.append(trt.PluginField(key, value, trt.PluginFieldType.UNKNOWN))
else:
fields.append(
trt.PluginField(
key,
np.array([value]),
_built_in_to_plugin_field_type[type(value)],
)
)
return trt.PluginFieldCollection(fields)
def get_output_shapes(self, inputs, shape_inputs, exprBuilder):
assert len(shape_inputs) == 0 # Shape inputs are not yet supported for QDPs
SymIntExpr._exprBuilder = exprBuilder
self.input_descs = []
for i in range(len(inputs)):
desc = TensorDesc()
inp = inputs[i]
desc.dtype = self.input_types[i]
desc.shape_expr = ShapeExprs(len(inp))
for j in range(len(inp)):
desc.shape_expr[j] = ShapeExpr(inp[j])
desc._immutable = True
self.input_descs.append(desc)
self.output_descs = self.register_function(*self.input_descs, **self.attrs)
if not isinstance(self.output_descs, Tuple):
self.output_descs = tuple([self.output_descs])
for idx, desc in enumerate(self.output_descs):
if desc.is_size_tensor:
desc._set_index(idx)
output_exprs = []
for i in range(len(self.output_descs)):
exprs = trt.DimsExprs(len(self.output_descs[i].shape_expr))
for j in range(len(exprs)):
exprs[j] = self.output_descs[i].shape_expr[j]._expr
output_exprs.append(exprs)
SymIntExpr._exprBuilder = None
return output_exprs
def configure_plugin(self, inputs, outputs):
self.curr_comb = _TypeFormatCombination()
self.curr_comb.types = [inp.desc.type for inp in inputs] + [out.desc.type for out in outputs]
self.curr_comb.layouts = [inp.desc.format for inp in inputs] + [out.desc.format for out in outputs]
def get_supported_format_combinations(self, in_out, num_inputs):
if self.autotune_function is not None:
if len(self.autotune_attr_names) > 0:
val = [self.attrs[k] for k in self.autotune_attr_names]
else:
val = ()
for i, desc in enumerate(in_out):
if i < num_inputs:
self.input_descs[i]._immutable = False
self.input_descs[i].shape = Shape(desc)
self.input_descs[i].format = desc.desc.format
self.input_descs[i].scale = desc.desc.scale
self.input_descs[i]._immutable = True
else:
self.output_descs[i - num_inputs]._immutable = False
self.output_descs[i - num_inputs].shape = Shape(desc)
self.output_descs[i - num_inputs].format = desc.desc.format
self.output_descs[i - num_inputs].scale = desc.desc.scale
self.output_descs[i - num_inputs]._immutable = True
self.autotune_combs = self.autotune_function(*self.input_descs, *val, self.output_descs)
if len(self.autotune_combs) == 0:
default_comb = [None] * len(in_out)
comb = _TypeFormatCombination(len(in_out))
for j in range(len(in_out)):
default_comb[j] = trt.PluginTensorDesc()
default_comb[j].type = (
self.input_types[j] if j < num_inputs else self.output_descs[j - num_inputs].dtype
)
default_comb[j].format = trt.TensorFormat.LINEAR
comb.types[j] = default_comb[j].type
comb.layouts[j] = default_comb[j].format
self.supported_combs[comb] = set()
return default_comb
all_combs = []
for comb in self.autotune_combs:
all_combs.extend(comb._get_combinations())
ret_supported_combs = []
self.supported_combs = {}
for i, comb in enumerate(all_combs):
value = self.supported_combs.get(comb)
if value is not None:
value.update(set(comb.tactics) if comb.tactics is not None else set())
else:
self.supported_combs[comb] = set(comb.tactics) if comb.tactics is not None else set()
for j in range(len(in_out)):
curr_comb = trt.PluginTensorDesc()
curr_comb.type = comb.types[j]
curr_comb.format = comb.layouts[j]
ret_supported_combs.append(curr_comb)
return ret_supported_combs
def get_aliased_input(self, output_index: int):
return self.aliased_map[output_index]
def get_valid_tactics(self):
tactics = self.supported_combs.get(self.curr_comb)
assert tactics is not None
return list(tactics)
def set_tactic(self, tactic):
self._tactic = tactic
class _TemplateJITPlugin(_TemplatePluginBase, trt.IPluginV3QuickRuntime):
def __init__(self, name, namespace, num_outputs):
super().__init__(name, namespace, num_outputs)
trt.IPluginV3QuickRuntime.__init__(self)
self.expects_tactic = False
def init(
self,
register_function,
attrs,
impl_attr_names,
impl_function,
autotune_attr_names,
autotune_function,
expects_tactic,
):
self.register_function = register_function
self.impl_function = impl_function
self.attrs = attrs
self.impl_attr_names = impl_attr_names
self.autotune_attr_names = autotune_attr_names
self.autotune_function = autotune_function
self.expects_tactic = expects_tactic
def get_capability_interface(self, type):
return self
def enqueue(
self,
input_desc,
output_desc,
inputs,
outputs,
in_strides,
out_strides,
stream,
):
input_tensors = [None] * (len(inputs))
aliased_input_idxs = list(self.aliased_map.values())
for i in range(len(inputs)):
input_tensors[i] = Tensor()
input_tensors[i].dtype = input_desc[i].type
input_tensors[i].shape = Shape(input_desc[i])
input_tensors[i].format = input_desc[i].format
input_tensors[i].scale = input_desc[i].scale
input_tensors[i].data_ptr = inputs[i]
input_tensors[i]._stream = stream
input_tensors[i]._read_only = i not in aliased_input_idxs
input_tensors[i].strides = in_strides[i]
output_tensors = [None] * (len(outputs))
for i in range(len(outputs)):
output_tensors[i] = Tensor()
output_tensors[i].dtype = output_desc[i].type
output_tensors[i].shape = Shape(output_desc[i])
output_tensors[i].format = output_desc[i].format
output_tensors[i].scale = output_desc[i].scale
output_tensors[i].data_ptr = outputs[i]
output_tensors[i]._stream = stream
output_tensors[i]._read_only = False
output_tensors[i].strides = out_strides[i]
for i, j in self.aliased_map.items():
output_tensors[i]._aliased_to = input_tensors[j]
input_tensors[j]._aliased_to = output_tensors[i]
for t in input_tensors:
t._immutable = True
for t in output_tensors:
t._immutable = True
if len(self.impl_attr_names) > 0:
val = [self.attrs[k] for k in self.impl_attr_names]
else:
val = ()
if self.expects_tactic:
self.impl_function(*input_tensors, *val, output_tensors, stream, self._tactic)
else:
self.impl_function(*input_tensors, *val, output_tensors, stream=stream)
def clone(self):
cloned_plugin = _TemplateJITPlugin(self.plugin_name, self.plugin_namespace, self.num_outputs)
cloned_plugin.__dict__.update(self.__dict__)
return cloned_plugin
if IS_AOT_ENABLED:
class _TemplateAOTPlugin(
_TemplatePluginBase,
trt.IPluginV3QuickAOTBuild,
):
def __init__(self, name, namespace, num_outputs):
_TemplatePluginBase.__init__(self, name, namespace, num_outputs)
trt.IPluginV3QuickAOTBuild.__init__(self)
self.kernel_map = {}
def set_tactic(self, tactic):
self._tactic = tactic
def init(
self,
register_function,
attrs,
aot_impl_attr_names,
aot_impl_function,
autotune_attr_names,
autotune_function,
):
self.register_function = register_function
self.aot_impl_function = aot_impl_function
self.attrs = attrs
self.aot_impl_attr_names = aot_impl_attr_names
self.autotune_attr_names = autotune_attr_names
self.autotune_function = autotune_function
def get_capability_interface(self, type):
return self
def get_kernel(self, inputDesc, outputDesc):
io_types = []
io_formats = []
for i, desc in enumerate(inputDesc):
io_types.append(desc.type)
io_formats.append(desc.format)
for i, desc in enumerate(outputDesc):
io_types.append(desc.type)
io_formats.append(desc.format)
key = (tuple(io_types), tuple(io_formats), self._tactic)
assert key in self.kernel_map, "key {} not in kernel_map".format(key)
kernel_name, ptx = self.kernel_map[key]
return kernel_name, ptx.encode() if isinstance(ptx, str) else ptx
def get_launch_params(self, inDimsExprs, in_out, num_inputs, launchParams, symExprSetter, exprBuilder):
SymIntExpr._exprBuilder = exprBuilder
if len(self.attrs) > 0:
_, val = zip(*self.attrs.items())
else:
val = ()
io_types = []
io_formats = []
for i, desc in enumerate(in_out):
if i < num_inputs:
self.input_descs[i]._immutable = False
self.input_descs[i].shape = Shape(desc)
self.input_descs[i].dtype = desc.desc.type
self.input_descs[i].format = desc.desc.format
self.input_descs[i].scale = desc.desc.scale
io_types.append(desc.desc.type)
io_formats.append(desc.desc.format)
self.input_descs[i]._immutable = True
else:
self.output_descs[i - num_inputs]._immutable = False
self.output_descs[i - num_inputs].shape = Shape(desc)
self.output_descs[i - num_inputs].dtype = desc.desc.type
self.output_descs[i - num_inputs].format = desc.desc.format
self.output_descs[i - num_inputs].scale = desc.desc.scale
io_types.append(desc.desc.type)
io_formats.append(desc.desc.format)
self.output_descs[i - num_inputs]._immutable = True
kernel_name, ptx, launch_params, extra_args = self.aot_impl_function(
*self.input_descs, *val, self.output_descs, self._tactic
)
if not isinstance(kernel_name, str) and not isinstance(kernel_name, bytes):
raise TypeError(f"Kernel name must be a 'str' or 'bytes'. Got: {type(kernel_name)}.")
if not isinstance(ptx, str) and not isinstance(ptx, bytes):
raise TypeError(f"PTX/CUBIN must be a 'str' or 'bytes'. Got: {type(ptx)}.")
if not isinstance(launch_params, KernelLaunchParams):
raise TypeError(
f"Launch params must be a 'tensorrt.plugin.KernelLaunchParams'. Got: {type(launch_params)}."
)
if not isinstance(extra_args, SymExprs):
raise TypeError(f"Extra args must be a 'tensorrt.plugin.SymIntExprs'. Got: {type(extra_args)}.")
launchParams.grid_x = launch_params.grid_x()
launchParams.grid_y = launch_params.grid_y()
launchParams.grid_z = launch_params.grid_z()
launchParams.block_x = launch_params.block_x()
launchParams.block_y = launch_params.block_y()
launchParams.block_z = launch_params.block_z()
launchParams.shared_mem = launch_params.shared_mem()
self.kernel_map[(tuple(io_types), tuple(io_formats), self._tactic)] = (kernel_name, ptx)
symExprSetter.nbSymExprs = len(extra_args)
for i, arg in enumerate(extra_args):
if not isinstance(arg, SymInt32):
raise TypeError(f"Extra args must be a 'tensorrt.plugin.SymInt32'. Got: {type(arg)}.")
symExprSetter[i] = arg()
SymIntExpr._exprBuilder = None
def get_timing_cache_id(self):
return ""
def clone(self):
cloned_plugin = _TemplateAOTPlugin(self.plugin_name, self.plugin_namespace, self.num_outputs)
cloned_plugin.__dict__.update(self.__dict__)
return cloned_plugin
python/packaging/bindings_wheel/tensorrt/plugin/_tensor.py
+1175
View File
File diff suppressed because it is too large Load Diff
python/packaging/bindings_wheel/tensorrt/plugin/_top_level.py
+132
View File
@@ -0,0 +1,132 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from typing import Union, Tuple
import tensorrt as trt
from ._tensor import ShapeExpr, TensorDesc, ShapeExprs, SizeTensorDesc
from ._export import public_api
# Miscellaneous top-level functions accessible through `tensorrt.plugin`
# Performs `trt.DimensionOperation.CEIL_DIV`
@public_api()
def cdiv(first: Union[int, ShapeExpr], second: Union[int, ShapeExpr]) -> ShapeExpr:
"""
Computes symbolic ceiling division of `first` by `second`
Args:
first (Union[int, ShapeExpr]): Dividend
second (Union[int, ShapeExpr]): Divisor
Raises:
ValueError: If both arguments are `int`\s or if `second` evaluates to 0
Returns:
ShapeExpr: Symbolic expression for the ceiling division of `first` by `second`
"""
if isinstance(first, int):
if isinstance(second, int):
raise ValueError("Both arguments cannot be 'int's")
first = ShapeExpr(first)
return first._op(trt.DimensionOperation.CEIL_DIV, second)
# Performs `trt.DimensionOperation.MAX`
@public_api()
def max(first: Union[int, ShapeExpr], second: Union[int, ShapeExpr]) -> ShapeExpr:
"""
Computes the maximum of `first` and `second`
Args:
first (Union[int, ShapeExpr]): First operand
second (Union[int, ShapeExpr]): Second operand
Raises:
ValueError: If both arguments are `int`\s
Returns:
ShapeExpr: Symbolic expression for the maximum of `first` and `second`
"""
if isinstance(first, int):
if isinstance(second, int):
raise ValueError("Both arguments cannot be 'int's")
first = ShapeExpr(first)
return first._op(trt.DimensionOperation.MAX, second)
# Performs `trt.DimensionOperation.MIN`
@public_api()
def min(first: Union[int, ShapeExpr], second: Union[int, ShapeExpr]) -> ShapeExpr:
"""
Computes the minimum of `first` and `second`
Args:
first (Union[int, ShapeExpr]): First operand
second (Union[int, ShapeExpr]): Second operand
Raises:
ValueError: If both arguments are `int`\s
Returns:
ShapeExpr: Symbolic expression for the minimum of `first` and `second`
"""
if isinstance(first, int):
if isinstance(second, int):
raise ValueError("Both arguments cannot be 'int's")
first = ShapeExpr(first)
return first._op(trt.DimensionOperation.MIN, second)
# Declare a size tensor descriptor with the specified autotune shape expression `opt` and `upper-bound` shape expression
@public_api()
def size_tensor(opt: ShapeExpr, upper_bound: ShapeExpr) -> SizeTensorDesc:
"""
Constructs a size tensor with the specified autotune shape expression `opt` and `upper_bound`
Args:
opt (ShapeExpr): Symbolic expression for the extent of this size tensor to use in the autotune process of the engine build
upper_bound (ShapeExpr): Symbolic expression for the upper-bound of this size tensor
Returns:
SizeTensorDesc: A tensor descriptor for a size tensor with the specified autotune extent and upper-bound
"""
return SizeTensorDesc(opt, upper_bound)
# Create a TensorDesc using shape expressions and a dtype
@public_api()
def from_shape_expr(shape_expr: Union[Tuple[Union[ShapeExpr, int]], ShapeExprs], dtype: trt.DataType) -> TensorDesc:
"""
Constructs a tensor descriptor with the specified shape expression and data type
Args:
shape_expr (Union[Tuple[Union[ShapeExpr, int]], ShapeExprs]): Expressions or constants denoting the shape of the tensor
dtype (trt.DataType): Data type of the tensor
Returns:
TensorDesc: Tensor descriptor with the specified shape expression and data type
"""
if isinstance(shape_expr, tuple):
shape_expr_ = ShapeExprs.from_tuple(shape_expr)
else:
shape_expr_ = shape_expr
return TensorDesc(shape_expr_, dtype)
python/packaging/bindings_wheel/tensorrt/plugin/_utils.py
+77
View File
@@ -0,0 +1,77 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import tensorrt as trt
import numpy as np
import typing
_numpy_to_plugin_field_type = {
np.dtype('int32'): trt.PluginFieldType.INT32,
np.dtype('int16'): trt.PluginFieldType.INT16,
np.dtype('int8'): trt.PluginFieldType.INT8,
np.dtype('bool'): trt.PluginFieldType.INT8,
np.dtype('int64'): trt.PluginFieldType.INT64,
np.dtype('float32'): trt.PluginFieldType.FLOAT32,
np.dtype('float64'): trt.PluginFieldType.FLOAT64,
np.dtype('float16'): trt.PluginFieldType.FLOAT16
}
_built_in_to_plugin_field_type = {
int: trt.PluginFieldType.INT64,
float: trt.PluginFieldType.FLOAT64,
bool: trt.PluginFieldType.INT8,
# str is handled separately, so not needed here
}
def _str_to_data_type(dtype: str) -> trt.DataType:
if dtype == "FP32":
return trt.DataType.FLOAT
if dtype == "FP16":
return trt.DataType.HALF
try:
return getattr(trt.DataType, dtype)
except KeyError:
raise ValueError(f"Unknown data type string '{dtype}'") from None
def _join_with(lst, middle = False, delim = ", "):
if len(lst) == 0:
return ""
ret = ""
if middle:
ret += ", "
ret += delim.join(lst)
return ret
def _is_npt_ndarray(annotation):
return (typing.get_origin(annotation) == np.ndarray) or (hasattr(annotation, "__origin__") and annotation.__origin__ == np.ndarray)
def _is_numpy_array(annotation):
return (annotation == np.ndarray) or _is_npt_ndarray(annotation)
def _infer_numpy_type(annotation):
assert _is_npt_ndarray(annotation)
annot_args = typing.get_args(annotation) or annotation.__args__
if len(annot_args) >= 2:
np_type = typing.get_args(annot_args[1]) or annot_args[1].__args__
if len(np_type) >= 1:
return np_type[0]
raise AttributeError("Improper annotation for numpy array. Annotate numpy array attributes using 'numpy.typing.NDArray[dtype]', where 'dtype' is the expected numpy dtype of the array.")
python/packaging/bindings_wheel/tensorrt/plugin/_validate.py
+475
View File
@@ -0,0 +1,475 @@
#
# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import inspect
import numpy as np
import typing
import types
from ._utils import _is_numpy_array, _join_with, _infer_numpy_type, _is_npt_ndarray
from ._tensor import TensorDesc, Tensor, SymExprs
from ._export import IS_AOT_ENABLED
if IS_AOT_ENABLED:
from ._tensor import KernelLaunchParams
from ._autotune import AutoTuneCombination
SERIALIZABLE_BUILTIN_TYPES = (int, float, bytes, bool, str)
SERIALIZABLE_NP_DTYPES = (
np.int8,
np.int16,
np.int32,
np.int64,
np.float16,
np.float32,
np.float64,
bool,
np.bool_,
)
# Reserve some namespaces for future use/avoid confusion
RESERVED_NAMESPACES = {
"",
"trt",
"tensorrt",
"std",
}
DISALLOWED_ATTR_NAMES = {
"outputs",
"stream",
"tactic",
}
def _validate_name_and_namespace(ns: str, name: str):
if "." in ns:
raise ValueError(
f"Provided namespace {ns} cannot have any '.' in trt.plugin.register(\"{ns}::{name}\", ...)"
)
if "." in name:
raise ValueError(
f"Provided name {name} cannot have any '.' in trt.plugin.register(\"{ns}::{name}\", ...)"
)
if ns in RESERVED_NAMESPACES:
raise ValueError(
f"Provided namespace {ns} is a reserved namespace"
)
# Parse `tensorrt.plugin.register` schema
def _parse_register_inputs(register_func, lazy_register):
tensor_names = []
input_attrs = (
dict()
) # order is important here but for Python >= 3.7, dict respects key order
schema_chunks = []
# TensorDescs and attribute args cannot be interspersed, so remember when we saw the first attribute arg
saw_first_attr = False
# Map of (attr_name: str) -> (is_builtin_type?: bool, type annotation: str)
attrs_types = {}
sig = inspect.signature(register_func)
for idx, (name, param) in enumerate(sig.parameters.items()):
if param.kind not in (
inspect.Parameter.POSITIONAL_OR_KEYWORD,
inspect.Parameter.KEYWORD_ONLY,
):
raise ValueError(
f"Argument {name} is not a positional-or-keyword or keyword-only arg"
)
# Type annotations are manadatory for `tensorrt.plugin.register` args
if param.annotation == inspect.Parameter.empty:
raise ValueError(
f"Argument {name} does not have a type annotation. Please mark as TensorDesc or one of the serializable attribute types."
)
# Presently, we do not support default values for attributes
if param.default is not inspect.Parameter.empty:
raise ValueError(
f"Argument {name} has a default value. Default values are not supported yet."
)
if issubclass(param.annotation, TensorDesc):
if saw_first_attr:
raise ValueError(
f"TensorDescs args and attribute args cannot be interspersed. Received function with signature {sig}."
)
tensor_names.append(name)
schema_chunks.append(f"TensorDesc {name}")
# At this point, we don't validate attribute types since we only care about the types of serializable attributes
# However, we memorize name and type so that we may validate that the autotune function maintains consistency
else:
if idx == 0:
raise ValueError(
f"TensorDescs args should come first, followed by attributes. Received function with signature {sig}."
)
if name in DISALLOWED_ATTR_NAMES:
raise ValueError(
f"'{name}' is not allowed as a plugin attribute name."
)
if param.annotation not in SERIALIZABLE_BUILTIN_TYPES:
if _is_numpy_array(param.annotation):
if not lazy_register:
if param.annotation == np.ndarray:
raise ValueError(
"If using non-lazy registration, annotate numpy array attributes using 'numpy.typing.NDArray[dtype]', where 'dtype' is the expected numpy dtype of the array."
)
if _is_npt_ndarray(param.annotation):
np_dtype = _infer_numpy_type(param.annotation)
if np_dtype not in SERIALIZABLE_NP_DTYPES:
raise ValueError(
f"Attribute '{name}' is not a supported numpy array type. Supported numpy arrays type are {SERIALIZABLE_NP_DTYPES}."
)
attrs_types[name] = (False, np_dtype)
else:
raise ValueError(
f"Attribute '{name}' of type {param.annotation} is not a supported serializable type. Supported types are {SERIALIZABLE_BUILTIN_TYPES} or numpy arrays of type {SERIALIZABLE_NP_DTYPES}."
)
else:
attrs_types[name] = (True, param.annotation)
saw_first_attr = True
schema_chunks.append(f"{param.annotation} {name}")
input_attrs[name] = param.annotation
return (
tensor_names,
input_attrs,
f"({_join_with(schema_chunks)})",
attrs_types,
)
def _parse_register_return(register_func):
sig = inspect.signature(register_func)
ret_annotation = sig.return_annotation
if ret_annotation == inspect.Parameter.empty:
raise ValueError(
f"No return annotation found for register function. Received signature {sig}."
)
if typing.get_origin(ret_annotation) is not tuple:
if not inspect.isclass(ret_annotation) or not issubclass(
ret_annotation, TensorDesc
):
raise ValueError(
f"Return argument is of type {ret_annotation}. Return types can only be TensorDesc or Tuple[TensorDesc]."
)
num_outputs = 1
else:
args = typing.get_args(ret_annotation)
for arg in args:
if not issubclass(arg, TensorDesc):
raise ValueError(
f"Return argument is of type {ret_annotation}. Return types can only be TensorDesc or Tuple[TensorDesc]."
)
num_outputs = len(args)
return num_outputs
def _validate_impl(impl_func, plugin_def):
impl_attr_names = []
found_tactic = False
sig = inspect.signature(impl_func)
registered_attr_names = plugin_def.input_attrs.keys()
# input arg annotations are optional, but we will validate if provided
for name, param in sig.parameters.items():
# tactic arg is optional in impl function. If specified, remember so that we can pass it during enqueue.
if name == "tactic":
found_tactic = True
if param.annotation != inspect.Parameter.empty:
if name == "outputs":
if typing.get_origin(param.annotation) is not tuple:
raise ValueError(
f"'outputs' should be of type Tuple[Tensor]. Received {param.annotation}."
)
args = typing.get_args(param.annotation)
for arg in args:
if not issubclass(arg, Tensor):
raise ValueError(
f"Argument for receiving output Tensor, '{name}' contains a {param.annotation}. '{name}' should be a Tuple[Tensor]."
)
elif name == "stream":
if not issubclass(param.annotation, int):
raise ValueError("'stream' input argument should be an int")
elif name == "tactic":
if not issubclass(param.annotation, int):
raise ValueError("'tactic' input argument should be an int")
elif issubclass(param.annotation, Tensor):
if name not in plugin_def.input_tensor_names:
raise ValueError(
f"Unexpected tensor '{name}' specified in autotune function. Expected one of {plugin_def.input_tensor_names}."
)
else:
if name not in plugin_def.input_attrs:
raise ValueError(
f"Unexpected attribute '{name}' specified in impl function. Expected one of {list(registered_attr_names)}."
)
if param.annotation != plugin_def.input_attrs[name]:
raise ValueError(
f"Attribute '{name}' has a type annotation different from the one specified at registration. Expected '{plugin_def.input_attrs[name]}'."
)
impl_attr_names.append(name)
else:
if name in plugin_def.input_attrs:
impl_attr_names.append(name)
# Expected attribute schema should be constructed in the order they appeared in the register function
expected_attr_schema_chunks = [
n for n in registered_attr_names if n in impl_attr_names
]
expected_schema = (
"("
+ _join_with(plugin_def.input_tensor_names)
+ _join_with(expected_attr_schema_chunks, True)
+ ", outputs, stream"
)
if found_tactic:
expected_schema += ", tactic)"
else:
expected_schema += ")"
if f"({', '.join(sig.parameters.keys())})" != expected_schema:
raise ValueError(
f"Signature of the impl function '{sig}' does not match the expected input arg schema: {expected_schema}"
)
# Return annotation is optional, but we will validate if one is specified
if sig.return_annotation != inspect.Parameter.empty and sig.return_annotation is not None:
raise ValueError("Return annotation should be None.")
return impl_attr_names, found_tactic
def _validate_aot_impl(aot_impl_func, plugin_def):
aot_impl_attr_names = []
sig = inspect.signature(aot_impl_func)
registered_attr_names = plugin_def.input_attrs.keys()
# input arg annotations are optional, but we will validate if provided
for name, param in sig.parameters.items():
if param.annotation != inspect.Parameter.empty:
if name == "outputs":
if typing.get_origin(param.annotation) is not tuple:
raise ValueError(
f"'outputs' should be of type Tuple[TensorDesc]. Received {param.annotation}."
)
args = typing.get_args(param.annotation)
for arg in args:
if not issubclass(arg, TensorDesc):
raise ValueError(
f"Argument for receiving output TensorDesc, '{name}' contains a {param.annotation}. '{name}' should be a Tuple[TensorDesc]."
)
elif name == "tactic":
if not issubclass(param.annotation, int):
raise ValueError("'tactic' input argument should be an int")
elif issubclass(param.annotation, TensorDesc):
if name not in plugin_def.input_tensor_names:
raise ValueError(
f"Unexpected tensor '{name}' specified in autotune function. Expected one of {plugin_def.input_tensor_names}."
)
else:
if name not in plugin_def.input_attrs:
raise ValueError(
f"Unexpected attribute '{name}' specified in aot_impl function. Expected one of {list(registered_attr_names)}."
)
if param.annotation != plugin_def.input_attrs[name]:
raise ValueError(
f"Attribute '{name}' has a type annotation different from the one specified at registration. Expected '{plugin_def.input_attrs[name]}'."
)
aot_impl_attr_names.append(name)
else:
if name in plugin_def.input_attrs:
aot_impl_attr_names.append(name)
# Expected attribute schema should be constructed in the order they appeared in the register function
expected_attr_schema_chunks = [
n for n in registered_attr_names if n in aot_impl_attr_names
]
expected_schema = (
"("
+ _join_with(plugin_def.input_tensor_names)
+ _join_with(expected_attr_schema_chunks, True)
+ ", outputs, tactic)"
)
if f"({', '.join(sig.parameters.keys())})" != expected_schema:
raise ValueError(
f"Signature of the aot_impl function '{sig}' does not match the expected input arg schema: {expected_schema}"
)
ret_annotation = sig.return_annotation
if ret_annotation == inspect.Parameter.empty:
raise ValueError(
f"No return annotation found for aot_impl function. Received signature {sig}."
)
expected_return_schema = "tuple[str | bytes, str | bytes, tensorrt.plugin.KernelLaunchParams, tensorrt.plugin.SymIntExprs]"
# Return annotation is optional, but we will validate if one is specified
if ret_annotation != inspect.Parameter.empty:
if typing.get_origin(ret_annotation) is not tuple:
raise ValueError(
f"Return annotation is {ret_annotation}. Expected {expected_return_schema}."
)
else:
args = typing.get_args(ret_annotation)
if len(args) != 4:
raise ValueError(
f"Return annotation is {ret_annotation}. Expected {expected_return_schema}."
)
def validate_union_str_or_bytes(index):
def validate_str_or_bytes(arg_):
if (arg_ is not str) and (arg_ is not bytes):
raise ValueError(
f"Return annotation for argument at {index} is '{arg_}'. Expected 'str' or 'bytes'."
)
orig = typing.get_origin(args[index])
# orig is `typing.Union` when annotation uses typing module (e.g, Union[str, bytes])
# orig is `types.UnionType` when annotation is of the new (3.10+) native syntax (e.g, str | bytes)
if orig is typing.Union or orig is types.UnionType:
for a in typing.get_args(args[index]):
validate_str_or_bytes(a)
else:
# when annoted with `str` or `bytes`
validate_str_or_bytes(args[index])
# kernel name should be str or bytes encoding
validate_union_str_or_bytes(0)
# kernel PTX should be str or bytes encoding
validate_union_str_or_bytes(1)
if not issubclass(args[2], KernelLaunchParams):
raise ValueError(f"Argument at index 2 of return annotation is '{args[2]}'. Expected 'tensorrt.plugin.KernelLaunchParams'.")
if not issubclass(args[3], SymExprs):
raise ValueError(f"Argument at index 3 of return annotation is '{args[3]}'. Expected a descendent of tensorrt.plugin.SymExprs.")
return aot_impl_attr_names
def _validate_autotune(autotune_func, plugin_def):
sig = inspect.signature(autotune_func)
registered_attr_names = plugin_def.input_attrs.keys()
autotune_attr_names = []
# input arg annotations are optional, but we will validate if provided
for name, param in sig.parameters.items():
if param.annotation != inspect.Parameter.empty:
if name == "outputs":
if typing.get_origin(param.annotation) is not tuple:
raise ValueError(
f"'outputs' should be of type Tuple[TensorDesc]. Received {param.annotation}."
)
args = typing.get_args(param.annotation)
for arg in args:
if not issubclass(arg, TensorDesc):
raise ValueError(
f"Argument for receiving output TensorDescs, '{name}' contains a {param.annotation}. '{name}' should be a Tuple[TensorDesc]."
)
elif issubclass(param.annotation, TensorDesc):
if name not in plugin_def.input_tensor_names:
raise ValueError(
f"Unexpected tensor '{name}' specified in autotune function. Expected one of {plugin_def.input_tensor_names}."
)
else:
if name not in plugin_def.input_attrs:
raise ValueError(
f"Unexpected attribute '{name}' specified in autotune function. Expected one of {list(registered_attr_names)}."
)
if param.annotation != plugin_def.input_attrs[name]:
raise ValueError(
f"Attribute '{name}' has a type annotation different from the one specified at registration. Expected '{plugin_def.input_attrs[name]}'."
)
autotune_attr_names.append(name)
else:
if name in plugin_def.input_attrs:
autotune_attr_names.append(name)
# Expected attribute schema should be constructed in the order they appeared in the register function
expected_attr_schema_chunks = [
n for n in registered_attr_names if n in autotune_attr_names
]
expected_schema = (
"("
+ _join_with(plugin_def.input_tensor_names)
+ _join_with(expected_attr_schema_chunks, True)
+ ", outputs)"
)
if f"({', '.join(sig.parameters.keys())})" != expected_schema:
raise ValueError(
f"Specified autotune function signature {sig} is not consistent with the expected input arg schema {expected_schema}."
)
ret_annotation = sig.return_annotation
# Return annotation is optional, but we will validate if one is specified
if ret_annotation != inspect.Parameter.empty:
if typing.get_origin(ret_annotation) is not list:
if not inspect.isclass(ret_annotation) or not issubclass(
ret_annotation, AutoTuneCombination
):
raise ValueError(
f"Return argument is of type {ret_annotation}. Return types can only be AutoTuneCombination or List[AutoTuneCombination]."
)
else:
args = typing.get_args(ret_annotation)
for arg in args:
if not issubclass(arg, AutoTuneCombination):
raise ValueError(
f"Return argument is of type {ret_annotation}. Return types can only be AutoTuneCombination or List[AutoTuneCombination]."
)
return autotune_attr_names
python/packaging/frontend_sdist/CMakeLists.txt
+91
View File
@@ -0,0 +1,91 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Adding a custom target to the all target requires passing the "ALL" keyword to the initial call.
# We use this variable so we can conditionally add the keyword based on the build options.
if(${TRT_BUILD_PYTHON_STANDALONE_WHEELS})
set(ENABLE_ALL "ALL")
else()
set(ENABLE_ALL "")
endif()
add_custom_target(tensorrt_frontend_sdist ${ENABLE_ALL})
# \brief Creates a target named tensorrt_frontend_sdist_${moduleName} which will build the Frontend SDist for that module.
#
# \param moduleName The module name to create the frontend for. One of "tensorrt", "tensorrt_dispatch", or "tensorrt_lean".
function(buildFrontendWheel moduleName)
set(filesTarget trt_wheel_files_frontend_${moduleName})
set(wheelTemplateFiles
tensorrt/__init__.py
tensorrt/plugin/__init__.py
tensorrt/plugin/_lib.py
tensorrt/plugin/_autotune.py
tensorrt/plugin/_export.py
tensorrt/plugin/_plugin_class.py
tensorrt/plugin/_tensor.py
tensorrt/plugin/_top_level.py
tensorrt/plugin/_utils.py
tensorrt/plugin/_validate.py
LICENSE.txt
pyproject.toml
)
# Expands all template files for the frontend for the target module. The python version is unused (and set to 0).
# File paths starting with "tensorrt/" are expanded into "${moduleName}/".
processWheelTemplates(frontend ${moduleName} 0 ${wheelTemplateFiles})
# Creates a new custom target, and makes trt_standalone_wheel_files depend on the new target.
add_custom_target(${filesTarget} DEPENDS ${generatedWheelFiles})
# Define the output directory for the wheel
set(sdistOutDir ${TRT_WHEEL_OUTPUT_DIR}/frontend)
set(sdistOutputFile ${sdistOutDir}/${moduleName}_cu${CUDAToolkit_VERSION_MAJOR}-${TensorRT_PACKAGE_VERSION}.tar.gz)
# Add a custom command to build the sdist using PEP 517 compliant build frontend.
add_custom_command(
OUTPUT ${sdistOutputFile}
COMMAND ${CMAKE_COMMAND} -E env ${TRT_PACKAGING_ENV}
${Python3_EXECUTABLE} -m build --sdist --no-isolation --config-setting=--quiet --outdir ${sdistOutDir}
WORKING_DIRECTORY ${generatedFileOutDir}
DEPENDS ${filesTarget} trt_packaging_requirements_installed
VERBATIM
)
set(sdistTarget tensorrt_frontend_sdist_${moduleName})
# Add a custom target for the wheel
add_custom_target(
${sdistTarget}
${ENABLE_ALL}
DEPENDS ${sdistOutputFile}
)
add_dependencies(tensorrt_frontend_sdist ${sdistTarget})
# Standalone wheels are only published to the internal tarfile as they are released separately.
install(FILES
${sdistOutputFile}
DESTINATION python_standalone
COMPONENT internal
OPTIONAL
)
endfunction()
foreach(moduleName IN LISTS TRT_PYTHON_MODULE_NAMES)
buildFrontendWheel(${moduleName})
endforeach()
add_dependencies(tensorrt_python_wheels tensorrt_frontend_sdist)
python/packaging/frontend_sdist/LICENSE.txt
+180
View File
@@ -0,0 +1,180 @@
Abstract
This document is the Software License Agreement (SLA) for NVIDIA TensorRT. This document contains specific license terms and conditions for NVIDIA TensorRT. By accepting this agreement, you agree to comply with all the terms and conditions applicable to the specific product(s) included herein.
If you are receiving TensorRT under the NVIDIA Prerelease License Agreement (also known as NPLA) or under the NVIDIA Software License Agreement (previously known as the NVIDIA Tegra Software License Agreement), your use of TensorRT is governed by such applicable terms and conditions. All other uses of TensorRT are governed by the terms and conditions of the below license agreement.
NVIDIA SOFTWARE LICENSE AGREEMENT
Important: READ BEFORE DOWNLOADING, INSTALLING, COPYING OR USING THE LICENSED SOFTWARE
This Software License Agreement ("SLA”), made and entered into as of the time and date of click through action (“Effective Date”),is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs the use of the NVIDIA computer software and the documentation made available for use with such NVIDIA software. By downloading, installing, copying, or otherwise using the NVIDIA software and/or documentation, you agree to be bound by the terms of this SLA. If you do not agree to the terms of this SLA, do not download, install, copy or use the NVIDIA software or documentation. IF YOU ARE ENTERING INTO THIS SLAON BEHALF OF A COMPANY OR OTHER LEGAL ENTITY, YOU REPRESENT THAT YOU HAVE THE LEGAL AUTHORITY TO BIND THE ENTITY TO THIS SLA, IN WHICH CASE “YOU” WILL MEAN THE ENTITY YOU REPRESENT. IF YOU DON’T HAVE SUCH AUTHORITY, OR IF YOU DON’T ACCEPT ALL THE TERMS AND CONDITIONS OF THIS SLA, THEN NVIDIA DOES NOT AGREETO LICENSE THE LICENSED SOFTWARETO YOU, AND YOU MAY NOT DOWNLOAD, INSTALL, COPY OR USE IT.
Preface
This document is the Software License Agreement (SLA) for NVIDIA TensorRT. This document contains specific license terms and conditions for NVIDIA TensorRT. By accepting this agreement, you agree to comply with all the terms and conditions applicable to the specific product(s) included herein.
If you are receiving TensorRT under the NVIDIA Prerelease License Agreement (also known as NPLA) or under the NVIDIA Software License Agreement (previously known as the NVIDIA Tegra Software License Agreement), your use of TensorRT is governed by such applicable terms and conditions. All other uses of TensorRT are governed by the terms and conditions of the below license agreement.
NVIDIA SOFTWARE LICENSE AGREEMENT
Important: READ BEFORE DOWNLOADING, INSTALLING, COPYING OR USING THE LICENSED SOFTWARE
This Software License Agreement ("SLA”), made and entered into as of the time and date of click through action (“Effective Date”),is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs the use of the NVIDIA computer software and the documentation made available for use with such NVIDIA software. By downloading, installing, copying, or otherwise using the NVIDIA software and/or documentation, you agree to be bound by the terms of this SLA. If you do not agree to the terms of this SLA, do not download, install, copy or use the NVIDIA software or documentation. IF YOU ARE ENTERING INTO THIS SLAON BEHALF OF A COMPANY OR OTHER LEGAL ENTITY, YOU REPRESENT THAT YOU HAVE THE LEGAL AUTHORITY TO BIND THE ENTITY TO THIS SLA, IN WHICH CASE “YOU” WILL MEAN THE ENTITY YOU REPRESENT. IF YOU DON’T HAVE SUCH AUTHORITY, OR IF YOU DON’T ACCEPT ALL THE TERMS AND CONDITIONS OF THIS SLA, THEN NVIDIA DOES NOT AGREETO LICENSE THE LICENSED SOFTWARETO YOU, AND YOU MAY NOT DOWNLOAD, INSTALL, COPY OR USE IT.
1. LICENSE.
1.1. License Grant
Subject to the terms of the AGREEMENT, NVIDIA hereby grants you a non-exclusive, non-transferable license, without the right to sublicense (except as expressly set forth in a Supplement), during the applicable license term unless earlier terminated as provided below, to have Authorized Users install and use the Software, including modifications (if expressly permitted in a Supplement), in accordance with the Documentation. You are only licensed to activate and use Licensed Software for which you a have a valid license, even if during the download or installation you are presented with other product options. No Orders are binding on NVIDIA until accepted by NVIDIA. Your Orders are subject to the AGREEMENT.
SLA Supplements: Certain Licensed Software licensed under this SLA may be subject to additional terms and conditions that will be presented to you in a Supplement for acceptance prior to the delivery of such Licensed Software under this SLA and the applicable Supplement. Licensed Software will only be delivered to you upon your acceptance of all applicable terms.
1.2. Limited Purpose Licenses
If your license is provided for one of the purposes indicated below, then notwithstanding contrary terms in License Grant or in a Supplement, such licenses are for internal use and do not include any right or license to sub-license and distribute the Licensed Software or its output in any way in any public release, however limited, and/or in any manner that provides third parties with use of or access to the Licensed Software or its functionality or output, including (but not limited to) external alpha or beta testing or development phases. Further:
Evaluation License. You may use evaluation licenses solely for your internal evaluation of the Licensed Software for broader adoption within your Enterprise or in connection with a NVIDIA product purchase decision, and such licenses have an expiration date as indicated by NVIDIA in its sole discretion (or ninety days from the date of download if no other duration is indicated).
Educational/Academic License. You may use educational/academic licenses solely for educational purposes and all users must be enrolled or employed by an academic institution. If you do not meet NVIDIA’s academic program requirements for educational institutions, you have no rights under this license.
Test/Development License. You may use test/development licenses solely for your internal development, testing and/or debugging of your software applications or for interoperability testing with the Licensed Software, and such licenses have an expiration date as indicated by NVIDIA in its sole discretion (or one year from the date of download if no other duration is indicated). NVIDIA Confidential Information under the AGREEMENT includes output from Licensed Software developer tools identified as “Pro” versions, where the output reveals functionality or performance data pertinent to NVIDIA hardware or software products.
1.3. Pre-Release Licenses
With respect to alpha, beta, preview, and other pre-release Software and Documentation (“Pre-Release Licensed Software”) delivered to you under the AGREEMENT you acknowledge and agree that such Pre-Release Licensed Software (i) may not be fully functional, may contain errors or design flaws, and may have reduced or different security, privacy, accessibility, availability, and reliability standards relative to commercially provided NVIDIA software and documentation, and (ii) use of such Pre-Release Licensed Software may result in unexpected results, loss of data, project delays or other unpredictable damage or loss. THEREFORE, PRE-RELEASE LICENSED SOFTWARE IS NOT INTENDED FOR USE, AND SHOULD NOT BE USED, IN PRODUCTION OR BUSINESS-CRITICAL SYSTEMS. NVIDIA has no obligation to make available a commercial version of any Pre-Release Licensed Software and NVIDIA has the right to abandon development of Pre-Release Licensed Software at any time without liability.
1.4. Enterprise and Contractor Usage
You may allow your Enterprise employees and Contractors to access and use the Licensed Software pursuant to the terms of the AGREEMENT solely to perform work on your behalf, provided further that with respect to Contractors: (i) you obtain a written agreement from each Contractor which contains terms and obligations with respect to access to and use of Licensed Software no less protective of NVIDIA than those set forth in the AGREEMENT, and (ii) such Contractor’s access and use expressly excludes any sublicensing or distribution rights for the Licensed Software. You are responsible for the compliance with the terms and conditions of the AGREEMENT by your Enterprise and Contractors. Any act or omission that, if committed by you, would constitute a breach of the AGREEMENT shall be deemed to constitute a breach of the AGREEMENT if committed by your Enterprise or Contractors.
1.5. Services
Except as expressly indicated in an Order, NVIDIA is under no obligation to provide support for the Licensed Software or to provide any patches, maintenance, updates or upgrades under the AGREEMENT. Unless patches, maintenance, updates or upgrades are provided with their separate governing terms and conditions, they constitute Licensed Software licensed to you under the AGREEMENT.
2. LIMITATIONS.
2.1. License Restrictions
Except as expressly authorized in the AGREEMENT, you agree that you will not (nor authorize third parties to): (i) copy and use Software that was licensed to you for use in one or more NVIDIA hardware products in other unlicensed products (provided that copies solely for backup purposes are allowed); (ii) reverse engineer, decompile, disassemble (except to the extent applicable laws specifically require that such activities be permitted) or attempt to derive the source code, underlying ideas, algorithm or structure of Software provided to you in object code form; (iii) sell, transfer, assign, distribute, rent, loan, lease, sublicense or otherwise make available the Licensed Software or its functionality to third parties (a) as an application services provider or service bureau, (b) by operating hosted/virtual system environments, (c) by hosting, time sharing or providing any other type of services, or (d) otherwise by means of the internet; (iv) modify, translate or otherwise create any derivative works of any Licensed Software; (v) remove, alter, cover or obscure any proprietary notice that appears on or with the Licensed Software or any copies thereof; (vi) use the Licensed Software, or allow its use, transfer, transmission or export in violation of any applicable export control laws, rules or regulations; (vii) distribute, permit access to, or sublicense the Licensed Software as a stand-alone product; (viii) bypass, disable, circumvent or remove any form of copy protection, encryption, security or digital rights management or authentication mechanism used by NVIDIA in connection with the Licensed Software, or use the Licensed Software together with any authorization code, serial number, or other copy protection device not supplied by NVIDIA directly or through an authorized reseller; (ix) use the Licensed Software for the purpose of developing competing products or technologies or assisting a third party in such activities; (x) use the Licensed Software with any system or application where the use or failure of such system or application can reasonably be expected to threaten or result in personal injury, death, or catastrophic loss including, without limitation, use in connection with any nuclear, avionics, navigation, military, medical, life support or other life critical application (“Critical Applications”), unless the parties have entered into a Critical Applications agreement; (xi) distribute any modification or derivative work you make to the Licensed Software under or by reference to the same name as used by NVIDIA; or (xii) use the Licensed Software in any manner that would cause the Licensed Software to become subject to an Open Source License. Nothing in the AGREEMENT shall be construed to give you a right to use, or otherwise obtain access to, any source code from which the Software or any portion thereof is compiled or interpreted. You acknowledge that NVIDIA does not design, test, manufacture or certify the Licensed Software for use in the context of a Critical Application and NVIDIA shall not be liable to you or any third party, in whole or in part, for any claims or damages arising from such use. You agree to defend, indemnify and hold harmless NVIDIA and its Affiliates, and their respective employees, contractors, agents, officers and directors, from and against any and all claims, damages, obligations, losses, liabilities, costs or debt, fines, restitutions and expenses (including but not limited to attorney’s fees and costs incident to establishing the right of indemnification) arising out of or related to you and your Enterprise, and their respective employees, contractors, agents, distributors, resellers, end users, officers and directors use of Licensed Software outside of the scope of the AGREEMENT or any other breach of the terms of the AGREEMENT.
2.2. Third Party License Obligations
You acknowledge and agree that the Licensed Software may include or incorporate third party technology (collectively “Third Party Components”), which is provided for use in or with the Software and not otherwise used separately. If the Licensed Software includes or incorporates Third Party Components, then the third-party pass-through terms and conditions (“Third Party Terms”) for the particular Third Party Component will be bundled with the Software or otherwise made available online as indicated by NVIDIA and will be incorporated by reference into the AGREEMENT. In the event of any conflict between the terms in the AGREEMENT and the Third Party Terms, the Third Party Terms shall govern. Copyright to Third Party Components are held by the copyright holders indicated in the copyright notices indicated in the Third Party Terms.
Audio/Video Encoders and Decoders: You acknowledge and agree that it is your sole responsibility to obtain any additional third party licenses required to make, have made, use, have used, sell, import, and offer for sale your products or services that include or incorporate any Third Party Components and content relating to audio and/or video encoders and decoders from, including but not limited to, Microsoft, Thomson, Fraunhofer IIS, Sisvel S.p.A., MPEG-LA, and Coding Technologies as NVIDIA does not grant to you under the AGREEMENT any necessary patent or other rights with respect to audio and/or video encoders and decoders.
2.3. Limited Rights
Your rights in the Licensed Software are limited to those expressly granted under the AGREEMENT and no other licenses are granted whether by implication, estoppel or otherwise. NVIDIA reserves all rights, title and interest in and to the Licensed Software not expressly granted under the AGREEMENT.
3. CONFIDENTIALITY
Neither party will use the other party’s Confidential Information, except as necessary for the performance of the AGREEMENT, nor will either party disclose such Confidential Information to any third party, except to personnel of NVIDIA and its Affiliates, you, your Enterprise, your Enterprise Contractors, and each party’s legal and financial advisors that have a need to know such Confidential Information for the performance of the AGREEMENT, provided that each such personnel, employee and Contractor is subject to a written agreement that includes confidentiality obligations consistent with those set forth herein. Each party will use all reasonable efforts to maintain the confidentiality of all of the other party’s Confidential Information in its possession or control, but in no event less than the efforts that it ordinarily uses with respect to its own Confidential Information of similar nature and importance. The foregoing obligations will not restrict either party from disclosing the other party’s Confidential Information or the terms and conditions of the AGREEMENT as required under applicable securities regulations or pursuant to the order or requirement of a court, administrative agency, or other governmental body, provided that the party required to make such disclosure (i) gives reasonable notice to the other party to enable it to contest such order or requirement prior to its disclosure (whether through protective orders or otherwise), (ii) uses reasonable effort to obtain confidential treatment or similar protection to the fullest extent possible to avoid such public disclosure, and (iii) discloses only the minimum amount of information necessary to comply with such requirements.
4. OWNERSHIP
You are not obligated to disclose to NVIDIA any modifications that you, your Enterprise or your Contractors make to the Licensed Software as permitted under the AGREEMENT. As between the parties, all modifications are owned by NVIDIA and licensed to you under the AGREEMENT unless otherwise expressly provided in a Supplement. The Licensed Software and all modifications owned by NVIDIA, and the respective Intellectual Property Rights therein, are and will remain the sole and exclusive property of NVIDIA or its licensors, whether the Licensed Software is separate from or combined with any other products or materials. You shall not engage in any act or omission that would impair NVIDIA’s and/or its licensors’ Intellectual Property Rights in the Licensed Software or any other materials, information, processes or subject matter proprietary to NVIDIA. NVIDIA’s licensors are intended third party beneficiaries with the right to enforce provisions of the AGREEMENT with respect to their Confidential Information and/or Intellectual Property Rights.
5. FEEDBACK
You have no obligation to provide Feedback to NVIDIA. However, NVIDIA and/or its Affiliates may use and include any Feedback that you provide to improve the Licensed Software or other NVIDIA products, technologies or materials. Accordingly, if you provide Feedback, you agree that NVIDIA and/or its Affiliates, at their option, may, and may permit their licensees, to make, have made, use, have used, reproduce, license, distribute and otherwise commercialize the Feedback in the Licensed Software or in other NVIDIA products, technologies or materials without the payment of any royalties or fees to you. All Feedback becomes the sole property of NVIDIA and may be used in any manner NVIDIA sees fit, and you hereby assign to NVIDIA all of your right, title and interest in and to any Feedback. NVIDIA has no obligation to respond to Feedback or to incorporate Feedback into the Licensed Software.
6. NO WARRANTIES
THE LICENSED SOFTWARE AND ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES ARE PROVIDED BY NVIDIA “AS IS” AND “WITH ALL FAULTS,” AND NVIDIA EXPRESSLY DISCLAIMS ALL OTHER WARRANTIES OF ANY KIND OR NATURE, WHETHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES OF OPERABILITY, CONDITION, VALUE, ACCURACY OF DATA, OR QUALITY, AS WELL AS ANY WARRANTIES OF MERCHANTABILITY, SYSTEM INTEGRATION, WORKMANSHIP, SUITABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, OR THE ABSENCE OF ANY DEFECTS THEREIN, WHETHER LATENT OR PATENT. NO WARRANTY IS MADE BY NVIDIA ON THE BASIS OF TRADE USAGE, COURSE OF DEALING OR COURSE OF TRADE. NVIDIA DOES NOT WARRANT THAT THE LICENSED SOFTWARE OR ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES PROVIDED BY NVIDIA UNDER THE AGREEMENT WILL MEET YOUR REQUIREMENTS OR THAT THE OPERATION THEREOF WILL BE UNINTERRUPTED OR ERROR-FREE, OR THAT ALL ERRORS WILL BE CORRECTED. YOU ACKNOWLEDGE THAT NVIDIA’S OBLIGATIONS UNDER THE AGREEMENT ARE FOR THE BENEFIT OF YOU ONLY. Nothing in this warranty section affects any statutory rights of consumers or other recipients to the extent that they cannot be waived or limited by contract under applicable law.
7. LIMITATION OF LIABILITY
TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA OR ITS LICENSORS SHALL NOT BE LIABLE FOR ANY SPECIAL, INCIDENTAL, PUNITIVE OR CONSEQUENTIAL DAMAGES, OR ANY LOST PROFITS, LOSS OF USE, LOSS OF DATA OR LOSS OF GOODWILL, OR THE COSTS OF PROCURING SUBSTITUTE PRODUCTS, ARISING OUT OF OR IN CONNECTION WITH THE AGREEMENT OR THE USE OR PERFORMANCE OF THE LICENSED SOFTWARE AND ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES PROVIDED BY NVIDIA UNDER THE AGREEMENT, WHETHER SUCH LIABILITY ARISES FROM ANY CLAIM BASED UPON BREACH OF CONTRACT, BREACH OF WARRANTY, TORT (INCLUDING NEGLIGENCE), PRODUCT LIABILITY OR ANY OTHER CAUSE OF ACTION OR THEORY OF LIABILITY. IN NO EVENT WILL NVIDIA’S TOTAL CUMULATIVE LIABILITY UNDER OR ARISING OUT OF THE AGREEMENT EXCEED THE NET AMOUNTS RECEIVED BY NVIDIA FOR YOUR USE OF THE PARTICULAR LICENSED SOFTWARE DURING THE TWELVE (12) MONTHS BEFORE THE LIABILITY AROSE (or up to US$10.00 if you acquired the Licensed Software for no charge). THE NATURE OF THE LIABILITY, THE NUMBER OF CLAIMS OR SUITS OR THE NUMBER OF PARTIES WITHIN YOUR ENTERPRISE THAT ACCEPTED THE TERMS OF THE AGREEMENT SHALL NOT ENLARGE OR EXTEND THIS LIMIT. THE FOREGOING LIMITATIONS SHALL APPLY REGARDLESS OF WHETHER NVIDIA OR ITS LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES AND REGARDLESS OF WHETHER ANY REMEDY FAILS ITS ESSENTIAL PURPOSE. The disclaimers, exclusions and limitations of liability set forth in the AGREEMENT form an essential basis of the bargain between the parties, and, absent any such disclaimers, exclusions or limitations of liability, the provisions of the AGREEMENT, including, without limitation, the economic terms, would be substantially different.
8. TERM AND TERMINATION.
8.1. AGREEMENT, Licenses and Services
This SLA shall become effective upon the Effective Date, each Supplement upon their acceptance, and both this SLA and Supplements shall continue in effect until your last access or use of the Licensed Software and/or services hereunder, unless earlier terminated as provided in this “Term and Termination” section. Each Licensed Software license ends at the earlier of (a) the expiration of the applicable license term, or (b) termination of such license or the AGREEMENT. Each service ends at the earlier of (x) the expiration of the applicable service term, (y) termination of such service or the AGREEMENT, or (z) expiration or termination of the associated license and no credit or refund will be provided upon the expiration or termination of the associated license for any service fees paid.
8.2. Termination and Effect of Expiration or Termination
NVIDIA may terminate the AGREEMENT in whole or in part: (i) if you breach any term of the AGREEMENT and fail to cure such breach within thirty (30) days following notice thereof from NVIDIA (or immediately if you violate NVIDIA’s Intellectual Property Rights); (ii) if you become the subject of a voluntary or involuntary petition in bankruptcy or any proceeding relating to insolvency, receivership, liquidation or composition for the benefit of creditors, if that petition or proceeding is not dismissed with prejudice within sixty (60) days after filing, or if you cease to do business; or (iii) if you commence or participate in any legal proceeding against NVIDIA, with respect to the Licensed Software that is the subject of the proceeding during the pendency of such legal proceeding. If you or your authorized NVIDIA reseller fail to pay license fees or service fees when due then NVIDIA may, in its sole discretion, suspend or terminate your license grants, services and any other rights provided under the AGREEMENT for the affected Licensed Software, in addition to any other remedies NVIDIA may have at law or equity. Upon any expiration or termination of the AGREEMENT, a license or a service provided hereunder, (a) any amounts owed to NVIDIA become immediately due and payable, (b) you must promptly discontinue use of the affected Licensed Software and/or service, and (c) you must promptly destroy or return to NVIDIA all copies of the affected Licensed Software and all portions thereof in your possession or control, and each party will promptly destroy or return to the other all of the other party’s Confidential Information within its possession or control. Upon written request, you will certify in writing that you have complied with your obligations under this section. Upon expiration or termination of the AGREEMENT all provisions survive except for the license grant provisions.
9. CONSENT TO COLLECTION AND USE OF INFORMATION.
You hereby agree and acknowledge that the Software may access, collect non-personally identifiable information about your Enterprise computer systems in order to properly optimize such systems for use with the Software. To the extent that you use the Software, you hereby consent to all of the foregoing, and represent and warrant that you have the right to grant such consent. In addition, you agree that you are solely responsible for maintaining appropriate data backups and system restore points for your Enterprise systems, and that NVIDIA will have no responsibility for any damage or loss to such systems (including loss of data or access) arising from or relating to (a) any changes to the configuration, application settings, environment variables, registry, drivers, BIOS, or other attributes of the systems (or any part of such systems) initiated through the Software; or (b) installation of any Software or third party software patches initiated through the Software. In certain systems you may change your system update preferences by unchecking "Automatically check for updates" in the "Preferences" tab of the control panel for the Software.
In connection with the receipt of the Licensed Software or services you may receive access to links to third party websites and services and the availability of those links does not imply any endorsement by NVIDIA. NVIDIA encourages you to review the privacy statements on those sites and services that you choose to visit so that you can understand how they may collect, use and share personal information of individuals. NVIDIA is not responsible or liable for: (i) the availability or accuracy of such links; or (ii) the products, services or information available on or through such links; or (iii) the privacy statements or practices of sites and services controlled by other companies or organizations.
To the extent that you or members of your Enterprise provide to NVIDIA during registration or otherwise personal information, you acknowledge that such information will be collected, used and disclosed by NVIDIA in accordance with NVIDIA's privacy policy, available at URL http://www.nvidia.com/object/privacy_policy.html.
10. GENERAL.
This SLA, any Supplements incorporated hereto, and Orders constitute the entire agreement of the parties with respect to the subject matter hereto and supersede all prior negotiations, conversations, or discussions between the parties relating to the subject matter hereto, oral or written, and all past dealings or industry custom. Any additional and/or conflicting terms and conditions on purchase order(s) or any other documents issued by you are null, void, and invalid. Any amendment or waiver under the AGREEMENT must be in writing and signed by representatives of both parties.
The AGREEMENT and the rights and obligations thereunder may not be assigned by you, in whole or in part, including by merger, consolidation, dissolution, operation of law, or any other manner, without written consent of NVIDIA, and any purported assignment in violation of this provision shall be void and of no effect. NVIDIA may assign, delegate or transfer the AGREEMENT and its rights and obligations hereunder, and if to a non-Affiliate you will be notified.
Each party acknowledges and agrees that the other is an independent contractor in the performance of the AGREEMENT, and each party is solely responsible for all of its employees, agents, contractors, and labor costs and expenses arising in connection therewith. The parties are not partners, joint ventures or otherwise affiliated, and neither has any authority to make any statements, representations or commitments of any kind to bind the other party without prior written consent.
Neither party will be responsible for any failure or delay in its performance under the AGREEMENT (except for any payment obligations) to the extent due to causes beyond its reasonable control for so long as such force majeure event continues in effect.
The AGREEMENT will be governed by and construed under the laws of the State of Delaware and the United States without regard to the conflicts of law provisions thereof and without regard to the United Nations Convention on Contracts for the International Sale of Goods. The parties consent to the personal jurisdiction of the federal and state courts located in Santa Clara County, California. You acknowledge and agree that a breach of any of your promises or agreements contained in the AGREEMENT may result in irreparable and continuing injury to NVIDIA for which monetary damages may not be an adequate remedy and therefore NVIDIA is entitled to seek injunctive relief as well as such other and further relief as may be appropriate. If any court of competent jurisdiction determines that any provision of the AGREEMENT is illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect. Unless otherwise specified, remedies are cumulative.
The Licensed Software has been developed entirely at private expense and is “commercial items” consisting of “commercial computer software” and “commercial computer software documentation” provided with RESTRICTED RIGHTS. Use, duplication or disclosure by the U.S. Government or a U.S. Government subcontractor is subject to the restrictions set forth in the AGREEMENT pursuant to DFARS 227.7202-3(a) or as set forth in subparagraphs (c)(1) and (2) of the Commercial Computer Software - Restricted Rights clause at FAR 52.227-19, as applicable. Contractor/manufacturer is NVIDIA, 2701 San Tomas Expressway, Santa Clara, CA 95050.
You acknowledge that the Licensed Software described under the AGREEMENT is subject to export control under the U.S. Export Administration Regulations (EAR) and economic sanctions regulations administered by the U.S. Department of Treasury’s Office of Foreign Assets Control (OFAC). Therefore, you may not export, reexport or transfer in-country the Licensed Software without first obtaining any license or other approval that may be required by BIS and/or OFAC. You are responsible for any violation of the U.S. or other applicable export control or economic sanctions laws, regulations and requirements related to the Licensed Software. By accepting this SLA, you confirm that you are not a resident or citizen of any country currently embargoed by the U.S. and that you are not otherwise prohibited from receiving the Licensed Software.
Any notice delivered by NVIDIA to you under the AGREEMENT will be delivered via mail, email or fax. Please direct your legal notices or other correspondence to NVIDIA Corporation, 2701 San Tomas Expressway, Santa Clara, California 95050, United States of America, Attention: Legal Department.
11. GLOSSARY OF TERMS
Certain capitalized terms, if not otherwise defined elsewhere in this SLA, shall have the meanings set forth below:
“Affiliate”
“Affiliate” means any legal entity that Owns, is Owned by, or is commonly Owned with a party. “Own” means having more than 50% ownership or the right to direct the management of the entity.
“AGREEMENT”
“AGREEMENT” means this SLA and all associated Supplements entered by the parties referencing this SLA.
“Authorized Users”
“Authorized Users” means your Enterprise individual employees and any of your Enterprise’s Contractors, subject to the terms of the “Enterprise and Contractors Usage” section.
“Confidential Information”
“Confidential Information” means the Licensed Software (unless made publicly available by NVIDIA without confidentiality obligations), and any NVIDIA business, marketing, pricing, research and development, know-how, technical, scientific, financial status, proposed new products or other information disclosed by NVIDIA to you which, at the time of disclosure, is designated in writing as confidential or proprietary (or like written designation), or orally identified as confidential or proprietary or is otherwise reasonably identifiable by parties exercising reasonable business judgment, as confidential. Confidential Information does not and will not include information that: (i) is or becomes generally known to the public through no fault of or breach of the AGREEMENT by the receiving party; (ii) is rightfully known by the receiving party at the time of disclosure without an obligation of confidentiality; (iii) is independently developed by the receiving party without use of the disclosing party’s Confidential Information; or (iv) is rightfully obtained by the receiving party from a third party without restriction on use or disclosure.
“Contractor”
“Contractor” means an individual who works primarily for your Enterprise on a contractor basis from your secure network. means an individual who works primarily for your Enterprise on a contractor basis from your secure network.
“Documentation”
“Documentation” means the NVIDIA documentation made available for use with the Software, including (without limitation) user manuals, datasheets, operations instructions, installation guides, release notes and other materials provided to you under the AGREEMENT.
“Enterprise”
“Enterprise” means you or any company or legal entity for which you accepted the terms of this SLA, and their subsidiaries of which your company or legal entity owns more than fifty percent (50%) of the issued and outstanding equity.
“Feedback”
“Feedback” means any and all suggestions, feature requests, comments or other feedback regarding the Licensed Software, including possible enhancements or modifications thereto.
“Intellectual Property Rights”
“Intellectual Property Rights” means all patent, copyright, trademark, trade secret, trade dress, trade names, utility models, mask work, moral rights, rights of attribution or integrity service marks, master recording and music publishing rights, performance rights, author’s rights, database rights, registered design rights and any applications for the protection or registration of these rights, or other intellectual or industrial property rights or proprietary rights, howsoever arising and in whatever media, whether now known or hereafter devised, whether or not registered, (including all claims and causes of action for infringement, misappropriation or violation and all rights in any registrations and renewals), worldwide and whether existing now or in the future.
“Licensed Software”
“Licensed Software” means Software, Documentation and all modifications owned by NVIDIA.
“Open Source License”
“Open Source License” includes, without limitation, a software license that requires as a condition of use, modification, and/or distribution of such software that the Software be (i) disclosed or distributed in source code form; (ii) be licensed for the purpose of making derivative works; or (iii) be redistributable at no charge.
“Order”
“Order” means a purchase order issued by you, a signed purchase agreement with you, or other ordering document issued by you to NVIDIA or a NVIDIA authorized reseller (including any on-line acceptance process) that references and incorporates the AGREEMENT and is accepted by NVIDIA.
“Software”
“Software” means the NVIDIA software programs licensed to you under the AGREEMENT including, without limitation, libraries, sample code, utility programs and programming code.
“Supplement”
“Supplement” means the additional terms and conditions beyond those stated in this SLA that apply to certain Licensed Software licensed hereunder.
12. TensorRT SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT
TensorRT SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT
The terms set forth in this TensorRT Supplement (“Supplement”) govern your use of the NVIDIA GPU inference engine (the “TensorRT Licensed Software”) under the terms of your software license agreement (“SLA”) as modified by this Supplement. This Supplement is an exhibit to the SLA and is hereby incorporated as an integral part thereto. Capitalized terms used but not defined herein shall have the meaning assigned to them in the SLA. In the event of conflict between the terms in this Supplement and the terms in the SLA, this Supplement shall control.
12.1. TensorRT DISTRIBUTION
Subject to the terms of the SLA and this Supplement, NVIDIA hereby grants you a non-exclusive, nontransferable license during the applicable license term unless earlier terminated pursuant to the SLA, to distribute the libnvinfer and libnvinfer_plugin libraries when delivered to you as part of the TensorRT Licensed Software in source code form or binary form (but not when provided to you as part of a hardware product), subject to the following: such distribution is solely in binary form to your licensees (“Customers”) only as a component of your own software products having additional material functionality beyond the TensorRT Licensed Software (each, a “Licensee Application"). Subject to the terms and conditions of the SLA and this Supplement, you may further authorize Customers to redistribute the libnvinfer and libnvinfer_plugin libraries as incorporated into a Licensee Application, solely in binary form, provided, however, that you shall require in your agreements with your Customers that their distributions be on terms at least as restrictive as those applicable for your use of such TensorRT Licensed Software within a Licensee Application. The expiration or termination of your licenses to the above described TensorRT Licensed Software under the SLA and this Supplement will not affect rights previously granted by you to recipients that were in compliance with the SLA and this Supplement.
In addition to the rights above, for parties that are developing software intended solely for use on Jetson development kits or Jetson modules and running Linux for Tegra software the following shall apply: TensorRT Licensed Software licensed hereunder may be distributed in its entirety, as provided by NVIDIA and without separation of its components, for you and/or your licensees to create software development kits for use only on the Jetson platform and running Linux for Tegra software. You shall require in your agreements with your licensees that their distributions be on terms at least as restrictive as those applicable for your distribution of TensorRT Licensed Software as described in this Section 1.
In addition to the rights above, for parties that are developing software intended solely for use on Jetson development kits or Jetson modules and running Linux for Tegra software the following shall apply: TensorRT Licensed Software licensed hereunder may be distributed in its entirety, as provided by NVIDIA and without separation of its components, for you and/or your licensees to create software development kits for use only on the Jetson platform and running Linux for Tegra software. You shall require in your agreements with your licensees that their distributions be on terms at least as restrictive as those applicable for your distribution of TensorRT Licensed Software as described in this Section 1.
12.2. LICENSE DURATION
Each TensorRT Licensed Software is licensed to you for an initial duration of one year starting from the date of delivery or download. The licenses granted will automatically renew for successive one year periods, provided that NVIDIA reserves the right to terminate licenses upon ninety days (90) days written notice to you prior to the commencement of a renewal year in addition to the termination rights set forth in the SLA.
12.3. EXPIRATION OF TERMINATION OF THIS SUPPLEMENT
Your failure to comply with the terms of this Supplement is ground for termination for breach by NVIDIA under the SLA. This Supplement will automatically expire or terminate upon the expiration or termination of your rights to TensorRT Licensed Software under the SLA or this Supplement.
Notices
Notice
This document is provided for information purposes only and shall not be regarded as a warranty of a certain functionality, condition, or quality of a product. NVIDIA Corporation (“NVIDIA”) makes no representations or warranties, expressed or implied, as to the accuracy or completeness of the information contained in this document and assumes no responsibility for any errors contained herein. NVIDIA shall have no liability for the consequences or use of such information or for any infringement of patents or other rights of third parties that may result from its use. This document is not a commitment to develop, release, or deliver any Material (defined below), code, or functionality.
NVIDIA reserves the right to make corrections, modifications, enhancements, improvements, and any other changes to this document, at any time without notice.
Customer should obtain the latest relevant information before placing orders and should verify that such information is current and complete.
NVIDIA products are sold subject to the NVIDIA standard terms and conditions of sale supplied at the time of order acknowledgement, unless otherwise agreed in an individual sales agreement signed by authorized representatives of NVIDIA and customer (“Terms of Sale”). NVIDIA hereby expressly objects to applying any customer general terms and conditions with regards to the purchase of the NVIDIA product referenced in this document. No contractual obligations are formed either directly or indirectly by this document.
NVIDIA products are not designed, authorized, or warranted to be suitable for use in medical, military, aircraft, space, or life support equipment, nor in applications where failure or malfunction of the NVIDIA product can reasonably be expected to result in personal injury, death, or property or environmental damage. NVIDIA accepts no liability for inclusion and/or use of NVIDIA products in such equipment or applications and therefore such inclusion and/or use is at customer’s own risk.
NVIDIA makes no representation or warranty that products based on this document will be suitable for any specified use. Testing of all parameters of each product is not necessarily performed by NVIDIA. It is customer’s sole responsibility to evaluate and determine the applicability of any information contained in this document, ensure the product is suitable and fit for the application planned by customer, and perform the necessary testing for the application in order to avoid a default of the application or the product. Weaknesses in customer’s product designs may affect the quality and reliability of the NVIDIA product and may result in additional or different conditions and/or requirements beyond those contained in this document. NVIDIA accepts no liability related to any default, damage, costs, or problem which may be based on or attributable to: (i) the use of the NVIDIA product in any manner that is contrary to this document or (ii) customer product designs.
No license, either expressed or implied, is granted under any NVIDIA patent right, copyright, or other NVIDIA intellectual property right under this document. Information published by NVIDIA regarding third-party products or services does not constitute a license from NVIDIA to use such products or services or a warranty or endorsement thereof. Use of such information may require a license from a third party under the patents or other intellectual property rights of the third party, or a license from NVIDIA under the patents or other intellectual property rights of NVIDIA.
Reproduction of information in this document is permissible only if approved in advance by NVIDIA in writing, reproduced without alteration and in full compliance with all applicable export laws and regulations, and accompanied by all associated conditions, limitations, and notices.
THIS DOCUMENT AND ALL NVIDIA DESIGN SPECIFICATIONS, REFERENCE BOARDS, FILES, DRAWINGS, DIAGNOSTICS, LISTS, AND OTHER DOCUMENTS (TOGETHER AND SEPARATELY, “MATERIALS”) ARE BEING PROVIDED “AS IS.” NVIDIA MAKES NO WARRANTIES, EXPRESSED, IMPLIED, STATUTORY, OR OTHERWISE WITH RESPECT TO THE MATERIALS, AND EXPRESSLY DISCLAIMS ALL IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTABILITY, AND FITNESS FOR A PARTICULAR PURPOSE. TO THE EXTENT NOT PROHIBITED BY LAW, IN NO EVENT WILL NVIDIA BE LIABLE FOR ANY DAMAGES, INCLUDING WITHOUT LIMITATION ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, PUNITIVE, OR CONSEQUENTIAL DAMAGES, HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY, ARISING OUT OF ANY USE OF THIS DOCUMENT, EVEN IF NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. Notwithstanding any damages that customer might incur for any reason whatsoever, NVIDIA’s aggregate and cumulative liability towards customer for the products described herein shall be limited in accordance with the Terms of Sale for the product.
VESA DisplayPort
DisplayPort and DisplayPort Compliance Logo, DisplayPort Compliance Logo for Dual-mode Sources, and DisplayPort Compliance Logo for Active Cables are trademarks owned by the Video Electronics Standards Association in the United States and other countries.
HDMI
HDMI, the HDMI logo, and High-Definition Multimedia Interface are trademarks or registered trademarks of HDMI Licensing LLC.
ARM
ARM, AMBA and ARM Powered are registered trademarks of ARM Limited. Cortex, MPCore and Mali are trademarks of ARM Limited. All other brands or product names are the property of their respective holders. "ARM" is used to represent ARM Holdings plc; its operating company ARM Limited; and the regional subsidiaries ARM Inc.; ARM KK; ARM Korea Limited.; ARM Taiwan Limited; ARM France SAS; ARM Consulting (Shanghai) Co. Ltd.; ARM Germany GmbH; ARM Embedded Technologies Pvt. Ltd.; ARM Norway, AS and ARM Sweden AB.
OpenCL
OpenCL is a trademark of Apple Inc. used under license to the Khronos Group Inc.
Trademarks
NVIDIA, the NVIDIA logo, and cuBLAS, CUDA, CUDA Toolkit, cuDNN, DALI, DIGITS, DGX, DGX-1, DGX-2, DGX Station, DLProf, GPU, JetPack, Jetson, Kepler, Maxwell, NCCL, Nsight Compute, Nsight Systems, NVCaffe, NVIDIA Ampere GPU architecture, NVIDIA Deep Learning SDK, NVIDIA Developer Program, NVIDIA GPU Cloud, NVLink, NVSHMEM, PerfWorks, Pascal, SDK Manager, T4, Tegra, TensorRT, TensorRT Inference Server, Tesla, TF-TRT, Triton Inference Server, Turing, and Volta are trademarks and/or registered trademarks of NVIDIA Corporation in the United States and other countries. Other company and product names may be trademarks of the respective companies with which they are associated.
Copyright
© 2021 NVIDIA Corporation. All rights reserved.
python/packaging/frontend_sdist/tensorrt/__init__.py
+19
View File
@@ -0,0 +1,19 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from ##TENSORRT_MODULE##_bindings import *
from ##TENSORRT_MODULE##_bindings import __version__
python/packaging/frontend_sdist/tensorrt/plugin/__init__.py
+18
View File
@@ -0,0 +1,18 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from ##TENSORRT_MODULE##_bindings.plugin import *
python/packaging/frontend_sdist/tensorrt/plugin/_autotune.py
+19
View File
@@ -0,0 +1,19 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from ##TENSORRT_MODULE##_bindings.plugin._autotune import *
python/packaging/frontend_sdist/tensorrt/plugin/_export.py
+19
View File
@@ -0,0 +1,19 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from ##TENSORRT_MODULE##_bindings.plugin._export import *
python/packaging/frontend_sdist/tensorrt/plugin/_lib.py
+19
View File
@@ -0,0 +1,19 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from ##TENSORRT_MODULE##_bindings.plugin._lib import *
python/packaging/frontend_sdist/tensorrt/plugin/_plugin_class.py
+19
View File
@@ -0,0 +1,19 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from ##TENSORRT_MODULE##_bindings.plugin._plugin_class import *
python/packaging/frontend_sdist/tensorrt/plugin/_tensor.py
+19
View File
@@ -0,0 +1,19 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from ##TENSORRT_MODULE##_bindings.plugin._tensor import *
python/packaging/frontend_sdist/tensorrt/plugin/_top_level.py
+19
View File
@@ -0,0 +1,19 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from ##TENSORRT_MODULE##_bindings.plugin._top_level import *
python/packaging/frontend_sdist/tensorrt/plugin/_utils.py
+19
View File
@@ -0,0 +1,19 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from ##TENSORRT_MODULE##_bindings.plugin._utils import *
python/packaging/frontend_sdist/tensorrt/plugin/_validate.py
+19
View File
@@ -0,0 +1,19 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from ##TENSORRT_MODULE##_bindings.plugin._validate import *
python/packaging/libs_wheel/CMakeLists.txt
+183
View File
@@ -0,0 +1,183 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Adding a custom target to the all target requires passing the "ALL" keyword to the initial call.
# We use this variable so we can conditionally add the keyword based on the build options.
if(${TRT_BUILD_PYTHON_STANDALONE_WHEELS})
set(ENABLE_ALL "ALL")
else()
set(ENABLE_ALL "")
endif()
add_custom_target(tensorrt_libs_wheels ${ENABLE_ALL})
# \brief Creates a target named tensorrt_libs_wheel_${moduleName} which will build the Standalone Libs Wheel for that module.
#
# \details The wheel is created by expanding all template files (from this directory) into the per-module build directory.
# Then, the relevant TRT libraries (e.g. libnvinfer.so) are copied into the same directory as the generated files.
# Finally, the wheel is built by running setup.py with the appropriate arguments in the binary directory.
#
# \param moduleName The module name to create the libs wheel for. One of "tensorrt", "tensorrt_dispatch", or "tensorrt_lean".
function(buildLibsWheel moduleName)
set(filesTarget trt_wheel_files_libs_${moduleName})
set(wheelTemplateFiles
tensorrt_libs/__init__.py
LICENSE.txt
pyproject.toml
)
# Expands all template files for the libs for the target module. The python version is unused (and set to 0).
# File paths starting with "tensorrt_libs/" are expanded into "${moduleName}_libs/".
processWheelTemplates(libs ${moduleName} 0 ${wheelTemplateFiles})
copyTRTBuildBackend()
# Determine which targets are packed into the wheel.
if(${moduleName} STREQUAL "tensorrt_dispatch")
set(moduleLibraryTargets tensorrt_dispatch_runtime)
elseif(${moduleName} STREQUAL "tensorrt_lean")
set(moduleLibraryTargets tensorrt_lean_runtime)
if(${TRT_BUILD_PLUGINS})
list(APPEND moduleLibraryTargets tensorrt_vc_plugins)
endif()
elseif(${moduleName} STREQUAL "tensorrt" OR ${moduleName} STREQUAL "tensorrt_rtx")
set(moduleLibraryTargets tensorrt)
get_all_fatbin_archs(KLIB_ARCHS KLIB_ARCHS_CROSS)
if(NOT ${TRT_BUILD_WINML})
if(NOT ${TRT_BUILD_SPLIT_KLIB})
list(APPEND moduleLibraryTargets tensorrt_builder_resource)
else()
foreach(ARCH IN LISTS KLIB_ARCHS)
list(APPEND moduleLibraryTargets tensorrt_builder_resource_${ARCH})
endforeach()
endif()
endif()
if(${TRT_BUILD_PLUGINS})
list(APPEND moduleLibraryTargets tensorrt_plugins)
endif()
if(${TRT_BUILD_ONNX_PARSER})
list(APPEND moduleLibraryTargets nvonnxparser)
endif()
if(NOT ${TRT_BUILD_SKIP_WIN_BUILDER_RESOURCE})
foreach(ARCH IN LISTS KLIB_ARCHS_CROSS)
list(APPEND moduleLibraryTargets tensorrt_builder_resource_win_${ARCH})
endforeach()
endif()
else()
message(FATAL_ERROR "Unknown module name: ${moduleName}. Expected 'tensorrt', 'tensorrt_dispatch', or 'tensorrt_lean'.")
endif()
# copy_standalone_libs.py needs a space-separated string of the files to copy.
set(moduleLibraryPaths "")
foreach(tgt IN LISTS moduleLibraryTargets)
list(APPEND moduleLibraryPaths "$<TARGET_FILE:${tgt}>")
endforeach()
string(JOIN " " moduleLibraryFiles ${moduleLibraryPaths})
# CMake needs a list of files output by the custom command.
# But it needs it at configure time, so we can't use generator expressions here.
set(moduleLibraryOutputFiles)
foreach(tgt IN LISTS moduleLibraryTargets)
# Get the library filename (configure time)
get_target_property(lib_name ${tgt} OUTPUT_NAME)
if(NOT lib_name)
set(lib_name ${tgt})
endif()
# Add extension based on platform
if(MSVC)
set(lib_filename "${lib_name}${CMAKE_SHARED_LIBRARY_SUFFIX}")
else()
set(lib_filename "${CMAKE_SHARED_LIBRARY_PREFIX}${lib_name}${CMAKE_SHARED_LIBRARY_SUFFIX}")
endif()
# Add to output files list
list(APPEND moduleLibraryOutputFiles "${generatedFileOutDir}/${moduleName}_libs/${lib_filename}")
endforeach()
# ARM Linux requires that we specify the page size for the patchelf call.
if(${TRT_BUILD_PLATFORM} STREQUAL ${TRT_PLATFORM_AARCH64})
set(pageSizeArgs "--page-size=65536")
else()
set(pageSizeArgs "")
endif()
# Copies the TRT libraries (nvinfer.so, nvinfer_dispatch.so, etc) into the same directory as the generated files.
add_custom_command(
OUTPUT ${moduleLibraryOutputFiles}
COMMAND
${Python3_EXECUTABLE} ${TensorRT_SOURCE_DIR}/scripts/wheel/copy_standalone_libs.py
--update
--follow-symlinks
--mode=$<CONFIG>
${pageSizeArgs}
--build-dir=${CMAKE_LIBRARY_OUTPUT_DIRECTORY} # Since we give absolute paths, this is mostly unused, but we'll keep it for consistency.
--output=${generatedFileOutDir}/${moduleName}_libs/
--trim-version
${moduleLibraryPaths}
COMMENT "Copying TRT libraries for ${moduleName} to ${generatedFileOutDir}/${moduleName}_libs/"
DEPENDS ${moduleLibraryTargets}
VERBATIM
)
# Creates a new custom target, and makes trt_standalone_wheel_files depend on the new target.
add_custom_target(${filesTarget} DEPENDS ${generatedWheelFiles} ${moduleLibraryOutputFiles} ${copiedBuildBackend})
get_wheel_platform(TRUE wheelPlatform)
# Define the output directory for the wheel
set(wheelOutDir ${TRT_WHEEL_OUTPUT_DIR}/libs)
set(wheelOutputFile ${wheelOutDir}/${moduleName}_cu${CUDAToolkit_VERSION_MAJOR}_libs-${TensorRT_PACKAGE_VERSION}-py3-none-${wheelPlatform}.whl)
# Add a custom command to build the wheel using PEP 517 compliant build frontend.
# The custom tensorrt_build_backend handles wheel tag customization via config settings.
add_custom_command(
OUTPUT ${wheelOutputFile}
COMMAND ${CMAKE_COMMAND} -E env ${TRT_PACKAGING_ENV}
${Python3_EXECUTABLE} -m build --wheel --no-isolation
--config-setting=--quiet
--config-setting=python-tag=py3
--config-setting=plat-name=${wheelPlatform}
--outdir ${wheelOutDir}
WORKING_DIRECTORY ${generatedFileOutDir}
COMMENT "Building wheel for ${moduleName}"
DEPENDS ${filesTarget} trt_packaging_requirements_installed
VERBATIM
)
set(wheelTarget tensorrt_libs_wheel_${moduleName})
# Add a custom target for the wheel
add_custom_target(
${wheelTarget}
${ENABLE_ALL}
DEPENDS ${wheelOutputFile}
)
add_dependencies(tensorrt_libs_wheels ${wheelTarget})
# Standalone wheels are only published to the internal tarfile as they are released separately.
install(FILES
${wheelOutputFile}
DESTINATION python_standalone
COMPONENT internal
OPTIONAL
)
endfunction()
foreach(moduleName IN LISTS TRT_PYTHON_MODULE_NAMES)
buildLibsWheel(${moduleName})
endforeach()
add_dependencies(tensorrt_python_wheels tensorrt_libs_wheels)
python/packaging/libs_wheel/LICENSE.txt
+180
View File
@@ -0,0 +1,180 @@
Abstract
This document is the Software License Agreement (SLA) for NVIDIA TensorRT. This document contains specific license terms and conditions for NVIDIA TensorRT. By accepting this agreement, you agree to comply with all the terms and conditions applicable to the specific product(s) included herein.
If you are receiving TensorRT under the NVIDIA Prerelease License Agreement (also known as NPLA) or under the NVIDIA Software License Agreement (previously known as the NVIDIA Tegra Software License Agreement), your use of TensorRT is governed by such applicable terms and conditions. All other uses of TensorRT are governed by the terms and conditions of the below license agreement.
NVIDIA SOFTWARE LICENSE AGREEMENT
Important: READ BEFORE DOWNLOADING, INSTALLING, COPYING OR USING THE LICENSED SOFTWARE
This Software License Agreement ("SLA”), made and entered into as of the time and date of click through action (“Effective Date”),is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs the use of the NVIDIA computer software and the documentation made available for use with such NVIDIA software. By downloading, installing, copying, or otherwise using the NVIDIA software and/or documentation, you agree to be bound by the terms of this SLA. If you do not agree to the terms of this SLA, do not download, install, copy or use the NVIDIA software or documentation. IF YOU ARE ENTERING INTO THIS SLAON BEHALF OF A COMPANY OR OTHER LEGAL ENTITY, YOU REPRESENT THAT YOU HAVE THE LEGAL AUTHORITY TO BIND THE ENTITY TO THIS SLA, IN WHICH CASE “YOU” WILL MEAN THE ENTITY YOU REPRESENT. IF YOU DON’T HAVE SUCH AUTHORITY, OR IF YOU DON’T ACCEPT ALL THE TERMS AND CONDITIONS OF THIS SLA, THEN NVIDIA DOES NOT AGREETO LICENSE THE LICENSED SOFTWARETO YOU, AND YOU MAY NOT DOWNLOAD, INSTALL, COPY OR USE IT.
Preface
This document is the Software License Agreement (SLA) for NVIDIA TensorRT. This document contains specific license terms and conditions for NVIDIA TensorRT. By accepting this agreement, you agree to comply with all the terms and conditions applicable to the specific product(s) included herein.
If you are receiving TensorRT under the NVIDIA Prerelease License Agreement (also known as NPLA) or under the NVIDIA Software License Agreement (previously known as the NVIDIA Tegra Software License Agreement), your use of TensorRT is governed by such applicable terms and conditions. All other uses of TensorRT are governed by the terms and conditions of the below license agreement.
NVIDIA SOFTWARE LICENSE AGREEMENT
Important: READ BEFORE DOWNLOADING, INSTALLING, COPYING OR USING THE LICENSED SOFTWARE
This Software License Agreement ("SLA”), made and entered into as of the time and date of click through action (“Effective Date”),is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs the use of the NVIDIA computer software and the documentation made available for use with such NVIDIA software. By downloading, installing, copying, or otherwise using the NVIDIA software and/or documentation, you agree to be bound by the terms of this SLA. If you do not agree to the terms of this SLA, do not download, install, copy or use the NVIDIA software or documentation. IF YOU ARE ENTERING INTO THIS SLAON BEHALF OF A COMPANY OR OTHER LEGAL ENTITY, YOU REPRESENT THAT YOU HAVE THE LEGAL AUTHORITY TO BIND THE ENTITY TO THIS SLA, IN WHICH CASE “YOU” WILL MEAN THE ENTITY YOU REPRESENT. IF YOU DON’T HAVE SUCH AUTHORITY, OR IF YOU DON’T ACCEPT ALL THE TERMS AND CONDITIONS OF THIS SLA, THEN NVIDIA DOES NOT AGREETO LICENSE THE LICENSED SOFTWARETO YOU, AND YOU MAY NOT DOWNLOAD, INSTALL, COPY OR USE IT.
1. LICENSE.
1.1. License Grant
Subject to the terms of the AGREEMENT, NVIDIA hereby grants you a non-exclusive, non-transferable license, without the right to sublicense (except as expressly set forth in a Supplement), during the applicable license term unless earlier terminated as provided below, to have Authorized Users install and use the Software, including modifications (if expressly permitted in a Supplement), in accordance with the Documentation. You are only licensed to activate and use Licensed Software for which you a have a valid license, even if during the download or installation you are presented with other product options. No Orders are binding on NVIDIA until accepted by NVIDIA. Your Orders are subject to the AGREEMENT.
SLA Supplements: Certain Licensed Software licensed under this SLA may be subject to additional terms and conditions that will be presented to you in a Supplement for acceptance prior to the delivery of such Licensed Software under this SLA and the applicable Supplement. Licensed Software will only be delivered to you upon your acceptance of all applicable terms.
1.2. Limited Purpose Licenses
If your license is provided for one of the purposes indicated below, then notwithstanding contrary terms in License Grant or in a Supplement, such licenses are for internal use and do not include any right or license to sub-license and distribute the Licensed Software or its output in any way in any public release, however limited, and/or in any manner that provides third parties with use of or access to the Licensed Software or its functionality or output, including (but not limited to) external alpha or beta testing or development phases. Further:
Evaluation License. You may use evaluation licenses solely for your internal evaluation of the Licensed Software for broader adoption within your Enterprise or in connection with a NVIDIA product purchase decision, and such licenses have an expiration date as indicated by NVIDIA in its sole discretion (or ninety days from the date of download if no other duration is indicated).
Educational/Academic License. You may use educational/academic licenses solely for educational purposes and all users must be enrolled or employed by an academic institution. If you do not meet NVIDIA’s academic program requirements for educational institutions, you have no rights under this license.
Test/Development License. You may use test/development licenses solely for your internal development, testing and/or debugging of your software applications or for interoperability testing with the Licensed Software, and such licenses have an expiration date as indicated by NVIDIA in its sole discretion (or one year from the date of download if no other duration is indicated). NVIDIA Confidential Information under the AGREEMENT includes output from Licensed Software developer tools identified as “Pro” versions, where the output reveals functionality or performance data pertinent to NVIDIA hardware or software products.
1.3. Pre-Release Licenses
With respect to alpha, beta, preview, and other pre-release Software and Documentation (“Pre-Release Licensed Software”) delivered to you under the AGREEMENT you acknowledge and agree that such Pre-Release Licensed Software (i) may not be fully functional, may contain errors or design flaws, and may have reduced or different security, privacy, accessibility, availability, and reliability standards relative to commercially provided NVIDIA software and documentation, and (ii) use of such Pre-Release Licensed Software may result in unexpected results, loss of data, project delays or other unpredictable damage or loss. THEREFORE, PRE-RELEASE LICENSED SOFTWARE IS NOT INTENDED FOR USE, AND SHOULD NOT BE USED, IN PRODUCTION OR BUSINESS-CRITICAL SYSTEMS. NVIDIA has no obligation to make available a commercial version of any Pre-Release Licensed Software and NVIDIA has the right to abandon development of Pre-Release Licensed Software at any time without liability.
1.4. Enterprise and Contractor Usage
You may allow your Enterprise employees and Contractors to access and use the Licensed Software pursuant to the terms of the AGREEMENT solely to perform work on your behalf, provided further that with respect to Contractors: (i) you obtain a written agreement from each Contractor which contains terms and obligations with respect to access to and use of Licensed Software no less protective of NVIDIA than those set forth in the AGREEMENT, and (ii) such Contractor’s access and use expressly excludes any sublicensing or distribution rights for the Licensed Software. You are responsible for the compliance with the terms and conditions of the AGREEMENT by your Enterprise and Contractors. Any act or omission that, if committed by you, would constitute a breach of the AGREEMENT shall be deemed to constitute a breach of the AGREEMENT if committed by your Enterprise or Contractors.
1.5. Services
Except as expressly indicated in an Order, NVIDIA is under no obligation to provide support for the Licensed Software or to provide any patches, maintenance, updates or upgrades under the AGREEMENT. Unless patches, maintenance, updates or upgrades are provided with their separate governing terms and conditions, they constitute Licensed Software licensed to you under the AGREEMENT.
2. LIMITATIONS.
2.1. License Restrictions
Except as expressly authorized in the AGREEMENT, you agree that you will not (nor authorize third parties to): (i) copy and use Software that was licensed to you for use in one or more NVIDIA hardware products in other unlicensed products (provided that copies solely for backup purposes are allowed); (ii) reverse engineer, decompile, disassemble (except to the extent applicable laws specifically require that such activities be permitted) or attempt to derive the source code, underlying ideas, algorithm or structure of Software provided to you in object code form; (iii) sell, transfer, assign, distribute, rent, loan, lease, sublicense or otherwise make available the Licensed Software or its functionality to third parties (a) as an application services provider or service bureau, (b) by operating hosted/virtual system environments, (c) by hosting, time sharing or providing any other type of services, or (d) otherwise by means of the internet; (iv) modify, translate or otherwise create any derivative works of any Licensed Software; (v) remove, alter, cover or obscure any proprietary notice that appears on or with the Licensed Software or any copies thereof; (vi) use the Licensed Software, or allow its use, transfer, transmission or export in violation of any applicable export control laws, rules or regulations; (vii) distribute, permit access to, or sublicense the Licensed Software as a stand-alone product; (viii) bypass, disable, circumvent or remove any form of copy protection, encryption, security or digital rights management or authentication mechanism used by NVIDIA in connection with the Licensed Software, or use the Licensed Software together with any authorization code, serial number, or other copy protection device not supplied by NVIDIA directly or through an authorized reseller; (ix) use the Licensed Software for the purpose of developing competing products or technologies or assisting a third party in such activities; (x) use the Licensed Software with any system or application where the use or failure of such system or application can reasonably be expected to threaten or result in personal injury, death, or catastrophic loss including, without limitation, use in connection with any nuclear, avionics, navigation, military, medical, life support or other life critical application (“Critical Applications”), unless the parties have entered into a Critical Applications agreement; (xi) distribute any modification or derivative work you make to the Licensed Software under or by reference to the same name as used by NVIDIA; or (xii) use the Licensed Software in any manner that would cause the Licensed Software to become subject to an Open Source License. Nothing in the AGREEMENT shall be construed to give you a right to use, or otherwise obtain access to, any source code from which the Software or any portion thereof is compiled or interpreted. You acknowledge that NVIDIA does not design, test, manufacture or certify the Licensed Software for use in the context of a Critical Application and NVIDIA shall not be liable to you or any third party, in whole or in part, for any claims or damages arising from such use. You agree to defend, indemnify and hold harmless NVIDIA and its Affiliates, and their respective employees, contractors, agents, officers and directors, from and against any and all claims, damages, obligations, losses, liabilities, costs or debt, fines, restitutions and expenses (including but not limited to attorney’s fees and costs incident to establishing the right of indemnification) arising out of or related to you and your Enterprise, and their respective employees, contractors, agents, distributors, resellers, end users, officers and directors use of Licensed Software outside of the scope of the AGREEMENT or any other breach of the terms of the AGREEMENT.
2.2. Third Party License Obligations
You acknowledge and agree that the Licensed Software may include or incorporate third party technology (collectively “Third Party Components”), which is provided for use in or with the Software and not otherwise used separately. If the Licensed Software includes or incorporates Third Party Components, then the third-party pass-through terms and conditions (“Third Party Terms”) for the particular Third Party Component will be bundled with the Software or otherwise made available online as indicated by NVIDIA and will be incorporated by reference into the AGREEMENT. In the event of any conflict between the terms in the AGREEMENT and the Third Party Terms, the Third Party Terms shall govern. Copyright to Third Party Components are held by the copyright holders indicated in the copyright notices indicated in the Third Party Terms.
Audio/Video Encoders and Decoders: You acknowledge and agree that it is your sole responsibility to obtain any additional third party licenses required to make, have made, use, have used, sell, import, and offer for sale your products or services that include or incorporate any Third Party Components and content relating to audio and/or video encoders and decoders from, including but not limited to, Microsoft, Thomson, Fraunhofer IIS, Sisvel S.p.A., MPEG-LA, and Coding Technologies as NVIDIA does not grant to you under the AGREEMENT any necessary patent or other rights with respect to audio and/or video encoders and decoders.
2.3. Limited Rights
Your rights in the Licensed Software are limited to those expressly granted under the AGREEMENT and no other licenses are granted whether by implication, estoppel or otherwise. NVIDIA reserves all rights, title and interest in and to the Licensed Software not expressly granted under the AGREEMENT.
3. CONFIDENTIALITY
Neither party will use the other party’s Confidential Information, except as necessary for the performance of the AGREEMENT, nor will either party disclose such Confidential Information to any third party, except to personnel of NVIDIA and its Affiliates, you, your Enterprise, your Enterprise Contractors, and each party’s legal and financial advisors that have a need to know such Confidential Information for the performance of the AGREEMENT, provided that each such personnel, employee and Contractor is subject to a written agreement that includes confidentiality obligations consistent with those set forth herein. Each party will use all reasonable efforts to maintain the confidentiality of all of the other party’s Confidential Information in its possession or control, but in no event less than the efforts that it ordinarily uses with respect to its own Confidential Information of similar nature and importance. The foregoing obligations will not restrict either party from disclosing the other party’s Confidential Information or the terms and conditions of the AGREEMENT as required under applicable securities regulations or pursuant to the order or requirement of a court, administrative agency, or other governmental body, provided that the party required to make such disclosure (i) gives reasonable notice to the other party to enable it to contest such order or requirement prior to its disclosure (whether through protective orders or otherwise), (ii) uses reasonable effort to obtain confidential treatment or similar protection to the fullest extent possible to avoid such public disclosure, and (iii) discloses only the minimum amount of information necessary to comply with such requirements.
4. OWNERSHIP
You are not obligated to disclose to NVIDIA any modifications that you, your Enterprise or your Contractors make to the Licensed Software as permitted under the AGREEMENT. As between the parties, all modifications are owned by NVIDIA and licensed to you under the AGREEMENT unless otherwise expressly provided in a Supplement. The Licensed Software and all modifications owned by NVIDIA, and the respective Intellectual Property Rights therein, are and will remain the sole and exclusive property of NVIDIA or its licensors, whether the Licensed Software is separate from or combined with any other products or materials. You shall not engage in any act or omission that would impair NVIDIA’s and/or its licensors’ Intellectual Property Rights in the Licensed Software or any other materials, information, processes or subject matter proprietary to NVIDIA. NVIDIA’s licensors are intended third party beneficiaries with the right to enforce provisions of the AGREEMENT with respect to their Confidential Information and/or Intellectual Property Rights.
5. FEEDBACK
You have no obligation to provide Feedback to NVIDIA. However, NVIDIA and/or its Affiliates may use and include any Feedback that you provide to improve the Licensed Software or other NVIDIA products, technologies or materials. Accordingly, if you provide Feedback, you agree that NVIDIA and/or its Affiliates, at their option, may, and may permit their licensees, to make, have made, use, have used, reproduce, license, distribute and otherwise commercialize the Feedback in the Licensed Software or in other NVIDIA products, technologies or materials without the payment of any royalties or fees to you. All Feedback becomes the sole property of NVIDIA and may be used in any manner NVIDIA sees fit, and you hereby assign to NVIDIA all of your right, title and interest in and to any Feedback. NVIDIA has no obligation to respond to Feedback or to incorporate Feedback into the Licensed Software.
6. NO WARRANTIES
THE LICENSED SOFTWARE AND ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES ARE PROVIDED BY NVIDIA “AS IS” AND “WITH ALL FAULTS,” AND NVIDIA EXPRESSLY DISCLAIMS ALL OTHER WARRANTIES OF ANY KIND OR NATURE, WHETHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES OF OPERABILITY, CONDITION, VALUE, ACCURACY OF DATA, OR QUALITY, AS WELL AS ANY WARRANTIES OF MERCHANTABILITY, SYSTEM INTEGRATION, WORKMANSHIP, SUITABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, OR THE ABSENCE OF ANY DEFECTS THEREIN, WHETHER LATENT OR PATENT. NO WARRANTY IS MADE BY NVIDIA ON THE BASIS OF TRADE USAGE, COURSE OF DEALING OR COURSE OF TRADE. NVIDIA DOES NOT WARRANT THAT THE LICENSED SOFTWARE OR ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES PROVIDED BY NVIDIA UNDER THE AGREEMENT WILL MEET YOUR REQUIREMENTS OR THAT THE OPERATION THEREOF WILL BE UNINTERRUPTED OR ERROR-FREE, OR THAT ALL ERRORS WILL BE CORRECTED. YOU ACKNOWLEDGE THAT NVIDIA’S OBLIGATIONS UNDER THE AGREEMENT ARE FOR THE BENEFIT OF YOU ONLY. Nothing in this warranty section affects any statutory rights of consumers or other recipients to the extent that they cannot be waived or limited by contract under applicable law.
7. LIMITATION OF LIABILITY
TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA OR ITS LICENSORS SHALL NOT BE LIABLE FOR ANY SPECIAL, INCIDENTAL, PUNITIVE OR CONSEQUENTIAL DAMAGES, OR ANY LOST PROFITS, LOSS OF USE, LOSS OF DATA OR LOSS OF GOODWILL, OR THE COSTS OF PROCURING SUBSTITUTE PRODUCTS, ARISING OUT OF OR IN CONNECTION WITH THE AGREEMENT OR THE USE OR PERFORMANCE OF THE LICENSED SOFTWARE AND ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES PROVIDED BY NVIDIA UNDER THE AGREEMENT, WHETHER SUCH LIABILITY ARISES FROM ANY CLAIM BASED UPON BREACH OF CONTRACT, BREACH OF WARRANTY, TORT (INCLUDING NEGLIGENCE), PRODUCT LIABILITY OR ANY OTHER CAUSE OF ACTION OR THEORY OF LIABILITY. IN NO EVENT WILL NVIDIA’S TOTAL CUMULATIVE LIABILITY UNDER OR ARISING OUT OF THE AGREEMENT EXCEED THE NET AMOUNTS RECEIVED BY NVIDIA FOR YOUR USE OF THE PARTICULAR LICENSED SOFTWARE DURING THE TWELVE (12) MONTHS BEFORE THE LIABILITY AROSE (or up to US$10.00 if you acquired the Licensed Software for no charge). THE NATURE OF THE LIABILITY, THE NUMBER OF CLAIMS OR SUITS OR THE NUMBER OF PARTIES WITHIN YOUR ENTERPRISE THAT ACCEPTED THE TERMS OF THE AGREEMENT SHALL NOT ENLARGE OR EXTEND THIS LIMIT. THE FOREGOING LIMITATIONS SHALL APPLY REGARDLESS OF WHETHER NVIDIA OR ITS LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES AND REGARDLESS OF WHETHER ANY REMEDY FAILS ITS ESSENTIAL PURPOSE. The disclaimers, exclusions and limitations of liability set forth in the AGREEMENT form an essential basis of the bargain between the parties, and, absent any such disclaimers, exclusions or limitations of liability, the provisions of the AGREEMENT, including, without limitation, the economic terms, would be substantially different.
8. TERM AND TERMINATION.
8.1. AGREEMENT, Licenses and Services
This SLA shall become effective upon the Effective Date, each Supplement upon their acceptance, and both this SLA and Supplements shall continue in effect until your last access or use of the Licensed Software and/or services hereunder, unless earlier terminated as provided in this “Term and Termination” section. Each Licensed Software license ends at the earlier of (a) the expiration of the applicable license term, or (b) termination of such license or the AGREEMENT. Each service ends at the earlier of (x) the expiration of the applicable service term, (y) termination of such service or the AGREEMENT, or (z) expiration or termination of the associated license and no credit or refund will be provided upon the expiration or termination of the associated license for any service fees paid.
8.2. Termination and Effect of Expiration or Termination
NVIDIA may terminate the AGREEMENT in whole or in part: (i) if you breach any term of the AGREEMENT and fail to cure such breach within thirty (30) days following notice thereof from NVIDIA (or immediately if you violate NVIDIA’s Intellectual Property Rights); (ii) if you become the subject of a voluntary or involuntary petition in bankruptcy or any proceeding relating to insolvency, receivership, liquidation or composition for the benefit of creditors, if that petition or proceeding is not dismissed with prejudice within sixty (60) days after filing, or if you cease to do business; or (iii) if you commence or participate in any legal proceeding against NVIDIA, with respect to the Licensed Software that is the subject of the proceeding during the pendency of such legal proceeding. If you or your authorized NVIDIA reseller fail to pay license fees or service fees when due then NVIDIA may, in its sole discretion, suspend or terminate your license grants, services and any other rights provided under the AGREEMENT for the affected Licensed Software, in addition to any other remedies NVIDIA may have at law or equity. Upon any expiration or termination of the AGREEMENT, a license or a service provided hereunder, (a) any amounts owed to NVIDIA become immediately due and payable, (b) you must promptly discontinue use of the affected Licensed Software and/or service, and (c) you must promptly destroy or return to NVIDIA all copies of the affected Licensed Software and all portions thereof in your possession or control, and each party will promptly destroy or return to the other all of the other party’s Confidential Information within its possession or control. Upon written request, you will certify in writing that you have complied with your obligations under this section. Upon expiration or termination of the AGREEMENT all provisions survive except for the license grant provisions.
9. CONSENT TO COLLECTION AND USE OF INFORMATION.
You hereby agree and acknowledge that the Software may access, collect non-personally identifiable information about your Enterprise computer systems in order to properly optimize such systems for use with the Software. To the extent that you use the Software, you hereby consent to all of the foregoing, and represent and warrant that you have the right to grant such consent. In addition, you agree that you are solely responsible for maintaining appropriate data backups and system restore points for your Enterprise systems, and that NVIDIA will have no responsibility for any damage or loss to such systems (including loss of data or access) arising from or relating to (a) any changes to the configuration, application settings, environment variables, registry, drivers, BIOS, or other attributes of the systems (or any part of such systems) initiated through the Software; or (b) installation of any Software or third party software patches initiated through the Software. In certain systems you may change your system update preferences by unchecking "Automatically check for updates" in the "Preferences" tab of the control panel for the Software.
In connection with the receipt of the Licensed Software or services you may receive access to links to third party websites and services and the availability of those links does not imply any endorsement by NVIDIA. NVIDIA encourages you to review the privacy statements on those sites and services that you choose to visit so that you can understand how they may collect, use and share personal information of individuals. NVIDIA is not responsible or liable for: (i) the availability or accuracy of such links; or (ii) the products, services or information available on or through such links; or (iii) the privacy statements or practices of sites and services controlled by other companies or organizations.
To the extent that you or members of your Enterprise provide to NVIDIA during registration or otherwise personal information, you acknowledge that such information will be collected, used and disclosed by NVIDIA in accordance with NVIDIA's privacy policy, available at URL http://www.nvidia.com/object/privacy_policy.html.
10. GENERAL.
This SLA, any Supplements incorporated hereto, and Orders constitute the entire agreement of the parties with respect to the subject matter hereto and supersede all prior negotiations, conversations, or discussions between the parties relating to the subject matter hereto, oral or written, and all past dealings or industry custom. Any additional and/or conflicting terms and conditions on purchase order(s) or any other documents issued by you are null, void, and invalid. Any amendment or waiver under the AGREEMENT must be in writing and signed by representatives of both parties.
The AGREEMENT and the rights and obligations thereunder may not be assigned by you, in whole or in part, including by merger, consolidation, dissolution, operation of law, or any other manner, without written consent of NVIDIA, and any purported assignment in violation of this provision shall be void and of no effect. NVIDIA may assign, delegate or transfer the AGREEMENT and its rights and obligations hereunder, and if to a non-Affiliate you will be notified.
Each party acknowledges and agrees that the other is an independent contractor in the performance of the AGREEMENT, and each party is solely responsible for all of its employees, agents, contractors, and labor costs and expenses arising in connection therewith. The parties are not partners, joint ventures or otherwise affiliated, and neither has any authority to make any statements, representations or commitments of any kind to bind the other party without prior written consent.
Neither party will be responsible for any failure or delay in its performance under the AGREEMENT (except for any payment obligations) to the extent due to causes beyond its reasonable control for so long as such force majeure event continues in effect.
The AGREEMENT will be governed by and construed under the laws of the State of Delaware and the United States without regard to the conflicts of law provisions thereof and without regard to the United Nations Convention on Contracts for the International Sale of Goods. The parties consent to the personal jurisdiction of the federal and state courts located in Santa Clara County, California. You acknowledge and agree that a breach of any of your promises or agreements contained in the AGREEMENT may result in irreparable and continuing injury to NVIDIA for which monetary damages may not be an adequate remedy and therefore NVIDIA is entitled to seek injunctive relief as well as such other and further relief as may be appropriate. If any court of competent jurisdiction determines that any provision of the AGREEMENT is illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect. Unless otherwise specified, remedies are cumulative.
The Licensed Software has been developed entirely at private expense and is “commercial items” consisting of “commercial computer software” and “commercial computer software documentation” provided with RESTRICTED RIGHTS. Use, duplication or disclosure by the U.S. Government or a U.S. Government subcontractor is subject to the restrictions set forth in the AGREEMENT pursuant to DFARS 227.7202-3(a) or as set forth in subparagraphs (c)(1) and (2) of the Commercial Computer Software - Restricted Rights clause at FAR 52.227-19, as applicable. Contractor/manufacturer is NVIDIA, 2701 San Tomas Expressway, Santa Clara, CA 95050.
You acknowledge that the Licensed Software described under the AGREEMENT is subject to export control under the U.S. Export Administration Regulations (EAR) and economic sanctions regulations administered by the U.S. Department of Treasury’s Office of Foreign Assets Control (OFAC). Therefore, you may not export, reexport or transfer in-country the Licensed Software without first obtaining any license or other approval that may be required by BIS and/or OFAC. You are responsible for any violation of the U.S. or other applicable export control or economic sanctions laws, regulations and requirements related to the Licensed Software. By accepting this SLA, you confirm that you are not a resident or citizen of any country currently embargoed by the U.S. and that you are not otherwise prohibited from receiving the Licensed Software.
Any notice delivered by NVIDIA to you under the AGREEMENT will be delivered via mail, email or fax. Please direct your legal notices or other correspondence to NVIDIA Corporation, 2701 San Tomas Expressway, Santa Clara, California 95050, United States of America, Attention: Legal Department.
11. GLOSSARY OF TERMS
Certain capitalized terms, if not otherwise defined elsewhere in this SLA, shall have the meanings set forth below:
“Affiliate”
“Affiliate” means any legal entity that Owns, is Owned by, or is commonly Owned with a party. “Own” means having more than 50% ownership or the right to direct the management of the entity.
“AGREEMENT”
“AGREEMENT” means this SLA and all associated Supplements entered by the parties referencing this SLA.
“Authorized Users”
“Authorized Users” means your Enterprise individual employees and any of your Enterprise’s Contractors, subject to the terms of the “Enterprise and Contractors Usage” section.
“Confidential Information”
“Confidential Information” means the Licensed Software (unless made publicly available by NVIDIA without confidentiality obligations), and any NVIDIA business, marketing, pricing, research and development, know-how, technical, scientific, financial status, proposed new products or other information disclosed by NVIDIA to you which, at the time of disclosure, is designated in writing as confidential or proprietary (or like written designation), or orally identified as confidential or proprietary or is otherwise reasonably identifiable by parties exercising reasonable business judgment, as confidential. Confidential Information does not and will not include information that: (i) is or becomes generally known to the public through no fault of or breach of the AGREEMENT by the receiving party; (ii) is rightfully known by the receiving party at the time of disclosure without an obligation of confidentiality; (iii) is independently developed by the receiving party without use of the disclosing party’s Confidential Information; or (iv) is rightfully obtained by the receiving party from a third party without restriction on use or disclosure.
“Contractor”
“Contractor” means an individual who works primarily for your Enterprise on a contractor basis from your secure network. means an individual who works primarily for your Enterprise on a contractor basis from your secure network.
“Documentation”
“Documentation” means the NVIDIA documentation made available for use with the Software, including (without limitation) user manuals, datasheets, operations instructions, installation guides, release notes and other materials provided to you under the AGREEMENT.
“Enterprise”
“Enterprise” means you or any company or legal entity for which you accepted the terms of this SLA, and their subsidiaries of which your company or legal entity owns more than fifty percent (50%) of the issued and outstanding equity.
“Feedback”
“Feedback” means any and all suggestions, feature requests, comments or other feedback regarding the Licensed Software, including possible enhancements or modifications thereto.
“Intellectual Property Rights”
“Intellectual Property Rights” means all patent, copyright, trademark, trade secret, trade dress, trade names, utility models, mask work, moral rights, rights of attribution or integrity service marks, master recording and music publishing rights, performance rights, author’s rights, database rights, registered design rights and any applications for the protection or registration of these rights, or other intellectual or industrial property rights or proprietary rights, howsoever arising and in whatever media, whether now known or hereafter devised, whether or not registered, (including all claims and causes of action for infringement, misappropriation or violation and all rights in any registrations and renewals), worldwide and whether existing now or in the future.
“Licensed Software”
“Licensed Software” means Software, Documentation and all modifications owned by NVIDIA.
“Open Source License”
“Open Source License” includes, without limitation, a software license that requires as a condition of use, modification, and/or distribution of such software that the Software be (i) disclosed or distributed in source code form; (ii) be licensed for the purpose of making derivative works; or (iii) be redistributable at no charge.
“Order”
“Order” means a purchase order issued by you, a signed purchase agreement with you, or other ordering document issued by you to NVIDIA or a NVIDIA authorized reseller (including any on-line acceptance process) that references and incorporates the AGREEMENT and is accepted by NVIDIA.
“Software”
“Software” means the NVIDIA software programs licensed to you under the AGREEMENT including, without limitation, libraries, sample code, utility programs and programming code.
“Supplement”
“Supplement” means the additional terms and conditions beyond those stated in this SLA that apply to certain Licensed Software licensed hereunder.
12. TensorRT SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT
TensorRT SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT
The terms set forth in this TensorRT Supplement (“Supplement”) govern your use of the NVIDIA GPU inference engine (the “TensorRT Licensed Software”) under the terms of your software license agreement (“SLA”) as modified by this Supplement. This Supplement is an exhibit to the SLA and is hereby incorporated as an integral part thereto. Capitalized terms used but not defined herein shall have the meaning assigned to them in the SLA. In the event of conflict between the terms in this Supplement and the terms in the SLA, this Supplement shall control.
12.1. TensorRT DISTRIBUTION
Subject to the terms of the SLA and this Supplement, NVIDIA hereby grants you a non-exclusive, nontransferable license during the applicable license term unless earlier terminated pursuant to the SLA, to distribute the libnvinfer and libnvinfer_plugin libraries when delivered to you as part of the TensorRT Licensed Software in source code form or binary form (but not when provided to you as part of a hardware product), subject to the following: such distribution is solely in binary form to your licensees (“Customers”) only as a component of your own software products having additional material functionality beyond the TensorRT Licensed Software (each, a “Licensee Application"). Subject to the terms and conditions of the SLA and this Supplement, you may further authorize Customers to redistribute the libnvinfer and libnvinfer_plugin libraries as incorporated into a Licensee Application, solely in binary form, provided, however, that you shall require in your agreements with your Customers that their distributions be on terms at least as restrictive as those applicable for your use of such TensorRT Licensed Software within a Licensee Application. The expiration or termination of your licenses to the above described TensorRT Licensed Software under the SLA and this Supplement will not affect rights previously granted by you to recipients that were in compliance with the SLA and this Supplement.
In addition to the rights above, for parties that are developing software intended solely for use on Jetson development kits or Jetson modules and running Linux for Tegra software the following shall apply: TensorRT Licensed Software licensed hereunder may be distributed in its entirety, as provided by NVIDIA and without separation of its components, for you and/or your licensees to create software development kits for use only on the Jetson platform and running Linux for Tegra software. You shall require in your agreements with your licensees that their distributions be on terms at least as restrictive as those applicable for your distribution of TensorRT Licensed Software as described in this Section 1.
In addition to the rights above, for parties that are developing software intended solely for use on Jetson development kits or Jetson modules and running Linux for Tegra software the following shall apply: TensorRT Licensed Software licensed hereunder may be distributed in its entirety, as provided by NVIDIA and without separation of its components, for you and/or your licensees to create software development kits for use only on the Jetson platform and running Linux for Tegra software. You shall require in your agreements with your licensees that their distributions be on terms at least as restrictive as those applicable for your distribution of TensorRT Licensed Software as described in this Section 1.
12.2. LICENSE DURATION
Each TensorRT Licensed Software is licensed to you for an initial duration of one year starting from the date of delivery or download. The licenses granted will automatically renew for successive one year periods, provided that NVIDIA reserves the right to terminate licenses upon ninety days (90) days written notice to you prior to the commencement of a renewal year in addition to the termination rights set forth in the SLA.
12.3. EXPIRATION OF TERMINATION OF THIS SUPPLEMENT
Your failure to comply with the terms of this Supplement is ground for termination for breach by NVIDIA under the SLA. This Supplement will automatically expire or terminate upon the expiration or termination of your rights to TensorRT Licensed Software under the SLA or this Supplement.
Notices
Notice
This document is provided for information purposes only and shall not be regarded as a warranty of a certain functionality, condition, or quality of a product. NVIDIA Corporation (“NVIDIA”) makes no representations or warranties, expressed or implied, as to the accuracy or completeness of the information contained in this document and assumes no responsibility for any errors contained herein. NVIDIA shall have no liability for the consequences or use of such information or for any infringement of patents or other rights of third parties that may result from its use. This document is not a commitment to develop, release, or deliver any Material (defined below), code, or functionality.
NVIDIA reserves the right to make corrections, modifications, enhancements, improvements, and any other changes to this document, at any time without notice.
Customer should obtain the latest relevant information before placing orders and should verify that such information is current and complete.
NVIDIA products are sold subject to the NVIDIA standard terms and conditions of sale supplied at the time of order acknowledgement, unless otherwise agreed in an individual sales agreement signed by authorized representatives of NVIDIA and customer (“Terms of Sale”). NVIDIA hereby expressly objects to applying any customer general terms and conditions with regards to the purchase of the NVIDIA product referenced in this document. No contractual obligations are formed either directly or indirectly by this document.
NVIDIA products are not designed, authorized, or warranted to be suitable for use in medical, military, aircraft, space, or life support equipment, nor in applications where failure or malfunction of the NVIDIA product can reasonably be expected to result in personal injury, death, or property or environmental damage. NVIDIA accepts no liability for inclusion and/or use of NVIDIA products in such equipment or applications and therefore such inclusion and/or use is at customer’s own risk.
NVIDIA makes no representation or warranty that products based on this document will be suitable for any specified use. Testing of all parameters of each product is not necessarily performed by NVIDIA. It is customer’s sole responsibility to evaluate and determine the applicability of any information contained in this document, ensure the product is suitable and fit for the application planned by customer, and perform the necessary testing for the application in order to avoid a default of the application or the product. Weaknesses in customer’s product designs may affect the quality and reliability of the NVIDIA product and may result in additional or different conditions and/or requirements beyond those contained in this document. NVIDIA accepts no liability related to any default, damage, costs, or problem which may be based on or attributable to: (i) the use of the NVIDIA product in any manner that is contrary to this document or (ii) customer product designs.
No license, either expressed or implied, is granted under any NVIDIA patent right, copyright, or other NVIDIA intellectual property right under this document. Information published by NVIDIA regarding third-party products or services does not constitute a license from NVIDIA to use such products or services or a warranty or endorsement thereof. Use of such information may require a license from a third party under the patents or other intellectual property rights of the third party, or a license from NVIDIA under the patents or other intellectual property rights of NVIDIA.
Reproduction of information in this document is permissible only if approved in advance by NVIDIA in writing, reproduced without alteration and in full compliance with all applicable export laws and regulations, and accompanied by all associated conditions, limitations, and notices.
THIS DOCUMENT AND ALL NVIDIA DESIGN SPECIFICATIONS, REFERENCE BOARDS, FILES, DRAWINGS, DIAGNOSTICS, LISTS, AND OTHER DOCUMENTS (TOGETHER AND SEPARATELY, “MATERIALS”) ARE BEING PROVIDED “AS IS.” NVIDIA MAKES NO WARRANTIES, EXPRESSED, IMPLIED, STATUTORY, OR OTHERWISE WITH RESPECT TO THE MATERIALS, AND EXPRESSLY DISCLAIMS ALL IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTABILITY, AND FITNESS FOR A PARTICULAR PURPOSE. TO THE EXTENT NOT PROHIBITED BY LAW, IN NO EVENT WILL NVIDIA BE LIABLE FOR ANY DAMAGES, INCLUDING WITHOUT LIMITATION ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, PUNITIVE, OR CONSEQUENTIAL DAMAGES, HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY, ARISING OUT OF ANY USE OF THIS DOCUMENT, EVEN IF NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. Notwithstanding any damages that customer might incur for any reason whatsoever, NVIDIA’s aggregate and cumulative liability towards customer for the products described herein shall be limited in accordance with the Terms of Sale for the product.
VESA DisplayPort
DisplayPort and DisplayPort Compliance Logo, DisplayPort Compliance Logo for Dual-mode Sources, and DisplayPort Compliance Logo for Active Cables are trademarks owned by the Video Electronics Standards Association in the United States and other countries.
HDMI
HDMI, the HDMI logo, and High-Definition Multimedia Interface are trademarks or registered trademarks of HDMI Licensing LLC.
ARM
ARM, AMBA and ARM Powered are registered trademarks of ARM Limited. Cortex, MPCore and Mali are trademarks of ARM Limited. All other brands or product names are the property of their respective holders. "ARM" is used to represent ARM Holdings plc; its operating company ARM Limited; and the regional subsidiaries ARM Inc.; ARM KK; ARM Korea Limited.; ARM Taiwan Limited; ARM France SAS; ARM Consulting (Shanghai) Co. Ltd.; ARM Germany GmbH; ARM Embedded Technologies Pvt. Ltd.; ARM Norway, AS and ARM Sweden AB.
OpenCL
OpenCL is a trademark of Apple Inc. used under license to the Khronos Group Inc.
Trademarks
NVIDIA, the NVIDIA logo, and cuBLAS, CUDA, CUDA Toolkit, cuDNN, DALI, DIGITS, DGX, DGX-1, DGX-2, DGX Station, DLProf, GPU, JetPack, Jetson, Kepler, Maxwell, NCCL, Nsight Compute, Nsight Systems, NVCaffe, NVIDIA Ampere GPU architecture, NVIDIA Deep Learning SDK, NVIDIA Developer Program, NVIDIA GPU Cloud, NVLink, NVSHMEM, PerfWorks, Pascal, SDK Manager, T4, Tegra, TensorRT, TensorRT Inference Server, Tesla, TF-TRT, Triton Inference Server, Turing, and Volta are trademarks and/or registered trademarks of NVIDIA Corporation in the United States and other countries. Other company and product names may be trademarks of the respective companies with which they are associated.
Copyright
© 2021 NVIDIA Corporation. All rights reserved.
python/packaging/libs_wheel/tensorrt_libs/__init__.py
+40
View File
@@ -0,0 +1,40 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import ctypes
import glob
import os
CURDIR = os.path.realpath(os.path.dirname(__file__))
def try_load(library):
try:
ctypes.CDLL(library)
except OSError:
pass
def try_load_libs_from_dir(path):
for lib in glob.iglob(os.path.join(path, "*.so*")):
try_load(lib)
for lib in glob.iglob(os.path.join(path, "*.dll*")):
try_load(lib)
# Try loading all packaged libraries. This is a nop if there are no libraries packaged.
try_load_libs_from_dir(CURDIR)
python/packaging/metapackage/CMakeLists.txt
+81
View File
@@ -0,0 +1,81 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Adding a custom target to the all target requires passing the "ALL" keyword to the initial call.
# We use this variable so we can conditionally add the keyword based on the build options.
if(${TRT_BUILD_PYTHON_STANDALONE_WHEELS})
set(ENABLE_ALL "ALL")
else()
set(ENABLE_ALL "")
endif()
add_custom_target(tensorrt_metapackage_sdist ${ENABLE_ALL})
# \brief Creates a target named tensorrt_metapackage_sdist_${moduleName} which will build the metapackage SDist for that module.
#
# \param moduleName The module name to create the metapackage for. One of "tensorrt", "tensorrt_dispatch", or "tensorrt_lean".
function(buildMetapackageWheel moduleName)
set(filesTarget trt_wheel_files_metapackage_${moduleName})
set(wheelTemplateFiles
LICENSE.txt
pyproject.toml
)
# Expands all template files for the metapackage for the target module. The python version is unused (and set to 0).
# File paths starting with "tensorrt/" are expanded into "${moduleName}/".
processWheelTemplates(metapackage ${moduleName} 0 ${wheelTemplateFiles})
# Creates a new custom target, and makes trt_standalone_wheel_files depend on the new target.
add_custom_target(${filesTarget} DEPENDS ${generatedWheelFiles})
# Define the output directory for the wheel
set(sdistOutDir ${TRT_WHEEL_OUTPUT_DIR}/metapackage)
set(sdistOutputFile ${sdistOutDir}/${moduleName}-${TensorRT_PACKAGE_VERSION}.tar.gz)
# Add a custom command to build the sdist using PEP 517 compliant build frontend.
add_custom_command(
OUTPUT ${sdistOutputFile}
COMMAND ${CMAKE_COMMAND} -E env ${TRT_PACKAGING_ENV}
${Python3_EXECUTABLE} -m build --sdist --no-isolation --config-setting=--quiet --outdir ${sdistOutDir}
WORKING_DIRECTORY ${generatedFileOutDir}
DEPENDS ${filesTarget} trt_packaging_requirements_installed
VERBATIM
)
set(sdistTarget tensorrt_metapackage_sdist_${moduleName})
# Add a custom target for the wheel
add_custom_target(
${sdistTarget}
${ENABLE_ALL}
DEPENDS ${sdistOutputFile}
)
add_dependencies(tensorrt_metapackage_sdist ${sdistTarget})
# Standalone wheels are only published to the internal tarfile as they are released separately.
install(FILES
${sdistOutputFile}
DESTINATION python_standalone
COMPONENT internal
OPTIONAL
)
endfunction()
foreach(moduleName IN LISTS TRT_PYTHON_MODULE_NAMES)
buildMetapackageWheel(${moduleName})
endforeach()
add_dependencies(tensorrt_python_wheels tensorrt_metapackage_sdist)
python/packaging/metapackage/LICENSE.txt
+180
View File
@@ -0,0 +1,180 @@
Abstract
This document is the Software License Agreement (SLA) for NVIDIA TensorRT. This document contains specific license terms and conditions for NVIDIA TensorRT. By accepting this agreement, you agree to comply with all the terms and conditions applicable to the specific product(s) included herein.
If you are receiving TensorRT under the NVIDIA Prerelease License Agreement (also known as NPLA) or under the NVIDIA Software License Agreement (previously known as the NVIDIA Tegra Software License Agreement), your use of TensorRT is governed by such applicable terms and conditions. All other uses of TensorRT are governed by the terms and conditions of the below license agreement.
NVIDIA SOFTWARE LICENSE AGREEMENT
Important: READ BEFORE DOWNLOADING, INSTALLING, COPYING OR USING THE LICENSED SOFTWARE
This Software License Agreement ("SLA”), made and entered into as of the time and date of click through action (“Effective Date”),is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs the use of the NVIDIA computer software and the documentation made available for use with such NVIDIA software. By downloading, installing, copying, or otherwise using the NVIDIA software and/or documentation, you agree to be bound by the terms of this SLA. If you do not agree to the terms of this SLA, do not download, install, copy or use the NVIDIA software or documentation. IF YOU ARE ENTERING INTO THIS SLAON BEHALF OF A COMPANY OR OTHER LEGAL ENTITY, YOU REPRESENT THAT YOU HAVE THE LEGAL AUTHORITY TO BIND THE ENTITY TO THIS SLA, IN WHICH CASE “YOU” WILL MEAN THE ENTITY YOU REPRESENT. IF YOU DON’T HAVE SUCH AUTHORITY, OR IF YOU DON’T ACCEPT ALL THE TERMS AND CONDITIONS OF THIS SLA, THEN NVIDIA DOES NOT AGREETO LICENSE THE LICENSED SOFTWARETO YOU, AND YOU MAY NOT DOWNLOAD, INSTALL, COPY OR USE IT.
Preface
This document is the Software License Agreement (SLA) for NVIDIA TensorRT. This document contains specific license terms and conditions for NVIDIA TensorRT. By accepting this agreement, you agree to comply with all the terms and conditions applicable to the specific product(s) included herein.
If you are receiving TensorRT under the NVIDIA Prerelease License Agreement (also known as NPLA) or under the NVIDIA Software License Agreement (previously known as the NVIDIA Tegra Software License Agreement), your use of TensorRT is governed by such applicable terms and conditions. All other uses of TensorRT are governed by the terms and conditions of the below license agreement.
NVIDIA SOFTWARE LICENSE AGREEMENT
Important: READ BEFORE DOWNLOADING, INSTALLING, COPYING OR USING THE LICENSED SOFTWARE
This Software License Agreement ("SLA”), made and entered into as of the time and date of click through action (“Effective Date”),is a legal agreement between you and NVIDIA Corporation ("NVIDIA") and governs the use of the NVIDIA computer software and the documentation made available for use with such NVIDIA software. By downloading, installing, copying, or otherwise using the NVIDIA software and/or documentation, you agree to be bound by the terms of this SLA. If you do not agree to the terms of this SLA, do not download, install, copy or use the NVIDIA software or documentation. IF YOU ARE ENTERING INTO THIS SLAON BEHALF OF A COMPANY OR OTHER LEGAL ENTITY, YOU REPRESENT THAT YOU HAVE THE LEGAL AUTHORITY TO BIND THE ENTITY TO THIS SLA, IN WHICH CASE “YOU” WILL MEAN THE ENTITY YOU REPRESENT. IF YOU DON’T HAVE SUCH AUTHORITY, OR IF YOU DON’T ACCEPT ALL THE TERMS AND CONDITIONS OF THIS SLA, THEN NVIDIA DOES NOT AGREETO LICENSE THE LICENSED SOFTWARETO YOU, AND YOU MAY NOT DOWNLOAD, INSTALL, COPY OR USE IT.
1. LICENSE.
1.1. License Grant
Subject to the terms of the AGREEMENT, NVIDIA hereby grants you a non-exclusive, non-transferable license, without the right to sublicense (except as expressly set forth in a Supplement), during the applicable license term unless earlier terminated as provided below, to have Authorized Users install and use the Software, including modifications (if expressly permitted in a Supplement), in accordance with the Documentation. You are only licensed to activate and use Licensed Software for which you a have a valid license, even if during the download or installation you are presented with other product options. No Orders are binding on NVIDIA until accepted by NVIDIA. Your Orders are subject to the AGREEMENT.
SLA Supplements: Certain Licensed Software licensed under this SLA may be subject to additional terms and conditions that will be presented to you in a Supplement for acceptance prior to the delivery of such Licensed Software under this SLA and the applicable Supplement. Licensed Software will only be delivered to you upon your acceptance of all applicable terms.
1.2. Limited Purpose Licenses
If your license is provided for one of the purposes indicated below, then notwithstanding contrary terms in License Grant or in a Supplement, such licenses are for internal use and do not include any right or license to sub-license and distribute the Licensed Software or its output in any way in any public release, however limited, and/or in any manner that provides third parties with use of or access to the Licensed Software or its functionality or output, including (but not limited to) external alpha or beta testing or development phases. Further:
Evaluation License. You may use evaluation licenses solely for your internal evaluation of the Licensed Software for broader adoption within your Enterprise or in connection with a NVIDIA product purchase decision, and such licenses have an expiration date as indicated by NVIDIA in its sole discretion (or ninety days from the date of download if no other duration is indicated).
Educational/Academic License. You may use educational/academic licenses solely for educational purposes and all users must be enrolled or employed by an academic institution. If you do not meet NVIDIA’s academic program requirements for educational institutions, you have no rights under this license.
Test/Development License. You may use test/development licenses solely for your internal development, testing and/or debugging of your software applications or for interoperability testing with the Licensed Software, and such licenses have an expiration date as indicated by NVIDIA in its sole discretion (or one year from the date of download if no other duration is indicated). NVIDIA Confidential Information under the AGREEMENT includes output from Licensed Software developer tools identified as “Pro” versions, where the output reveals functionality or performance data pertinent to NVIDIA hardware or software products.
1.3. Pre-Release Licenses
With respect to alpha, beta, preview, and other pre-release Software and Documentation (“Pre-Release Licensed Software”) delivered to you under the AGREEMENT you acknowledge and agree that such Pre-Release Licensed Software (i) may not be fully functional, may contain errors or design flaws, and may have reduced or different security, privacy, accessibility, availability, and reliability standards relative to commercially provided NVIDIA software and documentation, and (ii) use of such Pre-Release Licensed Software may result in unexpected results, loss of data, project delays or other unpredictable damage or loss. THEREFORE, PRE-RELEASE LICENSED SOFTWARE IS NOT INTENDED FOR USE, AND SHOULD NOT BE USED, IN PRODUCTION OR BUSINESS-CRITICAL SYSTEMS. NVIDIA has no obligation to make available a commercial version of any Pre-Release Licensed Software and NVIDIA has the right to abandon development of Pre-Release Licensed Software at any time without liability.
1.4. Enterprise and Contractor Usage
You may allow your Enterprise employees and Contractors to access and use the Licensed Software pursuant to the terms of the AGREEMENT solely to perform work on your behalf, provided further that with respect to Contractors: (i) you obtain a written agreement from each Contractor which contains terms and obligations with respect to access to and use of Licensed Software no less protective of NVIDIA than those set forth in the AGREEMENT, and (ii) such Contractor’s access and use expressly excludes any sublicensing or distribution rights for the Licensed Software. You are responsible for the compliance with the terms and conditions of the AGREEMENT by your Enterprise and Contractors. Any act or omission that, if committed by you, would constitute a breach of the AGREEMENT shall be deemed to constitute a breach of the AGREEMENT if committed by your Enterprise or Contractors.
1.5. Services
Except as expressly indicated in an Order, NVIDIA is under no obligation to provide support for the Licensed Software or to provide any patches, maintenance, updates or upgrades under the AGREEMENT. Unless patches, maintenance, updates or upgrades are provided with their separate governing terms and conditions, they constitute Licensed Software licensed to you under the AGREEMENT.
2. LIMITATIONS.
2.1. License Restrictions
Except as expressly authorized in the AGREEMENT, you agree that you will not (nor authorize third parties to): (i) copy and use Software that was licensed to you for use in one or more NVIDIA hardware products in other unlicensed products (provided that copies solely for backup purposes are allowed); (ii) reverse engineer, decompile, disassemble (except to the extent applicable laws specifically require that such activities be permitted) or attempt to derive the source code, underlying ideas, algorithm or structure of Software provided to you in object code form; (iii) sell, transfer, assign, distribute, rent, loan, lease, sublicense or otherwise make available the Licensed Software or its functionality to third parties (a) as an application services provider or service bureau, (b) by operating hosted/virtual system environments, (c) by hosting, time sharing or providing any other type of services, or (d) otherwise by means of the internet; (iv) modify, translate or otherwise create any derivative works of any Licensed Software; (v) remove, alter, cover or obscure any proprietary notice that appears on or with the Licensed Software or any copies thereof; (vi) use the Licensed Software, or allow its use, transfer, transmission or export in violation of any applicable export control laws, rules or regulations; (vii) distribute, permit access to, or sublicense the Licensed Software as a stand-alone product; (viii) bypass, disable, circumvent or remove any form of copy protection, encryption, security or digital rights management or authentication mechanism used by NVIDIA in connection with the Licensed Software, or use the Licensed Software together with any authorization code, serial number, or other copy protection device not supplied by NVIDIA directly or through an authorized reseller; (ix) use the Licensed Software for the purpose of developing competing products or technologies or assisting a third party in such activities; (x) use the Licensed Software with any system or application where the use or failure of such system or application can reasonably be expected to threaten or result in personal injury, death, or catastrophic loss including, without limitation, use in connection with any nuclear, avionics, navigation, military, medical, life support or other life critical application (“Critical Applications”), unless the parties have entered into a Critical Applications agreement; (xi) distribute any modification or derivative work you make to the Licensed Software under or by reference to the same name as used by NVIDIA; or (xii) use the Licensed Software in any manner that would cause the Licensed Software to become subject to an Open Source License. Nothing in the AGREEMENT shall be construed to give you a right to use, or otherwise obtain access to, any source code from which the Software or any portion thereof is compiled or interpreted. You acknowledge that NVIDIA does not design, test, manufacture or certify the Licensed Software for use in the context of a Critical Application and NVIDIA shall not be liable to you or any third party, in whole or in part, for any claims or damages arising from such use. You agree to defend, indemnify and hold harmless NVIDIA and its Affiliates, and their respective employees, contractors, agents, officers and directors, from and against any and all claims, damages, obligations, losses, liabilities, costs or debt, fines, restitutions and expenses (including but not limited to attorney’s fees and costs incident to establishing the right of indemnification) arising out of or related to you and your Enterprise, and their respective employees, contractors, agents, distributors, resellers, end users, officers and directors use of Licensed Software outside of the scope of the AGREEMENT or any other breach of the terms of the AGREEMENT.
2.2. Third Party License Obligations
You acknowledge and agree that the Licensed Software may include or incorporate third party technology (collectively “Third Party Components”), which is provided for use in or with the Software and not otherwise used separately. If the Licensed Software includes or incorporates Third Party Components, then the third-party pass-through terms and conditions (“Third Party Terms”) for the particular Third Party Component will be bundled with the Software or otherwise made available online as indicated by NVIDIA and will be incorporated by reference into the AGREEMENT. In the event of any conflict between the terms in the AGREEMENT and the Third Party Terms, the Third Party Terms shall govern. Copyright to Third Party Components are held by the copyright holders indicated in the copyright notices indicated in the Third Party Terms.
Audio/Video Encoders and Decoders: You acknowledge and agree that it is your sole responsibility to obtain any additional third party licenses required to make, have made, use, have used, sell, import, and offer for sale your products or services that include or incorporate any Third Party Components and content relating to audio and/or video encoders and decoders from, including but not limited to, Microsoft, Thomson, Fraunhofer IIS, Sisvel S.p.A., MPEG-LA, and Coding Technologies as NVIDIA does not grant to you under the AGREEMENT any necessary patent or other rights with respect to audio and/or video encoders and decoders.
2.3. Limited Rights
Your rights in the Licensed Software are limited to those expressly granted under the AGREEMENT and no other licenses are granted whether by implication, estoppel or otherwise. NVIDIA reserves all rights, title and interest in and to the Licensed Software not expressly granted under the AGREEMENT.
3. CONFIDENTIALITY
Neither party will use the other party’s Confidential Information, except as necessary for the performance of the AGREEMENT, nor will either party disclose such Confidential Information to any third party, except to personnel of NVIDIA and its Affiliates, you, your Enterprise, your Enterprise Contractors, and each party’s legal and financial advisors that have a need to know such Confidential Information for the performance of the AGREEMENT, provided that each such personnel, employee and Contractor is subject to a written agreement that includes confidentiality obligations consistent with those set forth herein. Each party will use all reasonable efforts to maintain the confidentiality of all of the other party’s Confidential Information in its possession or control, but in no event less than the efforts that it ordinarily uses with respect to its own Confidential Information of similar nature and importance. The foregoing obligations will not restrict either party from disclosing the other party’s Confidential Information or the terms and conditions of the AGREEMENT as required under applicable securities regulations or pursuant to the order or requirement of a court, administrative agency, or other governmental body, provided that the party required to make such disclosure (i) gives reasonable notice to the other party to enable it to contest such order or requirement prior to its disclosure (whether through protective orders or otherwise), (ii) uses reasonable effort to obtain confidential treatment or similar protection to the fullest extent possible to avoid such public disclosure, and (iii) discloses only the minimum amount of information necessary to comply with such requirements.
4. OWNERSHIP
You are not obligated to disclose to NVIDIA any modifications that you, your Enterprise or your Contractors make to the Licensed Software as permitted under the AGREEMENT. As between the parties, all modifications are owned by NVIDIA and licensed to you under the AGREEMENT unless otherwise expressly provided in a Supplement. The Licensed Software and all modifications owned by NVIDIA, and the respective Intellectual Property Rights therein, are and will remain the sole and exclusive property of NVIDIA or its licensors, whether the Licensed Software is separate from or combined with any other products or materials. You shall not engage in any act or omission that would impair NVIDIA’s and/or its licensors’ Intellectual Property Rights in the Licensed Software or any other materials, information, processes or subject matter proprietary to NVIDIA. NVIDIA’s licensors are intended third party beneficiaries with the right to enforce provisions of the AGREEMENT with respect to their Confidential Information and/or Intellectual Property Rights.
5. FEEDBACK
You have no obligation to provide Feedback to NVIDIA. However, NVIDIA and/or its Affiliates may use and include any Feedback that you provide to improve the Licensed Software or other NVIDIA products, technologies or materials. Accordingly, if you provide Feedback, you agree that NVIDIA and/or its Affiliates, at their option, may, and may permit their licensees, to make, have made, use, have used, reproduce, license, distribute and otherwise commercialize the Feedback in the Licensed Software or in other NVIDIA products, technologies or materials without the payment of any royalties or fees to you. All Feedback becomes the sole property of NVIDIA and may be used in any manner NVIDIA sees fit, and you hereby assign to NVIDIA all of your right, title and interest in and to any Feedback. NVIDIA has no obligation to respond to Feedback or to incorporate Feedback into the Licensed Software.
6. NO WARRANTIES
THE LICENSED SOFTWARE AND ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES ARE PROVIDED BY NVIDIA “AS IS” AND “WITH ALL FAULTS,” AND NVIDIA EXPRESSLY DISCLAIMS ALL OTHER WARRANTIES OF ANY KIND OR NATURE, WHETHER EXPRESS, IMPLIED OR STATUTORY, INCLUDING, BUT NOT LIMITED TO, ANY WARRANTIES OF OPERABILITY, CONDITION, VALUE, ACCURACY OF DATA, OR QUALITY, AS WELL AS ANY WARRANTIES OF MERCHANTABILITY, SYSTEM INTEGRATION, WORKMANSHIP, SUITABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, OR THE ABSENCE OF ANY DEFECTS THEREIN, WHETHER LATENT OR PATENT. NO WARRANTY IS MADE BY NVIDIA ON THE BASIS OF TRADE USAGE, COURSE OF DEALING OR COURSE OF TRADE. NVIDIA DOES NOT WARRANT THAT THE LICENSED SOFTWARE OR ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES PROVIDED BY NVIDIA UNDER THE AGREEMENT WILL MEET YOUR REQUIREMENTS OR THAT THE OPERATION THEREOF WILL BE UNINTERRUPTED OR ERROR-FREE, OR THAT ALL ERRORS WILL BE CORRECTED. YOU ACKNOWLEDGE THAT NVIDIA’S OBLIGATIONS UNDER THE AGREEMENT ARE FOR THE BENEFIT OF YOU ONLY. Nothing in this warranty section affects any statutory rights of consumers or other recipients to the extent that they cannot be waived or limited by contract under applicable law.
7. LIMITATION OF LIABILITY
TO THE MAXIMUM EXTENT PERMITTED BY LAW, NVIDIA OR ITS LICENSORS SHALL NOT BE LIABLE FOR ANY SPECIAL, INCIDENTAL, PUNITIVE OR CONSEQUENTIAL DAMAGES, OR ANY LOST PROFITS, LOSS OF USE, LOSS OF DATA OR LOSS OF GOODWILL, OR THE COSTS OF PROCURING SUBSTITUTE PRODUCTS, ARISING OUT OF OR IN CONNECTION WITH THE AGREEMENT OR THE USE OR PERFORMANCE OF THE LICENSED SOFTWARE AND ANY OTHER CONFIDENTIAL INFORMATION AND/OR SERVICES PROVIDED BY NVIDIA UNDER THE AGREEMENT, WHETHER SUCH LIABILITY ARISES FROM ANY CLAIM BASED UPON BREACH OF CONTRACT, BREACH OF WARRANTY, TORT (INCLUDING NEGLIGENCE), PRODUCT LIABILITY OR ANY OTHER CAUSE OF ACTION OR THEORY OF LIABILITY. IN NO EVENT WILL NVIDIA’S TOTAL CUMULATIVE LIABILITY UNDER OR ARISING OUT OF THE AGREEMENT EXCEED THE NET AMOUNTS RECEIVED BY NVIDIA FOR YOUR USE OF THE PARTICULAR LICENSED SOFTWARE DURING THE TWELVE (12) MONTHS BEFORE THE LIABILITY AROSE (or up to US$10.00 if you acquired the Licensed Software for no charge). THE NATURE OF THE LIABILITY, THE NUMBER OF CLAIMS OR SUITS OR THE NUMBER OF PARTIES WITHIN YOUR ENTERPRISE THAT ACCEPTED THE TERMS OF THE AGREEMENT SHALL NOT ENLARGE OR EXTEND THIS LIMIT. THE FOREGOING LIMITATIONS SHALL APPLY REGARDLESS OF WHETHER NVIDIA OR ITS LICENSORS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES AND REGARDLESS OF WHETHER ANY REMEDY FAILS ITS ESSENTIAL PURPOSE. The disclaimers, exclusions and limitations of liability set forth in the AGREEMENT form an essential basis of the bargain between the parties, and, absent any such disclaimers, exclusions or limitations of liability, the provisions of the AGREEMENT, including, without limitation, the economic terms, would be substantially different.
8. TERM AND TERMINATION.
8.1. AGREEMENT, Licenses and Services
This SLA shall become effective upon the Effective Date, each Supplement upon their acceptance, and both this SLA and Supplements shall continue in effect until your last access or use of the Licensed Software and/or services hereunder, unless earlier terminated as provided in this “Term and Termination” section. Each Licensed Software license ends at the earlier of (a) the expiration of the applicable license term, or (b) termination of such license or the AGREEMENT. Each service ends at the earlier of (x) the expiration of the applicable service term, (y) termination of such service or the AGREEMENT, or (z) expiration or termination of the associated license and no credit or refund will be provided upon the expiration or termination of the associated license for any service fees paid.
8.2. Termination and Effect of Expiration or Termination
NVIDIA may terminate the AGREEMENT in whole or in part: (i) if you breach any term of the AGREEMENT and fail to cure such breach within thirty (30) days following notice thereof from NVIDIA (or immediately if you violate NVIDIA’s Intellectual Property Rights); (ii) if you become the subject of a voluntary or involuntary petition in bankruptcy or any proceeding relating to insolvency, receivership, liquidation or composition for the benefit of creditors, if that petition or proceeding is not dismissed with prejudice within sixty (60) days after filing, or if you cease to do business; or (iii) if you commence or participate in any legal proceeding against NVIDIA, with respect to the Licensed Software that is the subject of the proceeding during the pendency of such legal proceeding. If you or your authorized NVIDIA reseller fail to pay license fees or service fees when due then NVIDIA may, in its sole discretion, suspend or terminate your license grants, services and any other rights provided under the AGREEMENT for the affected Licensed Software, in addition to any other remedies NVIDIA may have at law or equity. Upon any expiration or termination of the AGREEMENT, a license or a service provided hereunder, (a) any amounts owed to NVIDIA become immediately due and payable, (b) you must promptly discontinue use of the affected Licensed Software and/or service, and (c) you must promptly destroy or return to NVIDIA all copies of the affected Licensed Software and all portions thereof in your possession or control, and each party will promptly destroy or return to the other all of the other party’s Confidential Information within its possession or control. Upon written request, you will certify in writing that you have complied with your obligations under this section. Upon expiration or termination of the AGREEMENT all provisions survive except for the license grant provisions.
9. CONSENT TO COLLECTION AND USE OF INFORMATION.
You hereby agree and acknowledge that the Software may access, collect non-personally identifiable information about your Enterprise computer systems in order to properly optimize such systems for use with the Software. To the extent that you use the Software, you hereby consent to all of the foregoing, and represent and warrant that you have the right to grant such consent. In addition, you agree that you are solely responsible for maintaining appropriate data backups and system restore points for your Enterprise systems, and that NVIDIA will have no responsibility for any damage or loss to such systems (including loss of data or access) arising from or relating to (a) any changes to the configuration, application settings, environment variables, registry, drivers, BIOS, or other attributes of the systems (or any part of such systems) initiated through the Software; or (b) installation of any Software or third party software patches initiated through the Software. In certain systems you may change your system update preferences by unchecking "Automatically check for updates" in the "Preferences" tab of the control panel for the Software.
In connection with the receipt of the Licensed Software or services you may receive access to links to third party websites and services and the availability of those links does not imply any endorsement by NVIDIA. NVIDIA encourages you to review the privacy statements on those sites and services that you choose to visit so that you can understand how they may collect, use and share personal information of individuals. NVIDIA is not responsible or liable for: (i) the availability or accuracy of such links; or (ii) the products, services or information available on or through such links; or (iii) the privacy statements or practices of sites and services controlled by other companies or organizations.
To the extent that you or members of your Enterprise provide to NVIDIA during registration or otherwise personal information, you acknowledge that such information will be collected, used and disclosed by NVIDIA in accordance with NVIDIA's privacy policy, available at URL http://www.nvidia.com/object/privacy_policy.html.
10. GENERAL.
This SLA, any Supplements incorporated hereto, and Orders constitute the entire agreement of the parties with respect to the subject matter hereto and supersede all prior negotiations, conversations, or discussions between the parties relating to the subject matter hereto, oral or written, and all past dealings or industry custom. Any additional and/or conflicting terms and conditions on purchase order(s) or any other documents issued by you are null, void, and invalid. Any amendment or waiver under the AGREEMENT must be in writing and signed by representatives of both parties.
The AGREEMENT and the rights and obligations thereunder may not be assigned by you, in whole or in part, including by merger, consolidation, dissolution, operation of law, or any other manner, without written consent of NVIDIA, and any purported assignment in violation of this provision shall be void and of no effect. NVIDIA may assign, delegate or transfer the AGREEMENT and its rights and obligations hereunder, and if to a non-Affiliate you will be notified.
Each party acknowledges and agrees that the other is an independent contractor in the performance of the AGREEMENT, and each party is solely responsible for all of its employees, agents, contractors, and labor costs and expenses arising in connection therewith. The parties are not partners, joint ventures or otherwise affiliated, and neither has any authority to make any statements, representations or commitments of any kind to bind the other party without prior written consent.
Neither party will be responsible for any failure or delay in its performance under the AGREEMENT (except for any payment obligations) to the extent due to causes beyond its reasonable control for so long as such force majeure event continues in effect.
The AGREEMENT will be governed by and construed under the laws of the State of Delaware and the United States without regard to the conflicts of law provisions thereof and without regard to the United Nations Convention on Contracts for the International Sale of Goods. The parties consent to the personal jurisdiction of the federal and state courts located in Santa Clara County, California. You acknowledge and agree that a breach of any of your promises or agreements contained in the AGREEMENT may result in irreparable and continuing injury to NVIDIA for which monetary damages may not be an adequate remedy and therefore NVIDIA is entitled to seek injunctive relief as well as such other and further relief as may be appropriate. If any court of competent jurisdiction determines that any provision of the AGREEMENT is illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect. Unless otherwise specified, remedies are cumulative.
The Licensed Software has been developed entirely at private expense and is “commercial items” consisting of “commercial computer software” and “commercial computer software documentation” provided with RESTRICTED RIGHTS. Use, duplication or disclosure by the U.S. Government or a U.S. Government subcontractor is subject to the restrictions set forth in the AGREEMENT pursuant to DFARS 227.7202-3(a) or as set forth in subparagraphs (c)(1) and (2) of the Commercial Computer Software - Restricted Rights clause at FAR 52.227-19, as applicable. Contractor/manufacturer is NVIDIA, 2701 San Tomas Expressway, Santa Clara, CA 95050.
You acknowledge that the Licensed Software described under the AGREEMENT is subject to export control under the U.S. Export Administration Regulations (EAR) and economic sanctions regulations administered by the U.S. Department of Treasury’s Office of Foreign Assets Control (OFAC). Therefore, you may not export, reexport or transfer in-country the Licensed Software without first obtaining any license or other approval that may be required by BIS and/or OFAC. You are responsible for any violation of the U.S. or other applicable export control or economic sanctions laws, regulations and requirements related to the Licensed Software. By accepting this SLA, you confirm that you are not a resident or citizen of any country currently embargoed by the U.S. and that you are not otherwise prohibited from receiving the Licensed Software.
Any notice delivered by NVIDIA to you under the AGREEMENT will be delivered via mail, email or fax. Please direct your legal notices or other correspondence to NVIDIA Corporation, 2701 San Tomas Expressway, Santa Clara, California 95050, United States of America, Attention: Legal Department.
11. GLOSSARY OF TERMS
Certain capitalized terms, if not otherwise defined elsewhere in this SLA, shall have the meanings set forth below:
“Affiliate”
“Affiliate” means any legal entity that Owns, is Owned by, or is commonly Owned with a party. “Own” means having more than 50% ownership or the right to direct the management of the entity.
“AGREEMENT”
“AGREEMENT” means this SLA and all associated Supplements entered by the parties referencing this SLA.
“Authorized Users”
“Authorized Users” means your Enterprise individual employees and any of your Enterprise’s Contractors, subject to the terms of the “Enterprise and Contractors Usage” section.
“Confidential Information”
“Confidential Information” means the Licensed Software (unless made publicly available by NVIDIA without confidentiality obligations), and any NVIDIA business, marketing, pricing, research and development, know-how, technical, scientific, financial status, proposed new products or other information disclosed by NVIDIA to you which, at the time of disclosure, is designated in writing as confidential or proprietary (or like written designation), or orally identified as confidential or proprietary or is otherwise reasonably identifiable by parties exercising reasonable business judgment, as confidential. Confidential Information does not and will not include information that: (i) is or becomes generally known to the public through no fault of or breach of the AGREEMENT by the receiving party; (ii) is rightfully known by the receiving party at the time of disclosure without an obligation of confidentiality; (iii) is independently developed by the receiving party without use of the disclosing party’s Confidential Information; or (iv) is rightfully obtained by the receiving party from a third party without restriction on use or disclosure.
“Contractor”
“Contractor” means an individual who works primarily for your Enterprise on a contractor basis from your secure network. means an individual who works primarily for your Enterprise on a contractor basis from your secure network.
“Documentation”
“Documentation” means the NVIDIA documentation made available for use with the Software, including (without limitation) user manuals, datasheets, operations instructions, installation guides, release notes and other materials provided to you under the AGREEMENT.
“Enterprise”
“Enterprise” means you or any company or legal entity for which you accepted the terms of this SLA, and their subsidiaries of which your company or legal entity owns more than fifty percent (50%) of the issued and outstanding equity.
“Feedback”
“Feedback” means any and all suggestions, feature requests, comments or other feedback regarding the Licensed Software, including possible enhancements or modifications thereto.
“Intellectual Property Rights”
“Intellectual Property Rights” means all patent, copyright, trademark, trade secret, trade dress, trade names, utility models, mask work, moral rights, rights of attribution or integrity service marks, master recording and music publishing rights, performance rights, author’s rights, database rights, registered design rights and any applications for the protection or registration of these rights, or other intellectual or industrial property rights or proprietary rights, howsoever arising and in whatever media, whether now known or hereafter devised, whether or not registered, (including all claims and causes of action for infringement, misappropriation or violation and all rights in any registrations and renewals), worldwide and whether existing now or in the future.
“Licensed Software”
“Licensed Software” means Software, Documentation and all modifications owned by NVIDIA.
“Open Source License”
“Open Source License” includes, without limitation, a software license that requires as a condition of use, modification, and/or distribution of such software that the Software be (i) disclosed or distributed in source code form; (ii) be licensed for the purpose of making derivative works; or (iii) be redistributable at no charge.
“Order”
“Order” means a purchase order issued by you, a signed purchase agreement with you, or other ordering document issued by you to NVIDIA or a NVIDIA authorized reseller (including any on-line acceptance process) that references and incorporates the AGREEMENT and is accepted by NVIDIA.
“Software”
“Software” means the NVIDIA software programs licensed to you under the AGREEMENT including, without limitation, libraries, sample code, utility programs and programming code.
“Supplement”
“Supplement” means the additional terms and conditions beyond those stated in this SLA that apply to certain Licensed Software licensed hereunder.
12. TensorRT SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT
TensorRT SUPPLEMENT TO SOFTWARE LICENSE AGREEMENT
The terms set forth in this TensorRT Supplement (“Supplement”) govern your use of the NVIDIA GPU inference engine (the “TensorRT Licensed Software”) under the terms of your software license agreement (“SLA”) as modified by this Supplement. This Supplement is an exhibit to the SLA and is hereby incorporated as an integral part thereto. Capitalized terms used but not defined herein shall have the meaning assigned to them in the SLA. In the event of conflict between the terms in this Supplement and the terms in the SLA, this Supplement shall control.
12.1. TensorRT DISTRIBUTION
Subject to the terms of the SLA and this Supplement, NVIDIA hereby grants you a non-exclusive, nontransferable license during the applicable license term unless earlier terminated pursuant to the SLA, to distribute the libnvinfer and libnvinfer_plugin libraries when delivered to you as part of the TensorRT Licensed Software in source code form or binary form (but not when provided to you as part of a hardware product), subject to the following: such distribution is solely in binary form to your licensees (“Customers”) only as a component of your own software products having additional material functionality beyond the TensorRT Licensed Software (each, a “Licensee Application"). Subject to the terms and conditions of the SLA and this Supplement, you may further authorize Customers to redistribute the libnvinfer and libnvinfer_plugin libraries as incorporated into a Licensee Application, solely in binary form, provided, however, that you shall require in your agreements with your Customers that their distributions be on terms at least as restrictive as those applicable for your use of such TensorRT Licensed Software within a Licensee Application. The expiration or termination of your licenses to the above described TensorRT Licensed Software under the SLA and this Supplement will not affect rights previously granted by you to recipients that were in compliance with the SLA and this Supplement.
In addition to the rights above, for parties that are developing software intended solely for use on Jetson development kits or Jetson modules and running Linux for Tegra software the following shall apply: TensorRT Licensed Software licensed hereunder may be distributed in its entirety, as provided by NVIDIA and without separation of its components, for you and/or your licensees to create software development kits for use only on the Jetson platform and running Linux for Tegra software. You shall require in your agreements with your licensees that their distributions be on terms at least as restrictive as those applicable for your distribution of TensorRT Licensed Software as described in this Section 1.
In addition to the rights above, for parties that are developing software intended solely for use on Jetson development kits or Jetson modules and running Linux for Tegra software the following shall apply: TensorRT Licensed Software licensed hereunder may be distributed in its entirety, as provided by NVIDIA and without separation of its components, for you and/or your licensees to create software development kits for use only on the Jetson platform and running Linux for Tegra software. You shall require in your agreements with your licensees that their distributions be on terms at least as restrictive as those applicable for your distribution of TensorRT Licensed Software as described in this Section 1.
12.2. LICENSE DURATION
Each TensorRT Licensed Software is licensed to you for an initial duration of one year starting from the date of delivery or download. The licenses granted will automatically renew for successive one year periods, provided that NVIDIA reserves the right to terminate licenses upon ninety days (90) days written notice to you prior to the commencement of a renewal year in addition to the termination rights set forth in the SLA.
12.3. EXPIRATION OF TERMINATION OF THIS SUPPLEMENT
Your failure to comply with the terms of this Supplement is ground for termination for breach by NVIDIA under the SLA. This Supplement will automatically expire or terminate upon the expiration or termination of your rights to TensorRT Licensed Software under the SLA or this Supplement.
Notices
Notice
This document is provided for information purposes only and shall not be regarded as a warranty of a certain functionality, condition, or quality of a product. NVIDIA Corporation (“NVIDIA”) makes no representations or warranties, expressed or implied, as to the accuracy or completeness of the information contained in this document and assumes no responsibility for any errors contained herein. NVIDIA shall have no liability for the consequences or use of such information or for any infringement of patents or other rights of third parties that may result from its use. This document is not a commitment to develop, release, or deliver any Material (defined below), code, or functionality.
NVIDIA reserves the right to make corrections, modifications, enhancements, improvements, and any other changes to this document, at any time without notice.
Customer should obtain the latest relevant information before placing orders and should verify that such information is current and complete.
NVIDIA products are sold subject to the NVIDIA standard terms and conditions of sale supplied at the time of order acknowledgement, unless otherwise agreed in an individual sales agreement signed by authorized representatives of NVIDIA and customer (“Terms of Sale”). NVIDIA hereby expressly objects to applying any customer general terms and conditions with regards to the purchase of the NVIDIA product referenced in this document. No contractual obligations are formed either directly or indirectly by this document.
NVIDIA products are not designed, authorized, or warranted to be suitable for use in medical, military, aircraft, space, or life support equipment, nor in applications where failure or malfunction of the NVIDIA product can reasonably be expected to result in personal injury, death, or property or environmental damage. NVIDIA accepts no liability for inclusion and/or use of NVIDIA products in such equipment or applications and therefore such inclusion and/or use is at customer’s own risk.
NVIDIA makes no representation or warranty that products based on this document will be suitable for any specified use. Testing of all parameters of each product is not necessarily performed by NVIDIA. It is customer’s sole responsibility to evaluate and determine the applicability of any information contained in this document, ensure the product is suitable and fit for the application planned by customer, and perform the necessary testing for the application in order to avoid a default of the application or the product. Weaknesses in customer’s product designs may affect the quality and reliability of the NVIDIA product and may result in additional or different conditions and/or requirements beyond those contained in this document. NVIDIA accepts no liability related to any default, damage, costs, or problem which may be based on or attributable to: (i) the use of the NVIDIA product in any manner that is contrary to this document or (ii) customer product designs.
No license, either expressed or implied, is granted under any NVIDIA patent right, copyright, or other NVIDIA intellectual property right under this document. Information published by NVIDIA regarding third-party products or services does not constitute a license from NVIDIA to use such products or services or a warranty or endorsement thereof. Use of such information may require a license from a third party under the patents or other intellectual property rights of the third party, or a license from NVIDIA under the patents or other intellectual property rights of NVIDIA.
Reproduction of information in this document is permissible only if approved in advance by NVIDIA in writing, reproduced without alteration and in full compliance with all applicable export laws and regulations, and accompanied by all associated conditions, limitations, and notices.
THIS DOCUMENT AND ALL NVIDIA DESIGN SPECIFICATIONS, REFERENCE BOARDS, FILES, DRAWINGS, DIAGNOSTICS, LISTS, AND OTHER DOCUMENTS (TOGETHER AND SEPARATELY, “MATERIALS”) ARE BEING PROVIDED “AS IS.” NVIDIA MAKES NO WARRANTIES, EXPRESSED, IMPLIED, STATUTORY, OR OTHERWISE WITH RESPECT TO THE MATERIALS, AND EXPRESSLY DISCLAIMS ALL IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTABILITY, AND FITNESS FOR A PARTICULAR PURPOSE. TO THE EXTENT NOT PROHIBITED BY LAW, IN NO EVENT WILL NVIDIA BE LIABLE FOR ANY DAMAGES, INCLUDING WITHOUT LIMITATION ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, PUNITIVE, OR CONSEQUENTIAL DAMAGES, HOWEVER CAUSED AND REGARDLESS OF THE THEORY OF LIABILITY, ARISING OUT OF ANY USE OF THIS DOCUMENT, EVEN IF NVIDIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. Notwithstanding any damages that customer might incur for any reason whatsoever, NVIDIA’s aggregate and cumulative liability towards customer for the products described herein shall be limited in accordance with the Terms of Sale for the product.
VESA DisplayPort
DisplayPort and DisplayPort Compliance Logo, DisplayPort Compliance Logo for Dual-mode Sources, and DisplayPort Compliance Logo for Active Cables are trademarks owned by the Video Electronics Standards Association in the United States and other countries.
HDMI
HDMI, the HDMI logo, and High-Definition Multimedia Interface are trademarks or registered trademarks of HDMI Licensing LLC.
ARM
ARM, AMBA and ARM Powered are registered trademarks of ARM Limited. Cortex, MPCore and Mali are trademarks of ARM Limited. All other brands or product names are the property of their respective holders. "ARM" is used to represent ARM Holdings plc; its operating company ARM Limited; and the regional subsidiaries ARM Inc.; ARM KK; ARM Korea Limited.; ARM Taiwan Limited; ARM France SAS; ARM Consulting (Shanghai) Co. Ltd.; ARM Germany GmbH; ARM Embedded Technologies Pvt. Ltd.; ARM Norway, AS and ARM Sweden AB.
OpenCL
OpenCL is a trademark of Apple Inc. used under license to the Khronos Group Inc.
Trademarks
NVIDIA, the NVIDIA logo, and cuBLAS, CUDA, CUDA Toolkit, cuDNN, DALI, DIGITS, DGX, DGX-1, DGX-2, DGX Station, DLProf, GPU, JetPack, Jetson, Kepler, Maxwell, NCCL, Nsight Compute, Nsight Systems, NVCaffe, NVIDIA Ampere GPU architecture, NVIDIA Deep Learning SDK, NVIDIA Developer Program, NVIDIA GPU Cloud, NVLink, NVSHMEM, PerfWorks, Pascal, SDK Manager, T4, Tegra, TensorRT, TensorRT Inference Server, Tesla, TF-TRT, Triton Inference Server, Turing, and Volta are trademarks and/or registered trademarks of NVIDIA Corporation in the United States and other countries. Other company and product names may be trademarks of the respective companies with which they are associated.
Copyright
© 2021 NVIDIA Corporation. All rights reserved.
python/packaging/requirements.txt
+7
View File
@@ -0,0 +1,7 @@
# Required for building wheel files (PEP 517/518/621 compliant)
wheel==0.45.1
setuptools~=75.3.2; python_version<"3.12"
setuptools~=80.9.0; python_version>="3.12"
build==1.0.0
# tomli is needed by tensorrt_build_backend for Python < 3.11 (tomllib is built-in for 3.11+)
tomli==2.0.0; python_version<"3.11"
python/packaging/tensorrt_build_backend/__init__.py
+256
View File
@@ -0,0 +1,256 @@
#
# SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
TensorRT Custom Build Backend
A PEP 517 compliant build backend that wraps setuptools to provide:
1. Custom wheel tag support (python-tag, plat-name) via CLI config settings
2. Dynamic package naming based on wheel type (standalone, libs, frontend, etc.)
This backend reads configuration from [tool.tensorrt] in pyproject.toml and
computes the appropriate package names based on the wheel-type config setting
passed via the build frontend CLI.
Usage in pyproject.toml:
[build-system]
requires = ["setuptools>=61.0", "wheel"]
build-backend = "tensorrt_build_backend"
backend-path = ["."]
[tool.tensorrt]
base-name = "tensorrt" # Base module name
cuda-major = "12" # CUDA major version
Config settings (passed via `python -m build --config-setting=key=value`):
wheel-type: Type of wheel being built. One of:
- "binding": Non-standalone bindings (name = base-name)
- "binding_standalone": Standalone bindings (name = {base-name}_cu{cuda}_bindings)
- "libs": Libraries wheel (name = {base-name}_cu{cuda}_libs)
- "frontend": Frontend/metapackage (name = {base-name}_cu{cuda})
python-tag: Python tag for the wheel (e.g., "cp312")
plat-name: Platform tag for the wheel (e.g., "linux_x86_64")
"""
import shutil
import tempfile
from pathlib import Path
try:
import tomllib
except ImportError:
import tomli as tomllib
from setuptools.build_meta import (
build_sdist,
get_requires_for_build_sdist,
get_requires_for_build_wheel,
prepare_metadata_for_build_wheel,
)
from setuptools.build_meta import build_wheel as _setuptools_build_wheel
def _read_pyproject_toml():
"""Read and parse pyproject.toml from the current directory."""
pyproject_path = Path("pyproject.toml")
if not pyproject_path.exists():
return {}
with open(pyproject_path, "rb") as f:
return tomllib.load(f)
def _get_tensorrt_config():
"""
Get TensorRT-specific configuration from pyproject.toml.
Returns a dict with keys:
- base_name: Base module name (e.g., "tensorrt")
- cuda_major: CUDA major version (e.g., "12")
- version: Package version (e.g., "10.0.0.1")
"""
pyproject = _read_pyproject_toml()
trt_config = pyproject.get("tool", {}).get("tensorrt", {})
return {
"base_name": trt_config.get("base-name", "tensorrt"),
"cuda_major": trt_config.get("cuda-major", "12"),
"version": trt_config.get("version") or pyproject.get("project", {}).get("version", "0.0.0"),
}
def _compute_package_names(trt_config, wheel_type):
"""
Compute the distribution and import package names based on wheel type.
Args:
trt_config: Dict with base_name, cuda_major, version
wheel_type: One of "binding", "binding_standalone", "libs", "frontend"
Returns:
tuple: (distribution_name, import_name)
- distribution_name: Name for pip install (e.g., "tensorrt_cu12_bindings")
- import_name: Name for Python import (e.g., "tensorrt_bindings")
"""
base_name = trt_config["base_name"]
cuda_major = trt_config["cuda_major"]
if wheel_type == "binding_standalone":
# Standalone bindings: tensorrt_cu12_bindings / tensorrt_bindings
return f"{base_name}_cu{cuda_major}_bindings", f"{base_name}_bindings"
elif wheel_type == "libs":
# Libs wheel: tensorrt_cu12_libs / tensorrt
return f"{base_name}_cu{cuda_major}_libs", base_name
elif wheel_type == "frontend":
# Frontend wheel: tensorrt_cu12 / tensorrt
return f"{base_name}_cu{cuda_major}", base_name
else:
# Non-standalone binding or default: tensorrt / tensorrt
return base_name, base_name
def _get_wheel_tags(config_settings):
"""
Extract wheel tags from config_settings.
Returns:
tuple: (python_tag, plat_name) or (None, None) if not specified
"""
config_settings = config_settings or {}
python_tag = config_settings.get("python-tag")
plat_name = config_settings.get("plat-name")
return python_tag, plat_name
def _get_wheel_type(config_settings):
"""
Get the wheel type from config_settings.
Returns one of: "binding", "binding_standalone", "libs", "frontend", or None
Returns None if not set (passthrough mode - no dynamic naming).
"""
config_settings = config_settings or {}
return config_settings.get("wheel-type")
def _update_pyproject_for_build(dist_name, import_name):
"""
Update pyproject.toml in-place with computed package name and version.
This allows setuptools to pick up the correct dynamic values.
We modify the file temporarily during the build process.
Args:
dist_name: Distribution name (for pip install)
import_name: Import name (for Python import)
"""
pyproject_path = Path("pyproject.toml")
content = pyproject_path.read_text()
# Update name field - replace placeholder or any existing name
content = content.replace(
'name = "build-backend-placeholder"',
f'name = "{dist_name}"'
)
# Update packages.find.include to use import name
content = content.replace(
'include = ["build-backend-placeholder"]',
f'include = ["{import_name}*"]'
)
pyproject_path.write_text(content)
def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
"""
Build a wheel with TensorRT-specific customizations.
This is the main PEP 517 hook for building wheels. It:
1. If wheel-type config setting is set: computes package name dynamically and updates pyproject.toml
2. Applies custom python/platform tags if specified via config settings
3. Builds the wheel using setuptools
4. Moves the final wheel to the output directory
Config settings (passed via --config-setting):
wheel-type: Type of wheel being built (binding, binding_standalone, libs, frontend)
python-tag: Python tag for the wheel (e.g., cp312)
plat-name: Platform tag for the wheel (e.g., linux_x86_64)
When wheel-type is not set, this acts as a passthrough to setuptools,
only applying wheel tags if specified.
"""
wheel_type = _get_wheel_type(config_settings)
# Only do dynamic naming if wheel type is specified
if wheel_type:
trt_config = _get_tensorrt_config()
dist_name, import_name = _compute_package_names(trt_config, wheel_type)
_update_pyproject_for_build(dist_name, import_name)
# Get wheel tags
python_tag, plat_name = _get_wheel_tags(config_settings)
# Build config_settings to pass wheel tags directly to setuptools
build_config = dict(config_settings or {})
# Pass wheel options via --build-option (setuptools.build_meta convention)
build_options = []
if python_tag:
build_options.append(f"--python-tag={python_tag}")
if plat_name:
build_options.append(f"--plat-name={plat_name}")
if build_options:
# Merge with any existing build options
existing = build_config.get("--build-option", [])
if isinstance(existing, str):
existing = [existing]
build_config["--build-option"] = list(existing) + build_options
# Build wheel using setuptools to a temporary directory first
# This avoids conflicts with existing wheels in the output directory
with tempfile.TemporaryDirectory() as tmp_wheel_dir:
wheel_filename = _setuptools_build_wheel(
tmp_wheel_dir,
config_settings=build_config,
metadata_directory=metadata_directory
)
wheel_path = Path(tmp_wheel_dir) / wheel_filename
# Move the final wheel to the output directory
final_wheel_path = Path(wheel_directory) / wheel_path.name
# Remove existing wheel if present (handles parallel build conflicts)
if final_wheel_path.exists():
final_wheel_path.unlink()
shutil.move(str(wheel_path), str(final_wheel_path))
return final_wheel_path.name
# Re-export other hooks from setuptools.build_meta
__all__ = [
"build_wheel",
"build_sdist",
"get_requires_for_build_wheel",
"get_requires_for_build_sdist",
"prepare_metadata_for_build_wheel",
]
python/requirements.txt
+6
View File
@@ -0,0 +1,6 @@
# This file should contain the minimum possible packages to be able to import tensorrt and use it correctly.
# This must succeed during builds, so be VERY CAREFUL when you add packages here.
numpy==1.23.0; python_version >= "3.8" and python_version < "3.10"
numpy==1.23.1; python_version >= "3.10" and python_version < "3.12"
numpy==1.26.4; python_version >= "3.12"
##PYTHON_BUILDDIR##/##TENSORRT_MODULE##_bindings-py3.##PYTHON3_MINOR##/dist/##TENSORRT_MODULE##-##TENSORRT_PYTHON_VERSION##-cp3##PYTHON3_MINOR##-none-linux_##TARGET##.whl ; python_version=="3.##PYTHON3_MINOR##"
python/scripts/process_wheel_template.py
+82
View File
@@ -0,0 +1,82 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Copies a template file from a source directory (e.g. packaging/) to a destination (usually a build dir),
replacing variables (e.g. `##TENSORRT_VERSION##`) with concrete values.
Sub-directory structure is preserved.
This file is used by the CMake build when building the python wheels.
"""
import argparse
import glob
import os
def main():
parser = argparse.ArgumentParser(
description="Copy files from a source to a destination, replacing variables with concrete values"
)
parser.add_argument("--src-dir", help="The absolute path to the source directory.", required=True)
parser.add_argument("--dst-dir", help="The absolute path to the destination directory.", required=True)
parser.add_argument("--filepath", help="The template file to copy, relative to the source directory.", required=True)
parser.add_argument("--trt-module", help="The TensorRT module name. One of 'tensorrt', 'tensorrt_lean', or 'tensorrt_dispatch' for non-RTX, and 'tensorrt_rtx' for RTX builds.", required=True)
parser.add_argument("--trt-py-version", help="The version string for the python bindings being built. Usually `major.minor.patch.build`.", required=True)
parser.add_argument("--cuda-version", help="The Cuda version (major.minor).", required=True)
parser.add_argument("--trt-version", help="The TensorRT version (major.minor.patch).", required=True)
parser.add_argument("--plugin-disabled", help="Whether the plugin is disabled.", type=int, choices=[0,1], default=0, required=False)
parser.add_argument("--trt-nvinfer-name", help="The name of the nvinfer library.", required=True)
parser.add_argument("--trt-onnxparser-name", help="The name of the onnxparser library.", required=True)
args, _ = parser.parse_known_args()
if not os.path.isdir(args.src_dir):
raise ValueError(f"Provided src-dir {args.src_dir} is not a directory.")
if not os.path.isdir(args.dst_dir):
raise ValueError(f"Provided dst-dir {args.dst_dir} is not a directory.")
target_path = os.path.join(args.src_dir, args.filepath)
if not os.path.exists(target_path):
raise ValueError(f"Target file {target_path} does not exist.")
with open(target_path, 'r', encoding="utf-8") as file:
contents = file.read()
if args.trt_module == "tensorrt_rtx":
readme_text = """NVIDIA TensorRT RTX is an SDK for high-performance AI inference on NVIDIA RTX GPUs. It includes a Just-in-Time compiler for fast on-device inference optimizations that enable portable deployments and runtime performance specialization. It also introduces convenience features such as built-in CUDA graph support, runtime cache, and a simplified development workflow."""
else:
readme_text = """NVIDIA TensorRT is an SDK that facilitates high-performance machine learning inference. It is designed to work in a complementary fashion with training frameworks such as PyTorch. It focuses specifically on running an already-trained network quickly and efficiently on NVIDIA hardware."""
contents = contents.replace("##TENSORRT_README##", readme_text)
contents = contents.replace("##TENSORRT_MODULE##", args.trt_module)
contents = contents.replace("##TENSORRT_PYTHON_VERSION##", args.trt_py_version)
contents = contents.replace("##CUDA_MAJOR##", args.cuda_version.split(".")[0])
contents = contents.replace("##TENSORRT_MAJOR##", args.trt_version.split(".")[0])
contents = contents.replace("##TENSORRT_MINOR##", args.trt_version.split(".")[1])
contents = contents.replace("##TENSORRT_PLUGIN_DISABLED##", "True" if args.plugin_disabled == 1 else "False")
contents = contents.replace("##TENSORRT_NVINFER_NAME##", args.trt_nvinfer_name)
contents = contents.replace("##TENSORRT_ONNXPARSER_NAME##", args.trt_onnxparser_name)
dest_path = os.path.join(args.dst_dir, args.filepath)
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
with open(dest_path, 'w', encoding="utf-8") as of:
of.write(contents)
if __name__ == "__main__":
main()
python/src/CMakeLists.txt
+25
View File
@@ -0,0 +1,25 @@
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: This subdirectory is added multiple times, once for each binding library target.
# Indirectly call the current ADD_SOURCES_FUNCTION to populate target sources on the bindings lib that currently being setup.
cmake_language(CALL ${ADD_SOURCES_FUNCTION}
pyTensorRT.cpp
utils.cpp
)
add_subdirectory(infer ${SUBDIR_BINARY_DIR_PREFIX}/src/infer)
add_subdirectory(parsers ${SUBDIR_BINARY_DIR_PREFIX}/src/parsers)
python/src/infer/CMakeLists.txt
+30
View File
@@ -0,0 +1,30 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: This subdirectory is added multiple times, once for each binding library target.
# Indirectly call the current ADD_SOURCES_FUNCTION to populate target sources on the bindings lib that currently being setup.
cmake_language(CALL ${ADD_SOURCES_FUNCTION}
pyCore.cpp
pyPlugin.cpp
pyFoundationalTypes.cpp
)
# These sources are omitted from the lean/dispatch runtime bindings.
if(${TRT_PYTHON_IS_FULL_BINDINGS})
cmake_language(CALL ${ADD_SOURCES_FUNCTION}
pyGraph.cpp
)
endif()
python/src/infer/pyCore.cpp
+1894
View File
File diff suppressed because it is too large Load Diff
python/src/infer/pyFoundationalTypes.cpp
+318
View File
@@ -0,0 +1,318 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This contains the fundamental types, i.e. Dims, Weights, dtype
#include "ForwardDeclarations.h"
#include "utils.h"
#include <pybind11/numpy.h>
#include <pybind11/stl.h>
#include "infer/pyFoundationalTypesDoc.h"
#include <cuda_runtime_api.h>
namespace tensorrt
{
using namespace nvinfer1;
namespace lambdas
{
// For Weights
static const auto weights_datatype_constructor = [](DataType const& type) { return new Weights{type, nullptr, 0}; };
static const auto weights_pointer_constructor = [](DataType const& type, size_t const ptr, int64_t count) {
return new Weights{type, reinterpret_cast<void*>(ptr), count};
};
static const auto weights_numpy_constructor = [](py::array& arr) {
arr = py::array::ensure(arr);
// In order to construct a weights object, we must have a contiguous C-style array.
PY_ASSERT_VALUE_ERROR(
arr, "Could not convert NumPy array to Weights. Is it using a data type supported by TensorRT?");
PY_ASSERT_VALUE_ERROR((arr.flags() & py::array::c_style),
"Could not convert non-contiguous NumPy array to Weights. Please use numpy.ascontiguousarray() to fix this.");
return new Weights{utils::type(arr.dtype()), arr.data(), arr.size()};
};
// Helper to compare dims with any kind of Python Iterable.
template <typename DimsType, typename PyIterable>
bool dimsEqual(DimsType const& self, PyIterable& other)
{
if (static_cast<int64_t>(other.size()) != self.nbDims)
{
return false;
}
bool eq = true;
std::vector<int64_t> o = other.template cast<std::vector<int64_t>>();
for (int32_t i = 0; i < self.nbDims; ++i)
{
eq = eq && (self.d[i] == o[i]);
}
return eq;
}
// For base Dims class
static const auto dims_vector_constructor = [](std::vector<int64_t> const& in) {
// This is required, because otherwise MAX_DIMS will not be resolved at compile time.
int32_t const maxDims{static_cast<int32_t>(Dims::MAX_DIMS)};
PY_ASSERT_VALUE_ERROR(in.size() <= maxDims,
"Input length " + std::to_string(in.size()) + ". Max expected length is " + std::to_string(maxDims));
// Create the Dims object.
Dims* self = new Dims{};
self->nbDims = in.size();
for (int32_t i = 0; static_cast<size_t>(i) < in.size(); ++i)
self->d[i] = in[i];
return self;
};
static const auto dims_to_str = [](Dims const& self) {
if (self.nbDims == 0)
return std::string("()");
// Length 1 should followed by trailing comma, for tuple-like behavior.
if (self.nbDims == 1)
return "(" + std::to_string(self.d[0]) + ",)";
// Non-zero lengths
std::string temp = "(";
for (int32_t i = 0; i < self.nbDims - 1; ++i)
temp += std::to_string(self.d[i]) + ", ";
temp += std::to_string(self.d[self.nbDims - 1]) + ")";
return temp;
};
static const auto dims_len = [](Dims const& self) { return self.nbDims; };
// TODO: Add slicing support?
static const auto dims_getter = [](Dims const& self, int32_t const pyIndex) -> int64_t const& {
// Without these bounds checks, horrible infinite looping will occur.
int32_t const index{(pyIndex < 0) ? static_cast<int32_t>(self.nbDims) + pyIndex : pyIndex};
PY_ASSERT_INDEX_ERROR(index >= 0 && index < self.nbDims);
return self.d[index];
};
static const auto dims_getter_slice = [](Dims const& self, py::slice slice) {
int64_t start, stop, step, slicelength;
PY_ASSERT_VALUE_ERROR(
slice.compute(self.nbDims, &start, &stop, &step, &slicelength), "Incorrect getter slice dims");
// Disallow out-of-bounds things.
PY_ASSERT_INDEX_ERROR(stop <= self.nbDims);
py::tuple ret{slicelength};
for (int32_t i = start, index = 0; i < stop; i += step, ++index)
ret[index] = self.d[i];
return ret;
};
static const auto dims_setter = [](Dims& self, int32_t const pyIndex, int64_t const item) {
int32_t const index{(pyIndex < 0) ? static_cast<int32_t>(self.nbDims) + pyIndex : pyIndex};
PY_ASSERT_INDEX_ERROR(index >= 0 && index < self.nbDims);
self.d[index] = item;
};
static const auto dims_setter_slice = [](Dims& self, py::slice slice, Dims const& other) {
int64_t start, stop, step, slicelength;
PY_ASSERT_VALUE_ERROR(
slice.compute(self.nbDims, &start, &stop, &step, &slicelength), "Incorrect setter slice dims");
// Disallow out-of-bounds things.
PY_ASSERT_INDEX_ERROR(stop < self.nbDims);
for (int32_t i = start, index = 0; i < stop; i += step, ++index)
self.d[i] = other.d[index];
};
// For Dims2
static const auto dims2_vector_constructor = [](std::vector<int64_t> const& in) {
PY_ASSERT_VALUE_ERROR(in.size() == 2,
"Input length " + std::to_string(in.size()) + " not equal to expected Dims2 length, which is 2");
return new Dims2{in[0], in[1]};
};
// For DimsHW
static const auto dimshw_vector_constructor = [](std::vector<int64_t> const& in) {
PY_ASSERT_VALUE_ERROR(in.size() == 2,
"Input length " + std::to_string(in.size()) + " not equal to expected DimsHW length, which is 2");
return new DimsHW{in[0], in[1]};
};
// For Dims3
static const auto dims3_vector_constructor = [](std::vector<int64_t> const& in) {
PY_ASSERT_VALUE_ERROR(in.size() == 3,
"Input length " + std::to_string(in.size()) + " not equal to expected Dims3 length, which is 3");
return new Dims3{in[0], in[1], in[2]};
};
// For Dims4
static const auto dims4_vector_constructor = [](std::vector<int64_t> const& in) {
PY_ASSERT_VALUE_ERROR(in.size() == 4,
"Input length " + std::to_string(in.size()) + " not equal to expected Dims4 length, which is 4");
return new Dims4{in[0], in[1], in[2], in[3]};
};
// For IHostMemory
static const auto host_memory_buffer_interface = [](IHostMemory& self) -> py::buffer_info {
return py::buffer_info(self.data(), /* Pointer to buffer */
utils::size(self.type()), /* Size of one scalar */
py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
1, /* Number of dimensions */
{self.size()}, /* Buffer dimensions */
{utils::size(self.type())} /* Strides (in bytes) for each index */
);
};
} // namespace lambdas
void bindFoundationalTypes(py::module& m)
{
// Bind the top level DataType enum.
py::enum_<DataType>(m, "DataType", DataTypeDoc::descr, py::module_local())
.value("FLOAT", DataType::kFLOAT, DataTypeDoc::float32)
.value("HALF", DataType::kHALF, DataTypeDoc::float16)
.value("BF16", DataType::kBF16, DataTypeDoc::bfloat16)
.value("INT8", DataType::kINT8, DataTypeDoc::int8)
.value("INT32", DataType::kINT32, DataTypeDoc::int32)
.value("INT64", DataType::kINT64, DataTypeDoc::int64)
.value("BOOL", DataType::kBOOL, DataTypeDoc::boolean)
.value("UINT8", DataType::kUINT8, DataTypeDoc::uint8)
.value("FP8", DataType::kFP8, DataTypeDoc::fp8)
.value("INT4", DataType::kINT4, DataTypeDoc::int4)
.value("FP4", DataType::kFP4, DataTypeDoc::fp4)
.value("E8M0", DataType::kE8M0, DataTypeDoc::e8m0); // DataType
// Also create direct mappings (so we can call trt.float32, for example).
m.attr("float32") = DataType::kFLOAT;
m.attr("float16") = DataType::kHALF;
m.attr("bfloat16") = DataType::kBF16;
m.attr("int8") = DataType::kINT8;
m.attr("int32") = DataType::kINT32;
m.attr("int64") = DataType::kINT64;
m.attr("bool") = DataType::kBOOL;
m.attr("uint8") = DataType::kUINT8;
m.attr("fp8") = DataType::kFP8;
m.attr("int4") = DataType::kINT4;
m.attr("fp4") = DataType::kFP4;
m.attr("e8m0") = DataType::kE8M0;
py::enum_<WeightsRole>(m, "WeightsRole", WeightsRoleDoc::descr, py::module_local())
.value("KERNEL", WeightsRole::kKERNEL, WeightsRoleDoc::KERNEL)
.value("BIAS", WeightsRole::kBIAS, WeightsRoleDoc::BIAS)
.value("SHIFT", WeightsRole::kSHIFT, WeightsRoleDoc::SHIFT)
.value("SCALE", WeightsRole::kSCALE, WeightsRoleDoc::SCALE)
.value("CONSTANT", WeightsRole::kCONSTANT, WeightsRoleDoc::CONSTANT)
.value("ANY", WeightsRole::kANY, WeightsRoleDoc::ANY); // WeightsRole
// Weights
py::class_<Weights>(m, "Weights", WeightsDoc::descr, py::module_local())
// Can construct an empty weights object with type. Defaults to float32.
.def(py::init(lambdas::weights_datatype_constructor), "type"_a = DataType::kFLOAT, WeightsDoc::init_type)
.def(py::init(lambdas::weights_pointer_constructor), "type"_a, "ptr"_a, "count"_a, WeightsDoc::init_ptr)
// Allows for construction through any contiguous numpy array. It then keeps a pointer to that buffer
// (zero-copy).
.def(py::init(lambdas::weights_numpy_constructor), "a"_a, py::keep_alive<1, 2>(), WeightsDoc::init_numpy)
// Expose numpy-like attributes.
.def_property_readonly("dtype", [](Weights const& self) -> DataType { return self.type; })
.def_property_readonly("size", [](Weights const& self) { return self.count; })
.def_property_readonly("nbytes", [](Weights const& self) { return utils::size(self.type) * self.count; })
.def("numpy", utils::weights_to_numpy, py::return_value_policy::reference_internal, WeightsDoc::numpy)
.def("__len__", [](Weights const& self) { return static_cast<size_t>(self.count); }); // Weights
// Also allow implicit construction, so we can pass in numpy arrays instead of Weights.
py::implicitly_convertible<py::array, Weights>();
// Dims
py::class_<Dims>(m, "Dims", DimsDoc::descr, py::module_local())
.def(py::init<>())
// Allows for construction from python lists and tuples.
.def(py::init(lambdas::dims_vector_constructor), "shape"_a)
// static_cast is required here, or MAX_DIMS does not get pulled in until LOAD time.
.def_property_readonly_static(
"MAX_DIMS", [](py::object /*self*/) { return static_cast<int32_t>(Dims::MAX_DIMS); }, DimsDoc::MAX_DIMS)
// Allow for string representations (displays like a python tuple).
.def("__str__", lambdas::dims_to_str)
.def("__repr__", lambdas::dims_to_str)
// Allow direct comparisons with tuples and lists.
.def("__eq__", lambdas::dimsEqual<Dims, py::list>)
.def("__eq__", lambdas::dimsEqual<Dims, py::tuple>)
// These functions allow us to use Dims like an iterable.
.def("__len__", lambdas::dims_len)
.def("__getitem__", lambdas::dims_getter)
.def("__getitem__", lambdas::dims_getter_slice)
.def("__setitem__", lambdas::dims_setter)
.def("__setitem__", lambdas::dims_setter_slice); // Dims
// Make it possible to use tuples/lists in Python in place of Dims.
py::implicitly_convertible<std::vector<int64_t>, Dims>();
// 2D
py::class_<Dims2, Dims>(m, "Dims2", Dims2Doc::descr, py::module_local())
.def(py::init<>())
.def(py::init<int64_t, int64_t>(), "dim0"_a, "dim1"_a)
// Allows for construction from a tuple/list.
.def(py::init(lambdas::dims2_vector_constructor), "shape"_a); // Dims2
py::implicitly_convertible<std::vector<int64_t>, Dims2>();
py::class_<DimsHW, Dims2>(m, "DimsHW", DimsHWDoc::descr, py::module_local())
.def(py::init<>())
.def(py::init<int64_t, int64_t>(), "h"_a, "w"_a)
// Allows for construction from a tuple/list.
.def(py::init(lambdas::dimshw_vector_constructor), "shape"_a)
// Expose these functions as attributes in Python.
.def_property(
"h", [](DimsHW const& dims) { return dims.h(); }, [](DimsHW& dims, int64_t i) { dims.h() = i; })
.def_property(
"w", [](DimsHW const& dims) { return dims.w(); }, [](DimsHW& dims, int64_t i) { dims.w() = i; }); // DimsHW
py::implicitly_convertible<std::vector<int64_t>, DimsHW>();
// 3D
py::class_<Dims3, Dims>(m, "Dims3", Dims3Doc::descr, py::module_local())
.def(py::init<>())
.def(py::init<int64_t, int64_t, int64_t>(), "dim0"_a, "dim1"_a, "dim2"_a)
// Allows for construction from a tuple/list.
.def(py::init(lambdas::dims3_vector_constructor), "shape"_a); // Dims3
py::implicitly_convertible<std::vector<int64_t>, Dims3>();
// 4D
py::class_<Dims4, Dims>(m, "Dims4", Dims4Doc::descr, py::module_local())
.def(py::init<>())
.def(py::init<int64_t, int64_t, int64_t, int64_t>(), "dim0"_a, "dim1"_a, "dim2"_a, "dim3"_a)
// Allows for construction from a tuple/list.
.def(py::init(lambdas::dims4_vector_constructor), "shape"_a); // Dims4
py::implicitly_convertible<std::vector<int64_t>, Dims4>();
py::class_<IHostMemory>(m, "IHostMemory", py::buffer_protocol(), IHostMemoryDoc::descr, py::module_local())
.def_property_readonly("dtype", [](IHostMemory const& mem) { return mem.type(); })
.def_property_readonly("nbytes", [](IHostMemory const& mem) { return mem.size(); })
// Expose buffer interface.
.def_buffer(lambdas::host_memory_buffer_interface)
.def("__del__", &utils::doNothingDel<IHostMemory>); // IHostMemory
py::class_<InterfaceInfo>(m, "InterfaceInfo", InterfaceInfoDoc::descr, py::module_local())
.def_readwrite("kind", &InterfaceInfo::kind)
.def_readwrite("major", &InterfaceInfo::major)
.def_readwrite("minor", &InterfaceInfo::minor);
py::enum_<APILanguage>(m, "APILanguage", APILanguageDoc::descr, py::module_local())
.value("CPP", APILanguage::kCPP)
.value("PYTHON", APILanguage::kPYTHON);
py::class_<IVersionedInterface>(m, "IVersionedInterface", IVersionedInterfaceDoc::descr, py::module_local())
.def_property_readonly("api_language", &IVersionedInterface::getAPILanguage)
.def_property_readonly("interface_info", &IVersionedInterface::getInterfaceInfo);
}
} // namespace tensorrt
python/src/infer/pyGraph.cpp
+1216
View File
File diff suppressed because it is too large Load Diff
python/src/infer/pyPlugin.cpp
+3938
View File
File diff suppressed because it is too large Load Diff
python/src/parsers/CMakeLists.txt
+24
View File
@@ -0,0 +1,24 @@
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: This subdirectory is added multiple times, once for each binding library target.
# Indirectly call the current ADD_SOURCES_FUNCTION to populate target sources on the bindings lib that currently being setup.
# The parser is only included by the full "tensorrt" or "tensorrt_rtx" binding module.
if(${TRT_PYTHON_IS_FULL_BINDINGS})
cmake_language(CALL ${ADD_SOURCES_FUNCTION}
pyOnnx.cpp
)
endif()
python/src/parsers/pyOnnx.cpp
+278
View File
@@ -0,0 +1,278 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Implementation of PyBind11 Binding Code for OnnxParser
#include "ForwardDeclarations.h"
#include "NvOnnxParser.h"
#include "parsers/pyOnnxDoc.h"
#include "utils.h"
#include <pybind11/stl.h>
#include <pybind11/stl_bind.h>
#include <string>
#include <vector>
using namespace nvonnxparser;
namespace tensorrt
{
// Long lambda functions should go here rather than being inlined into the bindings (1 liners are OK).
namespace lambdas
{
// NOLINTBEGIN(readability-identifier-naming)
static auto const parse = [](IParser& self, py::buffer const& model, char const* path = nullptr) {
py::buffer_info info = model.request();
py::gil_scoped_release releaseGil{};
return self.parse(info.ptr, info.size * info.itemsize, path);
};
static auto const parseFromFile
= [](IParser& self, std::string const& model) { return self.parseFromFile(model.c_str(), 0); };
static auto const supportsModelV2 = [](IParser& self, py::buffer const& model, char const* path = nullptr) {
py::buffer_info info = model.request();
return self.supportsModelV2(info.ptr, info.size * info.itemsize, path);
};
static auto const isSubgraphSupported
= [](IParser& self, int64_t const index) { return self.isSubgraphSupported(index); };
static auto const getSubgraphNodes = [](IParser& self, int64_t const index) {
py::list py_nodes;
int64_t nb_nodes = 0;
int64_t* nodes = self.getSubgraphNodes(index, nb_nodes);
for (int64_t i = 0; i < nb_nodes; i++)
{
py_nodes.append(nodes[i]);
}
return py_nodes;
};
static auto const get_used_vc_plugin_libraries = [](IParser& self) {
std::vector<std::string> vcPluginLibs;
int64_t nbPluginLibs;
auto libCArray = self.getUsedVCPluginLibraries(nbPluginLibs);
if (nbPluginLibs < 0)
{
utils::throwPyError(PyExc_RuntimeError, "Internal error");
}
vcPluginLibs.reserve(nbPluginLibs);
for (int64_t i = 0; i < nbPluginLibs; ++i)
{
vcPluginLibs.emplace_back(std::string{libCArray[i]});
}
return vcPluginLibs;
};
static auto const pLoadModelProto = [](IParser& self, py::buffer const& model, char const* path = nullptr) {
py::buffer_info info = model.request();
return self.loadModelProto(info.ptr, info.size * info.itemsize, path);
};
static auto const pLoadInitializer = [](IParser& self, std::string const& name, size_t const ptr, size_t const count) {
return self.loadInitializer(name.c_str(), reinterpret_cast<void*>(ptr), count);
};
static auto const get_local_function_stack = [](IParserError& self) {
std::vector<std::string> localFunctionStack;
int32_t localFunctionStackSize = self.localFunctionStackSize();
if (localFunctionStackSize > 0)
{
auto localFunctionStackCArray = self.localFunctionStack();
localFunctionStack.reserve(localFunctionStackSize);
for (int32_t i = 0; i < localFunctionStackSize; ++i)
{
localFunctionStack.emplace_back(std::string{localFunctionStackCArray[i]});
}
}
return localFunctionStack;
};
static auto const refitFromBytes = [](IParserRefitter& self, py::buffer const& model, char const* path = nullptr) {
py::buffer_info info = model.request();
py::gil_scoped_release releaseGil{};
return self.refitFromBytes(info.ptr, info.size * info.itemsize, path);
};
static auto const refitFromFile
= [](IParserRefitter& self, std::string const& model) { return self.refitFromFile(model.c_str()); };
static auto const rLoadModelProto = [](IParserRefitter& self, py::buffer const& model, char const* path = nullptr) {
py::buffer_info info = model.request();
return self.loadModelProto(info.ptr, info.size * info.itemsize, path);
};
static auto const rLoadInitializer
= [](IParserRefitter& self, std::string const& name, size_t const ptr, size_t const count) {
return self.loadInitializer(name.c_str(), reinterpret_cast<void*>(ptr), count);
};
// NOLINTEND(readability-identifier-naming)
} // namespace lambdas
// Copies of functions from ONNX Parser code used for the Python Bindings.
namespace pyonnx2trt
{
inline char const* errorCodeStr(ErrorCode code)
{
switch (code)
{
case ErrorCode::kSUCCESS: return "SUCCESS";
case ErrorCode::kINTERNAL_ERROR: return "INTERNAL_ERROR";
case ErrorCode::kMEM_ALLOC_FAILED: return "MEM_ALLOC_FAILED";
case ErrorCode::kMODEL_DESERIALIZE_FAILED: return "MODEL_DESERIALIZE_FAILED";
case ErrorCode::kINVALID_VALUE: return "INVALID_VALUE";
case ErrorCode::kINVALID_GRAPH: return "INVALID_GRAPH";
case ErrorCode::kINVALID_NODE: return "INVALID_NODE";
case ErrorCode::kUNSUPPORTED_GRAPH: return "UNSUPPORTED_GRAPH";
case ErrorCode::kUNSUPPORTED_NODE: return "UNSUPPORTED_NODE";
case ErrorCode::kUNSUPPORTED_NODE_ATTR: return "UNSUPPORTED_NODE_ATTR";
case ErrorCode::kUNSUPPORTED_NODE_INPUT: return "UNSUPPORTED_NODE_INPUT";
case ErrorCode::kUNSUPPORTED_NODE_DATATYPE: return "UNSUPPORTED_NODE_DATATYPE";
case ErrorCode::kUNSUPPORTED_NODE_DYNAMIC: return "UNSUPPORTED_NODE_DYNAMIC";
case ErrorCode::kUNSUPPORTED_NODE_SHAPE: return "UNSUPPORTED_NODE_SHAPE";
case ErrorCode::kREFIT_FAILED: return "REFIT_FAILED";
}
return "UNKNOWN";
}
inline std::string const parserErrorStr(nvonnxparser::IParserError const* error)
{
std::string const nodeInfo = "In node " + std::to_string(error->node()) + " with name: " + error->nodeName()
+ " and operator: " + error->nodeOperator() + " ";
std::string const errorInfo
= std::string("(") + error->func() + "): " + errorCodeStr(error->code()) + ": " + error->desc();
if (error->code() == ErrorCode::kMODEL_DESERIALIZE_FAILED || error->code() == ErrorCode::kREFIT_FAILED)
{
return errorInfo.c_str();
}
return (nodeInfo + errorInfo).c_str();
}
} // namespace pyonnx2trt
void bindOnnx(py::module& m)
{
py::bind_vector<std::vector<size_t>>(m, "NodeIndices");
py::class_<IParser>(m, "OnnxParser", OnnxParserDoc::descr, py::module_local())
// Use a lambda to force correct resolution. Pybind doesn't resolve noexcept factory methods correctly as
// constructors. https://github.com/pybind/pybind11/issues/2856
.def(py::init([](nvinfer1::INetworkDefinition& network, nvinfer1::ILogger& logger) {
return nvonnxparser::createParser(network, logger);
}),
"network"_a, "logger"_a, OnnxParserDoc::init, py::keep_alive<1, 3>{}, py::keep_alive<2, 1>{})
.def("parse", lambdas::parse, "model"_a, "path"_a = nullptr, OnnxParserDoc::parse)
.def("parse_from_file", lambdas::parseFromFile, "model"_a, OnnxParserDoc::parse_from_file,
py::call_guard<py::gil_scoped_release>{})
.def("supports_operator", &IParser::supportsOperator, "op_name"_a, OnnxParserDoc::supports_operator)
.def("supports_model_v2", lambdas::supportsModelV2, "model"_a, "path"_a = nullptr,
OnnxParserDoc::supports_model_v2)
.def_property_readonly("num_subgraphs", &IParser::getNbSubgraphs)
.def("is_subgraph_supported", lambdas::isSubgraphSupported, "index"_a, OnnxParserDoc::is_subgraph_supported)
.def("get_subgraph_nodes", lambdas::getSubgraphNodes, "index"_a, OnnxParserDoc::get_subgraph_nodes)
.def_property_readonly("num_errors", &IParser::getNbErrors)
.def("get_error", &IParser::getError, "index"_a, OnnxParserDoc::get_error)
.def("clear_errors", &IParser::clearErrors, OnnxParserDoc::clear_errors)
.def_property("flags", &IParser::getFlags, &IParser::setFlags)
.def("clear_flag", &IParser::clearFlag, "flag"_a, OnnxParserDoc::clear_flag)
.def("set_flag", &IParser::setFlag, "flag"_a, OnnxParserDoc::set_flag)
.def("get_flag", &IParser::getFlag, "flag"_a, OnnxParserDoc::get_flag)
.def("get_layer_output_tensor", &IParser::getLayerOutputTensor, "name"_a, "i"_a,
OnnxParserDoc::get_layer_output_tensor)
.def("get_used_vc_plugin_libraries", lambdas::get_used_vc_plugin_libraries,
OnnxParserDoc::get_used_vc_plugin_libraries)
.def("__del__", &utils::doNothingDel<IParser>)
.def("load_model_proto", lambdas::pLoadModelProto, "model"_a, "path"_a = nullptr,
OnnxParserDoc::load_model_proto, py::call_guard<py::gil_scoped_release>{})
.def("load_initializer", lambdas::pLoadInitializer, "name"_a, "data"_a, "size"_a,
OnnxParserDoc::load_initializer)
.def("parse_model_proto", &IParser::parseModelProto, OnnxParserDoc::parse_model_proto)
.def("set_builder_config", &IParser::setBuilderConfig, "builder_config"_a, OnnxParserDoc::set_builder_config,
py::keep_alive<1, 2>{});
py::enum_<OnnxParserFlag>(m, "OnnxParserFlag", OnnxParserFlagDoc::descr, py::module_local())
.value("NATIVE_INSTANCENORM", OnnxParserFlag::kNATIVE_INSTANCENORM, OnnxParserFlagDoc::NATIVE_INSTANCENORM)
.value("ENABLE_UINT8_AND_ASYMMETRIC_QUANTIZATION_DLA",
OnnxParserFlag::kENABLE_UINT8_AND_ASYMMETRIC_QUANTIZATION_DLA,
OnnxParserFlagDoc::ENABLE_UINT8_AND_ASYMMETRIC_QUANTIZATION_DLA)
.value(
"REPORT_CAPABILITY_DLA", OnnxParserFlag::kREPORT_CAPABILITY_DLA, OnnxParserFlagDoc::REPORT_CAPABILITY_DLA)
.value("ENABLE_PLUGIN_OVERRIDE", OnnxParserFlag::kENABLE_PLUGIN_OVERRIDE,
OnnxParserFlagDoc::ENABLE_PLUGIN_OVERRIDE)
.value("ADJUST_FOR_DLA", OnnxParserFlag::kADJUST_FOR_DLA, OnnxParserFlagDoc::ADJUST_FOR_DLA);
py::enum_<ErrorCode>(m, "ErrorCode", ErrorCodeDoc::descr, py::module_local())
.value("SUCCESS", ErrorCode::kSUCCESS)
.value("INTERNAL_ERROR", ErrorCode::kINTERNAL_ERROR)
.value("MEM_ALLOC_FAILED", ErrorCode::kMEM_ALLOC_FAILED)
.value("MODEL_DESERIALIZE_FAILED", ErrorCode::kMODEL_DESERIALIZE_FAILED)
.value("INVALID_VALUE", ErrorCode::kINVALID_VALUE)
.value("INVALID_GRAPH", ErrorCode::kINVALID_GRAPH)
.value("INVALID_NODE", ErrorCode::kINVALID_NODE)
.value("UNSUPPORTED_GRAPH", ErrorCode::kUNSUPPORTED_GRAPH)
.value("UNSUPPORTED_NODE", ErrorCode::kUNSUPPORTED_NODE)
.value("UNSUPPORTED_NODE_ATTR", ErrorCode::kUNSUPPORTED_NODE_ATTR)
.value("UNSUPPORTED_NODE_INPUT", ErrorCode::kUNSUPPORTED_NODE_INPUT)
.value("UNSUPPORTED_NODE_DATATYPE", ErrorCode::kUNSUPPORTED_NODE_DATATYPE)
.value("UNSUPPORTED_NODE_DYNAMIC", ErrorCode::kUNSUPPORTED_NODE_DYNAMIC)
.value("UNSUPPORTED_NODE_SHAPE", ErrorCode::kUNSUPPORTED_NODE_SHAPE)
.value("REFIT_FAILED", ErrorCode::kREFIT_FAILED)
.def("__str__", &pyonnx2trt::errorCodeStr)
.def("__repr__", &pyonnx2trt::errorCodeStr);
py::class_<IParserError, std::unique_ptr<IParserError, py::nodelete>>(m, "ParserError", py::module_local())
.def("code", &IParserError::code, ParserErrorDoc::code)
.def("desc", &IParserError::desc, ParserErrorDoc::desc)
.def("file", &IParserError::file, ParserErrorDoc::file)
.def("line", &IParserError::line, ParserErrorDoc::line)
.def("func", &IParserError::func, ParserErrorDoc::func)
.def("node", &IParserError::node, ParserErrorDoc::node)
.def("node_name", &IParserError::nodeName, ParserErrorDoc::node_name)
.def("node_operator", &IParserError::nodeOperator, ParserErrorDoc::node_operator)
.def("local_function_stack", lambdas::get_local_function_stack, ParserErrorDoc::local_function_stack)
.def("local_function_stack_size", &IParserError::localFunctionStackSize,
ParserErrorDoc::local_function_stack_size)
.def("__str__", &pyonnx2trt::parserErrorStr)
.def("__repr__", &pyonnx2trt::parserErrorStr);
py::class_<IParserRefitter>(m, "OnnxParserRefitter", OnnxParserRefitterDoc::descr, py::module_local())
// Use a lambda to force correct resolution. Pybind doesn't resolve noexcept factory methods correctly as
// constructors. https://github.com/pybind/pybind11/issues/2856
.def(py::init([](nvinfer1::IRefitter& refitter, nvinfer1::ILogger& logger) {
return nvonnxparser::createParserRefitter(refitter, logger);
}),
"refitter"_a, "logger"_a, OnnxParserRefitterDoc::init, py::keep_alive<1, 3>{}, py::keep_alive<2, 1>{})
.def("refit_from_bytes", lambdas::refitFromBytes, "model"_a, "path"_a = nullptr,
OnnxParserRefitterDoc::refit_from_bytes)
.def("refit_from_file", lambdas::refitFromFile, "model"_a, OnnxParserRefitterDoc::refit_from_file,
py::call_guard<py::gil_scoped_release>{})
.def_property_readonly("num_errors", &IParserRefitter::getNbErrors)
.def("get_error", &IParserRefitter::getError, "index"_a, OnnxParserRefitterDoc::get_error)
.def("clear_errors", &IParserRefitter::clearErrors, OnnxParserRefitterDoc::clear_errors)
.def("load_model_proto", lambdas::rLoadModelProto, "model"_a, "path"_a = nullptr,
OnnxParserRefitterDoc::load_model_proto, py::call_guard<py::gil_scoped_release>{})
.def("load_initializer", lambdas::rLoadInitializer, "name"_a, "data"_a, "size"_a,
OnnxParserRefitterDoc::load_initializer)
.def("refit_model_proto", &IParserRefitter::refitModelProto, OnnxParserRefitterDoc::refit_model_proto);
// Free functions.
m.def("get_nv_onnx_parser_version", &getNvOnnxParserVersion, get_nv_onnx_parser_version);
}
} // namespace tensorrt
python/src/pyTensorRT.cpp
+63
View File
@@ -0,0 +1,63 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// This file contains top level bindings and defines the whole TRT Python package.
#include "ForwardDeclarations.h"
#include "NvInfer.h"
#include "pyTensorRTDoc.h"
#include <pybind11/stl_bind.h>
namespace tensorrt
{
PYBIND11_MODULE(TENSORRT_MODULE, m)
{
// Python strings can be automatically converted to FallbackStrings,
// whose lifetime is tied to TRT objects that reference, but do not own, a string.
// See ForwardDeclarations.h for more information about FallbackString.
// Note that we cannot allow Python to deallocate this string, hence the py::nodelete.
//
// Important: *All* Python enum and class interfaces exported by this module *must* be
// declared with py::module_local() scope, so that multiple TRT runtime modules
// (e.g. tensorrt_lean and tensorrt_dispatch) may be imported by the same Python script
// without conflicts.
//
// See https://pybind11.readthedocs.io/en/stable/advanced/classes.html#module-local-class-bindings
// for more information.
py::class_<FallbackString, std::unique_ptr<FallbackString, py::nodelete>>(m, "FallbackString", py::module_local())
.def(py::init<std::string>())
.def(py::init<py::str>());
py::implicitly_convertible<std::string, FallbackString>();
py::implicitly_convertible<py::str, FallbackString>();
// Make it so that we can use lists of PluginFields without creating unwanted copies.
// This is declared opaque in ForwardDeclarations.h
py::bind_vector<std::vector<nvinfer1::PluginField>>(m, "PluginFieldCollection");
// Order matters here - Dependencies must be resolved properly!
// TODO: Maybe use actual forward declarations and define functions later.
bindFoundationalTypes(m);
bindPlugin(m);
#if EXPORT_ALL_BINDINGS
bindGraph(m);
#endif
bindCore(m);
#if EXPORT_ALL_BINDINGS
// Parsers
bindOnnx(m);
#endif
}
} // namespace tensorrt
python/src/utils.cpp
+167
View File
@@ -0,0 +1,167 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "utils.h"
#if defined(_WIN32)
#if !defined(WIN32_LEAN_AND_MEAN)
#define WIN32_LEAN_AND_MEAN
#endif // defined(WIN32_LEAN_AND_MEAN)
#define NOMINMAX
#include <windows.h>
#else // defined(_WIN32)
#include <dlfcn.h>
#endif // defined(_WIN32)
namespace tensorrt
{
namespace utils
{
void issueDeprecationWarning(char const* useInstead)
{
std::string msg{"Use " + std::string{useInstead} + " instead."};
py::gil_scoped_acquire acquire{};
PyErr_WarnEx(PyExc_DeprecationWarning, msg.c_str(), 1);
}
void* nvdllOpen(char const* libName)
{
std::ostringstream fullLibName;
#if defined(_WIN32)
fullLibName << "nv" << libName << ".dll";
std::string strFullLibName = fullLibName.str();
return static_cast<void*>(LoadLibraryA(strFullLibName.c_str()));
#else // defined(_WIN32)
fullLibName << "lib" << libName << ".so.1";
std::string strFullLibName = fullLibName.str();
return dlopen(strFullLibName.c_str(), RTLD_LAZY);
#endif // defined(_WIN32)
}
void dllClose(void* handle)
{
if (handle)
{
#if defined(_WIN32)
FreeLibrary(static_cast<HMODULE>(handle));
#else // defined(_WIN32)
dlclose(handle);
#endif // defined(_WIN32)
}
}
void* dllGetSym(void* handle, char const* name)
{
#if defined(_WIN32)
return GetProcAddress(static_cast<HMODULE>(handle), name);
#else // defined(_WIN32)
return dlsym(handle, name);
#endif // defined(_WIN32)
}
// Returns the size in bytes of the specified data type.
size_t size(nvinfer1::DataType type)
{
switch (type)
{
case nvinfer1::DataType::kFLOAT: return 4;
case nvinfer1::DataType::kHALF: return 2;
case nvinfer1::DataType::kINT8: return 1;
case nvinfer1::DataType::kINT32: return 4;
case nvinfer1::DataType::kINT64: return 8;
case nvinfer1::DataType::kBOOL: return 1;
case nvinfer1::DataType::kUINT8: return 1;
case nvinfer1::DataType::kFP8: return 1;
case nvinfer1::DataType::kE8M0: return 1;
case nvinfer1::DataType::kBF16: return 2;
case nvinfer1::DataType::kINT4:
case nvinfer1::DataType::kFP4: break; // TRT-22011 - need to address sub-byte element size
}
return -1;
}
std::unique_ptr<py::dtype> nptype(nvinfer1::DataType type)
{
auto const makeDtype = [](char const* typeStr) { return std::make_unique<py::dtype>(typeStr); };
switch (type)
{
case nvinfer1::DataType::kFLOAT: return makeDtype("f4");
case nvinfer1::DataType::kHALF: return makeDtype("f2");
case nvinfer1::DataType::kINT8: return makeDtype("i1");
case nvinfer1::DataType::kINT32: return makeDtype("i4");
case nvinfer1::DataType::kINT64: return makeDtype("i8");
case nvinfer1::DataType::kBOOL: return makeDtype("b1");
case nvinfer1::DataType::kUINT8: return makeDtype("u1");
case nvinfer1::DataType::kFP8:
case nvinfer1::DataType::kBF16:
case nvinfer1::DataType::kINT4:
case nvinfer1::DataType::kFP4:
case nvinfer1::DataType::kE8M0: return nullptr;
}
return nullptr;
}
nvinfer1::DataType type(py::dtype const& type)
{
if (type.is(py::dtype("f4")))
{
return nvinfer1::DataType::kFLOAT;
}
else if (type.is(py::dtype("f2")))
{
return nvinfer1::DataType::kHALF;
}
else if (type.is(py::dtype("i8")))
{
return nvinfer1::DataType::kINT64;
}
else if (type.is(py::dtype("i4")))
{
return nvinfer1::DataType::kINT32;
}
else if (type.is(py::dtype("i1")))
{
return nvinfer1::DataType::kINT8;
}
else if (type.is(py::dtype("b1")))
{
return nvinfer1::DataType::kBOOL;
}
else if (type.is(py::dtype("u1")))
{
return nvinfer1::DataType::kUINT8;
}
int32_t constexpr kBITS_PER_BYTE{8};
std::stringstream ss{};
ss << "[TRT] [E] Could not implicitly convert NumPy data type: " << type.kind()
<< (type.itemsize() * kBITS_PER_BYTE) << " to TensorRT.";
std::cerr << ss.str() << std::endl;
PY_ASSERT_VALUE_ERROR(false, ss.str());
return nvinfer1::DataType::kFLOAT;
}
void throwPyError(PyObject* type, std::string const& message)
{
PyErr_SetString(type, message.data());
throw py::error_already_set();
}
} // namespace utils
} // namespace tensorrt