wehub/nvidia--tensorrt
1
0
Fork 0
Code Issues Pull Requests Actions 9 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution
Docker Image CI / build-ubuntu2004 (push) Waiting to run
Details

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:36:55 +08:00
commit c8a779b1bb
1887 changed files with 3245738 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/NvInfer.h
+11878
View File
File diff suppressed because it is too large Load Diff
include/NvInferImpl.h
+1388
View File
File diff suppressed because it is too large Load Diff
include/NvInferLegacyDims.h
+206
View File
@@ -0,0 +1,206 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NV_INFER_LEGACY_DIMS_H
#define NV_INFER_LEGACY_DIMS_H
#define NV_INFER_INTERNAL_INCLUDE 1
#include "NvInferRuntimeBase.h" // IWYU pragma: exports
#undef NV_INFER_INTERNAL_INCLUDE
//!
//! \file NvInferLegacyDims.h
//!
//! This file contains declarations of legacy dimensions types which use channel
//! semantics in their names, and declarations on which those types rely.
//!
//!
//! \namespace nvinfer1
//!
//! \brief The TensorRT API version 1 namespace.
//!
namespace nvinfer1
{
//!
//! \class Dims2
//!
//! \brief Descriptor for two-dimensional data.
//!
class Dims2 : public Dims
{
public:
//!
//! \brief Construct an empty Dims2 object.
//!
Dims2()
: Dims2(0, 0)
{
}
//!
//! \brief Construct a Dims2 from 2 elements.
//!
//! \param d0 The first element.
//! \param d1 The second element.
//!
Dims2(int64_t d0, int64_t d1)
{
nbDims = 2;
d[0] = d0;
d[1] = d1;
for (int64_t i{nbDims}; i < Dims::MAX_DIMS; ++i)
{
d[i] = 0;
}
}
};
//!
//! \class DimsHW
//!
//! \brief Descriptor for two-dimensional spatial data.
//!
class DimsHW : public Dims2
{
public:
//!
//! \brief Construct an empty DimsHW object.
//!
DimsHW()
: Dims2()
{
}
//!
//! \brief Construct a DimsHW given height and width.
//!
//! \param height the height of the data
//! \param width the width of the data
//!
DimsHW(int64_t height, int64_t width)
: Dims2(height, width)
{
}
//!
//! \brief Get the height.
//!
//! \return The height.
//!
int64_t& h()
{
return d[0];
}
//!
//! \brief Get the height.
//!
//! \return The height.
//!
int64_t h() const
{
return d[0];
}
//!
//! \brief Get the width.
//!
//! \return The width.
//!
int64_t& w()
{
return d[1];
}
//!
//! \brief Get the width.
//!
//! \return The width.
//!
int64_t w() const
{
return d[1];
}
};
//!
//! \class Dims3
//!
//! \brief Descriptor for three-dimensional data.
//!
class Dims3 : public Dims2
{
public:
//!
//! \brief Construct an empty Dims3 object.
//!
Dims3()
: Dims3(0, 0, 0)
{
}
//!
//! \brief Construct a Dims3 from 3 elements.
//!
//! \param d0 The first element.
//! \param d1 The second element.
//! \param d2 The third element.
//!
Dims3(int64_t d0, int64_t d1, int64_t d2)
: Dims2(d0, d1)
{
nbDims = 3;
d[2] = d2;
}
};
//!
//! \class Dims4
//!
//! \brief Descriptor for four-dimensional data.
//!
class Dims4 : public Dims3
{
public:
//!
//! \brief Construct an empty Dims4 object.
//!
Dims4()
: Dims4(0, 0, 0, 0)
{
}
//!
//! \brief Construct a Dims4 from 4 elements.
//!
//! \param d0 The first element.
//! \param d1 The second element.
//! \param d2 The third element.
//! \param d3 The fourth element.
//!
Dims4(int64_t d0, int64_t d1, int64_t d2, int64_t d3)
: Dims3(d0, d1, d2)
{
nbDims = 4;
d[3] = d3;
}
};
} // namespace nvinfer1
#endif // NV_INFER_LEGCY_DIMS_H
include/NvInferPlugin.h
+42
View File
@@ -0,0 +1,42 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NV_INFER_PLUGIN_H
#define NV_INFER_PLUGIN_H
#include "NvInfer.h"
#include "NvInferPluginUtils.h"
//!
//! \file NvInferPlugin.h
//!
//! This is the API for the Nvidia provided TensorRT plugins.
//!
extern "C"
{
//!
//! \brief Initialize and register all the existing TensorRT plugins to the Plugin Registry with an optional
//! namespace. The plugin library author should ensure that this function name is unique to the library. This
//! function should be called once before accessing the Plugin Registry.
//! \param logger Logger object to print plugin registration information
//! \param libNamespace Namespace used to register all the plugins in this library
//!
TENSORRTAPI bool initLibNvInferPlugins(void* logger, char const* libNamespace);
} // extern "C"
#endif // NV_INFER_PLUGIN_H
include/NvInferPluginBase.h
+291
View File
@@ -0,0 +1,291 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NV_INFER_PLUGIN_BASE_H
#define NV_INFER_PLUGIN_BASE_H
#if !defined(NV_INFER_INTERNAL_INCLUDE)
static_assert(false, "Do not directly include this file. Include NvInferRuntime.h or NvInferPluginUtils.h");
#endif
#define NV_INFER_INTERNAL_INCLUDE 1
#include "NvInferRuntimeBase.h" // IWYU pragma: exports
#undef NV_INFER_INTERNAL_INCLUDE
namespace nvinfer1
{
//!
//! \enum PluginFieldType
//!
//! \brief The possible field types for custom layer.
//!
enum class PluginFieldType : int32_t
{
//! FP16 field type.
kFLOAT16 = 0,
//! FP32 field type.
kFLOAT32 = 1,
//! FP64 field type.
kFLOAT64 = 2,
//! INT8 field type.
kINT8 = 3,
//! INT16 field type.
kINT16 = 4,
//! INT32 field type.
kINT32 = 5,
//! char field type.
kCHAR = 6,
//! nvinfer1::Dims field type.
kDIMS = 7,
//! Unknown field type.
kUNKNOWN = 8,
//! BF16 field type.
kBF16 = 9,
//! INT64 field type.
kINT64 = 10,
//! FP8 field type.
kFP8 = 11,
//! INT4 field type.
kINT4 = 12,
//! FP4 field type.
kFP4 = 13,
};
//!
//! \class PluginField
//!
//! \brief Structure containing plugin attribute field names and associated data
//! This information can be parsed to decode necessary plugin metadata
//!
//!
class PluginField
{
public:
//! Plugin field attribute name
AsciiChar const* name;
//! Plugin field attribute data
void const* data;
//! Plugin field attribute type
PluginFieldType type;
//! Number of data entries in the Plugin attribute
int32_t length;
PluginField(AsciiChar const* const name_ = nullptr, void const* const data_ = nullptr,
PluginFieldType const type_ = PluginFieldType::kUNKNOWN, int32_t const length_ = 0) noexcept
: name(name_)
, data(data_)
, type(type_)
, length(length_)
{
}
};
//!
//! \struct PluginFieldCollection
//!
//! \brief Plugin field collection struct.
//!
struct PluginFieldCollection
{
//! Number of PluginField entries.
int32_t nbFields{};
//! Pointer to PluginField entries.
PluginField const* fields{};
};
//!
//! \enum TensorRTPhase
//!
//! \brief Indicates a phase of operation of TensorRT
//!
enum class TensorRTPhase : int32_t
{
//! Build phase of TensorRT
kBUILD = 0,
//! Execution phase of TensorRT
kRUNTIME = 1
};
//!
//! \enum PluginCapabilityType
//!
//! \brief Enumerates the different capability types a IPluginV3 object may have
//!
enum class PluginCapabilityType : int32_t
{
//! Core capability. Every IPluginV3 object must have this.
kCORE = 0,
//! Build capability. IPluginV3 objects provided to TensorRT build phase must have this.
kBUILD = 1,
//! Runtime capability. IPluginV3 objects provided to TensorRT build and execution phases must have this.
kRUNTIME = 2
};
namespace v_1_0
{
class IPluginCapability : public IVersionedInterface
{
};
class IPluginResource : public IVersionedInterface
{
public:
//!
//! \brief Return version information associated with this interface. Applications must not override this method.
//!
InterfaceInfo getInterfaceInfo() const noexcept override
{
return InterfaceInfo{"IPluginResource", 1, 0};
}
//!
//! \brief Free the underlying resource
//!
//! This will only be called for IPluginResource objects that were produced from IPluginResource::clone()
//!
//! The IPluginResource object on which release() is called must still be in a clone-able state
//! after release() returns
//!
//! \return 0 for success, else non-zero
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: No; this method is not required to be thread-safe
//!
virtual int32_t release() noexcept = 0;
//!
//! \brief Clone the resource object
//!
//! \note Resource initialization (if any) may be skipped for non-cloned objects since only clones will be
//! registered by TensorRT
//!
//! \return Pointer to cloned object. nullptr if there was an issue.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes; this method is required to be thread-safe and may be called from multiple threads.
//!
virtual IPluginResource* clone() noexcept = 0;
~IPluginResource() noexcept override = default;
IPluginResource() = default;
IPluginResource(IPluginResource const&) = default;
IPluginResource(IPluginResource&&) = default;
IPluginResource& operator=(IPluginResource const&) & = default;
IPluginResource& operator=(IPluginResource&&) & = default;
}; // class IPluginResource
class IPluginCreatorInterface : public IVersionedInterface
{
public:
~IPluginCreatorInterface() noexcept override = default;
protected:
IPluginCreatorInterface() = default;
IPluginCreatorInterface(IPluginCreatorInterface const&) = default;
IPluginCreatorInterface(IPluginCreatorInterface&&) = default;
IPluginCreatorInterface& operator=(IPluginCreatorInterface const&) & = default;
IPluginCreatorInterface& operator=(IPluginCreatorInterface&&) & = default;
};
class IPluginV3 : public IVersionedInterface
{
public:
//!
//! \brief Return version information associated with this interface. Applications must not override this method.
//!
InterfaceInfo getInterfaceInfo() const noexcept override
{
return InterfaceInfo{"PLUGIN", 1, 0};
}
//! \brief Return a pointer to plugin object implementing the specified PluginCapabilityType.
//!
//! \note IPluginV3 objects added for the build phase (through addPluginV3()) must return valid objects for
//! PluginCapabilityType::kCORE, PluginCapabilityType::kBUILD and PluginCapabilityType::kRUNTIME.
//!
//! \note IPluginV3 objects added for the runtime phase must return valid objects for
//! PluginCapabilityType::kCORE and PluginCapabilityType::kRUNTIME.
//!
//! \see TensorRTPhase
//! \see IPluginCreatorV3One::createPlugin()
//!
virtual IPluginCapability* getCapabilityInterface(PluginCapabilityType type) noexcept = 0;
//!
//! \brief Clone the plugin object. This copies over internal plugin parameters and returns a new plugin object with
//! these parameters. The cloned object must be in a fully initialized state.
//!
//! \note The cloned object must return valid objects through getCapabilityInterface() for at least the same
//! PluginCapabilityTypes as the original object.
//!
//! \return A cloned plugin object in an initialized state with the same parameters as the current object.
//! nullptr must be returned if the cloning fails.
//!
virtual IPluginV3* clone() noexcept = 0;
};
} // namespace v_1_0
//!
//! \class IPluginResource
//!
//! \brief Interface for plugins to define custom resources that could be shared through the plugin registry
//!
//! \see IPluginRegistry::acquirePluginResource
//! \see IPluginRegistry::releasePluginResource
//!
using IPluginResource = v_1_0::IPluginResource;
//!
//! \class IPluginCreatorInterface
//!
//! \brief Base class for all plugin creator versions.
//!
//! \see IPluginCreator and IPluginRegistry
//!
using IPluginCreatorInterface = v_1_0::IPluginCreatorInterface;
//!
//! \class IPluginV3
//!
//! \brief Plugin class for the V3 generation of user-implemented layers.
//!
//! IPluginV3 acts as a wrapper around the plugin capability interfaces that define the actual behavior of the plugin.
//!
//! \see IPluginCapability
//! \see IPluginCreatorV3One
//! \see IPluginRegistry
//!
using IPluginV3 = v_1_0::IPluginV3;
//!
//! \class IPluginCapability
//!
//! \brief Base class for plugin capability interfaces
//!
//! IPluginCapability represents a split in TensorRT V3 plugins to sub-objects that expose different types of
//! capabilites a plugin may have, as opposed to a single interface which defines all capabilities and behaviors of a
//! plugin.
//!
//! \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
//!
//! \see PluginCapabilityType
//!
using IPluginCapability = v_1_0::IPluginCapability;
} // namespace nvinfer1
#endif /* NV_INFER_PLUGIN_BASE_H */
include/NvInferPluginUtils.h
+135
View File
@@ -0,0 +1,135 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NV_INFER_PLUGIN_UTILS_H
#define NV_INFER_PLUGIN_UTILS_H
#include "NvInferRuntimeCommon.h"
//!
//! \file NvInferPluginUtils.h
//!
//! This is the API for the Nvidia provided TensorRT plugin utilities.
//! It lists all the parameters utilized by the TensorRT plugins.
//!
namespace nvinfer1
{
namespace plugin
{
//!
//! \struct PriorBoxParameters
//!
//! \brief The PriorBox plugin layer generates the prior boxes of designated sizes and aspect ratios across all
//! dimensions (H x W).
//!
//! PriorBoxParameters defines a set of parameters for creating the PriorBox plugin layer.
//!
struct PriorBoxParameters
{
float *minSize; //!< Minimum box size in pixels. Can not be nullptr.
float *maxSize; //!< Maximum box size in pixels. Can be nullptr.
float *aspectRatios; //!< Aspect ratios of the boxes. Can be nullptr.
int32_t numMinSize; //!< Number of elements in minSize. Must be larger than 0.
int32_t numMaxSize; //!< Number of elements in maxSize. Can be 0 or same as numMinSize.
int32_t numAspectRatios; //!< Number of elements in aspectRatios. Can be 0.
bool flip; //!< If true, will flip each aspect ratio. For example,
//!< if there is an aspect ratio "r", the aspect ratio "1.0/r" will be generated as well.
bool clip; //!< If true, will clip the prior so that it is within [0,1].
float variance[4]; //!< Variance for adjusting the prior boxes.
int32_t imgH; //!< Image height. If 0, then the H dimension of the data tensor will be used.
int32_t imgW; //!< Image width. If 0, then the W dimension of the data tensor will be used.
float stepH; //!< Step in H. If 0, then (float)imgH/h will be used where h is the H dimension of the 1st input tensor.
float stepW; //!< Step in W. If 0, then (float)imgW/w will be used where w is the W dimension of the 1st input tensor.
float offset; //!< Offset to the top left corner of each cell.
};
//!
//! \struct RPROIParams
//!
//! \brief RPROIParams is used to create the RPROIPlugin instance.
//!
struct RPROIParams
{
int32_t poolingH; //!< Height of the output in pixels after ROI pooling on feature map.
int32_t poolingW; //!< Width of the output in pixels after ROI pooling on feature map.
int32_t featureStride; //!< Feature stride; ratio of input image size to feature map size.
//!< Assuming that max pooling layers in the neural network use square filters.
int32_t preNmsTop; //!< Number of proposals to keep before applying NMS.
int32_t nmsMaxOut; //!< Number of remaining proposals after applying NMS.
int32_t anchorsRatioCount; //!< Number of anchor box ratios.
int32_t anchorsScaleCount; //!< Number of anchor box scales.
float iouThreshold; //!< IoU (Intersection over Union) threshold used for the NMS step.
float minBoxSize; //!< Minimum allowed bounding box size before scaling, used for anchor box calculation.
float spatialScale; //!< Spatial scale between the input image and the last feature map.
};
//!
//! \struct GridAnchorParameters
//!
//! \brief The Anchor Generator plugin layer generates the prior boxes of designated sizes and aspect ratios across all dimensions (H x W).
//! GridAnchorParameters defines a set of parameters for creating the plugin layer for all feature maps.
//!
struct GridAnchorParameters
{
float minSize; //!< Scale of anchors corresponding to finest resolution.
float maxSize; //!< Scale of anchors corresponding to coarsest resolution.
float* aspectRatios; //!< List of aspect ratios to place on each grid point.
int32_t numAspectRatios; //!< Number of elements in aspectRatios.
int32_t H; //!< Height of feature map to generate anchors for.
int32_t W; //!< Width of feature map to generate anchors for.
float variance[4]; //!< Variance for adjusting the prior boxes.
};
//!
//! \brief When performing yolo9000, softmaxTree is helping to do softmax on confidence scores,
//! for element to get the precise classification through word-tree structured classification definition.
//!
struct softmaxTree
{
int32_t* leaf;
int32_t n;
int32_t* parent;
int32_t* child;
int32_t* group;
char** name;
int32_t groups;
int32_t* groupSize;
int32_t* groupOffset;
};
//!
//! \brief The Region plugin layer performs region proposal calculation.
//!
//! Generate 5 bounding boxes per cell (for yolo9000, generate 3 bounding boxes per cell).
//! For each box, calculating its probabilities of objects detections from 80 pre-defined classifications
//! (yolo9000 has 9418 pre-defined classifications, and these 9418 items are organized as work-tree structure).
//! RegionParameters defines a set of parameters for creating the Region plugin layer.
//!
struct RegionParameters
{
int32_t num; //!< Number of predicted bounding box for each grid cell.
int32_t coords; //!< Number of coordinates for a bounding box.
int32_t classes; //!< Number of classifications to be predicted.
softmaxTree* smTree; //!< Helping structure to do softmax on confidence scores.
};
} // namespace plugin
} // namespace nvinfer1
#endif // NV_INFER_PLUGIN_UTILS_H
include/NvInferRuntime.h
+5403
View File
File diff suppressed because it is too large Load Diff
include/NvInferRuntimeBase.h
+692
View File
@@ -0,0 +1,692 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NV_INFER_RUNTIME_BASE_H
#define NV_INFER_RUNTIME_BASE_H
#include "NvInferVersion.h"
#include <cstddef>
#include <cstdint>
#include <cuda_runtime_api.h>
// Items that are marked as deprecated will be removed in a future release.
#if __cplusplus >= 201402L
#define TRT_DEPRECATED [[deprecated]]
#define TRT_DEPRECATED_BECAUSE(REASON) [[deprecated(REASON)]]
#define TRT_DEPRECATED_ENUM TRT_DEPRECATED
#ifdef _MSC_VER
#define TRT_DEPRECATED_API __declspec(dllexport)
#else
#define TRT_DEPRECATED_API [[deprecated]] __attribute__((visibility("default")))
#endif
#else
#ifdef _MSC_VER
#define TRT_DEPRECATED
#define TRT_DEPRECATED_ENUM
#define TRT_DEPRECATED_API __declspec(dllexport)
#else
#define TRT_DEPRECATED __attribute__((deprecated))
#define TRT_DEPRECATED_ENUM
#define TRT_DEPRECATED_API __attribute__((deprecated, visibility("default")))
#endif
#define TRT_DEPRECATED_BECAUSE(REASON) TRT_DEPRECATED
#endif
//! A stand-in for `[[nodiscard]]` and `[[nodiscard(REASON)]]` that works with older compilers.
#if __cplusplus >= 201907L
#define TRT_NODISCARD [[nodiscard]]
#define TRT_NODISCARD_BECAUSE(REASON) [[nodiscard(REASON)]]
#elif __cplusplus >= 201603L
#define TRT_NODISCARD [[nodiscard]]
#define TRT_NODISCARD_BECAUSE(REASON) [[nodiscard]]
#else
#define TRT_NODISCARD
#define TRT_NODISCARD_BECAUSE(REASON)
#endif
// Defines which symbols are exported.
#ifdef TENSORRT_BUILD_LIB
#ifdef _MSC_VER
//On Windows, exports are controlled by .def files; not by dllexport.
#define TENSORRTAPI
#else
#define TENSORRTAPI __attribute__((visibility("default")))
#endif
#else
#define TENSORRTAPI
#endif
#define TRTNOEXCEPT
//!
//! \file NvInferRuntimeBase.h
//!
//! This file contains common definitions, data structures and interfaces shared between the standard and safe runtime.
//!
//! \warning Do not directly include this file. Instead include one of:
//! * NvInferRuntime.h (for the standard runtime)
//! * NvInferPluginUtils.h (for plugin utilities)
//!
#if !defined(NV_INFER_INTERNAL_INCLUDE)
static_assert(false, "Do not directly include this file. Include NvInferRuntime.h or NvInferPluginUtils.h");
#endif
//! Forward declare some CUDA types to avoid an include dependency.
extern "C"
{
//! Forward declaration of cublasContext to use in other interfaces.
struct cublasContext;
//! Forward declaration of cudnnContext to use in other interfaces.
struct cudnnContext;
}
//! Construct a single integer denoting TensorRT version.
//! Usable in preprocessor expressions.
#define NV_TENSORRT_VERSION_INT(major, minor, patch) ((major) *10000L + (minor) *100L + (patch) *1L)
//! TensorRT version as a single integer.
//! Usable in preprocessor expressions.
#define NV_TENSORRT_VERSION NV_TENSORRT_VERSION_INT(NV_TENSORRT_MAJOR, NV_TENSORRT_MINOR, NV_TENSORRT_PATCH)
//!
//! \namespace nvinfer1
//!
//! \brief The TensorRT API version 1 namespace.
//!
namespace nvinfer1
{
//! char_t is the type used by TensorRT to represent all valid characters.
using char_t = char;
//! AsciiChar is the type used by TensorRT to represent valid ASCII characters.
//! This type is widely used in automotive safety context.
using AsciiChar = char_t;
//! Forward declare IErrorRecorder and ILogger for use in other interfaces.
namespace v_1_0
{
class IErrorRecorder;
class ILogger;
} // namespace v_1_0
using IErrorRecorder = v_1_0::IErrorRecorder;
using ILogger = v_1_0::ILogger;
namespace impl
{
//! Declaration of EnumMaxImpl struct to store the exclusive upper bound of an enumeration type.
template <typename T>
struct EnumMaxImpl;
} // namespace impl
//! One greater than the maximum value of enumeration type T.
//! For example, if the highest enumerator in T has value 5, then EnumMax<T>() returns 6.
template <typename T>
constexpr int32_t EnumMax() noexcept
{
return impl::EnumMaxImpl<T>::kVALUE;
}
//!
//! \enum DataType
//! \brief The type of weights and tensors.
//! The datatypes other than kBOOL, kINT32, and kINT64 are "activation datatypes,"
//! as they often represent values corresponding to inference results.
//!
enum class DataType : int32_t
{
//! 32-bit floating point format.
kFLOAT = 0,
//! IEEE 16-bit floating-point format -- has a 5 bit exponent and 11 bit significand.
kHALF = 1,
//! Signed 8-bit integer representing a quantized floating-point value.
kINT8 = 2,
//! Signed 32-bit integer format.
kINT32 = 3,
//! 8-bit boolean. 0 = false, 1 = true, other values undefined.
kBOOL = 4,
//! Unsigned 8-bit integer format.
//! Cannot be used to represent quantized floating-point values.
//! Use the IdentityLayer to convert kUINT8 network-level inputs to {kFLOAT, kHALF} prior
//! to use with other TensorRT layers, or to convert intermediate output
//! before kUINT8 network-level outputs from {kFLOAT, kHALF} to kUINT8.
//! kUINT8 conversions are only supported for {kFLOAT, kHALF}.
//! kUINT8 to {kFLOAT, kHALF} conversion will convert the integer values
//! to equivalent floating point values.
//! {kFLOAT, kHALF} to kUINT8 conversion will convert the floating point values
//! to integer values by truncating towards zero. This conversion has undefined behavior for
//! floating point values outside the range [0.0F, 256.0F) after truncation.
//! kUINT8 conversions are not supported for {kINT8, kINT32, kBOOL}.
kUINT8 = 5,
//! Signed 8-bit floating point with
//! 1 sign bit, 4 exponent bits, 3 mantissa bits, and exponent-bias 7.
kFP8 = 6,
//! Brain float -- has an 8 bit exponent and 8 bit significand.
kBF16 = 7,
//! Signed 64-bit integer type.
kINT64 = 8,
//! Signed 4-bit integer type.
kINT4 = 9,
//! 4-bit floating point type
//! 1 bit sign, 2 bit exponent, 1 bit mantissa
kFP4 = 10,
//! Unsigned representation of exponent-only 8-bit floating point type for quantization scales
kE8M0 = 11,
};
namespace impl
{
//! One greater than the maximum value of DataType enum. \see DataType
template <>
struct EnumMaxImpl<DataType>
{
//! One greater than the maximum value of DataType enum.
static constexpr int32_t kVALUE = 12;
};
} // namespace impl
//!
//! \class Dims
//! \brief Structure to define the dimensions of a tensor.
//!
//! TensorRT can also return an "invalid dims" structure. This structure is
//! represented by nbDims == -1 and d[i] == 0 for all i.
//!
//! TensorRT can also return an "unknown rank" dims structure. This structure is
//! represented by nbDims == -1 and d[i] == -1 for all i.
//!
class Dims64
{
public:
//! The maximum rank (number of dimensions) supported for a tensor.
static constexpr int32_t MAX_DIMS{8};
//! The rank (number of dimensions).
int32_t nbDims;
//! The extent of each dimension.
int64_t d[MAX_DIMS];
};
//!
//! Alias for Dims64.
//!
using Dims = Dims64;
using InterfaceKind = char const*;
//!
//! \class InterfaceInfo
//!
//! \brief Version information associated with a TRT interface
//!
class InterfaceInfo
{
public:
InterfaceKind kind;
int32_t major;
int32_t minor;
};
//!
//! \enum APILanguage
//!
//! \brief Programming language used in the implementation of a TRT interface
//!
enum class APILanguage : int32_t
{
kCPP = 0,
kPYTHON = 1
};
namespace impl
{
//! One greater than the maximum value of APILanguage enum. \see APILanguage
template <>
struct EnumMaxImpl<APILanguage>
{
//! One greater than the maximum value of APILanguage enum.
static constexpr int32_t kVALUE = 2;
};
} // namespace impl
//!
//! \class IVersionedInterface
//!
//! \brief An Interface class for version control.
//!
class IVersionedInterface
{
public:
//!
//! \brief The language used to build the implementation of this Interface.
//!
//! Applications must not override this method.
//!
virtual APILanguage getAPILanguage() const noexcept
{
return APILanguage::kCPP;
}
//!
//! \brief Return version information associated with this interface. Applications must not override this method.
//!
virtual InterfaceInfo getInterfaceInfo() const noexcept = 0;
virtual ~IVersionedInterface() noexcept = default;
protected:
IVersionedInterface() = default;
IVersionedInterface(IVersionedInterface const&) = default;
IVersionedInterface(IVersionedInterface&&) = default;
IVersionedInterface& operator=(IVersionedInterface const&) & = default;
IVersionedInterface& operator=(IVersionedInterface&&) & = default;
};
//!
//! \enum ErrorCode
//!
//! \brief Error codes that can be returned by TensorRT during execution.
//!
enum class ErrorCode : int32_t
{
//!
//! Execution completed successfully.
//!
kSUCCESS = 0,
//!
//! An error that does not fall into any other category. This error is included for forward compatibility.
//!
kUNSPECIFIED_ERROR = 1,
//!
//! A non-recoverable TensorRT error occurred. TensorRT is in an invalid internal state when this error is
//! emitted and any further calls to TensorRT will result in undefined behavior.
//!
kINTERNAL_ERROR = 2,
//!
//! An argument passed to the function is invalid in isolation.
//! This is a violation of the API contract.
//!
kINVALID_ARGUMENT = 3,
//!
//! An error occurred when comparing the state of an argument relative to other arguments. For example, the
//! dimensions for concat differ between two tensors outside of the channel dimension. This error is triggered
//! when an argument is correct in isolation, but not relative to other arguments. This is to help to distinguish
//! from the simple errors from the more complex errors.
//! This is a violation of the API contract.
//!
kINVALID_CONFIG = 4,
//!
//! An error occurred when performing an allocation of memory on the host or the device.
//! A memory allocation error is normally fatal, but in the case where the application provided its own memory
//! allocation routine, it is possible to increase the pool of available memory and resume execution.
//!
kFAILED_ALLOCATION = 5,
//!
//! One, or more, of the components that TensorRT relies on did not initialize correctly.
//! This is a system setup issue.
//!
kFAILED_INITIALIZATION = 6,
//!
//! An error occurred during execution that caused TensorRT to end prematurely, either an asynchronous error,
//! user cancellation, or other execution errors reported by CUDA/DLA. In a dynamic system, the
//! data can be thrown away and the next frame can be processed or execution can be retried.
//! This is either an execution error or a memory error.
//!
kFAILED_EXECUTION = 7,
//!
//! An error occurred during execution that caused the data to become corrupted, but execution finished. Examples
//! of this error are NaN squashing or integer overflow. In a dynamic system, the data can be thrown away and the
//! next frame can be processed or execution can be retried.
//! This is either a data corruption error, an input error, or a range error.
//! This is not used in safety but may be used in standard.
//!
kFAILED_COMPUTATION = 8,
//!
//! TensorRT was put into a bad state by incorrect sequence of function calls. An example of an invalid state is
//! specifying a layer to be DLA only without GPU fallback, and that layer is not supported by DLA. This can occur
//! in situations where a service is optimistically executing networks for multiple different configurations
//! without checking proper error configurations, and instead throwing away bad configurations caught by TensorRT.
//! This is a violation of the API contract, but can be recoverable.
//!
//! Example of a recovery:
//! GPU fallback is disabled and conv layer with large filter(63x63) is specified to run on DLA. This will fail due
//! to DLA not supporting the large kernel size. This can be recovered by either turning on GPU fallback
//! or setting the layer to run on the GPU.
//!
kINVALID_STATE = 9,
//!
//! An error occurred due to the network not being supported on the device due to constraints of the hardware or
//! system. An example is running an unsafe layer in a safety certified context, or a resource requirement for the
//! current network is greater than the capabilities of the target device. The network is otherwise correct, but
//! the network and hardware combination is problematic. This can be recoverable.
//! Examples:
//! * Scratch space requests larger than available device memory and can be recovered by increasing allowed
//! workspace size.
//! * Tensor size exceeds the maximum element count and can be recovered by reducing the maximum batch size.
//!
kUNSUPPORTED_STATE = 10,
};
namespace impl
{
//! One greater than the maximum value of ErrorCode enum. \see ErrorCode
template <>
struct EnumMaxImpl<ErrorCode>
{
//! Declaration of kVALUE
static constexpr int32_t kVALUE = 11;
};
} // namespace impl
namespace v_1_0
{
class IErrorRecorder : public IVersionedInterface
{
public:
//!
//! \brief Return version information associated with this interface. Applications must not override this method.
//!
InterfaceInfo getInterfaceInfo() const noexcept override
{
return InterfaceInfo{"IErrorRecorder", 1, 0};
}
//!
//! \brief A typedef of a C-style string for reporting error descriptions.
//!
using ErrorDesc = char const*;
//!
//! \brief The length limit for an error description in bytes, excluding the '\0' string terminator.
//! Only applicable to safe runtime.
//! General error recorder implementation can use any size appropriate for the use case.
//!
static constexpr size_t kMAX_DESC_LENGTH{127U};
//!
//! \brief A typedef of a 32-bit integer for reference counting.
//!
using RefCount = int32_t;
IErrorRecorder() = default;
~IErrorRecorder() noexcept override = default;
// Public API used to retrieve information from the error recorder.
//!
//! \brief Return the number of errors
//!
//! Determines the number of errors that occurred between the current point in execution
//! and the last time that the clear() was executed. Due to the possibility of asynchronous
//! errors occurring, a TensorRT API can return correct results, but still register errors
//! with the Error Recorder. The value of getNbErrors() must increment by 1 after each reportError()
//! call until clear() is called, or the maximum number of errors that can be stored is exceeded.
//!
//! \return Returns the number of errors detected, or 0 if there are no errors.
//! If the upper bound of errors that can be stored is exceeded, the upper bound value must
//! be returned.
//!
//! For example, if the error recorder can store up to 16 error descriptions but reportError() has
//! been called 20 times, getNbErrors() must return 16.
//!
//! \see clear(), hasOverflowed()
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when multiple execution contexts are used during runtime.
//!
virtual int32_t getNbErrors() const noexcept = 0;
//!
//! \brief Returns the ErrorCode enumeration.
//!
//! \param errorIdx A 32-bit integer that indexes into the error array.
//!
//! The errorIdx specifies what error code from 0 to getNbErrors()-1 that the application
//! wants to analyze and return the error code enum.
//!
//! \return Returns the enum corresponding to errorIdx if errorIdx is in range (between 0 and getNbErrors()-1).
//! ErrorCode::kUNSPECIFIED_ERROR must be returned if errorIdx is not in range.
//!
//! \see getErrorDesc(), ErrorCode
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when multiple execution contexts are used during runtime.
//!
virtual ErrorCode getErrorCode(int32_t errorIdx) const noexcept = 0;
//!
//! \brief Returns a null-terminated C-style string description of the error.
//!
//! \param errorIdx A 32-bit integer that indexes into the error array.
//!
//! For the error specified by the idx value, return the string description of the error. The
//! error string is a null-terminated C-style string. In the safety context there is a
//! constant length requirement to remove any dynamic memory allocations and the error message
//! will be truncated if it exceeds kMAX_DESC_LENGTH bytes.
//! The format of the string is "<EnumAsStr> - <Description>".
//!
//! \return Returns a string representation of the error along with a description of the error if errorIdx is in
//! range (between 0 and getNbErrors()-1). An empty string will be returned if errorIdx is not in range.
//!
//! \see getErrorCode()
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when multiple execution contexts are used during runtime.
//!
virtual ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept = 0;
//!
//! \brief Determine if the error stack has overflowed.
//!
//! In the case when the number of errors is large, this function is used to query if one or more
//! errors have been dropped due to lack of storage capacity. This is especially important in the
//! automotive safety case where the internal error handling mechanisms cannot allocate memory.
//!
//! \return true if errors have been dropped due to overflowing the error stack.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when multiple execution contexts are used during runtime.
//!
virtual bool hasOverflowed() const noexcept = 0;
//!
//! \brief Clear the error stack on the error recorder.
//!
//! Removes all the tracked errors by the error recorder. The implementation must guarantee that after
//! this function is called, and as long as no error occurs, the next call to getNbErrors will return
//! zero and hasOverflowed will return false.
//!
//! \see getNbErrors(), hasOverflowed()
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when multiple execution contexts are used during runtime.
//!
virtual void clear() noexcept = 0;
// API used by TensorRT to report Error information to the application.
//!
//! \brief Report an error to the error recorder with the corresponding enum and description.
//!
//! \param val The error code enum that is being reported.
//! \param desc The string description of the error, which will be a NULL-terminated string.
//! For safety use cases its length is limited to kMAX_DESC_LENGTH bytes
//! (excluding the NULL terminator) and descriptions that exceed this limit will be silently truncated.
//!
//! Report an error to the user that has a given value and human readable description. The function returns false
//! if processing can continue, which implies that the reported error is not fatal. This does not guarantee that
//! processing continues, but provides a hint to TensorRT.
//! The desc C-string data is only valid during the call to reportError and may be immediately deallocated by the
//! caller when reportError returns. The implementation must not store the desc pointer in the ErrorRecorder object
//! or otherwise access the data from desc after reportError returns.
//!
//! \return True if the error is determined to be fatal and processing of the current function must end.
//!
//! \warning If the error recorder's maximum number of storable errors is exceeded, the error description will be
//! silently dropped and the value returned by getNbErrors() will not be incremented. However, the return
//! value will still signal whether the error must be considered fatal.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when multiple execution contexts are used during runtime.
//!
virtual bool reportError(ErrorCode val, ErrorDesc desc) noexcept = 0;
//!
//! \brief Increments the refcount for the current ErrorRecorder.
//!
//! Increments the reference count for the object by one and returns the current value. This reference count allows
//! the application to know that an object inside of TensorRT has taken a reference to the ErrorRecorder. TensorRT
//! guarantees that every call to IErrorRecorder::incRefCount() will be paired with a call to
//! IErrorRecorder::decRefCount() when the reference is released. It is undefined behavior to destruct the
//! ErrorRecorder when incRefCount() has been called without a corresponding decRefCount().
//!
//! \return The reference counted value after the increment completes.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when multiple execution contexts are used during runtime.
//!
virtual RefCount incRefCount() noexcept = 0;
//!
//! \brief Decrements the refcount for the current ErrorRecorder.
//!
//! Decrements the reference count for the object by one and returns the current value. This reference count allows
//! the application to know that an object inside of TensorRT has taken a reference to the ErrorRecorder. TensorRT
//! guarantees that every call to IErrorRecorder::decRefCount() will be preceded by a call to
//! IErrorRecorder::incRefCount(). It is undefined behavior to destruct the ErrorRecorder when incRefCount() has been
//! called without a corresponding decRefCount().
//!
//! \return The reference counted value after the decrement completes.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when multiple execution contexts are used during runtime.
//!
virtual RefCount decRefCount() noexcept = 0;
protected:
// @cond SuppressDoxyWarnings
IErrorRecorder(IErrorRecorder const&) = default;
IErrorRecorder(IErrorRecorder&&) = default;
IErrorRecorder& operator=(IErrorRecorder const&) & = default;
IErrorRecorder& operator=(IErrorRecorder&&) & = default;
// @endcond
}; // class IErrorRecorder
} // namespace v_1_0
//!
//! \class IErrorRecorder
//!
//! \brief Reference counted application-implemented error reporting interface for TensorRT objects.
//!
//! The error reporting mechanism is a user-defined object that interacts with the internal state of the object
//! that it is assigned to in order to determine information about abnormalities in execution. The error recorder
//! gets both an error enum that is more descriptive than pass/fail and also a string description that gives more
//! detail on the exact failure modes. In the safety context, the error strings are all limited to 128 bytes
//! or less in length, including the NULL terminator.
//!
//! The ErrorRecorder gets passed along to any class that is created from another class that has an ErrorRecorder
//! assigned to it. For example, assigning an ErrorRecorder to an IBuilder allows all INetwork's, ILayer's, and
//! ITensor's to use the same error recorder. For functions that have their own ErrorRecorder accessor functions.
//! This allows registering a different error recorder or de-registering of the error recorder for that specific
//! object.
//!
//! ErrorRecorder objects that are used in the safety runtime must define an implementation-dependent upper limit
//! of errors whose information can be stored, and drop errors above this upper limit. The limit must fit in int32_t.
//! The IErrorRecorder::hasOverflowed() method is used to signal that one or more errors have been dropped.
//!
//! The ErrorRecorder object implementation must be thread safe. All locking and synchronization is pushed to the
//! interface implementation and TensorRT does not hold any synchronization primitives when calling the interface
//! functions.
//!
//! The lifetime of the ErrorRecorder object must exceed the lifetime of all TensorRT objects that use it.
//!
using IErrorRecorder = v_1_0::IErrorRecorder;
//!
//! \enum TensorIOMode
//!
//! \brief Definition of tensor IO Mode.
//!
enum class TensorIOMode : int32_t
{
//! Tensor is not an input or output.
kNONE = 0,
//! Tensor is input to the engine.
kINPUT = 1,
//! Tensor is output by the engine.
kOUTPUT = 2
};
namespace impl
{
//! One greater than the maximum value of TensorIOMode enum. \see TensorIOMode
template <>
struct EnumMaxImpl<TensorIOMode>
{
// One greater than the maximum value of TensorIOMode enum
static constexpr int32_t kVALUE = 3;
};
} // namespace impl
} // namespace nvinfer1
//!
//! \brief Return the library version number.
//!
//! The format is as for TENSORRT_VERSION: (MAJOR * 100 + MINOR) * 100 + PATCH
//!
extern "C" TENSORRTAPI int32_t getInferLibVersion() noexcept;
#endif // NV_INFER_RUNTIME_BASE_H
include/NvInferRuntimeCommon.h
+253
View File
@@ -0,0 +1,253 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NV_INFER_RUNTIME_COMMON_H
#define NV_INFER_RUNTIME_COMMON_H
//!
//! \file NvInferRuntimeCommon.h
//!
//! This file provides the nvinfer1::IPluginRegistry interface, which will be moved to the NvInferRuntime.h header
//! in a future release.
//!
//! \warning This file will be removed in a future release.
//!
//! \warning Do not directly include this file. Instead include NvInferRuntime.h
//!
#define NV_INFER_INTERNAL_INCLUDE 1
#include "NvInferPluginBase.h"
#undef NV_INFER_INTERNAL_INCLUDE
#include "NvInferRuntimePlugin.h"
namespace nvinfer1
{
//!
//! \class IPluginRegistry
//!
//! \brief Single registration point for all plugins in an application. It is
//! used to find plugin implementations during engine deserialization.
//! Internally, the plugin registry is considered to be a singleton so all
//! plugins in an application are part of the same global registry.
//! Note that the plugin registry is only supported for plugins of type
//! IPluginV2 and should also have a corresponding IPluginCreator implementation.
//!
//! \see IPluginV2 and IPluginCreator
//!
//! \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
//!
//! \warning In the automotive safety context, be sure to call IPluginRegistry::setErrorRecorder() to register
//! an error recorder with the registry before using other methods in the registry.
//!
class IPluginRegistry
{
public:
//!
//! \brief Pointer for plugin library handle.
//!
using PluginLibraryHandle = void*;
// @cond SuppressDoxyWarnings
IPluginRegistry() = default;
IPluginRegistry(IPluginRegistry const&) = delete;
IPluginRegistry(IPluginRegistry&&) = delete;
IPluginRegistry& operator=(IPluginRegistry const&) & = delete;
IPluginRegistry& operator=(IPluginRegistry&&) & = delete;
// @endcond
protected:
virtual ~IPluginRegistry() noexcept = 0;
public:
//!
//! \brief Set the ErrorRecorder for this interface
//!
//! Assigns the ErrorRecorder to this interface. The ErrorRecorder will track all errors during execution.
//! This function will call incRefCount of the registered ErrorRecorder at least once. Setting
//! recorder to nullptr unregisters the recorder with the interface, resulting in a call to decRefCount if
//! a recorder has been registered.
//!
//! \param recorder The error recorder to register with this interface.
//!
//! \see getErrorRecorder()
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: No
//!
virtual void setErrorRecorder(IErrorRecorder* const recorder) noexcept = 0;
//!
//! \brief Get the ErrorRecorder assigned to this interface.
//!
//! Retrieves the assigned error recorder object for the given class. A default error recorder does not exist,
//! so a nullptr will be returned if setErrorRecorder has not been called, or an ErrorRecorder has not been
//! inherited.
//!
//! \return A pointer to the IErrorRecorder object that has been registered.
//!
//! \see setErrorRecorder()
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes
//!
virtual IErrorRecorder* getErrorRecorder() const noexcept = 0;
//!
//! \brief Return whether the parent registry will be searched if a plugin is not found in this registry
//! default: true
//!
//! \return bool variable indicating whether parent search is enabled.
//!
//! \see setParentSearchEnabled
//!
virtual bool isParentSearchEnabled() const = 0;
//!
//! \brief Set whether the parent registry will be searched if a plugin is not found in this registry.
//!
//! \param enabled The bool variable indicating whether parent search is enabled.
//!
//! \see isParentSearchEnabled
//!
virtual void setParentSearchEnabled(bool const enabled) = 0;
//!
//! \brief Load and register a shared library of plugins.
//!
//! \param pluginPath the plugin library path.
//!
//! \return The loaded plugin library handle. The call will fail and return
//! nullptr if any of the plugins are already registered.
//!
virtual PluginLibraryHandle loadLibrary(AsciiChar const* pluginPath) noexcept = 0;
//!
//! \brief Deregister plugins associated with a library. Any resources acquired when the library
//! was loaded will be released.
//!
//! \param handle the plugin library handle to deregister.
//!
virtual void deregisterLibrary(PluginLibraryHandle handle) noexcept = 0;
//!
//! \brief Register a plugin creator. Returns false if a plugin creator with the same type
//! is already registered.
//!
//! \warning The string pluginNamespace must be 1024 bytes or less including the NULL terminator and must be NULL
//! terminated.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes; calls to this method will be synchronized by a mutex.
//!
virtual bool registerCreator(IPluginCreatorInterface& creator, AsciiChar const* const pluginNamespace) noexcept = 0;
//!
//! \brief Return all registered plugin creators. Returns nullptr if none found.
//!
//! \warning If any plugin creators are registered or deregistered after calling this function, the returned pointer
//! is not guaranteed to be valid thereafter.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: No
//!
virtual IPluginCreatorInterface* const* getAllCreators(int32_t* const numCreators) const noexcept = 0;
//!
//! \brief Return a registered plugin creator based on plugin name, version, and namespace associated with the
//! plugin during network creation.
//!
//! \warning The strings pluginName, pluginVersion, and pluginNamespace must be 1024 bytes or less including the
//! NULL terminator and must be NULL terminated.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes
//!
virtual IPluginCreatorInterface* getCreator(AsciiChar const* const pluginName, AsciiChar const* const pluginVersion,
AsciiChar const* const pluginNamespace = "") noexcept = 0;
//!
//! \brief Deregister a previously registered plugin creator.
//!
//! Since there may be a desire to limit the number of plugins,
//! this function provides a mechanism for removing plugin creators registered in TensorRT.
//! The plugin creator that is specified by \p creator is removed from TensorRT and no longer tracked.
//!
//! \return True if the plugin creator was deregistered, false if it was not found in the registry or otherwise
//! could not be deregistered.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes
//!
virtual bool deregisterCreator(IPluginCreatorInterface const& creator) noexcept = 0;
//!
//! \brief Get a plugin resource
//! \param key Key for identifying the resource. Cannot be null.
//! \param resource A plugin resource object. The object will only need to be valid until this method returns, as
//! only a clone of this object will be registered by TRT. Cannot be null.
//!
//! \return Registered plugin resource object
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes; calls to this method will be synchronized by a mutex.
//!
virtual IPluginResource* acquirePluginResource(AsciiChar const* key, IPluginResource* resource) noexcept = 0;
//!
//! \brief Decrement reference count for the resource with this key
//! If reference count goes to zero after decrement, release() will be invoked on the resource, the key will
//! be deregistered and the resource object will be deleted
//!
//! \param key Key that was used to register the resource. Cannot be null.
//!
//! \return 0 for success, else non-zero
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes; calls to this method will be synchronized by a mutex.
//!
virtual int32_t releasePluginResource(AsciiChar const* key) noexcept = 0;
//!
//! \brief Return all registered plugin creators by searching starting from the current registry and following
//! parent registries recursively as long as isParentSearchEnabled() returns true.
//!
//! \param[out] numCreators Pointer to an integer where the number of registered plugin creators will be stored.
//!
//! \return A pointer to an array of IPluginCreatorInterface pointers. Returns nullptr if no creators are found.
//!
//! \warning If any plugin creators are registered or deregistered after calling this function, the returned pointer
//! is not guaranteed to remain valid.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: No
//!
virtual IPluginCreatorInterface* const* getAllCreatorsRecursive(int32_t* const numCreators) noexcept = 0;
};
inline IPluginRegistry::~IPluginRegistry() noexcept = default;
} // namespace nvinfer1
#endif /* NV_INFER_RUNTIME_COMMON_H */
include/NvInferRuntimePlugin.h
+925
View File
@@ -0,0 +1,925 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NV_INFER_RUNTIME_PLUGIN_H
#define NV_INFER_RUNTIME_PLUGIN_H
#define NV_INFER_INTERNAL_INCLUDE 1
#include "NvInferPluginBase.h"
#undef NV_INFER_INTERNAL_INCLUDE
//!
//! \file NvInferRuntimePlugin.h
//!
//! This file contains common definitions, data structures and interfaces that relate to plugins and are shared
//! between the standard and safe runtime.
//!
//! \warning Do not directly include this file. Instead include NvInferRuntime.h
//!
//!
//! \namespace nvinfer1
//!
//! \brief The TensorRT API version 1 namespace.
//!
namespace nvinfer1
{
enum class TensorFormat : int32_t;
namespace v_1_0
{
class IGpuAllocator;
} // namespace v_1_0
using IGpuAllocator = v_1_0::IGpuAllocator;
//!
//! \brief PluginFormat is reserved for backward compatibility.
//!
//! \see IPluginV2::supportsFormat()
//!
using PluginFormat = TensorFormat;
//!
//! \brief Bit at the plugin version to identify that it is a plugin.
//!
static constexpr int32_t kPLUGIN_VERSION_PYTHON_BIT = 0x40;
//!
//! \struct PluginTensorDesc
//!
//! \brief Fields that a plugin might see for an input or output.
//!
//! Scale is only valid when data type is DataType::kINT8. TensorRT will set
//! the value to -1.0F if it is invalid.
//!
//! \see IPluginV2IOExt::supportsFormatCombination
//! \see IPluginV2IOExt::configurePlugin
//!
struct PluginTensorDesc
{
//! Dimensions.
Dims dims;
//! \warning DataType:kBOOL and DataType::kUINT8 are not supported.
DataType type;
//! Tensor format.
TensorFormat format;
//! Scale for INT8 data type.
float scale;
};
//!
//! \struct PluginVersion
//!
//! \brief Definition of plugin versions.
//!
//! Tag for plug-in versions. Used in upper byte of getTensorRTVersion().
//!
//! \deprecated Deprecated in TensorRT 10.10. PluginVersion is used only in relation to IPluginV2-descendent plugin
//! interfaces, which are all deprecated.
//!
enum class PluginVersion : uint8_t
{
//! IPluginV2
kV2 TRT_DEPRECATED_ENUM = 0,
//! IPluginV2Ext
kV2_EXT TRT_DEPRECATED_ENUM = 1,
//! IPluginV2IOExt
kV2_IOEXT TRT_DEPRECATED_ENUM = 2,
//! IPluginV2DynamicExt
kV2_DYNAMICEXT TRT_DEPRECATED_ENUM = 3,
//! IPluginV2DynamicExt-based Python plugins
kV2_DYNAMICEXT_PYTHON TRT_DEPRECATED_ENUM = kPLUGIN_VERSION_PYTHON_BIT | 3
};
//!
//! \enum PluginCreatorVersion
//!
//! \brief Enum to identify version of the plugin creator.
//!
//! \deprecated Deprecated in TensorRT 10.10. PluginCreatorVersion is used only in relation to plugin creators based
//! off IPluginCreator, which is deprecated.
//!
enum class PluginCreatorVersion : int32_t
{
//! IPluginCreator
kV1 TRT_DEPRECATED_ENUM = 0,
//! IPluginCreator-based Python plugin creators
kV1_PYTHON TRT_DEPRECATED_ENUM = kPLUGIN_VERSION_PYTHON_BIT
};
//!
//! \class IPluginV2
//!
//! \brief Plugin class for user-implemented layers.
//!
//! Plugins are a mechanism for applications to implement custom layers. When
//! combined with IPluginCreator it provides a mechanism to register plugins and
//! look up the Plugin Registry during de-serialization.
//!
//! \see IPluginCreator
//! \see IPluginRegistry
//!
//! \deprecated Deprecated in TensorRT 8.5. Implement IPluginV3 instead.
//!
class TRT_DEPRECATED IPluginV2
{
public:
//!
//! \brief Return the API version with which this plugin was built.
//!
//! Do not override this method as it is used by the TensorRT library to maintain backwards-compatibility with
//! plugins.
//!
//! \return The TensorRT version in the format (major * 100 + minor) * 100 + patch.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, the implementation provided here is safe to call from any thread.
//!
virtual int32_t getTensorRTVersion() const noexcept
{
return NV_TENSORRT_VERSION;
}
//!
//! \brief Return the plugin type. Should match the plugin name returned by the corresponding plugin creator
//!
//! \see IPluginCreator::getPluginName()
//!
//! \warning The string returned must be NULL-terminated and have a length of 1024 bytes or less including the
//! NULL terminator.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual AsciiChar const* getPluginType() const noexcept = 0;
//!
//! \brief Return the plugin version. Should match the plugin version returned by the corresponding plugin creator
//!
//! \see IPluginCreator::getPluginVersion()
//!
//! \warning The string returned must be NULL-terminated and have a length of 1024 bytes or less including the
//! NULL terminator.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual AsciiChar const* getPluginVersion() const noexcept = 0;
//!
//! \brief Get the number of outputs from the layer.
//!
//! \return The number of outputs, which is a positive integer.
//!
//! This function is called by the implementations of INetworkDefinition and IBuilder. In particular, it is called
//! prior to any call to initialize().
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual int32_t getNbOutputs() const noexcept = 0;
//!
//! \brief Get the dimension of an output tensor.
//!
//! \param index The index of the output tensor. Will lie in the valid range (between 0 and getNbOutputs()-1
//! inclusive).
//! \param inputs The input tensor dimensions. Will be the start address of a Dims array of length nbInputDims.
//! \param nbInputDims The number of input tensors. Will be a non-negative integer.
//!
//! \return The output tensor dimensions if the index is in the valid range.
//! An invalid value of Dims{-1, {}} must be returned if the index is not in the valid range.
//!
//! This function is called by the implementations of INetworkDefinition and IBuilder. In particular, it is called
//! prior to any call to initialize().
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
//! \note In any non-IPluginV2DynamicExt plugin, batch size must not be included in the returned dimensions,
//! even if the plugin is expected to be run in a network with explicit batch mode enabled.
//! Please see the TensorRT Developer Guide for more details on how plugin inputs and outputs behave.
//!
virtual Dims getOutputDimensions(int32_t index, Dims const* inputs, int32_t nbInputDims) noexcept = 0;
//!
//! \brief Check format support.
//!
//! \param type DataType requested.
//! \param format PluginFormat requested.
//!
//! \return true if the plugin supports the type-format combination.
//!
//! This function is called by the implementations of INetworkDefinition, IBuilder, and
//! safe::ICudaEngine/ICudaEngine. In particular, it is called when creating an engine and when deserializing an
//! engine.
//!
//! \warning for the format field, the values PluginFormat::kCHW4, PluginFormat::kCHW16, and PluginFormat::kCHW32
//! will not be passed in, this is to keep backward compatibility with TensorRT 5.x series. Use PluginV2IOExt
//! or PluginV2DynamicExt for other PluginFormats.
//!
//! \warning DataType:kBOOL and DataType::kUINT8 are not supported.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual bool supportsFormat(DataType type, PluginFormat format) const noexcept = 0;
//!
//! \brief Configure the layer.
//!
//! This function is called by the builder prior to initialize(). It provides an opportunity for the layer to make
//! algorithm choices on the basis of its weights, dimensions, and maximum batch size.
//!
//! \param inputDims The input tensor dimensions. Will be the start address of a Dims array of length nbInputs.
//! \param nbInputs The number of inputs. Will be a non-negative integer.
//! \param outputDims The output tensor dimensions. Will be the start address of a Dims array of length nbOutputs.
//! \param nbOutputs The number of outputs. Will be a positive integer identical to the return value of
//! getNbOutputs().
//! \param type The data type selected for the engine.
//! \param format The format selected for the engine.
//! \param maxBatchSize The maximum batch size. Will be a positive integer.
//!
//! The dimensions passed here do not include the outermost batch size (i.e. for 2D image networks, they will be
//! 3-dimensional CHW dimensions).
//!
//! \warning for the format field, the values PluginFormat::kCHW4, PluginFormat::kCHW16, and PluginFormat::kCHW32
//! will not be passed in, this is to keep backward compatibility with TensorRT 5.x series. Use PluginV2IOExt
//! or PluginV2DynamicExt for other PluginFormats.
//!
//! \warning DataType:kBOOL and DataType::kUINT8 are not supported.
//!
//! \see clone()
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin. However, TensorRT
//! will not call this method from two threads simultaneously on a given clone of a plugin.
//!
virtual void configureWithFormat(Dims const* inputDims, int32_t nbInputs, Dims const* outputDims, int32_t nbOutputs,
DataType type, PluginFormat format, int32_t maxBatchSize) noexcept
= 0;
//!
//! \brief Initialize the layer for execution. This is called when the engine is created.
//!
//! \return 0 for success, else non-zero (which will cause engine termination).
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin or when using multiple
//! execution contexts using this plugin.
//!
virtual int32_t initialize() noexcept = 0;
//!
//! \brief Release resources acquired during plugin layer initialization. This is called when the engine is
//! destroyed.
//!
//! \see initialize()
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin or when using multiple
//! execution contexts using this plugin. However, TensorRT will not call this method from
//! two threads simultaneously on a given clone of a plugin.
//!
virtual void terminate() noexcept = 0;
//!
//! \brief Find the workspace size required by the layer.
//!
//! This function is called during engine startup, after initialize(). The workspace size returned must be
//! sufficient for any batch size up to the maximum.
//!
//! \param maxBatchSize The maximum batch size, which will be a positive integer.
//!
//! \return The workspace size in bytes, i.e. the device memory size that the plugin requires for its internal
//! computations.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin. However, TensorRT
//! will not call this method from two threads simultaneously on a given clone of a plugin.
//!
virtual size_t getWorkspaceSize(int32_t maxBatchSize) const noexcept = 0;
//!
//! \brief Execute the layer.
//!
//! \param batchSize The number of inputs in the batch.
//! \param inputs The memory for the input tensors. Will be an array of device addresses corresponding to input
//! tensors of length nbInputs, where nbInputs is the second parameter passed to configureWithFormat().
//! The i-th input tensor will have the dimensions inputDims[i], where inputDims is the first parameter
//! that was passed to configureWithFormat().
//! \param outputs The memory for the output tensors. Will be an array of device addresses corresponding to output
//! tensors of length getNbOutputs().
//! \param workspace Workspace for execution. Will be the start address of a device buffer whose length will be at
//! least getWorkspaceSize(batchSize).
//! \param stream The stream in which to execute the kernels. This will be a valid CUDA stream.
//!
//! \return 0 for success, else non-zero (which will cause engine termination).
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when multiple execution contexts are used during runtime.
//!
virtual int32_t enqueue(int32_t batchSize, void const* const* inputs, void* const* outputs, void* workspace,
cudaStream_t stream) noexcept
= 0;
//!
//! \brief Find the size of the serialization buffer required to store the plugin configuration in a binary file.
//!
//! \return The size of the serialization buffer in bytes.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual size_t getSerializationSize() const noexcept = 0;
//!
//! \brief Serialize the layer.
//!
//! \param buffer A pointer to a host buffer to serialize data. Size of buffer will be at least as large as the
//! value returned by getSerializationSize.
//!
//! \see getSerializationSize()
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual void serialize(void* buffer) const noexcept = 0;
//!
//! \brief Destroy the plugin object. This will be called when the network, builder or engine is destroyed.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual void destroy() noexcept = 0;
//!
//! \brief Clone the plugin object. This copies over internal plugin parameters and returns a new plugin object with
//! these parameters.
//!
//! The TensorRT runtime calls clone() to clone the plugin when an execution context is created for an engine,
//! after the engine has been created. The runtime does not call initialize() on the cloned plugin,
//! so the cloned plugin must be created in an initialized state.
//!
//! \return A cloned plugin object in an initialized state with the same parameters as the current object.
//! nullptr must be returned if the cloning fails, e.g. because of resource exhaustion.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin or when creating multiple
//! execution contexts.
//!
virtual IPluginV2* clone() const noexcept = 0;
//!
//! \brief Set the namespace that this plugin object belongs to. Ideally, all plugin
//! objects from the same plugin library must have the same namespace.
//!
//! \param pluginNamespace The namespace for the plugin object.
//!
//! \warning The string pluginNamespace will be NULL-terminated and have a length of 1024 bytes or less including the
//! NULL terminator.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual void setPluginNamespace(AsciiChar const* pluginNamespace) noexcept = 0;
//!
//! \brief Return the namespace of the plugin object.
//!
//! \return The namespace string that was passed to setPluginNamespace(), possibly after truncation to 1024 bytes
//! if a longer string was passed. An empty string must be returned as default value.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
// @cond SuppressDoxyWarnings
IPluginV2() = default;
virtual ~IPluginV2() noexcept = default;
// @endcond
protected:
// @cond SuppressDoxyWarnings
IPluginV2(IPluginV2 const&) = default;
IPluginV2(IPluginV2&&) = default;
IPluginV2& operator=(IPluginV2 const&) & = default;
IPluginV2& operator=(IPluginV2&&) & = default;
// @endcond
};
//!
//! \class IPluginV2Ext
//!
//! \brief Plugin class for user-implemented layers.
//!
//! Plugins are a mechanism for applications to implement custom layers. This
//! interface provides additional capabilities to the IPluginV2 interface by
//! supporting different output data types and broadcast across batches.
//!
//! \see IPluginV2
//!
//! \deprecated Deprecated in TensorRT 8.5. Implement IPluginV3 instead.
//!
class TRT_DEPRECATED IPluginV2Ext : public IPluginV2
{
public:
//!
//! \brief Return the DataType of the plugin output at the requested index.
//!
//! \param index The output tensor index in the valid range between 0 and getNbOutputs()-1.
//! \param inputTypes The data types of the input tensors, stored in an array of length nbInputs.
//! \param nbInputs The number of input tensors. Will be a non-negative integer.
//!
//! \return The data type of the output tensor with the provided index if the input tensors have the data types
//! provided in inputTypes, provided the output tensor index is in the valid range. DataType::kFLOAT must be
//! returned if the index is not in the valid range.
//!
//! The default behavior must be to return the type of the first input, or DataType::kFLOAT if the layer has no
//! inputs. The returned data type must have a format that is supported by the plugin.
//!
//! \see supportsFormat()
//!
//! \warning DataType:kBOOL and DataType::kUINT8 are not supported.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual nvinfer1::DataType getOutputDataType(
int32_t index, nvinfer1::DataType const* inputTypes, int32_t nbInputs) const noexcept
= 0;
//!
//! \brief Configure the layer with input and output data types.
//!
//! This function is called by the builder prior to initialize(). It provides an opportunity for the layer to make
//! algorithm choices on the basis of its weights, dimensions, data types and maximum batch size.
//!
//! \param inputDims The input tensor dimensions. Will be an array of length nbInputs.
//! \param nbInputs The number of inputs. Will be a non-negative integer.
//! \param outputDims The output tensor dimensions. Will be an array of length nbOutputs.
//! \param nbOutputs The number of outputs. Will be a positive integer.
//! \param inputTypes The data types selected for the plugin inputs. Will be an array of length nbInputs.
//! \param outputTypes The data types selected for the plugin outputs. Will be an array of length nbOutputs.
//! \param inputIsBroadcast True for each input that the plugin must broadcast across the batch.
//! Will be an array of length nbInputs.
//! \param outputIsBroadcast True for each output that TensorRT will broadcast across the batch.
//! Will be an array of length nbOutputs.
//! \param floatFormat The format selected for the engine for the floating point inputs/outputs.
//! \param maxBatchSize The maximum batch size. Will be a positive integer.
//!
//! The dimensions passed here do not include the outermost batch size (i.e. for 2D image networks, they will be
//! 3-dimensional CHW dimensions). When inputIsBroadcast or outputIsBroadcast is true, the outermost batch size for
//! that input or output must be treated as if it is one.
//! Index 'i' of inputIsBroadcast is true only if the input is semantically broadcast across the batch.
//! Index 'i' of outputIsBroadcast is true only if the output is semantically broadcast across the batch.
//!
//! \warning for the floatFormat field, the values PluginFormat::kCHW4, PluginFormat::kCHW16, and
//! PluginFormat::kCHW32 will not be passed in, this is to keep backward compatibility with TensorRT 5.x series. Use
//! PluginV2IOExt or PluginV2DynamicExt for other PluginFormats.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin. However, TensorRT
//! will not call this method from two threads simultaneously on a given clone of a plugin.
//!
virtual void configurePlugin(Dims const* inputDims, int32_t nbInputs, Dims const* outputDims, int32_t nbOutputs,
DataType const* inputTypes, DataType const* outputTypes, bool const* inputIsBroadcast,
bool const* outputIsBroadcast, PluginFormat floatFormat, int32_t maxBatchSize) noexcept
= 0;
IPluginV2Ext() = default;
~IPluginV2Ext() override = default;
//!
//! \brief Attach the plugin object to an execution context and grant the plugin the access to some context
//! resources.
//!
//! \param cudnn The cuDNN context handle of the execution context. Always nullptr.
//! \param cublas The cuBLAS context handle of the execution context. Always nullptr.
//! \param allocator The allocator used by the execution context
//!
//! This function is called automatically for each plugin when a new execution context is created. If the context
//! was created without resources, this method is not called until the resources are assigned. It is also called if
//! new resources are assigned to the context.
//!
//! If the plugin needs per-context resource, it can be allocated here.
//!
//! \note The cuDNN and cuBLAS handles are always nullptr. Plugins that need cuDNN or cuBLAS
//! should create their own handles.
//! The allocator pointer is unique to each building or execution context instance having overlapping lifetimes.
//! It can be used as a key to manage resources across plugin instances sharing the same context.
//! Plugins attached to different contexts will have different handles as their execution will not overlap.
//!
//! \see TacticSources
//! \see getPluginCudnnHandle(void* executionContextIdentifier)
//! \see getPluginCublasHandle(void* excecutionContextIdentifier)
//!
//! \note In the automotive safety context, the cuDNN and cuBLAS parameters will be nullptr because cuDNN and cuBLAS
//! are not used by the safe runtime.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual void attachToContext(
cudnnContext* /*cudnn*/, cublasContext* /*cublas*/, IGpuAllocator* /*allocator*/) noexcept
{
}
//!
//! \brief Detach the plugin object from its execution context.
//!
//! This function is called automatically for each plugin when an execution context is destroyed or the context
//! resources are unassigned from the context.
//!
//! If the plugin owns per-context resource, it can be released here.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual void detachFromContext() noexcept {}
//!
//! \brief Clone the plugin object. This copies over internal plugin parameters as well and returns a new plugin
//! object with these parameters. If the source plugin is pre-configured with configurePlugin(), the returned object
//! must also be pre-configured. The returned object must allow attachToContext() with a new execution context.
//! Cloned plugin objects can share the same per-engine immutable resource (e.g. weights) with the source object
//! (e.g. via ref-counting) to avoid duplication.
//!
//! \return A pointer to a cloned plugin object if cloning was successful, otherwise nullptr.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
IPluginV2Ext* clone() const noexcept override = 0;
protected:
// @cond SuppressDoxyWarnings
IPluginV2Ext(IPluginV2Ext const&) = default;
IPluginV2Ext(IPluginV2Ext&&) = default;
IPluginV2Ext& operator=(IPluginV2Ext const&) & = default;
IPluginV2Ext& operator=(IPluginV2Ext&&) & = default;
// @endcond
//!
//! \brief Return the API version with which this plugin was built. The
//! upper byte reserved by TensorRT and is used to differentiate this from IPluginV2.
//!
//! \return In the lower three bytes, the TensorRT version in the format
//! (major * 100 + minor) * 100 + patch.
//! In the upper byte, the value 1.
//!
//! Do not override this method as it is used by the TensorRT library to maintain backwards-compatibility with
//! plugins.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, the implementation provided here is safe to call from any thread.
//!
int32_t getTensorRTVersion() const noexcept override
{
return static_cast<int32_t>((static_cast<uint32_t>(PluginVersion::kV2_EXT) << 24U)
| (static_cast<uint32_t>(NV_TENSORRT_VERSION) & 0xFFFFFFU));
}
//!
//! \brief Derived classes must not implement this. In a C++11 API it would be final.
//!
//! IPluginV2Ext::configureWithFormat() is a NOP operation for all classes derived from IPluginV2Ext.
//! These classes call configurePlugin() instead.
//!
void configureWithFormat(Dims const* /*inputDims*/, int32_t /*nbInputs*/, Dims const* /*outputDims*/,
int32_t /*nbOutputs*/, DataType /*type*/, PluginFormat /*format*/, int32_t /*maxBatchSize*/) noexcept override
{
}
};
//!
//! \class IPluginV2IOExt
//!
//! \brief Plugin class for user-implemented layers.
//!
//! Plugins are a mechanism for applications to implement custom layers. This interface provides additional
//! capabilities to the IPluginV2Ext interface by extending different I/O data types and tensor formats.
//!
//! \see IPluginV2Ext
//!
//! \deprecated Deprecated in TensorRT 10.0. Implement IPluginV3 instead.
//!
class TRT_DEPRECATED IPluginV2IOExt : public IPluginV2Ext
{
public:
//!
//! \brief Configure the layer.
//!
//! This function is called by the builder prior to initialize(). It provides an opportunity for the layer to make
//! algorithm choices on the basis of the provided I/O PluginTensorDesc.
//!
//! \param in The input tensors attributes that are used for configuration.
//! \param nbInput Number of input tensors.
//! \param out The output tensors attributes that are used for configuration.
//! \param nbOutput Number of output tensors.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin. However, TensorRT
//! will not call this method from two threads simultaneously on a given clone of a plugin.
//!
virtual void configurePlugin(
PluginTensorDesc const* in, int32_t nbInput, PluginTensorDesc const* out, int32_t nbOutput) noexcept
= 0;
//!
//! \brief Return true if plugin supports the format and datatype for the input/output indexed by pos.
//!
//! For this method inputs are numbered 0..(nbInputs-1) and outputs are numbered nbInputs..(nbInputs+nbOutputs-1).
//! Using this numbering, pos is an index into InOut, where 0 <= pos < nbInputs+nbOutputs.
//!
//! TensorRT invokes this method to ask if the input/output indexed by pos supports the format/datatype specified
//! by inOut[pos].format and inOut[pos].type. The override must return true if that format/datatype at inOut[pos]
//! are supported by the plugin. If support is conditional on other input/output formats/datatypes, the plugin can
//! make its result conditional on the formats/datatypes in inOut[0..pos-1], which will be set to values
//! that the plugin supports. The override must not inspect inOut[pos+1..nbInputs+nbOutputs-1],
//! which will have invalid values. In other words, the decision for pos must be based on inOut[0..pos] only.
//!
//! Some examples:
//!
//! * A definition for a plugin that supports only FP16 NCHW:
//!
//! return inOut.format[pos] == TensorFormat::kLINEAR && inOut.type[pos] == DataType::kHALF;
//!
//! * A definition for a plugin that supports only FP16 NCHW for its two inputs,
//! and FP32 NCHW for its single output:
//!
//! return inOut.format[pos] == TensorFormat::kLINEAR &&
//! (inOut.type[pos] == (pos < 2 ? DataType::kHALF : DataType::kFLOAT));
//!
//! * A definition for a "polymorphic" plugin with two inputs and one output that supports
//! any format or type, but the inputs and output must have the same format and type:
//!
//! return pos == 0 || (inOut.format[pos] == inOut.format[0] && inOut.type[pos] == inOut.type[0]);
//!
//! Warning: TensorRT will stop asking for formats once it finds kFORMAT_COMBINATION_LIMIT on combinations.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin.
//!
virtual bool supportsFormatCombination(
int32_t pos, PluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) const noexcept
= 0;
// @cond SuppressDoxyWarnings
IPluginV2IOExt() = default;
~IPluginV2IOExt() override = default;
// @endcond
protected:
// @cond SuppressDoxyWarnings
IPluginV2IOExt(IPluginV2IOExt const&) = default;
IPluginV2IOExt(IPluginV2IOExt&&) = default;
IPluginV2IOExt& operator=(IPluginV2IOExt const&) & = default;
IPluginV2IOExt& operator=(IPluginV2IOExt&&) & = default;
// @endcond
//!
//! \brief Return the API version with which this plugin was built. The upper byte is reserved by TensorRT and is
//! used to differentiate this from IPluginV2 and IPluginV2Ext.
//!
//! Do not override this method as it is used by the TensorRT library to maintain backwards-compatibility with
//! plugins.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, the implementation provided here is safe to call from any thread.
//!
int32_t getTensorRTVersion() const noexcept override
{
return static_cast<int32_t>((static_cast<uint32_t>(PluginVersion::kV2_IOEXT) << 24U)
| (static_cast<uint32_t>(NV_TENSORRT_VERSION) & 0xFFFFFFU));
}
private:
// Following are obsolete base class methods, and must not be implemented or used.
//!
//! \brief Set plugin configuration.
//!
void configurePlugin(Dims const*, int32_t, Dims const*, int32_t, DataType const*, DataType const*, bool const*,
bool const*, PluginFormat, int32_t) noexcept final
{
}
//!
//! \brief Check if provided data type is supported.
//!
bool supportsFormat(DataType, PluginFormat) const noexcept final
{
return false;
}
};
namespace v_1_0
{
class TRT_DEPRECATED IPluginCreator : public IPluginCreatorInterface
{
public:
//!
//! \brief Return the plugin name.
//!
//! \warning The string returned must be NULL-terminated and have a length of 1024 bytes or less including
//! the NULL terminator.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin or when deserializing
//! multiple engines concurrently sharing plugins.
//!
virtual AsciiChar const* getPluginName() const noexcept = 0;
//!
//! \brief Return the plugin version.
//!
//! \warning The string returned must be NULL-terminated and have a length of 1024 bytes or less including
//! the NULL terminator.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin or when deserializing
//! multiple engines concurrently sharing plugins.
//!
virtual AsciiChar const* getPluginVersion() const noexcept = 0;
//!
//! \brief Return a list of fields that need to be passed to createPlugin.
//!
//! \see PluginFieldCollection
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin or when deserializing
//! multiple engines concurrently sharing plugins.
//!
virtual PluginFieldCollection const* getFieldNames() noexcept = 0;
//!
//! \brief Return a plugin object. Return nullptr in case of error.
//!
//! \param name A NULL-terminated name string of length 1024 or less, including the NULL terminator.
//! \param fc A pointer to a collection of fields needed for constructing the plugin.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin or when deserializing
//! multiple engines concurrently sharing plugins.
//!
virtual IPluginV2* createPlugin(AsciiChar const* name, PluginFieldCollection const* fc) noexcept = 0;
//!
//! \brief Called during deserialization of plugin layer. Return a plugin object.
//!
//! \param name A NULL-terminated name string of length 1024 or less, including the NULL terminator.
//! \param serialData The start address of a byte array with the serialized plugin representation.
//! \param serialLength The length in bytes of the byte array with the serialized plugin representation.
//!
//! \return A deserialized plugin object
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin or when deserializing
//! multiple engines concurrently sharing plugins.
//!
virtual IPluginV2* deserializePlugin(AsciiChar const* name, void const* serialData, size_t serialLength) noexcept
= 0;
//!
//! \brief Set the namespace of the plugin creator based on the plugin
//! library it belongs to. This can be set while registering the plugin creator.
//!
//! \param pluginNamespace A NULL-terminated namespace string of length 1024 or less, including the NULL terminator
//!
//! \see IPluginRegistry::registerCreator()
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin or when deserializing
//! multiple engines concurrently sharing plugins.
//!
virtual void setPluginNamespace(AsciiChar const* pluginNamespace) noexcept = 0;
//!
//! \brief Return the namespace of the plugin creator object.
//!
//! \warning The string returned must be NULL-terminated and have a length of 1024 bytes or less including the
//! NULL terminator.
//!
//! \usage
//! - Allowed context for the API call
//! - Thread-safe: Yes, this method is required to be thread-safe and may be called from multiple threads
//! when building networks on multiple devices sharing the same plugin or when deserializing
//! multiple engines concurrently sharing plugins.
//!
virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
IPluginCreator() = default;
~IPluginCreator() override = default;
protected:
// @cond SuppressDoxyWarnings
IPluginCreator(IPluginCreator const&) = default;
IPluginCreator(IPluginCreator&&) = default;
IPluginCreator& operator=(IPluginCreator const&) & = default;
IPluginCreator& operator=(IPluginCreator&&) & = default;
// @endcond
public:
//!
//! \brief Return version information associated with this interface. Applications must not override this method.
//!
InterfaceInfo getInterfaceInfo() const noexcept override
{
return InterfaceInfo{"PLUGIN CREATOR_V1", 1, 0};
}
};
} // namespace v_1_0
//!
//! \class IPluginCreator
//!
//! \brief Plugin creator class for user implemented layers.
//!
//! \see IPlugin and IPluginFactory
//!
//! \deprecated Deprecated in TensorRT 10.0. Please implement IPluginCreatorV3One
//! along with IPluginV3 plugins instead.
//!
using IPluginCreator = v_1_0::IPluginCreator;
} // namespace nvinfer1
#endif // NV_INFER_RUNTIME_PLUGIN_H
include/NvInferVersion.h
+45
View File
@@ -0,0 +1,45 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
//!
//! \file NvInferVersion.h
//!
//! Defines the TensorRT version
//!
#ifndef NV_INFER_VERSION_H
#define NV_INFER_VERSION_H
#define TRT_MAJOR_ENTERPRISE 11
#define TRT_MINOR_ENTERPRISE 1
#define TRT_PATCH_ENTERPRISE 0
#define TRT_BUILD_ENTERPRISE 106
#define NV_TENSORRT_MAJOR TRT_MAJOR_ENTERPRISE //!< TensorRT major version.
#define NV_TENSORRT_MINOR TRT_MINOR_ENTERPRISE //!< TensorRT minor version.
#define NV_TENSORRT_PATCH TRT_PATCH_ENTERPRISE //!< TensorRT patch version.
#define NV_TENSORRT_BUILD TRT_BUILD_ENTERPRISE //!< TensorRT build number.
#define NV_TENSORRT_LWS_MAJOR 0 //!< TensorRT LWS major version.
#define NV_TENSORRT_LWS_MINOR 0 //!< TensorRT LWS minor version.
#define NV_TENSORRT_LWS_PATCH 0 //!< TensorRT LWS patch version.
#define NV_TENSORRT_RELEASE_TYPE_EARLY_ACCESS 0 //!< An early access release
#define NV_TENSORRT_RELEASE_TYPE_RELEASE_CANDIDATE 1 //!< A release candidate
#define NV_TENSORRT_RELEASE_TYPE_GENERAL_AVAILABILITY 2 //!< A final release
#define NV_TENSORRT_RELEASE_TYPE NV_TENSORRT_RELEASE_TYPE_GENERAL_AVAILABILITY //!< TensorRT release type
#endif // NV_INFER_VERSION_H
include/NvOnnxConfig.h
+200
View File
@@ -0,0 +1,200 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NV_OnnxConfig_H
#define NV_OnnxConfig_H
#include "NvInfer.h"
namespace nvonnxparser
{
//!
//! \mainpage
//!
//! This is the API documentation for the Configuration Manager for Open Neural Network Exchange (ONNX) Parser for Nvidia TensorRT Inference Engine.
//! It provides information on individual functions, classes
//! and methods. Use the index on the left to navigate the documentation.
//!
//! Please see the accompanying user guide and samples for higher-level information and general advice on using ONNX Parser and TensorRT.
//!
//!
//! \file NvOnnxConfig.h
//!
//! This is the API file for the Configuration Manager for ONNX Parser for Nvidia TensorRT.
//!
//!
//! \class IOnnxConfig
//! \brief Configuration Manager Class.
//!
class IOnnxConfig
{
public:
virtual ~IOnnxConfig() noexcept = 0;
//!
//! \typedef Verbosity
//!
//! \brief Defines Verbosity level.
//!
typedef int32_t Verbosity;
//!
//! \brief Set the Model Data Type.
//!
//! Sets the Model DataType, one of the following: float -d 32 (default), half precision -d 16, and int8 -d 8 data
//! types.
//!
//! \see getModelDtype()
//!
virtual void setModelDtype(const nvinfer1::DataType) noexcept = 0;
//!
//! \brief Get the Model Data Type.
//!
//! \return the data type of the model.
//!
//! \see setModelDtype() and DataType
//!
virtual nvinfer1::DataType getModelDtype() const noexcept = 0;
//!
//! \brief Get the Model FileName.
//!
//! \return Return the Model Filename, as a null-terminated C-style string.
//!
//! \see setModelFileName()
//!
virtual char const* getModelFileName() const noexcept = 0;
//!
//! \brief Set the Model File Name.
//!
//! The Model File name contains the Network Description in ONNX pb format.
//!
//! This method copies the name string.
//!
//! \param onnxFilename The name.
//!
//! \see getModelFileName()
//!
virtual void setModelFileName(char const* onnxFilename) noexcept = 0;
//!
//! \brief Get the Verbosity Level.
//!
//! \return The Verbosity Level.
//!
//! \see addVerbosity(), reduceVerbosity()
//!
virtual Verbosity getVerbosityLevel() const noexcept = 0;
//!
//! \brief Increase the Verbosity Level.
//!
//! \return The Verbosity Level.
//!
//! \see reduceVerbosity(), setVerbosity(Verbosity)
//!
virtual void addVerbosity() noexcept = 0;
//!
//! \brief Reduce the Verbosity Level.
//!
//! \see addVerbosity(), setVerbosity(Verbosity)
//!
virtual void reduceVerbosity() noexcept = 0;
//!
//! \brief Set to specific verbosity Level.
//!
//! \see addVerbosity(), reduceVerbosity()
//!
virtual void setVerbosityLevel(Verbosity) noexcept = 0;
//!
//! \brief Returns the File Name of the Network Description as a Text File.
//!
//! \return Return the name of the file containing the network description converted to a plain text, used for
//! debugging purposes.
//!
//! \see setTextFilename()
//!
virtual char const* getTextFileName() const noexcept = 0;
//!
//! \brief Set the File Name of the Network Description as a Text File.
//!
//! This API allows setting a file name for the network description in plain text, equivalent of the ONNX protobuf.
//!
//! This method copies the name string.
//!
//! \param textFileName Name of the file.
//!
//! \see getTextFilename()
//!
virtual void setTextFileName(char const* textFileName) noexcept = 0;
//!
//! \brief Get the File Name of the Network Description as a Text File, including the weights.
//!
//! \return Return the name of the file containing the network description converted to a plain text, used for
//! debugging purposes.
//!
//! \see setFullTextFilename()
//!
virtual char const* getFullTextFileName() const noexcept = 0;
//!
//! \brief Set the File Name of the Network Description as a Text File, including the weights.
//!
//! This API allows setting a file name for the network description in plain text, equivalent of the ONNX protobuf.
//!
//! This method copies the name string.
//!
//! \param fullTextFileName Name of the file.
//!
//! \see getFullTextFilename()
//!
virtual void setFullTextFileName(char const* fullTextFileName) noexcept = 0;
//!
//! \brief Get whether the layer information will be printed.
//!
//! \return Returns whether the layer information will be printed.
//!
//! \see setPrintLayerInfo()
//!
virtual bool getPrintLayerInfo() const noexcept = 0;
//!
//! \brief Set whether the layer information will be printed.
//!
//! \see getPrintLayerInfo()
//!
virtual void setPrintLayerInfo(bool) noexcept = 0;
}; // class IOnnxConfig
inline IOnnxConfig::~IOnnxConfig() noexcept = default;
TENSORRTAPI IOnnxConfig* createONNXConfig();
} // namespace nvonnxparser
#endif
include/NvOnnxParser.h
+604
View File
@@ -0,0 +1,604 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef NV_ONNX_PARSER_H
#define NV_ONNX_PARSER_H
#include "NvInfer.h"
#include <stddef.h>
//!
//! \file NvOnnxParser.h
//!
//! This is the API for the ONNX Parser
//!
#define NV_ONNX_PARSER_MAJOR 0
#define NV_ONNX_PARSER_MINOR 1
#define NV_ONNX_PARSER_PATCH 0
static constexpr int32_t NV_ONNX_PARSER_VERSION
= ((NV_ONNX_PARSER_MAJOR * 10000) + (NV_ONNX_PARSER_MINOR * 100) + NV_ONNX_PARSER_PATCH);
//!
//! \namespace nvonnxparser
//!
//! \brief The TensorRT ONNX parser API namespace
//!
namespace nvonnxparser
{
//! \return the numerical value of the highest-valued enumerator for type T.
//! It must be specialized for each enum type that uses it.
template <typename T>
constexpr int32_t EnumMax() noexcept = delete;
//!
//! \enum ErrorCode
//!
//! \brief The type of error that the parser or refitter may return
//!
enum class ErrorCode : int
{
kSUCCESS = 0,
kINTERNAL_ERROR = 1,
kMEM_ALLOC_FAILED = 2,
kMODEL_DESERIALIZE_FAILED = 3,
kINVALID_VALUE = 4,
kINVALID_GRAPH = 5,
kINVALID_NODE = 6,
kUNSUPPORTED_GRAPH = 7,
kUNSUPPORTED_NODE = 8,
kUNSUPPORTED_NODE_ATTR = 9,
kUNSUPPORTED_NODE_INPUT = 10,
kUNSUPPORTED_NODE_DATATYPE = 11,
kUNSUPPORTED_NODE_DYNAMIC = 12,
kUNSUPPORTED_NODE_SHAPE = 13,
kREFIT_FAILED = 14
};
//! Specialization. See `nvonnxparser::EnumMax()` for details.
template <>
constexpr int32_t EnumMax<ErrorCode>() noexcept
{
return 14;
}
//!
//! \brief Represents one or more OnnxParserFlag values using binary OR
//! operations, e.g., 1U << OnnxParserFlag::kNATIVE_INSTANCENORM
//!
//! \see IParser::setFlags() and IParser::getFlags()
//!
using OnnxParserFlags
= uint32_t;
enum class OnnxParserFlag : int32_t
{
//! Parse the ONNX model into the INetworkDefinition with the intention of using TensorRT's native layer
//! implementation over the plugin implementation for InstanceNormalization nodes.
//! This flag is required when building version-compatible or hardware-compatible engines.
//! This flag is set to be ON by default.
kNATIVE_INSTANCENORM = 0,
//! Enable UINT8 as a quantization data type and asymmetric quantization with non-zero zero-point values
//! in Quantize and Dequantize nodes. This flag is set to be OFF by default.
//! The resulting engine must be built targeting DLA version >= 3.16.
kENABLE_UINT8_AND_ASYMMETRIC_QUANTIZATION_DLA = 1,
//! Parse the ONNX model with per-node validation for DLA. If the model is not fully supported by DLA, then
//! parsing will fail. If this flag is set, isSubGraphSupported() will also return capability in the context of DLA
//! support. When this flag is set, a valid IBuilderConfig must be provided to the parser via setBuilderConfig().
// This flag is set to be OFF by default.
kREPORT_CAPABILITY_DLA = 2,
//! Allow a loaded plugin with the same name as an ONNX operator type to override the default ONNX implementation,
//! even if the plugin namespace attribute is not set.
//! Useful for custom plugins that replace standard ONNX operators, such as alternative implementations for better
//! performance. This flag is set to be OFF by default.
kENABLE_PLUGIN_OVERRIDE = 3,
//! Opportunistically rewrite or modify layers to make them more amenable to running on DLA.
kADJUST_FOR_DLA = 4
};
//! Specialization. See `nvonnxparser::EnumMax()` for details.
template <>
constexpr int32_t EnumMax<OnnxParserFlag>() noexcept
{
return 4;
}
//!
//! \class IParserError
//!
//! \brief an object containing information about an error
//!
class IParserError
{
public:
//!
//!\brief the error code.
//!
virtual ErrorCode code() const = 0;
//!
//!\brief description of the error.
//!
virtual char const* desc() const = 0;
//!
//!\brief source file in which the error occurred.
//!
virtual char const* file() const = 0;
//!
//!\brief source line at which the error occurred.
//!
virtual int line() const = 0;
//!
//!\brief source function in which the error occurred.
//!
virtual char const* func() const = 0;
//!
//!\brief index of the ONNX model node in which the error occurred.
//!
virtual int node() const = 0;
//!
//!\brief name of the node in which the error occurred.
//!
virtual char const* nodeName() const = 0;
//!
//!\brief name of the node operation in which the error occurred.
//!
virtual char const* nodeOperator() const = 0;
//!
//!\brief A list of the local function names, from the top level down, constituting the current
//! stack trace in which the error occurred. A top-level node that is not inside any
//! local function would return a nullptr.
//!
virtual char const* const* localFunctionStack() const = 0;
//!
//!\brief The size of the stack of local functions at the point where the error occurred.
//! A top-level node that is not inside any local function would correspond to
// a stack size of 0.
//!
virtual int32_t localFunctionStackSize() const = 0;
protected:
virtual ~IParserError() {}
};
//!
//! \class IParser
//!
//! \brief an object for parsing ONNX models into a TensorRT network definition
//!
//! \warning If the ONNX model has a graph output with the same name as a graph input,
//! the output will be renamed by prepending "__".
//!
//! \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
//!
class IParser
{
public:
//!
//! \brief Parse a serialized ONNX model into the TensorRT network.
//! This method has very limited diagnostics. If parsing the serialized model
//! fails for any reason (e.g. unsupported IR version, unsupported opset, etc.)
//! it the user responsibility to intercept and report the error.
//! To obtain a better diagnostic, use the parseFromFile method below.
//!
//! \param serialized_onnx_model Pointer to the serialized ONNX model. Can be freed after this function returns.
//! \param serialized_onnx_model_size Size of the serialized ONNX model
//! in bytes
//! \param model_path Absolute path to the model file for loading external weights if required
//! \return true if the model was parsed successfully
//! \see getNbErrors() getError()
//!
virtual bool parse(
void const* serialized_onnx_model, size_t serialized_onnx_model_size, const char* model_path = nullptr) noexcept
= 0;
//!
//! \brief Parse an onnx model file, which can be a binary protobuf or a text onnx model
//! calls parse method inside.
//!
//! \param onnxModelFile name
//! \param verbosity Level
//!
//! \return true if the model was parsed successfully
//!
//!
virtual bool parseFromFile(char const* onnxModelFile, int verbosity) noexcept = 0;
//!
//!\brief Returns whether the specified operator may be supported by the
//! parser.
//!
//! Note that a result of true does not guarantee that the operator will be
//! supported in all cases (i.e., this function may return false-positives).
//!
//! \param op_name The name of the ONNX operator to check for support
//!
virtual bool supportsOperator(const char* op_name) const noexcept = 0;
//!
//!\brief Get the number of errors that occurred during prior calls to
//! \p parse
//!
//! \see getError() clearErrors() IParserError
//!
virtual int getNbErrors() const noexcept = 0;
//!
//!\brief Get an error that occurred during prior calls to \p parse
//!
//! \see getNbErrors() clearErrors() IParserError
//!
virtual IParserError const* getError(int index) const noexcept = 0;
//!
//!\brief Clear errors from prior calls to \p parse
//!
//! \see getNbErrors() getError() IParserError
//!
virtual void clearErrors() noexcept = 0;
virtual ~IParser() noexcept = default;
//!
//! \brief Query the plugin libraries needed to implement operations used by the parser in a version-compatible
//! engine.
//!
//! This provides a list of plugin libraries on the filesystem needed to implement operations
//! in the parsed network. If you are building a version-compatible engine using this network,
//! provide this list to IBuilderConfig::setPluginsToSerialize to serialize these plugins along
//! with the version-compatible engine, or, if you want to ship these plugin libraries externally
//! to the engine, ensure that IPluginRegistry::loadLibrary is used to load these libraries in the
//! appropriate runtime before deserializing the corresponding engine.
//!
//! \param[out] nbPluginLibs Returns the number of plugin libraries in the array, or -1 if there was an error.
//! \return Array of `nbPluginLibs` C-strings describing plugin library paths on the filesystem if nbPluginLibs > 0,
//! or nullptr otherwise. This array is owned by the IParser, and the pointers in the array are only valid until
//! the next call to parse() or parseFromFile().
//!
virtual char const* const* getUsedVCPluginLibraries(int64_t& nbPluginLibs) const noexcept = 0;
//!
//! \brief Set the parser flags.
//!
//! The flags are listed in the OnnxParserFlag enum.
//!
//! \param OnnxParserFlags The flags used when parsing an ONNX model.
//!
//! \note This function will override the previous set flags, rather than bitwise ORing the new flag.
//!
//! \see getFlags()
//!
virtual void setFlags(OnnxParserFlags onnxParserFlags) noexcept = 0;
//!
//! \brief Get the parser flags. Defaults to 0.
//!
//! \return The parser flags as a bitmask.
//!
//! \see setFlags()
//!
virtual OnnxParserFlags getFlags() const noexcept = 0;
//!
//! \brief clear a parser flag.
//!
//! clears the parser flag from the enabled flags.
//!
//! \see setFlags()
//!
virtual void clearFlag(OnnxParserFlag onnxParserFlag) noexcept = 0;
//!
//! \brief Set a single parser flag.
//!
//! Add the input parser flag to the already enabled flags.
//!
//! \see setFlags()
//!
virtual void setFlag(OnnxParserFlag onnxParserFlag) noexcept = 0;
//!
//! \brief Returns true if the parser flag is set
//!
//! \see getFlags()
//!
//! \return True if flag is set, false if unset.
//!
virtual bool getFlag(OnnxParserFlag onnxParserFlag) const noexcept = 0;
//!
//!\brief Return the i-th output ITensor object for the ONNX layer "name".
//!
//! Return the i-th output ITensor object for the ONNX layer "name".
//! If "name" is not found or i is out of range, return nullptr.
//! In the case of multiple nodes sharing the same name this function will return
//! the output tensors of the first instance of the node in the ONNX graph.
//!
//! \param name The name of the ONNX layer.
//!
//! \param i The index of the output. i must be in range [0, layer.num_outputs).
//!
virtual nvinfer1::ITensor const* getLayerOutputTensor(char const* name, int64_t i) noexcept = 0;
//!
//! \brief Check whether TensorRT supports a particular ONNX model.
//! If the function returns True, one can proceed to engine building
//! without having to call \p parse or \p parseFromFile.
//! Results can be queried through \p getNbSubgraphs, \p isSubgraphSupported,
//! \p getSubgraphNodes.
//!
//! \param serializedOnnxModel Pointer to the serialized ONNX model. Can be freed after this function returns.
//! \param serializedOnnxModelSize Size of the serialized ONNX model in bytes
//! \param modelPath Absolute path to the model file for loading external weights if required
//! \return true if the model is supported
//!
virtual bool supportsModelV2(
void const* serializedOnnxModel, size_t serializedOnnxModelSize, char const* modelPath = nullptr) noexcept = 0;
//!
//! \brief Get the number of subgraphs. Calling this function before calling \p supportsModelV2 results in undefined
//! behavior.
//!
//!
//! \return Number of subgraphs.
//!
virtual int64_t getNbSubgraphs() noexcept = 0;
//!
//! \brief Returns whether the subgraph is supported. Calling this function before calling \p supportsModelV2
//! results in undefined behavior.
//!
//!
//! \param index Index of the subgraph.
//! \return Whether the subgraph is supported.
//!
virtual bool isSubgraphSupported(int64_t const index) noexcept = 0;
//!
//! \brief Get the nodes of the specified subgraph. Calling this function before calling \p supportsModelV2 results
//! in undefined behavior.
//!
//!
//! \param index Index of the subgraph.
//! \param subgraphLength Returns the length of the subgraph as reference.
//!
//! \return Pointer to the subgraph nodes array. This pointer is owned by the Parser.
//!
virtual int64_t* getSubgraphNodes(int64_t const index, int64_t& subgraphLength) noexcept = 0;
//!
//! \brief Load a serialized ONNX model into the parser. Unlike the parse() or parseFromFile()
//! functions, this function does not immediately convert the model into a TensorRT
//! INetworkDefinition. Using this function allows users to provide their own initializers for the ONNX model
//! through the loadInitializer() function.
//!
//! Only one model can be loaded at a time. Subsequent calls to loadModelProto() will result in an error.
//!
//! To begin the conversion of the model into a TensorRT INetworkDefinition, use parseModelProto().
//!
//! \param serializedOnnxModel Pointer to the serialized ONNX model. Can be freed after this function returns.
//! \param serializedOnnxModelSize Size of the serialized ONNX model in bytes.
//! \param modelPath Absolute path to the model file for loading external weights if required.
//! \return true if the model was loaded successfully
//! \see getNbErrors() getError()
//!
virtual bool loadModelProto(
void const* serializedOnnxModel, size_t serializedOnnxModelSize, char const* modelPath = nullptr) noexcept = 0;
//!
//! \brief Prompt the ONNX parser to load an initializer with user-provided binary data.
//! The lifetime of the data must exceed the lifetime of the parser.
//!
//! All user-provided initializers must be provided prior to calling refitModelProto().
//!
//! This function can be called multiple times to specify the names of multiple initializers.
//!
//! Calling this function with an initializer previously specified will overwrite the previous instance.
//!
//!
//! This function will return false if initializer validation fails. Possible validation errors are:
//! * This function was called prior to loadModelProto().
//! * The requested initializer was not found in the model.
//! * The size of the data provided is different from the corresponding initializer in the model.
//!
//! \param name Name of the initializer.
//! \param data Binary data containing the values of the initializer.
//! \param size Size of the initializer in bytes.
//! \return true if the initializer was loaded successfully
//! \see loadModelProto()
//!
virtual bool loadInitializer(char const* name, void const* data, size_t size) noexcept = 0;
//! \brief Begin the parsing and conversion process of the loaded ONNX model into a TensorRT INetworkDefinition.
//!
//! \return true if conversion was successful
//! \see getNbErrors() getError() loadModelProto() loadModelProtoFromFile()
//!
virtual bool parseModelProto() noexcept = 0;
//!
//! \brief Set the BuilderConfig for the parser.
//!
//! \return true if the IBuilderConfig was set successfully, false otherwise.
//!
virtual bool setBuilderConfig(const nvinfer1::IBuilderConfig* const builderConfig) noexcept = 0;
};
//!
//! \class IParserRefitter
//!
//! \brief An interface designed to refit weights from an ONNX model.
//!
//! \warning Do not inherit from this class, as doing so will break forward-compatibility of the API and ABI.
//!
class IParserRefitter
{
public:
//!
//! \brief Load a serialized ONNX model from memory and perform weight refit.
//!
//! \param serializedOnnxModel Pointer to the serialized ONNX model
//! \param serializedOnnxModelSize Size of the serialized ONNX model
//! in bytes
//! \param modelPath Absolute path to the model file for loading external weights if required
//! \return true if all the weights in the engine were refit successfully.
//!
//! The serialized ONNX model must be identical to the one used to generate the engine
//! that will be refit.
//!
virtual bool refitFromBytes(
void const* serializedOnnxModel, size_t serializedOnnxModelSize, char const* modelPath = nullptr) noexcept
= 0;
//!
//! \brief Load and parse a ONNX model from disk and perform weight refit.
//!
//! \param onnxModelFile Path to the ONNX model to load from disk.
//!
//! \return true if the model was loaded successfully, and if all the weights in the engine were refit successfully.
//!
//! The provided ONNX model must be identical to the one used to generate the engine
//! that will be refit.
//!
virtual bool refitFromFile(char const* onnxModelFile) noexcept = 0;
//!
//!\brief Get the number of errors that occurred during prior calls to \p refitFromBytes or \p refitFromFile
//!
//! \see getError() IParserError
//!
virtual int32_t getNbErrors() const noexcept = 0;
//!
//!\brief Get an error that occurred during prior calls to \p refitFromBytes or \p refitFromFile
//!
//! \see getNbErrors() IParserError
//!
virtual IParserError const* getError(int32_t index) const noexcept = 0;
//!
//!\brief Clear errors from prior calls to \p refitFromBytes or \p refitFromFile
//!
//! \see getNbErrors() getError() IParserError
//!
virtual void clearErrors() = 0;
virtual ~IParserRefitter() noexcept = default;
//!
//! \brief Load a serialized ONNX model into the parser. Unlike the refit(), or refitFromFile()
//! functions, this function does not immediately begin the refit process. Using this function
//! allows users to provide their own initializers for the ONNX model through the loadInitializer() function.
//!
//! Only one model can be loaded at a time. Subsequent calls to loadModelProto() will result in an error.
//!
//! To begin the refit process, use refitModelProto().
//!
//! \param serializedOnnxModel Pointer to the serialized ONNX model. Can be freed after this function returns.
//! \param serializedOnnxModelSize Size of the serialized ONNX model in bytes.
//! \param modelPath Absolute path to the model file for loading external weights if required.
//! \return true if the model was loaded successfully
//! \see getNbErrors() getError()
//!
virtual bool loadModelProto(
void const* serializedOnnxModel, size_t serializedOnnxModelSize, char const* modelPath = nullptr) noexcept = 0;
//!
//! \brief Prompt the ONNX refitter to load an initializer with user-provided binary data.
//! The lifetime of the data must exceed the lifetime of the refitter.
//!
//! All user-provided initializers must be provided prior to calling refitModelProto().
//!
//! This function can be called multiple times to specify the names of multiple initializers.
//!
//! Calling this function with an initializer previously specified will overwrite the previous instance.
//!
//! This function will return false if initializer validation fails. Possible validation errors are:
//! * This function was called prior to loadModelProto()
//! * The requested initializer was not found in the model.
//! * The size of the data provided is different from the corresponding initializer in the model.
//!
//! \param name Name of the initializer.
//! \param data Binary data containing the values of the initializer.
//! \param size Size of the initializer in bytes.
//! \return true if the initializer was loaded successfully
//! \see loadModelProto()
//!
virtual bool loadInitializer(char const* name, void const* data, size_t size) noexcept = 0;
//! \brief Begin the refit process from the loaded ONNX model.
//!
//! \return true if refit was successful
//! \see getNbErrors() getError() loadModelProto()
//!
virtual bool refitModelProto() noexcept = 0;
};
} // namespace nvonnxparser
extern "C" TENSORRTAPI void* createNvOnnxParser_INTERNAL(void* network, void* logger, int version) noexcept;
extern "C" TENSORRTAPI void* createNvOnnxParserRefitter_INTERNAL(
void* refitter, void* logger, int32_t version) noexcept;
extern "C" TENSORRTAPI int getNvOnnxParserVersion() noexcept;
namespace nvonnxparser
{
namespace
{
//!
//! \brief Create a new parser object
//!
//! \param network The network definition that the parser will write to
//! \param logger The logger to use
//! \return a new parser object or NULL if an error occurred
//!
//! Any input dimensions that are constant should not be changed after parsing,
//! because correctness of the translation may rely on those constants.
//! Changing a dynamic input dimension, i.e. one that translates to -1 in
//! TensorRT, to a constant is okay if the constant is consistent with the model.
//! Each instance of the parser is designed to only parse one ONNX model once.
//!
//! \see IParser
//!
inline IParser* createParser(nvinfer1::INetworkDefinition& network, nvinfer1::ILogger& logger) noexcept
{
return static_cast<IParser*>(createNvOnnxParser_INTERNAL(&network, &logger, NV_ONNX_PARSER_VERSION));
}
//!
//! \brief Create a new ONNX refitter object
//!
//! \param refitter The Refitter object used to refit the model
//! \param logger The logger to use
//! \return a new ParserRefitter object or NULL if an error occurred
//!
//! \see IParserRefitter
//!
inline IParserRefitter* createParserRefitter(nvinfer1::IRefitter& refitter, nvinfer1::ILogger& logger) noexcept
{
return static_cast<IParserRefitter*>(
createNvOnnxParserRefitter_INTERNAL(&refitter, &logger, NV_ONNX_PARSER_VERSION));
}
} // namespace
} // namespace nvonnxparser
#endif // NV_ONNX_PARSER_H
include/impl/NvInferPythonPlugin.h
+603
View File
@@ -0,0 +1,603 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef TRT_PYTHON_IMPL_PLUGIN_H
#define TRT_PYTHON_IMPL_PLUGIN_H
#include "NvInfer.h"
//!
//! \file NvInferPythonPlugin.h
//!
//! This file contains definitions for supporting the `tensorrt.plugin` Python module
//!
//! \warning None of the defintions here are part of the TensorRT C++ API and may not follow semantic versioning rules.
//! TensorRT clients must not utilize them directly.
//!
namespace nvinfer1
{
//! \enum PluginArgType
//! \brief Numeric type of an extra kernel input argument in an AOT Python plugin
enum class PluginArgType : int32_t
{
//! Integer argument
kINT = 0,
};
//! \enum PluginArgDataType
//! \brief Data type of an extra kernel input argument in an AOT Python plugin
enum class PluginArgDataType : int32_t
{
//! 8-bit signed integer
kINT8 = 0,
//! 16-bit signed integer
kINT16 = 1,
//! 32-bit signed integer
kINT32 = 2,
};
//! \class ISymExpr
//! \brief Generic interface for a scalar symbolic expression implementable by a Python plugin / TensorRT Python backend
class ISymExpr
{
public:
//! \brief Get the type of the symbolic expression
virtual PluginArgType getType() const noexcept = 0;
//! \brief Get the data type of the symbolic expression
virtual PluginArgDataType getDataType() const noexcept = 0;
//! \brief Underlying symbolic expression
virtual void* getExpr() noexcept = 0;
};
//! Impl class for ISymExprs
class ISymExprsImpl
{
public:
virtual ISymExpr* getSymExpr(int32_t index) const noexcept = 0;
virtual bool setSymExpr(int32_t index, ISymExpr* symExpr) noexcept = 0;
virtual int32_t getNbSymExprs() const noexcept = 0;
virtual bool setNbSymExprs(int32_t count) noexcept = 0;
virtual ~ISymExprsImpl() noexcept = 0;
};
inline ISymExprsImpl::~ISymExprsImpl() noexcept = default;
//! \class ISymExprs
//! \brief Allows for a sequence of symbolic expressions to be communicated to the TensorRT backend
//! \note Clients must not implement this class.
//! \see ISymExpr
class ISymExprs
{
public:
//! \brief Get the symbolic expression at the given index
//! \return A pointer to the symbolic expression or nullptr if the index is out of range
ISymExpr* getSymExpr(int32_t index) const noexcept
{
return mImpl->getSymExpr(index);
}
//! \brief Set the symbolic expression at the given index
//! \return true if the index is in range and the symbolic expression was set successfully, false otherwise
bool setSymExpr(int32_t index, ISymExpr* symExpr) noexcept
{
return mImpl->setSymExpr(index, symExpr);
}
//! \brief Get the number of symbolic expressions
int32_t getNbSymExprs() const noexcept
{
return mImpl->getNbSymExprs();
}
//! \brief Set the number of symbolic expressions
//! \return true if the number of symbolic expressions was set successfully, false otherwise
bool setNbSymExprs(int32_t count) noexcept
{
return mImpl->setNbSymExprs(count);
}
protected:
ISymExprsImpl* mImpl{nullptr};
virtual ~ISymExprs() noexcept = 0;
};
inline ISymExprs::~ISymExprs() noexcept = default;
//! \enum QuickPluginCreationRequest
//! \brief Communicates preference when a quickly deployable plugin is to be added to the network
enum class QuickPluginCreationRequest : int32_t
{
//! No preference specified
kUNKNOWN = 0,
//! JIT plugin is preferred
kPREFER_JIT = 1,
//! AOT plugin is preferred
kPREFER_AOT = 2,
//! JIT plugin must be used. TensorRT should fail if a JIT implementation cannot be found.
kSTRICT_JIT = 3,
//! AOT plugin must be used. TensorRT should fail if an AOT implementation cannot be found.
kSTRICT_AOT = 4,
};
//! Impl class for IKernelLaunchParams
class IKernelLaunchParamsImpl
{
public:
virtual ISymExpr* getGridX() noexcept = 0;
virtual bool setGridX(ISymExpr* gridX) noexcept = 0;
virtual ISymExpr* getGridY() noexcept = 0;
virtual bool setGridY(ISymExpr* gridY) noexcept = 0;
virtual ISymExpr* getGridZ() noexcept = 0;
virtual bool setGridZ(ISymExpr* gridZ) noexcept = 0;
virtual ISymExpr* getBlockX() noexcept = 0;
virtual bool setBlockX(ISymExpr* blockX) noexcept = 0;
virtual ISymExpr* getBlockY() noexcept = 0;
virtual bool setBlockY(ISymExpr* blockY) noexcept = 0;
virtual ISymExpr* getBlockZ() noexcept = 0;
virtual bool setBlockZ(ISymExpr* blockZ) noexcept = 0;
virtual ISymExpr* getSharedMem() noexcept = 0;
virtual bool setSharedMem(ISymExpr* sharedMem) noexcept = 0;
virtual ~IKernelLaunchParamsImpl() noexcept = 0;
};
inline IKernelLaunchParamsImpl::~IKernelLaunchParamsImpl() noexcept = default;
//! \class IKernelLaunchParams
//! \brief Allows for kernel launch parameters to be communicated to the TensorRT backend
//! \note Clients must not implement this class.
class IKernelLaunchParams
{
public:
//! Get the X dimension of the grid
ISymExpr* getGridX() noexcept
{
return mImpl->getGridX();
}
//! \brief Set the X dimension of the grid
//! \return true if the grid's X dimension was set successfully, false otherwise
bool setGridX(ISymExpr* gridX) noexcept
{
return mImpl->setGridX(gridX);
}
//! Get the Y dimension of the grid
ISymExpr* getGridY() noexcept
{
return mImpl->getGridY();
}
//! \brief Set the Y dimension of the grid
//! \return true if the grid's Y dimension was set successfully, false otherwise
bool setGridY(ISymExpr* gridY) noexcept
{
return mImpl->setGridY(gridY);
}
//! Get the Z dimension of the grid
ISymExpr* getGridZ() noexcept
{
return mImpl->getGridZ();
}
//! \brief Set the Z dimension of the grid
//! \return true if the grid's Z dimension was set successfully, false otherwise
bool setGridZ(ISymExpr* gridZ) noexcept
{
return mImpl->setGridZ(gridZ);
}
//! \brief Get the X dimension of each thread block
ISymExpr* getBlockX() noexcept
{
return mImpl->getBlockX();
}
//! \brief Set the X dimension of each thread block
//! \return true if each thread block's X dimension was set successfully, false otherwise
bool setBlockX(ISymExpr* blockX) noexcept
{
return mImpl->setBlockX(blockX);
}
//! \brief Get the Y dimension of each thread block
ISymExpr* getBlockY() noexcept
{
return mImpl->getBlockY();
}
//! \brief Set the Y dimension of each thread block
//! \return true if each thread block's Y dimension was set successfully, false otherwise
bool setBlockY(ISymExpr* blockY) noexcept
{
return mImpl->setBlockY(blockY);
}
//! \brief Get the Z dimension of each thread block
ISymExpr* getBlockZ() noexcept
{
return mImpl->getBlockZ();
}
//! \brief Set the Z dimension of each thread block
//! \return true if each thread block's Z dimension was set successfully, false otherwise
bool setBlockZ(ISymExpr* blockZ) noexcept
{
return mImpl->setBlockZ(blockZ);
}
//! \brief Get the dynamic shared-memory per thread block in bytes
ISymExpr* getSharedMem() noexcept
{
return mImpl->getSharedMem();
}
//! \brief Set the dynamic shared-memory per thread block in bytes
//! \return true if the dynamic shared-memory per thread block was set successfully, false otherwise
bool setSharedMem(ISymExpr* sharedMem) noexcept
{
return mImpl->setSharedMem(sharedMem);
}
protected:
IKernelLaunchParamsImpl* mImpl{nullptr};
virtual ~IKernelLaunchParams() noexcept = 0;
};
inline IKernelLaunchParams::~IKernelLaunchParams() noexcept = default;
namespace v_1_0
{
class IPluginV3QuickCore : public IPluginCapability
{
public:
InterfaceInfo getInterfaceInfo() const noexcept override
{
return InterfaceInfo{"PLUGIN_V3QUICK_CORE", 1, 0};
}
virtual AsciiChar const* getPluginName() const noexcept = 0;
virtual AsciiChar const* getPluginVersion() const noexcept = 0;
virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
};
class IPluginV3QuickBuild : public IPluginCapability
{
public:
InterfaceInfo getInterfaceInfo() const noexcept override
{
return InterfaceInfo{"PLUGIN_V3QUICK_BUILD", 1, 0};
}
//!
//! \brief Provide the data types of the plugin outputs if the input tensors have the data types provided.
//!
//! \param outputTypes Pre-allocated array to which the output data types should be written.
//! \param nbOutputs The number of output tensors. This matches the value returned from getNbOutputs().
//! \param inputTypes The input data types.
//! \param inputRanks Ranks of the input tensors
//! \param nbInputs The number of input tensors.
//!
//! \return 0 for success, else non-zero
//!
virtual int32_t getOutputDataTypes(DataType* outputTypes, int32_t nbOutputs, DataType const* inputTypes,
int32_t const* inputRanks, int32_t nbInputs) const noexcept = 0;
//!
//! \brief Provide expressions for computing dimensions of the output tensors from dimensions of the input tensors.
//!
//! \param inputs Expressions for dimensions of the input tensors
//! \param nbInputs The number of input tensors
//! \param shapeInputs Expressions for values of the shape tensor inputs
//! \param nbShapeInputs The number of shape tensor inputs
//! \param outputs Pre-allocated array to which the output dimensions must be written
//! \param exprBuilder Object for generating new dimension expressions
//!
//! \return 0 for success, else non-zero
//!
virtual int32_t getOutputShapes(DimsExprs const* inputs, int32_t nbInputs, DimsExprs const* shapeInputs,
int32_t nbShapeInputs, DimsExprs* outputs, int32_t nbOutputs, IExprBuilder& exprBuilder) noexcept = 0;
//!
//! \brief Configure the plugin. Behaves similarly to `IPluginV3OneBuild::configurePlugin()`
//!
//! \return 0 for success, else non-zero
//!
virtual int32_t configurePlugin(DynamicPluginTensorDesc const* in, int32_t nbInputs,
DynamicPluginTensorDesc const* out, int32_t nbOutputs) noexcept = 0;
//!
//! \brief Get number of format combinations supported by the plugin for the I/O characteristics indicated by
//! `inOut`.
//!
virtual int32_t getNbSupportedFormatCombinations(
DynamicPluginTensorDesc const* inOut, int32_t nbInputs, int32_t nbOutputs) noexcept = 0;
//!
//! \brief Write all format combinations supported by the plugin for the I/O characteristics indicated by `inOut` to
//! `supportedCombinations`. It is guaranteed to have sufficient memory allocated for (nbInputs + nbOutputs) *
//! getNbSupportedFormatCombinations() `PluginTensorDesc`s.
//!
//! \return 0 for success, else non-zero
//!
virtual int32_t getSupportedFormatCombinations(DynamicPluginTensorDesc const* inOut, int32_t nbInputs,
int32_t nbOutputs, PluginTensorDesc* supportedCombinations, int32_t nbFormatCombinations) noexcept = 0;
//!
//! \brief Get the number of outputs from the plugin.
//!
virtual int32_t getNbOutputs() const noexcept = 0;
//!
//! \brief Communicates to TensorRT that the output at the specified output index is aliased to the input at the
//! returned index. Behaves similary to `v_2_0::IPluginV3OneBuild.getAliasedInput()`.
//!
virtual int32_t getAliasedInput(int32_t outputIndex) noexcept
{
return -1;
}
//!
//! \brief Query for any custom tactics that the plugin intends to use specific to the I/O characteristics indicated
//! by the immediately preceding call to `configurePlugin()`.
//!
//! \return 0 for success, else non-zero
//!
virtual int32_t getValidTactics(int32_t* tactics, int32_t nbTactics) noexcept
{
return 0;
}
//!
//! \brief Query for number of custom tactics related to the `getValidTactics()` call.
//!
virtual int32_t getNbTactics() noexcept
{
return 0;
}
//!
//! \brief Called to query the suffix to use for the timing cache ID. May be called anytime after plugin creation.
//!
virtual char const* getTimingCacheID() noexcept
{
return nullptr;
}
//!
//! \brief Query for a string representing the configuration of the plugin. May be called anytime after
//! plugin creation.
//!
virtual char const* getMetadataString() noexcept
{
return nullptr;
}
};
class IPluginV3QuickAOTBuild : public IPluginV3QuickBuild
{
public:
InterfaceInfo getInterfaceInfo() const noexcept override
{
return InterfaceInfo{"PLUGIN_V3QUICKAOT_BUILD", 1, 0};
}
//! \brief Get the launch parameters for the kernel to be used for the specified input and output types/formats and
//! any corresponding custom tactics.
//! If custom tactics are being advertised by the plugin, the corresponding tactic is the one specified by
//! the immediately preceding call to setTactic().
//!
//! \param inputs Expressions for dimensions of the input tensors
//! \param inOut The input and output tensors' attributes
//! \param nbInputs The number of input tensors
//! \param nbOutputs The number of output tensors
//! \param launchParams Interface which allows the specification of kernel launch parameters as symbolic expressions
//! of the input dimensions
//! \param extraArgs Interface which allows the specification of any scalar arguments to be
//! passed to the kernel, as symbolic expressions of the input dimensions
//! \param exprBuilder Object for generating new symbolic expressions
//!
//! \return 0 for success, else non-zero
//!
virtual int32_t getLaunchParams(DimsExprs const* inputs, DynamicPluginTensorDesc const* inOut, int32_t nbInputs,
int32_t nbOutputs, IKernelLaunchParams* launchParams, ISymExprs* extraArgs,
IExprBuilder& exprBuilder) noexcept = 0;
//!
//! \brief Get the compiled form for the kernel to be used for the specified input and output types/formats and any
//! corresponding custom tactics.
//! If custom tactics are being advertised by the plugin, the corresponding tactic is the one specified by
//! the immediately preceding call to setTactic().
//!
//! \param in The input tensors' attributes that are used for configuration.
//! \param nbInputs Number of input tensors.
//! \param out The output tensors' attributes that are used for configuration.
//! \param nbOutputs Number of output tensors.
//! \param kernelName The name for the kernel.
//! \param compiledKernel Compiled form of the kernel.
//! \param compiledKernelSize The size of the compiled kernel.
//!
//! \return 0 for success, else non-zero
//!
virtual int32_t getKernel(PluginTensorDesc const* in, int32_t nbInputs, PluginTensorDesc const* out,
int32_t nbOutputs, const char** kernelName, char** compiledKernel, int32_t* compiledKernelSize) noexcept = 0;
//!
//! \brief Set the tactic to be used in the subsequent call to enqueue(). Behaves similar to
//! IPluginV3OneRuntime::setTactic()
//!
//! \return 0 for success, else non-zero
//!
virtual int32_t setTactic(int32_t tactic) noexcept
{
return 0;
}
};
class IPluginV3QuickRuntime : public IPluginCapability
{
public:
InterfaceInfo getInterfaceInfo() const noexcept override
{
return InterfaceInfo{"PLUGIN_V3QUICK_RUNTIME", 1, 0};
}
//!
//! \brief Set the tactic to be used in the subsequent call to enqueue(). Behaves similar to
//! `IPluginV3OneRuntime::setTactic()`.
//!
//! \return 0 for success, else non-zero
//!
virtual int32_t setTactic(int32_t tactic) noexcept
{
return 0;
}
//!
//! \brief Execute the plugin.
//!
//! \param inputDesc how to interpret the memory for the input tensors.
//! \param outputDesc how to interpret the memory for the output tensors.
//! \param inputs The memory for the input tensors.
//! \param inputStrides Strides for input tensors.
//! \param outputStrides Strides for output tensors.
//! \param outputs The memory for the output tensors.
//! \param nbInputs Number of input tensors.
//! \param nbOutputs Number of output tensors.
//! \param stream The stream in which to execute the kernels.
//!
//! \return 0 for success, else non-zero
//!
virtual int32_t enqueue(PluginTensorDesc const* inputDesc, PluginTensorDesc const* outputDesc,
void const* const* inputs, void* const* outputs, Dims const* inputStrides, Dims const* outputStrides,
int32_t nbInputs, int32_t nbOutputs, cudaStream_t stream) noexcept = 0;
//!
//! \brief Get the plugin fields which should be serialized.
//!
virtual PluginFieldCollection const* getFieldsToSerialize() noexcept = 0;
};
class IPluginCreatorV3Quick : public IPluginCreatorInterface
{
public:
InterfaceInfo getInterfaceInfo() const noexcept override
{
return InterfaceInfo{"PLUGIN CREATOR_V3QUICK", 1, 0};
}
//!
//! \brief Return a plugin object. Return nullptr in case of error.
//!
//! \param name A NULL-terminated name string of length 1024 or less, including the NULL terminator.
//! \param namespace A NULL-terminated name string of length 1024 or less, including the NULL terminator.
//! \param fc A pointer to a collection of fields needed for constructing the plugin.
//! \param phase The TensorRT phase in which the plugin is being created
//! \param quickPluginCreationRequest Whether a JIT or AOT plugin should be created
//!
virtual IPluginV3* createPlugin(AsciiChar const* name, AsciiChar const* nspace, PluginFieldCollection const* fc,
TensorRTPhase phase, QuickPluginCreationRequest quickPluginCreationRequest) noexcept = 0;
//!
//! \brief Return a list of fields that need to be passed to createPlugin() when creating a plugin for use in the
//! TensorRT build phase.
//!
virtual PluginFieldCollection const* getFieldNames() noexcept = 0;
virtual AsciiChar const* getPluginName() const noexcept = 0;
virtual AsciiChar const* getPluginVersion() const noexcept = 0;
virtual AsciiChar const* getPluginNamespace() const noexcept = 0;
IPluginCreatorV3Quick() = default;
virtual ~IPluginCreatorV3Quick() = default;
protected:
IPluginCreatorV3Quick(IPluginCreatorV3Quick const&) = default;
IPluginCreatorV3Quick(IPluginCreatorV3Quick&&) = default;
IPluginCreatorV3Quick& operator=(IPluginCreatorV3Quick const&) & = default;
IPluginCreatorV3Quick& operator=(IPluginCreatorV3Quick&&) & = default;
};
} // namespace v_1_0
//!
//! \class IPluginV3QuickCore
//!
//! \brief Provides core capability (`IPluginCapability::kCORE`) for quickly-deployable TRT plugins
//!
//! \warning This class is strictly for the purpose of supporting quickly-deployable TRT Python plugins and is not part
//! of the public TensorRT C++ API. Users must not inherit from this class.
//!
using IPluginV3QuickCore = v_1_0::IPluginV3QuickCore;
//!
//! \class IPluginV3QuickBuild
//!
//! \brief Provides build capability (`IPluginCapability::kBUILD`) for quickly-deployable TRT plugins
//!
//! \warning This class is strictly for the purpose of supporting quickly-deployable TRT Python plugins and is not part
//! of the public TensorRT C++ API. Users must not inherit from this class.
//!
using IPluginV3QuickBuild = v_1_0::IPluginV3QuickBuild;
//!
//! \class IPluginV3QuickAOTBuild
//!
//! \brief Provides additional build capabilities for AOT quickly-deployable TRT plugins. Descends from
//! IPluginV3QuickBuild.
//!
//! \warning This class is strictly for the purpose of supporting quickly-deployable TRT Python plugins and is not part
//! of the public TensorRT C++ API. Users must not inherit from this class.
//!
using IPluginV3QuickAOTBuild = v_1_0::IPluginV3QuickAOTBuild;
//!
//! \class IPluginV3QuickRuntime
//!
//! \brief Provides runtime capability (`IPluginCapability::kRUNTIME`) for JIT quickly-deployable TRT plugins
//!
//! \warning This class is strictly for the purpose of supporting quickly-deployable TRT Python plugins and is not part
//! of the public TensorRT C++ API. Users must not inherit from this class.
//!
using IPluginV3QuickRuntime = v_1_0::IPluginV3QuickRuntime;
//!
//! \class IPluginCreatorV3Quick
//!
//! \warning This class is strictly for the purpose of supporting quickly-deployable TRT Python plugins and is not part
//! of the public TensorRT C++ API. Users must not inherit from this class.
//!
using IPluginCreatorV3Quick = v_1_0::IPluginCreatorV3Quick;
} // namespace nvinfer1
#endif // TRT_PYTHON_IMPL_PLUGIN_H