nvidia--tensorrt/samples/common/sampleInference.h

/*
 * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef TRT_SAMPLE_INFERENCE_H
#define TRT_SAMPLE_INFERENCE_H

#include "debugTensorWriter.h"
#include "sampleDevice.h"
#include "sampleEngines.h"
#include "sampleReporting.h"
#include "sampleUtils.h"

#include <functional>
#include <iostream>
#include <list>
#include <memory>
#include <string>
#include <vector>

#if ENABLE_UNIFIED_BUILDER
#include "safeCudaAllocator.h"
#endif
namespace sample
{
using LibraryPtr = std::unique_ptr<samplesCommon::DynamicLibrary>;

std::string const TRT_NVINFER_NAME = "nvinfer";
std::string const TRT_ONNXPARSER_NAME = "nvonnxparser";
std::string const TRT_LIB_SUFFIX = "";

#if !TRT_STATIC
#if defined(_WIN32)
std::string const kNVINFER_PLUGIN_LIBNAME
    = std::string{"nvinfer_plugin_"} + std::to_string(NV_TENSORRT_MAJOR) + std::string{".dll"};
std::string const kNVINFER_LIBNAME = std::string(TRT_NVINFER_NAME) + std::string{"_"}
    + std::to_string(NV_TENSORRT_MAJOR) + TRT_LIB_SUFFIX + std::string{".dll"};
std::string const kNVINFER_SAFE_LIBNAME
    = std::string{"nvinfer_safe_"} + std::to_string(NV_TENSORRT_MAJOR) + std::string{".dll"};
std::string const kNVONNXPARSER_LIBNAME = std::string(TRT_ONNXPARSER_NAME) + std::string{"_"}
    + std::to_string(NV_TENSORRT_MAJOR) + TRT_LIB_SUFFIX + std::string{".dll"};
std::string const kNVINFER_LEAN_LIBNAME
    = std::string{"nvinfer_lean_"} + std::to_string(NV_TENSORRT_MAJOR) + std::string{".dll"};
std::string const kNVINFER_DISPATCH_LIBNAME
    = std::string{"nvinfer_dispatch_"} + std::to_string(NV_TENSORRT_MAJOR) + std::string{".dll"};
#else
std::string const kNVINFER_PLUGIN_LIBNAME = std::string{"libnvinfer_plugin.so."} + std::to_string(NV_TENSORRT_MAJOR);
std::string const kNVINFER_LIBNAME
    = std::string{"lib"} + std::string(TRT_NVINFER_NAME) + std::string{".so."} + std::to_string(NV_TENSORRT_MAJOR);
std::string const kNVINFER_SAFE_LIBNAME = std::string{"libnvinfer_safe.so."} + std::to_string(NV_TENSORRT_MAJOR);
std::string const kNVONNXPARSER_LIBNAME
    = std::string{"lib"} + std::string(TRT_ONNXPARSER_NAME) + std::string{".so."} + std::to_string(NV_TENSORRT_MAJOR);
std::string const kNVINFER_LEAN_LIBNAME = std::string{"libnvinfer_lean.so."} + std::to_string(NV_TENSORRT_MAJOR);
std::string const kNVINFER_DISPATCH_LIBNAME
    = std::string{"libnvinfer_dispatch.so."} + std::to_string(NV_TENSORRT_MAJOR);
#endif

std::string const& getRuntimeLibraryName(RuntimeMode const mode);

template <typename FetchPtrs>
bool initLibrary(LibraryPtr& libPtr, std::string const& libName, FetchPtrs fetchFunc)
{
    if (libPtr != nullptr)
    {
        return true;
    }
    try
    {
        libPtr.reset(new samplesCommon::DynamicLibrary{libName});
        fetchFunc(libPtr.get());
    }
    catch (std::exception const& e)
    {
        libPtr.reset();
        sample::gLogError << "Could not load library " << libName << ": " << e.what() << std::endl;
        return false;
    }
    catch (...)
    {
        libPtr.reset();
        sample::gLogError << "Could not load library " << libName << std::endl;
        return false;
    }

    return true;
}
#endif // !TRT_STATIC

#if ENABLE_UNIFIED_BUILDER
namespace safe
{

//!
//! \brief Initialize the NVIDIA Inference Safe Runtime library
//!
//! This function dynamically loads the Safe TensorRT runtime library and initializes
//! function pointers for safe TensorRT operations. It is used to set up the safe runtime
//! environment for inference with safety-certified TensorRT engines.
//!
//! \return true if the safe runtime library was successfully loaded and initialized,
//!         false otherwise (e.g., in static builds or if library loading fails)
//!
bool initNvinferSafe();

//!
//! \brief Create a safe TRT graph from serialized engine data
//!
//! This function creates a safe TRT graph from serialized engine data. It is used to create
//! a safe TRT graph for inference with safety-certified TensorRT engines.
//!
//! \param graph: Pointer to the safe TRT graph to be created
//! \param blob: Pointer to the serialized engine data
//! \param size: Size of the serialized engine data
//! \param recorder: Reference to the safe recorder
//! \param useManaged: Flag indicating whether to use managed memory
//! \param allocator: Pointer to the safe memory allocator
//! \return Error code indicating the success or failure of the operation
//!
nvinfer1::ErrorCode createSafeTRTGraph(nvinfer2::safe::ITRTGraph*& graph, void const* blob, int64_t size,
    ISafeRecorder& recorder, bool useManaged, ISafeMemAllocator* allocator);

//!
//! \brief Destroy a safe TRT graph and release resources
//!
//! This function destroys a safe TRT graph and releases the associated resources. It is used to clean up
//! the safe TRT graph after inference with safety-certified TensorRT engines.
//!
//! \param graph: Pointer to the safe TRT graph to be destroyed
//! \return Error code indicating the success or failure of the operation
//!
nvinfer1::ErrorCode destroySafeTRTGraph(nvinfer2::safe::ITRTGraph*& graph);

//!
//! \brief Get the safe plugin registry for loading plugins
//!
//! This function retrieves the safe plugin registry for loading plugins. It is used to get the safe plugin registry
//! for loading plugins with safety-certified TensorRT engines.
//!
//! \param recorder: Reference to the safe recorder
//! \return Pointer to the safe plugin registry
//!
nvinfer2::safe::ISafePluginRegistry* getSafePluginRegistry(ISafeRecorder& recorder);
} // namespace safe
#endif

struct InferenceEnvironmentBase
{
    InferenceEnvironmentBase() = delete;
    virtual ~InferenceEnvironmentBase() = default;
    InferenceEnvironmentBase(InferenceEnvironmentBase const& other) = delete;
    InferenceEnvironmentBase(InferenceEnvironmentBase&& other) = delete;
    InferenceEnvironmentBase(BuildEnvironment& bEnv)
        : engine(std::move(bEnv.engine))
        , safe(bEnv.engine.isSafe())
        , cmdline(bEnv.cmdline)
    {
    }

    LazilyDeserializedEngine engine;
    std::unique_ptr<Profiler> profiler;
    std::vector<TrtDeviceBuffer>
        deviceMemory; //< Device memory used for inference when the allocation strategy is not static.
    std::unique_ptr<DebugTensorWriter> listener;
    bool error{false};
    bool accuracyFailed{false};                                 //< Set to true if any tensor accuracy exceeds threshold
    std::unordered_map<std::string, double> accuracyLossValues; //< Per-tensor accuracy values from the last validation

    bool safe{false};
    std::string cmdline;

#if !defined(_WIN32)
    //! Reference outputs for accuracy validation (tuner feature, Linux enterprise/auto-only).
    //! Map from tensor name to host buffer containing reference data.
    //! Guarded because MSVC cannot instantiate vector<unordered_map<string, unique_ptr<T>>>,
    //! and the tuner does not run on Windows or RTX/winjit.
    using RefOutputMap = std::unordered_map<std::string, std::unique_ptr<TrtHostBuffer>>;
    //! Vector of reference output maps, one for each refPair.
    std::vector<RefOutputMap> refOutputsAll;
#endif // !defined(_WIN32) && !TRT_WINML
};

struct InferenceEnvironmentStd : public InferenceEnvironmentBase
{
    InferenceEnvironmentStd() = delete;
    InferenceEnvironmentStd(InferenceEnvironmentStd const& other) = delete;
    InferenceEnvironmentStd(InferenceEnvironmentStd&& other) = delete;
    InferenceEnvironmentStd(BuildEnvironment& bEnv)
        : InferenceEnvironmentBase(bEnv)
    {
    }
    std::vector<std::unique_ptr<nvinfer1::IExecutionContext>> contexts;
    std::vector<std::unique_ptr<BindingsStd>> bindings;

    inline nvinfer1::IExecutionContext* getContext(int32_t streamIdx);

    //! Storage for input shape tensors.
    //!
    //! It's important that the addresses of the data do not change between the calls to
    //! setTensorAddress/setInputShape (which tells TensorRT where the input shape tensor is)
    //! and enqueueV3 (when TensorRT might use the input shape tensor).
    //!
    //! The input shape tensors could alternatively be handled via member bindings,
    //! but it simplifies control-flow to store the data here since it's shared across
    //! the bindings.
    std::list<std::vector<int64_t>> inputShapeTensorValues;
};

#if ENABLE_UNIFIED_BUILDER
// Forward declaration of BindingsSafe
class BindingsSafe;

struct InferenceEnvironmentSafe : public InferenceEnvironmentBase
{
    InferenceEnvironmentSafe() = delete;
    InferenceEnvironmentSafe(InferenceEnvironmentSafe const& other) = delete;
    InferenceEnvironmentSafe(InferenceEnvironmentSafe&& other) = delete;
    InferenceEnvironmentSafe(BuildEnvironment& bEnv)
        : InferenceEnvironmentBase(bEnv)
    {
    }

    std::vector<std::unique_ptr<BindingsSafe>> bindings;
    //! deleters for aux. streams, per cloned graph
    std::vector<std::shared_ptr<std::nullptr_t>> mAuxStreamsDeleters;
    std::vector<std::unique_ptr<nvinfer2::safe::ITRTGraph>> mClonedGraphs;
};
#endif

inline nvinfer1::IExecutionContext* InferenceEnvironmentStd::getContext(int32_t streamIdx)
{
    return contexts[streamIdx].get();
}

//!
//! \brief Set up contexts/graphs and bindings for inference
//!
bool setUpInference(InferenceEnvironmentBase& iEnv, InferenceOptions const& inference, SystemOptions const& system);

#if ENABLE_UNIFIED_BUILDER
//!
//! \brief Set up graphs and bindings for safe inference
//!
bool setUpSafeInference(InferenceEnvironmentSafe& iEnv, InferenceOptions const& inference, SystemOptions const& system);
#endif

//!
//! \brief Set up contexts and bindings for standard inference
//!
bool setUpStdInference(InferenceEnvironmentStd& iEnv, InferenceOptions const& inference, SystemOptions const& system);

//!
//! \brief Deserialize the engine and time how long it takes.
//!
bool timeDeserialize(InferenceEnvironmentBase& iEnv, SystemOptions const& sys);

//!
//! \brief Run inference and collect timing, return false if any error hit during inference
//!
bool runInference(InferenceOptions const& inference, InferenceEnvironmentBase& iEnv, int32_t device,
    std::vector<InferenceTrace>& trace, ReportingOptions const& reporting);

#if !defined(_WIN32)
//!
//! \brief Load reference outputs from files into InferenceEnvironmentBase::refOutputsAll.
//! \param pairIndex Index of the refPair to use (default 0 for backward compatibility).
//!
void loadRefOutputs(InferenceEnvironmentBase& iEnv, InferenceOptions const& inference,
    nvinfer1::IExecutionContext const& context, int64_t pairIndex = 0);

#if ENABLE_UNIFIED_BUILDER
//!
//! \brief Load reference outputs from files for safe inference.
//! \param pairIndex Index of the refPair to use (default 0 for backward compatibility).
//!
void loadRefOutputs(InferenceEnvironmentBase& iEnv, InferenceOptions const& inference,
    nvinfer2::safe::ITRTGraph const& graph, int64_t pairIndex = 0);
#endif
#endif // !defined(_WIN32) && !TRT_WINML

//!
//! \brief Get layer information of the engine.
//!
std::string getLayerInformation(
    nvinfer1::ICudaEngine* engine, nvinfer1::IExecutionContext* context, nvinfer1::LayerInformationFormat format);

struct Binding
{
    bool isInput{false};
    std::shared_ptr<IMirroredBuffer> buffer; // shared_ptr to allow aliasing between inputs and outputs
    std::unique_ptr<OutputAllocator> outputAllocator;
    int64_t volume{0};
    nvinfer1::DataType dataType{nvinfer1::DataType::kFLOAT};

    void fill(std::string const& fileName);

    void fill();

    void dump(std::ostream& os, nvinfer1::Dims dims, nvinfer1::Dims strides, int32_t vectorDim, int32_t spv,
        std::string const separator = " ") const;
};

struct TensorInfo
{
    int32_t bindingIndex{-1};
    char const* name{nullptr};
    nvinfer1::Dims dims{};
    bool isDynamic{};
    int32_t comps{-1};
    nvinfer1::Dims strides{};
    int32_t vectorDimIndex{-1};
    bool isInput{};
    nvinfer1::DataType dataType{};
    int64_t vol{-1};

    void updateVolume(int32_t batch)
    {
        vol = volume(dims, strides, vectorDimIndex, comps, batch);
    }
};

class BindingsBase
{
public:
    BindingsBase() = delete;
    explicit BindingsBase(bool useManaged)
        : mUseManaged(useManaged)
    {
    }

    void addBinding(
        TensorInfo const& tensorInfo, std::string const& fileName = "", char const* aliasedInputTensor = nullptr);

    void** getDeviceBuffers();

    void transferInputToDevice(TrtCudaStream& stream);

    void transferOutputToHost(TrtCudaStream& stream);

    void fill(int binding, std::string const& fileName)
    {
        mBindings[binding].fill(fileName);
    }

    void fill(int binding)
    {
        mBindings[binding].fill();
    }

    std::unordered_map<std::string, int> getInputBindings() const
    {
        auto isInput = [](Binding const& b) { return b.isInput; };
        return getBindings(isInput);
    }

    //! Fill input bindings from a name-to-file map.
    //!
    //! \param inputMap A map where:
    //!   - key: tensor name (e.g., "input", "input:0")
    //!   - value: file path containing the tensor data to load (e.g., "input_0.dat")
    //!
    //! For each entry in the map, looks up the tensor name in the input bindings
    //! and fills the binding buffer with data from the specified file.
    //! Entries with tensor names not found in input bindings are silently skipped.
    void fillInputsFromMap(std::unordered_map<std::string, std::string> const& inputMap)
    {
        auto inputBindings = getInputBindings();
        for (auto const& item : inputMap)
        {
            auto it = inputBindings.find(item.first);
            if (it != inputBindings.end())
            {
                fill(it->second, item.second);
            }
        }
    }

    std::unordered_map<std::string, int> getOutputBindings() const
    {
        auto isOutput = [](Binding const& b) { return !b.isInput; };
        return getBindings(isOutput);
    }

    std::unordered_map<std::string, int> getBindings() const
    {
        auto all = [](Binding const& b) { return true; };
        return getBindings(all);
    }

    std::unordered_map<std::string, int> getBindings(std::function<bool(Binding const&)> predicate) const;

    Binding const& getBinding(int32_t index) const
    {
        return mBindings.at(index);
    }

protected:
    std::unordered_map<std::string, int32_t> mNames;
    std::vector<Binding> mBindings;
    std::vector<void*> mDevicePointers;
    bool mUseManaged{false};
};

class BindingsStd : public BindingsBase
{
public:
    BindingsStd() = delete;
    explicit BindingsStd(bool useManaged)
        : BindingsBase(useManaged)
    {
    }

    void dumpInputs(nvinfer1::IExecutionContext const& context, std::ostream& os) const
    {
        auto isInput = [](Binding const& b) { return b.isInput; };
        dumpBindings(context, isInput, os);
    }

    void dumpOutputs(nvinfer1::IExecutionContext const& context, std::ostream& os) const
    {
        auto isOutput = [](Binding const& b) { return !b.isInput; };
        dumpBindings(context, isOutput, os);
    }

    void dumpBindings(nvinfer1::IExecutionContext const& context, std::ostream& os) const
    {
        auto all = [](Binding const& b) { return true; };
        dumpBindings(context, all, os);
    }

    void dumpBindings(nvinfer1::IExecutionContext const& context, std::function<bool(Binding const&)> predicate,
        std::ostream& os) const
    {
        for (auto const& n : mNames)
        {
            auto const name = n.first;
            auto const binding = n.second;
            if (predicate(mBindings[binding]))
            {
                os << n.first << ": (";
                dumpBindingDimensions(name, context, os);
                os << ")" << std::endl;

                dumpBindingValues(context, binding, os);
                os << std::endl;
            }
        }
    }

    void dumpBindingDimensions(
        std::string const& name, nvinfer1::IExecutionContext const& context, std::ostream& os) const;

    void dumpBindingValues(nvinfer1::IExecutionContext const& context, int32_t binding, std::ostream& os,
        std::string const& separator = " ", int32_t batch = 1) const;

    void dumpRawBindingToFiles(nvinfer1::IExecutionContext const& context, std::ostream& os) const;

    bool setTensorAddresses(nvinfer1::IExecutionContext& context) const;
};
#if ENABLE_UNIFIED_BUILDER
class BindingsSafe : public BindingsBase
{
public:
    BindingsSafe() = delete;
    explicit BindingsSafe(bool useManaged)
        : BindingsBase(useManaged)
    {
    }

    void dumpInputs(ITRTGraph const& graph, std::ostream& os) const
    {
        auto isInput = [](Binding const& b) { return b.isInput; };
        dumpBindings(graph, isInput, os);
    }

    void dumpOutputs(ITRTGraph const& graph, std::ostream& os) const
    {
        auto isOutput = [](Binding const& b) { return !b.isInput; };
        dumpBindings(graph, isOutput, os);
    }

    void dumpBindings(ITRTGraph const& graph, std::ostream& os) const
    {
        auto all = [](Binding const& b) { return true; };
        dumpBindings(graph, all, os);
    }

    void dumpBindings(ITRTGraph const& graph, std::function<bool(Binding const&)> predicate, std::ostream& os) const
    {
        for (auto const& n : mNames)
        {
            auto const name = n.first;
            auto const binding = n.second;
            if (predicate(mBindings[binding]))
            {
                os << n.first << ": (";
                dumpBindingDimensions(name, graph, os);
                os << ")" << std::endl;

                dumpBindingValues(graph, binding, os);
                os << std::endl;
            }
        }
    }

    void dumpBindingDimensions(std::string const& name, ITRTGraph const& graph, std::ostream& os) const;

    void dumpBindingValues(ITRTGraph const& graph, int32_t binding, std::ostream& os,
        std::string const& separator = " ", int32_t batch = 1) const;

    void dumpRawBindingToFiles(ITRTGraph& graph, std::ostream& os) const;

    bool setTensorAddresses(ITRTGraph& graph) const;
};
#endif

struct TaskInferenceEnvironment
{
    TaskInferenceEnvironment(std::string engineFile, InferenceOptions const& inference,
        ReportingOptions const& reporting, int32_t deviceId = 0,
        int32_t DLACore = -1, int32_t bs = batchNotProvided);
    InferenceOptions iOptions{};
    ReportingOptions rOptions{};
    int32_t device{defaultDevice};
    int32_t batch{batchNotProvided};
    std::unique_ptr<InferenceEnvironmentStd> iEnv;
    std::vector<InferenceTrace> trace;
};

bool runMultiTasksInference(std::vector<std::unique_ptr<TaskInferenceEnvironment>>& tEnvList);

} // namespace sample

#endif // TRT_SAMPLE_INFERENCE_H