chore: import upstream snapshot with attribution

samples/common/ArgVec.test.h

@@ -0,0 +1,58 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_SAMPLES_COMMON_ARGVEC_TEST_H
+#define TRT_SAMPLES_COMMON_ARGVEC_TEST_H
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+//! Wraps a list of argument strings as argc/argv for use with argument-parsing functions.
+//!
+//! CharPtrT controls the constness of the argv pointers:
+//!   - ArgVec<char*>        for APIs that take char** (e.g. argsToArgumentsMap)
+//!   - ArgVec<char const*>  for APIs that take char const* const* (e.g. getOptions)
+template <typename CharPtrT>
+struct ArgVec
+{
+    explicit ArgVec(std::initializer_list<char const*> strs)
+    {
+        mArgs.emplace_back("prog"); // argv[0] is the program name; argument parsers skip it.
+        mArgs.insert(mArgs.end(), strs.begin(), strs.end());
+        for (auto& s : mArgs)
+        {
+            mArgv.push_back(s.data());
+        }
+    }
+
+    [[nodiscard]] int32_t argc() const
+    {
+        return static_cast<int32_t>(mArgv.size());
+    }
+
+    [[nodiscard]] CharPtrT* argv()
+    {
+        return mArgv.data();
+    }
+
+private:
+    std::vector<std::string> mArgs;
+    std::vector<CharPtrT> mArgv;
+};
+
+#endif // TRT_SAMPLES_COMMON_ARGVEC_TEST_H

samples/common/BatchStream.h

@@ -0,0 +1,381 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef BATCH_STREAM_H
+#define BATCH_STREAM_H
+
+#include "NvInfer.h"
+#include "common.h"
+#include <algorithm>
+#include <stdio.h>
+#include <vector>
+
+class IBatchStream
+{
+public:
+    virtual void reset(int firstBatch) = 0;
+    virtual bool next() = 0;
+    virtual void skip(int skipCount) = 0;
+    virtual float* getBatch() = 0;
+    virtual float* getLabels() = 0;
+    virtual int getBatchesRead() const = 0;
+    virtual int getBatchSize() const = 0;
+    virtual nvinfer1::Dims getDims() const = 0;
+};
+
+class MNISTBatchStream : public IBatchStream
+{
+public:
+    MNISTBatchStream(int batchSize, int maxBatches, const std::string& dataFile, const std::string& labelsFile,
+        const std::vector<std::string>& directories)
+        : mBatchSize{batchSize}
+        , mMaxBatches{maxBatches}
+        , mDims{3, {1, 28, 28}} //!< We already know the dimensions of MNIST images.
+    {
+        readDataFile(samplesCommon::locateFile(dataFile, directories));
+        readLabelsFile(samplesCommon::locateFile(labelsFile, directories));
+    }
+
+    void reset(int firstBatch) override
+    {
+        mBatchCount = firstBatch;
+    }
+
+    bool next() override
+    {
+        if (mBatchCount >= mMaxBatches)
+        {
+            return false;
+        }
+        ++mBatchCount;
+        return true;
+    }
+
+    void skip(int skipCount) override
+    {
+        mBatchCount += skipCount;
+    }
+
+    float* getBatch() override
+    {
+        return mData.data() + (mBatchCount * mBatchSize * samplesCommon::volume(mDims));
+    }
+
+    float* getLabels() override
+    {
+        return mLabels.data() + (mBatchCount * mBatchSize);
+    }
+
+    int getBatchesRead() const override
+    {
+        return mBatchCount;
+    }
+
+    int getBatchSize() const override
+    {
+        return mBatchSize;
+    }
+
+    nvinfer1::Dims getDims() const override
+    {
+        return nvinfer1::Dims{4, {mBatchSize, mDims.d[0], mDims.d[1], mDims.d[2]}};
+    }
+
+private:
+    void readDataFile(const std::string& dataFilePath)
+    {
+        std::ifstream file{dataFilePath.c_str(), std::ios::binary};
+
+        int magicNumber, numImages, imageH, imageW;
+        file.read(reinterpret_cast<char*>(&magicNumber), sizeof(magicNumber));
+        // All values in the MNIST files are big endian.
+        magicNumber = samplesCommon::swapEndianness(magicNumber);
+        ASSERT(magicNumber == 2051 && "Magic Number does not match the expected value for an MNIST image set");
+
+        // Read number of images and dimensions
+        file.read(reinterpret_cast<char*>(&numImages), sizeof(numImages));
+        file.read(reinterpret_cast<char*>(&imageH), sizeof(imageH));
+        file.read(reinterpret_cast<char*>(&imageW), sizeof(imageW));
+
+        numImages = samplesCommon::swapEndianness(numImages);
+        imageH = samplesCommon::swapEndianness(imageH);
+        imageW = samplesCommon::swapEndianness(imageW);
+
+        // The MNIST data is made up of unsigned bytes, so we need to cast to float and normalize.
+        int numElements = numImages * imageH * imageW;
+        std::vector<uint8_t> rawData(numElements);
+        file.read(reinterpret_cast<char*>(rawData.data()), numElements * sizeof(uint8_t));
+        mData.resize(numElements);
+        std::transform(
+            rawData.begin(), rawData.end(), mData.begin(), [](uint8_t val) { return static_cast<float>(val) / 255.F; });
+    }
+
+    void readLabelsFile(const std::string& labelsFilePath)
+    {
+        std::ifstream file{labelsFilePath.c_str(), std::ios::binary};
+        int magicNumber, numImages;
+        file.read(reinterpret_cast<char*>(&magicNumber), sizeof(magicNumber));
+        // All values in the MNIST files are big endian.
+        magicNumber = samplesCommon::swapEndianness(magicNumber);
+        ASSERT(magicNumber == 2049 && "Magic Number does not match the expected value for an MNIST labels file");
+
+        file.read(reinterpret_cast<char*>(&numImages), sizeof(numImages));
+        numImages = samplesCommon::swapEndianness(numImages);
+
+        std::vector<uint8_t> rawLabels(numImages);
+        file.read(reinterpret_cast<char*>(rawLabels.data()), numImages * sizeof(uint8_t));
+        mLabels.resize(numImages);
+        std::transform(
+            rawLabels.begin(), rawLabels.end(), mLabels.begin(), [](uint8_t val) { return static_cast<float>(val); });
+    }
+
+    int mBatchSize{0};
+    int mBatchCount{0}; //!< The batch that will be read on the next invocation of next()
+    int mMaxBatches{0};
+    nvinfer1::Dims mDims{};
+    std::vector<float> mData{};
+    std::vector<float> mLabels{};
+};
+
+class BatchStream : public IBatchStream
+{
+public:
+    BatchStream(int batchSize, int maxBatches, std::string const& prefix, std::string const& suffix,
+        std::vector<std::string> const& directories)
+        : mBatchSize(batchSize)
+        , mMaxBatches(maxBatches)
+        , mPrefix(prefix)
+        , mSuffix(suffix)
+        , mDataDir(directories)
+    {
+        std::ifstream file(
+            samplesCommon::locateFile(mPrefix + std::string("0") + mSuffix, mDataDir).c_str(), std::ios::binary);
+        ASSERT(file.good());
+        int d[4];
+        file.read(reinterpret_cast<char*>(d), 4 * sizeof(int32_t));
+        mDims.nbDims = 4;  // The number of dimensions.
+        mDims.d[0] = d[0]; // Batch Size
+        mDims.d[1] = d[1]; // Channels
+        mDims.d[2] = d[2]; // Height
+        mDims.d[3] = d[3]; // Width
+        ASSERT(mDims.d[0] > 0 && mDims.d[1] > 0 && mDims.d[2] > 0 && mDims.d[3] > 0);
+
+        mImageSize = mDims.d[1] * mDims.d[2] * mDims.d[3];
+        mBatch.resize(mBatchSize * mImageSize, 0);
+        mLabels.resize(mBatchSize, 0);
+        mFileBatch.resize(mDims.d[0] * mImageSize, 0);
+        mFileLabels.resize(mDims.d[0], 0);
+    }
+
+    BatchStream(int batchSize, int maxBatches, std::string const& prefix, std::vector<std::string> const& directories)
+        : BatchStream(batchSize, maxBatches, prefix, ".batch", directories)
+    {
+    }
+
+    BatchStream(int batchSize, int maxBatches, nvinfer1::Dims const& dims, std::string const& listFile,
+        std::vector<std::string> const& directories)
+        : mBatchSize(batchSize)
+        , mMaxBatches(maxBatches)
+        , mDims(dims)
+        , mListFile(listFile)
+        , mDataDir(directories)
+    {
+        mImageSize = mDims.d[1] * mDims.d[2] * mDims.d[3];
+        mBatch.resize(mBatchSize * mImageSize, 0);
+        mLabels.resize(mBatchSize, 0);
+        mFileBatch.resize(mDims.d[0] * mImageSize, 0);
+        mFileLabels.resize(mDims.d[0], 0);
+    }
+
+    // Resets data members
+    void reset(int firstBatch) override
+    {
+        mBatchCount = 0;
+        mFileCount = 0;
+        mFileBatchPos = mDims.d[0];
+        skip(firstBatch);
+    }
+
+    // Advance to next batch and return true, or return false if there is no batch left.
+    bool next() override
+    {
+        if (mBatchCount == mMaxBatches)
+        {
+            return false;
+        }
+
+        for (int64_t csize = 1, batchPos = 0; batchPos < mBatchSize; batchPos += csize, mFileBatchPos += csize)
+        {
+            ASSERT(mFileBatchPos > 0 && mFileBatchPos <= mDims.d[0]);
+            if (mFileBatchPos == mDims.d[0] && !update())
+            {
+                return false;
+            }
+
+            // copy the smaller of: elements left to fulfill the request, or elements left in the file buffer.
+            csize = std::min<int64_t>(mBatchSize - batchPos, mDims.d[0] - mFileBatchPos);
+            std::copy_n(
+                getFileBatch() + mFileBatchPos * mImageSize, csize * mImageSize, getBatch() + batchPos * mImageSize);
+            std::copy_n(getFileLabels() + mFileBatchPos, csize, getLabels() + batchPos);
+        }
+        mBatchCount++;
+        return true;
+    }
+
+    // Skips the batches
+    void skip(int skipCount) override
+    {
+        if (mBatchSize >= mDims.d[0] && mBatchSize % mDims.d[0] == 0 && mFileBatchPos == mDims.d[0])
+        {
+            mFileCount += skipCount * mBatchSize / mDims.d[0];
+            return;
+        }
+
+        int x = mBatchCount;
+        for (int i = 0; i < skipCount; i++)
+        {
+            next();
+        }
+        mBatchCount = x;
+    }
+
+    float* getBatch() override
+    {
+        return mBatch.data();
+    }
+
+    float* getLabels() override
+    {
+        return mLabels.data();
+    }
+
+    int getBatchesRead() const override
+    {
+        return mBatchCount;
+    }
+
+    int getBatchSize() const override
+    {
+        return mBatchSize;
+    }
+
+    nvinfer1::Dims getDims() const override
+    {
+        return mDims;
+    }
+
+private:
+    float* getFileBatch()
+    {
+        return mFileBatch.data();
+    }
+
+    float* getFileLabels()
+    {
+        return mFileLabels.data();
+    }
+
+    bool update()
+    {
+        if (mListFile.empty())
+        {
+            std::string inputFileName
+                = samplesCommon::locateFile(mPrefix + std::to_string(mFileCount++) + mSuffix, mDataDir);
+            std::ifstream file(inputFileName.c_str(), std::ios::binary);
+            if (!file)
+            {
+                return false;
+            }
+            int d[4];
+            file.read(reinterpret_cast<char*>(d), 4 * sizeof(int32_t));
+            ASSERT(mDims.d[0] == d[0] && mDims.d[1] == d[1] && mDims.d[2] == d[2] && mDims.d[3] == d[3]);
+            file.read(reinterpret_cast<char*>(getFileBatch()), sizeof(float) * mDims.d[0] * mImageSize);
+            file.read(reinterpret_cast<char*>(getFileLabels()), sizeof(float) * mDims.d[0]);
+        }
+        else
+        {
+            std::vector<std::string> fNames;
+            std::ifstream file(samplesCommon::locateFile(mListFile, mDataDir), std::ios::binary);
+            if (!file)
+            {
+                return false;
+            }
+
+            sample::gLogInfo << "Batch #" << mFileCount << std::endl;
+            file.seekg(((mBatchCount * mBatchSize)) * 7);
+
+            for (int i = 1; i <= mBatchSize; i++)
+            {
+                std::string sName;
+                std::getline(file, sName);
+                sName = sName + ".ppm";
+                sample::gLogInfo << "Calibrating with file " << sName << std::endl;
+                fNames.emplace_back(sName);
+            }
+
+            mFileCount++;
+
+            const int imageC = 3;
+            const int imageH = 300;
+            const int imageW = 300;
+            std::vector<samplesCommon::PPM<imageC, imageH, imageW>> ppms(fNames.size());
+            for (uint32_t i = 0; i < fNames.size(); ++i)
+            {
+                readPPMFile(samplesCommon::locateFile(fNames[i], mDataDir), ppms[i]);
+            }
+
+            std::vector<float> data(samplesCommon::volume(mDims));
+            const float scale = 2.0 / 255.0;
+            const float bias = 1.0;
+            long int volChl = mDims.d[2] * mDims.d[3];
+
+            // Normalize input data
+            for (int i = 0, volImg = mDims.d[1] * mDims.d[2] * mDims.d[3]; i < mBatchSize; ++i)
+            {
+                for (int c = 0; c < mDims.d[1]; ++c)
+                {
+                    for (int j = 0; j < volChl; ++j)
+                    {
+                        data[i * volImg + c * volChl + j] = scale * float(ppms[i].buffer[j * mDims.d[1] + c]) - bias;
+                    }
+                }
+            }
+
+            std::copy_n(data.data(), mDims.d[0] * mImageSize, getFileBatch());
+        }
+
+        mFileBatchPos = 0;
+        return true;
+    }
+
+    int64_t mBatchSize{0};
+    int mMaxBatches{0};
+    int mBatchCount{0};
+    int mFileCount{0};
+    int mFileBatchPos{0};
+    int mImageSize{0};
+    std::vector<float> mBatch;         //!< Data for the batch
+    std::vector<float> mLabels;        //!< Labels for the batch
+    std::vector<float> mFileBatch;     //!< List of image files
+    std::vector<float> mFileLabels;    //!< List of label files
+    std::string mPrefix;               //!< Batch file name prefix
+    std::string mSuffix;               //!< Batch file name suffix
+    nvinfer1::Dims mDims;              //!< Input dimensions
+    std::string mListFile;             //!< File name of the list of image names
+    std::vector<std::string> mDataDir; //!< Directories where the files can be found
+};
+
+#endif

samples/common/CMakeLists.txt

@@ -0,0 +1,125 @@
+#
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+if(NOT NLOHMANN_JSON_INCLUDE_DIRS)
+    include(FetchNlohmannJson)
+endif()
+
+add_library(trt_samples_common STATIC
+    argsParser.h
+    BatchStream.h
+    bfloat16.cpp
+    bfloat16.h
+    bigInt.cpp
+    bigInt.h
+    buffers.h
+    common.cpp
+    common.h
+    debugTensorWriter.cpp
+    debugTensorWriter.h
+    ErrorRecorder.h
+    getOptions.cpp
+    getOptions.h
+    getoptWin.h
+    globalTimerKernel.cu
+    globalTimerKernel.h
+    half.h
+    logger.cpp
+    logger.h
+    logging.h
+    parserOnnxConfig.h
+    sampleConfig.h
+    sampleDevice.cpp
+    sampleDevice.h
+    sampleEngines.cpp
+    sampleEngines.h
+    sampleEntrypoints.h
+    sampleInference.cpp
+    sampleInference.h
+    sampleOptions.cpp
+    sampleOptions.h
+    sampleReporting.cpp
+    sampleReporting.h
+    sampleTuning.cpp
+    sampleTuning.h
+    sampleUtils.cpp
+    sampleUtils.h
+    safeCommon.h
+    safeCudaAllocator.h
+    safeErrorRecorder.h
+    streamReader.h
+)
+
+if(MSVC)
+    enable_language(C)
+    target_sources(trt_samples_common PRIVATE
+        getopt.c
+    )
+endif()
+
+if (${TRT_BUILD_TESTING})
+    include(GoogleTest)
+    enable_testing()
+
+    add_executable(trt_samples_common_test
+        bfloat16.test.cpp
+        getOptions.test.cpp
+        half.test.cpp
+        sampleOptions.test.cpp
+        sampleUtils.test.cpp
+    )
+
+    target_link_libraries(trt_samples_common_test PRIVATE
+        gtest_main
+        trt_samples_common
+    )
+
+    gtest_discover_tests(trt_samples_common_test DISCOVERY_MODE ${TRT_GTEST_DISCOVERY_MODE})
+endif() # TRT_BUILD_TESTING
+
+target_include_directories(trt_samples_common PUBLIC
+    ${CMAKE_CURRENT_LIST_DIR}
+    ${NLOHMANN_JSON_INCLUDE_DIRS}
+)
+
+target_link_libraries(trt_samples_common PUBLIC
+    tensorrt_headers
+    trt_shared
+    trt_global_definitions
+    Threads::Threads
+    $<COMPILE_ONLY:TRT_SAMPLES::tensorrt> # Each sample individually must determine its linkage to TRT.
+    TRT_SAMPLES::onnxparser
+)
+
+if(TARGET TRT::cudart)
+    target_link_libraries(trt_samples_common PUBLIC TRT::cudart)
+else()
+    target_link_libraries(trt_samples_common PUBLIC CUDA::cudart_static)
+endif()
+
+if(NOT WIN32 AND NOT ${CMAKE_SYSTEM_NAME} STREQUAL "QNX")
+    target_link_libraries(trt_samples_common PUBLIC dl)
+endif()
+
+target_link_libraries(trt_samples_common PUBLIC $<COMPILE_ONLY:CUDA::cupti>)
+
+# For statically-linked samples, we need to upgrade the link to always link TRT rather than letting the samples decide.
+if(${TRT_BUILD_SAMPLES_LINK_STATIC_TRT})
+    target_link_libraries(trt_samples_common PUBLIC
+        $<LINK_LIBRARY:WHOLE_ARCHIVE,TRT_SAMPLES::tensorrt> # Has to be whole archive so we keep the builder resources correctly.
+    )
+endif()

samples/common/ErrorRecorder.h

@@ -0,0 +1,138 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ERROR_RECORDER_H
+#define ERROR_RECORDER_H
+#include "NvInferRuntimeBase.h"
+#include "logger.h"
+#include <atomic>
+#include <cstdint>
+#include <exception>
+#include <mutex>
+#include <vector>
+
+using nvinfer1::IErrorRecorder;
+using nvinfer1::ErrorCode;
+
+//!
+//! A simple implementation of the IErrorRecorder interface for
+//! use by samples. This interface also can be used as a reference
+//! implementation.
+//! The sample Error recorder is based on a vector that pairs the error
+//! code and the error string into a single element. It also uses
+//! standard mutex's and atomics in order to make sure that the code
+//! works in a multi-threaded environment.
+//!
+class SampleErrorRecorder : public IErrorRecorder
+{
+    using errorPair = std::pair<ErrorCode, std::string>;
+    using errorStack = std::vector<errorPair>;
+
+public:
+    SampleErrorRecorder() = default;
+
+    ~SampleErrorRecorder() noexcept override {}
+    int32_t getNbErrors() const noexcept final
+    {
+        return mErrorStack.size();
+    }
+    ErrorCode getErrorCode(int32_t errorIdx) const noexcept final
+    {
+        return invalidIndexCheck(errorIdx) ? ErrorCode::kINVALID_ARGUMENT : (*this)[errorIdx].first;
+    }
+    IErrorRecorder::ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept final
+    {
+        return invalidIndexCheck(errorIdx) ? "errorIdx out of range." : (*this)[errorIdx].second.c_str();
+    }
+    // This class can never overflow since we have dynamic resize via std::vector usage.
+    bool hasOverflowed() const noexcept final
+    {
+        return false;
+    }
+
+    // Empty the errorStack.
+    void clear() noexcept final
+    {
+        try
+        {
+            // grab a lock so that there is no addition while clearing.
+            std::lock_guard<std::mutex> guard(mStackLock);
+            mErrorStack.clear();
+        }
+        catch (const std::exception& e)
+        {
+            sample::gLogFatal << "Internal Error: " << e.what() << std::endl;
+        }
+    }
+
+    //! Simple helper function that
+    bool empty() const noexcept
+    {
+        return mErrorStack.empty();
+    }
+
+    bool reportError(ErrorCode val, IErrorRecorder::ErrorDesc desc) noexcept final
+    {
+        try
+        {
+            std::lock_guard<std::mutex> guard(mStackLock);
+            sample::gLogError << "Error[" << static_cast<int32_t>(val) << "]: " << desc << std::endl;
+            mErrorStack.push_back(errorPair(val, desc));
+        }
+        catch (const std::exception& e)
+        {
+            sample::gLogFatal << "Internal Error: " << e.what() << std::endl;
+        }
+        // All errors are considered fatal.
+        return true;
+    }
+
+    // Atomically increment or decrement the ref counter.
+    IErrorRecorder::RefCount incRefCount() noexcept final
+    {
+        return ++mRefCount;
+    }
+    IErrorRecorder::RefCount decRefCount() noexcept final
+    {
+        return --mRefCount;
+    }
+
+private:
+    // Simple helper functions.
+    const errorPair& operator[](size_t index) const noexcept
+    {
+        return mErrorStack[index];
+    }
+
+    bool invalidIndexCheck(int32_t index) const noexcept
+    {
+        // By converting signed to unsigned, we only need a single check since
+        // negative numbers turn into large positive greater than the size.
+        size_t sIndex = index;
+        return sIndex >= mErrorStack.size();
+    }
+    // Mutex to hold when locking mErrorStack.
+    std::mutex mStackLock;
+
+    // Reference count of the class. Destruction of the class when mRefCount
+    // is not zero causes undefined behavior.
+    std::atomic<int32_t> mRefCount{0};
+
+    // The error stack that holds the errors recorded by TensorRT.
+    errorStack mErrorStack;
+};     // class SampleErrorRecorder
+#endif // ERROR_RECORDER_H

samples/common/README.md

@@ -0,0 +1,3 @@
+# samples/common
+
+Shared utility library (`trt_samples_common`) used by all TensorRT C++ samples and `trtexec`.

samples/common/argsParser.h

@@ -0,0 +1,162 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef TENSORRT_ARGS_PARSER_H
+#define TENSORRT_ARGS_PARSER_H
+
+#ifdef _MSC_VER
+#include "getOptWin.h"
+#else
+#include <getopt.h>
+#endif
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace samplesCommon
+{
+
+//!
+//! \brief The SampleParams structure groups the basic parameters required by
+//!        all sample networks.
+//!
+struct SampleParams
+{
+    int32_t batchSize{1};              //!< Number of inputs in a batch
+    int32_t dlaCore{-1};               //!< Specify the DLA core to run network on.
+    bool int8{false};                  //!< Allow runnning the network in Int8 mode.
+    bool fp16{false};                  //!< Allow running the network in FP16 mode.
+    bool bf16{false};                  //!< Allow running the network in BF16 mode.
+    std::vector<std::string> dataDirs; //!< Directory paths where sample data files are stored
+    std::vector<std::string> inputTensorNames;
+    std::vector<std::string> outputTensorNames;
+    std::string timingCacheFile; //!< Path to timing cache file
+};
+
+//!
+//! \brief The OnnxSampleParams structure groups the additional parameters required by
+//!         networks that use ONNX
+//!
+struct OnnxSampleParams : public SampleParams
+{
+    std::string onnxFileName; //!< Filename of ONNX file of a network
+};
+
+//!
+//! /brief Struct to maintain command-line arguments.
+//!
+struct Args
+{
+    bool runInInt8{false};
+    bool runInFp16{false};
+    bool runInBf16{false};
+    bool help{false};
+    int32_t useDLACore{-1};
+    int32_t batch{1};
+    std::vector<std::string> dataDirs;
+    std::string saveEngine;
+    std::string loadEngine;
+    bool rowOrder{true};
+    std::string timingCacheFile;
+};
+
+//!
+//! \brief Populates the Args struct with the provided command-line parameters.
+//!
+//! \throw invalid_argument if any of the arguments are not valid
+//!
+//! \return boolean If return value is true, execution can continue, otherwise program should exit
+//!
+inline bool parseArgs(Args& args, int32_t argc, char* argv[])
+{
+    while (1)
+    {
+        int32_t arg;
+        static struct option long_options[]
+            = {{"help", no_argument, 0, 'h'}, {"datadir", required_argument, 0, 'd'}, {"int8", no_argument, 0, 'i'},
+                {"fp16", no_argument, 0, 'f'}, {"bf16", no_argument, 0, 'z'}, {"columnOrder", no_argument, 0, 'c'},
+                {"saveEngine", required_argument, 0, 's'}, {"loadEngine", required_argument, 0, 'o'},
+                {"useDLACore", required_argument, 0, 'u'}, {"batch", required_argument, 0, 'b'},
+                {"timingCacheFile", required_argument, 0, 't'}, {nullptr, 0, nullptr, 0}};
+        int32_t option_index = 0;
+        arg = getopt_long(argc, argv, "hd:iu", long_options, &option_index);
+        if (arg == -1)
+        {
+            break;
+        }
+
+        switch (arg)
+        {
+        case 'h': args.help = true; return true;
+        case 'd':
+            if (optarg)
+            {
+                args.dataDirs.push_back(optarg);
+            }
+            else
+            {
+                std::cerr << "ERROR: --datadir requires option argument" << std::endl;
+                return false;
+            }
+            break;
+        case 's':
+            if (optarg)
+            {
+                args.saveEngine = optarg;
+            }
+            break;
+        case 'o':
+            if (optarg)
+            {
+                args.loadEngine = optarg;
+            }
+            break;
+        case 'i': args.runInInt8 = true; break;
+        case 'f': args.runInFp16 = true; break;
+        case 'z': args.runInBf16 = true; break;
+        case 'c': args.rowOrder = false; break;
+        case 'u':
+            if (optarg)
+            {
+                args.useDLACore = std::stoi(optarg);
+            }
+            break;
+        case 'b':
+            if (optarg)
+            {
+                args.batch = std::stoi(optarg);
+            }
+            break;
+        case 't':
+            if (optarg)
+            {
+                args.timingCacheFile = optarg;
+            }
+            else
+            {
+                std::cerr << "ERROR: --timingCacheFile requires option argument" << std::endl;
+                return false;
+            }
+            break;
+        default: return false;
+        }
+    }
+    return true;
+}
+
+} // namespace samplesCommon
+
+#endif // TENSORRT_ARGS_PARSER_H

samples/common/bfloat16.cpp

@@ -0,0 +1,60 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "bfloat16.h"
+#include <cstring>
+
+namespace sample
+{
+
+BFloat16::operator float() const
+{
+    static_assert(sizeof(uint32_t) == sizeof(float), "");
+    float val{0.F};
+    auto bits = static_cast<uint32_t>(mRep) << 16;
+    std::memcpy(&val, &bits, sizeof(uint32_t));
+    return val;
+}
+
+BFloat16::BFloat16(float x)
+{
+    static_assert(sizeof(uint32_t) == sizeof(float), "");
+    uint32_t bits{0};
+    std::memcpy(&bits, &x, sizeof(float));
+
+    // FP32 format: 1 sign bit, 8 bit exponent, 23 bit mantissa
+    // BF16 format: 1 sign bit, 8 bit exponent, 7 bit mantissa
+
+    // Mask for exponent
+    constexpr uint32_t exponent = 0xFFU << 23;
+
+    // Check if exponent is all 1s (NaN or infinite)
+    if ((bits & exponent) != exponent)
+    {
+        // x is finite - round to even
+        bits += 0x7FFFU + (bits >> 16 & 1);
+    }
+
+    mRep = static_cast<uint16_t>(bits >> 16);
+}
+
+BFloat16 operator+(BFloat16 x, BFloat16 y)
+{
+    return BFloat16(static_cast<float>(x) + static_cast<float>(y));
+}
+
+} // namespace sample

samples/common/bfloat16.h

@@ -0,0 +1,46 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <cstdint>
+
+namespace sample
+{
+
+//! Implements "Brain Floating Point": like an IEEE FP32,
+//! but the significand is only 7 bits instead of 23 bits.
+class BFloat16
+{
+public:
+    BFloat16()
+        : mRep(0)
+    {
+    }
+
+    // Rounds to even if there is a tie.
+    BFloat16(float x);
+
+    operator float() const;
+
+private:
+    //! Value stored in BFloat16 representation.
+    uint16_t mRep;
+};
+BFloat16 operator+(BFloat16 x, BFloat16 y);
+
+} // namespace sample

samples/common/bfloat16.test.cpp

@@ -0,0 +1,142 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "bfloat16.h"
+
+#include <gtest/gtest.h>
+
+#include <cmath>
+#include <cstdint>
+#include <limits>
+#include <sstream>
+#include <string>
+#include <type_traits>
+
+using sample::BFloat16;
+using NLF32 = std::numeric_limits<float>;
+
+TEST(BFloat16, Type)
+{
+    static_assert(sizeof(BFloat16) == sizeof(uint16_t), "BFloat16 should be 16 bits!");
+    static_assert(alignof(BFloat16) == alignof(uint16_t), "BFloat16 should be 16 bit aligned!");
+    EXPECT_EQ(BFloat16{}.operator float(), 0.0F);
+}
+
+TEST(BFloat16, Constructors)
+{
+    EXPECT_EQ(BFloat16{}, 0.0F);
+    EXPECT_EQ(BFloat16{1.0F}, 1.0F);
+    EXPECT_EQ(BFloat16{-1.0F}, -1.0F);
+    EXPECT_EQ(BFloat16{0.0F}, 0.0F);
+    EXPECT_EQ(BFloat16{0.5F}, 0.5F);
+    // Preserve sign bit, even for zero.
+    EXPECT_EQ(std::signbit(static_cast<float>(BFloat16{-0.0F})), std::signbit(-1.0F));
+    EXPECT_EQ(std::signbit(static_cast<float>(BFloat16{0.0F})), std::signbit(1.0F));
+}
+
+TEST(BFloat16, UnaryMinus)
+{
+    BFloat16 const bf16Pos = 2.5F;
+    BFloat16 const bf16Neg = -bf16Pos;
+    EXPECT_EQ(bf16Neg, -2.5F);
+
+    BFloat16 const bf16NegInput = -3.0F;
+    BFloat16 const bf16PosResult = -bf16NegInput;
+    EXPECT_EQ(bf16PosResult, 3.0F);
+
+    BFloat16 const bf16Zero = 0.0F;
+    BFloat16 const bf16NegZero = -bf16Zero;
+    EXPECT_EQ(bf16NegZero, 0.0F);
+}
+
+TEST(BFloat16, Addition)
+{
+    BFloat16 const a = 1.5F;
+    BFloat16 const b = 2.5F;
+    EXPECT_EQ(a + b, 4.0F);
+
+    BFloat16 const c = -1.0F;
+    BFloat16 const d = 3.0F;
+    EXPECT_EQ(c + d, 2.0F);
+
+    BFloat16 const e = 0.0F;
+    BFloat16 const f = 5.0F;
+    EXPECT_EQ(e + f, 5.0F);
+}
+
+TEST(BFloat16, FloatConversion)
+{
+    EXPECT_EQ(static_cast<float>(BFloat16{3.14159F}), 3.140625F);
+    EXPECT_EQ(static_cast<float>(BFloat16{1000.0F}), 1000.0F);
+    EXPECT_EQ(static_cast<float>(BFloat16{0.001F}), 0.0009994507F);
+    // Out-of-bounds conversion rounds to infinity.
+    EXPECT_TRUE(std::isinf(static_cast<float>(BFloat16{NLF32::max()})));
+}
+
+TEST(BFloat16, StreamOutput)
+{
+    auto toStr = [](auto const& value) {
+        std::stringstream ss;
+        ss << value;
+        return ss.str();
+    };
+    using namespace std::string_view_literals;
+    EXPECT_EQ(toStr(BFloat16(2.718F)), "2.71875"sv);
+    EXPECT_EQ(toStr(BFloat16(0.0F)), "0"sv);
+    // BFloat16 should match float stringification for special values.
+    EXPECT_EQ(toStr(BFloat16{NLF32::infinity()}), std::to_string(NLF32::infinity()));
+    EXPECT_EQ(toStr(-BFloat16{NLF32::infinity()}), std::to_string(-NLF32::infinity()));
+    EXPECT_EQ(toStr(BFloat16{NLF32::quiet_NaN()}), std::to_string(NLF32::quiet_NaN()));
+}
+
+TEST(BFloat16, NumericLimits)
+{
+    static_assert(!std::numeric_limits<BFloat16>::is_specialized);
+}
+
+TEST(BFloat16, TypeTraits)
+{
+    static_assert(!std::is_arithmetic_v<BFloat16>);
+    static_assert(!std::is_scalar_v<BFloat16>);
+}
+
+TEST(BFloat16, NaN)
+{
+    EXPECT_TRUE(std::isnan(static_cast<float>(BFloat16{NLF32::quiet_NaN()})));
+    EXPECT_TRUE(std::isnan(BFloat16{} / BFloat16{}));
+    EXPECT_TRUE(std::isnan(static_cast<float>(BFloat16{BFloat16{} / BFloat16{}})));
+    EXPECT_TRUE(std::isnan(BFloat16{NLF32::quiet_NaN()} / BFloat16{}));
+    EXPECT_TRUE(std::isnan(BFloat16{NLF32::infinity()} - BFloat16{NLF32::infinity()}));
+}
+
+TEST(BFloat16, EdgeCases)
+{
+    auto const bf16Large = BFloat16(1e38F);
+    EXPECT_FALSE(std::isinf(static_cast<float>(bf16Large)));
+    EXPECT_LT(BFloat16(1e30F), bf16Large);
+
+    auto const bf16Small = BFloat16(1e-38F);
+    EXPECT_LT(0.0F, static_cast<float>(bf16Small));
+    EXPECT_LT(bf16Small, BFloat16(1e-30F));
+}
+
+TEST(BFloat16, PrecisionAndRounding)
+{
+    // BFloat16 has lower precision than float; values are rounded (to even).
+    constexpr float kPRECISE_VALUE = 1.0F + 1e-7F;
+    EXPECT_NEAR(static_cast<float>(BFloat16{kPRECISE_VALUE}), kPRECISE_VALUE, 1e-3F);
+}

samples/common/bigInt.cpp

@@ -0,0 +1,202 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "bigInt.h"
+
+namespace sample
+{
+
+BigInt::BigInt(std::string const& str)
+{
+    if (str.empty())
+    {
+        throw std::invalid_argument("Empty string");
+    }
+
+    BigInt const ten(10);
+    for (char const c : str)
+    {
+        if (c < '0' || c > '9')
+        {
+            throw std::invalid_argument("Invalid decimal character in BigInt string");
+        }
+
+        auto [mulResult, mulOverflow] = multiplyWithOverflow(*this, ten);
+        if (mulOverflow)
+        {
+            throw std::overflow_error("Number too large for BigInt");
+        }
+
+        auto [addResult, addOverflow] = addWithOverflow(mulResult, BigInt(static_cast<uint64_t>(c - '0')));
+        if (addOverflow)
+        {
+            throw std::overflow_error("Number too large for BigInt");
+        }
+
+        *this = addResult;
+    }
+}
+
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
+std::pair<BigInt, bool> BigInt::multiplyWithOverflow(BigInt const& a, BigInt const& b) noexcept
+{
+    // Full multiplication into 2*kWordCount words
+    std::array<WordType, kWordCount * 2> result{};
+
+    for (uint64_t i = 0; i < kWordCount; ++i)
+    {
+        if (a.mWords[i] == 0)
+        {
+            continue;
+        }
+
+        WordType carry = 0;
+        for (uint64_t j = 0; j < kWordCount; ++j)
+        {
+            uint64_t const k = i + j;
+            if (k >= kWordCount * 2)
+            {
+                break;
+            }
+
+            // 64x64 → 128-bit multiply using four 32-bit half-words.
+            // Split: a = aHi*2^32 + aLo, b = bHi*2^32 + bLo
+            // Product = aHi*bHi*2^64 + (aHi*bLo + aLo*bHi)*2^32 + aLo*bLo
+            uint64_t const aLo = a.mWords[i] & 0xFFFFFFFF;
+            uint64_t const aHi = a.mWords[i] >> 32;
+            uint64_t const bLo = b.mWords[j] & 0xFFFFFFFF;
+            uint64_t const bHi = b.mWords[j] >> 32;
+
+            uint64_t const p0 = aLo * bLo;
+            uint64_t const p1 = aLo * bHi;
+            uint64_t const p2 = aHi * bLo;
+            uint64_t const p3 = aHi * bHi;
+
+            // Combine: prodLo = lower 64 bits, prodHi = upper 64 bits
+            uint64_t const mid = p1 + (p0 >> 32);
+            uint64_t const midCarry = (mid < p1) ? (uint64_t{1} << 32) : 0;
+            uint64_t const mid2 = mid + p2;
+            uint64_t const midCarry2 = (mid2 < mid) ? (uint64_t{1} << 32) : 0;
+
+            uint64_t prodLo = (mid2 << 32) | (p0 & 0xFFFFFFFF);
+            uint64_t prodHi = p3 + (mid2 >> 32) + midCarry + midCarry2;
+
+            // Add result[k] and carry to the 128-bit product.
+            prodLo += result[k];
+            if (prodLo < result[k])
+            {
+                ++prodHi;
+            }
+            prodLo += carry;
+            if (prodLo < carry)
+            {
+                ++prodHi;
+            }
+            result[k] = prodLo;
+            carry = prodHi;
+        }
+        if (i + kWordCount < kWordCount * 2)
+        {
+            result[i + kWordCount] += carry;
+        }
+    }
+
+    // Check for overflow (any non-zero word in upper half)
+    bool overflow = false;
+    for (uint64_t i = kWordCount; i < kWordCount * 2; ++i)
+    {
+        if (result[i] != 0)
+        {
+            overflow = true;
+            break;
+        }
+    }
+
+    // Copy lower half to result
+    BigInt low;
+    for (uint64_t i = 0; i < kWordCount; ++i)
+    {
+        low.mWords[i] = result[i];
+    }
+
+    return {low, overflow};
+}
+
+std::pair<BigInt, BigInt> BigInt::divideWithRemainder(BigInt const& dividend, BigInt const& divisor)
+{
+    if (divisor.isZero())
+    {
+        throw std::domain_error("Division by zero in BigInt");
+    }
+
+    if (dividend < divisor)
+    {
+        return {BigInt(), dividend};
+    }
+
+    if (dividend == divisor)
+    {
+        return {BigInt(1), BigInt()};
+    }
+
+    // Binary long division algorithm
+    BigInt quotient;
+    BigInt remainder;
+
+    int32_t const highBit = dividend.getHighestSetBit();
+
+    for (int32_t i = highBit; i >= 0; --i)
+    {
+        remainder <<= 1;
+        if (dividend.getBit(i))
+        {
+            remainder.setBit(0, true);
+        }
+
+        if (remainder >= divisor)
+        {
+            remainder -= divisor;
+            quotient.setBit(i, true);
+        }
+    }
+
+    return {quotient, remainder};
+}
+
+std::string BigInt::toString() const
+{
+    if (isZero())
+    {
+        return "0";
+    }
+
+    std::string result;
+    BigInt tmp = *this;
+    BigInt const ten(10);
+
+    while (!tmp.isZero())
+    {
+        auto [q, r] = divideWithRemainder(tmp, ten);
+        result += static_cast<char>('0' + r.mWords[0]);
+        tmp = q;
+    }
+
+    std::reverse(result.begin(), result.end());
+    return result;
+}
+
+} // namespace sample

samples/common/bigInt.h

@@ -0,0 +1,410 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_SAMPLE_BIG_INT_H
+#define TRT_SAMPLE_BIG_INT_H
+
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <stdexcept>
+#include <string>
+#include <utility>
+
+namespace sample
+{
+
+//!
+//! \class BigInt
+//! \brief A class for arbitrary-precision unsigned integers (8192 bits).
+//!
+//! This class provides support for very large unsigned integers, primarily used
+//! for counting and indexing in build path expression expansion where the number
+//! of combinations can be astronomically large (e.g., 2^1000 combinations).
+//!
+//! Key operations:
+//! - Construction from uint64_t or decimal string
+//! - Comparison operators for loop termination
+//! - Increment operator for loop counting
+//! - Division/modulo for mixed-radix index decomposition
+//! - String conversion for display
+//!
+class BigInt
+{
+public:
+    //! Number of bits in the integer (8192 = 128 * 64)
+    static constexpr uint64_t kBitCount = 8192;
+    //! Number of 64-bit words
+    static constexpr uint64_t kWordCount = kBitCount / 64; // 128 words
+    using WordType = uint64_t;
+
+    //! \brief Default constructor. Initializes to zero.
+    constexpr BigInt() noexcept = default;
+
+    //! \brief Construct from a 64-bit unsigned integer.
+    //! \param[in] value The initial value.
+    constexpr BigInt(uint64_t value) noexcept
+    {
+        mWords[0] = value;
+    }
+
+    //! \brief Construct from a decimal string.
+    //! \param[in] str The decimal string representation.
+    //! \throws std::invalid_argument If the string is empty or contains invalid characters.
+    //! \throws std::overflow_error If the number is too large.
+    explicit BigInt(std::string const& str);
+
+    // Default copy and move operations
+    constexpr BigInt(BigInt const&) noexcept = default;
+    constexpr BigInt& operator=(BigInt const&) noexcept = default;
+    constexpr BigInt(BigInt&&) noexcept = default;
+    constexpr BigInt& operator=(BigInt&&) noexcept = default;
+
+    //! \brief Check if the value is zero.
+    //! \return True if zero.
+    constexpr bool isZero() const noexcept
+    {
+        for (uint64_t i = 0; i < kWordCount; ++i)
+        {
+            if (mWords[i] != 0)
+            {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    //! \brief Get the bit value at a specific position.
+    //! \param[in] pos The bit position (0 = LSB).
+    //! \return The bit value.
+    constexpr bool getBit(uint64_t pos) const noexcept
+    {
+        if (pos >= kBitCount)
+        {
+            return false;
+        }
+        uint64_t const wordIdx = pos / 64;
+        uint64_t const bitIdx = pos % 64;
+        return (mWords[wordIdx] >> bitIdx) & 1;
+    }
+
+    //! \brief Set the bit value at a specific position.
+    //! \param[in] pos The bit position (0 = LSB).
+    //! \param[in] value The bit value to set.
+    constexpr void setBit(uint64_t pos, bool value = true) noexcept
+    {
+        if (pos >= kBitCount)
+        {
+            return;
+        }
+        uint64_t const wordIdx = pos / 64;
+        uint64_t const bitIdx = pos % 64;
+        if (value)
+        {
+            mWords[wordIdx] |= (WordType{1} << bitIdx);
+        }
+        else
+        {
+            mWords[wordIdx] &= ~(WordType{1} << bitIdx);
+        }
+    }
+
+    //! \brief Get the position of the highest set bit.
+    //! \return The position (0-indexed), or -1 if zero.
+    constexpr int32_t getHighestSetBit() const noexcept
+    {
+        for (int32_t i = kWordCount - 1; i >= 0; --i)
+        {
+            if (mWords[i] != 0)
+            {
+                // Count leading zeros portably (no compiler intrinsics).
+                uint64_t w = mWords[i];
+                int32_t bit = 63;
+                while (bit > 0 && (w & (uint64_t{1} << bit)) == 0)
+                {
+                    --bit;
+                }
+                return i * 64 + bit;
+            }
+        }
+        return -1;
+    }
+
+    // ========================================================================
+    // Comparison operators
+    // ========================================================================
+
+    constexpr bool operator==(BigInt const& other) const noexcept
+    {
+        // Manual element-by-element comparison (std::array::operator== is not constexpr in C++17)
+        for (uint64_t i = 0; i < kWordCount; ++i)
+        {
+            if (mWords[i] != other.mWords[i])
+            {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    constexpr bool operator!=(BigInt const& other) const noexcept
+    {
+        return !(*this == other);
+    }
+
+    //! \brief Less-than comparison.
+    //! Compares from most significant word down.
+    constexpr bool operator<(BigInt const& other) const noexcept
+    {
+        for (int32_t i = kWordCount - 1; i >= 0; --i)
+        {
+            if (mWords[i] < other.mWords[i])
+            {
+                return true;
+            }
+            if (mWords[i] > other.mWords[i])
+            {
+                return false;
+            }
+        }
+        return false;
+    }
+
+    constexpr bool operator<=(BigInt const& other) const noexcept
+    {
+        return !(other < *this);
+    }
+
+    constexpr bool operator>(BigInt const& other) const noexcept
+    {
+        return other < *this;
+    }
+
+    constexpr bool operator>=(BigInt const& other) const noexcept
+    {
+        return !(*this < other);
+    }
+
+    // ========================================================================
+    // Arithmetic operators
+    // ========================================================================
+
+    //! \brief Add with overflow detection.
+    //! \return Pair of (result, overflow_flag).
+    static constexpr std::pair<BigInt, bool> addWithOverflow(BigInt const& a, BigInt const& b) noexcept
+    {
+        BigInt result;
+        uint64_t carry = 0;
+        for (uint64_t i = 0; i < kWordCount; ++i)
+        {
+            // Add with carry using plain uint64_t. Overflow is detected by comparing
+            // the result against the operand: if sum < a then overflow occurred.
+            uint64_t sum = a.mWords[i] + b.mWords[i];
+            uint64_t c1 = (sum < a.mWords[i]) ? 1U : 0U;
+            uint64_t sum2 = sum + carry;
+            uint64_t c2 = (sum2 < sum) ? 1U : 0U;
+            result.mWords[i] = sum2;
+            carry = c1 + c2;
+        }
+        return {result, carry != 0};
+    }
+
+    //! \brief Subtract with underflow detection.
+    //! \return Pair of (result, underflow_flag).
+    static constexpr std::pair<BigInt, bool> subWithUnderflow(BigInt const& a, BigInt const& b) noexcept
+    {
+        BigInt result;
+        uint64_t borrow = 0;
+        for (uint64_t i = 0; i < kWordCount; ++i)
+        {
+            // Subtract with borrow using plain uint64_t.
+            // Borrow is detected by: if a < b+borrow, then we borrowed from the next word.
+            uint64_t sub = a.mWords[i] - b.mWords[i];
+            uint64_t b1 = (a.mWords[i] < b.mWords[i]) ? 1U : 0U;
+            uint64_t sub2 = sub - borrow;
+            uint64_t b2 = (sub < borrow) ? 1U : 0U;
+            result.mWords[i] = sub2;
+            borrow = b1 + b2;
+        }
+        return {result, borrow != 0};
+    }
+
+    //! \brief Multiply with overflow detection.
+    //! \return Pair of (result, overflow_flag).
+    static std::pair<BigInt, bool> multiplyWithOverflow(BigInt const& a, BigInt const& b) noexcept;
+
+    //! \brief Divide with remainder.
+    //! \param[in] dividend The dividend.
+    //! \param[in] divisor The divisor.
+    //! \return Pair of (quotient, remainder).
+    //! \throws std::domain_error If divisor is zero.
+    static std::pair<BigInt, BigInt> divideWithRemainder(BigInt const& dividend, BigInt const& divisor);
+
+    constexpr BigInt operator+(BigInt const& other) const noexcept
+    {
+        return addWithOverflow(*this, other).first;
+    }
+
+    constexpr BigInt& operator+=(BigInt const& other) noexcept
+    {
+        *this = *this + other;
+        return *this;
+    }
+
+    constexpr BigInt operator-(BigInt const& other) const noexcept
+    {
+        return subWithUnderflow(*this, other).first;
+    }
+
+    constexpr BigInt& operator-=(BigInt const& other) noexcept
+    {
+        *this = *this - other;
+        return *this;
+    }
+
+    BigInt operator*(BigInt const& other) const noexcept
+    {
+        return multiplyWithOverflow(*this, other).first;
+    }
+
+    BigInt operator/(BigInt const& other) const
+    {
+        return divideWithRemainder(*this, other).first;
+    }
+
+    BigInt operator%(BigInt const& other) const
+    {
+        return divideWithRemainder(*this, other).second;
+    }
+
+    // ========================================================================
+    // Shift operators (needed for division algorithm)
+    // ========================================================================
+
+    constexpr BigInt operator<<(uint64_t shift) const noexcept
+    {
+        if (shift >= kBitCount)
+        {
+            return BigInt();
+        }
+        if (shift == 0)
+        {
+            return *this;
+        }
+
+        BigInt result;
+        uint64_t const wordShift = shift / 64;
+        uint64_t const bitShift = shift % 64;
+
+        if (bitShift == 0)
+        {
+            for (uint64_t i = wordShift; i < kWordCount; ++i)
+            {
+                result.mWords[i] = mWords[i - wordShift];
+            }
+        }
+        else
+        {
+            for (uint64_t i = wordShift; i < kWordCount; ++i)
+            {
+                result.mWords[i] = mWords[i - wordShift] << bitShift;
+                if (i > wordShift)
+                {
+                    result.mWords[i] |= mWords[i - wordShift - 1] >> (64 - bitShift);
+                }
+            }
+        }
+        return result;
+    }
+
+    constexpr BigInt& operator<<=(uint64_t shift) noexcept
+    {
+        *this = *this << shift;
+        return *this;
+    }
+
+    // ========================================================================
+    // Increment/Decrement operators
+    // ========================================================================
+
+    //! \brief Pre-increment operator.
+    //! Handles carry propagation across words.
+    constexpr BigInt& operator++() noexcept
+    {
+        for (uint64_t i = 0; i < kWordCount; ++i)
+        {
+            if (++mWords[i] != 0)
+            {
+                break; // No carry, done
+            }
+            // Carry propagates to next word
+        }
+        return *this;
+    }
+
+    constexpr BigInt operator++(int32_t) noexcept
+    {
+        BigInt tmp = *this;
+        ++(*this);
+        return tmp;
+    }
+
+    //! \brief Pre-decrement operator.
+    //! Handles borrow propagation across words.
+    constexpr BigInt& operator--() noexcept
+    {
+        for (uint64_t i = 0; i < kWordCount; ++i)
+        {
+            if (mWords[i]-- != 0)
+            {
+                break; // No borrow, done
+            }
+            // Borrow propagates to next word
+        }
+        return *this;
+    }
+
+    constexpr BigInt operator--(int32_t) noexcept
+    {
+        BigInt tmp = *this;
+        --(*this);
+        return tmp;
+    }
+
+    // ========================================================================
+    // String conversion
+    // ========================================================================
+
+    //! \brief Convert to decimal string representation.
+    //! \return The decimal string.
+    std::string toString() const;
+
+    //! \brief Get the lowest 64 bits as uint64_t.
+    //! Useful when the value is known to fit in 64 bits.
+    constexpr uint64_t toUint64() const noexcept
+    {
+        return mWords[0];
+    }
+
+private:
+    std::array<WordType, kWordCount> mWords{};
+};
+
+} // namespace sample
+
+#endif // TRT_SAMPLE_BIG_INT_H

samples/common/buffers.h

@@ -0,0 +1,427 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef TENSORRT_BUFFERS_H
+#define TENSORRT_BUFFERS_H
+
+#include "NvInfer.h"
+#include "common.h"
+#include "half.h"
+#include <cassert>
+#include <cuda_runtime_api.h>
+#include <iostream>
+#include <iterator>
+#include <memory>
+#include <new>
+#include <numeric>
+#include <string>
+#include <vector>
+
+namespace samplesCommon
+{
+
+//!
+//! \brief  The GenericBuffer class is a templated class for buffers.
+//!
+//! \details This templated RAII (Resource Acquisition Is Initialization) class handles the allocation,
+//!          deallocation, querying of buffers on both the device and the host.
+//!          It can handle data of arbitrary types because it stores byte buffers.
+//!          The template parameters AllocFunc and FreeFunc are used for the
+//!          allocation and deallocation of the buffer.
+//!          AllocFunc must be a functor that takes in (void** ptr, size_t size)
+//!          and returns bool. ptr is a pointer to where the allocated buffer address should be stored.
+//!          size is the amount of memory in bytes to allocate.
+//!          The boolean indicates whether or not the memory allocation was successful.
+//!          FreeFunc must be a functor that takes in (void* ptr) and returns void.
+//!          ptr is the allocated buffer address. It must work with nullptr input.
+//!
+template <typename AllocFunc, typename FreeFunc>
+class GenericBuffer
+{
+public:
+    //!
+    //! \brief Construct an empty buffer.
+    //!
+    GenericBuffer(nvinfer1::DataType type = nvinfer1::DataType::kFLOAT)
+        : mSize(0)
+        , mCapacity(0)
+        , mType(type)
+        , mBuffer(nullptr)
+    {
+    }
+
+    //!
+    //! \brief Construct a buffer with the specified allocation size in bytes.
+    //!
+    GenericBuffer(size_t size, nvinfer1::DataType type)
+        : mSize(size)
+        , mCapacity(size)
+        , mType(type)
+    {
+        if (!allocFn(&mBuffer, this->nbBytes()))
+        {
+            throw std::bad_alloc();
+        }
+    }
+
+    GenericBuffer(GenericBuffer&& buf)
+        : mSize(buf.mSize)
+        , mCapacity(buf.mCapacity)
+        , mType(buf.mType)
+        , mBuffer(buf.mBuffer)
+    {
+        buf.mSize = 0;
+        buf.mCapacity = 0;
+        buf.mType = nvinfer1::DataType::kFLOAT;
+        buf.mBuffer = nullptr;
+    }
+
+    GenericBuffer& operator=(GenericBuffer&& buf)
+    {
+        if (this != &buf)
+        {
+            freeFn(mBuffer);
+            mSize = buf.mSize;
+            mCapacity = buf.mCapacity;
+            mType = buf.mType;
+            mBuffer = buf.mBuffer;
+            // Reset buf.
+            buf.mSize = 0;
+            buf.mCapacity = 0;
+            buf.mBuffer = nullptr;
+        }
+        return *this;
+    }
+
+    //!
+    //! \brief Returns pointer to underlying array.
+    //!
+    void* data()
+    {
+        return mBuffer;
+    }
+
+    //!
+    //! \brief Returns pointer to underlying array.
+    //!
+    const void* data() const
+    {
+        return mBuffer;
+    }
+
+    //!
+    //! \brief Returns the size (in number of elements) of the buffer.
+    //!
+    size_t size() const
+    {
+        return mSize;
+    }
+
+    //!
+    //! \brief Returns the size (in bytes) of the buffer.
+    //!
+    size_t nbBytes() const
+    {
+        return samplesCommon::getNbBytes(mType, size());
+    }
+
+    //!
+    //! \brief Resizes the buffer. This is a no-op if the new size is smaller than or equal to the current capacity.
+    //!
+    void resize(size_t newSize)
+    {
+        mSize = newSize;
+        if (mCapacity < newSize)
+        {
+            freeFn(mBuffer);
+            if (!allocFn(&mBuffer, this->nbBytes()))
+            {
+                throw std::bad_alloc{};
+            }
+            mCapacity = newSize;
+        }
+    }
+
+    //!
+    //! \brief Overload of resize that accepts Dims
+    //!
+    void resize(const nvinfer1::Dims& dims)
+    {
+        return this->resize(samplesCommon::volume(dims));
+    }
+
+    ~GenericBuffer()
+    {
+        freeFn(mBuffer);
+    }
+
+private:
+    size_t mSize{0}, mCapacity{0};
+    nvinfer1::DataType mType;
+    void* mBuffer;
+    AllocFunc allocFn;
+    FreeFunc freeFn;
+};
+
+class DeviceAllocator
+{
+public:
+    bool operator()(void** ptr, size_t size) const
+    {
+        return cudaMalloc(ptr, size) == cudaSuccess;
+    }
+};
+
+class DeviceFree
+{
+public:
+    void operator()(void* ptr) const
+    {
+        cudaFree(ptr);
+    }
+};
+
+class HostAllocator
+{
+public:
+    bool operator()(void** ptr, size_t size) const
+    {
+        *ptr = malloc(size);
+        return *ptr != nullptr;
+    }
+};
+
+class HostFree
+{
+public:
+    void operator()(void* ptr) const
+    {
+        free(ptr);
+    }
+};
+
+using DeviceBuffer = GenericBuffer<DeviceAllocator, DeviceFree>;
+using HostBuffer = GenericBuffer<HostAllocator, HostFree>;
+
+//!
+//! \brief  The ManagedBuffer class groups together a pair of corresponding device and host buffers.
+//!
+class ManagedBuffer
+{
+public:
+    DeviceBuffer deviceBuffer;
+    HostBuffer hostBuffer;
+};
+
+//!
+//! \brief  The BufferManager class handles host and device buffer allocation and deallocation.
+//!
+//! \details This RAII class handles host and device buffer allocation and deallocation,
+//!          memcpy between host and device buffers to aid with inference,
+//!          and debugging dumps to validate inference. The BufferManager class is meant to be
+//!          used to simplify buffer management and any interactions between buffers and the engine.
+//!
+class BufferManager
+{
+public:
+    static const size_t kINVALID_SIZE_VALUE = ~size_t(0);
+
+    //!
+    //! \brief Create a BufferManager for handling buffer interactions with engine, when the I/O tensor volumes
+    //! are provided
+    //!
+    BufferManager(
+        std::shared_ptr<nvinfer1::ICudaEngine> engine, std::vector<int64_t> const& volumes, int32_t batchSize = 0)
+        : mEngine(engine)
+        , mBatchSize(batchSize)
+    {
+        // Create host and device buffers
+        for (int32_t i = 0; i < mEngine->getNbIOTensors(); i++)
+        {
+            auto const name = engine->getIOTensorName(i);
+            mNames[name] = i;
+
+            nvinfer1::DataType type = mEngine->getTensorDataType(name);
+
+            std::unique_ptr<ManagedBuffer> manBuf{new ManagedBuffer()};
+            manBuf->deviceBuffer = DeviceBuffer(volumes[i], type);
+            manBuf->hostBuffer = HostBuffer(volumes[i], type);
+            void* deviceBuffer = manBuf->deviceBuffer.data();
+            mDeviceBindings.emplace_back(deviceBuffer);
+            mManagedBuffers.emplace_back(std::move(manBuf));
+        }
+    }
+
+    //!
+    //! \brief Create a BufferManager for handling buffer interactions with engine.
+    //!
+    BufferManager(std::shared_ptr<nvinfer1::ICudaEngine> engine, int32_t const batchSize = 0,
+        nvinfer1::IExecutionContext const* context = nullptr)
+        : mEngine(engine)
+        , mBatchSize(batchSize)
+    {
+        // Create host and device buffers
+        for (int32_t i = 0, e = mEngine->getNbIOTensors(); i < e; i++)
+        {
+            auto const name = engine->getIOTensorName(i);
+            mNames[name] = i;
+
+            auto dims = context ? context->getTensorShape(name) : mEngine->getTensorShape(name);
+            size_t vol = context || !mBatchSize ? 1 : static_cast<size_t>(mBatchSize);
+            nvinfer1::DataType type = mEngine->getTensorDataType(name);
+            int32_t vecDim = mEngine->getTensorVectorizedDim(name);
+            if (-1 != vecDim) // i.e., 0 != lgScalarsPerVector
+            {
+                int32_t scalarsPerVec = mEngine->getTensorComponentsPerElement(name);
+                dims.d[vecDim] = divUp(dims.d[vecDim], scalarsPerVec);
+                vol *= scalarsPerVec;
+            }
+            vol *= samplesCommon::volume(dims);
+            std::unique_ptr<ManagedBuffer> manBuf{new ManagedBuffer()};
+            manBuf->deviceBuffer = DeviceBuffer(vol, type);
+            manBuf->hostBuffer = HostBuffer(vol, type);
+            void* deviceBuffer = manBuf->deviceBuffer.data();
+            mDeviceBindings.emplace_back(deviceBuffer);
+            mManagedBuffers.emplace_back(std::move(manBuf));
+        }
+    }
+
+    //!
+    //! \brief Returns a vector of device buffers that you can use directly as
+    //!        bindings for the execute and enqueue methods of IExecutionContext.
+    //!
+    std::vector<void*>& getDeviceBindings()
+    {
+        return mDeviceBindings;
+    }
+
+    //!
+    //! \brief Returns a vector of device buffers.
+    //!
+    std::vector<void*> const& getDeviceBindings() const
+    {
+        return mDeviceBindings;
+    }
+
+    //!
+    //! \brief Returns the device buffer corresponding to tensorName.
+    //!        Returns nullptr if no such tensor can be found.
+    //!
+    void* getDeviceBuffer(std::string const& tensorName) const
+    {
+        return getBuffer(false, tensorName);
+    }
+
+    //!
+    //! \brief Returns the host buffer corresponding to tensorName.
+    //!        Returns nullptr if no such tensor can be found.
+    //!
+    void* getHostBuffer(std::string const& tensorName) const
+    {
+        return getBuffer(true, tensorName);
+    }
+
+    //!
+    //! \brief Returns the size of the host and device buffers that correspond to tensorName.
+    //!        Returns kINVALID_SIZE_VALUE if no such tensor can be found.
+    //!
+    size_t size(std::string const& tensorName) const
+    {
+        auto record = mNames.find(tensorName);
+        if (record == mNames.end())
+            return kINVALID_SIZE_VALUE;
+        return mManagedBuffers[record->second]->hostBuffer.nbBytes();
+    }
+
+    //!
+    //! \brief Copy the contents of input host buffers to input device buffers synchronously.
+    //!
+    void copyInputToDevice()
+    {
+        memcpyBuffers(true, false, false);
+    }
+
+    //!
+    //! \brief Copy the contents of output device buffers to output host buffers synchronously.
+    //!
+    void copyOutputToHost()
+    {
+        memcpyBuffers(false, true, false);
+    }
+
+    //!
+    //! \brief Copy the contents of input host buffers to input device buffers asynchronously.
+    //!
+    void copyInputToDeviceAsync(cudaStream_t const& stream = 0)
+    {
+        memcpyBuffers(true, false, true, stream);
+    }
+
+    //!
+    //! \brief Copy the contents of output device buffers to output host buffers asynchronously.
+    //!
+    void copyOutputToHostAsync(cudaStream_t const& stream = 0)
+    {
+        memcpyBuffers(false, true, true, stream);
+    }
+
+    ~BufferManager() = default;
+
+private:
+    void* getBuffer(bool const isHost, std::string const& tensorName) const
+    {
+        auto record = mNames.find(tensorName);
+        if (record == mNames.end())
+            return nullptr;
+        return (isHost ? mManagedBuffers[record->second]->hostBuffer.data()
+                       : mManagedBuffers[record->second]->deviceBuffer.data());
+    }
+
+    bool tensorIsInput(const std::string& tensorName) const
+    {
+        return mEngine->getTensorIOMode(tensorName.c_str()) == nvinfer1::TensorIOMode::kINPUT;
+    }
+
+    void memcpyBuffers(bool const copyInput, bool const deviceToHost, bool const async, cudaStream_t const& stream = 0)
+    {
+        for (auto const& n : mNames)
+        {
+            void* dstPtr = deviceToHost ? mManagedBuffers[n.second]->hostBuffer.data()
+                                        : mManagedBuffers[n.second]->deviceBuffer.data();
+            void const* srcPtr = deviceToHost ? mManagedBuffers[n.second]->deviceBuffer.data()
+                                              : mManagedBuffers[n.second]->hostBuffer.data();
+            size_t const byteSize = mManagedBuffers[n.second]->hostBuffer.nbBytes();
+            const cudaMemcpyKind memcpyType = deviceToHost ? cudaMemcpyDeviceToHost : cudaMemcpyHostToDevice;
+            if ((copyInput && tensorIsInput(n.first)) || (!copyInput && !tensorIsInput(n.first)))
+            {
+                if (async)
+                    CHECK(cudaMemcpyAsync(dstPtr, srcPtr, byteSize, memcpyType, stream));
+                else
+                    CHECK(cudaMemcpy(dstPtr, srcPtr, byteSize, memcpyType));
+            }
+        }
+    }
+
+    std::shared_ptr<nvinfer1::ICudaEngine> mEngine;              //!< The pointer to the engine
+    int mBatchSize;                                              //!< The batch size for legacy networks, 0 otherwise.
+    std::vector<std::unique_ptr<ManagedBuffer>> mManagedBuffers; //!< The vector of pointers to managed buffers
+    std::vector<void*> mDeviceBindings;              //!< The vector of device buffers needed for engine execution
+    std::unordered_map<std::string, int32_t> mNames; //!< The map of tensor name and index pairs
+};
+
+} // namespace samplesCommon
+
+#endif // TENSORRT_BUFFERS_H

samples/common/common.cpp

@@ -0,0 +1,52 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common.h"
+
+namespace samplesCommon
+{
+
+using namespace std::string_view_literals;
+
+std::optional<std::string_view> matchFlag(std::string_view arg, std::string_view flag)
+{
+    auto const start = arg.find_first_not_of(' ');
+    if (start == std::string_view::npos)
+    {
+        return std::nullopt;
+    }
+    arg.remove_prefix(start);
+    if (startsWith(arg, flag))
+    {
+        return arg.substr(flag.size());
+    }
+    return std::nullopt;
+}
+
+int32_t parseDLA(int32_t argc, char** argv)
+{
+    for (int32_t i = 1; i < argc; i++)
+    {
+        if (auto v = matchFlag(argv[i], "--useDLACore="sv))
+        {
+            return std::stoi(std::string{v.value()});
+        }
+    }
+    return -1;
+}
+
+} // namespace samplesCommon

samples/common/debugTensorWriter.cpp

@@ -0,0 +1,923 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "debugTensorWriter.h"
+#include "common.h"
+#include <algorithm>
+#include <cuda_bf16.h>
+#include <cuda_fp16.h>
+#if CUDA_VERSION >= 11060
+#include <cuda_fp8.h>
+#endif
+#if CUDA_VERSION >= 12070
+#include <cuda_fp4.h>
+#endif
+#include <cuda_runtime_api.h>
+#include <numeric>
+namespace sample
+{
+
+namespace
+{
+
+class Int4
+{
+public:
+    Int4() = default;
+    explicit Int4(int8_t val)
+        : mValue(val)
+    {
+    }
+
+    operator int64_t() const
+    {
+        return static_cast<int64_t>(mValue);
+    }
+
+private:
+    int8_t mValue{};
+};
+
+class Int4x2
+{
+public:
+    using StorageType = uint8_t;
+
+    Int4x2() = default;
+    explicit Int4x2(StorageType val)
+        : mRep(val)
+    {
+    }
+
+    // Get a single element
+    inline Int4 element(int32_t index) const
+    {
+        ASSERT(index == 0 || index == 1);
+        return Int4(index == 0 ? static_cast<int8_t>(mRep << 4) >> 4 : static_cast<int8_t>(mRep) >> 4);
+    }
+
+private:
+    StorageType mRep{};
+};
+
+#if CUDA_VERSION >= 12070
+using Fp4 = __nv_fp4_e2m1;
+
+class Fp4x2
+{
+public:
+    using StorageType = uint8_t;
+
+    Fp4x2() = default;
+    explicit Fp4x2(StorageType val)
+        : mRep(val)
+    {
+    }
+
+    // Get a single element
+    inline Fp4 element(int32_t index) const
+    {
+        ASSERT(index == 0 || index == 1);
+        int8_t bits = index == 0 ? static_cast<int8_t>(mRep << 4) >> 4 : static_cast<int8_t>(mRep) >> 4;
+        Fp4 fp4_el = *reinterpret_cast<Fp4*>(&bits);
+        return fp4_el;
+    }
+
+private:
+    StorageType mRep{};
+};
+#endif
+
+// Iterator that can handle packed format data (int4 and fp4)
+template <typename T>
+class DataIterator
+{
+public:
+#if CUDA_VERSION >= 12070
+    using value_type
+        = std::conditional_t<std::is_same_v<T, Int4x2>, Int4, std::conditional_t<std::is_same_v<T, Fp4x2>, Fp4, T>>;
+#else
+    using value_type = std::conditional_t<std::is_same_v<T, Int4x2>, Int4, T>;
+#endif
+
+    DataIterator(void const* data, int64_t volume, int64_t index = 0)
+        : mData(static_cast<uint8_t const*>(data))
+        , mVolume(volume)
+        , mIndex(index)
+    {
+    }
+
+    value_type operator*() const
+    {
+        if constexpr (std::is_same_v<T, Int4x2>)
+        {
+            // For Int4x2, each byte contains two 4-bit integers
+            Int4x2 packed(mData[mIndex / 2]);
+            return packed.element(mIndex % 2);
+        }
+#if CUDA_VERSION >= 12070
+        else if constexpr (std::is_same_v<T, Fp4x2>)
+        {
+            // For Fp4x2, each byte contains two 4-bit floating point numbers
+            Fp4x2 packed(mData[mIndex / 2]);
+            return packed.element(mIndex % 2);
+        }
+#endif
+        else
+        {
+            return reinterpret_cast<T const*>(mData)[mIndex];
+        }
+    }
+
+    DataIterator& operator++()
+    {
+        ++mIndex;
+        return *this;
+    }
+
+    DataIterator operator++(int)
+    {
+        DataIterator tmp = *this;
+        ++mIndex;
+        return tmp;
+    }
+
+    bool operator==(DataIterator const& other) const
+    {
+        return mIndex == other.mIndex;
+    }
+
+    bool operator!=(DataIterator const& other) const
+    {
+        return mIndex != other.mIndex;
+    }
+
+    DataIterator operator+(int64_t n) const
+    {
+        DataIterator tmp = *this;
+        tmp.mIndex += n;
+        return tmp;
+    }
+
+private:
+    uint8_t const* mData;
+    int64_t mVolume;
+    int64_t mIndex;
+};
+
+template <typename T>
+class DataRange
+{
+public:
+    using iterator = DataIterator<T>;
+    using value_type = typename iterator::value_type;
+
+    DataRange(void const* data, int64_t volume)
+        : mData(data)
+        , mVolume(volume)
+    {
+    }
+
+    iterator begin() const
+    {
+        return iterator(mData, mVolume, 0);
+    }
+    iterator end() const
+    {
+        return iterator(mData, mVolume, mVolume);
+    }
+
+private:
+    void const* mData;
+    int64_t mVolume;
+};
+
+template <typename T>
+static constexpr bool isFloatingPoint
+    = std::is_floating_point_v<T> || std::is_same_v<T, half> || std::is_same_v<T, nv_bfloat16>
+#if CUDA_VERSION >= 11060
+    || std::is_same_v<T, __nv_fp8_e4m3>
+#endif
+#if CUDA_VERSION >= 12070
+    || std::is_same_v<T, Fp4> || std::is_same_v<T, Fp4x2>
+#endif
+    ;
+
+constexpr int32_t kFLOATING_POINT_PRECISION = 6;
+constexpr int32_t kFLOATING_POINT_WIDTH = 13;
+
+std::string_view getDataTypeString(nvinfer1::DataType type)
+{
+    switch (type)
+    {
+    case nvinfer1::DataType::kBOOL: return "BOOL";
+    case nvinfer1::DataType::kINT4: return "INT4";
+    case nvinfer1::DataType::kINT8: return "INT8";
+    case nvinfer1::DataType::kINT32: return "INT32";
+    case nvinfer1::DataType::kINT64: return "INT64";
+    case nvinfer1::DataType::kUINT8: return "UINT8";
+    case nvinfer1::DataType::kFP4: return "FP4";
+    case nvinfer1::DataType::kFP8: return "FP8";
+    case nvinfer1::DataType::kE8M0: return "E8M0";
+    case nvinfer1::DataType::kHALF: return "HALF";
+    case nvinfer1::DataType::kBF16: return "BF16";
+    case nvinfer1::DataType::kFLOAT: return "FLOAT";
+    }
+    return "UNKNOWN";
+}
+
+template <typename T>
+void printTensorElements(T const* data, int64_t volume, std::ofstream& f)
+{
+    f << "        \"elements\": \"";
+    constexpr int32_t kPRINT_ELEMENTS_COUNT = 10;
+    int64_t firstHalf = std::min(static_cast<int64_t>(kPRINT_ELEMENTS_COUNT / 2), volume);
+    int64_t secondHalf = (volume > kPRINT_ELEMENTS_COUNT)
+        ? kPRINT_ELEMENTS_COUNT / 2
+        : std::max(static_cast<int64_t>(0), volume - kPRINT_ELEMENTS_COUNT / 2);
+
+    auto printElement = [&f](auto value) {
+        if constexpr (isFloatingPoint<T>)
+        {
+            f << static_cast<float>(value);
+        }
+        else
+        {
+            f << static_cast<int64_t>(value);
+        }
+    };
+
+    DataRange<T> range(data, volume);
+    auto it = range.begin();
+
+    // Print first half elements
+    std::string delimiter = "";
+    for (int64_t i = 0; i < firstHalf; ++i)
+    {
+        f << delimiter;
+        printElement(*it++);
+        delimiter = ", ";
+    }
+
+    // Add ellipsis if needed
+    f << (volume > kPRINT_ELEMENTS_COUNT ? ", ..." : "");
+
+    // Print last elements
+    it = range.begin() + (volume - secondHalf);
+    for (int64_t i = volume - secondHalf; i < volume; ++i)
+    {
+        f << ", ";
+        printElement(*it++);
+    }
+
+    f << "\"" << std::endl;
+}
+
+template <typename T>
+void processTensorSummary(void const* addr_host, int64_t volume, std::ofstream& f)
+{
+    DataRange<T> range(addr_host, volume);
+
+    if constexpr (isFloatingPoint<T>)
+    {
+        float minVal = std::numeric_limits<float>::max();
+        float maxVal = std::numeric_limits<float>::lowest();
+        double sum = 0.0;
+
+        for (auto value : range)
+        {
+            float val = static_cast<float>(value);
+            minVal = std::min(minVal, val);
+            maxVal = std::max(maxVal, val);
+            sum += val;
+        }
+        float avgVal = sum / volume;
+
+        // nan and inf turn into string in json
+        auto valueToStr = [](float val) -> std::string {
+            std::stringstream ss;
+            if (!std::isfinite(val))
+            {
+                ss << "\"" << val << "\"";
+            }
+            else
+            {
+                ss << val;
+            }
+            return ss.str();
+        };
+        f << "        \"min\": " << valueToStr(minVal) << "," << std::endl;
+        f << "        \"max\": " << valueToStr(maxVal) << "," << std::endl;
+        f << "        \"avg\": " << valueToStr(avgVal) << "," << std::endl;
+    }
+    else
+    {
+        // For integer types, use int64_t for min/max calculation
+        int64_t minVal = std::numeric_limits<int64_t>::max();
+        int64_t maxVal = std::numeric_limits<int64_t>::lowest();
+        int64_t sum = 0;
+
+        for (auto value : range)
+        {
+            int64_t val = static_cast<int64_t>(value);
+            minVal = std::min(minVal, val);
+            maxVal = std::max(maxVal, val);
+            sum += val;
+        }
+        double avgVal = static_cast<double>(sum) / volume;
+
+        f << "        \"min\": " << minVal << "," << std::endl;
+        f << "        \"max\": " << maxVal << "," << std::endl;
+        f << "        \"avg\": " << avgVal << "," << std::endl;
+    }
+
+    printTensorElements<T>(static_cast<T const*>(addr_host), volume, f);
+}
+
+std::string getCurrentTimeString()
+{
+    auto now = std::chrono::system_clock::now();
+    auto nowC = std::chrono::system_clock::to_time_t(now);
+    std::stringstream ss;
+    ss << std::put_time(std::localtime(&nowC), "%Y-%m-%dT%H:%M:%S%z");
+    return ss.str();
+}
+
+template <typename T>
+void writeTensorStringRecursive(T const* data, nvinfer1::Dims const& shape, int32_t currentDim, int64_t offset,
+    int64_t stride, std::ofstream& f, bool isFirstElement = true, int32_t indent = 0, int32_t maxWidth = 0)
+{
+    bool isLastDim = currentDim == shape.nbDims - 1;
+    if (isLastDim)
+    {
+        // Last dimension - print elements in a row
+        f << std::string(indent, ' ') << "[";
+        DataRange<T> range(data + offset, shape.d[currentDim]);
+        auto it = range.begin();
+        for (int32_t i = 0; i < shape.d[currentDim]; ++i)
+        {
+            if (i > 0)
+            {
+                f << " ";
+            }
+            if constexpr (isFloatingPoint<T>)
+            {
+                f << std::scientific << std::setprecision(kFLOATING_POINT_PRECISION) << std::setw(kFLOATING_POINT_WIDTH)
+                  << std::right << static_cast<float>(*it++);
+            }
+            else
+            {
+                f << std::setw(maxWidth) << static_cast<int64_t>(*it++);
+            }
+        }
+        f << "]" << std::endl;
+    }
+    else
+    {
+        // For higher dimensions, print each slice
+        f << std::string(indent, ' ') << "[" << std::endl;
+        for (int32_t i = 0; i < shape.d[currentDim]; ++i)
+        {
+            writeTensorStringRecursive(data, shape, currentDim + 1, offset + i * stride,
+                stride / shape.d[currentDim + 1], f, i == 0, indent + 1, maxWidth);
+        }
+        f << std::string(indent, ' ') << "]" << std::endl;
+    }
+}
+
+template <typename T>
+int32_t getMaxWidthInDimension(
+    T const* data, nvinfer1::Dims const& shape, int32_t currentDim, int64_t offset, int64_t stride)
+{
+    int32_t maxWidth = 0;
+    if (currentDim == shape.nbDims - 1)
+    {
+        // Last dimension - check each element
+        DataRange<T> range(data + offset, shape.d[currentDim]);
+        for (auto value : range)
+        {
+            std::stringstream ss;
+            ss << static_cast<int64_t>(value);
+            maxWidth = std::max(maxWidth, static_cast<int32_t>(ss.str().length()));
+        }
+    }
+    else
+    {
+        // For higher dimensions, check each slice
+        for (int64_t i = 0; i < shape.d[currentDim]; ++i)
+        {
+            maxWidth = std::max(maxWidth,
+                getMaxWidthInDimension(
+                    data, shape, currentDim + 1, offset + i * stride, stride / shape.d[currentDim + 1]));
+        }
+    }
+    return maxWidth;
+}
+
+template <typename T>
+void writeTensorString(
+    T const* data, nvinfer1::Dims const& shape, std::string_view tensorName, std::string const& fileName)
+{
+    sample::gLogVerbose << "Writing debug tensor '" << tensorName << "' to file '" << fileName << "'" << std::endl;
+
+    std::ofstream f(fileName, std::ios::out);
+    if (!f)
+    {
+        sample::gLogError << "Cannot open file for write: " << fileName << std::endl;
+        return;
+    }
+
+    if (shape.nbDims == 0)
+    {
+        f << "[]";
+        return;
+    }
+
+    int64_t totalElements = 1;
+    for (int32_t i = 0; i < shape.nbDims; ++i)
+    {
+        totalElements *= shape.d[i];
+    }
+
+    if (totalElements == 0)
+    {
+        f << "[]";
+        return;
+    }
+
+    // Calculate stride for the first dimension
+    int64_t stride = totalElements / shape.d[0];
+
+    // Calculate max width for proper alignment only for non-floating point types
+    int32_t maxWidth = 0;
+    if constexpr (!isFloatingPoint<T>)
+    {
+        maxWidth = getMaxWidthInDimension(data, shape, 0, 0, stride);
+    }
+
+    writeTensorStringRecursive(data, shape, 0, 0, stride, f, true, 0, maxWidth);
+    f << std::endl;
+}
+
+std::string writeStringFile(void const* addr_host, nvinfer1::DataType type, nvinfer1::Dims const& shape,
+    std::string const& tensorName, std::string const& prefix)
+{
+    std::string fileName = genFilenameSafeString(prefix + tensorName + ".str");
+
+    switch (type)
+    {
+    case nvinfer1::DataType::kBOOL:
+        writeTensorString(static_cast<bool const*>(addr_host), shape, tensorName, fileName);
+        break;
+    case nvinfer1::DataType::kINT4:
+        writeTensorString(reinterpret_cast<Int4x2 const*>(addr_host), shape, tensorName, fileName);
+        break;
+    case nvinfer1::DataType::kINT8:
+        writeTensorString(static_cast<int8_t const*>(addr_host), shape, tensorName, fileName);
+        break;
+    case nvinfer1::DataType::kINT32:
+        writeTensorString(static_cast<int32_t const*>(addr_host), shape, tensorName, fileName);
+        break;
+    case nvinfer1::DataType::kINT64:
+        writeTensorString(static_cast<int64_t const*>(addr_host), shape, tensorName, fileName);
+        break;
+    case nvinfer1::DataType::kUINT8:
+        writeTensorString(static_cast<uint8_t const*>(addr_host), shape, tensorName, fileName);
+        break;
+    case nvinfer1::DataType::kFP4:
+#if CUDA_VERSION >= 12070
+        writeTensorString(static_cast<Fp4x2 const*>(addr_host), shape, tensorName, fileName);
+        break;
+#else
+        sample::gLogWarning << "Unsupported data type kFP4 for tensor string dump in this CUDA version." << std::endl;
+        return "";
+#endif
+    case nvinfer1::DataType::kFP8:
+#if CUDA_VERSION >= 11060
+        writeTensorString(static_cast<__nv_fp8_e4m3 const*>(addr_host), shape, tensorName, fileName);
+        break;
+#else
+        sample::gLogWarning << "Unsupported data type kFP8 for tensor string dump in this CUDA version." << std::endl;
+        return "";
+#endif
+    case nvinfer1::DataType::kE8M0:
+        sample::gLogWarning << "Unsupported data type kE8M0 for tensor string dump." << std::endl;
+        return "";
+    case nvinfer1::DataType::kHALF:
+        writeTensorString(static_cast<half const*>(addr_host), shape, tensorName, fileName);
+        break;
+    case nvinfer1::DataType::kBF16:
+        writeTensorString(static_cast<nv_bfloat16 const*>(addr_host), shape, tensorName, fileName);
+        break;
+    case nvinfer1::DataType::kFLOAT:
+        writeTensorString(static_cast<float const*>(addr_host), shape, tensorName, fileName);
+        break;
+    }
+    return fileName;
+}
+
+std::string escapeJsonString(std::string_view str)
+{
+    std::string result;
+    result.reserve(str.length());
+    for (char c : str)
+    {
+        switch (c)
+        {
+        case '\\': result += "\\\\"; break;
+        case '\"': result += "\\\""; break;
+        case '\b': result += "\\b"; break;
+        case '\f': result += "\\f"; break;
+        case '\n': result += "\\n"; break;
+        case '\r': result += "\\r"; break;
+        case '\t': result += "\\t"; break;
+        default: result += c;
+        }
+    }
+    return result;
+}
+
+template <typename U, typename T>
+std::vector<U> convertBufferTo(T const* data, int64_t volume)
+{
+    std::vector<U> buffer(volume);
+    DataRange<T> range(data, volume);
+    int64_t i = 0;
+    for (auto value : range)
+    {
+        buffer[i++] = static_cast<U>(value);
+    }
+    return buffer;
+}
+
+} // namespace
+
+DebugTensorWriter::DebugTensorWriter(std::unordered_map<std::string, std::string> const& debugTensorFileNames,
+    std::vector<std::string> const& debugTensorFormats, std::string const& engineName, std::string const& cmdline)
+    : mDebugTensorFileNames(debugTensorFileNames)
+    , mDebugTensorFormats(debugTensorFormats)
+    , mEngineName(engineName)
+    , mCmdline(cmdline)
+{
+    // Create a summary file if "summary" format is requested
+    if (std::find(mDebugTensorFormats.begin(), mDebugTensorFormats.end(), "summary") != mDebugTensorFormats.end())
+    {
+        mSummaryFileName = "tensor_summary.json";
+        mSummaryFile.open(mSummaryFileName, std::ios::out);
+        if (mSummaryFile.is_open())
+        {
+            sample::gLogInfo << "Writing tensor summary to file: " << mSummaryFileName << std::endl;
+            writeSummaryHeader();
+        }
+        else
+        {
+            sample::gLogError << "Failed to open tensor summary file: " << mSummaryFileName << std::endl;
+        }
+    }
+}
+
+DebugTensorWriter::~DebugTensorWriter()
+{
+    // Close the summary file
+    if (mSummaryFile.is_open())
+    {
+        writeSummaryFooter();
+        mSummaryFile.close();
+    }
+}
+
+void DebugTensorWriter::writeSummaryHeader()
+{
+    mSummaryFile << "{" << std::endl;
+    mSummaryFile << "  \"metadata\": {" << std::endl;
+    mSummaryFile << "    \"title\": \"Tensor Summary Report\"," << std::endl;
+    mSummaryFile << "    \"time_generated\": \"" << getCurrentTimeString() << "\"," << std::endl;
+    mSummaryFile << "    \"engine_name\": \"" << mEngineName << "\"," << std::endl;
+    mSummaryFile << "    \"command_line\": \"" << escapeJsonString(mCmdline) << "\"" << std::endl;
+    mSummaryFile << "  }," << std::endl;
+    mSummaryFile << "  \"tensors\": [" << std::endl;
+}
+
+void DebugTensorWriter::writeSummaryFooter()
+{
+    mSummaryFile << std::endl << "  ]" << std::endl;
+    mSummaryFile << "}" << std::endl;
+}
+
+void DebugTensorWriter::writeSummary(std::string_view name, nvinfer1::Dims const& shape, nvinfer1::DataType type,
+    int64_t volume, void const* addr_host, std::string_view assignedFileName, std::string_view numpyFileName,
+    std::string_view stringFileName, std::string_view rawFileName)
+{
+    // Add comma separator if not the first tensor
+    if (!mFirstTensor)
+    {
+        mSummaryFile << "," << std::endl;
+    }
+    mFirstTensor = false;
+
+    // Write tensor information
+    mSummaryFile << "  {\n"
+                 << "    \"name\": \"" << name << "\",\n"
+                 << "    \"shape\": [";
+
+    for (int32_t i = 0; i < shape.nbDims; ++i)
+    {
+        if (i > 0)
+        {
+            mSummaryFile << ", ";
+        }
+        mSummaryFile << shape.d[i];
+    }
+
+    mSummaryFile << "],\n"
+                 << "    \"type\": \"" << getDataTypeString(type) << "\",\n";
+
+    // Write statistics
+    mSummaryFile << "    \"statistics\": {\n";
+
+    switch (type)
+    {
+    case nvinfer1::DataType::kBOOL: processTensorSummary<bool>(addr_host, volume, mSummaryFile); break;
+    case nvinfer1::DataType::kINT4: processTensorSummary<Int4x2>(addr_host, volume, mSummaryFile); break;
+    case nvinfer1::DataType::kINT8: processTensorSummary<int8_t>(addr_host, volume, mSummaryFile); break;
+    case nvinfer1::DataType::kINT32: processTensorSummary<int32_t>(addr_host, volume, mSummaryFile); break;
+    case nvinfer1::DataType::kINT64: processTensorSummary<int64_t>(addr_host, volume, mSummaryFile); break;
+    case nvinfer1::DataType::kUINT8: processTensorSummary<uint8_t>(addr_host, volume, mSummaryFile); break;
+    case nvinfer1::DataType::kFP4:
+#if CUDA_VERSION >= 12070
+        processTensorSummary<Fp4x2>(addr_host, volume, mSummaryFile);
+#else
+        sample::gLogWarning << "Unsupported data type kFP4 for tensor '" << name
+                            << "' summary dump in this CUDA version." << std::endl;
+#endif
+        break;
+    case nvinfer1::DataType::kFP8:
+#if CUDA_VERSION >= 11060
+        processTensorSummary<__nv_fp8_e4m3>(addr_host, volume, mSummaryFile);
+        break;
+#else
+        sample::gLogWarning << "Unsupported data type kFP8 for tensor '" << name
+                            << "' summary dump in this CUDA version." << std::endl;
+#endif
+        break;
+    case nvinfer1::DataType::kE8M0:
+        sample::gLogWarning << "Unsupported data type kE8M0 for tensor '" << name << "' summary dump." << std::endl;
+        break;
+    case nvinfer1::DataType::kHALF: processTensorSummary<half>(addr_host, volume, mSummaryFile); break;
+    case nvinfer1::DataType::kBF16: processTensorSummary<nv_bfloat16>(addr_host, volume, mSummaryFile); break;
+    case nvinfer1::DataType::kFLOAT: processTensorSummary<float>(addr_host, volume, mSummaryFile); break;
+    }
+
+    mSummaryFile << "    }";
+
+    // Write file information only if at least one file exists
+    if (!assignedFileName.empty() || !numpyFileName.empty() || !stringFileName.empty() || !rawFileName.empty())
+    {
+        mSummaryFile << ",\n    \"files\": {\n";
+        std::string delimiter = "";
+
+        if (!assignedFileName.empty())
+        {
+            mSummaryFile << delimiter << "      \"assigned\": \"" << escapeJsonString(assignedFileName) << "\"";
+            delimiter = ",\n";
+        }
+
+        if (!numpyFileName.empty())
+        {
+            mSummaryFile << delimiter << "      \"numpy\": \"" << escapeJsonString(numpyFileName) << "\"";
+            delimiter = ",\n";
+        }
+
+        if (!stringFileName.empty())
+        {
+            mSummaryFile << delimiter << "      \"string\": \"" << escapeJsonString(stringFileName) << "\"";
+            delimiter = ",\n";
+        }
+
+        if (!rawFileName.empty())
+        {
+            mSummaryFile << delimiter << "      \"raw\": \"" << escapeJsonString(rawFileName) << "\"";
+        }
+
+        mSummaryFile << "\n    }";
+    }
+
+    mSummaryFile << "\n  }";
+}
+
+bool writeNumpyFile(void const* addr_host, std::string_view dtype, nvinfer1::Dims const& shape, int64_t size,
+    std::string_view tensorName, std::string const& fileName)
+{
+    sample::gLogVerbose << "Writing debug tensor '" << tensorName << "' to numpy file '" << fileName << "'"
+                        << std::endl;
+
+    std::ofstream f(fileName, std::ios::out | std::ios::binary);
+    if (!f)
+    {
+        sample::gLogError << "Cannot open file for write: " << fileName << std::endl;
+        return false;
+    }
+
+    // Write numpy magic string and version
+    char magic[] = {'\x93', 'N', 'U', 'M', 'P', 'Y'};
+    char version[] = {'\x01', '\x00'};
+    f.write(magic, sizeof(magic));
+    f.write(version, sizeof(version));
+
+    // Construct header
+    std::stringstream header;
+    header << "{'descr': '" << dtype << "', 'fortran_order': False, 'shape': (";
+
+    for (int32_t i = 0; i < shape.nbDims; i++)
+    {
+        header << shape.d[i];
+        header << ", ";
+    }
+    header << "), }";
+
+    // Pad header to 16 bytes alignment
+    std::string headerStr = header.str();
+    int32_t headerLen = 10 + headerStr.length();
+    int32_t padding = 16 - ((headerLen + 1) % 16);
+    headerStr.append(padding, ' ');
+    headerStr += '\n';
+
+    // Write header length and header
+    uint16_t headerSize = headerStr.length();
+    f.write(reinterpret_cast<char*>(&headerSize), sizeof(uint16_t));
+    f.write(headerStr.c_str(), headerSize);
+
+    // Write data
+    f.write(static_cast<char const*>(addr_host), size);
+    f.close();
+
+    return true;
+}
+
+std::string writeNumpy(nvinfer1::DataType type, void const* addr_host, int64_t volume, nvinfer1::Dims const& shape,
+    std::string const& name, std::string const& prefix)
+{
+    std::string fileName = prefix + name;
+    std::string_view dtype = "";
+    void const* data = addr_host;
+    int64_t size = samplesCommon::getNbBytes(type, volume);
+    std::vector<float> floatBuffer;
+    std::vector<int8_t> int8Buffer;
+
+    auto convertToFloat = [&](std::vector<float> const& buffer) {
+        sample::gLogWarning << "Converting " << getDataTypeString(type) << " to float for numpy dump of tensor '"
+                            << name << "'." << std::endl;
+        dtype = "<f4";
+        data = buffer.data();
+        size = volume * sizeof(float);
+        fileName += "_to_float";
+    };
+
+    auto convertToInt8 = [&](std::vector<int8_t> const& buffer) {
+        sample::gLogWarning << "Converting " << getDataTypeString(type) << " to int8 for numpy dump of tensor '" << name
+                            << "'." << std::endl;
+        dtype = "<i1";
+        data = buffer.data();
+        size = volume * sizeof(int8_t);
+        fileName += "_to_int8";
+    };
+
+    switch (type)
+    {
+    case nvinfer1::DataType::kBOOL: dtype = "|b1"; break;
+    case nvinfer1::DataType::kINT4:
+        int8Buffer = convertBufferTo<int8_t>(reinterpret_cast<Int4x2 const*>(addr_host), volume);
+        convertToInt8(int8Buffer);
+        break;
+    case nvinfer1::DataType::kINT8: dtype = "<i1"; break;
+    case nvinfer1::DataType::kINT32: dtype = "<i4"; break;
+    case nvinfer1::DataType::kINT64: dtype = "<i8"; break;
+    case nvinfer1::DataType::kUINT8: dtype = "|u1"; break;
+    case nvinfer1::DataType::kFP4:
+#if CUDA_VERSION >= 12070
+        floatBuffer = convertBufferTo<float>(static_cast<Fp4x2 const*>(addr_host), volume);
+        convertToFloat(floatBuffer);
+#else
+        sample::gLogWarning << "Unsupported data type kFP4 for tensor '" << name << "' numpy dump in this CUDA version."
+                            << std::endl;
+        return "";
+#endif
+        break;
+    case nvinfer1::DataType::kFP8:
+#if CUDA_VERSION >= 11060
+        floatBuffer = convertBufferTo<float>(static_cast<__nv_fp8_e4m3 const*>(addr_host), volume);
+        convertToFloat(floatBuffer);
+#else
+        sample::gLogWarning << "Unsupported data type kFP8 for tensor '" << name << "' numpy dump in this CUDA version."
+                            << std::endl;
+        return "";
+#endif
+        break;
+    case nvinfer1::DataType::kE8M0:
+        sample::gLogWarning << "Unsupported data type kE8M0 for tensor '" << name << "' numpy dump." << std::endl;
+        return "";
+    case nvinfer1::DataType::kHALF: dtype = "<f2"; break;
+    case nvinfer1::DataType::kBF16:
+        floatBuffer = convertBufferTo<float>(static_cast<nv_bfloat16 const*>(addr_host), volume);
+        convertToFloat(floatBuffer);
+        break;
+    case nvinfer1::DataType::kFLOAT: dtype = "<f4"; break;
+    }
+
+    if (!dtype.empty())
+    {
+
+        fileName += ".npy";
+        fileName = genFilenameSafeString(fileName);
+        writeNumpyFile(data, dtype, shape, size, name, fileName);
+        return fileName;
+    }
+    return "";
+}
+
+bool DebugTensorWriter::processDebugTensor(void const* addr, nvinfer1::TensorLocation location, nvinfer1::DataType type,
+    nvinfer1::Dims const& shape, char const* name, cudaStream_t stream)
+{
+    CHECK(cudaStreamSynchronize(stream));
+    // Store data from callback.
+    auto volume = std::accumulate(shape.d, shape.d + shape.nbDims, 1LL, std::multiplies<int64_t>{});
+    int64_t size = samplesCommon::getNbBytes(type, volume);
+    std::vector<char> hostDataOut;
+    void const* addrHost = nullptr;
+    if (location == nvinfer1::TensorLocation::kDEVICE)
+    {
+        hostDataOut.resize(size);
+        CHECK(cudaMemcpy(hostDataOut.data(), addr, size, cudaMemcpyDeviceToHost));
+        addrHost = hostDataOut.data();
+    }
+    else
+    {
+        addrHost = addr;
+    }
+
+    std::string assignedFileName;
+    std::string numpyFileName;
+    std::string rawFileName;
+    std::string stringFileName;
+    auto it = mDebugTensorFileNames.find(name);
+    if (it != mDebugTensorFileNames.end())
+    {
+        assignedFileName = it->second;
+        std::ofstream f(assignedFileName, std::ios::out | std::ios::binary);
+        ASSERT(f && "Cannot open file for write");
+        sample::gLogVerbose << "Writing debug tensor '" << name << "' to file '" << assignedFileName << "'"
+                            << std::endl;
+        f.write(static_cast<char const*>(addrHost), size);
+        f.close();
+    }
+
+    std::stringstream ss;
+    ss << std::setw(4) << std::setfill('0') << mTensorIndex << "_";
+    std::string prefix = ss.str();
+
+    if (std::find(mDebugTensorFormats.begin(), mDebugTensorFormats.end(), "raw") != mDebugTensorFormats.end())
+    {
+        rawFileName = genFilenameSafeString(prefix + name + ".raw");
+        sample::gLogVerbose << "Writing debug tensor '" << name << "' to raw file '" << rawFileName << "'" << std::endl;
+        std::ofstream f(rawFileName, std::ios::out | std::ios::binary);
+        ASSERT(f && "Cannot open file for write");
+        f.write(static_cast<char const*>(addrHost), size);
+        f.close();
+    }
+
+    if (std::find(mDebugTensorFormats.begin(), mDebugTensorFormats.end(), "numpy") != mDebugTensorFormats.end())
+    {
+        numpyFileName = writeNumpy(type, addrHost, volume, shape, name, prefix);
+    }
+
+    if (std::find(mDebugTensorFormats.begin(), mDebugTensorFormats.end(), "string") != mDebugTensorFormats.end())
+    {
+        stringFileName = writeStringFile(addrHost, type, shape, name, prefix);
+    }
+
+    if (std::find(mDebugTensorFormats.begin(), mDebugTensorFormats.end(), "summary") != mDebugTensorFormats.end()
+        && mSummaryFile.is_open())
+    {
+        writeSummary(name, shape, type, volume, addrHost, assignedFileName, numpyFileName, stringFileName, rawFileName);
+        mSummaryFile.flush();
+    }
+
+    mTensorIndex++;
+    return true;
+}
+
+} // namespace sample

samples/common/debugTensorWriter.h

@@ -0,0 +1,59 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TENSORRT_DEBUG_TENSOR_WRITER_H
+#define TENSORRT_DEBUG_TENSOR_WRITER_H
+
+#include "NvInferRuntime.h"
+#include <fstream>
+#include <string>
+#include <unordered_map>
+#include <vector>
+namespace sample
+{
+
+class DebugTensorWriter : public nvinfer1::IDebugListener
+{
+public:
+    DebugTensorWriter(std::unordered_map<std::string, std::string> const& debugTensorFileNames,
+        std::vector<std::string> const& debugTensorFormats, std::string const& engineName = "",
+        std::string const& cmdline = "");
+    ~DebugTensorWriter() override;
+
+    bool processDebugTensor(void const* addr, nvinfer1::TensorLocation location, nvinfer1::DataType type,
+        nvinfer1::Dims const& shape, char const* name, cudaStream_t stream) override;
+
+private:
+    void writeSummaryHeader();
+    void writeSummaryFooter();
+    void writeSummary(std::string_view name, nvinfer1::Dims const& shape, nvinfer1::DataType type, int64_t volume,
+        void const* addr_host, std::string_view assignedFileName, std::string_view numpyFileName,
+        std::string_view stringFileName, std::string_view rawFileName);
+
+    std::unordered_map<std::string, std::string> mDebugTensorFileNames;
+    std::vector<std::string> mDebugTensorFormats;
+    std::string mSummaryFileName;
+    std::ofstream mSummaryFile;
+    bool mFirstTensor{true};
+    std::string mEngineName;
+    std::string mCmdline;
+    int32_t mTensorIndex{0};
+};
+
+} // namespace sample
+
+#endif // TENSORRT_DEBUG_TENSOR_WRITER_H

samples/common/dumpTFWts.py

@@ -0,0 +1,124 @@
+#!/usr/bin/python
+#
+# SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Script to dump TensorFlow weights in TRT v1 and v2 dump format.
+# The V1 format is for TensorRT 4.0. The V2 format is for TensorRT 4.0 and later.
+
+import sys
+import struct
+import argparse
+
+try:
+    import tensorflow as tf
+    from tensorflow.python import pywrap_tensorflow
+except ImportError as err:
+    sys.stderr.write("""Error: Failed to import module ({})""".format(err))
+    sys.exit()
+
+parser = argparse.ArgumentParser(description="TensorFlow Weight Dumper")
+
+parser.add_argument(
+    "-m",
+    "--model",
+    required=True,
+    help="The checkpoint file basename, example basename(model.ckpt-766908.data-00000-of-00001) -> model.ckpt-766908",
+)
+parser.add_argument("-o", "--output", required=True, help="The weight file to dump all the weights to.")
+parser.add_argument("-1", "--wtsv1", required=False, default=False, type=bool, help="Dump the weights in the wts v1.")
+
+opt = parser.parse_args()
+
+if opt.wtsv1:
+    print("Outputting the trained weights in TensorRT's wts v1 format. This format is documented as:")
+    print("Line 0: <number of buffers in the file>")
+    print("Line 1-Num: [buffer name] [buffer type] [buffer size] <hex values>")
+else:
+    print("Outputting the trained weights in TensorRT's wts v2 format. This format is documented as:")
+    print("Line 0: <number of buffers in the file>")
+    print("Line 1-Num: [buffer name] [buffer type] [(buffer shape{e.g. (1, 2, 3)}] <buffer shaped size bytes of data>")
+
+inputbase = opt.model
+outputbase = opt.output
+
+
+def float_to_hex(f):
+    return hex(struct.unpack("<I", struct.pack("<f", f))[0])
+
+
+def getTRTType(tensor):
+    if tf.as_dtype(tensor.dtype) == tf.float32:
+        return 0
+    if tf.as_dtype(tensor.dtype) == tf.float16:
+        return 1
+    if tf.as_dtype(tensor.dtype) == tf.int8:
+        return 2
+    if tf.as_dtype(tensor.dtype) == tf.int32:
+        return 3
+    print("Tensor data type of %s is not supported in TensorRT" % (tensor.dtype))
+    sys.exit()
+
+
+try:
+    # Open output file
+    if opt.wtsv1:
+        outputFileName = outputbase + ".wts"
+    else:
+        outputFileName = outputbase + ".wts2"
+    outputFile = open(outputFileName, "w")
+
+    # read vars from checkpoint
+    reader = pywrap_tensorflow.NewCheckpointReader(inputbase)
+    var_to_shape_map = reader.get_variable_to_shape_map()
+
+    # Record count of weights
+    count = 0
+    for key in sorted(var_to_shape_map):
+        count += 1
+    outputFile.write("%s\n" % (count))
+
+    # Dump the weights in either v1 or v2 format
+    for key in sorted(var_to_shape_map):
+        tensor = reader.get_tensor(key)
+        file_key = key.replace("/", "_")
+        typeOfElem = getTRTType(tensor)
+        val = tensor.shape
+        if opt.wtsv1:
+            val = tensor.size
+        print("%s %s %s " % (file_key, typeOfElem, val))
+        flat_tensor = tensor.flatten()
+        outputFile.write("%s 0 %s " % (file_key, val))
+        if opt.wtsv1:
+            for weight in flat_tensor:
+                hexval = float_to_hex(float(weight))
+                outputFile.write("%s " % (hexval[2:]))
+        else:
+            outputFile.write(flat_tensor.tobytes())
+        outputFile.write("\n")
+    outputFile.close()
+
+except Exception as e:  # pylint: disable=broad-except
+    print(str(e))
+    if "corrupted compressed block contents" in str(e):
+        print("It's likely that your checkpoint file has been compressed " "with SNAPPY.")
+        if "Data loss" in str(e) and (any([e in inputbase for e in [".index", ".meta", ".data"]])):
+            proposed_file = ".".join(inputbase.split(".")[0:-1])
+            v2_file_error_template = """
+           It's likely that this is a V2 checkpoint and you need to provide the filename
+           *prefix*.  Try removing the '.' and extension.  Try:
+           inspect checkpoint --file_name = {}"""
+            print(v2_file_error_template.format(proposed_file))

samples/common/getOptions.cpp

@@ -0,0 +1,286 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "getOptions.h"
+#include "logger.h"
+
+#include <algorithm>
+#include <cassert>
+#include <cctype>
+#include <cstring>
+#include <set>
+
+namespace nvinfer1::utility
+{
+
+namespace
+{
+
+using namespace std::string_view_literals;
+using sample::gLogWarning;
+
+//! Matching for TRTOptions is defined as follows:
+//!
+//! If A and B both have longName set, A matches B if and only if A.longName ==
+//! B.longName and (A.shortName == B.shortName if both have short name set).
+//!
+//! If A only has shortName set and B only has longName set, then A does not
+//! match B. It is assumed that when 2 TRTOptions are compared, one of them is
+//! the definition of a TRTOption in the input to getOptions. As such, if the
+//! definition only has shortName set, it will never be equal to a TRTOption
+//! that does not have shortName set (and same for longName).
+//!
+//! If A and B both have shortName set but B does not have longName set, A
+//! matches B if and only if A.shortName == B.shortName.
+//!
+//! If A has neither long or short name set, A matches B if and only if B has
+//! neither long or short name set.
+[[nodiscard]] bool matches(TRTOption const& a, TRTOption const& b)
+{
+    if (!a.longName.empty() && !b.longName.empty())
+    {
+        if (a.shortName != '\0' && b.shortName != '\0')
+        {
+            return (a.longName == b.longName) && (a.shortName == b.shortName);
+        }
+        return a.longName == b.longName;
+    }
+
+    // If only one of them is not set, this will return false anyway.
+    return a.shortName == b.shortName;
+}
+
+//! getTRTOptionIndex returns the index of a TRTOption in a vector of
+//! TRTOptions, -1 if not found.
+[[nodiscard]] int32_t getTRTOptionIndex(std::vector<TRTOption> const& options, TRTOption const& opt)
+{
+    auto it = std::find_if(
+        options.begin(), options.end(), [&opt](TRTOption const& option) { return matches(opt, option); });
+    return it != options.end() ? static_cast<int32_t>(std::distance(options.begin(), it)) : -1;
+}
+
+//! validateTRTOption will return a string containing an error message if options
+//! contain non-numeric characters, or if there are duplicate option names found.
+//! Otherwise, returns the empty string.
+[[nodiscard]] std::string validateTRTOption(
+    std::set<char> const& seenShortNames, std::set<std::string> const& seenLongNames, TRTOption const& opt)
+{
+    if (opt.shortName != '\0')
+    {
+        if (!std::isalnum(opt.shortName))
+        {
+            return "Short name '" + std::to_string(opt.shortName) + "' is non-alphanumeric";
+        }
+
+        if (seenShortNames.count(opt.shortName) != 0)
+        {
+            return "Short name '" + std::to_string(opt.shortName) + "' is a duplicate";
+        }
+    }
+
+    if (!opt.longName.empty())
+    {
+        for (char const& c : opt.longName)
+        {
+            if (!std::isalnum(c) && c != '-' && c != '_')
+            {
+                return "Long name '" + opt.longName + "' contains characters that are not '-', '_', or alphanumeric";
+            }
+        }
+
+        if (seenLongNames.count(opt.longName) != 0)
+        {
+            return "Long name '" + opt.longName + "' is a duplicate";
+        }
+    }
+    return "";
+}
+
+//! validateTRTOptions will return a string containing an error message if any
+//! options contain non-numeric characters, or if there are duplicate option
+//! names found. Otherwise, returns the empty string.
+[[nodiscard]] std::string validateTRTOptions(std::vector<TRTOption> const& options)
+{
+    std::set<char> seenShortNames;
+    std::set<std::string> seenLongNames;
+    for (size_t i = 0; i < options.size(); ++i)
+    {
+        std::string const errMsg = validateTRTOption(seenShortNames, seenLongNames, options[i]);
+        if (!errMsg.empty())
+        {
+            return "Error '" + errMsg + "' at TRTOption " + std::to_string(i);
+        }
+
+        seenShortNames.insert(options[i].shortName);
+        seenLongNames.insert(options[i].longName);
+    }
+    return "";
+}
+
+//! Structure to hold a parsed option and its inline value (if any)
+struct ParsedOption
+{
+    TRTOption opt;
+    std::string inlineValue;
+};
+
+//! Parse an option string (starting with '-' or '--') into a TRTOption and optional inline value.
+//! \param[in] argStr The option string to parse.
+//! \param[out] result The parsed option and inline value.
+//! \return error message if parsing fails, empty string otherwise.
+[[nodiscard]] std::string parseOptionString(std::string_view const argStr, ParsedOption& result)
+{
+    // C++23: Return a `std::expected<ParsedOption, std::string>` instead.
+    if (argStr.size() < 2)
+    {
+        return "Option string is too short";
+    }
+    if (argStr[1] != '-')
+    {
+        // Short option: must only have 1 char after the hyphen
+        if (argStr.size() > 2)
+        {
+            return "Short arg contains more than 1 character";
+        }
+        result = ParsedOption{TRTOption{argStr[1]}};
+        return {};
+    }
+    else
+    {
+        // Long option: extract name and check for --foo=bar syntax
+        auto longName = argStr.substr(2);
+        size_t const eqIndex = longName.find('=');
+
+        auto inlineValue = eqIndex != std::string_view::npos ? longName.substr(eqIndex + 1) : ""sv;
+
+        // Note: If `eqIndex == std::string_view::npos`, then `longName.substr(0, eqIndex)` is the entire string_view.
+        result = ParsedOption{TRTOption{{}, std::string{longName.substr(0, eqIndex)}}, std::string{inlineValue}};
+        return {};
+    }
+}
+
+//! Handle an option that requires a value. Returns error message if value cannot be obtained.
+//! Updates currentArgIdx if a value is consumed from the next argument.
+[[nodiscard]] std::string handleRequiredValue(TRTParsedArgs& parsedArgs, int32_t idx, std::string inlineValue,
+    std::string_view const argStr, int32_t& currentArgIdx, int32_t argc, char const* const* argv)
+{
+    // If we have an inline value (from --foo=bar), use it
+    if (!inlineValue.empty())
+    {
+        parsedArgs.values[idx].addOccurrence(std::move(inlineValue));
+        return {};
+    }
+
+    // Otherwise, consume the next argument as the value
+    if (currentArgIdx + 1 >= argc)
+    {
+        return "Last argument requires value, but none given";
+    }
+
+    std::string_view const nextArg(argv[currentArgIdx + 1]);
+    if (!nextArg.empty() && nextArg[0] == '-')
+    {
+        gLogWarning << "Warning: Using '" << nextArg << "' as a value for '" << argStr
+                    << "', Should this be its own flag?" << std::endl;
+    }
+
+    parsedArgs.values[idx].addOccurrence(std::string{nextArg});
+    ++currentArgIdx; // Next argument consumed
+    return {};
+}
+
+//! parseArgs parses an argument list and returns a TRTParsedArgs with the
+//! fields set accordingly. Assumes that options is validated.
+//! ErrMsg will be set if:
+//!     - an argument is null
+//!     - an argument is empty
+//!     - an argument does not have option (i.e. "-" and "--")
+//!     - a short argument has more than 1 character
+//!     - the last argument in the list requires a value
+[[nodiscard]] TRTParsedArgs parseArgs(
+    int32_t const argc, char const* const* const argv, std::vector<TRTOption> const& options)
+{
+    TRTParsedArgs parsedArgs;
+    parsedArgs.values.resize(options.size());
+
+    for (int32_t i = 1; i < argc; ++i) // index of current command-line argument
+    {
+        if (argv[i] == nullptr)
+        {
+            return TRTParsedArgs{"Null argument at index " + std::to_string(i)};
+        }
+
+        std::string_view const argStr(argv[i]);
+        if (argStr.empty())
+        {
+            return TRTParsedArgs{"Empty argument at index " + std::to_string(i)};
+        }
+
+        // No starting hyphen means it is a positional argument
+        if (argStr[0] != '-')
+        {
+            parsedArgs.positionalArgs.push_back(std::string{argStr});
+            continue;
+        }
+        if (argStr == "-"sv || argStr == "--"sv)
+        {
+            return TRTParsedArgs{"Argument does not specify an option at index " + std::to_string(i)};
+        }
+
+        // Parse the option string
+        ParsedOption parsed;
+        if (std::string const parseErr = parseOptionString(argStr, parsed); !parseErr.empty())
+        {
+            return TRTParsedArgs{parseErr + " at index " + std::to_string(i)};
+        }
+
+        // Find the option in the registered options list
+        int32_t const idx = getTRTOptionIndex(options, parsed.opt);
+        if (idx < 0)
+        {
+            continue;
+        }
+
+        // Handle value-required options vs. flag options
+        if (options[idx].valueRequired)
+        {
+            if (std::string valueErr = handleRequiredValue(parsedArgs, idx, parsed.inlineValue, argStr, i, argc, argv);
+                !valueErr.empty())
+            {
+                return TRTParsedArgs{std::move(valueErr)};
+            }
+        }
+        else
+        {
+            parsedArgs.values[idx].addOccurrence();
+        }
+    }
+    return parsedArgs;
+}
+
+} // namespace
+
+TRTParsedArgs getOptions(int32_t argc, char const* const* argv, std::vector<TRTOption> const& options)
+{
+    if (std::string errMsg = validateTRTOptions(options); !errMsg.empty())
+    {
+        return TRTParsedArgs{std::move(errMsg)};
+    }
+
+    return parseArgs(argc, argv, options);
+}
+} // namespace nvinfer1::utility

samples/common/getOptions.h

@@ -0,0 +1,135 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_GET_OPTIONS_H
+#define TRT_GET_OPTIONS_H
+
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace nvinfer1::utility
+{
+
+//! TRTOption defines a command line option. At least 1 of shortName and longName
+//! must be defined.
+//! If bool initialization is undefined behavior on your system, valueRequired
+//! must also be explicitly defined.
+//! helpText is optional.
+struct TRTOption
+{
+    char shortName{};     //!< Option name in short (single hyphen) form (e.g., -a, -b); '\0' for no short name.
+    std::string longName; //!< Option name in long (double hyphen) form (e.g., --foo, --bar); empty for no long name.
+    bool valueRequired{}; //!< True if a value is needed for an option (e.g., -N 4, --foo bar); false for not required.
+    std::string helpText; //!< Text to show when printing out the command usage
+};
+
+//! TRTParsedArgs is returned by getOptions after it has parsed a command line
+//! argument list (argv).
+struct TRTParsedArgs
+{
+    //! An error message if any errors occurred. Empty if no errors occurred.
+    std::string errMsg;
+
+    //! A value for an option.
+    struct Value
+    {
+        //! The number of occurrences of the option (for value-required options, this equals `values.size()`).
+        int32_t occurrences{};
+        //! The values for the option. For non-value args, will be empty.
+        std::vector<std::string> values;
+        //! Increment the number of occurrences (for a non-value arg).
+        void addOccurrence()
+        {
+            ++occurrences;
+        }
+        //! Append \p value and set \p occurrences to the number of values.
+        void addOccurrence(std::string value)
+        {
+            values.push_back(std::move(value));
+            occurrences = values.size();
+        }
+    };
+    //! A list of values for each option.
+    std::vector<Value> values;
+    //! Positional arguments that are passed in without an option (these must not start with a hyphen).
+    std::vector<std::string> positionalArgs;
+};
+
+//! Parse the input arguments passed to main() and extract options as well as
+//! positional arguments.
+//!
+//! Options are supposed to be passed to main() with a preceding hyphen '-'.
+//!
+//! If there is a single preceding hyphen, there should be exactly 1 character
+//! after the hyphen, which is interpreted as the option.
+//!
+//! If there are 2 preceding hyphens, the entire argument (without the hyphens)
+//! is interpreted as the option.
+//!
+//! If the option requires a value, the next argument is used as the value.
+//!
+//! Positional arguments must not start with a hyphen.
+//!
+//! If an argument requires a value, the next argument is interpreted as the
+//! value, even if it is the form of a valid option (i.e. --foo --bar will store
+//! "--bar" as a value for option "foo" if "foo" requires a value).
+//! We also support --name=value syntax. In this case, 'value' would be used as
+//! the value, NOT the next argument.
+//!
+//! For options:
+//!   { { 'a', "", false },
+//!     { 'b', "", false },
+//!     { 0, "cee", false },
+//!     { 'd', "", true },
+//!     { 'e', "", true },
+//!     { 'f', "foo", true } }
+//!
+//! ./main hello world -a -a --cee -d 12 -f 34
+//! and
+//! ./main hello world -a -a --cee -d 12 --foo 34
+//!
+//! will result in:
+//!
+//! TRTParsedArgs {
+//!      errMsg: "",
+//!      values: { { 2, {} },
+//!                { 0, {} },
+//!                { 1, {} },
+//!                { 1, {"12"} },
+//!                { 0, {} },
+//!                { 1, {"34"} } }
+//!      positionalArgs: {"hello", "world"},
+//! }
+//!
+//! Non-POSIX behavior:
+//!      - Does not support "-abcde" as a shorthand for "-a -b -c -d -e". Each
+//!        option must have its own hyphen prefix.
+//!      - Does not support -e12 as a shorthand for "-e 12". Values MUST be
+//!        whitespace-separated from the option it is for.
+//!
+//! @param[in] argc The number of arguments passed to main (including the
+//!            file name, which is disregarded)
+//! @param[in] argv The arguments passed to main (including the file name,
+//!            which is disregarded)
+//! @param[in] options List of TRTOptions to parse
+//! @return TRTParsedArgs. See TRTParsedArgs documentation for descriptions of
+//!         the fields.
+[[nodiscard]] TRTParsedArgs getOptions(int argc, char const* const* argv, std::vector<TRTOption> const& options);
+} // namespace nvinfer1::utility
+
+#endif // TRT_GET_OPTIONS_H

samples/common/getOptions.test.cpp

@@ -0,0 +1,145 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "getOptions.h"
+#include "ArgVec.test.h"
+
+#include <gtest/gtest.h>
+
+#include <string_view>
+
+using nvinfer1::utility::getOptions;
+using nvinfer1::utility::TRTOption;
+using namespace std::string_view_literals;
+
+using TestArgVec = ArgVec<char const*>;
+
+// The options used by several tests below, matching the header's worked example.
+static std::vector<TRTOption> const kEXAMPLE_OPTIONS{
+    {'a', "", false},
+    {'b', "", false},
+    {'\0', "cee", false},
+    {'d', "", true},
+    {'e', "", true},
+    {'f', "foo", true},
+};
+
+TEST(GetOptions, PositionalArgs)
+{
+    TestArgVec av{"hello", "world"};
+    auto const result = getOptions(av.argc(), av.argv(), kEXAMPLE_OPTIONS);
+    EXPECT_TRUE(result.errMsg.empty());
+    ASSERT_EQ(result.positionalArgs.size(), 2U);
+    EXPECT_EQ(result.positionalArgs[0], "hello"sv);
+    EXPECT_EQ(result.positionalArgs[1], "world"sv);
+}
+
+TEST(GetOptions, ShortFlag)
+{
+    TestArgVec av{"-a"};
+    auto const result = getOptions(av.argc(), av.argv(), kEXAMPLE_OPTIONS);
+    EXPECT_TRUE(result.errMsg.empty());
+    EXPECT_EQ(result.values[0].occurrences, 1); // 'a' is index 0
+}
+
+TEST(GetOptions, ShortFlagRepeated)
+{
+    TestArgVec av{"-a", "-a"};
+    auto const result = getOptions(av.argc(), av.argv(), kEXAMPLE_OPTIONS);
+    EXPECT_TRUE(result.errMsg.empty());
+    EXPECT_EQ(result.values[0].occurrences, 2);
+}
+
+TEST(GetOptions, LongFlag)
+{
+    TestArgVec av{"--cee"};
+    auto const result = getOptions(av.argc(), av.argv(), kEXAMPLE_OPTIONS);
+    EXPECT_TRUE(result.errMsg.empty());
+    EXPECT_EQ(result.values[2].occurrences, 1); // "cee" is index 2
+}
+
+TEST(GetOptions, ShortValueSpaceSeparated)
+{
+    TestArgVec av{"-d", "12"};
+    auto const result = getOptions(av.argc(), av.argv(), kEXAMPLE_OPTIONS);
+    EXPECT_TRUE(result.errMsg.empty());
+    ASSERT_EQ(result.values[3].occurrences, 1); // 'd' is index 3
+    EXPECT_EQ(result.values[3].values[0], "12"sv);
+}
+
+TEST(GetOptions, LongValueEqualsSign)
+{
+    TestArgVec av{"--foo=34"};
+    auto const result = getOptions(av.argc(), av.argv(), kEXAMPLE_OPTIONS);
+    EXPECT_TRUE(result.errMsg.empty());
+    ASSERT_EQ(result.values[5].occurrences, 1); // "foo" is index 5
+    EXPECT_EQ(result.values[5].values[0], "34"sv);
+}
+
+TEST(GetOptions, ExactExampleFromHeader)
+{
+    // ./main hello world -a -a --cee -d 12 -f 34
+    TestArgVec av{"hello", "world", "-a", "-a", "--cee", "-d", "12", "-f", "34"};
+    auto const result = getOptions(av.argc(), av.argv(), kEXAMPLE_OPTIONS);
+    EXPECT_TRUE(result.errMsg.empty());
+    EXPECT_EQ(result.values[0].occurrences, 2); // 'a'
+    EXPECT_EQ(result.values[1].occurrences, 0); // 'b'
+    EXPECT_EQ(result.values[2].occurrences, 1); // "cee"
+    ASSERT_EQ(result.values[3].occurrences, 1); // 'd'
+    EXPECT_EQ(result.values[3].values[0], "12"sv);
+    EXPECT_EQ(result.values[4].occurrences, 0); // 'e'
+    ASSERT_EQ(result.values[5].occurrences, 1); // "foo"/"f"
+    EXPECT_EQ(result.values[5].values[0], "34"sv);
+    ASSERT_EQ(result.positionalArgs.size(), 2U);
+    EXPECT_EQ(result.positionalArgs[0], "hello"sv);
+    EXPECT_EQ(result.positionalArgs[1], "world"sv);
+}
+
+TEST(GetOptions, UnknownOptionsIgnored)
+{
+    TestArgVec av{"--unknown-flag"};
+    auto const result = getOptions(av.argc(), av.argv(), kEXAMPLE_OPTIONS);
+    EXPECT_TRUE(result.errMsg.empty());
+    for (auto const& v : result.values)
+    {
+        EXPECT_EQ(v.occurrences, 0);
+    }
+}
+
+TEST(GetOptions, MissingRequiredValue)
+{
+    TestArgVec av{"-d"}; // 'd' requires a value but none is given
+    auto const result = getOptions(av.argc(), av.argv(), kEXAMPLE_OPTIONS);
+    EXPECT_FALSE(result.errMsg.empty());
+}
+
+TEST(GetOptions, DuplicateShortName)
+{
+    std::vector<TRTOption> const opts{{'a', "", false}, {'a', "other", false}};
+    TestArgVec av{};
+    auto const result = getOptions(av.argc(), av.argv(), opts);
+    EXPECT_FALSE(result.errMsg.empty());
+}
+
+TEST(GetOptions, EmptyOptions)
+{
+    TestArgVec av{"hello"};
+    auto const result = getOptions(av.argc(), av.argv(), {});
+    EXPECT_TRUE(result.errMsg.empty());
+    ASSERT_EQ(result.positionalArgs.size(), 1U);
+    EXPECT_EQ(result.positionalArgs[0], "hello"sv);
+}

samples/common/getopt.c

@@ -0,0 +1,568 @@
+/*	$OpenBSD: getopt_long.c,v 1.23 2007/10/31 12:34:57 chl Exp $	*/
+/*	$NetBSD: getopt_long.c,v 1.15 2002/01/31 22:43:40 tv Exp $	*/
+
+/*
+ * Copyright (c) 2002 Todd C. Miller <Todd.Miller@courtesan.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Sponsored in part by the Defense Advanced Research Projects
+ * Agency (DARPA) and Air Force Research Laboratory, Air Force
+ * Materiel Command, USAF, under agreement number F39502-99-1-0512.
+ */
+/*-
+ * Copyright (c) 2000 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Dieter Baron and Thomas Klausner.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "getoptWin.h"
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <windows.h>
+
+#define REPLACE_GETOPT /* use this getopt as the system getopt(3) */
+
+#ifdef REPLACE_GETOPT
+int opterr = 1;   /* if error message should be printed */
+int optind = 1;   /* index into parent argv vector */
+int optopt = '?'; /* character checked for validity */
+#undef optreset   /* see getopt.h */
+#define optreset __mingw_optreset
+int optreset; /* reset getopt */
+char* optarg; /* argument associated with option */
+#endif
+
+#define PRINT_ERROR ((opterr) && (*options != ':'))
+
+#define FLAG_PERMUTE 0x01  /* permute non-options to the end of argv */
+#define FLAG_ALLARGS 0x02  /* treat non-options as args to option "-1" */
+#define FLAG_LONGONLY 0x04 /* operate as getopt_long_only */
+
+/* return values */
+#define BADCH (int) '?'
+#define BADARG ((*options == ':') ? (int) ':' : (int) '?')
+#define INORDER (int) 1
+
+#ifndef __CYGWIN__
+#define __progname __argv[0]
+#else
+extern char __declspec(dllimport) * __progname;
+#endif
+
+#ifdef __CYGWIN__
+static char EMSG[] = "";
+#else
+#define EMSG ""
+#endif
+
+static int getopt_internal(int, char* const*, char const*, const struct option*, int*, int);
+static int parse_long_options(char* const*, char const*, const struct option*, int*, int);
+static int gcd(int, int);
+static void permute_args(int, int, int, char* const*);
+
+static char* place = EMSG; /* option letter processing */
+
+/* XXX: set optreset to 1 rather than these two */
+static int nonopt_start = -1; /* first non option argument (for permute) */
+static int nonopt_end = -1;   /* first option after non options (for permute) */
+
+/* Error messages */
+static char const recargchar[] = "option requires an argument -- %c";
+static char const recargstring[] = "option requires an argument -- %s";
+static char const ambig[] = "ambiguous option -- %.*s";
+static char const noarg[] = "option doesn't take an argument -- %.*s";
+static char const illoptchar[] = "unknown option -- %c";
+static char const illoptstring[] = "unknown option -- %s";
+
+static void _vwarnx(char const* fmt, va_list ap)
+{
+    (void) fprintf(stderr, "%s: ", __progname);
+    if (fmt != NULL)
+        (void) vfprintf(stderr, fmt, ap);
+    (void) fprintf(stderr, "\n");
+}
+
+static void warnx(char const* fmt, ...)
+{
+    va_list ap;
+    va_start(ap, fmt);
+    _vwarnx(fmt, ap);
+    va_end(ap);
+}
+
+/*
+ * Compute the greatest common divisor of a and b.
+ */
+static int gcd(int a, int b)
+{
+    int c;
+
+    c = a % b;
+    while (c != 0)
+    {
+        a = b;
+        b = c;
+        c = a % b;
+    }
+
+    return (b);
+}
+
+/*
+ * Exchange the block from nonopt_start to nonopt_end with the block
+ * from nonopt_end to opt_end (keeping the same order of arguments
+ * in each block).
+ */
+static void permute_args(int panonopt_start, int panonopt_end, int opt_end, char* const* nargv)
+{
+    int cstart, cyclelen, i, j, ncycle, nnonopts, nopts, pos;
+    char* swap;
+
+    /*
+     * compute lengths of blocks and number and size of cycles
+     */
+    nnonopts = panonopt_end - panonopt_start;
+    nopts = opt_end - panonopt_end;
+    ncycle = gcd(nnonopts, nopts);
+    cyclelen = (opt_end - panonopt_start) / ncycle;
+
+    for (i = 0; i < ncycle; i++)
+    {
+        cstart = panonopt_end + i;
+        pos = cstart;
+        for (j = 0; j < cyclelen; j++)
+        {
+            if (pos >= panonopt_end)
+                pos -= nnonopts;
+            else
+                pos += nopts;
+            swap = nargv[pos];
+            /* LINTED const cast */
+            ((char**) nargv)[pos] = nargv[cstart];
+            /* LINTED const cast */
+            ((char**) nargv)[cstart] = swap;
+        }
+    }
+}
+
+/*
+ * parse_long_options --
+ *	Parse long options in argc/argv argument vector.
+ * Returns -1 if short_too is set and the option does not match long_options.
+ */
+static int parse_long_options(
+    char* const* nargv, char const* options, const struct option* long_options, int* idx, int short_too)
+{
+    char *current_argv, *has_equal;
+    size_t current_argv_len;
+    int i, ambiguous, match;
+
+#define IDENTICAL_INTERPRETATION(_x, _y)                                                                               \
+    (long_options[(_x)].has_arg == long_options[(_y)].has_arg && long_options[(_x)].flag == long_options[(_y)].flag    \
+        && long_options[(_x)].val == long_options[(_y)].val)
+
+    current_argv = place;
+    match = -1;
+    ambiguous = 0;
+
+    optind++;
+
+    if ((has_equal = strchr(current_argv, '=')) != NULL)
+    {
+        /* argument found (--option=arg) */
+        current_argv_len = has_equal - current_argv;
+        has_equal++;
+    }
+    else
+        current_argv_len = strlen(current_argv);
+
+    for (i = 0; long_options[i].name; i++)
+    {
+        /* find matching long option */
+        if (strncmp(current_argv, long_options[i].name, current_argv_len))
+            continue;
+
+        if (strlen(long_options[i].name) == current_argv_len)
+        {
+            /* exact match */
+            match = i;
+            ambiguous = 0;
+            break;
+        }
+        /*
+         * If this is a known short option, don't allow
+         * a partial match of a single character.
+         */
+        if (short_too && current_argv_len == 1)
+            continue;
+
+        if (match == -1) /* partial match */
+            match = i;
+        else if (!IDENTICAL_INTERPRETATION(i, match))
+            ambiguous = 1;
+    }
+    if (ambiguous)
+    {
+        /* ambiguous abbreviation */
+        if (PRINT_ERROR)
+            warnx(ambig, (int) current_argv_len, current_argv);
+        optopt = 0;
+        return (BADCH);
+    }
+    if (match != -1)
+    { /* option found */
+        if (long_options[match].has_arg == no_argument && has_equal)
+        {
+            if (PRINT_ERROR)
+                warnx(noarg, (int) current_argv_len, current_argv);
+            /*
+             * XXX: GNU sets optopt to val regardless of flag
+             */
+            if (long_options[match].flag == NULL)
+                optopt = long_options[match].val;
+            else
+                optopt = 0;
+            return (BADARG);
+        }
+        if (long_options[match].has_arg == required_argument || long_options[match].has_arg == optional_argument)
+        {
+            if (has_equal)
+                optarg = has_equal;
+            else if (long_options[match].has_arg == required_argument)
+            {
+                /*
+                 * optional argument doesn't use next nargv
+                 */
+                optarg = nargv[optind++];
+            }
+        }
+        if ((long_options[match].has_arg == required_argument) && (optarg == NULL))
+        {
+            /*
+             * Missing argument; leading ':' indicates no error
+             * should be generated.
+             */
+            if (PRINT_ERROR)
+                warnx(recargstring, current_argv);
+            /*
+             * XXX: GNU sets optopt to val regardless of flag
+             */
+            if (long_options[match].flag == NULL)
+                optopt = long_options[match].val;
+            else
+                optopt = 0;
+            --optind;
+            return (BADARG);
+        }
+    }
+    else
+    { /* unknown option */
+        if (short_too)
+        {
+            --optind;
+            return (-1);
+        }
+        if (PRINT_ERROR)
+            warnx(illoptstring, current_argv);
+        optopt = 0;
+        return (BADCH);
+    }
+    if (idx)
+        *idx = match;
+    if (long_options[match].flag)
+    {
+        *long_options[match].flag = long_options[match].val;
+        return (0);
+    }
+    else
+        return (long_options[match].val);
+#undef IDENTICAL_INTERPRETATION
+}
+
+/*
+ * getopt_internal --
+ *	Parse argc/argv argument vector.  Called by user level routines.
+ */
+static int getopt_internal(
+    int nargc, char* const* nargv, char const* options, const struct option* long_options, int* idx, int flags)
+{
+    char const* oli; /* option letter list index */
+    int optchar, short_too;
+    static int posixly_correct = -1;
+
+    if (options == NULL)
+        return (-1);
+
+    /*
+     * XXX Some GNU programs (like cvs) set optind to 0 instead of
+     * XXX using optreset.  Work around this braindamage.
+     */
+    if (optind == 0)
+        optind = optreset = 1;
+
+    /*
+     * Disable GNU extensions if POSIXLY_CORRECT is set or options
+     * string begins with a '+'.
+     *
+     * CV, 2009-12-14: Check POSIXLY_CORRECT anew if optind == 0 or
+     *                 optreset != 0 for GNU compatibility.
+     */
+    if (posixly_correct == -1 || optreset != 0)
+        posixly_correct = (getenv("POSIXLY_CORRECT") != NULL);
+    if (*options == '-')
+        flags |= FLAG_ALLARGS;
+    else if (posixly_correct || *options == '+')
+        flags &= ~FLAG_PERMUTE;
+    if (*options == '+' || *options == '-')
+        options++;
+
+    optarg = NULL;
+    if (optreset)
+        nonopt_start = nonopt_end = -1;
+start:
+    if (optreset || !*place)
+    { /* update scanning pointer */
+        optreset = 0;
+        if (optind >= nargc)
+        { /* end of argument vector */
+            place = EMSG;
+            if (nonopt_end != -1)
+            {
+                /* do permutation, if we have to */
+                permute_args(nonopt_start, nonopt_end, optind, nargv);
+                optind -= nonopt_end - nonopt_start;
+            }
+            else if (nonopt_start != -1)
+            {
+                /*
+                 * If we skipped non-options, set optind
+                 * to the first of them.
+                 */
+                optind = nonopt_start;
+            }
+            nonopt_start = nonopt_end = -1;
+            return (-1);
+        }
+        if (*(place = nargv[optind]) != '-' || (place[1] == '\0' && strchr(options, '-') == NULL))
+        {
+            place = EMSG; /* found non-option */
+            if (flags & FLAG_ALLARGS)
+            {
+                /*
+                 * GNU extension:
+                 * return non-option as argument to option 1
+                 */
+                optarg = nargv[optind++];
+                return (INORDER);
+            }
+            if (!(flags & FLAG_PERMUTE))
+            {
+                /*
+                 * If no permutation wanted, stop parsing
+                 * at first non-option.
+                 */
+                return (-1);
+            }
+            /* do permutation */
+            if (nonopt_start == -1)
+                nonopt_start = optind;
+            else if (nonopt_end != -1)
+            {
+                permute_args(nonopt_start, nonopt_end, optind, nargv);
+                nonopt_start = optind - (nonopt_end - nonopt_start);
+                nonopt_end = -1;
+            }
+            optind++;
+            /* process next argument */
+            goto start;
+        }
+        if (nonopt_start != -1 && nonopt_end == -1)
+            nonopt_end = optind;
+
+        /*
+         * If we have "-" do nothing, if "--" we are done.
+         */
+        if (place[1] != '\0' && *++place == '-' && place[1] == '\0')
+        {
+            optind++;
+            place = EMSG;
+            /*
+             * We found an option (--), so if we skipped
+             * non-options, we have to permute.
+             */
+            if (nonopt_end != -1)
+            {
+                permute_args(nonopt_start, nonopt_end, optind, nargv);
+                optind -= nonopt_end - nonopt_start;
+            }
+            nonopt_start = nonopt_end = -1;
+            return (-1);
+        }
+    }
+
+    /*
+     * Check long options if:
+     *  1) we were passed some
+     *  2) the arg is not just "-"
+     *  3) either the arg starts with -- we are getopt_long_only()
+     */
+    if (long_options != NULL && place != nargv[optind] && (*place == '-' || (flags & FLAG_LONGONLY)))
+    {
+        short_too = 0;
+        if (*place == '-')
+            place++; /* --foo long option */
+        else if (*place != ':' && strchr(options, *place) != NULL)
+            short_too = 1; /* could be short option too */
+
+        optchar = parse_long_options(nargv, options, long_options, idx, short_too);
+        if (optchar != -1)
+        {
+            place = EMSG;
+            return (optchar);
+        }
+    }
+
+    if ((optchar = (int) *place++) == (int) ':' || (optchar == (int) '-' && *place != '\0')
+        || (oli = strchr(options, optchar)) == NULL)
+    {
+        /*
+         * If the user specified "-" and  '-' isn't listed in
+         * options, return -1 (non-option) as per POSIX.
+         * Otherwise, it is an unknown option character (or ':').
+         */
+        if (optchar == (int) '-' && *place == '\0')
+            return (-1);
+        if (!*place)
+            ++optind;
+        if (PRINT_ERROR)
+            warnx(illoptchar, optchar);
+        optopt = optchar;
+        return (BADCH);
+    }
+    if (long_options != NULL && optchar == 'W' && oli[1] == ';')
+    {
+        /* -W long-option */
+        if (*place) /* no space */
+            /* NOTHING */;
+        else if (++optind >= nargc)
+        { /* no arg */
+            place = EMSG;
+            if (PRINT_ERROR)
+                warnx(recargchar, optchar);
+            optopt = optchar;
+            return (BADARG);
+        }
+        else /* white space */
+            place = nargv[optind];
+        optchar = parse_long_options(nargv, options, long_options, idx, 0);
+        place = EMSG;
+        return (optchar);
+    }
+    if (*++oli != ':')
+    { /* doesn't take argument */
+        if (!*place)
+            ++optind;
+    }
+    else
+    { /* takes (optional) argument */
+        optarg = NULL;
+        if (*place) /* no white space */
+            optarg = place;
+        else if (oli[1] != ':')
+        { /* arg not optional */
+            if (++optind >= nargc)
+            { /* no arg */
+                place = EMSG;
+                if (PRINT_ERROR)
+                    warnx(recargchar, optchar);
+                optopt = optchar;
+                return (BADARG);
+            }
+            else
+                optarg = nargv[optind];
+        }
+        place = EMSG;
+        ++optind;
+    }
+    /* dump back option letter */
+    return (optchar);
+}
+
+#ifdef REPLACE_GETOPT
+/*
+ * getopt --
+ *	Parse argc/argv argument vector.
+ *
+ * [eventually this will replace the BSD getopt]
+ */
+int getopt(int nargc, char* const* nargv, char const* options)
+{
+
+    /*
+     * We don't pass FLAG_PERMUTE to getopt_internal() since
+     * the BSD getopt(3) (unlike GNU) has never done this.
+     *
+     * Furthermore, since many privileged programs call getopt()
+     * before dropping privileges it makes sense to keep things
+     * as simple (and bug-free) as possible.
+     */
+    return (getopt_internal(nargc, nargv, options, NULL, NULL, 0));
+}
+#endif /* REPLACE_GETOPT */
+
+/*
+ * getopt_long --
+ *	Parse argc/argv argument vector.
+ */
+int getopt_long(int nargc, char* const* nargv, char const* options, const struct option* long_options, int* idx)
+{
+
+    return (getopt_internal(nargc, nargv, options, long_options, idx, FLAG_PERMUTE));
+}
+
+/*
+ * getopt_long_only --
+ *	Parse argc/argv argument vector.
+ */
+int getopt_long_only(int nargc, char* const* nargv, char const* options, const struct option* long_options, int* idx)
+{
+
+    return (getopt_internal(nargc, nargv, options, long_options, idx, FLAG_PERMUTE | FLAG_LONGONLY));
+}

samples/common/getoptWin.h

@@ -0,0 +1,124 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __GETOPT_H__
+/**
+ * DISCLAIMER
+ * This file has no copyright assigned and is placed in the Public Domain.
+ * This file is a part of the w64 mingw-runtime package.
+ *
+ * The w64 mingw-runtime package and its code is distributed in the hope that it
+ * will be useful but WITHOUT ANY WARRANTY.  ALL WARRANTIES, EXPRESSED OR
+ * IMPLIED ARE HEREBY DISCLAIMED.  This includes but is not limited to
+ * warranties of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ */
+
+#define __GETOPT_H__
+
+/* All the headers include this file. */
+#include <crtdefs.h>
+
+#if defined(WINGETOPT_SHARED_LIB)
+#if defined(BUILDING_WINGETOPT_DLL)
+#define WINGETOPT_API __declspec(dllexport)
+#else
+#define WINGETOPT_API __declspec(dllimport)
+#endif
+#else
+#define WINGETOPT_API
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+    WINGETOPT_API extern int optind; /* index of first non-option in argv      */
+    WINGETOPT_API extern int optopt; /* single option character, as parsed     */
+    WINGETOPT_API extern int opterr; /* flag to enable built-in diagnostics... */
+    /* (user may set to zero, to suppress)    */
+
+    WINGETOPT_API extern char* optarg; /* pointer to argument of current option  */
+
+    extern int getopt(int nargc, char* const* nargv, char const* options);
+
+#ifdef _BSD_SOURCE
+/*
+ * BSD adds the non-standard `optreset' feature, for reinitialisation
+ * of `getopt' parsing.  We support this feature, for applications which
+ * proclaim their BSD heritage, before including this header; however,
+ * to maintain portability, developers are advised to avoid it.
+ */
+#define optreset __mingw_optreset
+    extern int optreset;
+#endif
+#ifdef __cplusplus
+}
+#endif
+/*
+ * POSIX requires the `getopt' API to be specified in `unistd.h';
+ * thus, `unistd.h' includes this header.  However, we do not want
+ * to expose the `getopt_long' or `getopt_long_only' APIs, when
+ * included in this manner.  Thus, close the standard __GETOPT_H__
+ * declarations block, and open an additional __GETOPT_LONG_H__
+ * specific block, only when *not* __UNISTD_H_SOURCED__, in which
+ * to declare the extended API.
+ */
+#endif /* !defined(__GETOPT_H__) */
+
+#if !defined(__UNISTD_H_SOURCED__) && !defined(__GETOPT_LONG_H__)
+#define __GETOPT_LONG_H__
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+    struct option /* specification for a long form option...	*/
+    {
+        char const* name; /* option name, without leading hyphens */
+        int has_arg;      /* does it take an argument?		*/
+        int* flag;        /* where to save its status, or NULL	*/
+        int val;          /* its associated status value		*/
+    };
+
+    enum /* permitted values for its `has_arg' field...	*/
+    {
+        no_argument = 0,   /* option never takes an argument	*/
+        required_argument, /* option always requires an argument	*/
+        optional_argument  /* option may take an argument		*/
+    };
+
+    extern int getopt_long(
+        int nargc, char* const* nargv, char const* options, const struct option* long_options, int* idx);
+    extern int getopt_long_only(
+        int nargc, char* const* nargv, char const* options, const struct option* long_options, int* idx);
+/*
+ * Previous MinGW implementation had...
+ */
+#ifndef HAVE_DECL_GETOPT
+/*
+ * ...for the long form API only; keep this for compatibility.
+ */
+#define HAVE_DECL_GETOPT 1
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* !defined(__UNISTD_H_SOURCED__) && !defined(__GETOPT_LONG_H__) */

samples/common/globalTimerKernel.cu

@@ -0,0 +1,46 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "globalTimerKernel.h"
+
+namespace
+{
+__global__ void readGlobalTimerKernel(uint64_t* timestamp)
+{
+    if (timestamp == nullptr)
+    {
+        return;
+    }
+    uint64_t ts;
+    asm volatile("mov.u64 %0, %%globaltimer;" : "=l"(ts));
+    *timestamp = ts;
+}
+} // namespace
+
+namespace sample
+{
+// NOTE: cudaGetLastError() only surfaces synchronous launch errors (invalid
+// configuration, bad stream, etc.). Any asynchronous execution errors from
+// the kernel itself — e.g. a dereference of a bad device pointer — will not
+// be reported here; they become visible only on a subsequent synchronization
+// point such as cudaEventRecord / cudaStreamSynchronize / cudaDeviceSynchronize.
+cudaError_t launchGlobalTimerKernel(uint64_t* dTimestamp, cudaStream_t stream) noexcept
+{
+    readGlobalTimerKernel<<<1, 1, 0, stream>>>(dTimestamp);
+    return cudaGetLastError();
+}
+} // namespace sample

samples/common/globalTimerKernel.h

@@ -0,0 +1,42 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_GLOBAL_TIMER_KERNEL_H
+#define TRT_GLOBAL_TIMER_KERNEL_H
+
+#include <cstdint>
+#include <cuda_runtime_api.h>
+
+namespace sample
+{
+//! Launch a single-thread kernel that writes the current value of the PTX
+//! %globaltimer register (GPU timer in ns) to \p dTimestamp on \p stream.
+//!
+//! Used as a replacement for cudaEventElapsedTime() when Confidential Compute
+//! is enabled, where cudaEventElapsedTime() is documented to be unreliable.
+//!
+//! \param dTimestamp Device pointer to a uint64_t. Must be non-null and point
+//!     to valid device memory reachable from \p stream.
+//! \param stream CUDA stream on which to launch the kernel.
+//! \return Result of \c cudaGetLastError() after the launch. This reports
+//!     synchronous launch errors only; asynchronous execution errors will
+//!     surface on a subsequent \c cudaEventRecord /
+//!     \c cudaStreamSynchronize / similar.
+[[nodiscard]] cudaError_t launchGlobalTimerKernel(uint64_t* dTimestamp, cudaStream_t stream) noexcept;
+} // namespace sample
+
+#endif // TRT_GLOBAL_TIMER_KERNEL_H

samples/common/half.test.cpp

@@ -0,0 +1,162 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "half.h"
+
+#include <gtest/gtest.h>
+
+#include <cmath>
+#include <limits>
+#include <sstream>
+#include <string>
+#include <type_traits>
+
+using half_float::half;
+using NLF32 = std::numeric_limits<float>;
+using NLHalf = std::numeric_limits<half>;
+
+TEST(Half, Type)
+{
+    static_assert(sizeof(half) == sizeof(uint16_t), "half should be 16 bits!");
+    static_assert(alignof(half) == alignof(uint16_t), "half should be 16 bit aligned!");
+    EXPECT_EQ(half{}.operator float(), 0.0F);
+}
+
+TEST(Half, Constructors)
+{
+    EXPECT_EQ(half{}, 0.0F);
+    EXPECT_EQ(half{1.0F}, 1.0F);
+    EXPECT_EQ(half{-1.0F}, -1.0F);
+    EXPECT_EQ(half{0.0F}, 0.0F);
+    EXPECT_EQ(half{0.5F}, 0.5F);
+    // Preserve sign bit, even for zero.
+    EXPECT_EQ(std::signbit(static_cast<float>(half{-0.0F})), std::signbit(-1.0F));
+    EXPECT_EQ(std::signbit(static_cast<float>(-half{0.0F})), std::signbit(-1.0F));
+    EXPECT_EQ(std::signbit(static_cast<float>(half{0.0F})), std::signbit(1.0F));
+}
+
+TEST(Half, UnaryMinus)
+{
+    half const pos(2.5F);
+    half const neg = -pos;
+    EXPECT_EQ(neg, -2.5F);
+
+    half const negInput(-3.0F);
+    half const posResult = -negInput;
+    EXPECT_EQ(posResult, 3.0F);
+
+    half const zero(0.0F);
+    half const negZero = -zero;
+    EXPECT_EQ(negZero, 0.0F);
+}
+
+TEST(Half, Addition)
+{
+    half const a(1.5F);
+    half const b(2.5F);
+    EXPECT_EQ(a + b, 4.0F);
+
+    half const c(-1.0F);
+    half const d(3.0F);
+    EXPECT_EQ(c + d, 2.0F);
+
+    half const e(0.0F);
+    half const f(5.0F);
+    EXPECT_EQ(e + f, 5.0F);
+}
+
+TEST(Half, FloatConversion)
+{
+    EXPECT_EQ(static_cast<float>(half{3.14159F}), 3.140625F);
+    EXPECT_EQ(static_cast<float>(half{1000.0F}), 1000.0F);
+    EXPECT_EQ(static_cast<float>(half{0.001F}), 0.0010004043F);
+    // Out-of-bounds conversion rounds to infinity.
+    EXPECT_TRUE(std::isinf(static_cast<float>(half{NLF32::max()})));
+}
+
+TEST(Half, StreamOutput)
+{
+    auto toStr = [](auto const& value) {
+        std::stringstream ss;
+        ss << value;
+        return ss.str();
+    };
+    using namespace std::string_view_literals;
+    EXPECT_EQ(toStr(half(2.718F)), "2.71875"sv);
+    EXPECT_EQ(toStr(half(0.0F)), "0"sv);
+    // half should match float stringification for special values.
+    EXPECT_EQ(toStr(half{NLF32::infinity()}), std::to_string(NLF32::infinity()));
+    EXPECT_EQ(toStr(-half{NLF32::infinity()}), std::to_string(-NLF32::infinity()));
+    EXPECT_EQ(toStr(half{NLF32::quiet_NaN()}), std::to_string(NLF32::quiet_NaN()));
+}
+
+TEST(Half, NumericLimits)
+{
+    static_assert(std::numeric_limits<half>::is_specialized);
+    EXPECT_FALSE(std::numeric_limits<half>::is_integer);
+    EXPECT_TRUE(std::numeric_limits<half>::has_infinity);
+
+    constexpr auto kINF = NLHalf::infinity();
+    EXPECT_TRUE(isinf(kINF));
+    EXPECT_LT(0.0F, kINF);
+    EXPECT_TRUE(isinf(-kINF));
+    EXPECT_LT(-kINF, 0.0F);
+
+    constexpr auto kMAX_VAL = NLHalf::max();
+    EXPECT_FALSE(isinf(kMAX_VAL));
+    EXPECT_EQ(kMAX_VAL, 65504.0F);
+    EXPECT_EQ(half(2.0F * 65505.0F), NLHalf::infinity());
+
+    constexpr auto kNAN = NLHalf::quiet_NaN();
+    EXPECT_TRUE(isnan(kNAN));
+    EXPECT_NE(kNAN, kNAN); // NaN is not equal to itself.
+}
+
+TEST(Half, TypeTraits)
+{
+    static_assert(!std::is_arithmetic_v<half>);
+    static_assert(!std::is_scalar_v<half>);
+}
+
+TEST(Half, NaN)
+{
+    EXPECT_TRUE(isnan(half{NLF32::quiet_NaN()}));
+    EXPECT_TRUE(isnan(half{} / half{}));
+    EXPECT_TRUE(isnan(half{half{} / half{}}));
+    EXPECT_TRUE(isnan(half{NLF32::quiet_NaN()} / half{}));
+    EXPECT_TRUE(isnan(half{NLF32::infinity()} - half{NLF32::infinity()}));
+}
+
+TEST(Half, EdgeCases)
+{
+    auto const f16Large = half(1e38F);
+    EXPECT_TRUE(std::isinf(static_cast<float>(f16Large)));
+    EXPECT_LT(NLHalf::max(), f16Large);
+
+    auto const f16Small = half(0x1p-24F); // smallest subnormal
+    EXPECT_LT(0.0F, f16Small);
+    // Note: this library uses HALF_ROUND_TIES_TO_EVEN=0 (ties away from zero) by default.
+    // 0x1p-24 / 2 = 0x1p-25, which is equidistant between 0 and 0x1p-24. Ties-away-from-zero
+    // rounds up to 0x1p-24 rather than down to 0 as IEEE round-to-nearest-even would.
+    EXPECT_EQ(f16Small, half(f16Small / 2.0F));
+}
+
+TEST(Half, PrecisionAndRounding)
+{
+    constexpr float kPRECISE_VALUE = 1.0F + 1e-7F;
+    EXPECT_NEAR(static_cast<float>(half{kPRECISE_VALUE}), kPRECISE_VALUE, 1e-3F);
+}

samples/common/logger.cpp

@@ -0,0 +1,41 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "logger.h"
+#include "ErrorRecorder.h"
+#include "logging.h"
+using namespace nvinfer1;
+SampleErrorRecorder gRecorder;
+namespace sample
+{
+Logger gLogger{Logger::Severity::kINFO};
+LogStreamConsumer gLogVerbose{LOG_VERBOSE(gLogger)};
+LogStreamConsumer gLogInfo{LOG_INFO(gLogger)};
+LogStreamConsumer gLogWarning{LOG_WARN(gLogger)};
+LogStreamConsumer gLogError{LOG_ERROR(gLogger)};
+LogStreamConsumer gLogFatal{LOG_FATAL(gLogger)};
+
+void setReportableSeverity(Logger::Severity severity)
+{
+    gLogger.setReportableSeverity(severity);
+    gLogVerbose.setReportableSeverity(severity);
+    gLogInfo.setReportableSeverity(severity);
+    gLogWarning.setReportableSeverity(severity);
+    gLogError.setReportableSeverity(severity);
+    gLogFatal.setReportableSeverity(severity);
+}
+} // namespace sample

samples/common/logger.h

@@ -0,0 +1,37 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LOGGER_H
+#define LOGGER_H
+
+#include "logging.h"
+
+class SampleErrorRecorder;
+extern SampleErrorRecorder gRecorder;
+namespace sample
+{
+extern Logger gLogger;
+extern LogStreamConsumer gLogVerbose;
+extern LogStreamConsumer gLogInfo;
+extern LogStreamConsumer gLogWarning;
+extern LogStreamConsumer gLogError;
+extern LogStreamConsumer gLogFatal;
+
+void setReportableSeverity(Logger::Severity severity);
+} // namespace sample
+
+#endif // LOGGER_H

samples/common/logging.h

@@ -0,0 +1,687 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TENSORRT_LOGGING_H
+#define TENSORRT_LOGGING_H
+
+#include "NvInferRuntime.h"
+#include "sampleOptions.h"
+#include <cassert>
+#include <ctime>
+#include <iomanip>
+#include <iostream>
+#include <mutex>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <utility>
+
+namespace sample
+{
+
+using Severity = nvinfer1::ILogger::Severity;
+
+class LogStreamConsumerBuffer : public std::stringbuf
+{
+public:
+    LogStreamConsumerBuffer(std::ostream& stream, const std::string& prefix, bool shouldLog)
+        : mOutput(stream)
+        , mPrefix(prefix)
+        , mShouldLog(shouldLog)
+    {
+    }
+
+    LogStreamConsumerBuffer(LogStreamConsumerBuffer&& other) noexcept
+        : mOutput(other.mOutput)
+        , mPrefix(other.mPrefix)
+        , mShouldLog(other.mShouldLog)
+    {
+    }
+    LogStreamConsumerBuffer(const LogStreamConsumerBuffer& other) = delete;
+    LogStreamConsumerBuffer() = delete;
+    LogStreamConsumerBuffer& operator=(const LogStreamConsumerBuffer&) = delete;
+    LogStreamConsumerBuffer& operator=(LogStreamConsumerBuffer&&) = delete;
+
+    ~LogStreamConsumerBuffer() override
+    {
+        // std::streambuf::pbase() gives a pointer to the beginning of the buffered part of the output sequence
+        // std::streambuf::pptr() gives a pointer to the current position of the output sequence
+        // if the pointer to the beginning is not equal to the pointer to the current position,
+        // call putOutput() to log the output to the stream
+        if (pbase() != pptr())
+        {
+            putOutput();
+        }
+    }
+
+    //!
+    //! synchronizes the stream buffer and returns 0 on success
+    //! synchronizing the stream buffer consists of inserting the buffer contents into the stream,
+    //! resetting the buffer and flushing the stream
+    //!
+    int32_t sync() override
+    {
+        putOutput();
+        return 0;
+    }
+
+    void putOutput()
+    {
+        if (mShouldLog)
+        {
+            // prepend timestamp
+            std::time_t timestamp = std::time(nullptr);
+            tm* tm_local = std::localtime(&timestamp);
+            mOutput << "[";
+            mOutput << std::setw(2) << std::setfill('0') << 1 + tm_local->tm_mon << "/";
+            mOutput << std::setw(2) << std::setfill('0') << tm_local->tm_mday << "/";
+            mOutput << std::setw(4) << std::setfill('0') << 1900 + tm_local->tm_year << "-";
+            mOutput << std::setw(2) << std::setfill('0') << tm_local->tm_hour << ":";
+            mOutput << std::setw(2) << std::setfill('0') << tm_local->tm_min << ":";
+            mOutput << std::setw(2) << std::setfill('0') << tm_local->tm_sec << "] ";
+            // std::stringbuf::str() gets the string contents of the buffer
+            // insert the buffer contents pre-appended by the appropriate prefix into the stream
+            mOutput << mPrefix << str();
+        }
+        // set the buffer to empty
+        str("");
+        // flush the stream
+        mOutput.flush();
+    }
+
+    void setShouldLog(bool shouldLog)
+    {
+        mShouldLog = shouldLog;
+    }
+
+private:
+    std::ostream& mOutput;
+    std::string mPrefix;
+    bool mShouldLog{};
+}; // class LogStreamConsumerBuffer
+
+//!
+//! \class LogStreamConsumerBase
+//! \brief Convenience object used to initialize LogStreamConsumerBuffer before std::ostream in LogStreamConsumer
+//!
+class LogStreamConsumerBase
+{
+public:
+    LogStreamConsumerBase(std::ostream& stream, const std::string& prefix, bool shouldLog)
+        : mBuffer(stream, prefix, shouldLog)
+    {
+    }
+
+protected:
+    std::mutex mLogMutex;
+    LogStreamConsumerBuffer mBuffer;
+}; // class LogStreamConsumerBase
+
+//!
+//! \class LogStreamConsumer
+//! \brief Convenience object used to facilitate use of C++ stream syntax when logging messages.
+//!  Order of base classes is LogStreamConsumerBase and then std::ostream.
+//!  This is because the LogStreamConsumerBase class is used to initialize the LogStreamConsumerBuffer member field
+//!  in LogStreamConsumer and then the address of the buffer is passed to std::ostream.
+//!  This is necessary to prevent the address of an uninitialized buffer from being passed to std::ostream.
+//!  Please do not change the order of the parent classes.
+//!
+class LogStreamConsumer : protected LogStreamConsumerBase, public std::ostream
+{
+public:
+    //!
+    //! \brief Creates a LogStreamConsumer which logs messages with level severity.
+    //!  Reportable severity determines if the messages are severe enough to be logged.
+    //!
+    LogStreamConsumer(nvinfer1::ILogger::Severity reportableSeverity, nvinfer1::ILogger::Severity severity)
+        : LogStreamConsumerBase(severityOstream(severity), severityPrefix(severity), severity <= reportableSeverity)
+        , std::ostream(&mBuffer) // links the stream buffer with the stream
+        , mShouldLog(severity <= reportableSeverity)
+        , mSeverity(severity)
+    {
+    }
+
+    LogStreamConsumer(LogStreamConsumer&& other) noexcept
+        : LogStreamConsumerBase(severityOstream(other.mSeverity), severityPrefix(other.mSeverity), other.mShouldLog)
+        , std::ostream(&mBuffer) // links the stream buffer with the stream
+        , mShouldLog(other.mShouldLog)
+        , mSeverity(other.mSeverity)
+    {
+    }
+    LogStreamConsumer(const LogStreamConsumer& other) = delete;
+    LogStreamConsumer() = delete;
+    ~LogStreamConsumer() override = default;
+    LogStreamConsumer& operator=(const LogStreamConsumer&) = delete;
+    LogStreamConsumer& operator=(LogStreamConsumer&&) = delete;
+
+    void setReportableSeverity(Severity reportableSeverity)
+    {
+        mShouldLog = mSeverity <= reportableSeverity;
+        mBuffer.setShouldLog(mShouldLog);
+    }
+
+    std::mutex& getMutex()
+    {
+        return mLogMutex;
+    }
+
+    bool getShouldLog() const
+    {
+        return mShouldLog;
+    }
+
+private:
+    static std::ostream& severityOstream(Severity severity)
+    {
+        return severity >= Severity::kINFO ? std::cout : std::cerr;
+    }
+
+    static std::string severityPrefix(Severity severity)
+    {
+        switch (severity)
+        {
+        case Severity::kINTERNAL_ERROR: return "[F] ";
+        case Severity::kERROR: return "[E] ";
+        case Severity::kWARNING: return "[W] ";
+        case Severity::kINFO: return "[I] ";
+        case Severity::kVERBOSE: return "[V] ";
+        default: assert(0); return "";
+        }
+    }
+
+    bool mShouldLog;
+    Severity mSeverity;
+}; // class LogStreamConsumer
+
+template <typename T>
+LogStreamConsumer& operator<<(LogStreamConsumer& logger, const T& obj)
+{
+    if (logger.getShouldLog())
+    {
+        std::lock_guard<std::mutex> guard(logger.getMutex());
+        auto& os = static_cast<std::ostream&>(logger);
+        os << obj;
+    }
+    return logger;
+}
+
+//!
+//! Special handling std::endl
+//!
+inline LogStreamConsumer& operator<<(LogStreamConsumer& logger, std::ostream& (*f)(std::ostream&) )
+{
+    if (logger.getShouldLog())
+    {
+        std::lock_guard<std::mutex> guard(logger.getMutex());
+        auto& os = static_cast<std::ostream&>(logger);
+        os << f;
+    }
+    return logger;
+}
+
+inline LogStreamConsumer& operator<<(LogStreamConsumer& logger, const nvinfer1::Dims& dims)
+{
+    if (logger.getShouldLog())
+    {
+        std::lock_guard<std::mutex> guard(logger.getMutex());
+        auto& os = static_cast<std::ostream&>(logger);
+        for (int32_t i = 0; i < dims.nbDims; ++i)
+        {
+            os << (i ? "x" : "") << dims.d[i];
+        }
+    }
+    return logger;
+}
+
+template <typename First, typename Second>
+inline LogStreamConsumer& operator<<(LogStreamConsumer& logger, const std::pair<First, Second>& value)
+{
+    if (logger.getShouldLog())
+    {
+        std::lock_guard<std::mutex> guard(logger.getMutex());
+        auto& os = static_cast<std::ostream&>(logger);
+        os << "(" << value.first << ", " << value.second << ")";
+    }
+    return logger;
+}
+
+//!
+//! \class Logger
+//!
+//! \brief Class which manages logging of TensorRT tools and samples
+//!
+//! \details This class provides a common interface for TensorRT tools and samples to log information to the console,
+//! and supports logging two types of messages:
+//!
+//! - Debugging messages with an associated severity (info, warning, error, or internal error/fatal)
+//! - Test pass/fail messages
+//!
+//! The advantage of having all samples use this class for logging as opposed to emitting directly to stdout/stderr is
+//! that the logic for controlling the verbosity and formatting of sample output is centralized in one location.
+//!
+//! In the future, this class could be extended to support dumping test results to a file in some standard format
+//! (for example, JUnit XML), and providing additional metadata (e.g. timing the duration of a test run).
+//!
+//! TODO: For backwards compatibility with existing samples, this class inherits directly from the nvinfer1::ILogger
+//! interface, which is problematic since there isn't a clean separation between messages coming from the TensorRT
+//! library and messages coming from the sample.
+//!
+//! In the future (once all samples are updated to use Logger::getTRTLogger() to access the ILogger) we can refactor the
+//! class to eliminate the inheritance and instead make the nvinfer1::ILogger implementation a member of the Logger
+//! object.
+//!
+class Logger : public nvinfer1::ILogger
+{
+public:
+    explicit Logger(Severity severity = Severity::kWARNING)
+        : mReportableSeverity(severity)
+    {
+    }
+
+    //!
+    //! \enum TestResult
+    //! \brief Represents the state of a given test
+    //!
+    enum class TestResult
+    {
+        kRUNNING,    //!< The test is running
+        kPASSED,     //!< The test passed
+        kFAILED,     //!< The test failed
+        kWAIVED,     //!< The test was waived
+        kTASK_BEGIN, //!< A sub-routine task has begun
+        kTASK_END,   //!< A sub-routine task completed successfully
+        kTASK_ABORT  //!< A sub-routine task was aborted (exception or validation failure)
+    };
+
+    //!
+    //! \brief Forward-compatible method for retrieving the nvinfer1::ILogger associated with this Logger
+    //! \return The nvinfer1::ILogger associated with this Logger
+    //!
+    //! TODO Once all samples are updated to use this method to register the logger with TensorRT,
+    //! we can eliminate the inheritance of Logger from ILogger
+    //!
+    nvinfer1::ILogger& getTRTLogger() noexcept
+    {
+        return *this;
+    }
+
+    //!
+    //! \brief Implementation of the nvinfer1::ILogger::log() virtual method
+    //!
+    //! Note samples should not be calling this function directly; it will eventually go away once we eliminate the
+    //! inheritance from nvinfer1::ILogger
+    //!
+    void log(Severity severity, const char* msg) noexcept override
+    {
+        LogStreamConsumer(mReportableSeverity, severity) << "[TRT] " << std::string(msg) << std::endl;
+    }
+
+    //!
+    //! \brief Method for controlling the verbosity of logging output
+    //!
+    //! \param severity The logger will only emit messages that have severity of this level or higher.
+    //!
+    void setReportableSeverity(Severity severity) noexcept
+    {
+        mReportableSeverity = severity;
+    }
+
+    //!
+    //! \brief Opaque handle that holds logging information for a particular test
+    //!
+    //! This object is an opaque handle to information used by the Logger to print test results.
+    //! The sample must call Logger::defineTest() in order to obtain a TestAtom that can be used
+    //! with Logger::reportTest{Start,End}().
+    //!
+    class TestAtom
+    {
+    public:
+        TestAtom(TestAtom&&) = default;
+
+        std::string getCmdline() const
+        {
+            return mCmdline;
+        }
+
+    private:
+        friend class Logger;
+
+        TestAtom(bool started, const std::string& name, const std::string& cmdline)
+            : mStarted(started)
+            , mName(name)
+            , mCmdline(cmdline)
+        {
+        }
+
+        bool mStarted;
+        std::string mName;
+        std::string mCmdline;
+    };
+
+    //!
+    //! \brief Define a test for logging
+    //!
+    //! \param[in] name The name of the test. This should be a string starting with
+    //!                  "TensorRT" and containing dot-separated strings containing
+    //!                  the characters [A-Za-z0-9_].
+    //!                  For example, "TensorRT.sample_googlenet"
+    //! \param[in] cmdline The command line used to reproduce the test
+    //
+    //! \return a TestAtom that can be used in Logger::reportTest{Start,End}().
+    //!
+    static TestAtom defineTest(const std::string& name, const std::string& cmdline)
+    {
+        return TestAtom(false, name, cmdline);
+    }
+
+    //!
+    //! \brief A convenience overloaded version of defineTest() that accepts an array of command-line arguments
+    //!        as input
+    //!
+    //! \param[in] name The name of the test
+    //! \param[in] argc The number of command-line arguments
+    //! \param[in] argv The array of command-line arguments (given as C strings)
+    //!
+    //! \return a TestAtom that can be used in Logger::reportTest{Start,End}().
+    //!
+    static TestAtom defineTest(const std::string& name, int32_t argc, char const* const* argv)
+    {
+        // Append TensorRT version as info
+        const std::string vname = name + " [TensorRT v" + std::to_string(NV_TENSORRT_VERSION) + "] [b"
+            + std::to_string(NV_TENSORRT_BUILD) + "]";
+        auto cmdline = genCmdlineString(argc, argv);
+        return defineTest(vname, cmdline);
+    }
+
+    //!
+    //! \brief Report that a test has started.
+    //!
+    //! \pre reportTestStart() has not been called yet for the given testAtom
+    //!
+    //! \param[in] testAtom The handle to the test that has started
+    //!
+    static void reportTestStart(TestAtom& testAtom)
+    {
+        reportTestResult(testAtom, TestResult::kRUNNING);
+        assert(!testAtom.mStarted);
+        testAtom.mStarted = true;
+    }
+
+    //!
+    //! \brief Report that a test has ended.
+    //!
+    //! \pre reportTestStart() has been called for the given testAtom
+    //!
+    //! \param[in] testAtom The handle to the test that has ended
+    //! \param[in] result The result of the test. Should be one of TestResult::kPASSED,
+    //!                   TestResult::kFAILED, TestResult::kWAIVED
+    //!
+    static void reportTestEnd(TestAtom const& testAtom, TestResult result)
+    {
+        assert(result != TestResult::kRUNNING);
+        assert(testAtom.mStarted);
+        reportTestResult(testAtom, result);
+    }
+
+    static int32_t reportPass(TestAtom const& testAtom)
+    {
+        reportTestEnd(testAtom, TestResult::kPASSED);
+        return EXIT_SUCCESS;
+    }
+
+    static int32_t reportFail(TestAtom const& testAtom)
+    {
+        reportTestEnd(testAtom, TestResult::kFAILED);
+        return EXIT_FAILURE;
+    }
+
+    static int32_t reportWaive(TestAtom const& testAtom)
+    {
+        reportTestEnd(testAtom, TestResult::kWAIVED);
+        return EXIT_SUCCESS;
+    }
+
+    //!
+    //! \brief Report that a sub-routine task has begun.
+    //!
+    //! Used by the tuning loop to mark the start of each iteration so external
+    //! tooling can detect iteration boundaries in the trtexec log stream.
+    //!
+    static void reportTaskBegin(TestAtom const& testAtom)
+    {
+        reportTestResult(testAtom, TestResult::kTASK_BEGIN);
+    }
+
+    //!
+    //! \brief Report that a sub-routine task has begun with iteration index and build route.
+    //! Prints a blank line before the banner for readability.
+    //!
+    //! Output example:
+    //!   &&&& TASK_BEGIN [iter=0] BuildRoute = '-match_ragged_mha=on -copy_ppg=off'
+    //!
+    static void reportTaskBegin(TestAtom const& /*testAtom*/, std::string const& index, std::string const& buildRoute)
+    {
+        reportTaskWithBuildRoute(
+            TestResult::kTASK_BEGIN, index, buildRoute, /*blankBefore=*/true, /*blankAfter=*/false);
+    }
+
+    //!
+    //! \brief Report that a sub-routine task completed successfully.
+    //!
+    static void reportTaskEnd(TestAtom const& testAtom)
+    {
+        reportTestResult(testAtom, TestResult::kTASK_END);
+    }
+
+    //!
+    //! \brief Report that a sub-routine task completed successfully, with iteration info.
+    //! Prints a blank line after the banner for readability.
+    //!
+    static void reportTaskEnd(TestAtom const& /*testAtom*/, std::string const& index, std::string const& buildRoute)
+    {
+        reportTaskWithBuildRoute(TestResult::kTASK_END, index, buildRoute, /*blankBefore=*/false, /*blankAfter=*/true);
+    }
+
+    //!
+    //! \brief Report that a sub-routine task was aborted (exception or validation failure).
+    //!
+    static void reportTaskAbort(TestAtom const& testAtom)
+    {
+        reportTestResult(testAtom, TestResult::kTASK_ABORT);
+    }
+
+    //!
+    //! \brief Report that a sub-routine task was aborted, with iteration info.
+    //! Prints a blank line after the banner for readability.
+    //!
+    static void reportTaskAbort(TestAtom const& /*testAtom*/, std::string const& index, std::string const& buildRoute)
+    {
+        reportTaskWithBuildRoute(
+            TestResult::kTASK_ABORT, index, buildRoute, /*blankBefore=*/false, /*blankAfter=*/true);
+    }
+
+    static int32_t reportTest(TestAtom const& testAtom, bool pass)
+    {
+        return pass ? reportPass(testAtom) : reportFail(testAtom);
+    }
+
+    Severity getReportableSeverity() const
+    {
+        return mReportableSeverity;
+    }
+
+private:
+    //!
+    //! \brief returns an appropriate string for prefixing a log message with the given severity
+    //!
+    static const char* severityPrefix(Severity severity)
+    {
+        switch (severity)
+        {
+        case Severity::kINTERNAL_ERROR: return "[F] ";
+        case Severity::kERROR: return "[E] ";
+        case Severity::kWARNING: return "[W] ";
+        case Severity::kINFO: return "[I] ";
+        case Severity::kVERBOSE: return "[V] ";
+        default: assert(0); return "";
+        }
+    }
+
+    //!
+    //! \brief returns an appropriate string for prefixing a test result message with the given result
+    //!
+    static const char* testResultString(TestResult result)
+    {
+        switch (result)
+        {
+        case TestResult::kRUNNING: return "RUNNING";
+        case TestResult::kPASSED: return "PASSED";
+        case TestResult::kFAILED: return "FAILED";
+        case TestResult::kWAIVED: return "WAIVED";
+        case TestResult::kTASK_BEGIN: return "TASK_BEGIN";
+        case TestResult::kTASK_END: return "TASK_END";
+        case TestResult::kTASK_ABORT: return "TASK_ABORT";
+        default: assert(0); return "";
+        }
+    }
+
+    //!
+    //! \brief Print a TASK_BEGIN/END/ABORT banner with iteration index and build route.
+    //!
+    //! Output format:
+    //!   &&&& TASK_BEGIN [iter=0] BuildRoute = '-match_ragged_mha=on -copy_ppg=off'
+    //!
+    static void reportTaskWithBuildRoute(
+        TestResult result, std::string const& index, std::string const& buildRoute, bool blankBefore, bool blankAfter)
+    {
+        auto& os = severityOstream(Severity::kINFO);
+        if (blankBefore)
+        {
+            os << std::endl;
+        }
+        os << "&&&& " << testResultString(result) << " [iter=" << index << "] BuildRoute = '" << buildRoute << "'"
+           << std::endl;
+        if (blankAfter)
+        {
+            os << std::endl;
+        }
+    }
+
+    //!
+    //! \brief returns an appropriate output stream (cout or cerr) to use with the given severity
+    //!
+    static std::ostream& severityOstream(Severity severity)
+    {
+        return severity >= Severity::kINFO ? std::cout : std::cerr;
+    }
+
+    //!
+    //! \brief method that implements logging test results
+    //!
+    static void reportTestResult(TestAtom const& testAtom, TestResult result)
+    {
+        severityOstream(Severity::kINFO) << "&&&& " << testResultString(result) << " " << testAtom.mName << " # "
+                                         << testAtom.mCmdline << std::endl;
+    }
+
+    //!
+    //! \brief generate a command line string from the given (argc, argv) values
+    //!        Note: It simply joins the arguments without proper escaping. If spaces is part
+    //!        of an argument, they will be joined with single space.
+    //!
+    static std::string genCmdlineString(int32_t argc, char const* const* argv)
+    {
+        std::stringstream ss;
+        for (int32_t i = 0; i < argc; i++)
+        {
+            if (i > 0)
+            {
+                ss << " ";
+            }
+            ss << argv[i];
+        }
+        return ss.str();
+    }
+
+    Severity mReportableSeverity;
+}; // class Logger
+
+namespace
+{
+//!
+//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kVERBOSE
+//!
+//! Example usage:
+//!
+//!     LOG_VERBOSE(logger) << "hello world" << std::endl;
+//!
+inline LogStreamConsumer LOG_VERBOSE(const Logger& logger)
+{
+    return LogStreamConsumer(logger.getReportableSeverity(), Severity::kVERBOSE);
+}
+
+//!
+//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kINFO
+//!
+//! Example usage:
+//!
+//!     LOG_INFO(logger) << "hello world" << std::endl;
+//!
+inline LogStreamConsumer LOG_INFO(const Logger& logger)
+{
+    return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINFO);
+}
+
+//!
+//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kWARNING
+//!
+//! Example usage:
+//!
+//!     LOG_WARN(logger) << "hello world" << std::endl;
+//!
+inline LogStreamConsumer LOG_WARN(const Logger& logger)
+{
+    return LogStreamConsumer(logger.getReportableSeverity(), Severity::kWARNING);
+}
+
+//!
+//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kERROR
+//!
+//! Example usage:
+//!
+//!     LOG_ERROR(logger) << "hello world" << std::endl;
+//!
+inline LogStreamConsumer LOG_ERROR(const Logger& logger)
+{
+    return LogStreamConsumer(logger.getReportableSeverity(), Severity::kERROR);
+}
+
+//!
+//! \brief produces a LogStreamConsumer object that can be used to log messages of severity kINTERNAL_ERROR
+//!        ("fatal" severity)
+//!
+//! Example usage:
+//!
+//!     LOG_FATAL(logger) << "hello world" << std::endl;
+//!
+inline LogStreamConsumer LOG_FATAL(const Logger& logger)
+{
+    return LogStreamConsumer(logger.getReportableSeverity(), Severity::kINTERNAL_ERROR);
+}
+} // anonymous namespace
+} // namespace sample
+#endif // TENSORRT_LOGGING_H

samples/common/parserOnnxConfig.h

@@ -0,0 +1,146 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef PARSER_ONNX_CONFIG_H
+#define PARSER_ONNX_CONFIG_H
+
+
+#include <cstring>
+#include <iostream>
+#include <string>
+
+#include "NvInfer.h"
+#include "NvOnnxConfig.h"
+#include "NvOnnxParser.h"
+
+#define ONNX_DEBUG 1
+
+/**
+ * \class ParserOnnxConfig
+ * \brief Configuration Manager Class Concrete Implementation
+ *
+ * \note:
+ *
+ */
+
+class ParserOnnxConfig : public nvonnxparser::IOnnxConfig
+{
+
+protected:
+    std::string mModelFilename{};
+    std::string mTextFilename{};
+    std::string mFullTextFilename{};
+    nvinfer1::DataType mModelDtype;
+    nvonnxparser::IOnnxConfig::Verbosity mVerbosity;
+    bool mPrintLayercInfo;
+
+public:
+    ParserOnnxConfig()
+        : mModelDtype(nvinfer1::DataType::kFLOAT)
+        , mVerbosity(static_cast<int>(nvinfer1::ILogger::Severity::kWARNING))
+        , mPrintLayercInfo(false)
+    {
+#ifdef ONNX_DEBUG
+        if (isDebug())
+        {
+            std::cout << " ParserOnnxConfig::ctor(): " << this << "\t" << std::endl;
+        }
+#endif
+    }
+
+    ~ParserOnnxConfig() override
+    {
+#ifdef ONNX_DEBUG
+        if (isDebug())
+        {
+            std::cout << "ParserOnnxConfig::dtor(): " << this << std::endl;
+        }
+#endif
+    }
+
+public:
+    void setModelDtype(const nvinfer1::DataType modelDtype) noexcept override
+    {
+        mModelDtype = modelDtype;
+    }
+
+    nvinfer1::DataType getModelDtype() const noexcept override
+    {
+        return mModelDtype;
+    }
+
+    const char* getModelFileName() const noexcept override
+    {
+        return mModelFilename.c_str();
+    }
+    void setModelFileName(const char* onnxFilename) noexcept override
+    {
+        mModelFilename = std::string(onnxFilename);
+    }
+    nvonnxparser::IOnnxConfig::Verbosity getVerbosityLevel() const noexcept override
+    {
+        return mVerbosity;
+    }
+    void addVerbosity() noexcept override
+    {
+        ++mVerbosity;
+    }
+    void reduceVerbosity() noexcept override
+    {
+        --mVerbosity;
+    }
+    void setVerbosityLevel(nvonnxparser::IOnnxConfig::Verbosity verbosity) noexcept override
+    {
+        mVerbosity = verbosity;
+    }
+
+    const char* getTextFileName() const noexcept override
+    {
+        return mTextFilename.c_str();
+    }
+    void setTextFileName(const char* textFilename) noexcept override
+    {
+        mTextFilename = std::string(textFilename);
+    }
+    const char* getFullTextFileName() const noexcept override
+    {
+        return mFullTextFilename.c_str();
+    }
+    void setFullTextFileName(const char* fullTextFilename) noexcept override
+    {
+        mFullTextFilename = std::string(fullTextFilename);
+    }
+    bool getPrintLayerInfo() const noexcept override
+    {
+        return mPrintLayercInfo;
+    }
+    void setPrintLayerInfo(bool src) noexcept override
+    {
+        mPrintLayercInfo = src;
+    } //!< get the boolean variable corresponding to the Layer Info, see getPrintLayerInfo()
+
+    virtual bool isDebug() const noexcept
+    {
+#if ONNX_DEBUG
+        return std::getenv("ONNX_DEBUG") != nullptr;
+#else
+        return false;
+#endif
+    }
+}; // class ParserOnnxConfig
+
+#endif // PARSER_ONNX_CONFIG_H

samples/common/safeCommon.h

@@ -0,0 +1,663 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TENSORRT_SAFE_COMMON_H
+#define TENSORRT_SAFE_COMMON_H
+
+#include "NvInferRuntimeBase.h"
+#include "NvInferSafeRecorder.h"
+#include "NvInferSafeRuntime.h"
+#include "cuda_runtime.h"
+#include <algorithm>
+#include <cerrno>
+#include <cmath>
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <memory>
+#include <numeric>
+#include <optional>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+// For safeLoadLibrary
+#ifdef _MSC_VER
+// Needed so that the max/min definitions in windows.h do not conflict with std::max/min.
+#define NOMINMAX
+#include <windows.h>
+#undef NOMINMAX
+#else
+#include <dlfcn.h>
+#endif
+#if IS_QNX_SAFE
+#include <cuda_runtime_api_safe_ex.h>
+#include <sys/procmgr.h>
+#endif // IS_QNX_SAFE
+
+using namespace nvinfer1;
+
+#undef CHECK_WITH_STREAM
+#define CHECK_WITH_STREAM(status, stream)                                                                              \
+    do                                                                                                                 \
+    {                                                                                                                  \
+        if ((status) != cudaSuccess)                                                                                   \
+        {                                                                                                              \
+            stream << "Cuda failure at " << __FILE__ << ":" << __LINE__ << ": " << cudaGetErrorString(status)          \
+                   << std::endl;                                                                                       \
+            exit(EXIT_FAILURE);                                                                                        \
+        }                                                                                                              \
+    } while (0)
+
+#undef CUDA_CHECK
+#define CUDA_CHECK(status) CHECK_WITH_STREAM(status, std::cerr)
+
+#define SAFE_LOG std::cerr
+
+inline std::string getTimestampStr()
+{
+    std::time_t timestamp = std::time(nullptr);
+    tm* tm_local = std::localtime(&timestamp);
+    std::stringstream ss;
+    ss << "[";
+    ss << std::setw(2) << std::setfill('0') << 1 + tm_local->tm_mon << "/";
+    ss << std::setw(2) << std::setfill('0') << tm_local->tm_mday << "/";
+    ss << std::setw(4) << std::setfill('0') << 1900 + tm_local->tm_year << "-";
+    ss << std::setw(2) << std::setfill('0') << tm_local->tm_hour << ":";
+    ss << std::setw(2) << std::setfill('0') << tm_local->tm_min << ":";
+    ss << std::setw(2) << std::setfill('0') << tm_local->tm_sec << "] ";
+    return ss.str();
+}
+
+inline void safeLogDebug(nvinfer2::safe::ISafeRecorder& recorder, std::string desc)
+{
+    desc = getTimestampStr() + "[D] " + desc;
+    recorder.reportDebug(desc.c_str());
+}
+
+inline void safeLogVerbose(nvinfer2::safe::ISafeRecorder& recorder, std::string desc)
+{
+    desc = getTimestampStr() + "[V] " + desc;
+    recorder.reportVerbose(desc.c_str());
+}
+
+inline void safeLogInfo(nvinfer2::safe::ISafeRecorder& recorder, std::string desc)
+{
+    desc = getTimestampStr() + "[I] " + desc;
+    recorder.reportInfo(desc.c_str());
+}
+
+inline void safeLogWarning(nvinfer2::safe::ISafeRecorder& recorder, std::string desc)
+{
+    desc = getTimestampStr() + "[W] " + desc;
+    recorder.reportWarn(desc.c_str());
+}
+
+inline void safeLogError(
+    nvinfer2::safe::ISafeRecorder& recorder, std::string desc, ErrorCode val = ErrorCode::kFAILED_EXECUTION)
+{
+    desc = getTimestampStr() + "[E] " + desc;
+    recorder.reportError(val, desc.c_str());
+}
+
+#undef SAFE_ASSERT
+#define SAFE_ASSERT(condition)                                                                                         \
+    do                                                                                                                 \
+    {                                                                                                                  \
+        if (!(condition))                                                                                              \
+        {                                                                                                              \
+            std::cerr << "Assertion failure: " << #condition << std::endl;                                             \
+            exit(EXIT_FAILURE);                                                                                        \
+        }                                                                                                              \
+    } while (0)
+
+#define SAFE_API_CALL(api_call, recorder)                                                                              \
+    do                                                                                                                 \
+    {                                                                                                                  \
+        std::stringstream ss;                                                                                          \
+        const ErrorCode ret = (api_call);                                                                              \
+        if (ret != ErrorCode::kSUCCESS)                                                                                \
+        {                                                                                                              \
+            ss << "SAFE API Error: [" << #api_call << "]: " << toString(ret);                                          \
+            safeLogError(recorder, ss.str(), ret);                                                                     \
+            throw ErrorCode{ret};                                                                                      \
+        }                                                                                                              \
+        ss << "SAFE API:[" << #api_call << "]: PASSED";                                                                \
+        safeLogVerbose(recorder, ss.str());                                                                            \
+    } while (0)
+
+#define CUDA_CALL(cuda_api_call, recorder)                                                                             \
+    do                                                                                                                 \
+    {                                                                                                                  \
+        std::stringstream ss;                                                                                          \
+        cudaError_t error = (cuda_api_call);                                                                           \
+        if (error != cudaSuccess)                                                                                      \
+        {                                                                                                              \
+            ss << "CUDA Error: [" << #cuda_api_call << "]: " << cudaGetErrorString(error);                             \
+            auto ret = ErrorCode::kFAILED_EXECUTION;                                                                   \
+            safeLogError(recorder, ss.str(), ret);                                                                     \
+            throw ErrorCode{ret};                                                                                      \
+        }                                                                                                              \
+        ss << "CUDA:[" << #cuda_api_call << "]: PASSED";                                                               \
+        safeLogVerbose(recorder, ss.str());                                                                            \
+    } while (0)
+
+inline std::string toString(ErrorCode ec)
+{
+    static const auto ecStrings = [] {
+        std::unordered_map<ErrorCode, std::string> result;
+#define INSERT_ELEMENT(p, s) result.emplace(p, s);
+        INSERT_ELEMENT(ErrorCode::kSUCCESS, "SUCCESS")
+        INSERT_ELEMENT(ErrorCode::kUNSPECIFIED_ERROR, "UNSPECIFIED_ERROR")
+        INSERT_ELEMENT(ErrorCode::kINTERNAL_ERROR, "INTERNAL_ERROR")
+        INSERT_ELEMENT(ErrorCode::kINVALID_ARGUMENT, "INVALID_ARGUMENT")
+        INSERT_ELEMENT(ErrorCode::kINVALID_CONFIG, "INVALID_CONFIG")
+        INSERT_ELEMENT(ErrorCode::kFAILED_ALLOCATION, "FAILED_ALLOCATION")
+        INSERT_ELEMENT(ErrorCode::kFAILED_INITIALIZATION, "FAILED_INITIALIZATION")
+        INSERT_ELEMENT(ErrorCode::kFAILED_EXECUTION, "FAILED_EXECUTION")
+        INSERT_ELEMENT(ErrorCode::kFAILED_COMPUTATION, "FAILED_COMPUTATION")
+        INSERT_ELEMENT(ErrorCode::kINVALID_STATE, "INVALID_STATE")
+        INSERT_ELEMENT(ErrorCode::kUNSUPPORTED_STATE, "UNSUPPORTED_STATE")
+#undef INSERT_ELEMENT
+        return result;
+    }();
+    return ecStrings.at(ec);
+}
+
+//! Locate path to file, given its filename or filepath suffix and possible dirs it might lie in.
+//! Function will also walk back MAX_DEPTH dirs from CWD to check for such a file path.
+inline std::string locateFile(
+    const std::string& filepathSuffix, const std::vector<std::string>& directories, bool reportError = true)
+{
+    const int MAX_DEPTH{10};
+    bool found{false};
+    std::string filepath;
+
+    for (auto& dir : directories)
+    {
+        if (!dir.empty() && dir.back() != '/')
+        {
+#ifdef _MSC_VER
+            filepath = dir + "\\" + filepathSuffix;
+#else
+            filepath = dir + "/" + filepathSuffix;
+#endif
+        }
+        else
+        {
+            filepath = dir + filepathSuffix;
+        }
+
+        for (int i = 0; i < MAX_DEPTH && !found; i++)
+        {
+            const std::ifstream checkFile(filepath);
+            found = checkFile.is_open();
+            if (found)
+            {
+                break;
+            }
+
+            filepath = "../" + filepath; // Try again in parent dir
+        }
+
+        if (found)
+        {
+            break;
+        }
+
+        filepath.clear();
+    }
+
+    // Could not find the file
+    if (filepath.empty())
+    {
+        const std::string dirList = std::accumulate(directories.begin() + 1, directories.end(), directories.front(),
+            [](const std::string& a, const std::string& b) { return a + "\n\t" + b; });
+        std::cout << "Could not find " << filepathSuffix << " in data directories:\n\t" << dirList << std::endl;
+
+        if (reportError)
+        {
+            std::cout << "&&&& FAILED" << std::endl;
+            exit(EXIT_FAILURE);
+        }
+    }
+
+    return filepath;
+}
+
+inline void readPGMFile(const std::string& fileName, uint8_t* buffer, int32_t inH, int32_t inW)
+{
+    std::ifstream infile(fileName, std::ifstream::binary);
+    SAFE_ASSERT(infile.is_open() && "Attempting to read from a file that is not open.");
+    std::string magic, w, h, max;
+    infile >> magic >> w >> h >> max;
+    infile.seekg(1, infile.cur);
+    infile.read(reinterpret_cast<char*>(buffer), inH * inW);
+}
+
+namespace samplesSafeCommon
+{
+//! Represents the compute capability of a device.
+//! This pertains to virtual architectures represented by the intermediate PTX format.
+//! This is distinct from the SM version.
+//! See https://forums.developer.nvidia.com/t/how-should-i-use-correctly-the-sm-xx-and-compute-xx/219160
+struct ComputeCapability
+{
+    int32_t major{};
+    int32_t minor{};
+
+    //! \return the compute capability of the CUDA device with the given \p deviceIndex.
+    [[nodiscard]] static ComputeCapability forDevice(int32_t deviceIndex)
+    {
+        int32_t major{0};
+        int32_t minor{0};
+        CUDA_CHECK(cudaDeviceGetAttribute(&major, cudaDevAttrComputeCapabilityMajor, deviceIndex));
+        CUDA_CHECK(cudaDeviceGetAttribute(&minor, cudaDevAttrComputeCapabilityMinor, deviceIndex));
+        return {major, minor};
+    }
+};
+
+inline int32_t getSmVersion()
+{
+    int32_t deviceIndex{};
+    CUDA_CHECK(cudaGetDevice(&deviceIndex));
+
+    auto const cc = ComputeCapability::forDevice(deviceIndex);
+    return ((cc.major << 8) | cc.minor);
+}
+
+inline bool isSmSafe()
+{
+    int32_t const smVersion = getSmVersion();
+    return smVersion == 0x0705 || smVersion == 0x0800 || smVersion == 0x0806 || smVersion == 0x0807
+        || smVersion == 0x0A00 || smVersion == 0x0B00;
+}
+
+inline int32_t calculateSoftmax(float* const prob, int32_t const numDigits)
+{
+    SAFE_ASSERT(prob != nullptr);
+    SAFE_ASSERT(numDigits == 10);
+    float sum{0.0F};
+    std::transform(prob, prob + numDigits, prob, [&sum](float v) -> float {
+        sum += exp(v);
+        return exp(v);
+    });
+
+    SAFE_ASSERT(sum != 0.0F);
+    std::transform(prob, prob + numDigits, prob, [sum](float v) -> float { return v / sum; });
+    int32_t idx = std::max_element(prob, prob + numDigits) - prob;
+    return idx;
+}
+
+//!
+//! \brief generate a command line string from the given (argc, argv) values
+//!        Note: It simply joins the arguments without proper escaping. If spaces is part
+//!        of an argument, they will be joined with single space.
+//!
+static std::string genCmdlineString(int32_t argc, char const* const* argv)
+{
+    std::stringstream ss;
+    for (int32_t i = 0; i < argc; i++)
+    {
+        if (i > 0)
+        {
+            ss << " ";
+        }
+        ss << argv[i];
+    }
+    return ss.str();
+}
+
+//!
+//! \enum TestResult
+//! \brief Represents the state of a given test
+//!
+enum class TestResult
+{
+    kFAILED, //!< The test failed
+    kPASSED, //!< The test passed
+};
+
+
+//!
+//! \brief method that implements logging test start
+//!
+inline void reportTestStart(std::string testName, int32_t argc, char const* const* argv)
+{
+    SAFE_LOG << "&&&& RUNNING " << testName << " [TensorRT v" << std::to_string(NV_TENSORRT_VERSION) << "] [b"
+             << std::to_string(NV_TENSORRT_BUILD) << "]" << " # " << genCmdlineString(argc, argv) << std::endl;
+}
+
+//!
+//! \brief method that implements logging test results
+//!
+inline void reportTestResult(std::string testName, TestResult result, int32_t argc, char const* const* argv)
+{
+    SAFE_LOG << "&&&& " << (result == TestResult::kPASSED ? "PASSED" : "FAILED") << " " << testName << " [TensorRT v"
+             << std::to_string(NV_TENSORRT_VERSION) << "] [b" << std::to_string(NV_TENSORRT_BUILD) << "]"
+             << " # " << genCmdlineString(argc, argv) << std::endl;
+}
+
+//!
+//! \class TrtCudaGraphSafe
+//! \brief Managed CUDA graph
+//!
+class TrtCudaGraphSafe
+{
+public:
+    explicit TrtCudaGraphSafe() = default;
+
+    TrtCudaGraphSafe(const TrtCudaGraphSafe&) = delete;
+
+    TrtCudaGraphSafe& operator=(const TrtCudaGraphSafe&) = delete;
+
+    TrtCudaGraphSafe(TrtCudaGraphSafe&&) = delete;
+
+    TrtCudaGraphSafe& operator=(TrtCudaGraphSafe&&) = delete;
+
+    ~TrtCudaGraphSafe()
+    {
+        if (mGraphExec)
+        {
+            cudaGraphExecDestroy(mGraphExec);
+        }
+    }
+
+    void beginCapture(cudaStream_t& stream)
+    {
+        CUDA_CHECK(cudaStreamBeginCapture(stream, cudaStreamCaptureModeThreadLocal));
+    }
+
+    bool launch(cudaStream_t& stream)
+    {
+        return cudaGraphLaunch(mGraphExec, stream) == cudaSuccess;
+    }
+
+    void endCapture(cudaStream_t& stream)
+    {
+        CUDA_CHECK(cudaStreamEndCapture(stream, &mGraph));
+        CUDA_CHECK(cudaGraphInstantiate(&mGraphExec, mGraph, nullptr, nullptr, 0));
+        CUDA_CHECK(cudaGraphDestroy(mGraph));
+    }
+
+    void endCaptureOnError(cudaStream_t& stream)
+    {
+        // There are two possibilities why stream capture would fail:
+        // (1) stream is in cudaErrorStreamCaptureInvalidated state.
+        // (2) TRT reports a failure.
+        // In case (1), the returning mGraph should be nullptr.
+        // In case (2), the returning mGraph is not nullptr, but it should not be used.
+        const auto ret = cudaStreamEndCapture(stream, &mGraph);
+        if (ret == cudaErrorStreamCaptureInvalidated)
+        {
+            SAFE_ASSERT(mGraph == nullptr);
+        }
+        else
+        {
+            SAFE_ASSERT(ret == cudaSuccess);
+            SAFE_ASSERT(mGraph != nullptr);
+            CUDA_CHECK(cudaGraphDestroy(mGraph));
+            mGraph = nullptr;
+        }
+        // Clean up any CUDA error.
+        cudaGetLastError();
+        SAFE_LOG << "The CUDA graph capture on the stream has failed." << std::endl;
+    }
+
+private:
+    cudaGraph_t mGraph{};
+    cudaGraphExec_t mGraphExec{};
+};
+
+inline void* safeLoadLibrary(const std::string& path)
+{
+#ifdef _MSC_VER
+    void* handle = LoadLibraryA(path.c_str());
+#else
+    int32_t flags{RTLD_LAZY};
+    void* handle = dlopen(path.c_str(), flags);
+#endif
+    if (handle == nullptr)
+    {
+#ifdef _MSC_VER
+        sample::gLogError << "Could not load plugin library: " << path << std::endl;
+#else
+        SAFE_LOG << "Could not load plugin library: " << path << ", due to: " << dlerror() << std::endl;
+#endif
+    }
+    return handle;
+}
+
+//!
+//! \class SafetyPluginAttribute
+//! \brief Represents a safety plugin with its namespace and name
+//!
+class SafetyPluginAttribute
+{
+public:
+    std::string pluginNamespace; //!< Plugin namespace (optional, can be empty)
+    std::string pluginName;      //!< Plugin name
+};
+
+//!
+//! \class SafetyPluginLibraryArgument
+//! \brief Represents a safety plugin library with its name and associated plugin attributes
+//!        Used for parsing command line arguments in the format: libraryName[namespace::pluginName1,pluginName2]
+//!
+class SafetyPluginLibraryArgument
+{
+public:
+    std::string libraryName;                        //!< Name of the plugin library
+    std::vector<SafetyPluginAttribute> pluginAttrs; //!< Vector of plugin attributes contained in this library
+};
+
+inline std::vector<std::string> safeSplitString(std::string str, char delimiter = ',')
+{
+    std::vector<std::string> splitVect;
+    std::stringstream ss(str);
+    std::string substr;
+
+    while (ss.good())
+    {
+        getline(ss, substr, delimiter);
+        splitVect.emplace_back(std::move(substr));
+    }
+    return splitVect;
+}
+
+// Safety plugin cmd argument example: safetyPluginLibrary[namespace::pluginName1,pluginName2]
+inline bool parseSafetyPluginArgument(std::string const& option, SafetyPluginLibraryArgument& args)
+{
+    auto const leftBracketIdx = option.find('[');
+    auto const rightBracketIdx = option.find(']');
+    if (leftBracketIdx == std::string::npos || rightBracketIdx == std::string::npos || leftBracketIdx > rightBracketIdx)
+    {
+        SAFE_LOG << "Invalid safety plugin argument: " << option << std::endl;
+        return false;
+    }
+    args.libraryName = option.substr(0, leftBracketIdx);
+    auto const pluginOptionStr = option.substr(leftBracketIdx + 1, rightBracketIdx - leftBracketIdx - 1);
+    auto const pluginOptions = safeSplitString(pluginOptionStr, ',');
+    if (args.libraryName.empty() || pluginOptions.empty())
+    {
+        SAFE_LOG << "Invalid safety plugin argument: " << option << std::endl;
+        return false;
+    }
+
+    auto parsePluginOption = [](std::string const& pluginOption) {
+        SafetyPluginAttribute attr{};
+        // Check if namespace is used, leave as empty if not exist
+        auto const sepratorIdx = pluginOption.find("::");
+        if (sepratorIdx == std::string::npos)
+        {
+            attr.pluginName = pluginOption;
+        }
+        else
+        {
+            attr.pluginNamespace = pluginOption.substr(0, sepratorIdx);
+            attr.pluginName = pluginOption.substr(sepratorIdx + 2, pluginOption.length() - sepratorIdx - 2);
+        }
+        return attr;
+    };
+
+    for (auto const& pluginOption : pluginOptions)
+    {
+        auto attr = parsePluginOption(pluginOption);
+        if (!attr.pluginName.empty())
+        {
+            args.pluginAttrs.push_back(attr);
+        }
+    }
+
+    return true;
+}
+
+//! \brief Check if arg is a command-line option with a value, and get the value.
+//! If arg matches the form --OPTION=VALUE (or -C=VALUE if singleChar is provided)
+//! and OPTION matches the provided name (or C matches the provided singleChar),
+//! extract the option VALUE.
+//! \param arg The argument to attempt to parse
+//! \param name The option name to match
+//! \param singleChar Single-character option, or std::nullopt if full name is required.
+//! \return If name matched, parsed string VALUE; otherwise std::nullopt.
+inline std::optional<std::string> parseString(std::string const& arg, std::string const& name, std::optional<char> singleChar = std::nullopt)
+{
+    for (std::string const& prefix : {
+             "--" + name + "=",
+             singleChar ? std::string{'-', *singleChar, '='} : "",
+         })
+    {
+        if (!prefix.empty() && prefix == arg.substr(0, prefix.size()))
+        {
+            return arg.substr(prefix.size());
+        }
+    }
+    return std::nullopt;
+}
+
+//! \brief Check if arg is command-line option without a value.
+//! Check whether arg matches the form --OPT (or -C if singleChar is provided)
+//! and OPT matches the provided name.
+//! \param arg The argument to attempt to parse
+//! \param name The option name to match
+//! \param singleChar Single-character version of OPT, or std::nullopt if full name is required.
+//! \return true on match, false otherwise.
+inline bool parseBool(std::string const& arg, std::string const& name, std::optional<char> singleChar = std::nullopt)
+{
+    return arg == "--" + name || (singleChar && arg == std::string{'-', *singleChar});
+}
+
+inline bool hasCpuOnlyInternalOption(std::string const& internalOptions)
+{
+    std::istringstream optionStream{internalOptions};
+    for (std::string option; optionStream >> option;)
+    {
+        if (option == "--cpu_only" || option.rfind("--cpu_only=", 0) == 0)
+        {
+            return true;
+        }
+    }
+    return false;
+}
+
+inline bool applyCpuOnlyMode()
+{
+#if !defined(_WIN32)
+    // The use of TRT_INTERNAL_OPTIONS is special to TensorRT 11.0 and will disappear in later releases.
+    char const* internalOptions = std::getenv("TRT_INTERNAL_OPTIONS");
+    std::string internalOptionsStr{internalOptions != nullptr ? internalOptions : ""};
+    if (hasCpuOnlyInternalOption(internalOptionsStr))
+    {
+        return true;
+    }
+
+    internalOptionsStr += " --cpu_only=1";
+    if (setenv("TRT_INTERNAL_OPTIONS", internalOptionsStr.c_str(), 1) != 0)
+    {
+        SAFE_LOG << "Failed to set TRT_INTERNAL_OPTIONS for CPU-only mode: " << std::strerror(errno) << std::endl;
+        return false;
+    }
+#endif
+    return true;
+}
+
+//! \brief Allocate \p graph's auxiliary CUDA streams and register them via setAuxStreams.
+//!
+//! The returned scope guard owns the streams; when the last reference is dropped, each
+//! stream is destroyed via cudaStreamDestroy. The pointed-to value (nullptr) is purposely
+//! opaque; only the deleter matters. The caller must keep the guard alive during the graph
+//! inference runs.
+[[nodiscard]] inline std::shared_ptr<std::nullptr_t> setUpAuxStreamsOn(
+    nvinfer2::safe::ITRTGraph& graph, nvinfer2::safe::ISafeRecorder& recorder)
+{
+    int32_t nbAuxStreams{};
+    SAFE_API_CALL(graph.getNbAuxStreams(nbAuxStreams), recorder);
+    std::vector<cudaStream_t> streams(nbAuxStreams);
+    for (auto& s : streams)
+    {
+        CUDA_CHECK(cudaStreamCreateWithFlags(&s, cudaStreamNonBlocking));
+    }
+    SAFE_API_CALL(graph.setAuxStreams(streams.data(), nbAuxStreams), recorder);
+    return {nullptr, [streams = std::move(streams)](void*) {
+                for (cudaStream_t s : streams)
+                {
+                    if (s)
+                    {
+                        (void) cudaStreamDestroy(s);
+                    }
+                }
+            }};
+}
+
+} // namespace samplesSafeCommon
+
+namespace safetyCompliance
+{
+inline void initSafeCuda()
+{
+    // According to CUDA initialization in NVIDIA CUDA SAFETY API REFERENCE FOR DRIVE OS
+    // We will need to do the following in order
+    // 1. Initialize the calling thread with CUDA specific information (Call any CUDA RT API identified as init)
+    // 2. Query/Configure and choose the desired CUDA device
+    // 3. CUDA context initialization. (Call cudaDeviceGetLimit or cuCtxCreate)
+    size_t stackSizeLimit = 0;
+    int32_t deviceIndex = 0;
+    CUDA_CHECK(cudaGetDevice(&deviceIndex));
+    CUDA_CHECK(cudaDeviceGetLimit(&stackSizeLimit, cudaLimitStackSize));
+#if IS_QNX_SAFE
+    CUDA_CHECK(cudaSafeExSelectAPIMode(cudaSafeExAPIModeAsilB));
+#endif // IS_QNX_SAFE
+}
+
+inline void setPromgrAbility()
+{
+#if IS_QNX_SAFE
+    // Comply with DEEPLRN_RES_117 on QNX-safe by dropping PROCMGR_AID_MEM_PHYS ability and locking out any further
+    // changes
+    procmgr_ability(
+        0, PROCMGR_ADN_NONROOT | PROCMGR_AOP_DENY | PROCMGR_AOP_LOCK | PROCMGR_AID_MEM_PHYS, PROCMGR_AID_EOL);
+#endif // IS_QNX_SAFE
+}
+
+} // namespace safetyCompliance
+
+#endif // TENSORRT_SAFE_COMMON_H

samples/common/safeCudaAllocator.h

@@ -0,0 +1,165 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SAFE_CUDA_ALLOCTOR_H
+#define SAFE_CUDA_ALLOCTOR_H
+
+#include "NvInferSafeRuntime.h" // TRTS-10206: NvInferSafeRuntime.h may be refactored
+#include "cuda_runtime.h"
+#include "safeCommon.h"
+#include "safeErrorRecorder.h"
+#include <cuda.h>
+
+namespace nvinfer2::safe
+{
+
+// In safe runtime, any single allocation size should not be greater than 16_GiB.
+constexpr uint64_t kMAXIMUM_SIZE{17179869184U};
+
+class SafeMemAllocator final : public ISafeMemAllocator
+{
+public:
+    static SafeMemAllocator& instance() noexcept
+    {
+        static SafeMemAllocator sDefaultAllocatorInstance{};
+        return sDefaultAllocatorInstance;
+    }
+
+    SafeMemAllocator(SafeMemAllocator const&) = delete;
+    SafeMemAllocator(SafeMemAllocator&&) = delete;
+    SafeMemAllocator& operator=(SafeMemAllocator const&) & = delete;
+    SafeMemAllocator& operator=(SafeMemAllocator&&) & = delete;
+
+private:
+    constexpr SafeMemAllocator() noexcept = default;
+    ~SafeMemAllocator() noexcept final = default;
+
+    void* allocate(uint64_t const size, uint64_t const alignment, MemoryPlacement const flags, MemoryUsage const usage,
+        ISafeRecorder& recorder) noexcept final
+    {
+        void* memory = nullptr;
+        SAFE_ASSERT(size <= kMAXIMUM_SIZE && "allocation size should not exceed 16_GiB.");
+        SAFE_ASSERT(alignment != 0U);
+        SAFE_ASSERT((alignment & (alignment - 1)) == 0 && "Memory alignment has to be power of 2");
+        try
+        {
+            switch (flags)
+            {
+            case MemoryPlacement::kCPU_PINNED:
+            {
+                if ((usage != MemoryUsage::kIMMUTABLE) && (usage != MemoryUsage::kIOTENSOR))
+                {
+                    safeLogWarning(
+                        recorder, "Memory usage for cpu-pinned memory should be either IMMUTABLE or IOTENSOR");
+                }
+                if (cudaHostAlloc(&memory, size, static_cast<uint32_t>(cudaHostAllocMapped)) != cudaSuccess)
+                {
+                    safeLogError(recorder, "Cannot allocate PINNED CPU/GPU memory with size = " + std::to_string(size));
+                    return nullptr;
+                }
+            }
+            break;
+
+            case MemoryPlacement::kGPU:
+            {
+                if (cudaMalloc(&memory, size) != cudaSuccess)
+                {
+                    safeLogError(recorder, "Cannot allocate GPU memory with size = " + std::to_string(size));
+                    return nullptr;
+                }
+            }
+            break;
+            case MemoryPlacement::kCPU:
+            {
+                // Use posix_memalign for aligned memory allocation on CPU
+                // Note: While aligned_alloc is available in C++17, we use posix_memalign
+                // for broader platform compatibility
+                SAFE_ASSERT(posix_memalign(&memory, alignment, size) == 0);
+            }
+            break;
+            case MemoryPlacement::kMANAGED:
+            case MemoryPlacement::kNONE:
+            {
+                safeLogError(recorder, "MemoryPlacement::kMANAGED and MemoryPlacement::kNONE are not allowed.");
+                return nullptr;
+            }
+            }
+
+            if (reinterpret_cast<uint64_t>(memory) % alignment != 0U)
+            {
+                safeLogError(recorder, "Allocated memory is not aligned with " + std::to_string(alignment) + " bytes.");
+                deallocate(memory, flags, recorder);
+                return nullptr;
+            }
+        }
+        catch (std::exception const& e)
+        {
+            safeLogError(recorder, e.what());
+            return nullptr;
+        }
+        return memory;
+    }
+
+    bool deallocate(void* const memory, MemoryPlacement const flags, ISafeRecorder& recorder) noexcept final
+    {
+        if (memory == nullptr)
+        {
+            safeLogWarning(recorder, "Attempting to free nullptr memory.");
+            return true;
+        }
+        try
+        {
+            switch (flags)
+            {
+            case MemoryPlacement::kCPU_PINNED:
+            {
+                return cudaFreeHost(memory) == cudaSuccess;
+            }
+            case MemoryPlacement::kGPU:
+            {
+                return cudaFree(memory) == cudaSuccess;
+            }
+            case MemoryPlacement::kCPU:
+            {
+                free(memory);
+                return true;
+            }
+            case MemoryPlacement::kMANAGED:
+            case MemoryPlacement::kNONE:
+            {
+                safeLogError(recorder, "MemoryPlacement::kMANAGED and MemoryPlacement::kNONE are not allowed.");
+                return false;
+            }
+            }
+        }
+        catch (std::exception const& e)
+        {
+            safeLogError(recorder, e.what());
+            return false;
+        }
+        return false;
+    }
+};
+
+inline ISafeMemAllocator& getSafeMemAllocator() noexcept
+{
+    return SafeMemAllocator::instance();
+}
+
+} // namespace nvinfer2::safe
+
+#endif // SAFE_CUDA_ALLOCTOR_H

samples/common/safeErrorRecorder.h

@@ -0,0 +1,294 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SAFE_ERROR_RECORDER_H
+#define SAFE_ERROR_RECORDER_H
+
+#include "NvInferSafeRecorder.h"
+#include "safeCommon.h"
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <exception>
+#include <iostream>
+#include <memory>
+#include <mutex>
+#include <string_view>
+
+#if ENABLE_NVLOG
+#include <nvos_s3_tegra_log.h>
+#endif // ENABLE_NVLOG
+
+namespace sample
+{
+using namespace nvinfer2::safe;
+
+namespace detail
+{
+//! Copy the contents of a std::string_view into a buffer, truncating if it doesn't fit, and always null-terminating the
+//! result (unless dst == nullptr or dstSize == 0). Returns the number of bytes written, including the null terminator.
+//! Behavior is undefined if dstSize < 0.
+//! Behavior is undefined if dst == nullpltr but dstSize > 0.
+inline int64_t truncatingCopyAsCString(std::string_view const src, AsciiChar* const dst, int64_t const dstSize)
+{
+    SAFE_ASSERT(0 <= dstSize);
+    SAFE_ASSERT(dst != nullptr || dstSize == 0);
+    if (dstSize == 0)
+    {
+        return 0;
+    }
+    SAFE_ASSERT(0 < dstSize); //< We at least have room for a null terminator.
+    auto const toWrite = std::min(static_cast<int64_t>(src.size()), dstSize - 1);
+    std::copy_n(src.data(), toWrite, dst);
+    dst[toWrite] = '\0';
+    return toWrite + 1;
+}
+
+//! Copy the contents of a std::string_view into a fixed-size buffer, null terminated (unless the buffer has zero size)
+//! and return it.
+template <typename TArray>
+[[nodiscard]] constexpr TArray truncatedCopyAsCString(std::string_view const src)
+{
+    TArray result{};
+    // Expecting TArray to be a std::array or similar, constructing to its size and having a static size:
+    SAFE_ASSERT(std::tuple_size<TArray>::value == result.size());
+    truncatingCopyAsCString(src, result.data(), result.size());
+    return result;
+}
+
+#if ENABLE_NVLOG
+//! \return a severity number corresponding to an `nvinfer2::safe::Severity`.
+//! Unlisted enumerators map to `NVOS_LOG_SEVERITY_INFO`
+[[nodiscard]] constexpr uint8_t toNvOsLogSeverity(nvinfer2::safe::Severity sev)
+{
+    switch (sev)
+    {
+    case nvinfer2::safe::Severity::kINFO: return NVOS_LOG_SEVERITY_INFO;
+    case nvinfer2::safe::Severity::kWARNING: return NVOS_LOG_SEVERITY_WARNING;
+    case nvinfer2::safe::Severity::kVERBOSE: return NVOS_LOG_SEVERITY_DEBUG1;
+    default: return NVOS_LOG_SEVERITY_INFO;
+    }
+}
+#endif // ENABLE_NVLOG
+} // namespace detail
+
+//! The SampleSafeRecorder implementation of the ISafeRecorder interface.
+class SampleSafeRecorder : public ISafeRecorder
+{
+    using DescHolder = std::array<AsciiChar, ISafeRecorder::kMAX_SAFE_DESC_LENGTH + 1U>;
+    using errorPair = std::pair<ErrorCode, DescHolder>;
+
+public:
+    SampleSafeRecorder(nvinfer2::safe::Severity severity = nvinfer2::safe::Severity::kINFO, int32_t index = -1,
+        char const* filename = "TRTErrors.log")
+        : ISafeRecorder(severity, index)
+    {
+        fatalErrorLogFile = fopen(filename, "w+");
+        if (fatalErrorLogFile == nullptr)
+        {
+            std::cerr << "Failed to open error log file: " << filename << std::endl;
+        }
+    }
+
+    virtual ~SampleSafeRecorder() noexcept
+    {
+        if (mRefCount != 0)
+        {
+            reportError(ErrorCode::kINTERNAL_ERROR, "Non-zero reference count for recorder upon deallocation.");
+        }
+    }
+
+    int32_t getNbErrors() const noexcept final
+    {
+        return nbErrors;
+    }
+    ErrorCode getErrorCode(int32_t errorIdx) const noexcept final
+    {
+        return invalidIndexCheck(errorIdx) ? ErrorCode::kINVALID_ARGUMENT : (*this)[errorIdx].first;
+    };
+    ErrorDesc getErrorDesc(int32_t errorIdx) const noexcept final
+    {
+        return invalidIndexCheck(errorIdx) ? "ErrorIdx is out of range." : (*this)[errorIdx].second.data();
+    }
+
+    bool hasOverflowed() const noexcept final
+    {
+        return (nbErrors >= kMAX_NB_ERRORS);
+    }
+
+    int32_t getMaxNbErrors() const
+    {
+        return kMAX_NB_ERRORS;
+    }
+
+    // Empty the errorStack.
+    void clear() noexcept final
+    {
+        try
+        {
+            // grab a lock so that there is no addition while clearing.
+            std::lock_guard<std::mutex> guard(mStackLock);
+            nbErrors = 0;
+        }
+        catch (std::exception const& e)
+        {
+#if ENABLE_NVLOG
+            NvOsDebugPrintStr(NVOS_LOG_CODE_START, NVOS_LOG_SEVERITY_ERROR, e.what());
+#else
+            std::cerr << "Internal Error: " << e.what() << std::endl;
+#endif // ENABLE_NVLOG
+        }
+    };
+
+    //! Simple helper function that checks if the error stack is empty.
+    bool empty() const noexcept
+    {
+        return (nbErrors == 0);
+    }
+
+    bool reportError(ErrorCode val, ErrorDesc desc) noexcept final
+    {
+        try
+        {
+            std::string_view const descView = desc; //< This implicitly calls strlen once.
+            std::cerr << descView << std::endl;
+            DescHolder descArr = detail::truncatedCopyAsCString<DescHolder>(descView);
+
+            std::lock_guard<std::mutex> guard(mStackLock);
+#if ENABLE_NVLOG
+            NvOsDebugPrintStrInt(
+                NVOS_LOG_CODE_START, NVOS_LOG_SEVERITY_ERROR, descArr.data(), static_cast<int32_t>(val));
+#else
+            // Only write to the array if there's space available
+            if (nbErrors < kMAX_NB_ERRORS)
+            {
+                mErrorStack.at(nbErrors) = errorPair(val, descArr);
+                nbErrors++;
+            }
+#endif // ENABLE_NVLOG
+        }
+        catch (std::exception const& e)
+        {
+#if ENABLE_NVLOG
+            NvOsDebugPrintStr(NVOS_LOG_CODE_START, NVOS_LOG_SEVERITY_ERROR, e.what());
+#else
+            // `std::ofstream` uses heap allocation which is not allowed for safe samples
+            // Hence, C functions are used here to write data to file.
+            if (fatalErrorLogFile != nullptr)
+            {
+                setbuf(fatalErrorLogFile, NULL);
+                fwrite(e.what(), strlen(e.what()), 1, fatalErrorLogFile);
+                fwrite("\n", 1, 1, fatalErrorLogFile);
+                fflush(fatalErrorLogFile);
+            }
+            std::cerr << e.what() << std::endl;
+#endif // ENABLE_NVLOG
+        }
+        // All errors are considered fatal.
+        return true;
+    }
+
+    bool reportInfo(ErrorDesc desc) noexcept final
+    {
+        return reportIfSevere(nvinfer2::safe::Severity::kINFO, desc);
+    }
+
+    bool reportWarn(ErrorDesc desc) noexcept final
+    {
+        return reportIfSevere(nvinfer2::safe::Severity::kWARNING, desc);
+    }
+
+    bool reportVerbose(ErrorDesc desc) noexcept final
+    {
+        return reportIfSevere(nvinfer2::safe::Severity::kVERBOSE, desc);
+    }
+
+    bool reportDebug(ErrorDesc desc) noexcept final
+    {
+        return reportIfSevere(nvinfer2::safe::Severity::kDEBUG, desc);
+    }
+
+    // Atomically increment or decrement the ref counter.
+    RefCount incRefCount() noexcept final
+    {
+        return ++mRefCount;
+    }
+    RefCount decRefCount() noexcept final
+    {
+        return --mRefCount;
+    }
+
+private:
+    // Simple helper functions.
+    errorPair const& operator[](int32_t index) const noexcept
+    {
+        return mErrorStack[index];
+    }
+
+    bool invalidIndexCheck(int32_t index) const noexcept
+    {
+        return index >= nbErrors;
+    }
+
+    bool reportIfSevere(nvinfer2::safe::Severity msgSev, ErrorDesc desc) noexcept
+    {
+#if ENABLE_NVLOG
+        if (mSeverity >= msgSev)
+        {
+            auto const severity = detail::toNvOsLogSeverity(msgSev);
+            std::lock_guard<std::mutex> guard(mStackLock);
+            NvOsDebugPrintStr(NVOS_LOG_CODE_START, severity, desc);
+            return true;
+        }
+        return false;
+#else
+        if (mSeverity >= msgSev)
+        {
+            std::lock_guard<std::mutex> guard(mStackLock);
+            std::cout << desc << std::endl;
+            return true;
+        }
+        return false;
+#endif // ENABLE_NVLOG
+    }
+
+    // Used to store the logs that are Fatal
+    FILE* fatalErrorLogFile;
+
+    // Mutex to hold when locking mErrorStack for thread safety.
+    std::mutex mStackLock;
+
+    // Reference count of the class. Destruction of the class when mRefCount
+    // is not zero causes undefined behavior.
+    std::atomic<int32_t> mRefCount{0};
+
+    // Number of errors that occurred so far.
+    int32_t nbErrors{0};
+
+    // Maximum number of errors that can be stored.
+    static constexpr int32_t kMAX_NB_ERRORS = 10;
+
+    // The error stack that holds the errors recorded by TensorRT.
+    std::array<errorPair, kMAX_NB_ERRORS> mErrorStack;
+}; // class SampleRecorder
+
+} // namespace sample
+
+#endif // SAFE_ERROR_RECORDER_H

samples/common/sampleConfig.h

@@ -0,0 +1,291 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SampleConfig_H
+#define SampleConfig_H
+
+#include <cstring>
+#include <iostream>
+#include <string>
+
+#include "NvInfer.h"
+#include "NvOnnxConfig.h"
+class SampleConfig : public nvonnxparser::IOnnxConfig
+{
+public:
+    enum class InputDataFormat : int
+    {
+        kASCII = 0,
+        kPPM = 1
+    };
+
+private:
+    std::string mModelFilename;
+    std::string mEngineFilename;
+    std::string mTextFilename;
+    std::string mFullTextFilename;
+    std::string mImageFilename;
+    std::string mReferenceFilename;
+    std::string mOutputFilename;
+    std::string mTimingCacheFilename;
+    int64_t mLabel{-1};
+    int64_t mMaxBatchSize{32};
+    int64_t mUseDLACore{-1};
+    nvinfer1::DataType mModelDtype{nvinfer1::DataType::kFLOAT};
+    bool mTF32{true};
+    Verbosity mVerbosity{static_cast<int>(nvinfer1::ILogger::Severity::kWARNING)};
+    bool mPrintLayercInfo{false};
+    bool mDebugBuilder{false};
+    InputDataFormat mInputDataFormat{InputDataFormat::kASCII};
+    uint64_t mTopK{0};
+    float mFailurePercentage{-1.0F};
+    float mTolerance{0.0F};
+    float mAbsTolerance{1e-5F};
+
+public:
+    SampleConfig()
+    {
+#ifdef ONNX_DEBUG
+        if (isDebug())
+        {
+            std::cout << " SampleConfig::ctor(): " << this << "\t" << std::endl;
+        }
+#endif
+    }
+
+    ~SampleConfig() override
+    {
+#ifdef ONNX_DEBUG
+        if (isDebug())
+        {
+            std::cout << "SampleConfig::dtor(): " << this << std::endl;
+        }
+#endif
+    }
+
+public:
+    void setModelDtype(const nvinfer1::DataType mdt) noexcept override
+    {
+        mModelDtype = mdt;
+    }
+
+    nvinfer1::DataType getModelDtype() const noexcept override
+    {
+        return mModelDtype;
+    }
+
+    bool getTF32() const noexcept
+    {
+        return mTF32;
+    }
+
+    void setTF32(bool enabled) noexcept
+    {
+        mTF32 = enabled;
+    }
+
+    const char* getModelFileName() const noexcept override
+    {
+        return mModelFilename.c_str();
+    }
+
+    void setModelFileName(const char* onnxFilename) noexcept override
+    {
+        mModelFilename = std::string(onnxFilename);
+    }
+    Verbosity getVerbosityLevel() const noexcept override
+    {
+        return mVerbosity;
+    }
+    void addVerbosity() noexcept override
+    {
+        ++mVerbosity;
+    }
+    void reduceVerbosity() noexcept override
+    {
+        --mVerbosity;
+    }
+    void setVerbosityLevel(Verbosity v) noexcept override
+    {
+        mVerbosity = v;
+    }
+    const char* getEngineFileName() const noexcept
+    {
+        return mEngineFilename.c_str();
+    }
+    void setEngineFileName(const char* engineFilename) noexcept
+    {
+        mEngineFilename = std::string(engineFilename);
+    }
+    const char* getTextFileName() const noexcept override
+    {
+        return mTextFilename.c_str();
+    }
+    void setTextFileName(const char* textFilename) noexcept override
+    {
+        mTextFilename = std::string(textFilename);
+    }
+    const char* getFullTextFileName() const noexcept override
+    {
+        return mFullTextFilename.c_str();
+    }
+    void setFullTextFileName(const char* fullTextFilename) noexcept override
+    {
+        mFullTextFilename = std::string(fullTextFilename);
+    }
+    void setLabel(int64_t label) noexcept
+    {
+        mLabel = label;
+    } //!<  set the Label
+
+    int64_t getLabel() const noexcept
+    {
+        return mLabel;
+    } //!<  get the Label
+
+    bool getPrintLayerInfo() const noexcept override
+    {
+        return mPrintLayercInfo;
+    }
+
+    void setPrintLayerInfo(bool b) noexcept override
+    {
+        mPrintLayercInfo = b;
+    } //!< get the boolean variable corresponding to the Layer Info, see getPrintLayerInfo()
+
+    void setMaxBatchSize(int64_t maxBatchSize) noexcept
+    {
+        mMaxBatchSize = maxBatchSize;
+    } //!<  set the Max Batch Size
+    int64_t getMaxBatchSize() const noexcept
+    {
+        return mMaxBatchSize;
+    } //!<  get the Max Batch Size
+
+    void setUseDLACore(int64_t UseDLACore) noexcept
+    {
+        mUseDLACore = UseDLACore;
+    } //!<  set the DLA core to use
+    int64_t getUseDLACore() const noexcept
+    {
+        return mUseDLACore;
+    } //!<  get the DLA core to use
+
+    void setDebugBuilder() noexcept
+    {
+        mDebugBuilder = true;
+    } //!<  enable the Debug info, while building the engine.
+    bool getDebugBuilder() const noexcept
+    {
+        return mDebugBuilder;
+    } //!<  get the boolean variable, corresponding to the debug builder
+
+    const char* getImageFileName() const noexcept //!<  set Image file name (PPM or ASCII)
+    {
+        return mImageFilename.c_str();
+    }
+    void setImageFileName(const char* imageFilename) noexcept //!< get the Image file name
+    {
+        mImageFilename = std::string(imageFilename);
+    }
+    const char* getReferenceFileName() const noexcept
+    {
+        return mReferenceFilename.c_str();
+    }
+    void setReferenceFileName(const char* referenceFilename) noexcept //!<  set reference file name
+    {
+        mReferenceFilename = std::string(referenceFilename);
+    }
+
+    void setInputDataFormat(InputDataFormat idt) noexcept
+    {
+        mInputDataFormat = idt;
+    } //!<  specifies expected data format of the image file (PPM or ASCII)
+    InputDataFormat getInputDataFormat() const noexcept
+    {
+        return mInputDataFormat;
+    } //!<  returns the expected data format of the image file.
+
+    const char* getOutputFileName() const noexcept //!<  specifies the file to save the results
+    {
+        return mOutputFilename.c_str();
+    }
+    void setOutputFileName(const char* outputFilename) noexcept //!<  get the output file name
+    {
+        mOutputFilename = std::string(outputFilename);
+    }
+
+    uint64_t getTopK() const noexcept
+    {
+        return mTopK;
+    }
+    void setTopK(uint64_t topK) noexcept
+    {
+        mTopK = topK;
+    } //!<  If this options is specified, return the K top probabilities.
+
+    float getFailurePercentage() const noexcept
+    {
+        return mFailurePercentage;
+    }
+
+    void setFailurePercentage(float f) noexcept
+    {
+        mFailurePercentage = f;
+    }
+
+    float getAbsoluteTolerance() const noexcept
+    {
+        return mAbsTolerance;
+    }
+
+    void setAbsoluteTolerance(float a) noexcept
+    {
+        mAbsTolerance = a;
+    }
+
+    float getTolerance() const noexcept
+    {
+        return mTolerance;
+    }
+
+    void setTolerance(float t) noexcept
+    {
+        mTolerance = t;
+    }
+
+    const char* getTimingCacheFilename() const noexcept
+    {
+        return mTimingCacheFilename.c_str();
+    }
+
+    void setTimingCacheFileName(const char* timingCacheFilename) noexcept
+    {
+        mTimingCacheFilename = std::string(timingCacheFilename);
+    }
+
+    bool isDebug() const noexcept
+    {
+#if ONNX_DEBUG
+        return std::getenv("ONNX_DEBUG") != nullptr;
+#else
+        return false;
+#endif
+    }
+}; // class SampleConfig
+
+#endif

samples/common/sampleDevice.cpp

@@ -0,0 +1,188 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "sampleDevice.h"
+
+#include <iomanip>
+
+#if !defined(_WIN32)
+#include <dlfcn.h>
+#endif
+
+namespace sample
+{
+
+namespace
+{
+// Subset of NVML types/constants needed to query Confidential Compute state.
+// Declared locally so we do not introduce a build-time dependency on nvml.h
+// or libnvidia-ml; functions are resolved via dlopen at runtime.
+using NvmlReturnT = int32_t;
+constexpr NvmlReturnT kNVML_SUCCESS = 0;
+constexpr uint32_t kNVML_CC_FEATURE_ENABLED = 1;
+
+struct NvmlConfComputeSystemState
+{
+    uint32_t environment;
+    uint32_t ccFeature;
+    uint32_t devToolsMode;
+};
+
+using NvmlInitFn = NvmlReturnT (*)();
+using NvmlShutdownFn = NvmlReturnT (*)();
+using NvmlGetCcStateFn = NvmlReturnT (*)(NvmlConfComputeSystemState*);
+
+bool queryConfidentialCompute()
+{
+#if defined(_WIN32)
+    return false;
+#else
+    void* handle = dlopen("libnvidia-ml.so.1", RTLD_LAZY | RTLD_LOCAL);
+    if (handle == nullptr)
+    {
+        return false;
+    }
+
+    auto init = reinterpret_cast<NvmlInitFn>(dlsym(handle, "nvmlInit_v2"));
+    auto shutdown = reinterpret_cast<NvmlShutdownFn>(dlsym(handle, "nvmlShutdown"));
+    auto getState = reinterpret_cast<NvmlGetCcStateFn>(dlsym(handle, "nvmlSystemGetConfComputeState"));
+
+    bool enabled = false;
+    if (init != nullptr && shutdown != nullptr && getState != nullptr && init() == kNVML_SUCCESS)
+    {
+        NvmlConfComputeSystemState state{};
+        if (getState(&state) == kNVML_SUCCESS && state.ccFeature == kNVML_CC_FEATURE_ENABLED)
+        {
+            enabled = true;
+        }
+        shutdown();
+    }
+
+    dlclose(handle);
+    return enabled;
+#endif
+}
+} // namespace
+
+bool isConfidentialComputeEnabled()
+{
+    static bool const kCC_ENABLED = queryConfidentialCompute();
+    return kCC_ENABLED;
+}
+
+// Construct GPU UUID string in the same format as nvidia-smi does.
+std::string getUuidString(cudaUUID_t uuid)
+{
+    constexpr int32_t kUUID_SIZE = sizeof(cudaUUID_t);
+    static_assert(kUUID_SIZE == 16, "Unexpected size for cudaUUID_t!");
+
+    std::ostringstream ss;
+    std::vector<int32_t> const splits = {0, 4, 6, 8, 10, kUUID_SIZE};
+
+    ss << "GPU" << std::hex << std::setfill('0');
+    for (int32_t splitIdx = 0; splitIdx < static_cast<int32_t>(splits.size()) - 1; ++splitIdx)
+    {
+        ss << "-";
+        for (int32_t byteIdx = splits[splitIdx]; byteIdx < splits[splitIdx + 1]; ++byteIdx)
+        {
+            ss << std::setw(2) << +static_cast<uint8_t>(uuid.bytes[byteIdx]);
+        }
+    }
+    return ss.str();
+}
+
+void setCudaDevice(int32_t device, std::ostream& os)
+{
+    os << "=== Device Information ===" << std::endl;
+
+    // Get the number of visible GPUs.
+    int32_t nbDevices{-1};
+    CHECK(cudaGetDeviceCount(&nbDevices));
+
+    if (nbDevices <= 0)
+    {
+        os << "Cannot find any available devices (GPUs)!" << std::endl;
+        exit(EXIT_FAILURE);
+    }
+
+    // Print out the GPU name and PCIe bus ID of each GPU.
+    os << "Available Devices: " << std::endl;
+    cudaDeviceProp properties{};
+    for (int32_t deviceIdx = 0; deviceIdx < nbDevices; ++deviceIdx)
+    {
+        cudaDeviceProp tempProperties;
+        CHECK(cudaGetDeviceProperties(&tempProperties, deviceIdx));
+
+        // clang-format off
+        os << "  Device " << deviceIdx << ": \"" << tempProperties.name << "\" UUID: "
+           << getUuidString(tempProperties.uuid) << std::endl;
+        // clang-format on
+
+        // Record the properties of the desired GPU.
+        if (deviceIdx == device)
+        {
+            properties = tempProperties;
+        }
+    }
+
+    // Exit with error if the requested device ID does not exist.
+    if (device < 0 || device >= nbDevices)
+    {
+        os << "Cannot find device ID " << device << "!" << std::endl;
+        exit(EXIT_FAILURE);
+    }
+
+    // Set to the corresponding GPU.
+    CHECK(cudaSetDevice(device));
+
+    // clang-format off
+    os << "Selected Device: "      << properties.name                                               << std::endl;
+    os << "Selected Device ID: "   << device                                                        << std::endl;
+    os << "Selected Device UUID: " << getUuidString(properties.uuid)                                << std::endl;
+    os << "Compute Capability: "   << properties.major << "." << properties.minor                   << std::endl;
+    os << "SMs: "                  << properties.multiProcessorCount                                << std::endl;
+    os << "Device Global Memory: " << (properties.totalGlobalMem >> 20) << " MiB"                   << std::endl;
+    os << "Shared Memory per SM: " << (properties.sharedMemPerMultiprocessor >> 10) << " KiB"       << std::endl;
+    os << "Memory Bus Width: "     << properties.memoryBusWidth << " bits"
+                        << " (ECC " << (properties.ECCEnabled != 0 ? "enabled" : "disabled") << ")" << std::endl;
+    int32_t clockRate = 0;
+    int32_t memoryClockRate = 0;
+    CHECK(cudaDeviceGetAttribute(&clockRate, cudaDevAttrClockRate, device));
+    CHECK(cudaDeviceGetAttribute(&memoryClockRate, cudaDevAttrMemoryClockRate, device));
+    os << "Application Compute Clock Rate: "   << clockRate / 1000000.0F << " GHz"       << std::endl;
+    os << "Application Memory Clock Rate: "    << memoryClockRate / 1000000.0F << " GHz" << std::endl;
+    os << std::endl;
+    os << "Note: The application clock rates do not reflect the actual clock rates that the GPU is "
+                                                                         << "currently running at." << std::endl;
+    // clang-format on
+}
+
+int32_t getCudaDriverVersion()
+{
+    int32_t version{-1};
+    CHECK(cudaDriverGetVersion(&version));
+    return version;
+}
+
+int32_t getCudaRuntimeVersion()
+{
+    int32_t version{-1};
+    CHECK(cudaRuntimeGetVersion(&version));
+    return version;
+}
+
+} // namespace sample

samples/common/sampleDevice.h

@@ -0,0 +1,633 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_SAMPLE_DEVICE_H
+#define TRT_SAMPLE_DEVICE_H
+
+#include <cassert>
+#include <cstdint>
+#include <cuda.h>
+#include <cuda_runtime.h>
+#include <iostream>
+#include <thread>
+
+#include "common.h"
+#include "globalTimerKernel.h"
+#include "sampleUtils.h"
+
+namespace sample
+{
+
+//! True when Confidential Compute is enabled on the current system. Cached on
+//! the first call. When true, TrtCudaEvent falls back to a GPU global-timer
+//! kernel because cudaEventElapsedTime() is unreliable under CC (see nvbug
+//! 5598617, mirrors TRT-LLM PR #11657).
+[[nodiscard]] bool isConfidentialComputeEnabled();
+
+class TrtCudaEvent;
+
+namespace
+{
+
+void cudaSleep(void* sleep)
+{
+    std::this_thread::sleep_for(std::chrono::duration<float, std::milli>(*static_cast<float*>(sleep)));
+}
+
+} // namespace
+
+//!
+//! \class TrtCudaStream
+//! \brief Managed CUDA stream
+//!
+class TrtCudaStream
+{
+public:
+    TrtCudaStream()
+    {
+        CHECK(cudaStreamCreate(&mStream));
+    }
+
+    TrtCudaStream(const TrtCudaStream&) = delete;
+
+    TrtCudaStream& operator=(const TrtCudaStream&) = delete;
+
+    TrtCudaStream(TrtCudaStream&&) = delete;
+
+    TrtCudaStream& operator=(TrtCudaStream&&) = delete;
+
+    ~TrtCudaStream()
+    {
+        CHECK(cudaStreamDestroy(mStream));
+    }
+
+    cudaStream_t get() const
+    {
+        return mStream;
+    }
+
+    void synchronize()
+    {
+        CHECK(cudaStreamSynchronize(mStream));
+    }
+
+    void wait(TrtCudaEvent& event);
+
+    void sleep(float* ms)
+    {
+        CHECK(cudaLaunchHostFunc(mStream, cudaSleep, ms));
+    }
+
+private:
+    cudaStream_t mStream{};
+};
+
+//!
+//! \class TrtCudaEvent
+//! \brief Managed CUDA event
+//!
+class TrtCudaEvent
+{
+public:
+    explicit TrtCudaEvent(bool blocking = true)
+    {
+        const uint32_t flags = blocking ? cudaEventBlockingSync : cudaEventDefault;
+        CHECK(cudaEventCreateWithFlags(&mEvent, flags));
+        if (isConfidentialComputeEnabled())
+        {
+            CHECK(cudaMalloc(&mDeviceTimestamp, sizeof(uint64_t)));
+        }
+    }
+
+    TrtCudaEvent(TrtCudaEvent const&) = delete;
+
+    TrtCudaEvent& operator=(TrtCudaEvent const&) = delete;
+
+    TrtCudaEvent(TrtCudaEvent&&) = delete;
+
+    TrtCudaEvent& operator=(TrtCudaEvent&&) = delete;
+
+    ~TrtCudaEvent()
+    {
+        CHECK(cudaEventDestroy(mEvent));
+        if (mDeviceTimestamp != nullptr)
+        {
+            CHECK(cudaFree(mDeviceTimestamp));
+        }
+    }
+
+    cudaEvent_t get() const
+    {
+        return mEvent;
+    }
+
+    void record(TrtCudaStream const& stream)
+    {
+        if (mDeviceTimestamp != nullptr)
+        {
+            CHECK(launchGlobalTimerKernel(mDeviceTimestamp, stream.get()));
+        }
+        CHECK(cudaEventRecord(mEvent, stream.get()));
+    }
+
+    void synchronize() const
+    {
+        CHECK(cudaEventSynchronize(mEvent));
+    }
+
+    // Returns time elapsed time in milliseconds
+    float operator-(const TrtCudaEvent& e) const
+    {
+        // Synchronize both events to ensure they have completed before calculating elapsed time
+        synchronize();
+        e.synchronize();
+
+        if (mDeviceTimestamp != nullptr && e.mDeviceTimestamp != nullptr)
+        {
+            // Confidential Compute path: read %globaltimer values captured at record().
+            // cudaEventElapsedTime() is unreliable under CC (nvbug 5598617); the global
+            // timer kernel reads the same underlying register directly.
+            // Use signed int64_t so the subtraction is well-defined if the events
+            // are ever measured out of order (otherwise the unsigned->signed cast
+            // is implementation-defined in C++17).
+            int64_t endNs{0};
+            int64_t startNs{0};
+            CHECK(cudaMemcpy(&endNs, mDeviceTimestamp, sizeof(int64_t), cudaMemcpyDeviceToHost));
+            CHECK(cudaMemcpy(&startNs, e.mDeviceTimestamp, sizeof(int64_t), cudaMemcpyDeviceToHost));
+            return static_cast<float>(endNs - startNs) / 1.0e6F;
+        }
+
+        float time{0};
+        CHECK(cudaEventElapsedTime(&time, e.get(), get()));
+        return time;
+    }
+
+private:
+    cudaEvent_t mEvent{};
+    uint64_t* mDeviceTimestamp{nullptr};
+};
+
+inline void TrtCudaStream::wait(TrtCudaEvent& event)
+{
+    CHECK(cudaStreamWaitEvent(mStream, event.get(), 0));
+}
+
+//!
+//! \class TrtCudaGraph
+//! \brief Managed CUDA graph
+//!
+class TrtCudaGraph
+{
+public:
+    explicit TrtCudaGraph() = default;
+
+    TrtCudaGraph(const TrtCudaGraph&) = delete;
+
+    TrtCudaGraph& operator=(const TrtCudaGraph&) = delete;
+
+    TrtCudaGraph(TrtCudaGraph&&) = delete;
+
+    TrtCudaGraph& operator=(TrtCudaGraph&&) = delete;
+
+    ~TrtCudaGraph()
+    {
+        if (mGraphExec)
+        {
+            cudaGraphExecDestroy(mGraphExec);
+        }
+    }
+
+    void beginCapture(TrtCudaStream& stream)
+    {
+        CHECK(cudaStreamBeginCapture(stream.get(), cudaStreamCaptureModeThreadLocal));
+    }
+
+    bool launch(TrtCudaStream& stream)
+    {
+        return cudaGraphLaunch(mGraphExec, stream.get()) == cudaSuccess;
+    }
+
+    void endCapture(TrtCudaStream& stream)
+    {
+        CHECK(cudaStreamEndCapture(stream.get(), &mGraph));
+        CHECK(cudaGraphInstantiate(&mGraphExec, mGraph, nullptr, nullptr, 0));
+        CHECK(cudaGraphDestroy(mGraph));
+    }
+
+    void endCaptureOnError(TrtCudaStream& stream)
+    {
+        // There are two possibilities why stream capture would fail:
+        // (1) stream is in cudaErrorStreamCaptureInvalidated state.
+        // (2) TRT reports a failure.
+        // In case (1), the returning mGraph should be nullptr.
+        // In case (2), the returning mGraph is not nullptr, but it should not be used.
+        const auto ret = cudaStreamEndCapture(stream.get(), &mGraph);
+        if (ret == cudaErrorStreamCaptureInvalidated)
+        {
+            assert(mGraph == nullptr);
+        }
+        else
+        {
+            CHECK(ret);
+            assert(mGraph != nullptr);
+            CHECK(cudaGraphDestroy(mGraph));
+            mGraph = nullptr;
+        }
+        // Clean up any CUDA error.
+        cudaGetLastError();
+        sample::gLogWarning << "The CUDA graph capture on the stream has failed." << std::endl;
+    }
+
+private:
+    cudaGraph_t mGraph{};
+    cudaGraphExec_t mGraphExec{};
+};
+
+//!
+//! \class TrtCudaBuffer
+//! \brief Managed buffer for host and device
+//!
+template <typename A, typename D>
+class TrtCudaBuffer
+{
+public:
+    TrtCudaBuffer() = default;
+
+    TrtCudaBuffer(const TrtCudaBuffer&) = delete;
+
+    TrtCudaBuffer& operator=(const TrtCudaBuffer&) = delete;
+
+    TrtCudaBuffer(TrtCudaBuffer&& rhs)
+    {
+        reset(rhs.mPtr, rhs.mSize);
+        rhs.mPtr = nullptr;
+        rhs.mSize = 0;
+    }
+
+    TrtCudaBuffer& operator=(TrtCudaBuffer&& rhs)
+    {
+        if (this != &rhs)
+        {
+            reset(rhs.mPtr, rhs.mSize);
+            rhs.mPtr = nullptr;
+            rhs.mSize = 0;
+        }
+        return *this;
+    }
+
+    ~TrtCudaBuffer()
+    {
+        reset();
+    }
+
+    TrtCudaBuffer(size_t size)
+    {
+        A()(&mPtr, size);
+        mSize = size;
+    }
+
+    void allocate(size_t size)
+    {
+        reset();
+        A()(&mPtr, size);
+        mSize = size;
+    }
+
+    void reset(void* ptr = nullptr, size_t size = 0)
+    {
+        if (mPtr)
+        {
+            D()(mPtr);
+        }
+        mPtr = ptr;
+        mSize = size;
+    }
+
+    void* get() const
+    {
+        return mPtr;
+    }
+
+    size_t getSize() const
+    {
+        return mSize;
+    }
+
+private:
+    void* mPtr{nullptr};
+    size_t mSize{0};
+};
+
+struct DeviceAllocator
+{
+    void operator()(void** ptr, size_t size)
+    {
+        CHECK(cudaMalloc(ptr, size));
+    }
+};
+
+struct DeviceDeallocator
+{
+    void operator()(void* ptr)
+    {
+        CHECK(cudaFree(ptr));
+    }
+};
+
+struct ManagedAllocator
+{
+    void operator()(void** ptr, size_t size)
+    {
+        CHECK(cudaMallocManaged(ptr, size));
+    }
+};
+
+struct HostAllocator
+{
+    //! Attempts to allocate size bytes on host, pointing *ptr to the start.
+    //! First attempts to allocate pinned memory using cudaMallocHost(ptr, size), failing that, warns to gLogWarning and
+    //! falls back to ::operator new(size) to allocate pageable memory. If that still fails, an exception may be thrown.
+    void operator()(void** ptr, size_t size)
+    {
+        // Try allocating pinned host memory.
+        cudaError_t ret = cudaMallocHost(ptr, size);
+
+        // If we cannot allocate pinned host memory, allocate pageable host memory instead and print a warning.
+        if (ret != cudaSuccess)
+        {
+            // Clean up the last cuda error.
+            (void) cudaGetLastError();
+
+            sample::gLogWarning << "cudaMallocHost() call with ptr=" << ptr << " and size=" << size
+                                << " returns a cuda error: " << cudaGetErrorString(ret) << std::endl;
+            sample::gLogWarning << "Allocate pageable host memory instead of pinned host memory. H2D and D2H copy "
+                                   "latencies may become longer."
+                                << std::endl;
+            *ptr = ::operator new(size);
+
+            // Make sure there is no remaining cuda error at this point.
+            CHECK(cudaGetLastError());
+        }
+    }
+};
+
+struct HostDeallocator
+{
+    //! Attempts to deallocate the host memory allocated by HostAllocator.
+    //! It first checks if ptr is a pinned or pageable host memory. If pinned, call cudaFreeHost() to free it. If
+    //! pageable, call ::operator delete() to free it. If ptr is neither of them, an error is printed and the program
+    //! exits.
+    void operator()(void* ptr)
+    {
+        // Check if the host memory pointer is pinned or pageable.
+        cudaPointerAttributes attrs;
+        CHECK(cudaPointerGetAttributes(&attrs, ptr));
+
+        // If pinned, call cudaFreeHost() to deallocate it. Under Confidential
+        // Compute, memory returned by cudaMallocHost() may be reported as
+        // cudaMemoryTypeManaged; it must still be released via cudaFreeHost.
+        if (attrs.type == cudaMemoryTypeHost || attrs.type == cudaMemoryTypeManaged)
+        {
+            CHECK(cudaFreeHost(ptr));
+        }
+        // If pageable, delete it directly.
+        else if (attrs.type == cudaMemoryTypeUnregistered)
+        {
+            ::operator delete(ptr);
+        }
+        // The host memory pointer should not be of any other types.
+        else
+        {
+            sample::gLogError << "Unexpected cuda memory type:" << static_cast<int32_t>(attrs.type) << std::endl;
+            exit(EXIT_FAILURE);
+        }
+    }
+};
+
+using TrtDeviceBuffer = TrtCudaBuffer<DeviceAllocator, DeviceDeallocator>;
+using TrtManagedBuffer = TrtCudaBuffer<ManagedAllocator, DeviceDeallocator>;
+
+using TrtHostBuffer = TrtCudaBuffer<HostAllocator, HostDeallocator>;
+
+//!
+//! \class MirroredBuffer
+//! \brief Coupled host and device buffers
+//!
+class IMirroredBuffer
+{
+public:
+    //!
+    //! Allocate memory for the mirrored buffer give the size
+    //! of the allocation.
+    //!
+    virtual void allocate(size_t size) = 0;
+
+    //!
+    //! Get the pointer to the device side buffer.
+    //!
+    //! \return pointer to device memory or nullptr if uninitialized.
+    //!
+    virtual void* getDeviceBuffer() const = 0;
+
+    //!
+    //! Get the pointer to the host side buffer.
+    //!
+    //! \return pointer to host memory or nullptr if uninitialized.
+    //!
+    virtual void* getHostBuffer() const = 0;
+
+    //!
+    //! Copy the memory from host to device.
+    //!
+    virtual void hostToDevice(TrtCudaStream& stream) = 0;
+
+    //!
+    //! Copy the memory from device to host.
+    //!
+    virtual void deviceToHost(TrtCudaStream& stream) = 0;
+
+    //!
+    //! Interface to get the size of the memory
+    //!
+    //! \return the size of memory allocated.
+    //!
+    virtual size_t getSize() const = 0;
+
+    //!
+    //! Virtual destructor declaraion
+    //!
+    virtual ~IMirroredBuffer() = default;
+
+}; // class IMirroredBuffer
+
+//!
+//! Class to have a separate memory buffer for discrete device and host allocations.
+//!
+class DiscreteMirroredBuffer : public IMirroredBuffer
+{
+public:
+    void allocate(size_t size) override
+    {
+        mSize = size;
+        mHostBuffer.allocate(size);
+        mDeviceBuffer.allocate(size);
+    }
+
+    void* getDeviceBuffer() const override
+    {
+        return mDeviceBuffer.get();
+    }
+
+    void* getHostBuffer() const override
+    {
+        return mHostBuffer.get();
+    }
+
+    void hostToDevice(TrtCudaStream& stream) override
+    {
+        CHECK(cudaMemcpyAsync(mDeviceBuffer.get(), mHostBuffer.get(), mSize, cudaMemcpyHostToDevice, stream.get()));
+    }
+
+    void deviceToHost(TrtCudaStream& stream) override
+    {
+        CHECK(cudaMemcpyAsync(mHostBuffer.get(), mDeviceBuffer.get(), mSize, cudaMemcpyDeviceToHost, stream.get()));
+    }
+
+    size_t getSize() const override
+    {
+        return mSize;
+    }
+
+private:
+    size_t mSize{0};
+    TrtHostBuffer mHostBuffer;
+    TrtDeviceBuffer mDeviceBuffer;
+}; // class DiscreteMirroredBuffer
+
+//!
+//! Class to have a unified memory buffer for embedded devices.
+//!
+class UnifiedMirroredBuffer : public IMirroredBuffer
+{
+public:
+    void allocate(size_t size) override
+    {
+        mSize = size;
+        mBuffer.allocate(size);
+    }
+
+    void* getDeviceBuffer() const override
+    {
+        return mBuffer.get();
+    }
+
+    void* getHostBuffer() const override
+    {
+        return mBuffer.get();
+    }
+
+    void hostToDevice(TrtCudaStream& stream) override
+    {
+        // Does nothing since we are using unified memory.
+    }
+
+    void deviceToHost(TrtCudaStream& stream) override
+    {
+        // Does nothing since we are using unified memory.
+    }
+
+    size_t getSize() const override
+    {
+        return mSize;
+    }
+
+private:
+    size_t mSize{0};
+    TrtManagedBuffer mBuffer;
+}; // class UnifiedMirroredBuffer
+
+//!
+//! Class to allocate memory for outputs with data-dependent shapes. The sizes of those are unknown so pre-allocation is
+//! not possible.
+//!
+class OutputAllocator : public nvinfer1::IOutputAllocator
+{
+public:
+    //! Construct, using buffer as the backing storage:
+    explicit OutputAllocator(std::shared_ptr<IMirroredBuffer> buffer)
+        : mBuffer{std::move(buffer)}
+    {
+        ASSERT(mBuffer);
+    }
+
+    ~OutputAllocator() override = default;
+
+    void* reallocateOutput(
+        char const* tensorName, void* currentMemory, uint64_t size, uint64_t alignment) noexcept override
+    {
+        // Some memory allocators return nullptr when allocating zero bytes, but TensorRT requires a non-null ptr
+        // even for empty tensors, so allocate a dummy byte.
+        size = std::max(size, static_cast<uint64_t>(1));
+        if (size > mSize)
+        {
+            mBuffer->allocate(roundUp(size, alignment));
+            mSize = size;
+        }
+        return mBuffer->getDeviceBuffer();
+    }
+
+    //! IMirroredBuffer does not implement Async allocation, hence this is just a wrap around
+    void* reallocateOutputAsync(char const* tensorName, void* currentMemory, uint64_t size, uint64_t alignment,
+        cudaStream_t /*stream*/) noexcept override
+    {
+        return reallocateOutput(tensorName, currentMemory, size, alignment);
+    }
+
+    void notifyShape(char const* tensorName, nvinfer1::Dims const& dims) noexcept override
+    {
+        mFinalDims = dims;
+    }
+
+    IMirroredBuffer* getBuffer()
+    {
+        return mBuffer.get();
+    }
+
+    nvinfer1::Dims getFinalDims()
+    {
+        return mFinalDims;
+    }
+
+private:
+    std::shared_ptr<IMirroredBuffer> mBuffer;
+    uint64_t mSize{};
+    nvinfer1::Dims mFinalDims;
+};
+
+//! Set the GPU to run the inference on.
+void setCudaDevice(int32_t device, std::ostream& os);
+
+//! Get the CUDA version of the current CUDA driver.
+int32_t getCudaDriverVersion();
+
+//! Get the CUDA version of the current CUDA runtime.
+int32_t getCudaRuntimeVersion();
+
+
+} // namespace sample
+
+#endif // TRT_SAMPLE_DEVICE_H

samples/common/sampleEngines.h

@@ -0,0 +1,382 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_SAMPLE_ENGINES_H
+#define TRT_SAMPLE_ENGINES_H
+
+#include "NvInfer.h"
+#include "NvOnnxParser.h"
+#include "sampleEntrypoints.h"
+#include "sampleOptions.h"
+#include "sampleUtils.h"
+#include "streamReader.h"
+#include <functional>
+#include <iostream>
+#include <vector>
+
+namespace sample
+{
+
+//! \brief Callback invoked after standard builder configuration, before engine build.
+//! Custom tools can use this to apply additional builder configuration on top of trtexec's.
+using PostConfigCallback = std::function<void(
+    nvinfer1::IBuilder&, nvinfer1::IBuilderConfig&, BuildOptions const&, SystemOptions const&)>;
+
+struct Parser
+{
+    std::unique_ptr<nvonnxparser::IParser> onnxParser;
+
+    operator bool() const
+    {
+        return onnxParser != nullptr;
+    }
+};
+
+//!
+//! \brief Helper struct to faciliate engine serialization and deserialization. It does not own the underlying memory.
+//!
+struct EngineBlob
+{
+    EngineBlob(void* engineData, size_t engineSize)
+        : data(engineData)
+        , size(engineSize)
+    {
+    }
+    void* data{};
+    size_t size{};
+    bool empty() const
+    {
+        return size == 0;
+    }
+};
+
+//!
+//! \brief A helper class to hold a serialized engine (std or safe) and only deserialize it when being accessed.
+//!
+class LazilyDeserializedEngine
+{
+public:
+    //!
+    //! \brief Delete default constructor to make sure isSafe and DLACore are always set.
+    //!
+    LazilyDeserializedEngine() = delete;
+
+    //!
+    //! \brief Constructor of LazilyDeserializedEngine.
+    //!
+    LazilyDeserializedEngine(bool isSafe, bool versionCompatible, int32_t DLACore, std::string const& tempdir,
+        nvinfer1::TempfileControlFlags tempfileControls, std::string const& leanDLLPath)
+        : mIsSafe(isSafe)
+        , mVersionCompatible(versionCompatible)
+        , mDLACore(DLACore)
+        , mTempdir(tempdir)
+        , mTempfileControls(tempfileControls)
+        , mLeanDLLPath(leanDLLPath)
+    {
+        // Only one of these is relevant for any given trtexec call.
+        // Enabled using  --asyncFileReader flag.
+        mAsyncFileReader = std::make_unique<samplesCommon::AsyncStreamReader>();
+    }
+
+    //!
+    //! \brief Move from another LazilyDeserializedEngine.
+    //!
+    LazilyDeserializedEngine(LazilyDeserializedEngine&& other) = default;
+
+    //!
+    //! \brief Delete copy constructor.
+    //!
+    LazilyDeserializedEngine(LazilyDeserializedEngine const& other) = delete;
+
+    //!
+    //! \brief Get the pointer to the ICudaEngine. Triggers deserialization if not already done so.
+    //!
+    nvinfer1::ICudaEngine* get();
+
+    //! \overload nvinfer1::ICudaEngine* get();
+    [[nodiscard]] nvinfer1::ICudaEngine* operator->()
+    {
+        return this->get();
+    }
+
+    //!
+    //! \brief Get the pointer to the ICudaEngine and release the ownership.
+    //!
+    nvinfer1::ICudaEngine* release();
+
+    //!
+    //! \brief Check Safe DLA engine built with kDLA_STANDALONE should not be run via TRT
+    //!
+    bool checkDLASafe();
+
+    //!
+    //! \brief Get the underlying blob storing serialized engine.
+    //!
+    EngineBlob const getBlob() const
+    {
+        ASSERT(!(mAsyncFileReader && mAsyncFileReader->isOpen())
+            && "Attempting to access the glob when there is an open async file reader!");
+        if (!mEngineBlob.empty())
+        {
+            // 'EngineBlob' is a non-owning view over a byte buffer. We intentionally avoid copying the data here.
+            // 'EngineBlob' stores a non-const pointer (`void*`) for legacy reasons, but callers must treat the buffer
+            // as read-only (e.g. writing to disk / passing to deserialize APIs).
+            return EngineBlob{static_cast<void*>(const_cast<uint8_t*>(mEngineBlob.data())), mEngineBlob.size()};
+        }
+        if (mEngineBlobHostMemory != nullptr && mEngineBlobHostMemory->size() > 0)
+        {
+            return EngineBlob{mEngineBlobHostMemory->data(), mEngineBlobHostMemory->size()};
+        }
+        ASSERT(false && "Attempting to access an empty engine!");
+        return EngineBlob{nullptr, 0};
+    }
+
+    //!
+    //! \brief Get the underlying blob storing serialized engine if present, otherwise return an empty blob.
+    //!
+    //! Unlike getBlob(), this function does NOT assert if the blob is empty. This is useful for optional artifacts
+    //! such as kernel text generated via `trtexec --dumpKernelText`.
+    //!
+    EngineBlob const getBlobOrEmpty() const
+    {
+        ASSERT(!(mAsyncFileReader && mAsyncFileReader->isOpen())
+            && "Attempting to access the glob when there is an open async file reader!");
+        if (!mEngineBlob.empty())
+        {
+            // NOTE: `EngineBlob` is a non-owning view over a byte buffer. We intentionally avoid copying the data here.
+            // `EngineBlob` stores a non-const pointer (`void*`) for legacy reasons, but callers must treat the buffer
+            // as read-only (e.g. writing to disk / passing to deserialize APIs).
+            return EngineBlob{static_cast<void*>(const_cast<uint8_t*>(mEngineBlob.data())), mEngineBlob.size()};
+        }
+        if (mEngineBlobHostMemory != nullptr && mEngineBlobHostMemory->size() > 0)
+        {
+            return EngineBlob{mEngineBlobHostMemory->data(), mEngineBlobHostMemory->size()};
+        }
+        return EngineBlob{nullptr, 0};
+    }
+
+    //!
+    //! \brief Set the underlying blob storing the serialized engine without duplicating IHostMemory.
+    //!
+    void setBlob(std::unique_ptr<nvinfer1::IHostMemory>& data)
+    {
+        ASSERT(data.get() && data->size() > 0);
+        mEngineBlobHostMemory = std::move(data);
+        mEngine.reset();
+    }
+
+    //!
+    //! \brief Set the underlying blob storing the serialized engine without duplicating vector memory.
+    //!
+    void setBlob(std::vector<uint8_t>&& engineBlob)
+    {
+        mEngineBlob = std::move(engineBlob);
+        mEngine.reset();
+    }
+
+    //!
+    //! \brief Release the underlying blob without deleting the deserialized engine.
+    //!
+    void releaseBlob()
+    {
+        mEngineBlob.clear();
+        mEngineBlobHostMemory.reset();
+    }
+
+    //!
+    //! \brief Get the file stream reader used for deserialization
+    //!
+    samplesCommon::AsyncStreamReader& getAsyncFileReader()
+    {
+        ASSERT(mAsyncFileReader);
+        return *mAsyncFileReader;
+    }
+
+    //!
+    //! \brief Get if safe mode is enabled.
+    //!
+    bool isSafe()
+    {
+        return mIsSafe;
+    }
+
+    void setDynamicPlugins(std::vector<std::string> const& dynamicPlugins)
+    {
+        mDynamicPlugins = dynamicPlugins;
+    }
+
+private:
+    bool mIsSafe{false};
+    bool mVersionCompatible{false};
+    int32_t mDLACore{-1};
+    std::vector<uint8_t> mEngineBlob;
+    std::unique_ptr<samplesCommon::AsyncStreamReader> mAsyncFileReader;
+
+    // Directly use the host memory of a serialized engine instead of duplicating the engine in CPU memory.
+    std::unique_ptr<nvinfer1::IHostMemory> mEngineBlobHostMemory;
+
+    std::string mTempdir{};
+    nvinfer1::TempfileControlFlags mTempfileControls{getTempfileControlDefaults()};
+    std::string mLeanDLLPath{};
+    std::vector<std::string> mDynamicPlugins;
+
+    //! \name Owned TensorRT objects
+    //! Per TensorRT object lifetime requirements as outlined in the developer guide,
+    //! the runtime must remain live while any engines created by the runtime are live.
+    //! DO NOT ADJUST the declaration order here: runtime -> (engine).
+    //! Destruction occurs in reverse declaration order: (engine) -> runtime.
+    //!@{
+
+    //! The runtime used to track parent of mRuntime if one exists.
+    //! Needed to load mRuntime if lean.so is supplied through file system path.
+    std::unique_ptr<nvinfer1::IRuntime> mParentRuntime{};
+
+    //! The runtime that is used to deserialize the engine.
+    std::unique_ptr<nvinfer1::IRuntime> mRuntime{};
+
+    //! If mIsSafe is false, this points to the deserialized std engine
+    std::unique_ptr<nvinfer1::ICudaEngine> mEngine{};
+
+    //!@}
+};
+
+struct BuildEnvironment
+{
+    BuildEnvironment() = delete;
+    BuildEnvironment(BuildEnvironment const& other) = delete;
+    BuildEnvironment(BuildEnvironment&& other) = delete;
+    BuildEnvironment(bool isSafe, bool versionCompatible, int32_t DLACore, std::string const& tempdir,
+        nvinfer1::TempfileControlFlags tempfileControls, std::string const& leanDLLPath = "",
+        std::string const& cmdline = "")
+        : engine(isSafe, versionCompatible, DLACore, tempdir, tempfileControls, leanDLLPath)
+        , kernelText(false, false, -1, "", tempfileControls, "")
+        , cmdline(cmdline)
+    {
+    }
+
+    //! \name Owned TensorRT objects
+    //! Per TensorRT object lifetime requirements as outlined in the developer guide,
+    //! factory objects must remain live while the objects created by those factories
+    //! are live (with the exception of builder -> engine).
+    //! DO NOT ADJUST the declaration order here: builder -> builder config -> network -> parser.
+    //! Destruction occurs in reverse declaration order: parser -> network -> builder config -> builder.
+    //!@{
+
+    //! The builder used to build the engine.
+    std::unique_ptr<nvinfer1::IBuilder> builder;
+
+    // Builder config used to build the engine.
+    std::unique_ptr<nvinfer1::IBuilderConfig> builderConfig;
+
+    //! The network used by the builder.
+    std::unique_ptr<nvinfer1::INetworkDefinition> network;
+
+    //! The parser used to specify the network.
+    Parser parser;
+
+    //! The engine.
+    LazilyDeserializedEngine engine;
+
+    //! The kernel CPP text.
+    LazilyDeserializedEngine kernelText;
+
+    //! The command line string.
+    std::string cmdline;
+    //!@}
+};
+
+//!
+//! \brief Log refittable layers and weights of a refittable engine
+//!
+void dumpRefittable(nvinfer1::ICudaEngine& engine);
+
+//!
+//! \brief Load a serialized engine
+//!
+//! \return Pointer to the engine loaded or nullptr if the operation failed
+//!
+nvinfer1::ICudaEngine* loadEngine(std::string const& engine, int32_t DLACore, std::ostream& err);
+
+//!
+//! \brief Save an engine into a file
+//!
+//! \return boolean Return true if the engine was successfully saved
+//!
+bool saveEngine(nvinfer1::ICudaEngine const& engine, std::string const& fileName, std::ostream& err);
+
+//!
+//! \brief Create an engine from model or serialized file, and optionally save engine
+//!
+//! \return Pointer to the engine created or nullptr if the creation failed
+//!
+bool getEngineBuildEnv(
+    ModelOptions const& model, BuildOptions const& build, SystemOptions& sys, BuildEnvironment& env, std::ostream& err, PostConfigCallback const& postConfigHook = nullptr);
+
+//!
+//! \brief Create a serialized network
+//!
+//! \return Pointer to a host memory for a serialized network
+//!
+nvinfer1::IHostMemory* networkToSerialized(const BuildOptions& build, const SystemOptions& sys,
+    nvinfer1::IBuilder& builder, nvinfer1::INetworkDefinition& network, std::ostream& err);
+
+//!
+//! \brief Tranfer model to a serialized network
+//!
+//! \return Pointer to a host memory for a serialized network
+//!
+nvinfer1::IHostMemory* modelToSerialized(
+    const ModelOptions& model, const BuildOptions& build, const SystemOptions& sys, std::ostream& err);
+
+//!
+//! \brief Serialize network and save it into a file
+//!
+//! \return boolean Return true if the network was successfully serialized and saved
+//!
+bool serializeAndSave(
+    const ModelOptions& model, const BuildOptions& build, const SystemOptions& sys, std::ostream& err);
+
+//!
+//! \brief Refit an engine using the weights from the specified ONNX model.
+//!
+//! \return boolean Return true if the engine was successfully refit from the model.
+//!
+bool refitFromOnnx(nvinfer1::ICudaEngine& engine, std::string onnxModelFile, bool multiThreading);
+
+//!
+//! \brief Refit an engine using the weights from the INetworkDefintiion and report the amount of time it took.
+//!
+//! \return boolean Return true if the engine was successfully refit from the INetworkDefinition.
+//!
+bool timeRefit(nvinfer1::INetworkDefinition const& network, nvinfer1::ICudaEngine& engine, bool multiThreading);
+
+//! \brief Check if safe runtime is loaded.
+[[nodiscard]] bool hasSafeRuntime();
+
+//!
+//! \brief Run consistency check on serialized engine.
+//!
+[[nodiscard]] bool checkSafeEngine(
+    void const* serializedEngine, int64_t const engineSize, std::vector<std::string> const& pluginBuildLibPath);
+
+bool loadStreamingEngineToBuildEnv(std::string const& engine, BuildEnvironment& env, std::ostream& err);
+
+bool loadEngineToBuildEnv(std::string const& engine, BuildEnvironment& env, std::ostream& err, SystemOptions const& sys,
+    bool const enableConsistency);
+} // namespace sample
+
+#endif // TRT_SAMPLE_ENGINES_H

samples/common/sampleEntrypoints.h

@@ -0,0 +1,107 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_SAMPLE_ENTRYPOINTS_H
+#define TRT_SAMPLE_ENTRYPOINTS_H
+
+//! \file sampleEntrypoints.h
+//!
+//! Declares and conditionally defines entrypoints needed to create base TensorRT objects, depending
+//! on whether the given sample uses TRT at link time or dynamically.  Since common code is built once
+//! and shared across all samples (both link-time and dynamic TRT), it does not define these entrypoints,
+//! so each sample must define them individually.
+//!
+//! Samples that use TRT at link time can define DEFINE_TRT_ENTRYPOINTS before including this header to
+//! pick up the definitions here.
+
+#include "NvInfer.h"
+#include "NvOnnxParser.h"
+#include "logger.h"
+
+extern nvinfer1::IBuilder* createBuilder();
+extern nvinfer1::IRuntime* createRuntime();
+extern nvinfer1::IRefitter* createRefitter(nvinfer1::ICudaEngine& engine);
+extern nvonnxparser::IParser* createONNXParser(nvinfer1::INetworkDefinition& network);
+extern nvonnxparser::IParserRefitter* createONNXRefitter(nvinfer1::IRefitter& refitter);
+
+#if !defined(DEFINE_TRT_ENTRYPOINTS)
+#define DEFINE_TRT_ENTRYPOINTS 0
+#endif
+
+// Allow opting out of individual entrypoints that are unused by the sample
+#if !defined(DEFINE_TRT_BUILDER_ENTRYPOINT)
+#define DEFINE_TRT_BUILDER_ENTRYPOINT 1
+#endif
+#if !defined(DEFINE_TRT_RUNTIME_ENTRYPOINT)
+#define DEFINE_TRT_RUNTIME_ENTRYPOINT 1
+#endif
+#if !defined(DEFINE_TRT_REFITTER_ENTRYPOINT)
+#define DEFINE_TRT_REFITTER_ENTRYPOINT 1
+#endif
+#if !defined(DEFINE_TRT_ONNX_PARSER_ENTRYPOINT)
+#define DEFINE_TRT_ONNX_PARSER_ENTRYPOINT 1
+#endif
+
+#if DEFINE_TRT_ENTRYPOINTS
+nvinfer1::IBuilder* createBuilder()
+{
+#if DEFINE_TRT_BUILDER_ENTRYPOINT
+    return nvinfer1::createInferBuilder(sample::gLogger.getTRTLogger());
+#else
+    return {};
+#endif
+}
+
+nvinfer1::IRuntime* createRuntime()
+{
+#if DEFINE_TRT_RUNTIME_ENTRYPOINT
+    return nvinfer1::createInferRuntime(sample::gLogger.getTRTLogger());
+#else
+    return {};
+#endif
+}
+
+nvinfer1::IRefitter* createRefitter(nvinfer1::ICudaEngine& engine)
+{
+#if DEFINE_TRT_REFITTER_ENTRYPOINT
+    return nvinfer1::createInferRefitter(engine, sample::gLogger.getTRTLogger());
+#else
+    return {};
+#endif
+}
+
+nvonnxparser::IParser* createONNXParser(nvinfer1::INetworkDefinition& network)
+{
+#if DEFINE_TRT_ONNX_PARSER_ENTRYPOINT
+    return nvonnxparser::createParser(network, sample::gLogger.getTRTLogger());
+#else
+    return {};
+#endif
+}
+
+nvonnxparser::IParserRefitter* createONNXRefitter(nvinfer1::IRefitter& refitter)
+{
+#if DEFINE_TRT_ONNX_PARSER_ENTRYPOINT
+    return nvonnxparser::createParserRefitter(refitter, sample::gLogger.getTRTLogger());
+#else
+    return {};
+#endif
+}
+
+#endif // DEFINE_TRT_ENTRYPOINTS
+
+#endif // TRT_SAMPLE_ENTRYPOINTS_H

samples/common/sampleInference.h

@@ -0,0 +1,546 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_SAMPLE_INFERENCE_H
+#define TRT_SAMPLE_INFERENCE_H
+
+#include "debugTensorWriter.h"
+#include "sampleDevice.h"
+#include "sampleEngines.h"
+#include "sampleReporting.h"
+#include "sampleUtils.h"
+
+#include <functional>
+#include <iostream>
+#include <list>
+#include <memory>
+#include <string>
+#include <vector>
+
+#if ENABLE_UNIFIED_BUILDER
+#include "safeCudaAllocator.h"
+#endif
+namespace sample
+{
+using LibraryPtr = std::unique_ptr<samplesCommon::DynamicLibrary>;
+
+std::string const TRT_NVINFER_NAME = "nvinfer";
+std::string const TRT_ONNXPARSER_NAME = "nvonnxparser";
+std::string const TRT_LIB_SUFFIX = "";
+
+#if !TRT_STATIC
+#if defined(_WIN32)
+std::string const kNVINFER_PLUGIN_LIBNAME
+    = std::string{"nvinfer_plugin_"} + std::to_string(NV_TENSORRT_MAJOR) + std::string{".dll"};
+std::string const kNVINFER_LIBNAME = std::string(TRT_NVINFER_NAME) + std::string{"_"}
+    + std::to_string(NV_TENSORRT_MAJOR) + TRT_LIB_SUFFIX + std::string{".dll"};
+std::string const kNVINFER_SAFE_LIBNAME
+    = std::string{"nvinfer_safe_"} + std::to_string(NV_TENSORRT_MAJOR) + std::string{".dll"};
+std::string const kNVONNXPARSER_LIBNAME = std::string(TRT_ONNXPARSER_NAME) + std::string{"_"}
+    + std::to_string(NV_TENSORRT_MAJOR) + TRT_LIB_SUFFIX + std::string{".dll"};
+std::string const kNVINFER_LEAN_LIBNAME
+    = std::string{"nvinfer_lean_"} + std::to_string(NV_TENSORRT_MAJOR) + std::string{".dll"};
+std::string const kNVINFER_DISPATCH_LIBNAME
+    = std::string{"nvinfer_dispatch_"} + std::to_string(NV_TENSORRT_MAJOR) + std::string{".dll"};
+#else
+std::string const kNVINFER_PLUGIN_LIBNAME = std::string{"libnvinfer_plugin.so."} + std::to_string(NV_TENSORRT_MAJOR);
+std::string const kNVINFER_LIBNAME
+    = std::string{"lib"} + std::string(TRT_NVINFER_NAME) + std::string{".so."} + std::to_string(NV_TENSORRT_MAJOR);
+std::string const kNVINFER_SAFE_LIBNAME = std::string{"libnvinfer_safe.so."} + std::to_string(NV_TENSORRT_MAJOR);
+std::string const kNVONNXPARSER_LIBNAME
+    = std::string{"lib"} + std::string(TRT_ONNXPARSER_NAME) + std::string{".so."} + std::to_string(NV_TENSORRT_MAJOR);
+std::string const kNVINFER_LEAN_LIBNAME = std::string{"libnvinfer_lean.so."} + std::to_string(NV_TENSORRT_MAJOR);
+std::string const kNVINFER_DISPATCH_LIBNAME
+    = std::string{"libnvinfer_dispatch.so."} + std::to_string(NV_TENSORRT_MAJOR);
+#endif
+
+std::string const& getRuntimeLibraryName(RuntimeMode const mode);
+
+template <typename FetchPtrs>
+bool initLibrary(LibraryPtr& libPtr, std::string const& libName, FetchPtrs fetchFunc)
+{
+    if (libPtr != nullptr)
+    {
+        return true;
+    }
+    try
+    {
+        libPtr.reset(new samplesCommon::DynamicLibrary{libName});
+        fetchFunc(libPtr.get());
+    }
+    catch (std::exception const& e)
+    {
+        libPtr.reset();
+        sample::gLogError << "Could not load library " << libName << ": " << e.what() << std::endl;
+        return false;
+    }
+    catch (...)
+    {
+        libPtr.reset();
+        sample::gLogError << "Could not load library " << libName << std::endl;
+        return false;
+    }
+
+    return true;
+}
+#endif // !TRT_STATIC
+
+#if ENABLE_UNIFIED_BUILDER
+namespace safe
+{
+
+//!
+//! \brief Initialize the NVIDIA Inference Safe Runtime library
+//!
+//! This function dynamically loads the Safe TensorRT runtime library and initializes
+//! function pointers for safe TensorRT operations. It is used to set up the safe runtime
+//! environment for inference with safety-certified TensorRT engines.
+//!
+//! \return true if the safe runtime library was successfully loaded and initialized,
+//!         false otherwise (e.g., in static builds or if library loading fails)
+//!
+bool initNvinferSafe();
+
+//!
+//! \brief Create a safe TRT graph from serialized engine data
+//!
+//! This function creates a safe TRT graph from serialized engine data. It is used to create
+//! a safe TRT graph for inference with safety-certified TensorRT engines.
+//!
+//! \param graph: Pointer to the safe TRT graph to be created
+//! \param blob: Pointer to the serialized engine data
+//! \param size: Size of the serialized engine data
+//! \param recorder: Reference to the safe recorder
+//! \param useManaged: Flag indicating whether to use managed memory
+//! \param allocator: Pointer to the safe memory allocator
+//! \return Error code indicating the success or failure of the operation
+//!
+nvinfer1::ErrorCode createSafeTRTGraph(nvinfer2::safe::ITRTGraph*& graph, void const* blob, int64_t size,
+    ISafeRecorder& recorder, bool useManaged, ISafeMemAllocator* allocator);
+
+//!
+//! \brief Destroy a safe TRT graph and release resources
+//!
+//! This function destroys a safe TRT graph and releases the associated resources. It is used to clean up
+//! the safe TRT graph after inference with safety-certified TensorRT engines.
+//!
+//! \param graph: Pointer to the safe TRT graph to be destroyed
+//! \return Error code indicating the success or failure of the operation
+//!
+nvinfer1::ErrorCode destroySafeTRTGraph(nvinfer2::safe::ITRTGraph*& graph);
+
+//!
+//! \brief Get the safe plugin registry for loading plugins
+//!
+//! This function retrieves the safe plugin registry for loading plugins. It is used to get the safe plugin registry
+//! for loading plugins with safety-certified TensorRT engines.
+//!
+//! \param recorder: Reference to the safe recorder
+//! \return Pointer to the safe plugin registry
+//!
+nvinfer2::safe::ISafePluginRegistry* getSafePluginRegistry(ISafeRecorder& recorder);
+} // namespace safe
+#endif
+
+struct InferenceEnvironmentBase
+{
+    InferenceEnvironmentBase() = delete;
+    virtual ~InferenceEnvironmentBase() = default;
+    InferenceEnvironmentBase(InferenceEnvironmentBase const& other) = delete;
+    InferenceEnvironmentBase(InferenceEnvironmentBase&& other) = delete;
+    InferenceEnvironmentBase(BuildEnvironment& bEnv)
+        : engine(std::move(bEnv.engine))
+        , safe(bEnv.engine.isSafe())
+        , cmdline(bEnv.cmdline)
+    {
+    }
+
+    LazilyDeserializedEngine engine;
+    std::unique_ptr<Profiler> profiler;
+    std::vector<TrtDeviceBuffer>
+        deviceMemory; //< Device memory used for inference when the allocation strategy is not static.
+    std::unique_ptr<DebugTensorWriter> listener;
+    bool error{false};
+    bool accuracyFailed{false};                                 //< Set to true if any tensor accuracy exceeds threshold
+    std::unordered_map<std::string, double> accuracyLossValues; //< Per-tensor accuracy values from the last validation
+
+    bool safe{false};
+    std::string cmdline;
+
+#if !defined(_WIN32)
+    //! Reference outputs for accuracy validation (tuner feature, Linux enterprise/auto-only).
+    //! Map from tensor name to host buffer containing reference data.
+    //! Guarded because MSVC cannot instantiate vector<unordered_map<string, unique_ptr<T>>>,
+    //! and the tuner does not run on Windows or RTX/winjit.
+    using RefOutputMap = std::unordered_map<std::string, std::unique_ptr<TrtHostBuffer>>;
+    //! Vector of reference output maps, one for each refPair.
+    std::vector<RefOutputMap> refOutputsAll;
+#endif // !defined(_WIN32) && !TRT_WINML
+};
+
+struct InferenceEnvironmentStd : public InferenceEnvironmentBase
+{
+    InferenceEnvironmentStd() = delete;
+    InferenceEnvironmentStd(InferenceEnvironmentStd const& other) = delete;
+    InferenceEnvironmentStd(InferenceEnvironmentStd&& other) = delete;
+    InferenceEnvironmentStd(BuildEnvironment& bEnv)
+        : InferenceEnvironmentBase(bEnv)
+    {
+    }
+    std::vector<std::unique_ptr<nvinfer1::IExecutionContext>> contexts;
+    std::vector<std::unique_ptr<BindingsStd>> bindings;
+
+    inline nvinfer1::IExecutionContext* getContext(int32_t streamIdx);
+
+    //! Storage for input shape tensors.
+    //!
+    //! It's important that the addresses of the data do not change between the calls to
+    //! setTensorAddress/setInputShape (which tells TensorRT where the input shape tensor is)
+    //! and enqueueV3 (when TensorRT might use the input shape tensor).
+    //!
+    //! The input shape tensors could alternatively be handled via member bindings,
+    //! but it simplifies control-flow to store the data here since it's shared across
+    //! the bindings.
+    std::list<std::vector<int64_t>> inputShapeTensorValues;
+};
+
+#if ENABLE_UNIFIED_BUILDER
+// Forward declaration of BindingsSafe
+class BindingsSafe;
+
+struct InferenceEnvironmentSafe : public InferenceEnvironmentBase
+{
+    InferenceEnvironmentSafe() = delete;
+    InferenceEnvironmentSafe(InferenceEnvironmentSafe const& other) = delete;
+    InferenceEnvironmentSafe(InferenceEnvironmentSafe&& other) = delete;
+    InferenceEnvironmentSafe(BuildEnvironment& bEnv)
+        : InferenceEnvironmentBase(bEnv)
+    {
+    }
+
+    std::vector<std::unique_ptr<BindingsSafe>> bindings;
+    //! deleters for aux. streams, per cloned graph
+    std::vector<std::shared_ptr<std::nullptr_t>> mAuxStreamsDeleters;
+    std::vector<std::unique_ptr<nvinfer2::safe::ITRTGraph>> mClonedGraphs;
+};
+#endif
+
+inline nvinfer1::IExecutionContext* InferenceEnvironmentStd::getContext(int32_t streamIdx)
+{
+    return contexts[streamIdx].get();
+}
+
+//!
+//! \brief Set up contexts/graphs and bindings for inference
+//!
+bool setUpInference(InferenceEnvironmentBase& iEnv, InferenceOptions const& inference, SystemOptions const& system);
+
+#if ENABLE_UNIFIED_BUILDER
+//!
+//! \brief Set up graphs and bindings for safe inference
+//!
+bool setUpSafeInference(InferenceEnvironmentSafe& iEnv, InferenceOptions const& inference, SystemOptions const& system);
+#endif
+
+//!
+//! \brief Set up contexts and bindings for standard inference
+//!
+bool setUpStdInference(InferenceEnvironmentStd& iEnv, InferenceOptions const& inference, SystemOptions const& system);
+
+//!
+//! \brief Deserialize the engine and time how long it takes.
+//!
+bool timeDeserialize(InferenceEnvironmentBase& iEnv, SystemOptions const& sys);
+
+//!
+//! \brief Run inference and collect timing, return false if any error hit during inference
+//!
+bool runInference(InferenceOptions const& inference, InferenceEnvironmentBase& iEnv, int32_t device,
+    std::vector<InferenceTrace>& trace, ReportingOptions const& reporting);
+
+#if !defined(_WIN32)
+//!
+//! \brief Load reference outputs from files into InferenceEnvironmentBase::refOutputsAll.
+//! \param pairIndex Index of the refPair to use (default 0 for backward compatibility).
+//!
+void loadRefOutputs(InferenceEnvironmentBase& iEnv, InferenceOptions const& inference,
+    nvinfer1::IExecutionContext const& context, int64_t pairIndex = 0);
+
+#if ENABLE_UNIFIED_BUILDER
+//!
+//! \brief Load reference outputs from files for safe inference.
+//! \param pairIndex Index of the refPair to use (default 0 for backward compatibility).
+//!
+void loadRefOutputs(InferenceEnvironmentBase& iEnv, InferenceOptions const& inference,
+    nvinfer2::safe::ITRTGraph const& graph, int64_t pairIndex = 0);
+#endif
+#endif // !defined(_WIN32) && !TRT_WINML
+
+//!
+//! \brief Get layer information of the engine.
+//!
+std::string getLayerInformation(
+    nvinfer1::ICudaEngine* engine, nvinfer1::IExecutionContext* context, nvinfer1::LayerInformationFormat format);
+
+struct Binding
+{
+    bool isInput{false};
+    std::shared_ptr<IMirroredBuffer> buffer; // shared_ptr to allow aliasing between inputs and outputs
+    std::unique_ptr<OutputAllocator> outputAllocator;
+    int64_t volume{0};
+    nvinfer1::DataType dataType{nvinfer1::DataType::kFLOAT};
+
+    void fill(std::string const& fileName);
+
+    void fill();
+
+    void dump(std::ostream& os, nvinfer1::Dims dims, nvinfer1::Dims strides, int32_t vectorDim, int32_t spv,
+        std::string const separator = " ") const;
+};
+
+struct TensorInfo
+{
+    int32_t bindingIndex{-1};
+    char const* name{nullptr};
+    nvinfer1::Dims dims{};
+    bool isDynamic{};
+    int32_t comps{-1};
+    nvinfer1::Dims strides{};
+    int32_t vectorDimIndex{-1};
+    bool isInput{};
+    nvinfer1::DataType dataType{};
+    int64_t vol{-1};
+
+    void updateVolume(int32_t batch)
+    {
+        vol = volume(dims, strides, vectorDimIndex, comps, batch);
+    }
+};
+
+class BindingsBase
+{
+public:
+    BindingsBase() = delete;
+    explicit BindingsBase(bool useManaged)
+        : mUseManaged(useManaged)
+    {
+    }
+
+    void addBinding(
+        TensorInfo const& tensorInfo, std::string const& fileName = "", char const* aliasedInputTensor = nullptr);
+
+    void** getDeviceBuffers();
+
+    void transferInputToDevice(TrtCudaStream& stream);
+
+    void transferOutputToHost(TrtCudaStream& stream);
+
+    void fill(int binding, std::string const& fileName)
+    {
+        mBindings[binding].fill(fileName);
+    }
+
+    void fill(int binding)
+    {
+        mBindings[binding].fill();
+    }
+
+    std::unordered_map<std::string, int> getInputBindings() const
+    {
+        auto isInput = [](Binding const& b) { return b.isInput; };
+        return getBindings(isInput);
+    }
+
+    //! Fill input bindings from a name-to-file map.
+    //!
+    //! \param inputMap A map where:
+    //!   - key: tensor name (e.g., "input", "input:0")
+    //!   - value: file path containing the tensor data to load (e.g., "input_0.dat")
+    //!
+    //! For each entry in the map, looks up the tensor name in the input bindings
+    //! and fills the binding buffer with data from the specified file.
+    //! Entries with tensor names not found in input bindings are silently skipped.
+    void fillInputsFromMap(std::unordered_map<std::string, std::string> const& inputMap)
+    {
+        auto inputBindings = getInputBindings();
+        for (auto const& item : inputMap)
+        {
+            auto it = inputBindings.find(item.first);
+            if (it != inputBindings.end())
+            {
+                fill(it->second, item.second);
+            }
+        }
+    }
+
+    std::unordered_map<std::string, int> getOutputBindings() const
+    {
+        auto isOutput = [](Binding const& b) { return !b.isInput; };
+        return getBindings(isOutput);
+    }
+
+    std::unordered_map<std::string, int> getBindings() const
+    {
+        auto all = [](Binding const& b) { return true; };
+        return getBindings(all);
+    }
+
+    std::unordered_map<std::string, int> getBindings(std::function<bool(Binding const&)> predicate) const;
+
+    Binding const& getBinding(int32_t index) const
+    {
+        return mBindings.at(index);
+    }
+
+protected:
+    std::unordered_map<std::string, int32_t> mNames;
+    std::vector<Binding> mBindings;
+    std::vector<void*> mDevicePointers;
+    bool mUseManaged{false};
+};
+
+class BindingsStd : public BindingsBase
+{
+public:
+    BindingsStd() = delete;
+    explicit BindingsStd(bool useManaged)
+        : BindingsBase(useManaged)
+    {
+    }
+
+    void dumpInputs(nvinfer1::IExecutionContext const& context, std::ostream& os) const
+    {
+        auto isInput = [](Binding const& b) { return b.isInput; };
+        dumpBindings(context, isInput, os);
+    }
+
+    void dumpOutputs(nvinfer1::IExecutionContext const& context, std::ostream& os) const
+    {
+        auto isOutput = [](Binding const& b) { return !b.isInput; };
+        dumpBindings(context, isOutput, os);
+    }
+
+    void dumpBindings(nvinfer1::IExecutionContext const& context, std::ostream& os) const
+    {
+        auto all = [](Binding const& b) { return true; };
+        dumpBindings(context, all, os);
+    }
+
+    void dumpBindings(nvinfer1::IExecutionContext const& context, std::function<bool(Binding const&)> predicate,
+        std::ostream& os) const
+    {
+        for (auto const& n : mNames)
+        {
+            auto const name = n.first;
+            auto const binding = n.second;
+            if (predicate(mBindings[binding]))
+            {
+                os << n.first << ": (";
+                dumpBindingDimensions(name, context, os);
+                os << ")" << std::endl;
+
+                dumpBindingValues(context, binding, os);
+                os << std::endl;
+            }
+        }
+    }
+
+    void dumpBindingDimensions(
+        std::string const& name, nvinfer1::IExecutionContext const& context, std::ostream& os) const;
+
+    void dumpBindingValues(nvinfer1::IExecutionContext const& context, int32_t binding, std::ostream& os,
+        std::string const& separator = " ", int32_t batch = 1) const;
+
+    void dumpRawBindingToFiles(nvinfer1::IExecutionContext const& context, std::ostream& os) const;
+
+    bool setTensorAddresses(nvinfer1::IExecutionContext& context) const;
+};
+#if ENABLE_UNIFIED_BUILDER
+class BindingsSafe : public BindingsBase
+{
+public:
+    BindingsSafe() = delete;
+    explicit BindingsSafe(bool useManaged)
+        : BindingsBase(useManaged)
+    {
+    }
+
+    void dumpInputs(ITRTGraph const& graph, std::ostream& os) const
+    {
+        auto isInput = [](Binding const& b) { return b.isInput; };
+        dumpBindings(graph, isInput, os);
+    }
+
+    void dumpOutputs(ITRTGraph const& graph, std::ostream& os) const
+    {
+        auto isOutput = [](Binding const& b) { return !b.isInput; };
+        dumpBindings(graph, isOutput, os);
+    }
+
+    void dumpBindings(ITRTGraph const& graph, std::ostream& os) const
+    {
+        auto all = [](Binding const& b) { return true; };
+        dumpBindings(graph, all, os);
+    }
+
+    void dumpBindings(ITRTGraph const& graph, std::function<bool(Binding const&)> predicate, std::ostream& os) const
+    {
+        for (auto const& n : mNames)
+        {
+            auto const name = n.first;
+            auto const binding = n.second;
+            if (predicate(mBindings[binding]))
+            {
+                os << n.first << ": (";
+                dumpBindingDimensions(name, graph, os);
+                os << ")" << std::endl;
+
+                dumpBindingValues(graph, binding, os);
+                os << std::endl;
+            }
+        }
+    }
+
+    void dumpBindingDimensions(std::string const& name, ITRTGraph const& graph, std::ostream& os) const;
+
+    void dumpBindingValues(ITRTGraph const& graph, int32_t binding, std::ostream& os,
+        std::string const& separator = " ", int32_t batch = 1) const;
+
+    void dumpRawBindingToFiles(ITRTGraph& graph, std::ostream& os) const;
+
+    bool setTensorAddresses(ITRTGraph& graph) const;
+};
+#endif
+
+struct TaskInferenceEnvironment
+{
+    TaskInferenceEnvironment(std::string engineFile, InferenceOptions const& inference,
+        ReportingOptions const& reporting, int32_t deviceId = 0,
+        int32_t DLACore = -1, int32_t bs = batchNotProvided);
+    InferenceOptions iOptions{};
+    ReportingOptions rOptions{};
+    int32_t device{defaultDevice};
+    int32_t batch{batchNotProvided};
+    std::unique_ptr<InferenceEnvironmentStd> iEnv;
+    std::vector<InferenceTrace> trace;
+};
+
+bool runMultiTasksInference(std::vector<std::unique_ptr<TaskInferenceEnvironment>>& tEnvList);
+
+} // namespace sample
+
+#endif // TRT_SAMPLE_INFERENCE_H

samples/common/sampleOptions.h

@@ -0,0 +1,582 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_SAMPLE_OPTIONS_H
+#define TRT_SAMPLE_OPTIONS_H
+
+#include <array>
+#include <iostream>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include "NvInfer.h"
+
+#if ENABLE_UNIFIED_BUILDER
+#include "safeCommon.h"
+#endif
+
+namespace sample
+{
+
+// Build default params
+constexpr int32_t defaultAvgTiming{8};
+constexpr int32_t defaultMaxAuxStreams{-1};
+constexpr int32_t defaultBuilderOptimizationLevel{-1};
+constexpr int32_t defaultTilingOptimizationLevel{static_cast<int32_t>(nvinfer1::TilingOptimizationLevel::kNONE)};
+constexpr int32_t defaultMaxTactics{-1};
+
+// System default params
+constexpr int32_t defaultDevice{0};
+
+// Inference default params
+constexpr int32_t defaultBatch{1};
+constexpr int32_t batchNotProvided{0};
+constexpr int32_t defaultStreams{1};
+constexpr int32_t defaultIterations{10};
+constexpr int32_t defaultOptProfileIndex{0};
+constexpr float defaultWarmUp{200.F};
+constexpr float defaultDuration{3.F};
+constexpr float defaultSleep{};
+constexpr float defaultIdle{};
+constexpr float defaultPersistentCacheRatio{0};
+
+// Reporting default params
+constexpr int32_t defaultAvgRuns{10};
+constexpr std::array<float, 3> defaultPercentiles{90, 95, 99};
+
+
+enum class ModelFormat
+{
+    kANY,
+    kONNX
+};
+
+enum class SparsityFlag
+{
+    kDISABLE,
+    kENABLE,
+    kFORCE
+};
+
+enum class TimingCacheMode
+{
+    kDISABLE,
+    kLOCAL,
+    kGLOBAL
+};
+
+enum class MemoryAllocationStrategy
+{
+    kSTATIC,  //< Allocate device memory based on max size across all profiles.
+    kPROFILE, //< Allocate device memory based on max size of the current profile.
+    kRUNTIME, //< Allocate device memory based on the current input shapes.
+};
+
+enum class AccuracyValidationAlgorithm
+{
+    kL0,               //< L0 accuracy algorithm
+    kL1,               //< L1 accuracy algorithm
+    kL2,               //< L2 accuracy algorithm
+    kLInf,             //< LInf accuracy algorithm
+    kCosineSimilarity, //< Cosine similarity accuracy algorithm
+};
+
+enum class TuningSearchAlgorithm
+{
+    kFAST,       //< Fast searching algorithm: baseline + one-off variations (linear in #knobs)
+    kEXHAUSTIVE, //< Exhaustive: all combinations enumerated (product of knob value-counts)
+    kMIXED,      //< Two-phase: fast scan, then exhaustive over knobs that improved performance
+};
+
+//! \brief Convert a TuningSearchAlgorithm enum to its CLI / cache-file string.
+//!   kFAST       -> "fast"
+//!   kEXHAUSTIVE -> "full"
+//!   kMIXED      -> "mixed"
+inline std::string toString(TuningSearchAlgorithm algo)
+{
+    switch (algo)
+    {
+    case TuningSearchAlgorithm::kFAST: return "fast";
+    case TuningSearchAlgorithm::kEXHAUSTIVE: return "full";
+    case TuningSearchAlgorithm::kMIXED: return "mixed";
+    }
+    return "unknown";
+}
+
+//!
+//! \enum RuntimeMode
+//!
+//! \brief Used to dictate which TensorRT runtime library to dynamically load.
+//!
+enum class RuntimeMode
+{
+    //! Maps to libnvinfer.so or nvinfer.dll
+    kFULL,
+
+    //! Maps to libnvinfer_dispatch.so or nvinfer_dispatch.dll
+    kDISPATCH,
+
+    //! Maps to libnvinfer_lean.so or nvinfer_lean.dll
+    kLEAN,
+
+    //! Maps to libnvinfer_safe.so or nvinfer_safe.dll
+    kSAFE,
+};
+
+inline std::ostream& operator<<(std::ostream& os, RuntimeMode const mode)
+{
+    switch (mode)
+    {
+    case RuntimeMode::kFULL:
+    {
+        os << "full";
+        break;
+    }
+    case RuntimeMode::kDISPATCH:
+    {
+        os << "dispatch";
+        break;
+    }
+    case RuntimeMode::kLEAN:
+    {
+        os << "lean";
+        break;
+    }
+    case RuntimeMode::kSAFE:
+    {
+        os << "safe";
+        break;
+    }
+    }
+
+    return os;
+}
+
+using Arguments = std::unordered_multimap<std::string, std::pair<std::string, int32_t>>;
+
+//! An IO format specification.
+struct IOFormat
+{
+    nvinfer1::TensorFormats formats{};
+};
+
+using ShapeRange = std::array<std::vector<int64_t>, nvinfer1::EnumMax<nvinfer1::OptProfileSelector>()>;
+
+using LayerDeviceTypes = std::unordered_map<std::string, nvinfer1::DeviceType>;
+using DecomposableAttentions = std::unordered_map<std::string, bool>;
+
+using StringSet = std::unordered_set<std::string>;
+
+class WeightStreamingBudget
+{
+public:
+    static constexpr int64_t kDISABLE{-2};
+    static constexpr int64_t kAUTOMATIC{-1};
+    int64_t bytes{kDISABLE};
+    double percent{static_cast<double>(100.0)};
+
+    bool isDisabled()
+    {
+        return bytes == kDISABLE && percent == kDISABLE;
+    }
+};
+
+class Options
+{
+public:
+    virtual ~Options() = default;
+    virtual void parse(Arguments& arguments) = 0;
+};
+
+class BaseModelOptions : public Options
+{
+public:
+    ModelFormat format{ModelFormat::kANY};
+    std::string model;
+
+    void parse(Arguments& arguments) override;
+
+    static void help(std::ostream& out);
+};
+
+class ModelOptions : public Options
+{
+public:
+    BaseModelOptions baseModel;
+    std::string prototxt;
+    std::vector<std::string> outputs;
+
+    void parse(Arguments& arguments) override;
+
+    static void help(std::ostream& out);
+};
+
+constexpr nvinfer1::TempfileControlFlags getTempfileControlDefaults()
+{
+    using F = nvinfer1::TempfileControlFlag;
+    return (1U << static_cast<uint32_t>(F::kALLOW_TEMPORARY_FILES))
+        | (1U << static_cast<uint32_t>(F::kALLOW_IN_MEMORY_FILES));
+}
+
+class BuildOptions : public Options
+{
+public:
+    // Unit in MB.
+    double workspace{-1.0};
+    // Unit in MB.
+    double dlaSRAM{-1.0};
+    // Unit in MB.
+    double dlaLocalDRAM{-1.0};
+    // Unit in MB.
+    double dlaGlobalDRAM{-1.0};
+    // Unit in KB.
+    double tacticSharedMem{-1.0};
+    int32_t avgTiming{defaultAvgTiming};
+    bool tf32{true};
+    bool stronglyTyped{true};
+    bool directIO{false};
+    LayerDeviceTypes layerDeviceTypes;
+    DecomposableAttentions decomposableAttentions;
+    StringSet debugTensors;
+    bool markUnfusedTensorsAsDebugTensors{false};
+    StringSet debugTensorStates;
+    bool safe{false};
+    bool consistency{false};
+    bool dumpKernelText{false};
+    bool buildDLAStandalone{false};
+    bool allowGPUFallback{false};
+    bool skipInference{false};
+    bool save{false};
+    bool saveAllEngines{false}; //!< Save per-iteration engines as <engine>.iter<N> during tuning
+    bool load{false};
+    bool asyncFileReader{false};
+    bool refittable{false};
+    bool stripWeights{false};
+    bool versionCompatible{false};
+    bool pluginInstanceNorm{false};
+    bool enableUInt8AsymmetricQuantizationDLA{false};
+    bool reportCapabilityDLA{false};
+    bool adjustForDLA{false};
+    bool enablePluginOverride{false};
+    bool excludeLeanRuntime{false};
+    bool disableCompilationCache{false};
+    bool enableMonitorMemory{false};
+    bool cpuOnly{false};
+    int32_t builderOptimizationLevel{defaultBuilderOptimizationLevel};
+    int32_t maxTactics{defaultMaxTactics};
+    SparsityFlag sparsity{SparsityFlag::kDISABLE};
+    nvinfer1::ProfilingVerbosity profilingVerbosity{nvinfer1::ProfilingVerbosity::kLAYER_NAMES_ONLY};
+    std::string engine;
+    using ShapeProfile = std::unordered_map<std::string, ShapeRange>;
+    std::vector<ShapeProfile> optProfiles;
+    std::vector<IOFormat> inputFormats;
+    std::vector<IOFormat> outputFormats;
+    nvinfer1::TacticSources enabledTactics{0};
+    nvinfer1::TacticSources disabledTactics{0};
+    TimingCacheMode timingCacheMode{TimingCacheMode::kLOCAL};
+    std::string timingCacheFile{};
+    bool errorOnTimingCacheMiss{false};
+    // C++11 does not automatically generate hash function for enum class.
+    // Use int32_t to support C++11 compilers.
+    std::unordered_map<int32_t, bool> previewFeatures;
+    nvinfer1::HardwareCompatibilityLevel hardwareCompatibilityLevel{nvinfer1::HardwareCompatibilityLevel::kNONE};
+    nvinfer1::RuntimePlatform runtimePlatform{nvinfer1::RuntimePlatform::kSAME_AS_BUILD};
+    std::string tempdir{};
+    nvinfer1::TempfileControlFlags tempfileControls{getTempfileControlDefaults()};
+    RuntimeMode useRuntime{RuntimeMode::kFULL};
+    std::string leanDLLPath{};
+    std::string buildRoute{}; //!< --setBuildRoute=<route> string; passed to IBuilderConfig::setBuildRoute()
+    int32_t maxAuxStreams{defaultMaxAuxStreams};
+    bool getPlanVersionOnly{false};
+
+    bool allowWeightStreaming{false};
+
+    int32_t tilingOptimizationLevel{defaultTilingOptimizationLevel};
+    int64_t l2LimitForTiling{-1};
+    bool distributiveIndependence{false};
+    std::string remoteAutoTuningConfig{};
+
+    void parse(Arguments& arguments) override;
+
+    static void help(std::ostream& out);
+};
+
+class SystemOptions : public Options
+{
+public:
+    int32_t device{defaultDevice};
+    int32_t DLACore{-1};
+    bool enableStaticPlugins{true};
+    bool ignoreParsedPluginLibs{false};
+    std::vector<std::string> plugins;
+    std::vector<std::string> setPluginsToSerialize;
+    std::vector<std::string> dynamicPlugins;
+#if ENABLE_UNIFIED_BUILDER
+    std::vector<samplesSafeCommon::SafetyPluginLibraryArgument> safetyPlugins;
+#endif
+
+    void parse(Arguments& arguments) override;
+
+    static void help(std::ostream& out, bool enableStaticPlugins = true);
+};
+
+class InferenceOptions : public Options
+{
+public:
+    int32_t batch{batchNotProvided};
+    int32_t iterations{defaultIterations};
+    int32_t infStreams{defaultStreams};
+    int32_t optProfileIndex{defaultOptProfileIndex};
+    float warmup{defaultWarmUp};
+    float duration{defaultDuration};
+    float sleep{defaultSleep};
+    float idle{defaultIdle};
+    float persistentCacheRatio{defaultPersistentCacheRatio};
+    float atol{1e-5};                  // Element-wise accuracy threshold absolute tolerance
+    float rtol{1e-5};                  // Element-wise accuracy threshold relative tolerance
+    float accuracyThresholdEndToEnd{}; // End-to-end accuracy threshold should not have default value
+                                       // because it depends on the model and accuracy validation algorithm
+    bool overlap{true};
+    bool includeTransfers{false};
+    bool useManaged{false};
+    bool spin{true};
+    bool threads{false};
+    bool graph{true};
+    bool timeDeserialize{false};
+    bool timeRefit{false};
+    bool setOptProfile{false};
+    // Reference pairs for accuracy validation with multiple test cases.
+    // Each pair contains (inputMap, refOutputMap) indexed by pair number.
+    // refPairs[0] is used as the primary pair (for --loadInputs/--loadRefOutputs without --refPair).
+    using RefPair = std::pair<std::unordered_map<std::string, std::string>,
+        std::unordered_map<std::string, std::string>>; // pair of (inputMap, refOutputMap)
+    std::vector<RefPair> refPairs{1};                  // Initialize with one empty pair
+    AccuracyValidationAlgorithm accuracyValidationAlgorithm{AccuracyValidationAlgorithm::kL0};
+    using ShapeProfile = std::unordered_map<std::string, std::vector<int64_t>>;
+    ShapeProfile shapes;
+    nvinfer1::ProfilingVerbosity nvtxVerbosity{nvinfer1::ProfilingVerbosity::kLAYER_NAMES_ONLY};
+    MemoryAllocationStrategy memoryAllocationStrategy{MemoryAllocationStrategy::kSTATIC};
+    std::unordered_map<std::string, std::string> debugTensorFileNames;
+    std::vector<std::string> dumpAlldebugTensorFormats;
+    WeightStreamingBudget weightStreamingBudget;
+    std::string refitOnnxModel;
+
+    void parse(Arguments& arguments) override;
+
+    static void help(std::ostream& out);
+};
+
+class ReportingOptions : public Options
+{
+public:
+    bool verbose{false};
+    int32_t avgs{defaultAvgRuns};
+    std::vector<float> percentiles{defaultPercentiles.begin(), defaultPercentiles.end()};
+    bool refit{false};
+    bool output{false};
+    bool dumpRawBindings{false};
+    bool profile{false};
+    bool layerInfo{false};
+    bool optProfileInfo{false};
+    std::string exportTimes;
+    std::string exportOutput;
+    std::string exportProfile;
+    std::string exportLayerInfo;
+
+    void parse(Arguments& arguments) override;
+
+    static void help(std::ostream& out);
+};
+
+class SafeBuilderOptions : public Options
+{
+public:
+    std::string serialized{};
+    std::string onnxModelFile{};
+    bool help{false};
+    bool verbose{false};
+    std::vector<IOFormat> inputFormats;
+    std::vector<IOFormat> outputFormats;
+    bool int8{false};
+    bool fp8{false};
+    bool int4{false};
+    std::vector<std::string> plugins;
+    bool consistency{false};
+    bool standard{false};
+    TimingCacheMode timingCacheMode{TimingCacheMode::kLOCAL};
+    std::string timingCacheFile{};
+    SparsityFlag sparsity{SparsityFlag::kDISABLE};
+    int32_t avgTiming{defaultAvgTiming};
+
+    void parse(Arguments& arguments) override;
+
+    static void printHelp(std::ostream& out);
+};
+
+//! \brief Options for `--tuneBuildRoutes`-driven autotuning of build routes.
+//!
+//! The tuning loop is implemented in samples/trtexec/trtexec.cpp by forking
+//! the trtexec process for each route; each child runs `runOnceBuildAndInfer`
+//! with `--setBuildRoute=<route>` injected by the parent.
+class TuningOptions : public Options
+{
+public:
+    std::string tuningCacheFile{"best_config.json"};
+    std::string tuningExpr{};                                          //!< --tuneBuildRoutes
+    std::string tuningExprFile{};                                      //!< --tuneBuildRouteFile
+    TuningSearchAlgorithm tuningSearchAlgorithm{TuningSearchAlgorithm::kFAST};
+    int64_t timeout{-1};                                               //!< --tuningTimeOut (s); -1 = no timeout
+    bool helpBuildRoute{false};                                        //!< --helpBuildRoute (short-circuit)
+    std::string helpBuildRouteKnob{};                                  //!< --helpBuildRoute=<knob> filter
+    bool continueFromCache{false};                                     //!< --continue
+    bool dryRun{false};                                                //!< --dryRun (enumerate, don't build)
+    //! \brief Hidden parent->child IPC channel.
+    //!
+    //! When set, runOnceBuildAndInfer writes a small JSON to this path containing
+    //! gpu_time_ms, accuracy_failed, and per-tensor accuracy_loss before returning.
+    //! Injected into the child's argv by the tuning loop; never shown in --help.
+    std::string tuningResultFile{};                                    //!< --tuningResultFile=<path>
+
+    void parse(Arguments& arguments) override;
+    static void help(std::ostream& out);
+};
+
+class AllOptions : public Options
+{
+public:
+    ModelOptions model;
+    BuildOptions build;
+    SystemOptions system;
+    InferenceOptions inference;
+    ReportingOptions reporting;
+    TuningOptions tuning;
+    bool helps{false};
+
+    void parse(Arguments& arguments) override;
+
+    static void help(std::ostream& out, bool enableStaticPlugins = true);
+};
+
+class TaskInferenceOptions : public Options
+{
+public:
+    std::string engine;
+    int32_t device{defaultDevice};
+    int32_t DLACore{-1};
+    int32_t batch{batchNotProvided};
+    bool graph{true};
+    float persistentCacheRatio{defaultPersistentCacheRatio};
+    void parse(Arguments& arguments) override;
+    static void help(std::ostream& out);
+};
+
+Arguments argsToArgumentsMap(int32_t argc, char* argv[]);
+
+bool parseHelp(Arguments& arguments);
+
+void helpHelp(std::ostream& out);
+
+// Functions to print options
+
+std::ostream& operator<<(std::ostream& os, BaseModelOptions const& options);
+
+std::ostream& operator<<(std::ostream& os, IOFormat const& format);
+
+std::ostream& operator<<(std::ostream& os, ShapeRange const& dims);
+
+std::ostream& operator<<(std::ostream& os, ModelOptions const& options);
+
+std::ostream& operator<<(std::ostream& os, BuildOptions const& options);
+
+std::ostream& operator<<(std::ostream& os, SystemOptions const& options);
+
+std::ostream& operator<<(std::ostream& os, InferenceOptions const& options);
+
+std::ostream& operator<<(std::ostream& os, ReportingOptions const& options);
+
+std::ostream& operator<<(std::ostream& os, AllOptions const& options);
+
+std::ostream& operator<<(std::ostream& os, SafeBuilderOptions const& options);
+
+std::ostream& operator<<(std::ostream& os, nvinfer1::DataType dtype);
+
+std::ostream& operator<<(std::ostream& os, nvinfer1::DeviceType devType);
+
+
+inline std::ostream& operator<<(std::ostream& os, nvinfer1::Dims const& dims)
+{
+    for (int32_t i = 0; i < dims.nbDims; ++i)
+    {
+        os << (i ? "x" : "") << dims.d[i];
+    }
+    return os;
+}
+inline std::ostream& operator<<(std::ostream& os, const nvinfer1::WeightsRole role)
+{
+    switch (role)
+    {
+    case nvinfer1::WeightsRole::kKERNEL:
+    {
+        os << "Kernel";
+        break;
+    }
+    case nvinfer1::WeightsRole::kBIAS:
+    {
+        os << "Bias";
+        break;
+    }
+    case nvinfer1::WeightsRole::kSHIFT:
+    {
+        os << "Shift";
+        break;
+    }
+    case nvinfer1::WeightsRole::kSCALE:
+    {
+        os << "Scale";
+        break;
+    }
+    case nvinfer1::WeightsRole::kCONSTANT:
+    {
+        os << "Constant";
+        break;
+    }
+    case nvinfer1::WeightsRole::kANY:
+    {
+        os << "Any";
+        break;
+    }
+    }
+
+    return os;
+}
+
+inline std::ostream& operator<<(std::ostream& os, std::vector<int64_t> const& vec)
+{
+    for (int32_t i = 0, e = static_cast<int32_t>(vec.size()); i < e; ++i)
+    {
+        os << (i ? "x" : "") << vec[i];
+    }
+    return os;
+}
+
+} // namespace sample
+
+#endif // TRT_SAMPLES_OPTIONS_H

samples/common/sampleOptions.test.cpp

@@ -0,0 +1,81 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "sampleOptions.h"
+#include "ArgVec.test.h"
+
+#include <gtest/gtest.h>
+
+#include <string_view>
+
+using namespace sample;
+using namespace std::string_view_literals;
+
+using TestArgVec = ArgVec<char*>;
+
+TEST(ArgsToArgumentsMap, Empty)
+{
+    TestArgVec av{};
+    auto const args = argsToArgumentsMap(av.argc(), av.argv());
+    EXPECT_TRUE(args.empty());
+}
+
+TEST(ArgsToArgumentsMap, FlagArg)
+{
+    TestArgVec av{"--verbose"};
+    auto const args = argsToArgumentsMap(av.argc(), av.argv());
+    ASSERT_EQ(args.count("--verbose"), 1U);
+    EXPECT_EQ(args.find("--verbose")->second.first, ""sv);
+}
+
+TEST(ArgsToArgumentsMap, KeyValueArg)
+{
+    TestArgVec av{"--onnx=model.onnx"};
+    auto const args = argsToArgumentsMap(av.argc(), av.argv());
+    ASSERT_EQ(args.count("--onnx"), 1U);
+    EXPECT_EQ(args.find("--onnx")->second.first, "model.onnx"sv);
+}
+
+TEST(ArgsToArgumentsMap, MultipleArgs)
+{
+    TestArgVec av{"--onnx=model.onnx", "--fp16", "--batch=4"};
+    auto const args = argsToArgumentsMap(av.argc(), av.argv());
+    ASSERT_EQ(args.count("--onnx"), 1U);
+    ASSERT_EQ(args.count("--fp16"), 1U);
+    ASSERT_EQ(args.count("--batch"), 1U);
+    EXPECT_EQ(args.find("--onnx")->second.first, "model.onnx"sv);
+    EXPECT_EQ(args.find("--fp16")->second.first, ""sv);
+    EXPECT_EQ(args.find("--batch")->second.first, "4"sv);
+}
+
+TEST(ArgsToArgumentsMap, ValueWithEquals)
+{
+    // Values can themselves contain '='; only the first '=' is the key/value separator.
+    TestArgVec av{"--key=a=b"};
+    auto const args = argsToArgumentsMap(av.argc(), av.argv());
+    ASSERT_EQ(args.count("--key"), 1U);
+    EXPECT_EQ(args.find("--key")->second.first, "a=b"sv);
+}
+
+TEST(ArgsToArgumentsMap, ArgPositionRecorded)
+{
+    // argsToArgumentsMap records the original argv index (1-based, skipping argv[0]).
+    TestArgVec av{"--onnx=model.onnx", "--fp16"};
+    auto const args = argsToArgumentsMap(av.argc(), av.argv());
+    EXPECT_EQ(args.find("--onnx")->second.second, 1);
+    EXPECT_EQ(args.find("--fp16")->second.second, 2);
+}

samples/common/sampleReporting.cpp

@@ -0,0 +1,691 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <algorithm>
+#include <exception>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <numeric>
+#include <utility>
+
+#include "sampleInference.h"
+#include "sampleOptions.h"
+#include "sampleReporting.h"
+
+#if ENABLE_UNIFIED_BUILDER
+#include "NvInferSafeRuntime.h"
+#include "bfloat16.h"
+#if CUDA_VERSION >= 11060
+#include <cuda_fp8.h>
+#endif
+#endif
+
+using namespace nvinfer1;
+
+namespace sample
+{
+
+namespace
+{
+
+//!
+//! \brief Find percentile in an ascending sequence of timings
+//! \note percentile must be in [0, 100]. Otherwise, an exception is thrown.
+//!
+template <typename T>
+float findPercentile(float percentile, std::vector<InferenceTime> const& timings, T const& toFloat)
+{
+    int32_t const all = static_cast<int32_t>(timings.size());
+    int32_t const exclude = static_cast<int32_t>((1 - percentile / 100) * all);
+    if (timings.empty())
+    {
+        return std::numeric_limits<float>::infinity();
+    }
+    if (percentile < 0.F || percentile > 100.F)
+    {
+        throw std::runtime_error("percentile is not in [0, 100]!");
+    }
+    return toFloat(timings[std::max(all - 1 - exclude, 0)]);
+}
+
+//!
+//! \brief Find median in a sorted sequence of timings
+//!
+template <typename T>
+float findMedian(std::vector<InferenceTime> const& timings, T const& toFloat)
+{
+    if (timings.empty())
+    {
+        return std::numeric_limits<float>::infinity();
+    }
+
+    int32_t const m = timings.size() / 2;
+    if (timings.size() % 2)
+    {
+        return toFloat(timings[m]);
+    }
+
+    return (toFloat(timings[m - 1]) + toFloat(timings[m])) / 2;
+}
+
+//!
+//! \brief Find coefficient of variance (which is std / mean) in a sorted sequence of timings given the mean
+//!
+template <typename T>
+float findCoeffOfVariance(std::vector<InferenceTime> const& timings, T const& toFloat, float mean)
+{
+    if (timings.empty())
+    {
+        return 0;
+    }
+
+    if (mean == 0.F)
+    {
+        return std::numeric_limits<float>::infinity();
+    }
+
+    auto const metricAccumulator = [toFloat, mean](float acc, InferenceTime const& a) {
+        float const diff = toFloat(a) - mean;
+        return acc + diff * diff;
+    };
+    float const variance = std::accumulate(timings.begin(), timings.end(), 0.F, metricAccumulator) / timings.size();
+
+    return std::sqrt(variance) / mean * 100.F;
+}
+
+inline InferenceTime traceToTiming(const InferenceTrace& a)
+{
+    return InferenceTime(
+        (a.enqEnd - a.enqStart), (a.h2dEnd - a.h2dStart), (a.computeEnd - a.computeStart), (a.d2hEnd - a.d2hStart));
+}
+
+inline std::string dimsToString(Dims const& shape)
+{
+    std::stringstream ss;
+
+    if (shape.nbDims == 0)
+    {
+        ss << "scalar";
+    }
+    else
+    {
+        for (int32_t i = 0; i < shape.nbDims; i++)
+        {
+            ss << shape.d[i] << (i != shape.nbDims - 1 ? "x" : "");
+        }
+    }
+    return ss.str();
+}
+
+} // namespace
+
+void printProlog(int32_t warmups, int32_t timings, float warmupMs, float benchTimeMs, std::ostream& os)
+{
+    os << "Warmup completed " << warmups << " queries over " << warmupMs << " ms" << std::endl;
+    os << "Timing trace has " << timings << " queries over " << benchTimeMs / 1000 << " s" << std::endl;
+}
+
+void printTiming(std::vector<InferenceTime> const& timings, int32_t runsPerAvg, std::ostream& os)
+{
+    int64_t count = 0;
+    InferenceTime sum;
+
+    os << std::endl;
+    os << "=== Trace details ===" << std::endl;
+    os << "Trace averages of " << runsPerAvg << " runs:" << std::endl;
+
+    // Show only the first N lines and the last N lines, where N = kTIMING_PRINT_THRESHOLD.
+    constexpr int64_t kTIMING_PRINT_THRESHOLD{200};
+    int64_t const maxNbTimings{kTIMING_PRINT_THRESHOLD * runsPerAvg};
+
+    for (int64_t idx = 0, size = timings.size(); idx < size; ++idx)
+    {
+        // Omit some latency printing to avoid very long logs.
+        if (size > 2 * maxNbTimings && idx == maxNbTimings)
+        {
+            os << "... Omitting " << (size - 2 * maxNbTimings) << " lines" << std::endl;
+            idx = size - kTIMING_PRINT_THRESHOLD * runsPerAvg - 1;
+        }
+
+        sum += timings[idx];
+
+        if (++count == runsPerAvg)
+        {
+            // clang-format off
+            os << "Average on " << runsPerAvg << " runs - GPU latency: " << sum.compute / runsPerAvg
+               << " ms - Host latency: " << sum.latency() / runsPerAvg << " ms (enqueue " << sum.enq / runsPerAvg
+               << " ms)" << std::endl;
+            // clang-format on
+            count = 0;
+            sum.enq = 0;
+            sum.h2d = 0;
+            sum.compute = 0;
+            sum.d2h = 0;
+        }
+    }
+}
+
+void printMetricExplanations(std::ostream& os)
+{
+    os << std::endl;
+    os << "=== Explanations of the performance metrics ===" << std::endl;
+    os << "Total Host Walltime: the host walltime from when the first query (after warmups) is enqueued to when the "
+          "last query is completed."
+       << std::endl;
+    os << "GPU Compute Time: the GPU latency to execute the kernels for a query." << std::endl;
+    os << "Total GPU Compute Time: the summation of the GPU Compute Time of all the queries. If this is significantly "
+          "shorter than Total Host Walltime, the GPU may be under-utilized because of host-side overheads or data "
+          "transfers."
+       << std::endl;
+    os << "Throughput: the observed throughput computed by dividing the number of queries by the Total Host Walltime. "
+          "If this is significantly lower than the reciprocal of GPU Compute Time, the GPU may be under-utilized "
+          "because of host-side overheads or data transfers."
+       << std::endl;
+    os << "Enqueue Time: the host latency to enqueue a query. If this is longer than GPU Compute Time, the GPU may be "
+          "under-utilized."
+       << std::endl;
+    os << "H2D Latency: the latency for host-to-device data transfers for input tensors of a single query."
+       << std::endl;
+    os << "D2H Latency: the latency for device-to-host data transfers for output tensors of a single query."
+       << std::endl;
+    os << "Latency: the summation of H2D Latency, GPU Compute Time, and D2H Latency. This is the latency to infer a "
+          "single query."
+       << std::endl;
+}
+
+PerformanceResult getPerformanceResult(std::vector<InferenceTime> const& timings,
+    std::function<float(InferenceTime const&)> metricGetter, std::vector<float> const& percentiles)
+{
+    auto const metricComparator
+        = [metricGetter](InferenceTime const& a, InferenceTime const& b) { return metricGetter(a) < metricGetter(b); };
+    auto const metricAccumulator = [metricGetter](float acc, InferenceTime const& a) { return acc + metricGetter(a); };
+    std::vector<InferenceTime> newTimings = timings;
+    std::sort(newTimings.begin(), newTimings.end(), metricComparator);
+    PerformanceResult result;
+    result.min = metricGetter(newTimings.front());
+    result.max = metricGetter(newTimings.back());
+    result.mean = std::accumulate(newTimings.begin(), newTimings.end(), 0.0F, metricAccumulator) / newTimings.size();
+    result.median = findMedian(newTimings, metricGetter);
+    for (auto percentile : percentiles)
+    {
+        result.percentiles.emplace_back(findPercentile(percentile, newTimings, metricGetter));
+    }
+    result.coeffVar = findCoeffOfVariance(newTimings, metricGetter, result.mean);
+    return result;
+}
+
+void printEpilog(std::vector<InferenceTime> const& timings, float walltimeMs, std::vector<float> const& percentiles,
+    int32_t batchSize, int32_t infStreams, std::ostream& osInfo, std::ostream& osWarning, std::ostream& osVerbose)
+{
+    float const throughput = batchSize * timings.size() / walltimeMs * 1000;
+
+    auto const getLatency = [](InferenceTime const& t) { return t.latency(); };
+    auto const latencyResult = getPerformanceResult(timings, getLatency, percentiles);
+
+    auto const getEnqueue = [](InferenceTime const& t) { return t.enq; };
+    auto const enqueueResult = getPerformanceResult(timings, getEnqueue, percentiles);
+
+    auto const getH2d = [](InferenceTime const& t) { return t.h2d; };
+    auto const h2dResult = getPerformanceResult(timings, getH2d, percentiles);
+
+    auto const getCompute = [](InferenceTime const& t) { return t.compute; };
+    auto const gpuComputeResult = getPerformanceResult(timings, getCompute, percentiles);
+
+    auto const getD2h = [](InferenceTime const& t) { return t.d2h; };
+    auto const d2hResult = getPerformanceResult(timings, getD2h, percentiles);
+
+    auto const toPerfString = [&](const PerformanceResult& r) {
+        std::stringstream s;
+        s << "min = " << r.min << " ms, max = " << r.max << " ms, mean = " << r.mean << " ms, "
+          << "median = " << r.median << " ms";
+        for (int32_t i = 0, n = percentiles.size(); i < n; ++i)
+        {
+            s << ", percentile(" << percentiles[i] << "%) = " << r.percentiles[i] << " ms";
+        }
+        return s.str();
+    };
+
+    osInfo << std::endl;
+    osInfo << "=== Performance summary ===" << std::endl;
+    osInfo << "Throughput: " << throughput << " qps" << std::endl;
+    osInfo << "Latency: " << toPerfString(latencyResult) << std::endl;
+    osInfo << "Enqueue Time: " << toPerfString(enqueueResult) << std::endl;
+    osInfo << "H2D Latency: " << toPerfString(h2dResult) << std::endl;
+    osInfo << "GPU Compute Time: " << toPerfString(gpuComputeResult) << std::endl;
+    osInfo << "D2H Latency: " << toPerfString(d2hResult) << std::endl;
+    osInfo << "Total Host Walltime: " << walltimeMs / 1000 << " s" << std::endl;
+    osInfo << "Total GPU Compute Time: " << gpuComputeResult.mean * timings.size() / 1000 << " s" << std::endl;
+
+    // Report warnings if the throughput is bound by other factors than GPU Compute Time.
+    constexpr float kENQUEUE_BOUND_REPORTING_THRESHOLD{0.8F};
+    if (enqueueResult.median > kENQUEUE_BOUND_REPORTING_THRESHOLD * gpuComputeResult.median)
+    {
+        osWarning
+            << "* Throughput may be bound by Enqueue Time rather than GPU Compute and the GPU may be under-utilized."
+            << std::endl;
+        osWarning << "  If you are using --noCudaGraph, removing it may increase throughput." << std::endl;
+    }
+    if (h2dResult.median >= gpuComputeResult.median)
+    {
+        osWarning << "* Throughput may be bound by host-to-device transfers for the inputs rather than GPU Compute and "
+                     "the GPU may be under-utilized."
+                  << std::endl;
+        osWarning << "  Consider removing --includeDataTransfers to disable data transfers and potentially increase "
+                     "throughput."
+                  << std::endl;
+    }
+    if (d2hResult.median >= gpuComputeResult.median)
+    {
+        osWarning << "* Throughput may be bound by device-to-host transfers for the outputs rather than GPU Compute "
+                     "and the GPU may be under-utilized."
+                  << std::endl;
+        osWarning << "  Consider removing --includeDataTransfers to disable data transfers and potentially increase "
+                     "throughput."
+                  << std::endl;
+    }
+
+    // Report warnings if the GPU Compute Time is unstable.
+    constexpr float kUNSTABLE_PERF_REPORTING_THRESHOLD{2.5F};
+    if (gpuComputeResult.coeffVar > kUNSTABLE_PERF_REPORTING_THRESHOLD)
+    {
+        osWarning << "* GPU compute time is unstable, with coefficient of variance = " << gpuComputeResult.coeffVar
+                  << "%." << std::endl;
+        osWarning << "  If not already in use, locking GPU clock frequency may improve the stability." << std::endl;
+    }
+
+    // Report warnings if multiple inference streams are used.
+    if (infStreams > 1)
+    {
+        osWarning << "* Multiple inference streams are used. Latencies may not be accurate since inferences may run in "
+                  << "  parallel. Please use \"Throughput\" as the performance metric instead." << std::endl;
+    }
+
+    // Explain what the metrics mean.
+    osInfo << "Explanations of the performance metrics are printed in the verbose logs." << std::endl;
+    printMetricExplanations(osVerbose);
+
+    osInfo << std::endl;
+}
+
+void printPerformanceReport(std::vector<InferenceTrace> const& trace, ReportingOptions const& reportingOpts,
+    InferenceOptions const& infOpts, std::ostream& osInfo, std::ostream& osWarning, std::ostream& osVerbose)
+{
+    int32_t batchSize = infOpts.batch;
+    float const warmupMs = infOpts.warmup;
+    auto const isNotWarmup = [&warmupMs](const InferenceTrace& a) { return a.computeStart >= warmupMs; };
+    auto const noWarmup = std::find_if(trace.begin(), trace.end(), isNotWarmup);
+    int32_t const warmups = noWarmup - trace.begin();
+    float const benchTime = trace.back().d2hEnd - noWarmup->h2dStart;
+    // treat inference with explicit batch as a single query and report the throughput
+    batchSize = batchSize ? batchSize : 1;
+    printProlog(warmups * batchSize, (trace.size() - warmups) * batchSize, warmupMs, benchTime, osInfo);
+
+    std::vector<InferenceTime> timings(trace.size() - warmups);
+    std::transform(noWarmup, trace.end(), timings.begin(), traceToTiming);
+    printTiming(timings, reportingOpts.avgs, osInfo);
+    printEpilog(
+        timings, benchTime, reportingOpts.percentiles, batchSize, infOpts.infStreams, osInfo, osWarning, osVerbose);
+
+    if (!reportingOpts.exportTimes.empty())
+    {
+        exportJSONTrace(trace, reportingOpts.exportTimes, warmups);
+    }
+}
+
+//! Printed format:
+//! [ value, ...]
+//! value ::= { "start enq : time, "end enq" : time, "start h2d" : time, "end h2d" : time, "start compute" : time,
+//!             "end compute" : time, "start d2h" : time, "end d2h" : time, "h2d" : time, "compute" : time,
+//!             "d2h" : time, "latency" : time }
+//!
+void exportJSONTrace(std::vector<InferenceTrace> const& trace, std::string const& fileName, int32_t const nbWarmups)
+{
+    std::ofstream os(fileName, std::ofstream::trunc);
+    os << "[" << std::endl;
+    char const* sep = "  ";
+    for (auto iter = trace.begin() + nbWarmups; iter < trace.end(); ++iter)
+    {
+        auto const& t = *iter;
+        InferenceTime const it(traceToTiming(t));
+        os << sep << "{ ";
+        sep = ", ";
+        // clang-format off
+        os << "\"startEnqMs\" : "     << t.enqStart     << sep << "\"endEnqMs\" : "     << t.enqEnd     << sep
+           << "\"startH2dMs\" : "     << t.h2dStart     << sep << "\"endH2dMs\" : "     << t.h2dEnd     << sep
+           << "\"startComputeMs\" : " << t.computeStart << sep << "\"endComputeMs\" : " << t.computeEnd << sep
+           << "\"startD2hMs\" : "     << t.d2hStart     << sep << "\"endD2hMs\" : "     << t.d2hEnd     << sep
+           << "\"h2dMs\" : "          << it.h2d         << sep << "\"computeMs\" : "    << it.compute   << sep
+           << "\"d2hMs\" : "          << it.d2h         << sep << "\"latencyMs\" : "    << it.latency() << " }"
+           << std::endl;
+        // clang-format on
+    }
+    os << "]" << std::endl;
+}
+
+void Profiler::reportLayerTime(char const* layerName, float timeMs) noexcept
+{
+    if (mIterator == mLayers.end())
+    {
+        bool const first = !mLayers.empty() && mLayers.begin()->name == layerName;
+        mUpdatesCount += mLayers.empty() || first;
+        if (first)
+        {
+            mIterator = mLayers.begin();
+        }
+        else
+        {
+            mLayers.emplace_back();
+            mLayers.back().name = layerName;
+            mIterator = mLayers.end() - 1;
+        }
+    }
+
+    mIterator->timeMs.push_back(timeMs);
+    ++mIterator;
+}
+
+void Profiler::print(std::ostream& os) const noexcept
+{
+    std::string const nameHdr("   Layer");
+    std::string const timeHdr("   Time(ms)");
+    std::string const avgHdr("     Avg.(ms)");
+    std::string const medHdr("   Median(ms)");
+    std::string const percentageHdr("   Time(%)");
+
+    float const totalTimeMs = getTotalTime();
+
+    auto const timeLength = timeHdr.size();
+    auto const avgLength = avgHdr.size();
+    auto const medLength = medHdr.size();
+    auto const percentageLength = percentageHdr.size();
+
+    os << std::endl
+       << "=== Profile (" << mUpdatesCount << " iterations ) ===" << std::endl
+       << timeHdr << avgHdr << medHdr << percentageHdr << nameHdr << std::endl;
+
+    for (auto const& p : mLayers)
+    {
+        if (p.timeMs.empty() || getTotalTime(p) == 0.F)
+        {
+            // there is no point to print profiling for layer that didn't run at all
+            continue;
+        }
+        // clang-format off
+        os << std::setw(timeLength) << std::fixed << std::setprecision(2) << getTotalTime(p)
+           << std::setw(avgLength) << std::fixed << std::setprecision(4) << getAvgTime(p)
+           << std::setw(medLength) << std::fixed << std::setprecision(4) << getMedianTime(p)
+           << std::setw(percentageLength) << std::fixed << std::setprecision(1) << getTotalTime(p) / totalTimeMs * 100
+           << "   " << p.name << std::endl;
+    }
+    {
+        os << std::setw(timeLength) << std::fixed << std::setprecision(2)
+           << totalTimeMs << std::setw(avgLength) << std::fixed << std::setprecision(4) << totalTimeMs / mUpdatesCount
+           << std::setw(medLength) << std::fixed << std::setprecision(4) << getMedianTime()
+           << std::setw(percentageLength) << std::fixed << std::setprecision(1) << 100.0
+           << "   Total" << std::endl;
+        // clang-format on
+    }
+    os << std::endl;
+}
+
+void Profiler::exportJSONProfile(std::string const& fileName) const noexcept
+{
+    std::ofstream os(fileName, std::ofstream::trunc);
+    os << "[" << std::endl << "  { \"count\" : " << mUpdatesCount << " }" << std::endl;
+
+    auto const totalTimeMs = getTotalTime();
+
+    for (auto const& l : mLayers)
+    {
+        // clang-format off
+        os << ", {" << R"( "name" : ")"      << l.name << R"(")"
+                       R"(, "timeMs" : )"     << getTotalTime(l)
+           <<          R"(, "averageMs" : )"  << getAvgTime(l)
+           <<          R"(, "medianMs" : )"  << getMedianTime(l)
+           <<          R"(, "percentage" : )" << getTotalTime(l) / totalTimeMs * 100
+           << " }"  << std::endl;
+        // clang-format on
+    }
+    os << "]" << std::endl;
+}
+
+void dumpInputs(nvinfer1::IExecutionContext const& context, BindingsStd const& bindings, std::ostream& os)
+{
+    os << "Input Tensors:" << std::endl;
+    bindings.dumpInputs(context, os);
+}
+
+void dumpOutputs(nvinfer1::IExecutionContext const& context, BindingsStd const& bindings, std::ostream& os)
+{
+    bindings.dumpOutputs(context, os);
+}
+
+void dumpRawBindingsToFiles(nvinfer1::IExecutionContext const& context, BindingsStd const& bindings, std::ostream& os)
+{
+    bindings.dumpRawBindingToFiles(context, os);
+}
+
+void exportJSONOutput(
+    nvinfer1::IExecutionContext const& context, BindingsStd const& bindings, std::string const& fileName, int32_t batch)
+{
+    std::ofstream os(fileName, std::ofstream::trunc);
+    std::string sep = "  ";
+    auto const output = bindings.getOutputBindings();
+    os << "[" << std::endl;
+    for (auto const& binding : output)
+    {
+        // clang-format off
+        os << sep << R"({ "name" : ")" << binding.first << "\"" << std::endl;
+        sep = ", ";
+        os << "  " << sep << R"("dimensions" : ")";
+        bindings.dumpBindingDimensions(binding.first, context, os);
+        os << "\"" << std::endl;
+        os << "  " << sep << "\"values\" : [ ";
+        bindings.dumpBindingValues(context, binding.second, os, sep, batch);
+        os << " ]" << std::endl << "  }" << std::endl;
+        // clang-format on
+    }
+    os << "]" << std::endl;
+}
+
+void exportJSONOutput(nvinfer1::IExecutionContext const& context, BindingsStd const& bindings,
+    std::string const& fileName, int32_t batch);
+
+#if ENABLE_UNIFIED_BUILDER
+void dumpSafeOutputs(nvinfer2::safe::ITRTGraph const& graph, BindingsSafe const& bindings, std::ostream& os)
+{
+    bindings.dumpOutputs(graph, os);
+}
+
+void dumpSafeRawBindingsToFiles(nvinfer2::safe::ITRTGraph const& graph, BindingsSafe const& bindings, std::ostream& os)
+{
+    bindings.dumpRawBindingToFiles(const_cast<nvinfer2::safe::ITRTGraph&>(graph), os);
+}
+
+void exportSafeJSONOutput(
+    nvinfer2::safe::ITRTGraph const& graph, BindingsSafe const& bindings, std::string const& fileName, int32_t batch)
+{
+    std::ofstream os(fileName, std::ofstream::trunc);
+    std::string sep = "  ";
+    auto const output = bindings.getOutputBindings();
+    os << "[" << std::endl;
+    for (auto const& binding : output)
+    {
+        // clang-format off
+        os << sep << R"({ "name" : ")" << binding.first << "\"" << std::endl;
+        sep = ", ";
+        os << "  " << sep << R"("dimensions" : ")";
+        bindings.dumpBindingDimensions(binding.first, graph, os);
+        os << "\"" << std::endl;
+        os << "  " << sep << "\"values\" : [ ";
+        bindings.dumpBindingValues(graph, binding.second, os, sep, batch);
+        os << " ]" << std::endl << "  }" << std::endl;
+        // clang-format on
+    }
+    os << "]" << std::endl;
+}
+
+void exportSafeJSONOutput(
+    nvinfer2::safe::ITRTGraph const& graph, BindingsSafe const& bindings, std::string const& fileName, int32_t batch);
+#endif
+
+void printLayerInfo(
+    ReportingOptions const& reporting, nvinfer1::ICudaEngine* engine, nvinfer1::IExecutionContext* context)
+{
+    if (reporting.layerInfo)
+    {
+        sample::gLogInfo << "Layer Information:" << std::endl;
+        sample::gLogInfo << getLayerInformation(engine, context, nvinfer1::LayerInformationFormat::kONELINE)
+                         << std::flush;
+    }
+    if (!reporting.exportLayerInfo.empty())
+    {
+        std::ofstream os(reporting.exportLayerInfo, std::ofstream::trunc);
+        os << getLayerInformation(engine, context, nvinfer1::LayerInformationFormat::kJSON) << std::flush;
+    }
+}
+
+void printOptimizationProfileInfo(ReportingOptions const& reporting, nvinfer1::ICudaEngine const* engine)
+{
+    if (reporting.optProfileInfo)
+    {
+        sample::gLogInfo << "Optimization Profile Information:" << std::endl;
+        for (int32_t i = 0; i < engine->getNbOptimizationProfiles(); i++)
+        {
+            for (int32_t j = 0, e = engine->getNbIOTensors(); j < e; j++)
+            {
+                auto const tensorName = engine->getIOTensorName(j);
+
+                if (engine->getTensorIOMode(tensorName) == nvinfer1::TensorIOMode::kINPUT)
+                {
+                    auto tensorMinShape = engine->getProfileShape(tensorName, i, nvinfer1::OptProfileSelector::kMIN);
+                    auto tensorOptShape = engine->getProfileShape(tensorName, i, nvinfer1::OptProfileSelector::kOPT);
+                    auto tensorMaxShape = engine->getProfileShape(tensorName, i, nvinfer1::OptProfileSelector::kMAX);
+
+                    sample::gLogInfo << "Model input " << tensorName << " (profile " << i << "): "
+                                     << "min=" << dimsToString(tensorMinShape)
+                                     << ", opt=" << dimsToString(tensorOptShape)
+                                     << ", max=" << dimsToString(tensorMaxShape) << std::endl;
+                }
+            }
+        }
+    }
+}
+
+void printPerformanceProfile(ReportingOptions const& reporting, InferenceEnvironmentBase& iEnv)
+{
+    if (reporting.profile)
+    {
+        iEnv.profiler->print(sample::gLogInfo);
+    }
+    if (!reporting.exportProfile.empty())
+    {
+        iEnv.profiler->exportJSONProfile(reporting.exportProfile);
+    }
+
+    // Print an warning about total per-layer latency when auxiliary streams are used.
+    if (!iEnv.safe && (reporting.profile || !reporting.exportProfile.empty()))
+    {
+        int32_t const nbAuxStreams = iEnv.engine->getNbAuxStreams();
+        if (nbAuxStreams > 0)
+        {
+            sample::gLogWarning << "The engine uses " << nbAuxStreams << " auxiliary streams, so the \"Total\" latency "
+                                << "may not be accurate because some layers may have run in parallel!" << std::endl;
+        }
+    }
+}
+
+namespace details
+{
+void dump(std::unique_ptr<nvinfer1::IExecutionContext> const& context, std::unique_ptr<BindingsStd> const& binding,
+    ReportingOptions const& reporting, int32_t batch)
+{
+    if (!context)
+    {
+        sample::gLogError << "Empty context! Skip printing outputs." << std::endl;
+        return;
+    }
+    if (reporting.output)
+    {
+        dumpOutputs(*context, *binding, sample::gLogInfo);
+    }
+    if (reporting.dumpRawBindings)
+    {
+        dumpRawBindingsToFiles(*context, *binding, sample::gLogInfo);
+    }
+    if (!reporting.exportOutput.empty())
+    {
+        exportJSONOutput(*context, *binding, reporting.exportOutput, batch);
+    }
+}
+
+#if ENABLE_UNIFIED_BUILDER
+void safeDump(std::unique_ptr<nvinfer2::safe::ITRTGraph> const& graph, std::unique_ptr<BindingsSafe> const& binding,
+    ReportingOptions const& reporting, int32_t batch)
+{
+    if (!graph)
+    {
+        sample::gLogError << "Empty safe graph! Skip printing outputs." << std::endl;
+        return;
+    }
+    if (reporting.output)
+    {
+        dumpSafeOutputs(*graph, *binding, sample::gLogInfo);
+    }
+    if (reporting.dumpRawBindings)
+    {
+        dumpSafeRawBindingsToFiles(*graph, *binding, sample::gLogInfo);
+    }
+    if (!reporting.exportOutput.empty())
+    {
+        exportSafeJSONOutput(*graph, *binding, reporting.exportOutput, batch);
+    }
+}
+#endif
+
+} // namespace details
+
+void printOutput(ReportingOptions const& reporting, InferenceEnvironmentBase const& iEnv, int32_t batch)
+{
+    if (iEnv.safe)
+    {
+#if ENABLE_UNIFIED_BUILDER
+        auto const& binding = static_cast<const InferenceEnvironmentSafe&>(iEnv).bindings.at(0);
+        if (!binding)
+        {
+            sample::gLogError << "Empty bindings! Skip printing outputs." << std::endl;
+            return;
+        }
+        auto const& graph = static_cast<const InferenceEnvironmentSafe&>(iEnv).mClonedGraphs.at(0);
+        details::safeDump(graph, binding, reporting, batch);
+#else
+        sample::gLogWarning << "Safe mode is not supported! Skip printing outputs." << std::endl;
+#endif
+        return;
+    }
+    auto const& binding = static_cast<const InferenceEnvironmentStd&>(iEnv).bindings.at(0);
+    if (!binding)
+    {
+        sample::gLogError << "Empty bindings! Skip printing outputs." << std::endl;
+        return;
+    }
+    auto const& context = static_cast<const InferenceEnvironmentStd&>(iEnv).contexts.at(0);
+    details::dump(context, binding, reporting, batch);
+}
+
+} // namespace sample

samples/common/sampleReporting.h

@@ -0,0 +1,298 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_SAMPLE_REPORTING_H
+#define TRT_SAMPLE_REPORTING_H
+
+#include <functional>
+#include <iostream>
+#include <numeric>
+
+#include "sampleOptions.h"
+
+namespace sample
+{
+
+class BindingsStd;
+
+//!
+//! \struct InferenceTime
+//! \brief Measurement times in milliseconds
+//!
+struct InferenceTime
+{
+    InferenceTime(float q, float i, float c, float o)
+        : enq(q)
+        , h2d(i)
+        , compute(c)
+        , d2h(o)
+    {
+    }
+
+    InferenceTime() = default;
+    InferenceTime(InferenceTime const&) = default;
+    InferenceTime(InferenceTime&&) = default;
+    InferenceTime& operator=(InferenceTime const&) = default;
+    InferenceTime& operator=(InferenceTime&&) = default;
+    ~InferenceTime() = default;
+
+    float enq{0};     // Enqueue
+    float h2d{0};     // Host to Device
+    float compute{0}; // Compute
+    float d2h{0};     // Device to Host
+
+    // ideal latency
+    float latency() const
+    {
+        return h2d + compute + d2h;
+    }
+};
+
+//!
+//! \struct InferenceTrace
+//! \brief Measurement points in milliseconds
+//!
+struct InferenceTrace
+{
+    InferenceTrace(int32_t s, float es, float ee, float is, float ie, float cs, float ce, float os, float oe)
+        : stream(s)
+        , enqStart(es)
+        , enqEnd(ee)
+        , h2dStart(is)
+        , h2dEnd(ie)
+        , computeStart(cs)
+        , computeEnd(ce)
+        , d2hStart(os)
+        , d2hEnd(oe)
+    {
+    }
+
+    InferenceTrace() = default;
+    InferenceTrace(InferenceTrace const&) = default;
+    InferenceTrace(InferenceTrace&&) = default;
+    InferenceTrace& operator=(InferenceTrace const&) = default;
+    InferenceTrace& operator=(InferenceTrace&&) = default;
+    ~InferenceTrace() = default;
+
+    int32_t stream{0};
+    float enqStart{0};
+    float enqEnd{0};
+    float h2dStart{0};
+    float h2dEnd{0};
+    float computeStart{0};
+    float computeEnd{0};
+    float d2hStart{0};
+    float d2hEnd{0};
+};
+
+inline InferenceTime operator+(InferenceTime const& a, InferenceTime const& b)
+{
+    return InferenceTime(a.enq + b.enq, a.h2d + b.h2d, a.compute + b.compute, a.d2h + b.d2h);
+}
+
+inline InferenceTime operator+=(InferenceTime& a, InferenceTime const& b)
+{
+    return a = a + b;
+}
+
+//!
+//! \struct PerformanceResult
+//! \brief Performance result of a performance metric
+//!
+struct PerformanceResult
+{
+    float min{0.F};
+    float max{0.F};
+    float mean{0.F};
+    float median{0.F};
+    std::vector<float> percentiles;
+    float coeffVar{0.F}; // coefficient of variation
+};
+
+//!
+//! \brief Print benchmarking time and number of traces collected
+//!
+void printProlog(int32_t warmups, int32_t timings, float warmupMs, float walltime, std::ostream& os);
+
+//!
+//! \brief Print a timing trace
+//!
+void printTiming(std::vector<InferenceTime> const& timings, int32_t runsPerAvg, std::ostream& os);
+
+//!
+//! \brief Print the performance summary of a trace
+//!
+void printEpilog(std::vector<InferenceTime> const& timings, std::vector<float> const& percentiles, int32_t batchSize,
+    std::ostream& osInfo, std::ostream& osWarning, std::ostream& osVerbose);
+
+//!
+//! \brief Get the result of a specific performance metric from a trace
+//!
+PerformanceResult getPerformanceResult(std::vector<InferenceTime> const& timings,
+    std::function<float(InferenceTime const&)> metricGetter, std::vector<float> const& percentiles);
+
+//!
+//! \brief Print the explanations of the performance metrics printed in printEpilog() function.
+//!
+void printMetricExplanations(std::ostream& os);
+
+//!
+//! \brief Print and summarize a timing trace
+//!
+void printPerformanceReport(std::vector<InferenceTrace> const& trace, ReportingOptions const& reportingOpts,
+    InferenceOptions const& infOpts, std::ostream& osInfo, std::ostream& osWarning, std::ostream& osVerbose);
+
+//!
+//! \brief Export a timing trace to JSON file
+//!
+void exportJSONTrace(
+    std::vector<InferenceTrace> const& InferenceTime, std::string const& fileName, int32_t const nbWarmups);
+
+//!
+//! \brief Print input tensors to stream
+//!
+void dumpInputs(nvinfer1::IExecutionContext const& context, BindingsStd const& bindings, std::ostream& os);
+
+//!
+//! \brief Print output tensors to stream
+//!
+void dumpOutputs(nvinfer1::IExecutionContext const& context, BindingsStd const& bindings, std::ostream& os);
+
+void dumpRawBindingsToFiles(nvinfer1::IExecutionContext const& context, BindingsStd const& bindings, std::ostream& os);
+
+//!
+//! \brief Export output tensors to JSON file
+//!
+void exportJSONOutput(nvinfer1::IExecutionContext const& context, BindingsStd const& bindings,
+    std::string const& fileName, int32_t batch);
+
+//!
+//! \struct LayerProfile
+//! \brief Layer profile information
+//!
+struct LayerProfile
+{
+    std::string name;
+    std::vector<float> timeMs;
+};
+
+//!
+//! \class Profiler
+//! \brief Collect per-layer profile information, assuming times are reported in the same order
+//!
+class Profiler : public nvinfer1::IProfiler
+{
+
+public:
+    void reportLayerTime(char const* layerName, float timeMs) noexcept override;
+
+    void print(std::ostream& os) const noexcept;
+
+    //!
+    //! \brief Export a profile to JSON file
+    //!
+    void exportJSONProfile(std::string const& fileName) const noexcept;
+
+private:
+    float getTotalTime() const noexcept
+    {
+        auto const plusLayerTime = [](float accumulator, LayerProfile const& lp) {
+            return accumulator + std::accumulate(lp.timeMs.begin(), lp.timeMs.end(), 0.F, std::plus<float>());
+        };
+        return std::accumulate(mLayers.begin(), mLayers.end(), 0.0F, plusLayerTime);
+    }
+
+    float getMedianTime() const noexcept
+    {
+        if (mLayers.empty())
+        {
+            return 0.F;
+        }
+        std::vector<float> totalTime;
+        for (size_t run = 0; run < mLayers[0].timeMs.size(); ++run)
+        {
+            auto const layerTime
+                = [&run](float accumulator, LayerProfile const& lp) { return accumulator + lp.timeMs[run]; };
+            auto t = std::accumulate(mLayers.begin(), mLayers.end(), 0.F, layerTime);
+            totalTime.push_back(t);
+        }
+        return median(totalTime);
+    }
+
+    float getMedianTime(LayerProfile const& p) const noexcept
+    {
+        return median(p.timeMs);
+    }
+
+    static float median(std::vector<float> vals)
+    {
+        if (vals.empty())
+        {
+            return 0.F;
+        }
+        std::sort(vals.begin(), vals.end());
+        if (vals.size() % 2U == 1U)
+        {
+            return vals[vals.size() / 2U];
+        }
+        return (vals[vals.size() / 2U - 1U] + vals[vals.size() / 2U]) * 0.5F;
+    }
+
+    //! return the total runtime of given layer profile
+    float getTotalTime(LayerProfile const& p) const noexcept
+    {
+        auto const& vals = p.timeMs;
+        return std::accumulate(vals.begin(), vals.end(), 0.F, std::plus<float>());
+    }
+
+    float getAvgTime(LayerProfile const& p) const noexcept
+    {
+        return getTotalTime(p) / p.timeMs.size();
+    }
+
+    std::vector<LayerProfile> mLayers;
+    std::vector<LayerProfile>::iterator mIterator{mLayers.begin()};
+    int32_t mUpdatesCount{0};
+};
+
+//!
+//! \brief Print layer info to logger or export it to output JSON file.
+//!
+void printLayerInfo(
+    ReportingOptions const& reporting, nvinfer1::ICudaEngine* engine, nvinfer1::IExecutionContext* context);
+
+//!
+//! \brief Print optimization profile info to logger.
+//!
+void printOptimizationProfileInfo(ReportingOptions const& reporting, nvinfer1::ICudaEngine const* engine);
+
+//! Forward declaration.
+struct InferenceEnvironmentBase;
+
+//!
+//! \brief Print per-layer perf profile data to logger or export it to output JSON file.
+//!
+void printPerformanceProfile(ReportingOptions const& reporting, InferenceEnvironmentBase& iEnv);
+
+//!
+//! \brief Print binding output values to logger or export them to output JSON file.
+//!
+void printOutput(ReportingOptions const& reporting, InferenceEnvironmentBase const& iEnv, int32_t batch);
+
+} // namespace sample
+
+#endif // TRT_SAMPLE_REPORTING_H

samples/common/sampleTuning.cpp

@@ -0,0 +1,718 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "sampleTuning.h"
+#include "common.h"
+#include "nlohmann/json.hpp"
+
+#include <algorithm>
+#include <cstring>
+#include <set>
+#include <sstream>
+#include <unordered_set>
+
+namespace sample
+{
+
+std::vector<std::string> splitPipeDelimited(std::string const& str)
+{
+    std::vector<std::string> result;
+    std::stringstream ss(str);
+    std::string token;
+    while (std::getline(ss, token, '|'))
+    {
+        uint64_t const start = token.find_first_not_of(" \t");
+        uint64_t const end = token.find_last_not_of(" \t");
+        if (start != std::string::npos && end != std::string::npos)
+        {
+            result.push_back(token.substr(start, end - start + 1));
+        }
+    }
+    return result;
+}
+
+// ============================================================================
+// BuildRouteKnobDatabase Implementation
+// ============================================================================
+
+bool BuildRouteKnobDatabase::loadFromJsonString(std::string const& jsonStr)
+{
+    mKnobs.clear();
+    mKnobOrder.clear();
+    mTunerVersion = "unknown";
+
+    if (jsonStr.empty())
+    {
+        return false;
+    }
+
+    try
+    {
+        // Parse JSON using nlohmann/json
+        nlohmann::json const root = nlohmann::json::parse(jsonStr);
+
+        // Extract tuner version from the top-level JSON object.
+        if (root.contains("tuner_version") && root["tuner_version"].is_string())
+        {
+            mTunerVersion = root["tuner_version"].get<std::string>();
+        }
+
+        // Check for "tuner_options" array
+        if (!root.contains("tuner_options") || !root["tuner_options"].is_array())
+        {
+            return false;
+        }
+
+        // Iterate over tuner_options array
+        for (auto const& item : root["tuner_options"])
+        {
+            if (!item.is_object())
+            {
+                continue;
+            }
+
+            BuildRouteKnobDef knob;
+
+            // Extract fields from JSON object
+            if (item.contains("option") && item["option"].is_string())
+            {
+                knob.mOption = item["option"].get<std::string>();
+            }
+            if (item.contains("allowed_values") && item["allowed_values"].is_string())
+            {
+                knob.mAllowedValues = item["allowed_values"].get<std::string>();
+            }
+            if (item.contains("default_value") && item["default_value"].is_string())
+            {
+                knob.mDefaultValue = item["default_value"].get<std::string>();
+            }
+            if (item.contains("help") && item["help"].is_string())
+            {
+                knob.mHelp = item["help"].get<std::string>();
+            }
+
+            // Parse allowed values and add to database
+            if (!knob.mOption.empty())
+            {
+                knob.mValues = parseAllowedValues(knob.mAllowedValues);
+                knob.mIsBounded = !knob.mValues.empty();
+                mKnobOrder.push_back(knob.mOption);
+                mKnobs[knob.mOption] = std::move(knob);
+            }
+        }
+    }
+    catch (nlohmann::json::exception const&)
+    {
+        // JSON parsing failed
+        return false;
+    }
+
+    return !mKnobs.empty();
+}
+
+std::string BuildRouteKnobDatabase::buildDefaultPath() const
+{
+    // Build a space-separated string of "knob=default_value" pairs,
+    // using the insertion order preserved from the original JSON.
+    std::string result;
+    for (auto const& name : mKnobOrder)
+    {
+        auto it = mKnobs.find(name);
+        if (it == mKnobs.end())
+        {
+            continue;
+        }
+        if (!result.empty())
+        {
+            result += " ";
+        }
+        result += it->second.mOption + "=" + it->second.mDefaultValue;
+    }
+    return result;
+}
+
+bool BuildRouteKnobDatabase::hasKnob(std::string const& knobName) const
+{
+    return mKnobs.find(knobName) != mKnobs.end();
+}
+
+BuildRouteKnobDef const* BuildRouteKnobDatabase::getKnob(std::string const& knobName) const
+{
+    auto const it = mKnobs.find(knobName);
+    return it != mKnobs.end() ? &it->second : nullptr;
+}
+
+bool BuildRouteKnobDatabase::validateValues(std::string const& knobName, std::vector<std::string> const& values) const
+{
+    BuildRouteKnobDef const* knob = getKnob(knobName);
+    if (knob == nullptr)
+    {
+        return false;
+    }
+    if (!knob->mIsBounded)
+    {
+        return false;
+    }
+
+    std::set<std::string> const allowed(knob->mValues.begin(), knob->mValues.end());
+    return std::ranges::all_of(values, [&allowed](auto const& v) { return allowed.contains(v); });
+}
+
+bool BuildRouteKnobDatabase::isBounded(std::string const& knobName) const
+{
+    BuildRouteKnobDef const* knob = getKnob(knobName);
+    return knob != nullptr && knob->mIsBounded;
+}
+
+std::string BuildRouteKnobDatabase::getDefaultValue(std::string const& knobName) const
+{
+    BuildRouteKnobDef const* knob = getKnob(knobName);
+    return knob != nullptr ? knob->mDefaultValue : std::string();
+}
+
+std::vector<std::string> BuildRouteKnobDatabase::parseAllowedValues(std::string const& allowedStr)
+{
+    // Find the bracket pattern: -knob=[val1|val2|val3]
+    uint64_t const bracketStart = allowedStr.find('[');
+    uint64_t const bracketEnd = allowedStr.find(']');
+
+    if (bracketStart == std::string::npos || bracketEnd == std::string::npos || bracketEnd <= bracketStart)
+    {
+        return {}; // No brackets: unbounded or unknown format (e.g., "=int32_t")
+    }
+
+    std::string const valuesStr = allowedStr.substr(bracketStart + 1, bracketEnd - bracketStart - 1);
+
+    // Range patterns like "..." indicate unbounded
+    if (valuesStr.find("...") != std::string::npos)
+    {
+        return {};
+    }
+
+    return splitPipeDelimited(valuesStr);
+}
+
+// ============================================================================
+// BuildRouteExprParser Implementation
+// ============================================================================
+
+BuildRouteExprParser::BuildRouteExprParser(BuildRouteKnobDatabase const& db)
+    : mDb(db)
+{
+}
+
+std::string const& BuildRouteExprParser::getError() const noexcept
+{
+    return mError;
+}
+
+std::optional<std::vector<BuildRouteParsedExpr>> BuildRouteExprParser::parse(std::string const& input) const
+{
+    mError.clear();
+
+    if (input.empty())
+    {
+        mError = "Empty input";
+        return std::nullopt;
+    }
+
+    std::vector<std::string> const tokens = tokenize(input);
+    if (tokens.empty())
+    {
+        mError = "No expressions found";
+        return std::nullopt;
+    }
+
+    std::vector<BuildRouteParsedExpr> result;
+    result.reserve(tokens.size());
+
+    for (auto const& token : tokens)
+    {
+        auto expr = parseExpr(token);
+        if (!expr)
+        {
+            return std::nullopt;
+        }
+        result.push_back(std::move(*expr));
+    }
+
+    return result;
+}
+
+std::vector<std::string> BuildRouteExprParser::tokenize(std::string const& input) const
+{
+    // Split input by spaces, but keep bracketed content together
+    // E.g., "-opt1=[a|b] -opt2=c" -> ["-opt1=[a|b]", "-opt2=c"]
+
+    std::vector<std::string> tokens;
+    std::string current;
+    int32_t bracketDepth = 0;
+
+    for (char const c : input)
+    {
+        if (c == '[')
+        {
+            ++bracketDepth;
+            current += c;
+        }
+        else if (c == ']')
+        {
+            --bracketDepth;
+            current += c;
+        }
+        else if (c == ' ' && bracketDepth == 0)
+        {
+            // Split point - space outside brackets
+            if (!current.empty())
+            {
+                tokens.push_back(std::move(current));
+                current.clear();
+            }
+        }
+        else
+        {
+            current += c;
+        }
+    }
+
+    // Add final token if any
+    if (!current.empty())
+    {
+        tokens.push_back(std::move(current));
+    }
+
+    return tokens;
+}
+
+// NOLINTNEXTLINE(readability-function-cognitive-complexity)
+std::optional<BuildRouteParsedExpr> BuildRouteExprParser::parseExpr(std::string const& expr) const
+{
+    BuildRouteParsedExpr result;
+
+    // Find the '=' separator
+    uint64_t const eqPos = expr.find('=');
+    if (eqPos == std::string::npos)
+    {
+        mError = "Invalid expression (no '='): " + expr;
+        return std::nullopt;
+    }
+
+    // Extract knob name (before '=')
+    result.mKnobName = expr.substr(0, eqPos);
+
+    // Trim whitespace from knob name
+    uint64_t start = result.mKnobName.find_first_not_of(" \t");
+    uint64_t end = result.mKnobName.find_last_not_of(" \t");
+    if (start != std::string::npos && end != std::string::npos)
+    {
+        result.mKnobName = result.mKnobName.substr(start, end - start + 1);
+    }
+
+    // Validate knob exists in database
+    if (!mDb.hasKnob(result.mKnobName))
+    {
+        mError = "Unknown knob: " + result.mKnobName;
+        return std::nullopt;
+    }
+
+    // Extract value part (after '=')
+    std::string valueStr = expr.substr(eqPos + 1);
+
+    // Check if this is a bracketed value list or a fixed value
+    uint64_t const bracketStart = valueStr.find('[');
+    uint64_t const bracketEnd = valueStr.find(']');
+
+    if (bracketStart != std::string::npos && bracketEnd != std::string::npos && bracketEnd > bracketStart)
+    {
+        // Bracketed value list: -knob=[val1|val2|val3]
+
+        // Validate: nothing should appear before the opening bracket
+        std::string const beforeBracket = valueStr.substr(0, bracketStart);
+        uint64_t nonSpace = beforeBracket.find_first_not_of(" \t");
+        if (nonSpace != std::string::npos)
+        {
+            mError = "Invalid expression format (unexpected content before '['): " + expr;
+            return std::nullopt;
+        }
+
+        // Validate: nothing should appear after the closing bracket
+        std::string const afterBracket = valueStr.substr(bracketEnd + 1);
+        nonSpace = afterBracket.find_first_not_of(" \t");
+        if (nonSpace != std::string::npos)
+        {
+            mError = "Invalid expression format (unexpected content after ']'): " + expr;
+            return std::nullopt;
+        }
+
+        // Extract and parse values inside brackets
+        std::string const valuesInBrackets = valueStr.substr(bracketStart + 1, bracketEnd - bracketStart - 1);
+
+        // Check for unbounded patterns
+        if (valuesInBrackets.find("...") != std::string::npos)
+        {
+            mError = "Unbounded expression not allowed: " + expr;
+            return std::nullopt;
+        }
+
+        // Split by '|' and trim each value
+        result.mValues = splitPipeDelimited(valuesInBrackets);
+
+        if (result.mValues.empty())
+        {
+            mError = "Empty value list in expression: " + expr;
+            return std::nullopt;
+        }
+
+        // Check for duplicates
+        std::unordered_set<std::string> seenValues;
+        for (auto const& val : result.mValues)
+        {
+            auto [iter, inserted] = seenValues.insert(val);
+            if (!inserted)
+            {
+                mError = "Duplicate value '" + val + "' in expression: " + expr;
+                return std::nullopt;
+            }
+        }
+
+        // Validate values against the knob's allowed values
+        if (!mDb.validateValues(result.mKnobName, result.mValues))
+        {
+            if (!mDb.isBounded(result.mKnobName))
+            {
+                mError = "Knob has unbounded values (int32_t): " + result.mKnobName;
+                return std::nullopt;
+            }
+            mError = "Invalid value(s) for knob: " + result.mKnobName;
+            return std::nullopt;
+        }
+
+        result.mIsFixed = false;
+    }
+    else
+    {
+        // Fixed value: -knob=value
+
+        // Trim whitespace from value
+        start = valueStr.find_first_not_of(" \t");
+        end = valueStr.find_last_not_of(" \t");
+        if (start != std::string::npos && end != std::string::npos)
+        {
+            valueStr = valueStr.substr(start, end - start + 1);
+        }
+
+        // Check for unbounded type keyword
+        if (valueStr == "int32_t")
+        {
+            mError = "Unbounded expression 'int32_t' not allowed: " + expr;
+            return std::nullopt;
+        }
+
+        result.mValues.push_back(valueStr);
+        result.mIsFixed = true;
+
+        // Validate fixed value if knob is bounded
+        if (mDb.isBounded(result.mKnobName))
+        {
+            if (!mDb.validateValues(result.mKnobName, result.mValues))
+            {
+                mError = "Invalid value for knob: " + result.mKnobName + "=" + valueStr;
+                return std::nullopt;
+            }
+        }
+    }
+
+    return result;
+}
+
+// ============================================================================
+// TuningContext Implementation
+// ============================================================================
+
+BigInt TuningContext::count() const
+{
+    if (parsedExprs.empty())
+    {
+        return BigInt(0);
+    }
+
+    switch (searchAlgorithm)
+    {
+    case TuningSearchAlgorithm::kEXHAUSTIVE:
+    {
+        // Product of all value list sizes
+        BigInt total(1);
+        for (auto const& expr : parsedExprs)
+        {
+            total = total * BigInt(expr.mValues.size());
+        }
+        return total;
+    }
+    case TuningSearchAlgorithm::kFAST:
+    case TuningSearchAlgorithm::kMIXED:
+    {
+        // 1 (baseline) + sum of non-default values per variable knob.
+        // For mixed mode, this returns the phase 1 (fast scan) count only.
+        // Phase 2 count is determined dynamically after phase 1 completes.
+        ASSERT(parsedExprs.size() == defaultValues.size());
+        BigInt total(1);
+        for (uint64_t i = 0; i < parsedExprs.size(); ++i)
+        {
+            if (parsedExprs[i].mIsFixed)
+            {
+                continue;
+            }
+            for (auto const& val : parsedExprs[i].mValues)
+            {
+                if (val != defaultValues[i])
+                {
+                    ++total;
+                }
+            }
+        }
+        return total;
+    }
+    default:
+    {
+        throw std::invalid_argument("Unsupported tuning search algorithm");
+    }
+    }
+}
+
+std::string TuningContext::getPathAtIndex(BigInt const& index) const
+{
+    // Helper lambda: build a space-separated knob=value string from per-knob value selections.
+    // Spaces are required because the Myelin compiler parses each knob as a separate option.
+    // Used by both exhaustive and fast modes.
+    auto buildString = [this](auto const& getValueAtPosition) -> std::string {
+        std::string result;
+        for (uint64_t j = 0; j < parsedExprs.size(); ++j)
+        {
+            if (j > 0)
+            {
+                result += " ";
+            }
+            result += parsedExprs[j].mKnobName + "=" + getValueAtPosition(j);
+        }
+        return result;
+    };
+
+    switch (searchAlgorithm)
+    {
+    case TuningSearchAlgorithm::kEXHAUSTIVE:
+    {
+        // Reverse mixed-radix decomposition: decompose index into per-knob value indices.
+        // Work from right to left (least significant to most significant).
+        BigInt const total = count();
+        if (index >= total)
+        {
+            throw std::out_of_range("Index " + index.toString() + " is out of range [0, " + total.toString() + ")");
+        }
+
+        std::vector<uint64_t> valueIndices(parsedExprs.size());
+        BigInt current = index;
+        for (int64_t i = static_cast<int64_t>(parsedExprs.size()) - 1; i >= 0; --i)
+        {
+            BigInt const base(parsedExprs[i].mValues.size());
+            valueIndices[i] = (current % base).toUint64();
+            current = current / base;
+        }
+
+        return buildString([&](uint64_t j) -> std::string const& { return parsedExprs[j].mValues[valueIndices[j]]; });
+    }
+    case TuningSearchAlgorithm::kFAST:
+    case TuningSearchAlgorithm::kMIXED:
+    {
+        // Fast/mixed mode: index 0 is the baseline (all defaults), index 1..N are one-off variations
+        // iterating from last knob to first, skipping default values.
+        // For mixed mode, this handles phase 1 only. Phase 2 uses a separate TuningContext.
+        ASSERT(parsedExprs.size() == defaultValues.size());
+
+        auto baselineValue = [this](uint64_t j) -> std::string const& {
+            return parsedExprs[j].mIsFixed ? parsedExprs[j].mValues[0] : defaultValues[j];
+        };
+
+        // Index 0: pure baseline
+        if (index == BigInt(0))
+        {
+            return buildString(baselineValue);
+        }
+
+        // Index > 0: find which knob is varied and to what value
+        auto knob = identifyVariedKnob(*this, index);
+        if (!knob)
+        {
+            throw std::out_of_range("Index " + index.toString() + " is out of range for fast expansion");
+        }
+        return buildString([&](uint64_t j) -> std::string const& {
+            return static_cast<int64_t>(j) == knob->first ? knob->second : baselineValue(j);
+        });
+    }
+    default:
+    {
+        throw std::invalid_argument("Unsupported tuning search algorithm");
+    }
+    }
+}
+
+// ============================================================================
+// Mixed Search Phase 2 Context Builder
+// ============================================================================
+
+TuningContext buildMixedPhase2Context(
+    TuningContext const& phase1Context, std::vector<MixedSearchKnobResult> const& positiveKnobs)
+{
+    // Build a set of knob indices that are "positive" (showed improvement in phase 1).
+    // For these knobs, we keep their full value lists. All other knobs become fixed to baseline.
+    std::set<int32_t> positiveIndices;
+    for (auto const& knob : positiveKnobs)
+    {
+        positiveIndices.insert(knob.knobIndex);
+    }
+
+    TuningContext phase2;
+    phase2.searchAlgorithm = TuningSearchAlgorithm::kEXHAUSTIVE;
+    phase2.tunerVersion = phase1Context.tunerVersion;
+    phase2.defaultBuildRoute = phase1Context.defaultBuildRoute;
+
+    for (uint64_t i = 0; i < phase1Context.parsedExprs.size(); ++i)
+    {
+        BuildRouteParsedExpr expr;
+        expr.mKnobName = phase1Context.parsedExprs[i].mKnobName;
+
+        if (positiveIndices.count(static_cast<int32_t>(i)) > 0 && !phase1Context.parsedExprs[i].mIsFixed)
+        {
+            // Positive knob: keep full value list for exhaustive expansion.
+            expr.mValues = phase1Context.parsedExprs[i].mValues;
+            expr.mIsFixed = false;
+        }
+        else
+        {
+            // Non-positive or fixed knob: lock to baseline default value.
+            expr.mValues = {phase1Context.parsedExprs[i].mIsFixed ? phase1Context.parsedExprs[i].mValues[0]
+                                                                  : phase1Context.defaultValues[i]};
+            expr.mIsFixed = true;
+        }
+
+        phase2.parsedExprs.push_back(std::move(expr));
+        phase2.defaultValues.push_back(phase1Context.defaultValues[i]);
+    }
+
+    phase2.totalCount = phase2.count();
+    return phase2;
+}
+
+void collectPositiveKnobFromResult(bool crashed, double gpuTimeMs, double baselineGpuTimeMs, BigInt const& index,
+    TuningContext const& ctx, std::vector<MixedSearchKnobResult>& positiveKnobs)
+{
+    if (!crashed && gpuTimeMs > 0.0 && gpuTimeMs < baselineGpuTimeMs)
+    {
+        auto knob = identifyVariedKnob(ctx, index);
+        if (knob)
+        {
+            positiveKnobs.push_back({knob->first, knob->second, gpuTimeMs});
+        }
+    }
+}
+
+std::optional<std::pair<int32_t, std::string>> identifyVariedKnob(TuningContext const& ctx, BigInt const& index)
+{
+    // In fast/mixed mode, index 0 is baseline. Indices 1..N are one-off variations
+    // iterating knobs right-to-left, skipping default values. This reverses that mapping.
+    if (index == BigInt(0))
+    {
+        return std::nullopt; // Baseline, no knob varied
+    }
+
+    BigInt remaining = index;
+    --remaining;
+    for (int64_t i = static_cast<int64_t>(ctx.parsedExprs.size()) - 1; i >= 0; --i)
+    {
+        if (ctx.parsedExprs[i].mIsFixed)
+        {
+            continue;
+        }
+        for (auto const& val : ctx.parsedExprs[i].mValues)
+        {
+            if (val == ctx.defaultValues[i])
+            {
+                continue;
+            }
+            if (remaining == BigInt(0))
+            {
+                return std::make_pair(static_cast<int32_t>(i), val);
+            }
+            --remaining;
+        }
+    }
+    return std::nullopt; // Index out of range
+}
+
+bool isTuningOnlyArg(char const* arg)
+{
+    // Tuning-only flags that the parent interprets and that must not appear on the child argv.
+    // The child runs a plain single-route trtexec build, so the parent strips these and
+    // re-injects the canonical child trio (--setBuildRoute, --saveEngine, --tuningResultFile).
+    static constexpr char const* kTUNING_STRIP_PREFIXES[] = {
+        "--tuneBuildRoutes",
+        "--tuneBuildRouteFile",
+        "--tuningSearch",
+        "--tuningCacheFile",
+        "--tuningTimeOut",
+        "--saveAllEngines",
+        "--continue",
+        "--dryRun",
+        // The parent will inject its own --setBuildRoute, --saveEngine, --tuningResultFile.
+        "--setBuildRoute",
+        "--saveEngine",
+        "--tuningResultFile",
+    };
+    return std::any_of(std::begin(kTUNING_STRIP_PREFIXES), std::end(kTUNING_STRIP_PREFIXES), [arg](char const* prefix) {
+        auto const len = std::strlen(prefix);
+        return std::strncmp(arg, prefix, len) == 0 && (arg[len] == '\0' || arg[len] == '=');
+    });
+}
+
+std::vector<char*> buildTuningChildArgv(int32_t argc, char** argv, std::string const& route,
+    std::string const& enginePath, std::string const& resultJsonPath, std::vector<std::string>& storage)
+{
+    storage.clear();
+    storage.reserve(argc + 3);
+    // Always include argv[0] (the trtexec executable path) verbatim.
+    storage.emplace_back(argv[0]);
+    for (int32_t i = 1; i < argc; ++i)
+    {
+        if (argv[i] != nullptr && !isTuningOnlyArg(argv[i]))
+        {
+            storage.emplace_back(argv[i]);
+        }
+    }
+    storage.emplace_back("--setBuildRoute=" + route);
+    storage.emplace_back("--saveEngine=" + enginePath);
+    storage.emplace_back("--tuningResultFile=" + resultJsonPath);
+
+    std::vector<char*> out;
+    out.reserve(storage.size() + 1);
+    for (auto& s : storage)
+    {
+        out.emplace_back(s.data());
+    }
+    out.emplace_back(nullptr);
+    return out;
+}
+
+} // namespace sample

samples/common/sampleTuning.h

@@ -0,0 +1,356 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_SAMPLE_TUNING_H
+#define TRT_SAMPLE_TUNING_H
+
+#include "bigInt.h"
+#include "logger.h"
+#include "sampleOptions.h"
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace sample
+{
+
+//! \brief Split a pipe-delimited string into trimmed tokens.
+//!
+//! Given a string like "val1|val2|val3", returns {"val1", "val2", "val3"}.
+//! Each token is trimmed of leading/trailing whitespace. Empty tokens
+//! (after trimming) are skipped.
+//!
+//! \param[in] str The pipe-delimited string to split.
+//! \return Vector of trimmed token strings.
+std::vector<std::string> splitPipeDelimited(std::string const& str);
+
+// ============================================================================
+// Build Route Expression Parser and Expander
+// ============================================================================
+//
+// These classes implement parsing and expansion of build route expressions
+// for the --tuneBuildRoutes feature in trtexec. The expression syntax supports:
+//   - Variable values: "-knob=[val1|val2|val3]"
+//   - Fixed values: "-knob=fixed_value"
+//
+// Two expansion modes are supported:
+//   - Full (exhaustive): All combinations enumerated (exponential in number of knobs)
+//   - Fast (one-off): Baseline (all defaults) + one-knob-different variations (linear)
+//
+// Example:
+//   Expression: "-opt1=[on|off] -opt2=[0|1|2]"
+//   Full expansion (6 combinations):
+//     [0]: -opt1=on -opt2=0
+//     [1]: -opt1=on -opt2=1
+//     [2]: -opt1=on -opt2=2
+//     [3]: -opt1=off -opt2=0
+//     [4]: -opt1=off -opt2=1
+//     [5]: -opt1=off -opt2=2
+//   Fast expansion (with defaults opt1=off, opt2=2):
+//     [0]: -opt1=off -opt2=2  (baseline)
+//     [1]: -opt1=off -opt2=0  (opt2 changed)
+//     [2]: -opt1=off -opt2=1  (opt2 changed)
+//     [3]: -opt1=on -opt2=2   (opt1 changed)
+// ============================================================================
+
+//! \struct BuildRouteKnobDef
+//! \brief Represents a single knob/option definition from the JSON configuration.
+struct BuildRouteKnobDef
+{
+    std::string mOption;              //!< Option name, e.g., "-kgen:tiling"
+    std::string mAllowedValues;       //!< Allowed values string, e.g., "-kgen:tiling=[0|1|2]"
+    std::string mDefaultValue;        //!< Default value
+    std::string mHelp;                //!< Help text
+    std::vector<std::string> mValues; //!< Parsed allowed values (empty if unbounded)
+    bool mIsBounded{false};           //!< True if values form a closed set
+};
+
+//! \struct BuildRouteParsedExpr
+//! \brief Represents a parsed expression from user input.
+struct BuildRouteParsedExpr
+{
+    std::string mKnobName;            //!< Knob name, e.g., "-kgen:tiling"
+    std::vector<std::string> mValues; //!< Values to expand, e.g., ["0", "1", "2"]
+    bool mIsFixed{false};             //!< True if this is a fixed value (no expansion)
+};
+
+//! \class BuildRouteKnobDatabase
+//! \brief Database of knob definitions loaded from getAllBuildRoutes() JSON output.
+//!
+//! This class parses the JSON output from IBuilderConfig::getAllBuildRoutes()
+//! and provides lookup and validation functions for knob definitions.
+class BuildRouteKnobDatabase
+{
+public:
+    //! \brief Default constructor.
+    BuildRouteKnobDatabase() = default;
+
+    //! \brief Destructor.
+    ~BuildRouteKnobDatabase() = default;
+
+    // Non-copyable, movable
+    BuildRouteKnobDatabase(BuildRouteKnobDatabase const&) = delete;
+    BuildRouteKnobDatabase& operator=(BuildRouteKnobDatabase const&) = delete;
+    BuildRouteKnobDatabase(BuildRouteKnobDatabase&&) = default;
+    BuildRouteKnobDatabase& operator=(BuildRouteKnobDatabase&&) = default;
+
+    //! \brief Load knob definitions from a JSON string.
+    //! \param[in] jsonStr JSON string from getAllBuildRoutes().
+    //! \return True if loading succeeded, false otherwise.
+    bool loadFromJsonString(std::string const& jsonStr);
+
+    //! \brief Check if a knob exists in the database.
+    //! \param[in] knobName The knob name to check.
+    //! \return True if the knob exists.
+    bool hasKnob(std::string const& knobName) const;
+
+    //! \brief Get the knob definition for a given knob name.
+    //! \param[in] knobName The knob name to look up.
+    //! \return Pointer to the KnobDef, or nullptr if not found.
+    BuildRouteKnobDef const* getKnob(std::string const& knobName) const;
+
+    //! \brief Validate that all values are allowed for a knob.
+    //! \param[in] knobName The knob name.
+    //! \param[in] values The values to validate.
+    //! \return True if all values are valid.
+    bool validateValues(std::string const& knobName, std::vector<std::string> const& values) const;
+
+    //! \brief Check if a knob has bounded (finite) values.
+    //! \param[in] knobName The knob name.
+    //! \return True if the knob has bounded values.
+    bool isBounded(std::string const& knobName) const;
+
+    //! \brief Get the default value for a knob.
+    //! \param[in] knobName The knob name.
+    //! \return The default value string, or empty string if not found.
+    std::string getDefaultValue(std::string const& knobName) const;
+
+    //! \brief Get the tuner version string extracted from the JSON.
+    //! \return Tuner version (e.g. "2.17.83"), or "unknown" if not available.
+    std::string const& getTunerVersion() const
+    {
+        return mTunerVersion;
+    }
+
+    //! \brief Build the default build route string from all knob defaults.
+    //! Returns a space-separated string of "knob=default_value" pairs,
+    //! preserving the insertion order from the original JSON.
+    //! Example: "-conv_use_long_w=on -reshape_ppg=on -transpose_ppg=on ..."
+    std::string buildDefaultPath() const;
+
+private:
+    //! \brief Parse allowed values from an allowed_values string.
+    //! \param[in] allowedStr The allowed_values string from JSON.
+    //! \return Vector of parsed values (empty if unbounded).
+    static std::vector<std::string> parseAllowedValues(std::string const& allowedStr);
+
+    std::unordered_map<std::string, BuildRouteKnobDef> mKnobs; //!< Map of knob names to definitions.
+    std::vector<std::string> mKnobOrder;                       //!< Insertion order of knob names.
+    std::string mTunerVersion{"unknown"};                      //!< Tuner version from JSON.
+};
+
+//! \class BuildRouteExprParser
+//! \brief Parser for build route expression strings.
+//!
+//! Parses expressions like "-opt1=[a|b] -opt2=fixed -opt3=[0|1|2]"
+//! into a list of BuildRouteParsedExpr structures for expansion.
+class BuildRouteExprParser
+{
+public:
+    //! \brief Construct a parser with a reference to a knob database.
+    //! \param[in] db Reference to the BuildRouteKnobDatabase for validation.
+    explicit BuildRouteExprParser(BuildRouteKnobDatabase const& db);
+
+    //! \brief Destructor.
+    ~BuildRouteExprParser() = default;
+
+    // Non-copyable, non-movable (holds reference)
+    BuildRouteExprParser(BuildRouteExprParser const&) = delete;
+    BuildRouteExprParser& operator=(BuildRouteExprParser const&) = delete;
+    BuildRouteExprParser(BuildRouteExprParser&&) = delete;
+    BuildRouteExprParser& operator=(BuildRouteExprParser&&) = delete;
+
+    //! \brief Parse an input string into a list of expressions.
+    //! \param[in] input The input string containing expressions.
+    //! \return Optional vector of BuildRouteParsedExpr, or nullopt on error.
+    std::optional<std::vector<BuildRouteParsedExpr>> parse(std::string const& input) const;
+
+    //! \brief Get the last error message.
+    //! \return The error message string.
+    std::string const& getError() const noexcept;
+
+private:
+    //! \brief Parse a single expression.
+    //! \param[in] expr The expression string.
+    //! \return Optional BuildRouteParsedExpr, or nullopt on error.
+    std::optional<BuildRouteParsedExpr> parseExpr(std::string const& expr) const;
+
+    //! \brief Tokenize input into individual expressions.
+    //! \param[in] input The input string.
+    //! \return Vector of token strings.
+    std::vector<std::string> tokenize(std::string const& input) const;
+
+    BuildRouteKnobDatabase const& mDb; //!< Reference to the knob database.
+    mutable std::string mError;        //!< Last error message.
+};
+
+//! \struct TuningContext
+//! \brief Context for build route tuning when --tuneBuildRoutes is specified.
+//!
+//! Uses lazy expansion: build route strings are computed on-demand from
+//! the parsed expressions and an index, without storing all strings in memory.
+//! This supports arbitrarily large expansion spaces (full mode with BigInt count).
+struct TuningContext
+{
+    //! Parsed expressions from --tuneBuildRoutes.
+    std::vector<BuildRouteParsedExpr> parsedExprs;
+
+    //! Default values for each knob (from knob database, needed for fast mode).
+    std::vector<std::string> defaultValues;
+
+    //! Total number of configurations.
+    BigInt totalCount;
+
+    //! Search algorithm (determines expansion strategy).
+    TuningSearchAlgorithm searchAlgorithm{TuningSearchAlgorithm::kFAST};
+
+    //! Tuner version string from the knob database JSON (e.g. "2.17.83").
+    std::string tunerVersion{"unknown"};
+
+    //! Default build route string (all knobs at their default values).
+    std::string defaultBuildRoute;
+
+    //! \brief Compute total configuration count based on searchAlgorithm.
+    //! - Exhaustive: product of all value list sizes.
+    //! - Fast: 1 (baseline) + sum of non-default values per variable knob.
+    BigInt count() const;
+
+    //! \brief Get the build route string for a given index (lazy).
+    //! Throws std::out_of_range if index is out of range.
+    //! - Exhaustive: reverse mixed-radix decomposition.
+    //! - Fast: baseline at index 0, one-off variations at index 1..N.
+    std::string getPathAtIndex(BigInt const& index) const;
+};
+
+// ============================================================================
+// Mixed Search Algorithm Support
+// ============================================================================
+//
+// Mixed search runs in two phases:
+//   Phase 1 (fast scan): baseline + one-off variations (same as --tuningSearch=fast).
+//     After each one-off iteration, if it passed accuracy, didn't crash, and was
+//     faster than the baseline, the knob is recorded as a "positive" knob.
+//   Phase 2 (full combinatorial): if >1 positive knobs found, run 2^n combinations
+//     of the top-N positive knobs. Other knobs stay at baseline defaults.
+//
+// Total iterations: fast_count + (2^n if n > 1, else 0).
+
+//! Maximum number of positive knobs to carry into phase 2 of mixed search.
+constexpr int32_t kMixedSearchMaxPositiveKnobs = 10;
+
+//! \struct MixedSearchKnobResult
+//! \brief Records a single knob variation that improved performance in phase 1.
+//! Used to select which knobs enter phase 2 of mixed search.
+struct MixedSearchKnobResult
+{
+    int32_t knobIndex; //!< Index into TuningContext::parsedExprs
+    std::string value; //!< The non-default value that was tested
+    double gpuTimeMs;  //!< GPU time achieved (lower is better)
+};
+
+//! \brief Build a TuningContext for phase 2 of mixed search.
+//!
+//! Constructs a new exhaustive-mode TuningContext where only the positive knobs
+//! are variable (with their full value lists from phase 1) and all other knobs
+//! are fixed to their baseline default values.
+//!
+//! \param phase1Context The original TuningContext from phase 1 (fast mode).
+//! \param positiveKnobs The knobs that showed improvement in phase 1.
+//! \return A new TuningContext configured for exhaustive search over positive knobs.
+TuningContext buildMixedPhase2Context(
+    TuningContext const& phase1Context, std::vector<MixedSearchKnobResult> const& positiveKnobs);
+
+//! \brief Identify which knob was varied at a given fast-mode iteration index.
+//!
+//! In fast mode, index 0 is the baseline (all defaults). Indices 1..N are one-off
+//! variations, iterating knobs right-to-left and skipping default values.
+//! This function reverses that mapping: given a one-off index (must be > 0),
+//! returns the (knobIndex, value) pair, or nullopt if the index is out of range.
+//!
+//! Example: with 2 binary knobs (defaults "on", "on"), values [on|off] each:
+//!   index 1 → knob 1 changed to "off"  → returns {1, "off"}
+//!   index 2 → knob 0 changed to "off"  → returns {0, "off"}
+//!
+//! \param ctx   The TuningContext (must be fast or mixed mode).
+//! \param index The one-off iteration index (must be > 0).
+//! \return (knobIndex, value) pair, or nullopt if index is invalid.
+std::optional<std::pair<int32_t, std::string>> identifyVariedKnob(TuningContext const& ctx, BigInt const& index);
+
+//! \brief Collect a positive knob from an iteration result (used in mixed search).
+//!
+//! If the iteration did not crash, achieved positive GPU time, and was faster than the
+//! baseline, identifies which knob was varied (using identifyVariedKnob) and appends
+//! it to the positiveKnobs vector. Called from both the live tuning loop (phase 1) and
+//! from the --continue cache reconstruction path.
+//!
+//! Example: if iteration 3 changed knob 1 to "off" and achieved 1.2ms vs baseline 1.5ms,
+//! this appends {knobIndex=1, value="off", gpuTimeMs=1.2} to positiveKnobs.
+//!
+//! \param[in] crashed          Whether the iteration crashed.
+//! \param[in] gpuTimeMs        GPU time achieved by this iteration.
+//! \param[in] baselineGpuTimeMs Baseline GPU time (index 0).
+//! \param[in] index            The iteration index (must be > 0 for one-off variations).
+//! \param[in] ctx              The TuningContext (fast/mixed mode).
+//! \param[in,out] positiveKnobs Vector to append the positive knob result to.
+void collectPositiveKnobFromResult(bool crashed, double gpuTimeMs, double baselineGpuTimeMs, BigInt const& index,
+    TuningContext const& ctx, std::vector<MixedSearchKnobResult>& positiveKnobs);
+
+// Exit codes for child process to distinguish failure modes (used when --tuneBuildRoutes expands to multiple configs)
+constexpr int32_t kChildExitSuccess = 0;
+constexpr int32_t kChildExitFailure = 1;            // runOnceBuildAndInfer returned failure
+constexpr int32_t kChildExitStdException = 100;     // Caught std::exception
+constexpr int32_t kChildExitUnknownException = 101; // Caught unknown exception
+
+//! \struct TuningIterationResult
+//! \brief Captures results from a single tuning iteration for the tuning cache file.
+struct TuningIterationResult
+{
+    double gpuTimeMs{0.0};                                      //!< Mean GPU compute time in milliseconds
+    bool accuracyFailed{false};                                 //!< True if accuracy exceeded the threshold
+    std::unordered_map<std::string, double> accuracyLossValues; //!< Per-tensor accuracy values
+};
+
+//! \brief True if `arg` is a tuning-only parent flag that should be stripped from
+//! the child argv. Matches both bare flag and flag=value forms. Tuning-only means
+//! the parent loop interprets it; the child runs a plain single-route build.
+[[nodiscard]] bool isTuningOnlyArg(char const* arg);
+
+//! \brief Build a child argv for one tuning iteration: copies argv with tuning-only
+//! flags removed and appends `--setBuildRoute=<route>`, `--saveEngine=<enginePath>`,
+//! `--tuningResultFile=<resultJsonPath>`. String storage is owned by `storage` so
+//! the returned `char*` pointers stay valid until the caller's execvp() completes.
+//! The returned vector is nullptr-terminated, ready for execvp().
+[[nodiscard]] std::vector<char*> buildTuningChildArgv(int32_t argc, char** argv, std::string const& route,
+    std::string const& enginePath, std::string const& resultJsonPath, std::vector<std::string>& storage);
+
+} // namespace sample
+
+#endif // TRT_SAMPLE_TUNING_H

samples/common/sampleUtils.h

@@ -0,0 +1,338 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TRT_SAMPLE_UTILS_H
+#define TRT_SAMPLE_UTILS_H
+
+#include <cmath>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <numeric>
+#include <random>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include <cuda.h>
+#include <cuda_fp16.h>
+#include <optional>
+
+#include "NvInfer.h"
+
+#include "common.h"
+#include "logger.h"
+#include "logging.h"
+#include "sampleOptions.h"
+
+#define SMP_RETVAL_IF_FALSE(condition, msg, retval, err)                                                               \
+    {                                                                                                                  \
+        if ((condition) == false)                                                                                      \
+        {                                                                                                              \
+            (err) << (msg) << std::endl;                                                                               \
+            return retval;                                                                                             \
+        }                                                                                                              \
+    }
+
+namespace sample
+{
+
+template <typename T>
+inline T roundUp(T m, T n)
+{
+    return ((m + n - 1) / n) * n;
+}
+
+//! comps is the number of components in a vector. Ignored if vecDim < 0.
+int64_t volume(nvinfer1::Dims const& dims, nvinfer1::Dims const& strides, int32_t vecDim, int32_t comps, int32_t batch);
+
+using samplesCommon::volume;
+
+nvinfer1::Dims toDims(std::vector<int64_t> const& vec);
+
+template <typename T, typename std::enable_if<std::is_integral<T>::value, bool>::type = true>
+void fillBuffer(void* buffer, int64_t volume, int32_t min, int32_t max);
+
+template <typename T, typename std::enable_if<!std::is_integral<T>::value, bool>::type = true>
+void fillBuffer(void* buffer, int64_t volume, float min, float max);
+
+template <typename T>
+void dumpBuffer(void const* buffer, std::string const& separator, std::ostream& os, nvinfer1::Dims const& dims,
+    nvinfer1::Dims const& strides, int32_t vectorDim, int32_t spv);
+
+void dumpInt4Buffer(void const* buffer, std::string const& separator, std::ostream& os, Dims const& dims,
+    Dims const& strides, int32_t vectorDim, int32_t spv);
+
+void loadFromFile(std::string const& fileName, char* dst, size_t size);
+
+std::vector<std::string> splitToStringVec(std::string const& option, char separator, int64_t maxSplit = -1);
+
+bool broadcastIOFormats(std::vector<IOFormat> const& formats, size_t nbBindings, bool isInput = true);
+
+int32_t getCudaDriverVersion();
+
+int32_t getCudaRuntimeVersion();
+
+void sparsify(nvinfer1::INetworkDefinition& network, std::vector<std::vector<int8_t>>& sparseWeights);
+void sparsify(nvinfer1::Weights const& weights, int32_t k, int32_t rs, std::vector<int8_t>& sparseWeights);
+
+// Walk the weights elements and overwrite (at most) 2 out of 4 elements to 0.
+template <typename T>
+void sparsify(T const* values, int64_t count, int32_t k, int32_t rs, std::vector<int8_t>& sparseWeights);
+
+template <typename L>
+void setSparseWeights(L& l, int32_t k, int32_t rs, std::vector<int8_t>& sparseWeights);
+
+// Sparsify the weights of Constant layers that are fed to MatMul via Shuffle layers.
+// Forward analysis on the API graph to determine which weights to sparsify.
+void sparsifyMatMulKernelWeights(
+    nvinfer1::INetworkDefinition& network, std::vector<std::vector<int8_t>>& sparseWeights);
+
+template <typename T>
+void transpose2DWeights(void* dst, void const* src, int32_t const m, int32_t const n);
+
+//! A helper function to match a target string with a pattern where the pattern can contain up to one wildcard ('*')
+//! character that matches to any strings.
+bool matchStringWithOneWildcard(std::string const& pattern, std::string const& target);
+
+//! A helper method to find an item from an unordered_map. If the exact match exists, this is identical to
+//! map.find(target). If the exact match does not exist, it returns the first plausible match, taking up to one wildcard
+//! into account. If there is no plausible match, then it returns map.end().
+template <typename T>
+typename std::unordered_map<std::string, T>::const_iterator findPlausible(
+    std::unordered_map<std::string, T> const& map, std::string const& target)
+{
+    auto res = map.find(target);
+    if (res == map.end())
+    {
+        res = std::find_if(
+            map.begin(), map.end(), [&](typename std::unordered_map<std::string, T>::value_type const& item) {
+                return matchStringWithOneWildcard(item.first, target);
+            });
+    }
+    return res;
+}
+
+// ==== Common argument parsing utilities ====
+
+//! Validate that a value is not empty, log error if it is
+bool validateNonEmpty(std::string const& value, std::string const& flagName);
+
+//! Validate remote auto tuning config format
+bool validateRemoteAutoTuningConfig(std::string const& config);
+
+//! Ensure directory path ends with '/'
+inline std::string normalizeDirectoryPath(std::string const& dirPath)
+{
+    std::string result = dirPath;
+    if (!result.empty() && result.back() != '/')
+    {
+        result.push_back('/');
+    }
+    return result;
+}
+
+//! Sanitizes the remote auto tuning config string by removing sensitive credentials
+//! Removes usernames and passwords from URL-style config strings for security.
+//! Example: "ssh://user:pass@host:22" becomes "ssh://***:***@host:22"
+std::string sanitizeRemoteAutoTuningConfig(std::string const& config);
+
+//! Sanitizes command line arguments for logging, removing sensitive credentials
+//! Processes argv array and sanitizes sensitive arguments like remoteAutoTuningConfig
+//! @param argc Number of arguments
+//! @param argv Array of argument strings
+//! @return Vector of sanitized argument strings
+std::vector<std::string> sanitizeArgv(int32_t argc, char** argv);
+
+//! Interface for accuracy validation
+//! This interface provides a way to calculate the accuracy gap between the actual and reference outputs.
+//! Since all the return value is a "loss value", the lower the return value, the better accuracy it is.
+template <typename T>
+class IAccuracyValidator
+{
+public:
+    virtual ~IAccuracyValidator() = default;
+    virtual double calculateAccuracy(std::vector<T> const& actual, std::vector<T> const& reference) = 0;
+};
+
+//! L0 accuracy validator calculates element-wise accuracy using the PyTorch/NumPy allclose formula.
+//! An element matches if: |actual[i] - ref[i]| <= atol + rtol * |ref[i]|
+//! accuracy = (number of mismatching elements) / N
+//! Returns the mismatch ratio (0.0 means perfect match, 1.0 means all elements mismatch).
+template <typename T>
+class L0AccuracyValidator : public IAccuracyValidator<T>
+{
+public:
+    L0AccuracyValidator(double atol, double rtol)
+        : mAtol(atol)
+        , mRtol(rtol)
+    {
+    }
+
+    double calculateAccuracy(std::vector<T> const& actual, std::vector<T> const& reference) override;
+
+private:
+    double mAtol;
+    double mRtol;
+};
+
+//! L1 accuracy validator calculates mean absolute error.
+//! accuracy = Sum(|actual[i] - ref[i]|) / N
+//! Returns the mean absolute error (0.0 means perfect match).
+template <typename T>
+class L1AccuracyValidator : public IAccuracyValidator<T>
+{
+public:
+    double calculateAccuracy(std::vector<T> const& actual, std::vector<T> const& reference) override;
+};
+
+//! L2 accuracy validator calculates mean squared error.
+//! accuracy = Sum(|actual[i] - ref[i]|^2) / N
+//! Returns the mean squared error (0.0 means perfect match).
+template <typename T>
+class L2AccuracyValidator : public IAccuracyValidator<T>
+{
+public:
+    double calculateAccuracy(std::vector<T> const& actual, std::vector<T> const& reference) override;
+};
+
+//! LInf accuracy validator calculates maximum absolute error.
+//! accuracy = Max(|actual[i] - ref[i]|)
+//! Returns the max absolute error (0.0 means perfect match).
+template <typename T>
+class LInfAccuracyValidator : public IAccuracyValidator<T>
+{
+public:
+    double calculateAccuracy(std::vector<T> const& actual, std::vector<T> const& reference) override;
+};
+
+//! Cosine similarity validator calculates 1 - cosine_similarity.
+//! cosine_sim = Sum(actual[i] * ref[i]) / (sqrt(Sum(actual[i]^2)) * sqrt(Sum(ref[i]^2)))
+//! accuracy loss = 1 - cosine_sim
+//! Returns 1 - cosine_similarity (0.0 means perfect match).
+template <typename T>
+class CosineSimilarityValidator : public IAccuracyValidator<T>
+{
+public:
+    double calculateAccuracy(std::vector<T> const& actual, std::vector<T> const& reference) override;
+};
+
+//! \brief Get human-readable name string for an accuracy validation algorithm.
+//! \param[in] algorithm The accuracy validation algorithm enum value.
+//! \return Name string (e.g., "L0", "L1", "Cosine").
+inline std::string getAlgorithmName(AccuracyValidationAlgorithm algorithm)
+{
+    switch (algorithm)
+    {
+    case AccuracyValidationAlgorithm::kL0: return "L0";
+    case AccuracyValidationAlgorithm::kL1: return "L1";
+    case AccuracyValidationAlgorithm::kL2: return "L2";
+    case AccuracyValidationAlgorithm::kLInf: return "LInf";
+    case AccuracyValidationAlgorithm::kCosineSimilarity: return "Cosine";
+    default: return "Unknown";
+    }
+}
+
+//! \brief Factory function to create an accuracy validator based on algorithm type.
+//! \param[in] algorithm The accuracy validation algorithm to use.
+//! \param[in] atol Absolute tolerance (only used by L0 algorithm).
+//! \param[in] rtol Relative tolerance (only used by L0 algorithm).
+//! \return Unique pointer to the appropriate IAccuracyValidator implementation.
+template <typename T>
+std::unique_ptr<IAccuracyValidator<T>> createAccuracyValidator(
+    AccuracyValidationAlgorithm algorithm, float atol = 1e-5F, float rtol = 1e-5F)
+{
+    switch (algorithm)
+    {
+    case AccuracyValidationAlgorithm::kL0: return std::make_unique<L0AccuracyValidator<T>>(atol, rtol);
+    case AccuracyValidationAlgorithm::kL1: return std::make_unique<L1AccuracyValidator<T>>();
+    case AccuracyValidationAlgorithm::kL2: return std::make_unique<L2AccuracyValidator<T>>();
+    case AccuracyValidationAlgorithm::kLInf: return std::make_unique<LInfAccuracyValidator<T>>();
+    case AccuracyValidationAlgorithm::kCosineSimilarity: return std::make_unique<CosineSimilarityValidator<T>>();
+    }
+    ASSERT(false && "Unknown Accuracy Validation Algorithm");
+    return nullptr;
+}
+
+//! \brief Cheap argv pre-scan. Returns true if some `argv[i]` exactly equals `flag`
+//! or starts with `flag` + "=". Used by main() before option parsing to dispatch
+//! between trtexec single-run mode and the tuning loop.
+[[nodiscard]] bool peekArg(int32_t argc, char** argv, char const* flag);
+
+// ============================================================================
+// Tuning cache I/O (used by --tuneBuildRoutes / --continue).
+// Header is a single JSON object on line 1; iterations are JSON Lines after.
+// ============================================================================
+
+//! \brief Reconstruct a shell-safe command line string from argc/argv.
+std::string buildShellQuotedCmdLine(int32_t argc, char** argv);
+
+//! \brief Resolve a file path to an absolute path using POSIX realpath().
+//! Empty input or realpath() failure returns the input unchanged.
+std::string resolveAbsolutePath(std::string const& path);
+
+//! \brief Write the tuning cache file header (line 1, JSON object).
+void writeTuningCacheHeader(std::string const& cacheFilePath, AllOptions const& options, int32_t argc, char** argv,
+    std::string const& tunerVersion, std::string const& defaultBuildRoute);
+
+//! \brief Append one iteration line to the cache file. Fields: iter, build_route, crash,
+//! error_message, accuracy_loss, gpu_time. Crashed iterations have null accuracy/gpu.
+void writeTuningCacheIteration(std::string const& cacheFilePath, uint64_t iter, std::string const& buildRoute,
+    bool crashed, std::string const& errorMessage, std::unordered_map<std::string, double> const& accuracyLossValues,
+    double gpuTimeMs);
+
+//! \struct TuningCacheHeader
+//! \brief Parsed contents of the cache header, returned by readTuningCacheHeader().
+struct TuningCacheHeader
+{
+    std::vector<std::string> argv;          //!< Original command line with file paths absolute.
+    std::string tuningExpr;                  //!< Expanded --tuneBuildRoutes expression.
+    int64_t completedIterations{0};          //!< Number of iteration lines after the header.
+};
+
+//! \brief Read and parse the cache file's header line + count completed iteration lines.
+std::optional<TuningCacheHeader> readTuningCacheHeader(std::string const& cacheFilePath);
+
+//! \brief Rebuild argv for a --continue resume. argv[0] is replaced with currentExePath;
+//! --tuneBuildRoutes is set to the cached expanded expression; --continue and
+//! --tuningCacheFile are stripped from the stored argv and the cache path is re-appended.
+std::vector<std::string> reconstructArgvFromCacheHeader(
+    TuningCacheHeader const& header, std::string const& currentExePath, std::string const& cacheFilePath);
+
+//! \struct CachedIterationResult
+//! \brief Minimal per-iteration fields from the cache, used to reconstruct mixed-mode positive knobs.
+struct CachedIterationResult
+{
+    bool crashed{true};
+    double gpuTimeMs{0.0};
+};
+
+//! \brief Read up to maxIterations iteration lines from the cache and extract (crashed, gpu_time).
+std::vector<CachedIterationResult> readCachedIterationResults(std::string const& cacheFilePath, int64_t maxIterations);
+
+namespace tuningCache
+{
+constexpr char const* kIter = "iter";
+constexpr char const* kBuildRoute = "build_route";
+constexpr char const* kCrash = "crash";
+constexpr char const* kErrorMessage = "error_message";
+constexpr char const* kAccuracyLoss = "accuracy_loss";
+constexpr char const* kGpuTime = "gpu_time";
+} // namespace tuningCache
+
+} // namespace sample
+#endif // TRT_SAMPLE_UTILS_H

samples/common/sampleUtils.test.cpp

@@ -0,0 +1,155 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "sampleUtils.h"
+
+#include <gtest/gtest.h>
+
+#include <string_view>
+
+using namespace sample;
+using namespace std::string_view_literals;
+
+TEST(RoundUp, ExactMultiple)
+{
+    EXPECT_EQ(roundUp(4, 4), 4);
+    EXPECT_EQ(roundUp(8, 4), 8);
+    EXPECT_EQ(roundUp(0, 4), 0);
+}
+
+TEST(RoundUp, NeedsRounding)
+{
+    EXPECT_EQ(roundUp(1, 4), 4);
+    EXPECT_EQ(roundUp(5, 4), 8);
+    EXPECT_EQ(roundUp(7, 4), 8);
+}
+
+TEST(SplitToStringVec, SingleToken)
+{
+    auto const v = splitToStringVec("hello", ',');
+    ASSERT_EQ(v.size(), 1U);
+    EXPECT_EQ(v[0], "hello"sv);
+}
+
+TEST(SplitToStringVec, MultipleTokens)
+{
+    auto const v = splitToStringVec("a,b,c", ',');
+    ASSERT_EQ(v.size(), 3U);
+    EXPECT_EQ(v[0], "a"sv);
+    EXPECT_EQ(v[1], "b"sv);
+    EXPECT_EQ(v[2], "c"sv);
+}
+
+TEST(SplitToStringVec, EmptyString)
+{
+    auto const v = splitToStringVec("", ',');
+    EXPECT_TRUE(v.empty());
+}
+
+TEST(SplitToStringVec, MaxSplit)
+{
+    // maxSplit=1 means at most one split; the rest of the string is the second element.
+    auto const v = splitToStringVec("a:b:c", ':', 1);
+    ASSERT_EQ(v.size(), 2U);
+    EXPECT_EQ(v[0], "a"sv);
+    EXPECT_EQ(v[1], "b:c"sv);
+}
+
+TEST(SplitToStringVec, TrailingSeparator)
+{
+    auto const v = splitToStringVec("a,b,", ',');
+    ASSERT_EQ(v.size(), 3U);
+    EXPECT_EQ(v[0], "a"sv);
+    EXPECT_EQ(v[1], "b"sv);
+    EXPECT_EQ(v[2], ""sv);
+}
+
+TEST(MatchStringWithOneWildcard, ExactMatch)
+{
+    EXPECT_TRUE(matchStringWithOneWildcard("hello", "hello"));
+    EXPECT_FALSE(matchStringWithOneWildcard("hello", "world"));
+    EXPECT_FALSE(matchStringWithOneWildcard("hello", "hello2"));
+}
+
+TEST(MatchStringWithOneWildcard, WildcardMatchesAnything)
+{
+    EXPECT_TRUE(matchStringWithOneWildcard("*", "anything"));
+    EXPECT_TRUE(matchStringWithOneWildcard("*", ""));
+}
+
+TEST(MatchStringWithOneWildcard, PrefixWildcard)
+{
+    EXPECT_TRUE(matchStringWithOneWildcard("hello*", "hello"));
+    EXPECT_TRUE(matchStringWithOneWildcard("hello*", "hello world"));
+    EXPECT_FALSE(matchStringWithOneWildcard("hello*", "world"));
+}
+
+TEST(MatchStringWithOneWildcard, SuffixWildcard)
+{
+    EXPECT_TRUE(matchStringWithOneWildcard("*world", "world"));
+    EXPECT_TRUE(matchStringWithOneWildcard("*world", "hello world"));
+    EXPECT_FALSE(matchStringWithOneWildcard("*world", "hello"));
+}
+
+TEST(MatchStringWithOneWildcard, MiddleWildcard)
+{
+    EXPECT_TRUE(matchStringWithOneWildcard("he*ld", "held"));
+    EXPECT_TRUE(matchStringWithOneWildcard("he*ld", "hello world"));
+    EXPECT_FALSE(matchStringWithOneWildcard("he*ld", "hello"));
+}
+
+TEST(NormalizeDirectoryPath, AlreadyNormalized)
+{
+    EXPECT_EQ(normalizeDirectoryPath("/some/path/"), "/some/path/"sv);
+}
+
+TEST(NormalizeDirectoryPath, MissingTrailingSlash)
+{
+    EXPECT_EQ(normalizeDirectoryPath("/some/path"), "/some/path/"sv);
+}
+
+TEST(NormalizeDirectoryPath, EmptyString)
+{
+    EXPECT_EQ(normalizeDirectoryPath(""), ""sv);
+}
+
+TEST(SanitizeRemoteAutoTuningConfig, Empty)
+{
+    EXPECT_EQ(sanitizeRemoteAutoTuningConfig(""), ""sv);
+}
+
+TEST(SanitizeRemoteAutoTuningConfig, NoCredentials)
+{
+    // No @ means no credentials section; returned as-is.
+    EXPECT_EQ(sanitizeRemoteAutoTuningConfig("ssh://host:22"), "ssh://host:22"sv);
+}
+
+TEST(SanitizeRemoteAutoTuningConfig, UsernameOnly)
+{
+    EXPECT_EQ(sanitizeRemoteAutoTuningConfig("ssh://user@host:22"), "ssh://***@host:22"sv);
+}
+
+TEST(SanitizeRemoteAutoTuningConfig, UsernameAndPassword)
+{
+    EXPECT_EQ(sanitizeRemoteAutoTuningConfig("ssh://user:pass@host:22"), "ssh://***@host:22"sv);
+}
+
+TEST(SanitizeRemoteAutoTuningConfig, WithQueryParams)
+{
+    EXPECT_EQ(sanitizeRemoteAutoTuningConfig("ssh://admin:secret@server.com:22?timeout=30"),
+        "ssh://***@server.com:22?timeout=30"sv);
+}

samples/common/streamReader.h

@@ -0,0 +1,112 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef STREAM_READER_H
+#define STREAM_READER_H
+
+#include "NvInferRuntime.h"
+#include "sampleUtils.h"
+#include <fstream>
+
+namespace samplesCommon
+{
+
+//! Implements the TensorRT IStreamReaderV2 interface to allow deserializing an engine directly from the plan file.
+//! Supports seeking to a position within the file, and reading directly to device pointers.
+//! This implementation is not optimized, and will not provide performance improvements over the existing reader.
+class AsyncStreamReader final : public nvinfer1::IStreamReaderV2
+{
+public:
+    bool open(std::string const& filepath)
+    {
+        mFile.open(filepath, std::ios::binary);
+        return mFile.is_open();
+    }
+
+    void close()
+    {
+        if (mFile.is_open())
+        {
+            mFile.close();
+        }
+    }
+
+    ~AsyncStreamReader() final
+    {
+        close();
+    }
+
+    bool seek(int64_t offset, nvinfer1::SeekPosition where) noexcept final
+    {
+        switch (where)
+        {
+        case (nvinfer1::SeekPosition::kSET): mFile.seekg(offset, std::ios_base::beg); break;
+        case (nvinfer1::SeekPosition::kCUR): mFile.seekg(offset, std::ios_base::cur); break;
+        case (nvinfer1::SeekPosition::kEND): mFile.seekg(offset, std::ios_base::end); break;
+        }
+        return mFile.good();
+    }
+
+    int64_t read(void* destination, int64_t nbBytes, cudaStream_t stream) noexcept final
+    {
+        if (!mFile.good())
+        {
+            return -1;
+        }
+
+        cudaPointerAttributes attributes;
+        ASSERT(cudaPointerGetAttributes(&attributes, destination) == cudaSuccess);
+
+        // from CUDA 11 onward, host pointers are return cudaMemoryTypeUnregistered
+        if (attributes.type == cudaMemoryTypeHost || attributes.type == cudaMemoryTypeUnregistered)
+        {
+            mFile.read(static_cast<char*>(destination), nbBytes);
+            return mFile.gcount();
+        }
+        else if (attributes.type == cudaMemoryTypeDevice)
+        {
+            // Set up a temp buffer to read into if reading into device memory.
+            std::unique_ptr<char[]> tmpBuf{new char[nbBytes]};
+            mFile.read(tmpBuf.get(), nbBytes);
+            // cudaMemcpyAsync into device storage.
+            ASSERT(cudaMemcpyAsync(destination, tmpBuf.get(), nbBytes, cudaMemcpyHostToDevice, stream) == cudaSuccess);
+            // No race between the copying and freeing of tmpBuf, because cudaMemcpyAsync will
+            // return once the pageable buffer has been copied to the staging memory for DMA transfer
+            // to device memory.
+            return mFile.gcount();
+        }
+        return -1;
+    }
+
+    void reset()
+    {
+        ASSERT(mFile.good());
+        mFile.seekg(0);
+    }
+
+    bool isOpen() const
+    {
+        return mFile.is_open();
+    }
+
+private:
+    std::ifstream mFile;
+};
+
+} // namespace samplesCommon
+
+#endif // STREAM_READER_H